diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..d4144ee --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,24 @@ +# Dependabot (TS8). Turning on Dependabot alerts and security updates is a repository setting; this file only +# configures version updates. +version: 2 +updates: + # The workflows pin every action to a commit SHA with its tag in a comment; Dependabot updates both together. + # One grouped pull request a week. The Flatpak and Debian container digests in release.yml aren't covered: + # update those by hand + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: ["*"] + + # Report only: open-pull-requests-limit 0 turns off version-update pull requests for pyproject.toml's + # dependencies, while a vulnerable one is still reported (Dependabot alerts and security updates, once turned on + # in the repository's settings). rnfs relies on RNS and LXMF internals, so a new version is taken deliberately: + # packaging/update-lock.py, packaging/flatpak/generate-sources.py and tests/test_rns_compat.py (CONTRIBUTING.md) + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 060ce92..1367a00 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,10 +2,16 @@ name: release # Push a tag vX.Y.Z (matching rnfs.__version__) to build every package and publish a GitHub Release with them, # SHA256SUMS and signed build provenance. Run it by hand (workflow_dispatch, on any branch or tag) to build the -# same artifacts without releasing. +# same artifacts without releasing. It also builds them, without releasing, for a pull request that changes the +# packaging or the workflows and once a week, so a build that only a release runs (the Flatpak, the Windows exe, +# the arm64 .deb, the Debian 12 smoke test) is known to work before a tag is pushed. on: push: tags: ["v*"] + pull_request: + paths: ["packaging/**", "pyproject.toml", ".github/**"] + schedule: + - cron: "23 5 * * 1" # Mondays, 05:23 UTC workflow_dispatch: permissions: @@ -17,9 +23,10 @@ permissions: # --require-hashes, and builds run without build isolation, so nothing unpinned is fetched for a build. jobs: # The whole tests workflow (unit and end-to-end tests on every OS and Python, lint, packaging smoke tests); - # the release waits for it + # the release waits for it. A pull request runs tests.yml itself, so it isn't run twice tests: uses: ./.github/workflows/tests.yml + if: github.event_name != 'pull_request' version: runs-on: ubuntu-latest @@ -39,6 +46,28 @@ jobs: exit 1 fi echo "version=$version" >> "$GITHUB_OUTPUT" + # The release notes are CHANGELOG.md's section for this version (`## X.Y.Z`, ended by the next `## `). A tag + # needs it to be the first section and no longer marked "(unreleased)"; any other run takes the version's + # section as it stands + - name: Release notes from CHANGELOG.md (and check the version's section is released) + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + first=$(grep -m1 '^## ' CHANGELOG.md) + if [ "${GITHUB_REF_TYPE}" = tag ] && [ "$first" != "## $VERSION" ]; then + echo "::error::CHANGELOG.md's first section is \"$first\", not \"## $VERSION\" (still unreleased?)" + exit 1 + fi + awk -v heading="## $VERSION" ' + /^## / { if (on) exit; on = ($0 == heading || index($0, heading " (") == 1); next } + on' CHANGELOG.md > release-notes.md + grep -q . release-notes.md || { echo "::error::CHANGELOG.md has no section for $VERSION"; exit 1; } + cat release-notes.md + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: notes # not release-*: it isn't one of the release's files + path: release-notes.md + if-no-files-found: error python: name: wheel and sdist @@ -118,32 +147,10 @@ jobs: path: dist/*.deb if-no-files-found: error - flatpak-sources: - name: flatpak sources - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - with: - persist-credentials: false # no token left in .git/config for the build code that runs next - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: "3.12" - # The Flatpak's wheels must be the ones packaging/requirements.lock pins (what every other package - # installs): regenerate them from the lock and fail if that changes the committed files - - name: Check the pinned Flatpak wheels match packaging/requirements.lock - run: | - python -I packaging/flatpak/generate-sources.py - git diff --exit-code -- packaging/flatpak || { - echo "::error::packaging/flatpak/*.json is out of date: run python3 packaging/flatpak/generate-sources.py and commit" - exit 1 - } - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: flatpak-sources - path: packaging/flatpak/*.json - + # The committed packaging/flatpak/*.json are built: the tests workflow (tests.yml's flatpak-sources job, which + # the release waits for) checks they match packaging/requirements.lock flatpak: - needs: [version, flatpak-sources] + needs: version runs-on: ubuntu-latest container: image: ghcr.io/flathub-infra/flatpak-github-actions:freedesktop-25.08@sha256:c91a88269eee8fcdd73ecf43064232fbebe27411f926f63f38bdf70c51f2ceae @@ -152,16 +159,14 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: persist-credentials: false # no token left in .git/config for the build code that runs next - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - with: - name: flatpak-sources - path: packaging/flatpak - uses: flatpak/flatpak-github-actions/flatpak-builder@79327416609af08178ad73b352877e51450790b3 # v6 with: manifest-path: packaging/flatpak/io.github.afit21.rnfs.yml bundle: rnfs.flatpak arch: x86_64 - cache-key: flatpak-builder-${{ github.sha }} + # No Actions cache: any default-branch workflow can write one, and a restored cache would go into the + # attested bundle + cache: false upload-artifact: false - name: Install and smoke test the bundle # Best effort: running sandboxed apps inside the CI container depends on the runner's kernel settings @@ -171,6 +176,8 @@ jobs: flatpak run io.github.afit21.rnfs --version if command -v python3 >/dev/null; then mkdir -p "$HOME/smoke-tmp" + # The sandbox reaches only ~/.rnfs, ~/.reticulum and Downloads unless granted more + flatpak override --user --filesystem="$HOME/smoke-tmp" io.github.afit21.rnfs TMPDIR="$HOME/smoke-tmp" python3 packaging/smoke_test.py flatpak run io.github.afit21.rnfs fi - name: Name it for the release @@ -237,6 +244,10 @@ jobs: pattern: release-* path: artifacts merge-multiple: true + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: notes + path: notes - name: Checksums working-directory: artifacts run: | @@ -252,8 +263,9 @@ jobs: GH_TOKEN: ${{ github.token }} VERSION: ${{ needs.version.outputs.version }} # A release already made for this tag on GitHub's release page gets the files attached rather than - # failing; otherwise the release is created here. Files already attached are never replaced (no - # --clobber): users may have downloaded and checked them, so a re-run with the same names fails instead + # failing (its notes are left as they are); otherwise the release is created here, with the version's + # CHANGELOG.md section as its notes. Files already attached are never replaced (no --clobber): users may + # have downloaded and checked them, so a re-run with the same names fails instead run: | if gh release view "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then gh release upload "$GITHUB_REF_NAME" artifacts/* --repo "$GITHUB_REPOSITORY" @@ -261,6 +273,6 @@ jobs: gh release create "$GITHUB_REF_NAME" artifacts/* \ --repo "$GITHUB_REPOSITORY" \ --title "rnfs $VERSION" \ - --generate-notes \ + --notes-file notes/release-notes.md \ --verify-tag fi diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4cd5ce6..53993ae 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,17 +37,46 @@ jobs: python -m pip check - name: Unit tests run: python -m unittest discover tests - # On Windows the daemons are started in their own process groups and stopped with CTRL_BREAK_EVENT + # On Windows the daemons are started in their own process groups and stopped with CTRL_BREAK_EVENT. Every + # module gated on RNFS_E2E is listed here (test_packaging checks it): the embedded node and the security + # regressions that need real RNS included - name: End-to-end tests (real daemons over local TCP) env: RNFS_E2E: "1" - run: python -m unittest tests.test_e2e tests.test_client - timeout-minutes: 10 + run: >- + python -m unittest tests.test_e2e tests.test_client tests.test_embedded tests.security.test_h1_real_rns + tests.security.test_m1_real_rns tests.security.test_h2_display_page_pins + timeout-minutes: 20 - name: Command works run: rnfs --version - # ruff with the rules in pyproject.toml (pyflakes, syntax errors, a few bugbear checks), and no merge-conflict - # markers left in any file + # Line coverage of the whole suite, the real-RNS tests included: a report only, with no threshold. The daemons + # the end-to-end tests start run in their own processes, which aren't measured + coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - name: Install (and coverage, from packaging/build-tools.lock) + run: | + python -m pip install --require-hashes --only-binary=:all: -r packaging/requirements.lock + python -m pip install --require-hashes --only-binary=:all: -r packaging/build-tools.lock + python -m pip install --no-deps --no-build-isolation . + python -m pip check + - name: Tests under coverage + env: + RNFS_E2E: "1" + run: python -m coverage run --source=rnfs -m unittest discover tests + timeout-minutes: 30 + - name: Report + run: python -m coverage report --sort=cover + + # ruff with the rules in pyproject.toml (pyflakes, syntax errors, a few bugbear checks, broad excepts, late + # imports, stale noqa comments), and no merge-conflict markers left in any file lint: runs-on: ubuntu-latest steps: @@ -65,6 +94,27 @@ jobs: - name: No merge-conflict markers run: "! git grep -nE '^(<<<<<<<|>>>>>>>)( |$)|^=======$' -- . ':!*.lock'" + # The Flatpak's wheels must be the ones packaging/requirements.lock pins (what every other package installs): + # regenerate them from the lock and fail if that changes the committed files. Run here, not only at release, so + # a lock bump that forgets them fails before a vX.Y.Z tag is pushed; the release builds the committed files + flatpak-sources: + name: flatpak sources + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false # no token left in .git/config for the build code that runs next + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - name: Check the pinned Flatpak wheels match packaging/requirements.lock + run: | + python -I packaging/flatpak/generate-sources.py + git diff --exit-code -- packaging/flatpak || { + echo "::error::packaging/flatpak/*.json is out of date: run python3 packaging/flatpak/generate-sources.py and commit" + exit 1 + } + # The packages the release workflow builds, smoke-tested: wheel/sdist, .deb, install.sh, PyInstaller binary packaging: runs-on: ubuntu-latest diff --git a/0.1.5-audit.md b/0.1.5-audit.md new file mode 100644 index 0000000..c27a8da --- /dev/null +++ b/0.1.5-audit.md @@ -0,0 +1,522 @@ +# rnfs 0.1.5 security audit + +**Date:** 2026-10-17. **Code:** `main` at `5bd3075`, which is v0.1.5 plus the API proof and connection commits. +**Scope:** everything under `rnfs/`, the NomadNet companion pages, `examples/`, and the install, packaging and +release pipeline. + +## Summary + +There are no Critical or High findings. + +- **Medium (7):** two are remotely reachable (CONT-1 needs a deny-list setting, DHT-1 does not). One leads to + local code execution if the rnfs dir is ever loose (API-1). Two are local filesystem or provenance issues + (CONT-2, FEAT-5), one is a CI cache-poisoning path (SUP-1), and one is a CPU-cost asymmetry on the server + (NET-1). +- **Low:** 13 findings. **Info:** 15. +- **Earlier fixes still hold.** The H1, M1–M4, L1–L17, N1, NEW-1 to NEW-8, I2 and R1/R2 regression tests in + `tests/security/` cover fixes that held up under re-testing. Wherever a residual issue was found (L12-R, L6, + I2-R1), it is listed below. + +| ID | Severity | Title | Attacker | Verified | +|---|---|---|---|---| +| [CONT-1](#cont-1-medium-a-128-kb-dag-makes-the-file-hash-deny-check-hash-up-to-2-bytes) | Medium | A 128 KB DAG makes the file-hash deny check hash up to 2^56 bytes | Remote publisher, plus a web page or user action | PoC, re-run | +| [DHT-1](#dht-1-medium-a-stalled-lookup-leaves-keeper-name-queries-1-s) | Medium | A stalled lookup leaves keeper `/name` queries 1 s, censoring offline owners' names | Remote Sybils near a name key | PoC, re-run | +| [API-1](#api-1-medium-posix-never-checks-rnfstoml-or-existing-subdirectories) | Medium | POSIX never checks `rnfs.toml` or existing subdirectories (spec §11 MUST) | Local user, if the rnfs dir was ever loose | PoC, re-run | +| [FEAT-5](#feat-5-medium-a-name-link-quietly-falls-back-to-the-senders-cid) | Medium | A name link quietly falls back to the sender's CID | Any LXMF sender | PoC, re-run | +| [CONT-2](#cont-2-medium-on-windows-low-on-posix-write-out-follows-a-swapped-directory) | Medium (Windows) / Low (POSIX) | Write-out follows a swapped directory, so files land outside the output | Local user who can rename inside the output | PoC, re-run | +| [SUP-1](#sup-1-medium-the-release-flatpak-build-restores-a-poisonable-actions-cache) | Medium | The release Flatpak build restores a poisonable Actions cache | Code execution in any default-branch workflow | Read from code | +| [NET-1](#net-1-medium-cancelling-a-blocks-response-after-its-first-parts-gets-no-cooldown) | Medium | Cancelling a `/blocks` response after its first parts gets no cooldown | Anonymous peer | PoC (logic; cost measured on fakes) | +| [DHT-2](#dht-2-low-a-replayed-announce-inherits-the-newest-emission-time) | Low | A replayed announce inherits the newest emission time | Neighbour holding old announces | PoC, re-run | +| [DHT-3](#dht-3-low-a-followed-names-owner-can-fill-the-followers-disk) | Low | A followed name's owner can fill the follower's disk | Owner of a followed name | PoC, re-run | +| [FEAT-1](#feat-1-low-one-auto-fetch-sender-can-push-out-anothers-pinned-messages) | Low | One auto-fetch sender can push out another's pinned messages | Trusted auto-fetch sender | PoC, re-run | +| [FEAT-4](#feat-4-low-queued-auto-fetches-still-run-after-the-sender-is-removed) | Low | Queued auto-fetches still run after the sender is removed | Formerly trusted sender | PoC, re-run | +| [FEAT-3](#feat-3-low-many-identities-starve-all-strangers-messages) | Low | Many identities starve all strangers' messages | Many LXMF identities | PoC, re-run | +| [FEAT-2](#feat-2-low-display-names-can-fake-a-trusted-address-in-rnfs-inbox) | Low | Display names can fake a trusted address in `rnfs inbox` | Any LXMF sender | PoC, re-run | +| [API-2](#api-2-low-before-the-proof-a-fake-listener-chooses-the-clients-error) | Low | Before the proof, a fake listener chooses the client's error | Local process on the port while the daemon is down | PoC, re-run | +| [NET-2](#net-2-low-single-packet-blocks-responses-skip-pacing) | Low | Single-packet `/blocks` responses skip pacing | Anonymous peer on a large-MTU path | PoC | +| [NET-3](#net-3-low-request-packets-on-outgoing-links-are-decoded-before-any-rnfs-check) | Low | Request packets on outgoing links are decoded before any rnfs check | Node we link to | PoC (rnfs side); RNS side read from source | +| [SUP-2](#sup-2-low-installsh-doesnt-tie-the-checked-sdist-to-the-requested-tag) | Low | `install.sh` doesn't tie the checked sdist to the requested tag | Release-asset write access | Read from code | +| [SUP-3](#sup-3-low-provenance-checking-is-off-by-default) | Low | Provenance checking is off by default | Release-asset write access | Read from code | +| [SUP-4](#sup-4-low-the-rnfs-name-on-pypi-is-unclaimed) | Low | The `rnfs` name on PyPI is unclaimed | Anyone | Checked against PyPI | +| [SUP-5](#sup-5-low-the-flatpak-sandbox-is-nominal) | Low | The Flatpak sandbox is nominal (`--filesystem=home`, `--device=all`) | Compromised daemon | Read from code | + +**How the findings were verified:** +- **"PoC, re-run":** an audit agent wrote and ran a proof of concept, and it was run again for this report. +- **"Read from code":** the finding comes from reading the code and was checked by a second reader, but there + is no runnable PoC. +- **PoC location:** the PoCs are in the session scratchpad and will not survive it. Port each one into + `tests/security/` with its fix, following the existing naming (`test_cont1_*.py` and so on). + +--- + +## Medium + +### CONT-1 (Medium): a 128 KB DAG makes the file-hash deny check hash up to 2^56 bytes + +- **Where:** + - [node.py:799](rnfs/node.py#L799) `_check_file` hashes the whole file, through `walker.read_range`, before + anything is sent. + - `read_range` has no output budget. Repeated subtrees are read again each time they appear, and + `SmallLeaves` exempts large leaves from the limit that would otherwise apply. +- **Reached from:** + - `Node._read`: `cat`, gateway GET, and gateway type sniffing at `gateway.py:605`; + - `node_content.export_stream`: the gateway's `?format=archive`; + - `import_archive`'s check (`node.py:1616`). +- **Precondition:** the deny list has at least one `sha256:` entry, locally or from a subscribed `deny_lists` + source ([node.py:803](rnfs/node.py#L803)). +- **Attack:** + 1. The attacker publishes 6 blocks (about 128 KB): one 64 KiB chunk, then five file nodes. Each node lists + the node below it 256 times, so the root declares 2^56 bytes. + 2. The victim makes a 1-byte Range GET through the gateway, or runs `cat`, `?format=archive`, or + `rnfs import` of the 128 KB archive. A web page or rnfs page the victim opens can fire the gateway + request, since the gateway listens on 127.0.0.1. + - `check_disk` only runs while the DAG is incomplete. The attacker can complete it first by requesting each + block with `?format=raw`, which never hashes DAG nodes. A declared size below the victim's free space + skips that step entirely. +- **Impact:** + - Each request pins a core: about a year of hashing at 1.7 GB/s. + - Each request also holds one of the 32 `max_connections` slots, and keeps hashing after the client + disconnects. A handful of requests exhaust the gateway and API. + - `cat` and `import` hang. `get` is safe, because `_check_free_space` bounds it first. +- **Verified:** re-run. A 1-byte gateway read hashed 1 GiB in 0.6 s before the PoC stopped it, and importing + the 128 KB archive did the same. +- **Fix:** + - Cap the bytes hashed for the deny check, at `max_download_size` or a new `deny_hash_max_bytes`. Refuse + files over the cap by policy. + - Or refuse files whose declared size is more than K times the bytes of their distinct blocks. + - Give `read_range` an optional output-byte budget either way. + +### DHT-1 (Medium): a stalled lookup leaves keeper `/name` queries 1 s + +- **Where:** + - [keepers.py:809](rnfs/features/keepers.py#L809): `deadline = time.monotonic() + max(1.0, lookup.remaining())`. + - `enough()` (around `:745`). + - `dht/lookup.py` `_iterate`, which has no bound on rounds or progress: only `lookup_timeout` (600 s) ends + it. +- **Attacker:** a few hundred announced Sybil server IDs ground close to `name_key(name)`. Grinding runs at + about 31k IDs/s per core, so 17 matching bits took 5 s. Each Sybil answers `/find` slowly, returning fresh + Sybil IDs that are closer to the key, so the beta closest nodes never all finish answering. +- **Impact:** + - For a typical name with fewer than 3 × alpha keepers, `enough()` never fires. The provider search uses + the whole `lookup_timeout`, and the `/name` queries then get 1 s. + - A link to a real keeper needs seconds to establish, so its answer is dropped. + - As a result, every keeper-fallback resolve, follow check and `retry_missing` for that name fails. Each one + also blocks its caller for more than 10 minutes. + - That is exactly the case keepers exist for: the owner is offline. + - A slow radio mesh can starve the queries the same way with no attacker. +- **Verified:** re-run. Without Sybils, sequence 2 resolved in 1.5 s. With 491 Sybils stalling the lookup it + resolved nothing after 6.0 s, even though the keeper was asked. +- **Fix (wire-compatible):** + - Give the `/name` queries their own time budget: at least `path_timeout`, plus link setup, plus the RNS + request timeout. + - Query candidates as they are found, or stop the provider search once candidates exist and part of the + budget has passed. + - In `Lookup`, end after N rounds without the beta-closest set changing, or cap the number of queries. + +### API-1 (Medium): POSIX never checks `rnfs.toml` or existing subdirectories + +- **Where:** + - [config.py:274-282](rnfs/config.py#L274-L282): `winacl.secure_settings` returns `None` off Windows + ([winacl.py:363-365](rnfs/winacl.py#L363-L365)). + - `node.py:138-139`: `makedirs(..., exist_ok=True)`, with no owner or mode check on subdirectories. + - `rnfs.toml` is read at `cli.py:186`, before `lock_rnfs_dir` tightens the root at `:200`. +- **What the spec requires:** §11 says `rnfs.toml` MUST be checked through its open handle: refused if + another user owns it, made private if others can access it. Windows does this; POSIX does nothing. + Subdirectories that already exist (`state/`, `names/`, `storage/`, `blocks/`) are never checked or + tightened. +- **Attacker:** a local user who could write in the rnfs dir at some point before the daemon tightened it, + for example a group-writable or shared `--dir`. Tightening the root to 0700 later does not remove their + access to subdirectories: they can keep a directory fd open, or keep subdirectories they created. +- **Impact:** + - A planted `rnfs.toml` can set `reticulum_dir` to a config with a `PipeInterface` `command=`. That is + code execution as the victim. + - Write access to `blocks/` lets them plant blocks: stored raw blocks are trusted by length + (`fetcher._check_stored`). + - The identity file (I2-R1) already got this protection, from the same starting condition. +- **Verified:** re-run. A 0666 `rnfs.toml` loads with no warning. After `lock_rnfs_dir` the root is 0700 but + its subdirectories stay 0777. That ownership isn't checked comes from reading the code; changing a file's + owner needs root. +- **Fix:** + - On POSIX, open `rnfs.toml` with `O_NOFOLLOW` and `fstat` it. Refuse if `st_uid != geteuid()`, and + `fchmod` it to 0600 with a warning if group or other bits are set. + - In `_make_private_dirs`, `lstat` each subdirectory. Refuse a symlink, a non-directory or a foreign owner, + and `chmod` loose ones to 0700. + +### FEAT-5 (Medium): a name link quietly falls back to the sender's CID + +- **Where:** `features/lxmf.py` `_resolve` (around `:1300-1320`) and `_print_links` (around `:1683`). +- **Attacker:** any LXMF sender, strangers included. +- **Attack:** + - The rnfs field of the message carries both `name` and `cid`. + - `rnfs inbox` shows only `rnfs:name//...`, never the CID. + - When the user runs `rnfs inbox get N` while the name can't be resolved (owner offline, no keeper + reachable: common on a mesh), `_resolve` silently uses the sender's CID. The only trace is a NOTICE in the + daemon log. +- **Impact:** a stranger can pose as a well-known name, such as a project's release name. The user saves the + attacker's file under the shown file name, believing it came from that name. The content still matches its + CID, but its origin is misrepresented. Auto-fetch has the same fallback, though only for trusted senders. +- **Verified:** re-run. The CLI never shows the CID, and `inbox get` writes `installer.sh` with the attacker's + content. +- **Fix:** + - When the user runs `inbox get` and the name can't be resolved, fail, unless they pass an explicit + `--use-sent-cid`. + - Show the sent CID next to name links in `rnfs inbox`. + - Report any fallback as a warning in the command's result. + +### CONT-2 (Medium on Windows, Low on POSIX): write-out follows a swapped directory + +- **Where:** `content/walker.py` `write_out` (around `:490-527`). + - It runs `os.mkdir(path)`, then writes that directory's children later, by path, after all its siblings. + - Nothing checks that the directory is still the one it created. +- **Attacker:** a local user who can rename entries inside the output tree. + - On POSIX, a group member, if the victim uses umask 002 in a shared setgid directory. + - On Windows, any Authenticated User, if the output is under a folder whose inherited ACL grants Modify. + That is the default for folders created directly under `C:\`. +- **Impact:** + - Between the mkdir and the child writes, the attacker swaps `partial/sub` for a symlink or junction. + - `get` then creates the DAG's files, executable bit included, in a directory the attacker chooses + (`~/.config/autostart`, the Startup folder). That can lead to code execution as the victim. + - `open(..., "xb")` stops existing files being overwritten, but not new files being created. +- **Verified:** re-run. The PoC does the swap deterministically, and `evil.desktop` is written into the + attacker's directory. +- **Fix:** + - Create the `.rnfs-get-*.partial` root as 0700 (on Windows, with `winacl.make_private_dir`), and relax its + mode just before `rename_new`. + - Where the platform allows, create entries relative to the parent's fd: `mkdir(dir_fd=)` and + `os.open(O_CREAT|O_EXCL|O_NOFOLLOW, dir_fd=)`. Reopen each new directory with `O_DIRECTORY|O_NOFOLLOW` + and check its dev/ino. + +### SUP-1 (Medium): the release Flatpak build restores a poisonable Actions cache + +- **Where:** [release.yml:159-165](.github/workflows/release.yml#L159-L165). The flatpak-builder action at the + pinned SHA (v6) defaults to `cache: true`. + - It restores the exact key `flatpak-builder--x86_64`, and otherwise falls back to the prefix + `flatpak-builder-x86_64`. + - A tag run can restore caches saved from the default branch. +- **Attacker:** code execution in any default-branch workflow run, even a `contents: read` job. The runner + token can write the cache whatever `permissions:` says. A `workflow_dispatch` of `release.yml` on main also + works. +- **Impact:** + - Poisoned cached modules or ccache output end up in `rnfs.flatpak`. + - The bundle is then checksummed and attested as genuine. + - The `release` environment gate reviews only the publish step, not what was built. +- **Fix:** set `cache: false` on the step, or `restore-cache: false` and `save-cache: false`. + +### NET-1 (Medium): cancelling a `/blocks` response after its first parts gets no cooldown + +- **Where:** `network/server.py:1207-1228` (`_concluded` / `_sent_any`). +- **What happens:** the L17 cooldown applies only when a Resource is rejected with nothing sent. A requester + that cancels after any part arrives can ask again at once. +- **Impact:** each round makes the node read and SHA-256 every block, pack them, then build and encrypt a + whole Resource with its hashmap: + - about 1 MiB per response once the link has a rate estimate; + - about 192 KiB before it has one. + - The requester only has to receive the first window of parts. The cost asymmetry is CPU, not bandwidth. +- **Background:** `test_l17_rejected_responses.py` deliberately treats a rejection mid-transfer as honest, so + this is a design residual, but its cost hadn't been weighed. +- **Verified:** PoC on the server test harness. + - A reject with nothing sent gets a cooldown. A cancel after progress, or a reject after progress, gets none, + and the same link takes a new `/blocks` slot at once. + - A 1 MiB response costs the responder about 0.5 ms of build CPU per round (re-hash and pack), against about + 0.003 ms for the requester: roughly 185×. + - RNS's Resource encryption and hashmap build add responder-only cost on top, which wasn't measured live. + - One link is limited by round trips (one `/blocks` in flight per link), so it forces only a few to tens of + rounds a second. Scaling the attack takes many links, up to `max_incoming_links` = 128. + - Rated Medium for the asymmetry. The absolute cost per link is modest. +- **Fix:** + - Treat a response the requester cancelled after receiving only a small fraction (under one window, or under + 5%) as no progress. Apply a short, growing cooldown per link and per identity. + - Or cap how many never-completed responses a link may cause per minute. + +--- + +## Low + +### DHT-2 (Low): a replayed announce inherits the newest emission time +- **Where:** `names.py:888-894` (`_absorb_recalled` → `_announce_heard`). +- **Mechanism:** + - RNS's `Identity.remember` overwrites the stored app_data for any validly signed replay. + - The path table keeps the newer announce's random blob, and `emitted_time` reads that blob. + - So an old record is paired with the new emission time, and looks fresh. +- **Attacker:** a neighbour holding old announces, against a resolver with no floor for the name. For + example, a first resolve, or a name that was only heard and then pushed out. +- **Impact:** + - A CLI or API resolve returns and persists the old sequence with age 0, and keepers that hold the newer + record are not asked. + - The same applies to deny-list and contacts resolves, and to the `max_age` fast path. + - The spec's statement that a first resolution can't detect a replay is not true here: RNS already holds + proof of a newer emission. +- **Verified:** re-run (returns sequence 3; no keepers queried). +- **Fix (wire-compatible):** get the remembered announce's own emission time from its packet hash, as + `keepers` already does with `known_destinations[name][1]` → `Transport.get_cached_packet`. If it is older + than the path's, treat the record as a replay: use its own emission time as heard, and ask keepers. + +### DHT-3 (Low): a followed name's owner can fill the follower's disk +- **Where:** `features/follow.py:316` (`fetcher.open(target, pinned=True)`). The default + `max_download_size` is None, and pinned fetches never stop at, or evict for, `max_storage`. +- **Impact:** + - A followed name can pin content of any size, up to the disk reserve. + - During a target swap it briefly holds about twice that. + - Unlike LXMF auto-fetch, there is no per-source quota. +- **Verified:** re-run (`max_storage` 64 KiB, 4.2 MB pinned). +- **Fix:** add a per-follow size cap (for example `follow_max_size`, defaulting to a fraction of + `max_storage`), or count follow pins against `max_storage`. + +### FEAT-1 (Low): one auto-fetch sender can push out another's pinned messages +- **Where:** `features/lxmf.py:599-614` (`MessageLog.add`), `_fetch_and_pin` (around `:1505`), and `received` + (`per_from=None` for auto senders). +- **Mechanism:** + - Re-linking a CID the sender already pins records "pinned" without using quota, and those records are + protected. + - Auto senders have no per-sender cap, so about 500 such messages fill the inbox. + - After that, each new message evicts the oldest record overall, including another sender's pinned message, + and `_release` unpins it. +- **Impact:** this breaks the spec's threat-table promise (spec line 1001, "auto-fetched pins never pushed out + by others' messages"). +- **Verified:** re-run. The attacker used 47 bytes of quota; the victim's message and its `lxmf:` pin are gone. +- **Fix:** + - When every record is protected, drop the incoming sender's own oldest record, or refuse the message. + - Cap auto senders per sender as well (a share of `INBOX_LIMIT`). + - Don't count repeated records of the same CID as protecting. + +### FEAT-4 (Low): queued auto-fetches still run after the sender is removed +- **Where:** `features/lxmf.py` `_requeue_auto_fetches` (around `:1404`) and `auto_fetch` (around `:1445`). + Neither checks that the sender is still in `lxmf_auto_fetch_from`. +- **Impact:** after a restart, up to 64 of a removed sender's queued links are fetched and pinned anyway. + `_load_remote_pins` already filters remote pins against the current list; this path doesn't. +- **Verified:** re-run (state "pinned", `lxmf:` pin held). +- **Fix:** skip, and mark "refused", any queued record whose sender is no longer listed. Check both when + re-queuing and inside `auto_fetch()`. + +### FEAT-3 (Low): many identities starve all strangers' messages +- **Where:** `features/lxmf.py:1263` (`_untrusted_token`). +- **Mechanism:** the L12-R allowance limits each identity, but strangers share one node-wide bucket (32 burst, + then 1 a minute). LXMF identities cost one announce each. One message a minute, each from a fresh identity, + keeps the bucket empty. +- **Impact:** every other stranger's message is dropped. Trusted senders are unaffected. +- **Verified:** re-run (0 of 36 strangers' messages kept over a simulated 6 hours). +- **Fix:** + - Announce and enforce an LXMF `stamp_cost` (proof of work) for untrusted senders. + `_announce_app_data` currently announces `None`. + - At least, document that this design only stops a single identity. + +### FEAT-2 (Low): display names can fake a trusted address in `rnfs inbox` +- **Where:** `features/lxmf.py` `_run_inbox` (around `:1695`). +- **Mechanism:** the sender's display name comes from their own announce. It is printed first, unmarked, and + up to 64 printable characters fit `alice ` plus padding. The real address is pushed off to the + right. Contact nicks and trust are never shown. +- **Verified:** re-run: + ``` + #7 2023-11-15 08:43 alice <51515151515151515151515151515151> direct + ``` +- **Fix:** print the address first and the display name quoted after it. Show the contact nick for contacts, + and mark every other sender as untrusted. + +### API-2 (Low): before the proof, a fake listener chooses the client's error +- **Where:** `apiclient.py:336`. `verify_daemon` calls `_send`, which turns the status into an exception + through `error_for` (`:296-302`) before the proof is compared. +- **Impact:** + - While the daemon is down, a process listening on its port can answer 451, 404 or 507 with any message. + - `rnfs.Client` then raises `Denied`, `NotFound` or `StorageFull` carrying the attacker's text, instead of + `AuthFailed`. Apps act on that result. + - The token is never sent. + - The proof exchange also has no default timeout, so the fake listener can hang a client indefinitely. +- **Verified:** re-run. +- **Fix:** check `X-Rnfs-Proof` before mapping any status; any response without a valid proof becomes + `AuthFailed`. Bound the proof exchange with a short timeout. + +### NET-2 (Low): single-packet `/blocks` responses skip pacing +- **Where:** `network/server.py:961-984` (`single = packed <= mdu` returns the response body directly) and + `:946` (`cap` is None without a rate estimate). +- **Impact:** single-packet responses free their slot at once and never produce a rate estimate. + - `max_upload_rate` therefore never applies to a requester that only asks for blocks fitting one packet. + The only per-link limit left is the 2000/s request bucket. + - This needs a large-MTU path: TCP interfaces allow about 16 KiB, Backbone about 32 KiB. + - `upload_budget` still charges these responses in full. + - It matches the spec's wording ("once it has a rate estimate"), but an operator who sets `max_upload_rate` + would not expect it. +- **Verified:** PoC on a fake link with a 32000-byte MDU and `max_upload_rate` = 1000 bit/s. 50 single-packet + responses (1 MB in all) were served with no pacing delay, and the rate estimate stayed None. A multi-packet + response afterwards got a 480 s wait. +- **Fix:** pace single-packet responses too, using `unmeasured_bitrate` or the interface bitrate when there is + no estimate. + +### NET-3 (Low): request packets on outgoing links are decoded before any rnfs check +- **Where:** `network/guards.py:245-264`, which guards only response-context packets. +- **Mechanism:** on links rnfs opens, the remote destination has no `max_request_size`. RNS therefore + msgpack-decodes any request packet, up to the link MDU, and starts a thread for it before finding there is no + handler. +- **Impact:** some memory and thread use, which any node rnfs contacts can cause. +- **Verified:** PoC. `guarded_receive` forwards a request packet on an outgoing link to RNS, while it drops an + oversized response on the same link. The RNS 1.5.6 side (decode plus a thread for each packet, `Link.py` + around lines 994-1006) is read from source, not run live. Since the attacker must be a node rnfs chose to link + to, it is a modest amplifier. +- **Fix:** in `guarded_receive`, drop request-context packets on rnfs.dht links this node opened, without + reading them. This mirrors the existing drop of responses on incoming links. + +### SUP-2 (Low): `install.sh` doesn't tie the checked sdist to the requested tag +- **Where:** `install.sh:350-357` and `:534-537`. + - `gh attestation verify` gets `--repo` and `--signer-workflow`, but no `--source-ref`. + - The wheel loop accepts any `rnfs-*.whl`. +- **Impact:** with release-asset write access, an attacker can rename an older, genuinely attested sdist (a + known-vulnerable version) to the new version. It passes both SHA256SUMS and the attestation check. +- **Fix:** + - Pass `--source-ref refs/tags/$RELEASE_TAG` (and `--deny-self-hosted-runners`) where `gh` supports them. + - Require the wheel to be named `rnfs-${RELEASE_TAG#v}-*.whl`. + +### SUP-3 (Low): provenance checking is off by default +- **Where:** `install.sh:363-370`. +- **What happens:** without a logged-in `gh`, which covers most `curl | sh` users and root, the only check is + SHA256SUMS from the same release. That only shows the download is complete. This is documented, and + `RNFS_REQUIRE_ATTESTATION=1` makes the check required. +- **Fix:** require the check whenever `gh` is available, and say plainly in the README that `curl | sh` + depends on who can write GitHub release assets. + +### SUP-4 (Low): the `rnfs` name on PyPI is unclaimed +- **What happens:** `https://pypi.org/pypi/rnfs/json` returns 404, while README.md:322 says plain + `pip install` works. Anyone can register `rnfs`, and a user running `pip install rnfs` would get their code. +- **Fix:** register the name, either by publishing through trusted publishing or with a placeholder that + points to GitHub. + +### SUP-5 (Low): the Flatpak sandbox is nominal +- **Where:** `packaging/flatpak/io.github.afit21.rnfs.yml:19,23` grants `--filesystem=home` and + `--device=all`. +- **Impact:** rnfs parses untrusted network input. If the daemon is compromised, it can write `~/.bashrc` and + escape the sandbox. +- **Fix:** narrow it to `~/.rnfs:create`, `~/.reticulum:create` and `xdg-download` (or use portals), and make + `--device=all` opt-in. + +### CONT-2 on POSIX +See [CONT-2](#cont-2-medium-on-windows-low-on-posix-write-out-follows-a-swapped-directory). On POSIX it is Low, +because the default umask keeps others out of the output tree. + +--- + +## Info + +| ID | Note | +|---|---| +| CONT-3 | `SharedFolder.scan` follows a symlink at `files/` (by design). If the owner points it at a directory whose parent another user can write, that user can retarget it at `~/.rnfs/storage`. The identity file would then be pinned and served, though only to someone who knows its unguessable CID. Possible fix: warn when the folder's parents aren't owner-only, or pin the folder's dev/ino. | +| CONT-4 / API-6 | A `sha256:` deny entry doesn't stop a multi-chunk file being served chunk by chunk (`?format=raw`, `/blocks`). Documented in spec §8 ("deny the CID for that"). | +| API-3 | Unauthenticated callers also get the kept-open proof connection (`api.py:753-766`). This doubles how long the accepted slot-exhaustion DoS can hold a slot (about 2 × `request_timeout`). The keep-open is otherwise bounded to exactly one more request. | +| API-4 | The fallback reconnect (`apiclient.py:324-326`) keeps the old proof-then-reconnect race for pre-0.1.5 daemons. Fall back only when `daemon_version < 0.1.5`, and drop the fallback later. | +| API-5 | A user HEAD, or a range not starting at 0, sniffs the first 512 bytes of a denied stored chunk (`gateway.py:607` → `node_content.stored_head:174`, which bypasses the deny list). Only the Content-Type leaks. A HEAD of a hash-denied file answers 200 where GET answers 451. | +| API-7 | A missing `Sec-Fetch-Site` header counts as the user's own request. Old browsers without Fetch Metadata (Safari < 16.4, Firefox < 90) therefore reopen the L6 `?format=` storage oracle. | +| DHT-4 | In private mode, name records are still announced network-wide, so outsiders learn members' `[sequence, CID]` and can link name changes. Content stays member-only. Document it, or warn on `name publish` in private mode. | +| DHT-5 | A node's own provider record can be evicted from its own store ([providers.py:385](rnfs/dht/providers.py#L385)) after about 1000 Sybil stores to the key (50 of 50 trials). Exempt `own_id` when choosing what to evict. | +| DHT-6 | The K = 10 closest IDs to any key can be ground in seconds (about 31k IDs/s per core), which makes eclipsing a key cheap. Already listed in spec §10 Limits; noted here only for the cost. Fixing it needs a design change. | +| NET-4 | A malicious provider can hold one of our `/blocks` fetches for hours: with no rate estimate the stall deadline assumes 100 bit/s, so 64 KiB takes about 4.4 h. Matches the spec; whether it matters depends on the fetcher racing other providers. | +| NET-5 | On the responder, `link.rtt` is whatever the requester claims (RNS takes the max). It is used only capped at `PROOF_GRACE` (5 s); any future server-side use must stay capped. | +| FEAT-6 | `printable()` escapes terminal controls, BiDi overrides and zero-width characters. It lets through visible-but-blank characters (U+3164, U+FFA0, U+2800), combining marks and literal backslashes, so names can look alike or look blank. | +| FEAT-7 | `examples/rnfs_share.py` prints names and errors unescaped (local content only). `remote.Client._report` passes daemon messages to logging unescaped. | +| FEAT-8 | `companion-nomadnet-sites/file-index-node/index.mu` has no lock or rate limit, unlike the display page. Each visit runs `rnfs pin ls`. | +| SUP-6 | The systemd user unit lacks `ProtectSystem=full`, `PrivateTmp` and `UMask=0077`. The deb postinst runs `python3 -m compileall` without `-I`. Plain-`http://` `RNFS_SOURCE` URLs are accepted without a warning. No Authenticode, apt or Flatpak signing, and no SBOM. | + +--- + +## Checked and found sound + +**Network:** +- Request parsing is strict: types, duplicate CIDs, the 55-key `/provide` cap, bool kept apart from int. RNS + enforces the 1 KiB request limit before decoding. +- Handler exceptions are contained by `_guarded`. +- Response parsing is bounds-checked, with nothing allowed after the response. +- The Resource advertisement guard refuses request Resources, oversize transfers, multi-segment, metadata and + compressed advertisements. +- Identity binding holds for `/provide` and LinkPool. +- Private mode is enforced on every handler. +- `/pin` is checked against the allowlist before parsing. +- `/blocks` serves only DAGs this node provides, checks the deny list first, and has no storage oracle. +- Bookkeeping maps are bounded. +- H1, M1, M2 and L17 have not regressed. + +**Content:** +- CIDs: only the exact raw and rnfs-dag sha256 prefixes are accepted, the text form is canonical, and no CID + is built from untrusted bytes. +- DAG decoding is strict, with a byte-identical re-encode check, and validates names, order, kinds and size + overflow. +- The walker checks each block against what was expected and dedupes repeated nodes, so repeated CIDs can't + blow up exponentially. +- Export names are sanitised for POSIX and Windows, including ADS, reserved names and trailing dots or spaces. + Writes use O_EXCL. +- Archives cap and verify everything before use. +- The fetcher verifies blocks and drops unrequested ones. +- Sealing is encrypt-then-MAC with a constant-time HMAC check before unpadding, HKDF-separated keys, a random + IV per segment, and tags that bind the header, index and final flag. +- The builder and shared folder use fd-relative `O_NOFOLLOW` opens with a dev/ino recheck, and skip hard + links, so the L8 and L8-R2 fixes are complete on POSIX. +- `winacl` was reviewed but not run on Windows. + +**DHT and names:** +- A name record's signature covers every field, and the destination hash is checked before the signature. +- Records are canonical msgpack. +- The rollback, floor and sequence rules hold, and far-future times are clamped. +- The L4/L4r and M3/M3r1/M3r2 fixes are complete. +- A 120-seed provider-store fuzz (`fuzz_providers.py`), with clock jumps, retain, save and reload, held every + accounting invariant. +- Routing admission, bans and pruning hold. +- Lookup replies are trimmed, and requests for unknown IDs are budgeted. +- Private-mode membership checks hold. +- Keeper storage is bounded, and records are re-verified. + +**API and gateway:** +- The token is 256 bits, compared with `compare_digest`, and checked before any body is read. Only + `/api/status` is open, and it returns just the version. +- The challenge proof is bound to the port. +- The token file is written atomically with O_EXCL, O_NOFOLLOW and mode 0600. +- Host is matched exactly, which blocks DNS rebinding. No CORS headers anywhere. +- Every gateway response, including `http.server`'s own errors, gets `CSP: sandbox` and `nosniff`. +- `filename*` is percent-encoded, so there is no CRLF injection. +- All SQL is parametrised, and the state DB and its WAL/SHM files are 0600. +- The debug log is opened O_EXCL, mode 0600, and contains no secrets. +- Deny-list CIDs are canonical, so there is no alternate-encoding bypass, and they are checked on every path. + +**Features and CLI:** +- LXMF accepts only signature-validated messages. +- Strangers can never trigger fetches or pins. +- Inbox, outbox and the hold queue are bounded (L12, L13 and L13-R complete). +- The auto-fetch quota holds. +- Jobs are re-validated when loaded. +- Contact trust is per kind. +- Every untrusted string the CLI prints goes through `printable`. +- The page scripts validate their parameters, call subprocesses with argument lists and no shell, and escape + Micron output correctly. + +**Supply chain:** +- All 52 lock pins are `==` pinned with `--hash`, matching PyPI exactly. None are yanked or carry a known CVE. +- Installs use `--require-hashes --only-binary=:all:`. +- Every Action is pinned by SHA (verified against the upstream tags), and containers by digest. +- No `pull_request_target`, and no `${{ }}` values spliced into `run:`. +- `persist-credentials: false`, and permissions are least-privilege. +- SHA256SUMS is attested. +- `install.sh` uses `-I`, `mktemp`, safe tar and zip extraction, and refuses to overwrite files it didn't + make. +- The service files run as the user, never as root. + +--- + +## Method + +- **Six agents in parallel**, one per attack surface: network server; content and filesystem; DHT, names and + keepers; local API and gateway; LXMF, features and CLI; supply chain. +- Each read the relevant parts of `specification.md` and the existing `tests/security/` regression tests + first. The aim was to check old fixes for completeness rather than report them again, and to focus on code + new in 0.1.5 (about 7.8k changed lines). +- PoCs ran against the real code with the test suite's fakes. pytest isn't installed in this environment, so + they run under `unittest`. +- Every finding marked "PoC, re-run" was run again for this report. +- **Existing regression suite:** `python3 -m unittest discover -s tests/security -p "test_*.py"` ran 474 tests, + all OK, with 35 skipped. The skips are the RNFS_E2E real-RNS cases. +- Not covered: + - live tests over real RNS (the RNFS_E2E suites); + - running `winacl` on Windows; + - the GitHub repository settings: required reviewers on the `release` environment, and tag rulesets. + +## Suggested fix order + +1. **CONT-1, API-1, CONT-2:** small, contained changes that close a remote DoS and two paths to local code + execution. +2. **SUP-1, SUP-4:** a one-line CI change, and claiming the PyPI name. +3. **DHT-1, FEAT-5, FEAT-1, FEAT-4:** all wire-compatible. +4. **NET-1, NET-2, NET-3, DHT-2, DHT-3:** these need small design decisions (cooldown policy, a follow quota). +5. Then the remaining Low and Info items. FEAT-3 needs a decision on LXMF stamps. diff --git a/0.1.6-plan.md b/0.1.6-plan.md new file mode 100644 index 0000000..740f5aa --- /dev/null +++ b/0.1.6-plan.md @@ -0,0 +1,1001 @@ +# 0.1.6 plan + +The work for 0.1.6, alongside the feature items in [roadmap.md](roadmap.md). It merges three inputs from 2026-10-17: +- **Security fixes** for [0.1.5-audit.md](0.1.5-audit.md) (IDs `CONT-*`, `DHT-*`, `API-*`, `FEAT-*`, `NET-*`, `SUP-*`). +- **Maintainability work** from [maintainability-audit0.1.5.md](maintainability-audit0.1.5.md) (IDs `TS*`, `XC*`, `CT*`, `NW*`, `DH*`, `ND*`, `FT*`). +- **Performance work** from five read-only reviews (IDs `A*`–`E*`, `S*`). + +Every item uses its source ID, so its full write-up can be found in that document. A design pass checked each +security and maintainability item against the tree at `5bd3075` (v0.1.5) and confirmed it. The line numbers +below are from that tree. Where the design pass corrected an audit, this plan says so. + +**Rules for every item:** +- No wire changes: request paths (including feature paths such as keepers' `/name`), messages, CIDs, DAG bytes, + chunk boundaries, provider and name records, and announce app_data stay byte-identical. 0.1.6 nodes must work + with 0.1.5, 0.1.4 and 0.1.3 nodes. +- Server-side policy may tighten (cooldowns, pacing), but only in ways an honest 0.1.5 fetcher already handles. +- The local HTTP API stays compatible, status codes included. Where an item changes a default, it is listed + under [Decisions](#decisions-needed-before-starting). +- Local state may change only with an automatic one-way upgrade. Downgrades need not work. +- rnfs is the core, and user-facing features belong in companion apps. +- Every security fix ports its PoC into `tests/security/test__*.py`. + +**Numbers:** they come from the reviewers' NVMe machine and from simulated links (3 providers, 40 Mbit/s, 50 ms +RTT). Treat them as relative. Re-measure the items marked **(measure on Pi)** on an SD card before and after. + +--- + +## Decisions needed before starting + +Each row has a recommended answer. Record the decision in the last column. An implementer who meets a row still +marked *open* builds the items that don't depend on it and asks about the rest. + +| # | Question | Recommended | Blocks | Decision | +|---|---|---|---|---| +| Q1 | **B:** may spec §8 change from "never written under a CID name" to "a damaged recent block is detected and removed after a power loss"? | Decide after the B benchmark | B | Deferred to 0.1.7 (maintainer, 2026-10-18): see B's gate note | +| Q2 | **CONT-1:** while the deny list has `sha256:` entries, refuse files whose declared size exceeds 16 × their distinct leaf bytes + 64 MiB (451), or use a byte cap? The ratio refuses honest, highly repetitive files over 64 MiB, such as a zero-filled disk image. | Ratio | CONT-1, API-5 | Ratio (maintainer, 2026-10-17) | +| Q3 | **FEAT-5:** `POST /api/lxmf/inbox/get` fails closed (404) on an unresolvable name unless `use_sent_cid` is set. This changes the endpoint's default. | Yes | FEAT-5 | Yes (maintainer, 2026-10-17) | +| Q4 | **NET-1:** cool down a `/blocks` requester that rejects or cancels before a quarter of the response is sent, using the flat existing 300 s cooldown with no growth and no strike counter? | Yes | NET-1 | Yes (maintainer, 2026-10-17) | +| Q5 | **NET-2:** without a rate estimate, apply only an explicit `max_upload_rate`, not a cap derived from `unmeasured_bitrate`? | Yes | NET-2 | Yes (maintainer, 2026-10-17) | +| Q6 | **API-1:** refuse a symlinked `rnfs.toml`; refuse subdirectories owned by another user, root included; also check RNS's `config` when `reticulum_dir` is the rnfs dir; scan a once-loose `blocks/` for foreign-owned files (warning only)? | Yes to all | API-1 | Yes to all (maintainer, 2026-10-17) | +| Q7 | **API-7:** a request with no `Sec-Fetch-Site` counts as the user's own only if its `User-Agent` doesn't start with `Mozilla/`. Old browsers then lose `?format=` downloads; curl and scripts keep them. | Yes | API-7 | Yes (maintainer, 2026-10-17) | +| Q8 | **SUP-3:** `install.sh` stops when `gh` has attestation support but isn't logged in, unless `RNFS_REQUIRE_ATTESTATION=0`. Root installs usually hit this. | Yes, the error names the opt-out | SUP-3 | Yes, the error names the opt-out (maintainer, 2026-10-17) | +| Q9 | **SUP-5:** narrow the Flatpak sandbox to `~/.rnfs`, `~/.reticulum` and `xdg-download`, with `--device=all` opt-in through `flatpak override`. Existing Flatpak users lose access to other folders and serial RNodes until they override. | Yes, with a CHANGELOG upgrade note | SUP-5 | Yes, with a CHANGELOG upgrade note (maintainer, 2026-10-17) | +| Q10 | **DHT-1:** cap each `Lookup` run at 80 queries and stop after 9 answers in a row with no progress, tuned against the 2000-node simulation? | Yes | DHT-1 | Yes (maintainer, 2026-10-17) | +| Q11 | **DHT-3:** a new `follow_quota` setting, default 1 GiB, matching `remote_pin_quota`? | Yes | DHT-3 | Yes (maintainer, 2026-10-17) | +| Q12 | **FEAT-1:** limit each auto-fetch sender to 1/n of the inbox's record and byte limits (n = number of auto senders)? | Yes | FEAT-1 | Yes (maintainer, 2026-10-17) | +| Q13 | **CONT-2 (Windows):** after a `get`, restore inheritance from the parent, or leave the output private? | Restore | CONT-2 | Restore (maintainer, 2026-10-17) | +| Q14 | **CONT-3:** refuse a shared folder that resolves into the rnfs dir (or an ancestor of it), and only warn for loose ancestors? | Yes | CONT-3 | Yes (maintainer, 2026-10-17) | +| Q15 | **FEAT-2:** is the `rnfs inbox` text format a stable interface? The fix reorders it. | No; scripts use the API | FEAT-2 | No; scripts use the API (maintainer, 2026-10-17) | +| Q16 | **SUP-6:** add `ProtectSystem`/`PrivateTmp` to the default systemd unit? In a user unit they need unprivileged user namespaces. | No; document them as an opt-in drop-in | SUP-6 | No; document them as an opt-in drop-in (maintainer, 2026-10-17) | + +**Maintainer-only actions** (an implementing agent can't do these): +- Claim `rnfs` on PyPI (SUP-4), either by uploading the attested v0.1.5 files or by setting up trusted publishing + with a `pypi` environment. A "pending publisher" alone does not reserve the name. +- In GitHub settings: give the `release` environment required reviewers and limit it to `v*` tags. Consider a + tag-protection rule for `v*`, and limiting who can run `workflow_dispatch`. +- Turn on Dependabot alerts. +- Run the Pi SD-card benchmarks (B, and the **(measure on Pi)** items). +- Your own `~/.rnfs` has a 0664 `rnfs.toml` and 0775 `blocks/`, `state/`, `storage/` and `files/`. API-1 tightens + these at the next start; `chmod` them now if you like. + +--- + +## Order of work + +Each phase is a few PRs. A phase may start once the phases it depends on have merged. Inside a phase, items that +touch different files can run in parallel. + +| Phase | Contents | Why here | +|---|---|---| +| **0. Safety nets** | SUP-1, TS1, TS2, TS3, TS10, TS4 | Every later change is checked against these. TS2 goes before any schema change, TS3 before any fetcher or protocol change. | +| **1. Security: Medium** | CONT-1 → API-5; API-1 with ND6; CONT-2; NW2 step 1 → NET-1; FT2 extract → DHT-1; FEAT-5 | Remote DoS, two paths to local code execution, and a provenance issue. | +| **2. Security: Low and Info** | NET-2, NET-3, NET-5; DHT-3, DHT-5; FT1 → FEAT-4, FEAT-1, FEAT-2, FEAT-3; API-2, API-4, API-7; CONT-3, CONT-4; DHT-4; FEAT-6, FEAT-7, FEAT-8; SUP-2, SUP-3, SUP-5, SUP-6 | Small and mostly independent. | +| **3. Quick wins** | Perf A3, C1, C2, S1, S2, S4, S6–S9 (S3 goes with XC2 in phase 4, S5 waits for DH2). Maintainability batch M1. | Low-risk speed-ups, and cleanups that make later phases safer. | +| **4. Lifecycle, state, write path** | XC5 → XC2 (with S3); XC8 `blockstore.matches`; B (after Q1) | D1 adds a store; B changes the write path. | +| **5. Storage** | CT1, CT4, CT2 → D1, D2, D3 | Needs CONT-1 and CONT-2 (walker), XC5 and XC2 (D1's store) and CONT-3 (D1's scan guard). | +| **6. Fetcher** | XC7 FakeClient, NW1 subset, NW5 → A1, A2, A4; then NET-4 (stretch) | Needs TS3, CONT-1 and NET-1/NET-2. | +| **7. Names, DHT, node** | DH1 step 1, `_NameFloor`, XC6 names waiter, DHT-2 → E1; DH2 → S5; E2 (after CONT-1); E3 | Needs DHT-1. | +| **8. Shared helpers and docs** | XC3, XC4 steps 1–2, ND3, ND4, FT4, FT5, rest of XC8; TS5–TS9, TS11, TS12 | Can run any time after phase 3. FT4/FT5 wait for the FEAT and DHT-3 fixes. | +| **9. Release** | [Release checklist](#release-checklist) | | + +--- + +# Part 1: Security fixes + +Every fix here is local policy or local code. None changes the wire. Sizes: S is under a day, M is a few days. + +## Medium + +### CONT-1. Bound the deny-list file-hash check (S, low) +- **Where:** + - [node.py:799-813](rnfs/node.py#L799-L813) `_check_file` hashes the whole declared size. + - Its callers: `_read` (`cat`, gateway GET and sniffing), the sealed `get`, `write_out` through `get`, + `node_content.export_stream` and `import_archive`. +- **Change (assumes Q2 = ratio):** + - New `walker.leaf_bytes(expect, get, stop_at)`. It walks file nodes only, each distinct node once, and adds + `child.size` once per distinct raw child CID. It never reads raw blocks, and it stops early at `stop_at`. + - `_check_file(expect, get)` now takes a getter, not a reader. In order: + 1. A raw digest, or a remembered hash, works as today. + 2. Otherwise compute `allowed = DENY_HASH_RATIO × leaf_bytes + DENY_HASH_SLACK`, with constants in + `denylist.py` set to 16 and 64 MiB. + 3. If the declared size is over `allowed`, raise `Denied` (451): "can't be checked against this node's + deny-list file hashes: it repeats its content more than 16×". + 4. Otherwise hash as now. + - Use `Denied` rather than `TooLarge`, so `_refusing` collects what a refused import stored. + - In `_read`, when `deny.hashes` is set, call `fetch.fetch_dag(...)` before the check, then pass `fetch.get`. + This adds `check_disk`. + - With the pre-check in place, `read_range` doesn't need a byte budget. +- **Docs:** + - spec §8, deny-list bullet: the refusal rule and its 451. + - spec §10, "Size lies" row. + - spec §12, local defaults: the two constants. + - design.md deny-list section. +- **Tests:** `test_cont1_deny_hash_amplification.py`: + - The 6-block PoC through `cat`, a gateway `Range: bytes=0-0` request, `?format=archive` and import: each + gives 451/`Denied` quickly, and a counting getter shows no raw block was read. + - A 32 MiB repeated-chunk file, inside the slack, is still served. + - An honest denied file still gives 451. + - With no hash entries, nothing is walked. +- **Order:** land before or with E2 and API-5, which edit the same functions. +- **Done:** `5454d5d` + +### API-1. Check `rnfs.toml` and existing subdirectories on POSIX (S–M, medium: new startup refusals) +- **Where:** + - [config.py:274-282](rnfs/config.py#L274-L282); `winacl.secure_settings` does nothing off Windows. + - [node.py:112-139](rnfs/node.py#L112-L139). + - `cli.py:187` and `embedded.py:68` read the settings before the root is locked. + - Also unchecked: `lxmf/`, and RNS's `config` when `reticulum_dir` is the rnfs dir. +- **Change:** + - A new `fsutil.check_owned_fd(path, fd, info, what) -> warning | None`, shared with + `network/identity._check_private`. + - **`rnfs.toml`:** + 1. `os.open(O_RDONLY|O_NOFOLLOW|O_NONBLOCK|O_CLOEXEC)`. `FileNotFoundError` means no settings, which replaces + the `isfile` check; `ELOOP` means refuse. + 2. `fstat`: refuse anything that isn't a regular file, or whose `st_uid != geteuid()`. + 3. If `mode & 0o077`, `fchmod 0600` and add a warning. + 4. `tomllib.load` from that fd, capped at 1 MiB. + 5. The same check on `/config` when `reticulum_dir` is the root. + - **Subdirectories:** `_make_private_dirs(root)` opens each of `state`, `blocks`, `storage`, `names` and `lxmf` + with `O_DIRECTORY|O_NOFOLLOW`, creating it 0700 if missing. A symlink, a non-directory or a foreign owner is + refused with `RnfsError` naming `chown`. A loose one is `fchmod`ed to 0700 with a NOTICE. When the old mode had + group or other write, one `lstat` pass warns about foreign-owned entries. + - **Order at startup:** root, then settings, then subdirectories, then the lock. This applies in both + `run_daemon` and `rnfs.Node`. `_make_private_dirs` takes the root, not a `Config`. + - **Warnings without importing RNS** (this is also ND6): + - `winacl.secure_settings` returns its reason instead of logging. + - `Config.load` stores warnings on `config.warnings`, a plain attribute rather than a dataclass field, so + rnfs.toml can't set it. + - The daemon prints them with `cli.warn`; `rnfs.Node` logs them; the CLI client prints them; `rnfs.Client` + raises `warnings.warn(RuntimeWarning)`. + - **Windows:** also run `winacl.tighten(d, directory=True)` on existing subdirectories. +- **Docs:** spec §11. Existing subdirectories MUST be real directories owned by the user, and loose ones are made + owner-only. A symlinked `rnfs.toml` is refused. CHANGELOG: dotfile setups that symlink `rnfs.toml` stop working. +- **Tests:** + - `test_api1_settings_file.py`: 0666 becomes 0600 with a warning. A symlink or FIFO is refused without + blocking. A foreign owner is refused (patch `geteuid`). A `python -I` subprocess shows that loading the client + with a loose file doesn't import RNS. + - `test_api1_subdirs.py`: 0777 becomes 0700. A symlink, a file or a foreign owner gives `RnfsError` before + Reticulum starts. +- **Done:** `939ecae` (with ND6; `winacl.secure_settings` was replaced by `fsutil.check_owned_fd`, which returns the + warning) + +### CONT-2. Make `get` write-out immune to directory swaps (M, medium: rewrites the write-out loop) +- **Where:** + - [walker.py:464-532](rnfs/content/walker.py#L464-L532): `os.mkdir(dest)` at :492, and children created by + path at :511/:513. + - The caller is `node.get` (`node.py:1052-1087`). Get jobs go through it too. +- **Change (POSIX, where `os.supports_dir_fd` and `O_DIRECTORY`/`O_NOFOLLOW` exist):** + - **Partial root:** + 1. `mkdir`, then open it `O_DIRECTORY|O_NOFOLLOW`. + 2. Check its owner and `fchmod 0700`. Using mkdir then fchmod avoids the process-global umask and keeps + setgid. + 3. Require the directory to be empty. + 4. Record its dev, ino and mode. + - **Entries:** + - Directories: `mkdir(name, dir_fd=parent)`, then reopen with `O_DIRECTORY|O_NOFOLLOW` and record dev/ino. + - Files: `os.open(name, O_WRONLY|O_CREAT|O_EXCL|O_NOFOLLOW|O_CLOEXEC, 0o666, dir_fd=parent)`. + - **Bounded fds:** the stack holds relative components plus dev/ino, not fds. A popped directory is reopened from + the root fd one component at a time with `O_NOFOLLOW`. A mismatch raises + `ValueError("output was changed while being written")`. Write order and warnings stay as they are. + - **Finish:** `fchmod` the root back to its original mode. `get()` checks `lstat(partial)` against the recorded + identity before `rename_new`. +- **Windows:** + - Create the partial root with `winacl.make_private_dir`; its inherited protected DACL keeps others out. + - After each `open("xb")`, check `fsutil.is_below(final_path(fd), real_root)`. + - Before `rename_new`, a new `winacl.inherit_from_parent(path)` restores inheritance (Q13). +- **Unchanged:** single-file DAGs, which use `open("xb")` in a parent the user chose. +- **Docs:** spec §11 "Output from other nodes"; design.md (the `get` row and the partials paragraph). +- **Tests:** `test_cont2_write_out_swap.py`: + - The deterministic swap PoC: a symlink fails, and nothing appears in the target. A swap to a real directory + fails on dev/ino. + - The root is 0700 during write-out and has its normal mode afterwards. + - An entry planted before the chmod aborts the get. + - 10k subdirectories don't exhaust fds. + - Windows logic through a fake `winacl` backend (the `test_l7_windows_acl` style). +- **Done:** `04b3e2e` (the Windows path is tested only through the fake backend; on Windows a file created through + a junction is left, empty, where the junction led) + +### NET-1. Cool down responses cancelled before real progress (S, low) +- **Where:** + - [server.py:1207-1238](rnfs/network/server.py#L1207-L1238) (`_concluded`, `_sent_any`); the cooldown check is + at :902. + - On RNS 1.5.6, any receiver cancel arrives as `REJECTED`. A link teardown arrives as `FAILED`. +- **Do NW2 step 1 first:** document which thread or lock owns each `Response` field. It's documentation only. +- **Change (assumes Q4):** + - Replace `_sent_any` with `_made_progress(response, resource)`: + - `sent = max(response.sent, resource.get_progress() × size)`; + - return `sent >= NO_PROGRESS_FRACTION × size`, with `NO_PROGRESS_FRACTION = 1/4`; + - if progress or the size can't be read, count it as progress, as today. + - In `_concluded`'s `REJECTED` branch, call `self.cooldowns.add(link_id, identity)` when there was no + progress. Reuse `Cooldowns` unchanged: flat `blocks_cooldown`, per link and per identity. + - `FAILED` (link closed) deliberately gets no cooldown: in private mode an honest early-closing reader would cool + its own identity down. Document that as a residual. +- **Honest 0.1.5 fetchers:** + - They never call `Resource.cancel()`: stalls and early closes close the link. + - The only honest RCL is RNS running out of retries on a lossy path. That peer then moves to another provider + after `max_failures`. +- **Constraint for plan A1:** an early-closing reader must close the link, never call `Resource.cancel()`. + Otherwise 0.1.6 would cool itself down at its peers. +- **Docs:** spec §3 Cooldown bullet, the threat table, and Limits. +- **Tests:** + - `test_net1_cancelled_responses.py`: + - a reject at 1% and at one window cools down the link and the identity; + - one response is built per cooldown; + - 25% and 50% get no cooldown; + - a link close at 1% gets no cooldown; + - `get_progress` raising gets no cooldown. + - Move `reject()` from `test_l17` into `_server_helpers`. `test_l17` keeps passing; only its docstrings change. +- **Done:** `0ec517b` (NW2 step 1: `258afae`) + +### DHT-1. Give keeper `/name` queries their own budget, and bound lookups (M, medium) +- **Where:** + - [keepers.py:702-839](rnfs/features/keepers.py#L702-L839): `enough` at 746 and the + `max(1.0, lookup.remaining())` deadline at 809. + - [lookup.py:211-335](rnfs/dht/lookup.py#L211-L335): only "complete", "no node left" or the deadline ends a run. +- **Do FT2's extraction first,** as a behaviour-neutral commit: a `_KeeperQuery` class (`next_batch`, `ask`, + `settled`) replacing the six closures. The behaviour changes then go in a separate commit. +- **Change (assumes Q10):** + 1. **`/name` budget:** each batch waits `max(lookup.remaining(), name_wait)` from when it starts, with + `name_wait = path_timeout + links.ESTABLISH_GRACE + 5 × RNS.Link.ESTABLISHMENT_TIMEOUT_PER_HOP + + links.DEADLINE_FACTOR × protocol.REQUEST_TIMEOUT_WITHOUT_RTT`. That is 87.5 s at the defaults. + 2. **Soft search budget:** + - Stop the provider search when `enough()` holds, or when candidates exist and `lookup_timeout / 5` has passed. + - If the batches don't settle and candidates run out with time left, resume `find_providers` and ask the new + candidates. + - Don't ask candidates as they arrive: reply order must not choose the keepers (spec §9). + 3. **Lookup bounds,** applied in `_iterate` as it stands (DH4's `_Run` is deferred): + - a new `max_queries` parameter, defaulting to 8 × K (80) per run; + - stop after 3 × window answers in a row (9) that don't shorten the bit length of the beta-th closest distance. +- **Docs:** spec §6 Termination (the cap and the no-progress stop); spec §9 Resolve (the soft budget, resume, and + the separate `/name` wait). +- **Tests:** + - `test_dht1_keeper_query_budget.py`: + - slow Sybils that keep returning closer IDs, plus a keeper whose `/name` takes 3 s: the keeper's record is + returned; + - total time is bounded; + - resume finds a later candidate. + - `test_lookup.py`: the adversarial run ends at the cap or on no progress, and honest `find_closest` results on + a simulated 2000-node network are unchanged. +- **Peers see:** fewer `/find` requests. +- **Done:** `73c8295` (FT2 extraction: `fbc4716`) + +### FEAT-5. Don't fall back to the sender's CID silently (S–M, low apart from Q3) +- **Where:** + - [lxmf.py:1281-1301](rnfs/features/lxmf.py#L1281-L1301): `_resolve` swallows NotFound and returns `link.cid`. + - `inbox_get` at :1331. + - `_print_links` (:1659) never shows the CID. +- **Change:** + - `_resolve(link, persist, allow_sent_cid)` returns `(cid, names, fell_back)`. + - `inbox_get(..., use_sent_cid=False)` checks names while planning, before writing anything or submitting a job. + An unresolvable name raises NotFound, naming `--use-sent-cid`. With the flag, the result item carries a + `warnings` entry, which `cli.print_warnings` already shows. + - CLI: `rnfs inbox get --use-sent-cid`. API: an optional `use_sent_cid` on `POST /api/lxmf/inbox/get`. + - `_print_links` adds "sent as ``" to name links. + - Auto-fetch (trusted senders) keeps the fallback, records `"sent_cid": true`, and the listing says "auto-fetch + pinned (sent CID; name unresolved)". +- **Docs:** spec §13 "Fetching a link"; the §11 CLI table; design.md:878; CHANGELOG (the API default change). +- **Tests:** `test_feat5_name_link_fallback.py`: + - unresolvable: NotFound, no file and no job; + - with the flag: the file plus a warning; + - a resolvable name always wins; + - the API without the flag answers 404; + - the listing shows the sent CID; + - auto-fetch marks `sent_cid`. +- **Done:** `64ab89a` + +### SUP-1. Turn off the Flatpak build cache in releases (XS) +- **Where:** [release.yml:159-165](.github/workflows/release.yml#L159-L165). The pinned action defaults `cache`, + `restore-cache` and `save-cache` to true. +- **Change:** set `cache: false`. Add a `test_packaging` guard that every `flatpak-builder` step sets it. +- **Order:** do it first; it's in phase 0. +- **Done:** `6fd0cc7` + +## Low + +### NET-2. Pace single-packet responses when an explicit cap is set (S, low–medium) +- **Correction:** the cause is [server.py:946](rnfs/network/server.py#L946). `cap` is None without a rate + estimate. `pacer.started` already runs for single-packet responses. +- **Change (assumes Q5):** use `cap = self.config.effective_max_upload_rate(slow, rate)` unconditionally. That + applies an explicit `max_upload_rate` with no estimate; the default cap stays None. `upload_budget` doesn't + change. +- **Honest 0.1.5 fetchers:** waits up to `MAX_PACING_WAIT` are held rather than refused. A cap far below the link + rate makes them move on, as multi-packet responses already do. +- **Docs:** spec §3 (:234) and the config table (:1339): an explicit `max_upload_rate` applies to every `/blocks` + response. +- **Tests:** + - `test_net2_single_packet_pacing.py`; + - a large-MTU case in `test_pacing.py`; + - update [test_server.py:670-672](tests/test_server.py#L670-L672), which expects no wait. +- **Done:** `c0ec333` + +### NET-3. Drop request packets on links this node opened (S, low) +- **Where:** [guards.py:245-264](rnfs/network/guards.py#L245-L264). +- **Change:** at the top of `guarded_receive`, drop without calling RNS when all of these hold: + - `context == REQUEST` and `packet_type == DATA`; + - `is_dht_link(link)`; + - `link.initiator` is True. + + Log it through a `_ThrottledLog` at VERBOSE. LXMF and host-program links are untouched. +- **Docs:** spec §3 (:232) and the guards.py docstring. +- **Tests:** `test_net3_outgoing_requests.py`: + - an initiator dht link: dropped, with neither `receive` nor `decrypt` called; + - an incoming dht link: passed; + - an LXMF initiator link: passed; + - the existing response check still applies. +- **Done:** `81afb98` + +### DHT-3. Per-follow quota (S, low) +- **Where:** [follow.py:300](rnfs/features/follow.py#L300) `fetcher.open(target, pinned=True)`. +- **Change (assumes Q11):** + - A new `follow_quota` setting, default 1 GiB; 0 disables follow fetches. + - After `fetch.resolve()`, if `size_of(expect) > follow_quota`, raise TooLarge. The follow's state then shows the + failure, and the old target stays pinned. + - Set `fetch.max_dag_bytes = follow_quota`. + - Peak use is 2 × the quota during a swap; document it. + - No migration: targets already pinned over the quota stay. +- **Docs:** spec §9 Following limits; the config table; CHANGELOG. Update `test_l5_follow_cap.py`'s docstring. +- **Tests:** `test_dht3_follow_quota.py`. +- **Done:** `63a0157` + +### DHT-2. Detect a replayed announce's own emission time (S–M, low–medium): phase 7, with E1 +- **Correction:** the audit's suggestion on its own usually finds nothing. RNS caches only announces that take the + path, and `clean_announce_cache` drops the rest. +- **Change:** + - **`RnsNetwork.recalled_emission(name) -> (emitted | None, takes_path)`:** + 1. Look up the remembered packet hash (`known_destinations[name][1]`) with + `Transport.get_cached_packet(..., "announce")`. + 2. Check the packet's type, its destination and that its app_data matches. Then read the emission time from + the random blob. + 3. If it isn't cached: when the remembered hash equals the path's packet (`path_table[name][6]`), use today's + `emitted_time`. Otherwise return `(None, False)`. + - **In `_absorb_recalled` and `_heard`:** a remembered announce that is older than the path's, or didn't take the + path, is a *suspected replay*. Its heard time is its own emission time if known, else + `min(heard, path_emitted)`. A suspect flag makes the resolve ask keepers. + - **Interaction with E1:** E1's refresh cache must not mark a suspected record as fresh. +- **RNS internals:** add the new ones to `test_rns_compat.INTERNALS`: + - `IDX_PT_PACKET == 6`; + - `remember` setting `entry[1]`; + - the announce cache being written only on the `should_add` path. +- **Docs:** spec §9 Resolve; soften §9 Limits ("can't detect a replay when RNS holds no newer path"). +- **Tests:** `test_dht2_replayed_announce.py`: + - the path holds sequence 4 and the recalled announce is sequence 3: keepers are asked and 4 is returned; + - with no keepers: 3 is returned with its real age; + - the path's own announce: no regression; + - the expired-path case. +- **Done:** `c31372b` (a remembered announce is compared by record with the path's cached announce, since a transit + node's path response repeating it has another packet hash; no cached announce on either side means no evidence) + +### FEAT-1. Auto-fetch senders get a share of the inbox (S–M, medium: eviction code) +- **Where:** + - [lxmf.py:543-582](rnfs/features/lxmf.py#L543-L582) `MessageLog.add`: :573 picks the oldest record of any sender. + - `received` (:1180) passes `per_from=None` for auto senders. + - `_fetch_and_pin` (:1512). +- **Change (assumes Q12):** + - `add(..., share=(max_records, max_bytes))` limits the entry's sender, counting its protected records too. + `received` passes `(INBOX_LIMIT // n, MAX_INBOX_BYTES // n)` for auto senders. + - (a) While the sender is over its share, drop its own records in this order: oldest unprotected; then oldest + whose CIDs another of its own records also holds; then oldest. + - (b) When every other record is protected, the victim may only be the same sender's own. If it has none, refuse + the entry. Another sender's protected record is never chosen. + - No migration needed. +- **Docs:** spec §13 inbox paragraph; the threat table (line 1001). +- **Tests:** + - `test_feat1_auto_sender_share.py`: the PoC (600 re-links of one CID) leaves the victim's record and pin intact, + and the share holds by count and by bytes. + - `test_l12_lxmf_inbox.py` stays green. +- **Order:** after FT1's constructor fix. S6 changes `_fetch_and_pin` nearby, in a separate later commit. +- **Done:** `81dc3a1` + +### FEAT-4. Refuse queued auto-fetches from removed senders (S, low) +- **Where:** [lxmf.py:1407-1413](rnfs/features/lxmf.py#L1407-L1413) and `auto_fetch` (:1446). +- **Change:** + - `_refuse_unlisted()`, at the end of `__init__` (after FT1), marks unlisted senders' `queued` records "refused". + This runs even if LXMF fails to start. + - `_requeue_auto_fetches` skips unlisted senders, and `auto_fetch` checks the list again. +- **Docs:** one sentence in spec §13. +- **Tests:** `test_feat4_removed_auto_sender.py`, including the case with LXMF off. +- **Done:** `58fe37a` + +### FEAT-3. Document the stranger-bucket limit; add a counter (XS) +- LXMF stamps would change announce app_data, and refuse senders that hadn't heard the new announce. That is an + interop break, so it's deferred (see [Not doing](#not-doing-in-016)). +- **Change:** a `status()["untrusted_dropped"]` counter, so apps can suggest adding contacts. A spec §10 Limits + row: "many identities can keep the strangers' bucket empty; contacts are unaffected." +- **Tests:** `test_feat3_untrusted_dropped_counter.py`. +- **Done:** `6738659` + +### FEAT-2. Address first in `rnfs inbox` (S, low) +- **Where:** [lxmf.py:1688-1701](rnfs/features/lxmf.py#L1688-L1701). +- **Change:** + - `inbox_list()` adds computed `trusted` and `contact` fields. The API change is additive. + - A new `contacts.nick_of(ident, kind)`. + - Each line starts with the address and says `untrusted`, `trusted` or `contact `, then the display name, + quoted and passed through `printable_label` (FEAT-6). Show `?` against older daemons. +- **Tests:** `test_feat2_inbox_sender_line.py`. +- **Done:** `3707a08` (the line keeps `#N` and the time first, then the address; the display name is last) + +### API-2. Check the daemon's proof before mapping any status (S, low) +- **Where:** [apiclient.py:331-339](rnfs/apiclient.py#L331-L339); `_send` raises at :302, before the proof is + checked. +- **Change:** + - Split `_send` into `_exchange` (connection errors only) and `_raise_for`. + - `verify_daemon` reads at most 64 KiB, then checks `X-Rnfs-Proof` first. + - Without a valid proof: a 503 becomes `Busy`, with a fixed client message; every other status becomes + `AuthFailed`. The listener's text is never shown. + - `PROOF_TIMEOUT = 15` s for connecting and proving, then restore `self.timeout`. + - Keep `error_for` as the only status mapping, so XC4 can drop in later. +- **Tests:** `test_api2_proof_first.py`. +- **Done:** `dda4603` (the proof header is checked before any of the body is read) + +### SUP-2. Tie the installed sdist to the requested tag (S, low) +- **Where:** `install.sh:350-362` and the wheel loop at :535-537. +- **Change:** + - Read `gh attestation verify --help` once. Add `--source-ref=refs/tags/$RELEASE_TAG` and + `--deny-self-hosted-runners` only when the help text lists them. + - With `RELEASE_TAG` set, require `rnfs-${RELEASE_TAG#v}-*.whl`. + - Add a `test_packaging` check for both. +- **Done:** `67bfafe` + +### SUP-3. Require provenance when `gh` can check it (S, medium; Q8) +- `install.sh:363-370`. Stop if `gh` has the attestation command but isn't logged in, unless + `RNFS_REQUIRE_ATTESTATION=0`. With no `gh` at all, keep the warning. +- README: one sentence saying that `curl | sh` without `gh` trusts whoever can write the release assets. +- **Done:** `c64ebf5` + +### SUP-4. Claim the PyPI name (maintainer action) +- The code side: README.md:322 says plainly that pip installs resolve dependencies unpinned. +- Optional: a `pypi` job in release.yml (trusted publishing) that uploads the attested `release-python` artifact. +- **Done (code side):** `4ac4b89`. The optional pypi job was left out; claiming the name is the maintainer's. + +### SUP-5. Narrow the Flatpak sandbox (S; Q9) +- In `packaging/flatpak/io.github.afit21.rnfs.yml:19,23`: + - Replace `--filesystem=home` with `~/.rnfs:create`, `~/.reticulum:create` and `xdg-download`. + - Drop `--device=all`. +- Document `flatpak override --user --device=all` and `--filesystem=...` in the manifest comment and the README. +- **Done:** `021c7c2` (release.yml's best-effort smoke test grants itself its temporary folder) + +## Info + +| ID | Change | Size | +|---|---|---| +| API-5 | `node_content.stored_head` wraps its getter to raise `Denied` for denied CIDs. With hash entries, it runs CONT-1's bounded `_check_file` against stored blocks only (NotFound means "not wholly stored", so it never fetches). A HEAD of a stored denied file then answers 451, like GET. Spec §11 gateway bullet. `test_api5_head_deny.py`. **Done:** `cb5c93f` | S | +| CONT-3 | After opening the shared folder, resolve it with `fsutil.final_path(fd)`. Refuse, with a warning shown once, if it is the rnfs root, inside `storage/`, `names/`, `state/` or `blocks/`, or an ancestor of the root. `names/` holds private keys, so this is worse than the audit says. Warn once for an ancestor others can write to (Q14). Spec §8. `test_cont3_shared_folder_target.py`. D1's cache must sit behind this guard. **Done:** `3fb49ae`, `8d30c61` (a refused folder is scanned as empty, releasing its pins; `lxmf/` refused too) | S | +| CONT-4 / API-6 | Docs only, plus a NOTICE suggesting `rnfs deny add ` when `_check_file` finds a denied hash. Turning remembered denied hashes into CID denies is deferred. **Done:** `c61a324` | S | +| API-3 | Document in spec §11 that the proof keep-open doubles how long a slot can be held. No code change. **Done:** `feba0f0` | XS | +| API-4 | Reconnect only when the proved `daemon_version` is below 0.1.5. Otherwise raise `ConnectionFailed`. Drop the fallback in 0.1.7. `test_api4_old_daemon_fallback.py`. **Done:** `b565ee8` | S | +| API-7 | Apply the User-Agent rule (Q7) in `gateway.limited()` ([gateway.py:449](rnfs/gateway.py#L449)). Spec §11 (:1196, :1222-1225). `test_api7_missing_fetch_metadata.py`. **Done:** `0d010b1` | S | +| DHT-4 | Document in spec §3 and §10 that name announces are visible outside a private network. `name publish` in private mode warns, and the API result gains an optional `warning` key. **Done:** `509c056` | S | +| DHT-5 | In `providers._within_limits` (:385) and `_make_room` (:451), `own_id` doesn't count toward `max_per_key` and is never the victim. `test_dht5_own_provider_record.py` (1000 Sybil stores, 50 seeds). **Done:** `4b72ea7` | S | +| DHT-6 | Deferred: it needs an ID-derivation design change. Add the measured grinding cost to spec §10 Limits. **Done:** `4441878` | XS | +| NET-4 | End-game duplication, requester-side. When `pending` is empty and a provider is idle, re-request a batch that has been in flight longer than `deadline_slack`. The first verified copy wins. Closing the slow request doesn't count as a stall. Spec §7 step 7 ("MAY"). `test_net4_dripping_provider.py`. **Phase 6 stretch, after A1/A4; otherwise 0.1.7.** **Done:** `afa3820`, `ac9ed4f` (20.6 → 3.3 s against a dripping provider) | M | +| NET-5 | Add a `_proof_wait(link)` helper: a finite, non-bool rtt ≥ 0, capped at `PROOF_GRACE`, on `self.clock`. All server-side rtt use goes through it. Add a test with rtt values of 1e9, inf, nan and True. **Done:** `db27b8b` | S | +| FEAT-6 | A new `fsutil.printable_label()` for untrusted labels. It escapes `\`, the blank fillers (U+3164, U+FFA0, U+115F/1160, U+2800), a leading combining mark or more than two in a row, and an all-blank result. `printable()` stays, since it prints Windows paths. `test_feat6_printable_label.py`. **Done:** `0c97eb7` | XS | +| FEAT-7 | Pass `examples/rnfs_share.py` (lines 26, 31, 46, 51, 85) and `remote.py:67` through `printable`. Covered by FT5's escaping test. **Done:** `71d7cb2` | XS | +| FEAT-8 | `file-index-node/index.mu`: copy the display page's `take_lock`, cache the rendered page for 60 s (written atomically), and serve the cache when it is fresh or the lock is busy. Update the README. **Done:** `be571fc` | S | +| SUP-6 | Don't add `UMask=0077`: `get` output relies on the umask (walker.py:548). `ProtectSystem` and `PrivateTmp` go in only as a documented drop-in (Q16). Use `python3 -I -m compileall` in `build-deb.sh:200`. `install.sh` warns on plain `http://` `RNFS_SOURCE`. Widen `test_unit_matches_contrib` to every non-comment `[Unit]`/`[Service]` line except `ExecStart`. **Done:** `c00f8d7` | S | + +--- + +# Part 2: Maintainability + +The design pass mostly confirmed the audit's phasing, with these corrections: +- **NW2 step 1** (document `Response` field ownership) moves into 0.1.6, before NET-1. +- **XC5, XC2 and CT2** come before D1. Otherwise D1 adds a 12th hand-copied store, and its cache hit can't reach + the deny check. +- **FT2's `_KeeperQuery` extraction** moves into 0.1.6 as the base for DHT-1. +- **XC8's token-bucket item is dropped.** lxmf's buckets take from both or neither; `mesh.TokenBucket` goes into + debt. That frees NW6 to delete `TokenBucket.take`. +- **CT4:** the flag is `_maybe_garbage`, not `_garbage`. +- **TS3:** the core paths are `/find`, `/provide`, `/blocks` and `/pin` (constants.py:36-39). `/name` is keepers' + feature path (`keepers.NAME_PATH`), and it exists in v0.1.3. + +## Tests, CI and docs + +| ID | Change | Phase | Size | +|---|---|---|---| +| TS1 | The E2E step runs `tests.test_e2e tests.test_client tests.test_embedded tests.security.test_h1_real_rns tests.security.test_m1_real_rns tests.security.test_h2_display_page_pins`, with `timeout-minutes: 20`. Add a `test_packaging` guard: every module whose code reads `environ.get("RNFS_E2E")` appears in that step. Expect a round of fixes on Windows and macOS. **Done:** `1ede316` | 0 | S | +| TS2 | See [TS2 detail](#ts2-detail). **Done:** `37e1053` | 0 | S–M | +| TS3 | See [TS3 detail](#ts3-detail). **Done:** `cd3313b` | 0 | M | +| TS10 | Add `BLE001`, `E402` and `RUF100` together, since RUF100 alone gives 80 false hits. Add `extension = { mu = "python" }` and `extend-include` for `companion-nomadnet-sites/*/index.mu`, and per-file BLE001 ignores for `tests/**`. Then triage the ~62 rnfs sites: give each a `noqa: BLE001 - ` or narrow it, together with XC4 step 3. Fix the 3 RUF100 hits, `log.py:15` (E402) and the `noqa: B902` in test_builder.py:30. The rest of B (34 hits) waits. **Done:** `cf171ff` | 0 | S–M | +| TS4 | Add `coverage` to `BUILD_TOOLS` (update-lock.py:53) and regenerate build-tools.lock. One Linux job runs `coverage run -m unittest discover tests` with `RNFS_E2E=1`, report only. **Done:** `00bb67b` | 0 | S | +| TS5 | Move the Flatpak sources diff (release.yml:121-139) into tests.yml. Optionally run the release builds on PRs touching `packaging/**`, `pyproject.toml` or `.github/**`, and weekly. **Done:** `25e7cb8` (PR and weekly builds too) | 8 | S | +| TS6 | CONTRIBUTING.md: tests and their env vars (`RNFS_E2E`, `RNFS_FUZZ_EXAMPLES`, `RNFS_BENCH`), lint, locks, releasing (this plan's checklist), adding a schema step, adding an API route. A test that CHANGELOG's first `## X.Y.Z` equals `__version__`. release.yml:264 uses `--notes-file` from that section instead of `--generate-notes`. **Done:** `d094d19` | 8 | S | +| TS7 | A `test_packaging` test that the tests.yml Python matrix agrees with `requires-python`, `BUILD_PYTHONS`/`RESOLVE_ARGS`, `RUNTIME_PYTHONS`, `build-deb.sh`'s lists and `install.sh:154`, and that README's `pyinstaller==` agrees with build-tools.lock. **Done:** `faa62a4` | 8 | S | +| TS8 | `.github/dependabot.yml` for `github-actions` (and `pip`, report only). The composite install action is deferred. **Done:** `e0df4e3` | 8 | XS | +| TS9 | An index from audit ID to module to fix commit (`tests/security/README.md`), and the seven stale `file:line` docstrings changed to cite functions. The regrouping is deferred. **Done:** `188229d` (94 rows; NET-4's row added with phase 6's merge) | 8 | S | +| TS11 | Update roadmap.md (0.1.5 done; the client library and change notifications shipped). Add a version and "wire-identical since 0.1.3" note to the spec, backed by TS3. Optionally, a generated route table with a doc-sync test (see ND4). **Done:** `4252b6c` | 8 | S | +| TS12 | Add `*.egg-info`, `.ruff_cache`, `.venv` and `build-dir` to the Flatpak manifest's skip list. Empty `DESCRIPTIONS` in `file-index-node/index.mu:21-26`, keeping one commented example. Fix the plist comment ("three places"). Moving the gif is your choice. **Done:** `208176a` (README badge alt text fixed too) | 8 | XS | + +### TS2 detail +- **Generator:** `tests/fixtures/rnfs-0.1.5/generate.py`, modelled on 0.1.4's. + 1. `git archive v0.1.5` into a temp dir, first on `sys.path`; assert `__version__ == "0.1.5"`. + 2. Copy the 0.1.4 fixture's `state/`, `names/` and `files/`, and start an offline 0.1.5 node at T0 + 1 day. That + migrates the JSON. + 3. Add 0.1.5-only rows: deny entries (a CID and a `sha256:`), `deny_lists`, `deny_hashes` and meta. + 4. Close (checkpoint), then `VACUUM`. + 5. Write `expected.json` with every table's rows. +- **Commit:** `state/rnfs.db`, the `.migrated` files, `names/`, `files/`, `expected.json` and a README. +- **`test_upgrade_015.py`:** + - `user_version == len(SCHEMA)`; + - `quick_check` passes; + - every expected row is present; + - a restart changes nothing; + - nothing is set aside. +- **Golden test:** + - `RELEASED_0_1_5` is the SHA-256 of `json.dumps([[label, list(stmts)] for label, stmts in SCHEMA[:16]])`. + - The fixture's `sqlite_master.sql` for those tables equals a fresh database's. +- **Python steps (ND5):** widen the step type to `str | Callable[[tx], None]`. `_upgrade` (statedb.py:233-235) + calls a callable step with the transaction. The digest covers only the released SQL prefix. +- **Done:** `37e1053` + +### TS3 detail +- **Generator:** `tests/fixtures/wire/generate.py TREE`, run on `git archive v0.1.3` and `git archive v0.1.4`. + Assert the two runs are identical, then write `vectors.json` as a list of `{name, inputs, hex}`. +- **Encoding:** `hex` is `umsgpack.packb(encoder(...))` of the request or response data only; RNS owns the + envelope. `encode_mode` and `encode_record` already return bytes. +- **Inputs:** fixed keys and CIDs, both raw and DAG. +- **Cases:** + - `/find`: a request; a response empty and with K entries. + - `/provide`: with and without withdrawn. + - `/blocks`: a request; a response with bytes and with None entries. + - `/pin`: true and false, plus the reply. + - keepers' `/name`: request and response. + - Announce mode: server and client. + - Name records: sequence 0, 1 and 2^64 − 1. + - LXMF field: one link and several. +- **Test:** for every vector, `packb(encode(inputs)) == bytes` and `parse(unpackb(bytes)) == inputs`. +- **Done:** `cd3313b` (`tests/test_wire.py`; the v0.1.5 tree gives the same bytes too) + +## Code + +**In 0.1.6.** "Before" names the plan item each one must precede. Every item is behaviour-neutral unless it says +otherwise. + +| ID | Change | Proof | Batch | Before | Size, risk | +|---|---|---|---|---|---| +| FT1 (ctor) | Move the attribute block ([lxmf.py:836-847](rnfs/features/lxmf.py#L836-L847)) and `_lock` above the `try` at :825. **Done:** `b758369` | A new test builds the feature with an unclean inbox and checks every attribute. | M1 | FEAT-1, FEAT-4 | S, low | +| NW4 + NW6 | **NW4:** `_in_slot(link_id, fn)` owns `start_request`/`finish_request`, called after each handler's pre-checks; not `_guarded`, which would change the order. **NW6:** delete `AirtimePolicy.spend` and `TokenBucket.take`, moving their tests to `reserve`/`charge`/`refund`. Factor `protocol.body_refusal()` out of `response_refusal`; `_feature_handler` checks its own responses and answers False if refused. Add `/name` to protocol.py's table. **Done:** `bd2b585` (NW4), `832cd6e` (NW6) | A handler that raises frees its slot; a test over keepers' `/name` responses. | M1 | — | S, low | +| XC4 step 3 | Log at WARNING, with `exc_info` and inside the `except`, at fetcher.py:963, lookup.py:352, links.py:487, mesh.py:99 and peers.py. Move `guards._ThrottledLog` to `rnfs.log.ThrottledLog` and reuse it at server.py:820. Pair with TS10's triage. **Done:** `13f432f` | Logging only. | M1 | A1 | S, low | +| XC6 (tests) | Drop test_publisher.py:618. In test_fetcher.py:755, replace `time.sleep(5)` with an Event released in `tearDown`. **Done:** `357f10a` | — | M1 | — | XS | +| ND4 (tests) | A test that every `ROUTES` value names an `ApiHandler` method, and that the docstring table (api.py:9-35) equals `ROUTES` plus feature `api_routes()`. Remove the stale per-handler parameter lists. **Done:** `913cb7c` | — | M1 | — | S | +| small | DH3: the `ProvidedItem = Cid \| DhtKey` alias and fixed hints. DH4: fix the comment at lookup.py:359. XC8: `chunker.FASTCDC_MAX` (keeping the `MAX_SIZE` alias) with an import-time `<= MAX_BLOCK_SIZE` assert, and `walker.FILE_TYPE`/`DIRECTORY_TYPE` at the literal sites. CT5: a test that `archive_size == len(export)` at the 255/256 and 65535/65536 boundaries. **Done:** DH3 `ebf54fb`, DH4 `f8d000c`, XC8 `33aeb1d` and `0cd296a`, CT5 `cf19230` | — | M1 | — | XS each | +| NW2 step 1 | Document which thread or lock owns each `Response` field ([server.py:233-319](rnfs/network/server.py#L233-L319)). **Done:** `258afae` | Docs only. | phase 1 | NET-1 | S | +| FT2 (extract) | `_KeeperQuery` (`next_batch`, `ask`, `settled`) in place of the closures in `ask_keepers`. **Done:** `fbc4716` | test_names' keepers tests, plus a new direct test (keepers has no test file). | phase 1 | DHT-1 | S–M, med | +| XC5 | `self._stores` filled in `_setup`, behind a `save()`/`close()` Protocol; `_save_state`, `stop` and `_close_state` loop over it. `stop()` becomes a list of guarded `(label, fn)` steps. `Feature.stop(deadline=None)` (call-compatible) with base `_spawn`/`_join(deadline)`; lxmf and jobs drop their 2 s. Remote-pin workers are tracked, and their fetches watch `stop_event`. **Done:** `c08ec37` (store list), `d928762` (stop steps, deadline), `5921441` (follow, deny, keepers join) | A raising `announcer.stop` still closes the db and releases the destination; test_embedded's in-process restart. | M2 | D1, XC2 | M, med | +| XC2 (part) | `statedb.RowStore(db, table, key_cols, *, durable, mode="write_through"\|"snapshot")` owns the dirty set, save lock and restore-on-failure. Add `open_store()` and `reconcile_pins()`. Migrate pin_names (with S3), contacts, jobs and publisher, each keeping its current durability. Tables and LEGACY strings are unchanged. **Done:** `07c0d38` (RowStore), `1f414bc` (pin_names, with S3), `8218829` (contacts), `d3846e8` (jobs), `a21781e` (publisher) | A failing-transaction test per mode; TS2's fixture. | M2 | D1 | M, low–med | +| XC8 `matches` | `blockstore.matches(cid, block)` for the five copies of the validity check. **Done:** `88427f1` | — | M2 | B | XS | +| CT1 | `child_cids`/`names` on dag nodes; `walker.index_child()` used by `walker.step` and `Storage._step_in`; one `_walk_path` generator for `_path_cids` and `_warm`; a `_NO_MEMO` sentinel. **Done:** `f3fc39d` | `_warm` and `_path_cids` reach the same CIDs; test_new5. | M3 | D3 (after CONT-1/2) | M, med | +| CT4 (part) | `_newly_incomplete(targets, act)` (four copies at storage.py:901/989/1028/1050), `_drop_records`, `_set_record`, `_cache_row`/`_parse_cache_row`. **Done:** `a8b918b` | test_storage. | M3 | D3 | M, low–med | +| CT2 | A frozen `builder.BuildOptions(chunker, check)` passed through `build_file`, `build_open`, `add_path`, `add_stream` and `SharedFolder.scan`. `using()`/`checking()` stay as wrappers until every caller has moved. **Done:** `a92edf6` | A build on a worker thread with explicit options still denies; the fastcdc CID vectors. | M3 | D1 (after CONT-1) | M, low–med | +| XC7 (part) | `tests/fakes/client.py` (one FakeClient with `link_mdu`) and a shared quiet-log fixture. `link_mdu` becomes required at fetcher.py:929. Extend test_fake_contract to FakeClient↔PeerClient and FakeBackend↔Backend, which makes `served_block_size` and `feature_requests` required. **Done:** `beebbf9`, `a724ed8` (the server calls the backend's methods directly) | The suite. | M4 | A1 | M, low | +| NW1 (part) | A pure `_DagBudget` (absorbing `_RangeBudget`, `_room`, `_reserve`, `_fits`), passed to `ensure(expects, budget=)` explicitly; NamedTuple events (`ResultEvent`, `ProviderEvent`, `LookupDone`). `progress()` writes only `_heard_at`, and the fetch thread uses the max. `_Providers` and `_Download` go with A1/A4 or later. **Done:** `b2c0111` | test_fetcher; TS3. | M4 | A1, A2, A4 | M, med | +| NW5 | `rates.response_capacity()` and `rates.default_upload_cap()`, used by `config.effective_max_upload_rate`, `server.blocks_budget` and the fetcher. `_pace`'s factors get a comment. **Done:** `80a8ce4` (exactly today's numbers) | — | M4 | A1 (after NET-1/2) | S, low | +| DH1 (part) | `_refresh_name(name, *, ask="always"\|"if_unheard")` shared by `publish` and `resolve`. A `_NameFloor` class (`{name: (sequence, resolved, used)}`) owning trim, load, snapshot, save and the timer. The table is unchanged. **Done:** `7bc4648` | test_names' floor and keepers tests; a floor round-trip test. | M5 | E1 (after DHT-1, with DHT-2) | M, med | +| XC6 (names) | Inject a `wait(event, timeout)` along with the clock into `NameManager._refresh` (names.py:858-867). Swapping in `self._monotonic` alone would hang test_names. **Done:** `171503f` (test_names 34.1 → 26.6 s) | test_names gets about 7 s faster. | M5 | E1 | S | +| DH2 | One `_insert(..., loading)` for `store` and `_load`; `_check_consistency()` in test_providers' `tearDown`; one randomized store/withdraw/expire/reload test. **Done:** `24329b9` | — | M5 | S5 | S–M, low | +| XC3 | A stdlib `rnfs/ids.py` with `parse_hex16(text, what, *, brackets=False, upper=True)`; `config._HEX_ID` kept as an alias. `Config.member_set` and `Config.remote_pin_set` used at node.py:397, links.py:206, node.py:1413 and server.py:684. Keep the accept-set of the persisted-row validators (`storage._IDENTITY_HASH`, `jobs._NODE_ID`), because `_load` deletes rows that fail. `parse_node_id` narrows (no inner whitespace). **Done:** `d042534`; the rest of XC8 (`MISSED_ANNOUNCES`, `dag.KIND_NAMES`): `dc15914` | — | M6 | — | S–M, low | +| XC4 steps 1–2 | `errors.BlockMismatch(RnfsError, ValueError)`; `SealError(RnfsError, ValueError)`; export these and `NotRecipient`. One ordered `errors.STATUSES` table from which `api.ERROR_STATUSES` and `apiclient.STATUS_ERRORS` derive; a test that `gateway.ERROR_MESSAGES` covers every status. The API logs `BlockMismatch` 400s at WARNING. In the shared folder, an unexpected ValueError is transient (`RETRY_AFTER`) and logged, not permanent. **Done:** `d68bf4a` | A test that pins every class's status. | M6 | — (after API-2) | S–M, low–med | +| ND3 | `edit.check_entry_target()` called in `RnfsNode.dir_rm` (before fetching) and in FakeNode; drop the copies in api.py and embedded.py. Leave the "pin is a bool" check where it is. **Done:** `d53b0d8` | Status 400 unchanged. | M6 | — | S, low | +| ND4 (client) | Typed `ApiConnection` methods for the 12 missing core routes, and remote.py's hand-written paths switched to them. **Done:** `a3830f2` | test_public over both backends. | M6 | — | S–M, low | +| FT4 | `Feature.requires`/`optional`, checked at load; `node.feature(name)` typed by a `ContactsApi` Protocol with a contract test; `lxmf._recipient` calls `contacts.resolve_id`. **Done:** `3a83b64` | — | M6 | — (after FEAT-2/5) | S–M, low | +| FT5 (part) | Validate feature API routes, request paths and core collisions at load with one error style. DummyFeature lifecycle tests. Extend the escaping test to jobs, peers and deny (covers FEAT-7). The dead-code list. **Done:** `eb5a2a2` (`_RoundRobin.clear` kept: a security test uses it) | — | M6 | — | S, low | + +ND6 is done inside API-1. **Done:** `939ecae` + +--- + +# Part 3: Performance + +The items are unchanged from the performance reviews, except for the **Depends on** lines added by this merge. + +## A. Fetcher (phase 6) + +### A1. `read_range` stops between windows (measure on Pi) +- **Problem:** [fetcher.py:86](rnfs/fetcher.py#L86) sets `WINDOW = 32` chunks. Each window has to finish before + the next one is requested (around :605–629). + - With 64 KiB chunks a window is only about 2 responses, so a third provider is never used and every link sits + idle at each window boundary. FastCDC's smaller chunks make it worse. + - Measured: a 48 MiB read took 8.97 s, against an ideal of 3.36 s, with links busy 51% of the time. +- **Change:** + - Request the next window before the current one is handed back to the reader. + - [fetcher.py:1069](rnfs/fetcher.py#L1069) currently drops verified blocks that are no longer `outstanding`. + Store them instead. + - If that is too invasive, a fallback: size the window as `2 × max_parallel × chunks per response`, not a + fixed 32 (use NW5's `response_capacity`). +- **Constraint (NET-1):** a reader that gives up early closes its links. It must never call `Resource.cancel()`. +- **Remote load:** a reader that gives up early wastes up to one extra window. +- **Expected:** with A2, 8.97 s → 4.66 s (links 92% busy). `WINDOW = 128` alone gives 5.83 s. +- **Depends on:** TS3, XC7, NW1, NW5, CONT-1 (walker). +- **Tests:** + - A multi-provider `read_range` keeps every provider busy across window boundaries. + - An early-closing reader cancels cleanly, by closing its links. +- **Done:** `0789612`, `ac9ed4f` (a link another fetch uses is left open and its request finishes). Simulated 48 MiB + read: 8.06 → 5.95 s on NVMe (links 57% → 77% busy), 6.12 → 4.73 s on tmpfs (97% busy). WINDOW stays 32. + +### A2. Block writes delay the next request (measure on Pi) +- **Problem:** [fetcher.py:1070-1073](rnfs/fetcher.py#L1070-L1073) runs `_accept` → `session.put` → + `BlockStore.put` (with fsync) for every block of a response. All of that happens on the fetch thread before + `_assign` sends that provider its next request. + - Measured: 63 ms per 16 × 64 KiB response on NVMe. SD cards and eMMC are typically 5–10× slower. +- **Change:** + - Verify and count the response's blocks, call `_assign`, then write them through a pool such as + `Session.writer()`. + - Wait for every write before `ensure` returns, so durability and protection stay as they are. + - `StorageFull` and `ValueError` (`BlockMismatch` after XC4) are raised slightly later, from that wait. +- **Depends on B:** if B removes the per-block fsync, most of this cost goes away. Re-measure after B before + doing A2. +- **Deferred to 0.1.7** with B (Q1): re-measure after B, as this entry says. +- **Expected (with today's fsync):** + - 48 MiB `fetch_dag`: 5.21 → 4.43 s. + - `read_range`: 8.97 → 6.37 s. + - Tree of 8 KiB files: 1.35 → 0.75 s. +- **Remote load:** none. + +### A3. Re-queueing scans the pending deque (phase 3) +- **Problem:** at [fetcher.py:845](rnfs/fetcher.py#L845), `cid not in pending` scans the whole deque. That + happens for every CID of a BUSY, FAILED or STALLED result, and for every CID the responder's budget cut. + - Measured: 20 ms per result at 16k pending (1 GiB); about 200 ms at 10 GiB, all on the fetch thread. +- **Change:** keep a `set` alongside `pending`, updated wherever `pending` is. +- **Tests:** existing fetcher tests. Add one for requeue ordering with a large pending queue. +- **Done:** `e54837a` (22.3 → 0.011 ms per BUSY result at 16k pending; 242 → 0.015 ms at 164k) + +### A4. `fetch_dag` and `fetch_listing` wait for each tree level +- **Problem:** at [fetcher.py:527-539](rnfs/fetcher.py#L527-L539) and :553–563, the last request of each tree + level leaves the other providers idle for up to one response. + - Each level's leftovers also cost a separate small request. On a single LoRa provider that is a full extra + request cycle plus the pacing wait. + - The directory-tree simulation kept links only 30% busy. +- **Change:** + - Queue a block's unseen children as soon as it is accepted, inside `_download`. + - Count `max_entries` per block rather than per level. `_room` and `_reserve` already work per CID (in + `_DagBudget` after NW1). +- **Remote load:** fewer, fuller requests. +- **Complexity:** medium (not prototyped). Keep the walk order the resume logic depends on. +- **Depends on:** as A1. CONT-1's `_read` now calls `fetch_dag` when deny hashes are set, so A4 speeds that up. +- **Done:** `a4eb2e7` (small gains in simulation: tree of 768 × 8 KiB files 2.47 → 2.39 s; LoRa-like single + provider 51.0 → 48.9 s, one request fewer; most of the "30% busy" was the per-block fsync, A2/B's territory) + +--- + +## B. Block write durability (Tier 1 #2): decision needed (Q1, phase 4) + +### What IPFS (Kubo) does +- **Same storage design:** Kubo stores blocks in flatfs, one file per block written to a temp file then renamed, + in sharded directories. That is the same design as rnfs's `BlockStore`. Kubo still calls flatfs "the most + battle-tested and reliable datastore". +- **Kubo used to sync like rnfs:** until v0.33, flatfs ran with `sync: true`, fsyncing every block file and its + directory. That is what rnfs does today. +- **Kubo now syncs at the durability point:** since v0.33, new repos get `sync: false`. The release notes say + flushing on every block write is usually unnecessary, because Kubo flushes before and after critical + operations like pinning. Users who want the old behaviour set `sync: true` and accept slower bulk adds. +- **Batching:** Kubo writes in batches of 128 nodes or 20 MiB (`Import.BatchMaxNodes`, `Import.BatchMaxSize`). + A batch writes all its temp files, renames them, then syncs each touched directory once. +- **Concurrency:** flatfs caps concurrent batch writes at 16 (`maxConcurrentPuts`). Its reasons: macOS's + default limit of 256 file descriptors, HDDs, and memory. rnfs's `WRITE_THREADS = 16` already matches, so + **don't raise it to 64**: that would hit the macOS limit and hurt HDDs. +- **Corruption checks:** Kubo depends on content addressing for this. flatfs is "safe only for content-addressed + data", and `Datastore.HashOnRead` is off by default. + +### Libraries +- No Python library batches durable writes of many small files. `atomicwrites` and similar packages still do one + fsync per file. +- The library route is an embedded key-value store, where one commit covers many blocks. Examples are LMDB + (`lmdb`, a new native dependency) or SQLite (already shipped). This is Kubo's pebble/badger profiles. +- Taking that route means migrating the blockstore format and dealing with eviction and space reclaim. Kubo + itself still defaults to flatfs. **Not recommended for 0.1.6.** + +### Recommendation: copy Kubo's model instead of tuning thread counts +rnfs already has one durability point: `BlockStore.sync()`. It runs before every pin save +([storage.py:300](rnfs/content/storage.py#L300)) and cache save +([storage.py:323](rnfs/content/storage.py#L323)). + +1. **No fsync on put:** `BlockStore.put` ([blockstore.py:73](rnfs/content/blockstore.py#L73)) writes the temp + file and renames it, without the fsync, and adds the path to an unsynced set. +2. **Sync at the durability point:** `BlockStore.sync()` fsyncs every file in that set (at most 16 at a time, + `fdatasync` where the OS has it), then the directory. + - Optional, Linux only: a single `syncfs()` through ctypes could replace the per-file loop. Only adopt it if + the measurement shows a clear win. +3. **Sync outside the lock:** `Session.writer()` close syncs its own blocks before returning, outside the storage + lock. `_save_pins` runs `sync()` while holding the lock, so it should find nothing left to do. + - The fetcher's blocks get synced at its `record_fetch` or pin step, as today. +4. **Keeping "no empty or short file under a CID name":** today's per-file fsync is what guarantees this + ([specification.md](specification.md) §8). + - Before the first unsynced write, create `blocks/.dirty` (one sync). Remove it after `sync()` completes. + - At startup, if `.dirty` exists, hash-check the blocks modified since it was created (with a slack for + clock steps). Delete any that don't match; they get fetched again. Use XC8's `blockstore.matches`. + - Only an unclean shutdown pays for this check. + - API-1 makes `blocks/` owner-only, so a recent block can't be planted by another local user in the meantime. +5. **Docs:** + - specification.md §8: the pin and cache guarantee is unchanged. Add that a damaged recent block left by a + power loss is found and removed at the next start. + - Update design.md's storage section and `BlockStore`'s docstring to match. + +### Decision gate before implementing +Benchmark a 256 MiB add, and a fetch of the same size, on NVMe and on a Pi 4 SD card in these variants: +- today's code; +- `fdatasync` only; +- deferred per-file fsync; +- deferred `syncfs`. + +Go ahead with the deferred design only if it is at least about 1.5× faster on the SD card. If it isn't, fall +back to `fdatasync` (about +15%) and keep 16 threads. + +**Decision (Q1, 2026-10-18): deferred to 0.1.7.** No Pi run; a model of an SD card stood in for it. An SD card +handles one flush at a time, a few to tens of ms each, so for 256 MiB (~4100 blocks of 64 KiB at ~20 MB/s): +today's per-block fsync takes ~22–115 s (flush latency 2–25 ms); deferred per-file fsync about the same (its NVMe +gain came from 16 flushes in parallel, which an SD card can't do); deferred `syncfs` ~13 s (1.7–9×). So the gate +would pass only in the `syncfs` form, which needs: Linux only (other platforms keep per-block fsync); a plain +filesystem (on a stacked one such as ecryptfs `syncfs` writes nothing back, so rnfs must detect it and fall back); +and the `.dirty` recovery path. A durability change of that size isn't rushed into a security release. 0.1.7: B +in the `syncfs` form with those conditions, measured on a Pi, then A2. + +**Harness done:** `1dcb7e0` (`tests/bench_block_writes.py`; run `python3 tests/bench_block_writes.py --dir +`). **Gate pending:** the Pi SD-card run (maintainer). NVMe results, plain ext4, median +seconds (ratio to today): + +| workload | today | fdatasync | deferred fsync | deferred syncfs | +|---|---|---|---|---| +| add, 256 MiB | 2.80 | 3.80 (noisy; min 1.27 vs 1.58) | 3.58 (0.78×) | 0.54 (5.2×) | +| fetch, 256 MiB | 14.5 | 25.9 (noisy; min 11.3 vs 11.4) | 4.15 (3.5×) | 0.65 (22×) | +| tree add, 4096 × 8 KiB | 2.64 | 2.91 | 3.78 (0.70×) | 0.57 (4.7×) | +| tree fetch | 24.9 | 23.3 | 3.52 (7.1×) | 0.27 (92×) | + +On ecryptfs (an encrypted home) `syncfs()` returns without writing anything back, so the syncfs variant isn't +durable there; the harness warns about stacked filesystems. Deferred fsync is slower for adds (today's 16 writer +threads overlap their fsyncs with building the DAG) and much faster for fetches (today syncs each block on the +fetch thread). fdatasync alone gains nothing measurable on NVMe. + +--- + +## C. Reticulum daemon calls (phase 3) + +When rnfs runs against a shared rnsd instance, each of these calls opens a new authenticated RPC connection: +- Linux (Unix socket): about 0.08 ms each. +- macOS and Windows (TCP): about **41 ms** each. + +### C1. Cache path info in `mesh.py` +- **Where it's called:** + - [mesh.py:84](rnfs/mesh.py#L84) `RnsPaths.info` calls `get_next_hop_if_name` for each node. + - The fetcher calls it once per open link at the start of every fetch, and once per provider (via + [fetcher.py:882](rnfs/fetcher.py#L882)). + - `rnfs peers` calls it once per peer, for up to 1000 peers. +- **Change:** + - Cache `PathInfo` per node ID for 10–30 s in a bounded dict, next to the existing `_stats` cache. + - In `peers()`, when attached to a shared instance, fetch `get_path_table()` once and map it, instead of one + RPC per peer. +- **Expected:** 20 open links currently add about 0.8 s before the first block request on macOS or Windows. + That drops to about 0. A full peers listing goes from seconds to milliseconds. +- **Done:** `0f38807`, `57cb0f3` (the fetch's open links in one `infos` call). Measured over TCP: 1000-peer listing + 41.4 s → 95 ms; 20 open links' lookups 825 ms → one RPC. Only the next-hop interface is cached (15 s); hops and + `has_path` stay fresh. + +### C2. Don't mark destinations "used" on read-only checks +- **What the flag does:** with `_no_use=True` (RNS 1.5.6), `RNS.Identity.recall*` skips the extra + `_used_destination_data` RPC. A status read then also no longer extends the destination's retention in rnsd. + [node_content.py:67](rnfs/node_content.py#L67) already uses it. +- **Add it to the read-only checks:** + - [peers.py:121](rnfs/features/peers.py#L121) `_recalled_mode`, which runs for every peer listed. + - [node.py:213](rnfs/node.py#L213) `_rns_is_known`. + - [node.py:606](rnfs/node.py#L606) `_is_member_node`, which in private mode runs on every routing + admission. +- **links.py:** + - Keep the "used" marking on a real contact. + - The wait loop at [links.py:341](rnfs/network/links.py#L341) polls every 0.1 s for up to `path_timeout`. + Poll with `_no_use=True`, then do one normal recall when the identity shows up. +- **Debug log:** [links.py:285](rnfs/network/links.py#L285) calls `path_description()` (RNS lookups, over RPC + on a shared instance) inside a `LOG_DEBUG` f-string even when debug logging is off. Only build that string + when debug logging is on. +- **Done:** `bc39bdb` (1000 recalls over TCP: 41.5 s → 3.2 ms; in RNS 1.5.6 `path_description` is local, guarded + anyway) + +--- + +## D. Content and storage scaling (phase 5) + +### D1. Shared folder re-imports a whole top-level directory +- **Problem:** at [shared_folder.py:226-243](rnfs/content/shared_folder.py#L226-L243), the signature covers + the whole tree. One changed file reruns `build_open` on everything. + - Measured: 0.44 s for 256 MiB / 5,000 files with one 4 KiB change. A 50 GB photo folder would re-hash 50 GB + for each new photo. +- **Change:** a per-file cache of `(dev, ino, size, mtime_ns, exec bit)` → `(CID, size, content SHA-256)`. On a + match where `is_complete(CID)` holds: + - reuse the CID; + - protect it in the session; + - still run the deny check on the digest, passed explicitly through CT2's `BuildOptions.check`. +- **Trust model:** the same as today's signature. CONT-3's target guard runs before any cache reuse. +- **Storage:** start in memory. Persisting it (so a restart doesn't re-hash everything) needs a new statedb + table with an upgrade migration. Build it as a `RowStore` (XC2), registered in the node's store list (XC5); + TS2's digest test then shows the released steps were left alone. +- **Depends on:** CT2, XC5, XC2, CONT-3, TS2. +- **Tests:** + - Changing one file re-hashes only that file. + - A file renamed in place, or touched with the same size, is handled correctly. + - A newly denied digest is still refused. +- **Done:** `60941cc` (in memory), `d677df6` (persisted: schema step 17, `shared_folder_files`). 1 GiB, 5000 files, + one 4 KiB change: 1.20 → 0.10 s. A hit also requires no denied block in the file. + +### D2. `dir set` / `dir rm` walk the whole tree to check depth +- **Problem:** at [edit.py:39](rnfs/content/edit.py#L39), `walker.height` decodes every node of the edited + tree. + - Measured: about 1 s per edit at 16 GiB (about 60 ms/GiB), so building a directory one entry at a time is + O(n²). +- **Done:** `f109d25` (memo keyed by the checked expectation, in `walker.py`, at most 32 768; 16 GiB tree: an + edit 0.9 s → 0.3 ms after the first) +- **Change:** memoise height by CID; content is immutable, so the memo never goes stale. + - Keep it in `Storage` next to `_links`, or in a bounded LRU. + - [walker.py:406](rnfs/content/walker.py#L406) consults it, so unchanged subtrees are O(1). + +### D3. Storage walks complete DAGs again under the lock +- **Problem:** `_reach` ([storage.py:627](rnfs/content/storage.py#L627)), `_evict_to` (:796), `dag_blocks` + (:832), `is_complete` (:921) and `dag_bytes` (:1081) all re-walk whole DAGs. + - `_reach` costs about 1.1 µs per block while holding `_lock`: about 1.8 s per 100 GiB pinned, on every + eviction pass, blocking every storage call. + - `is_complete` on a 4 GiB DAG missing its first chunk takes 0.35 s, and eviction repeats it for every cached + target. +- **Change:** + - In `_reach`, take the union of the `_complete` frozenset for complete roots instead of walking. + - Memoise the byte total of complete roots, which helps `dag_bytes` and `source_usage`. + - For incomplete DAGs, remember the first missing CID found. The DAG stays incomplete while that block is + absent, so `is_complete` becomes O(1) until it arrives. + - Invalidate on delete, as `_complete` already is (through CT4's `_newly_incomplete`). +- **Depends on:** CT1, CT4, and CT6's injectable block-read hook if test_storage's patches of `_reach` and others + break. +- **Done:** `34f72e8` (CT6's hook wasn't needed). 548k blocks: eviction lock hold 930–985 → 272–279 ms; + `is_complete` of a file missing its first chunk 530 ms → 0.002 ms after the first. Found, not fixed (behaviour + kept): an older cache record whose target directory holds a newer record's path node keeps its record while the + rest of its DAG is evicted. + +--- + +## E. Node, publisher and names (phase 7) + +### E1. Gateway name resolves repeat network work (Tier 1 #4) +- **Problem:** in [names.py:797-843](rnfs/names.py#L797-L843), `resolve(max_age=GATEWAY_NAME_AGE)` (60 s) + judges freshness by `record.heard`. + - That time is capped at the announce's emission time plus 2 × `announce_interval`. + - So once an owner has been offline for more than 12 h (the case keepers exist for), the record never counts + as fresh. + - Every gateway request then sends a path request, waits up to `path_timeout` (30 s), and runs + `keepers.query` (a full `find_providers` lookup plus `/name` queries). + - Simultaneous resolves of the same name aren't merged. A page with N assets under `/name//…` does all of + that N times. +- **Change:** + - Keep a per-name monotonic "last network refresh" time, and skip the refresh while it is within `max_age`. + `contacts.py` does the same with `EXPAND_AGE`. + - Merge in-flight refreshes: one caller runs `_refresh_name` (DH1) and the others wait on an `Event`, then do + only the local final step. `persist` only affects that local step. + - The refresh time is recorded even when nothing is found, which gives a short negative cache for names that + don't resolve. It is **not** recorded when DHT-2 flags the record as a suspected replay. + - CLI calls (`max_age=0`) never use the cache, but may join a refresh already running. +- **Depends on:** DH1 step 1, `_NameFloor`, the XC6 names waiter, DHT-1, DHT-2. +- **Remote load:** lower (fewer path requests, `/find` and `/name` calls). +- **Behaviour:** a name republished moments ago may show the older record on the gateway for up to 60 s. +- **Done:** `07c11d1` (20 simultaneous resolves: 20 → 1 path requests and keeper queries; 10 sequential 15.0 → + 1.5 s). Keyed on when the refresh ended; applies to follow and deny-list resolves too. + +### E2. Avoid re-checking whether a file is sealed, and repeating `pin ls` work +- **Sealed check:** `_info` / `_read` ([node.py:1098](rnfs/node.py#L1098), :1150) call `open_sealed` for every + plain-file read. `read_file` (behind `/api/cat`) does it twice. + - Change: keep a bounded LRU of file CIDs known not to be sealed (it never goes stale), and pass the + `SealedFile` from `_info` on to `_read`. + - Expected: a 64 KiB read goes from 1.3 to 0.5 ms. + - **Depends on:** CONT-1, which edits `_read`. +- **`pin_ls`:** [node.py:1242](rnfs/node.py#L1242) re-reads and decodes each pin's root block, and for files + also re-resolves the file and reads its first bytes. + - Change: remember type, inside-name and sealed status per CID, bounded and pruned along with the pins. + - Expected: 137–207 ms at 2000 pins drops to 3–4 ms on repeat calls. +- **Done:** `ed86aaa` (sealed check: 64 KiB read 0.32 → 0.19 ms here), `628ae2d` (`pin_ls`: 68 → 3.8 ms at 2000 + pins) + +### E3. Publisher work after every add +- **Problem:** + - `refresh()` → `_compute` ([publisher.py:414](rnfs/publisher.py#L414)) copies `storage.pins()`, including + every source set, after every add. That was 6 of the 11 ms per add at 3000 pins. + - `_provide_again` (:499) scans all pending withdrawals for each added CID. That costs about 150 ms for + 10k CIDs × 300 nodes, on the first refresh after start or on a mass re-pin. +- **Change:** + - Add a cheap storage accessor: a cached frozenset of pinned CIDs, rebuilt only when pins change. Use it + instead of the full copy. + - Build the set of pending-withdrawal keys once per `refresh()`, and skip CIDs whose key isn't in it. +- **Done:** `19ae2bc` (first refresh, 10k CIDs × 300 nodes: 71 → 14.5 ms) + +--- + +## Low-risk performance fixes (phase 3 unless noted) + +| # | Item | Where | Change | +|---|---|---|---| +| S1 | **Job rate samples grow without bound** (Tier 3 #1) | [jobs.py:117](rnfs/features/jobs.py#L117) | Prune samples older than `RATE_WINDOW` in `block()` when appending, keeping one base sample. Today this is about 6–10 MB a day for a job nobody polls. **Done:** `05710e1` (86 401 samples (10 MB) a day → 301) | +| S2 | Block-index scan at startup | [cid.py:60](rnfs/content/cid.py#L60), `blockstore.py:111` | A scan-only constructor that checks canonical form cheaply (length, lowercase base32 alphabet, zero trailing bits) and keeps the input as the string form. `Cid.parse` for API and wire input stays as it is. 9.8 → about 7 µs per block, so about 3 s saved per million blocks. **Done:** `e7f5626` (10.4 → 4.5 µs per block) | +| S3 | Second durable commit on add (**phase 4, with XC2's pin_names migration**) | `node.py:992/1012`, `pin_names` | Write the pin's display name in the pin's transaction, or with `durable=False`. A power loss can then lose at most a display name, never a pin. About 10% of an add. **Done:** `1f414bc` (`durable=False`; 4 KiB add 12.9 → 10.5 ms) | +| S4 | `RoutingTable._prune` on every announce | [routing.py:94](rnfs/dht/routing.py#L94) | Prune expired bans and drops from the front of insertion-ordered dicts (pop and re-insert on update), or at most once a second. 202 µs → about 1 µs per `heard()` with 3000 dropped entries. **Done:** `c7f05e5` (201 → 0.86 µs per `heard()` with 3000 drops) | +| S5 | Provider expiry clamps every record (**phase 7, after DH2**) | [providers.py:641](rnfs/dht/providers.py#L641) | Track the highest clock value seen, and only run `_clamp` after the clock has gone backwards. Halves the 17 ms pass at 100k records, which runs under the store lock. **Done:** `9e495a4` (100k records: 16.6 → 8.7 ms per pass) | +| S6 | LXMF auto-fetch copies every pin (**after FEAT-1**) | `lxmf.py:1510` | `storage.has_pin(cid, source)`: a dict lookup under the lock instead of `pins()`. Can share E3's accessor. **Done:** `8afce50` (1.96 ms → 0.24 µs at 10k pins) | +| S7 | Idle 1 Hz wakeups | [lxmf.py:1000](rnfs/features/lxmf.py#L1000), [cli.py:273](rnfs/cli.py#L273) | In LXMF, when the outbound and held queues are empty, wait until the next announce or sync is due, and `_wake.set()` from `send()` and `_hold()`. In the daemon main loop, `wait(None)` on POSIX, keeping 1 s on Windows for signals. Saves wakeups on Pi and battery nodes. **Done:** `cb944c4` (120 fewer wakeups a minute idle) | +| S8 | Second `rnfs.dht` announce handler | [peers.py:38](rnfs/features/peers.py#L38) | Call `traffic.heard` from the node's existing `DhtAnnounceHandler` (`node.py:357`) and drop the second registration. RNS starts a thread per handler per announce, so this halves the threads. **Done:** `fcd55d8` (2 → 1 handler threads per announce) | +| S9 | `rnfs.Client.open()` has no read-ahead | [interface.py:34](rnfs/interface.py#L34) | Double the refill size (64 KiB up to 1–4 MiB) while reads are sequential, and reset it on `seek`. Sequential reads currently run at 31 MB/s, against 335 MB/s for a single `read()`. Random reads still fetch only what they cover. **Done:** `d35ccfe` (sequential 64 KiB reads: Client 33 → 560 MB/s, Node 79 → 1219 MB/s) | + +--- + +## Not doing in 0.1.6 + +| Item | Why not | +|---|---| +| **FEAT-3 LXMF stamps** | Announcing a `stamp_cost` changes announce app_data, and enforcing it refuses senders that haven't heard the new announce. For 0.1.7: an opt-in `lxmf_stamp_cost` (default 0, so the announce is byte-identical), with `enforce_stamps` off and a valid stamp used as a priority for strangers rather than a gate. | +| **DHT-6 ID grinding** | Raising the cost of IDs changes how they are derived (a wire and design change). A later local mitigation: keep one of alpha's slots for the closest unqueried senior node. | +| **B** (block write durability) and **A2** | Q1 deferred them (2026-10-18): the gain on an SD card needs `syncfs`, Linux-only and with a fallback on stacked filesystems, plus the power-loss recovery path. See B's decision note. | +| **NET-4**, if phase 6 runs long | It depends on the A-series fetcher changes. Otherwise 0.1.7. | +| **CONT-4 CID denies from remembered hashes** | M-sized, because it needs a change trigger in `_apply_deny`. Chunks must never be auto-denied. | +| **API-4 dropping the fallback** | It would break newer clients against deb-installed 0.1.4 daemons. Gate it on version in 0.1.6 (above) and drop it in 0.1.7. | +| **SUP-6 signing (Authenticode, apt, Flatpak) and an SBOM** | Needs keys and infrastructure. | +| **ND1, FT1 package split, rest of DH1, DH5 legacy moves** | Large, and in the files with the most churn and open security fixes. 0.1.7, each in its own release. | +| **NW2 lock and NW3 split** | `/blocks` is mid-fix for NET-1/2. Step 1 (docs) is in 0.1.6. | +| **XC1 feature boundary** | Decides how easily features move to companion apps. It also covers moving cli.py and the feature CLIs onto `ApiConnection`, and deduplicating the add options. 0.1.7. | +| **XC2 for lxmf, follow and keepers; XC6 waits in server and links** | Wait for FEAT-*, DHT-3, DHT-1 and NET-1 to settle. | +| **DH3 `_Round`, DH4 `_Run`, NW1 `_Providers`/`_Download`** | No 0.1.6 item needs them, and DH4 must wait for DHT-1's bounds to settle. | +| **CT5 headers module, XC8 `fmt.py`** | Little gain, and `fmt.py` changes CLI output. | +| **XC8 token buckets** | lxmf's two-level buckets don't fit `mesh.TokenBucket`. | +| **TS8 composite install action, TS9 regrouping, rest of ruff B** | Large renames or rewrites while the fetcher work is in progress. | +| Hash each block once (trusted put path) | About 10% on add and fetch, but a "skip verification" path is a security boundary that must never reach unverified callers. Needs its own review. | +| `dag._decode` canonical check without re-encoding | About 40% of decode time, but it changes validation code: the accepted set of encodings must stay exactly the same. Needs fuzzing against the current check first. | +| Gateway resolves `index.html` directly | A missing entry and an unreachable block both raise `NotFound` today, so it needs a new error type. Only helps directories over 128 KiB. | +| Pacing sends about one RTT earlier on slow links | Changes timing seen by remote peers (handler threads held longer). | +| Even split of the final batch across providers | Sends more requests to remote peers, for a small gain at the end of a fetch. | +| Raise `WRITE_THREADS` to 64 | See B: flatfs keeps 16 for macOS's file-descriptor limit and HDDs. | +| Merge the gateway's 3 fetch sessions; `next_due` SHA-256; `walker.step` building `Expect`s; `BlockStore.delete` stat | Negligible. | + +Already measured as fine, so not worth revisiting: +- DHT lookups and the routing table's `closest`; +- statedb (WAL, primary-key access, `executemany`); +- denylist frozensets; +- CLI startup (about 50 ms, and RNS isn't imported); +- logging cost when the level is off; +- API and gateway streaming; +- the chunkers; +- sealing. + +--- + +## Release checklist +- [ ] Every row of [Decisions](#decisions-needed-before-starting) is answered, and the maintainer-only actions + are done or deliberately left. *Q2–Q16 answered 2026-10-17; Q1 answered 2026-10-18 (B and A2 deferred to + 0.1.7). Left for later: PyPI and the Pi benchmarks. GitHub settings and Dependabot alerts: the maintainer's.* +- [x] Wire: TS3's v0.1.3/v0.1.4 vectors pass byte-for-byte. +- [x] Interop: 0.1.6 against 0.1.5 and 0.1.4 nodes (fixtures in `tests/fixtures/`, `tests/test_rns_compat.py`). + Fetch, provide, names and keepers behave the same. A 0.1.5 fetcher against a 0.1.6 server sees no new + refusals in an honest transfer (NET-1, NET-2). *`tests/interop.py` (`65c11f6`): every scenario passed both + ways against v0.1.5 and v0.1.4 (providers, get/cat, transfers with and without `max_upload_rate` with zero + refusals, failures or cooldowns, names, keepers, LXMF). Local TCP only: no slow or LoRa links.* +- [x] Upgrade: TS2's `test_upgrade_015` passes, including D1's new table if it is persisted. The released-schema + digest is unchanged. *(Schema step 17, `shared_folder_files`.)* +- [x] Every security fix has its `tests/security/test__*.py` regression test, and the TS9 index lists it. +- [x] CI runs every `RNFS_E2E` module (TS1). Lint passes with BLE001, E402 and RUF100 (TS10). +- [x] ~~B: the power-loss recovery path is tested (a leftover `.dirty`, a truncated recent block, an old block + left alone).~~ *Not applicable: B is deferred to 0.1.7 (Q1).* +- [ ] Before/after numbers on NVMe and a Pi SD card for A1, A2, B, C1, D1 and D3, recorded in CHANGELOG.md. + *NVMe numbers recorded for A1, C1, D1, D3 (CHANGELOG) and B's variants (above). A2 and B deferred to 0.1.7. + Pi numbers deliberately left for later.* +- [x] specification.md updated for every item that lists a spec change, plus §8 if B goes ahead, and a version + and compatibility note (TS11). *(§8 unchanged: B hasn't gone ahead.)* +- [x] design.md updated (storage write path, fetcher pipelining, names refresh cache, mesh cache, deny-hash bound, + write-out, dir checks). *(The storage write path is unchanged while B waits.)* +- [x] CHANGELOG.md 0.1.6 entry stating "no wire-format or CID changes", with a **Security** section and the + user-visible changes: + - FEAT-5's API default; + - API-1's refusals; + - API-7; + - SUP-3; + - SUP-5; + - FEAT-2's inbox format; + - `follow_quota`. +- [x] CONTRIBUTING.md holds this checklist (TS6), and CHANGELOG's version equals `__version__`. *(Bumped to + 0.1.6 with README's release links.)* + +## Sources (for B) +- [Kubo v0.33 changelog: flatfs `sync` default, `WriteThrough`, `BlockKeyCacheSize`, batch sizes](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.33.md) +- [Kubo default datastore spec (`config/init.go`)](https://github.com/ipfs/kubo/blob/master/config/init.go) +- [Kubo config reference: `Import.BatchMaxNodes`, `Import.BatchMaxSize`, `Datastore.HashOnRead`](https://github.com/ipfs/kubo/blob/master/docs/config.md) +- [go-ds-flatfs source: batch put and commit, `maxConcurrentPuts`](https://github.com/ipfs/go-ds-flatfs/blob/master/flatfs.go) +- [go-ipld-format batch (`ParallelBatchCommits`)](https://pkg.go.dev/github.com/ipfs/go-ipld-format) and + [kubo#4299 on tuning it](https://github.com/ipfs/kubo/issues/4299) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4114a83..e0caa48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,304 @@ [specification.md](specification.md) is the authority on behaviour. +## 0.1.6 + +Security fixes, maintainability work and performance work. No wire-format or CID changes: these nodes, 0.1.5, +0.1.4 and 0.1.3 nodes work together. The upgrade of an `` is automatic and one-way. + +Upgrade notes (each is described in full below): +- **Settings and the rnfs dir:** a symlinked `rnfs.toml`, or one or a subdirectory owned by another user, stops the + start; a loose one (such as a 0664 `rnfs.toml` or 0775 `blocks/`) is made owner-only, with a warning. +- **The shared folder:** one that leads into the rnfs dir (or a directory holding it) is refused, and its + pins with source `folder` are released. +- **Following names:** a followed name's target may take at most `follow_quota` (new, default 1 GiB). +- **Deny lists:** while the deny list has `sha256:` entries, a file repeating its content more than 16 times + (beyond 64 MiB) can't be read or imported (451). +- **Local API:** `POST /api/lxmf/inbox/get` answers 404 for a name link whose name can't be resolved, unless the + request sets `use_sent_cid`. +- **Gateway:** browsers that send no `Sec-Fetch-Site` (Safari before 16.4, Firefox before 90) are treated as web + pages, so `?format=` downloads give them 409. +- **`rnfs inbox`:** its message line is reordered (address and trust first, the display name last, quoted). +- **install.sh:** it stops when the GitHub CLI can check provenance but isn't logged in; log in with + `gh auth login`, or set `RNFS_REQUIRE_ATTESTATION=0`. +- **Flatpak:** after updating, the Flatpak no longer sees the rest of your home folder or any device. Grant what + you used again: `flatpak override --user --device=all io.github.afit21.rnfs` for an RNode or other serial + interface, and `flatpak override --user --filesystem= io.github.afit21.rnfs` for other folders, + `~/.config/reticulum` included if your Reticulum config is there. +- **State:** the state database gains a table (the shared folder's file cache); an older rnfs can't open it + afterwards. + +Security: +- The release workflow's Flatpak build no longer uses the GitHub Actions cache. Any workflow run on the default + branch can write that cache, and a poisoned one would have gone into the attested `.flatpak` bundle. +- `rnfs inbox get` no longer quietly fetches the CID a sender put in a name link when the name can't be + resolved. The sender chose that CID and nothing ties it to the name, so any sender could pose as a + well-known name. It now fails before writing anything, unless you pass `--use-sent-cid`; then the result + carries a warning. Auto-fetch (trusted senders only) keeps the fallback, and records it. +- A `/blocks` requester that rejects or cancels a response before a quarter of it was sent now cools down for + `blocks_cooldown`, per link and per identity, as one that rejected it unsent already did. Before, a requester + could take the first window of each response, cancel it and ask again at once, so the node built a whole + response per window. Closing the link still starts no cooldown, so an honest reader that gives up early isn't + held back. +- While the deny list has file hashes (`sha256:` entries), a file that declares more than 16 times the bytes of + its distinct chunks, plus 64 MiB, is refused (451) before it is hashed. Hashing reads every repeat of + repeated content, so six blocks (128 KB) declaring 2^56 bytes made a one-byte `cat`, a gateway range request, + `?format=archive` or an `import` hash for about a year, holding a gateway slot. `cat` and the gateway now fetch + a file's whole DAG before checking it (unless its hash is already known), so `max_download_size` and the disk + reserve apply as they do for `get`. +- A gateway HEAD (or a range not starting at byte 0) of a stored file the deny list blocks answers 451, like + GET. Before, it took its Content-Type from a denied stored chunk, and a HEAD of a file with a denied hash + answered 200. +- A name whose owner is offline can still be resolved through its keepers when Sybil nodes near its keeper key + stall the DHT lookup. They could answer `/find` slowly with ever closer IDs, so the search used all of + `lookup_timeout` and the keepers then had one second to answer. Each batch of `/name` queries now has its own + budget (87.5 s at the defaults); the search for keepers stops once some are known and `lookup_timeout` / 5 has + passed, and resumes if their answers don't settle the name. +- A neighbour replaying an older, validly signed name announce no longer makes its record look current: rnfs + reads the replayed announce's own emission time from RNS's announce cache, treats one older than the path's + (or carrying another record) as a suspected replay, gives its real age and asks keepers. +- A provider can no longer hold a fetch for hours by dripping a `/blocks` response: once nothing is left to + request and a provider is idle, a batch in flight longer than `deadline_slack` is asked of it too, the first + verified copy wins, and the slow request's link is closed (left to finish if another fetch uses it) without + counting as a stall. Simulated: 20.6 → 3.3 s. +- `get` of a directory can no longer be redirected by someone who can rename inside the output swapping one of + its directories for a link or junction (or another directory) while it is written. On Linux and macOS the + output being written is private (0700, setgid kept) and must start empty, every entry is created relative to + its parent's descriptor without following links, and each directory is checked by device and inode before + anything is written into it. On Windows it gets a private access list while it is written, and each file's + final path must be inside it. A swap fails the get with "output was changed while being written". +- On Linux and macOS, `rnfs.toml` is now checked before its settings are used, as on Windows: a symlink, a FIFO + or anything else that isn't a regular file is refused, and so is a file another user owns; one others may + access is made 0600, with a warning; at most 1 MiB of it is read. Its settings choose the Reticulum config, and + with it programs rnfs starts, so anyone who could write it could run code as you. Reticulum's `config` gets the + same check when `reticulum_dir` is the rnfs dir. +- The rnfs dir's subdirectories (`state`, `blocks`, `storage`, `names`, `lxmf`) are checked at every start: a + symlink, a non-directory or one another user owns stops the start before Reticulum starts; a loose one is made + 0700, with a notice; and if others could write to it, entries another user owns are reported. Before, a user + who could once write in a loose rnfs dir kept their access to `blocks/` and `names/` after it was tightened. +- The shared folder is no longer scanned when `files/` leads into the rnfs dir. Before, a symlinked `files/` that + led to `storage/` or `names/` (or that someone who could write to a directory above it retargeted there) + imported, pinned and provided the node's identity or the name keys, and one that led to the rnfs dir or a + directory above it shared everything in it. A folder that resolves, as opened, to the rnfs dir, a directory + holding it, or anything inside `storage/`, `names/`, `state/`, `blocks/` or `lxmf/` is now refused with a + warning and scanned as empty: nothing is imported, and every pin with source `folder` is released, so keys a + 0.1.5 node already shared that way stop being provided at the first scan after the upgrade. On Linux and macOS, + a directory above the folder that other users can change gets a warning, once. +- A node no longer reads request packets sent to it on `rnfs.dht` links it opened. That end of a link has no + request handlers, yet RNS decoded each request (up to the link's packet size) and started a thread for it + before finding nothing to run, for any node rnfs linked to. LXMF and host-program links are unchanged. +- An explicit `max_upload_rate` now applies to every `/blocks` response, including one that fits in one packet. + Such responses never give the link a rate estimate, and the cap applied only once there was one, so on a + large-MTU path (RNS's TCP interfaces) a requester asking only for single-packet answers was never paced. Waits + of up to 5.6 s are held and longer ones refused as busy, as for larger responses. The default cap (half the + rate estimate, on slow links) is unchanged. +- The wait for a previous `/blocks` response's proof uses the round-trip time the requester claims only if it is + a finite, non-negative number, and still at most 5 s. Before, a negative claim gave an honest proof no grace, + and a value that wasn't a number made the request handler fail. +- `install.sh` ties a release install to the tag it asked for: where `gh` supports them, the provenance check + passes `--source-ref=refs/tags/` and `--deny-self-hosted-runners`, and only an `rnfs--*.whl` of + that tag's version is installed. Before, an older release's genuinely attested sdist, renamed to the new + release's file name, passed both checks. +- `install.sh` stops when the GitHub CLI has `gh attestation` but isn't logged in (root installs usually hit + this), instead of quietly checking `SHA256SUMS` only. Log in with `gh auth login`, or set + `RNFS_REQUIRE_ATTESTATION=0` to install with the checksum only. Without `gh` it still warns, and the README now + says that `curl | sh` without `gh` trusts whoever can write the release's files. +- The Flatpak's sandbox is narrowed to `~/.rnfs`, `~/.reticulum` and the Downloads folder, with no devices + (before: your whole home folder and every device, so a compromised daemon could write `~/.bashrc`). See the + upgrade note below. +- `install.sh` warns when `RNFS_SOURCE` is, or names, a plain `http://` URL, and the `.deb`'s postinst + byte-compiles with `python3 -I`. +- The file-index-node NomadNet page renders at most once a minute, one visit at a time, and serves its cached + page meanwhile (before, every visit ran `rnfs pin ls`). +- `examples/rnfs_share.py` and `rnfs.Client`'s log escape names and daemon messages, so a name can't drive the + terminal. +- `rnfs.Client` and the CLI check the daemon's proof before believing anything else the API port answers: while + the daemon is down, a program listening on its port can no longer make the client raise `Denied`, `NotFound` + or `StorageFull` with its own text. Without a valid proof a 503 is `Busy` and anything else `AuthFailed`, both + in the client's own words. Connecting and proving time out after 15 s (or the client's timeout, if shorter), + and at most 64 KiB of the proof response is read. +- After the proof, the client sends the request on a new connection only to a daemon that proved a version below + 0.1.5 (those close every connection); a newer daemon that closes the proved connection gets + `ConnectionFailed`, and nothing is sent. The fallback for 0.1.4 daemons goes in 0.1.7. +- A followed name can no longer fill the follower's disk: its new target is fetched only if its size, and the + bytes of its DAG's blocks, fit in `follow_quota` (default 1 GiB); otherwise the check fails, the follow's state + says why, and the previous target stays pinned. While one target replaces another both are held, so a followed + name can use up to twice the quota for a time. Targets already pinned over the quota stay. +- An auto-fetch sender can no longer push another auto-fetch sender's pinned messages out of the inbox. + Re-linking a CID it already pins costs no quota, so about 500 such messages filled the inbox with protected + records, and each further message dropped the oldest record of anyone, releasing its pin. Each of the n senders + in `lxmf_auto_fetch_from` now keeps at most 1/n of the inbox's messages and bytes, messages holding pins + included, and when every other message holds pins, a new one may push out only its own sender's. +- Auto-fetches still queued for a sender removed from `lxmf_auto_fetch_from` are refused at the next start (with + LXMF off too) and when their turn comes. Before, up to 64 of them were fetched and pinned after a restart. +- `rnfs inbox` shows each sender's address before its display name, marked `untrusted`, `trusted` or + `contact `; the display name, which the sender chooses, comes last, quoted and escaped. Before, a display + name like `alice ` came first and pushed the real address off to the right. +- A node's own provider records can no longer be evicted from its own store: about 1000 Sybil stores to a key it + provides pushed its record out, and a full newcomer pool could take it too. Its own records don't count toward + the publishers a key keeps and are never chosen for eviction. + +Changed: +- `POST /api/lxmf/inbox/get` answers 404 for a name link whose name can't be resolved, unless the request sets + `use_sent_cid` (before, it fetched the sender's CID). +- While the deny list has file hashes, a highly repetitive file over 64 MiB (a zero-filled disk image, say) can't + be read or imported (451). The user's own gateway HEAD of a file that isn't wholly stored is then typed by its + name only, not by its first bytes. +- On Windows, a directory `get` output is private to the user while it is written, then inherits its folder's + access list again before it gets its final name (if that fails, a warning is logged and it stays private). +- A symlinked `rnfs.toml` is refused: dotfile setups that symlink it into the rnfs dir stop working (put the + file itself there). A loose `rnfs.toml` (such as 0664) or subdirectory (such as 0775) is tightened at the next + start, with a warning; it isn't refused. +- The daemon and `rnfs.Node` check, in order, the rnfs dir, the settings, the subdirectories, then take the lock. + The CLI prints settings warnings, and `rnfs.Client` raises them as `RuntimeWarning`. On Windows, a loose + `rnfs.toml` no longer makes the CLI or `rnfs.Client` load RNS. +- When a file is refused because its SHA-256 is on the deny list, the log suggests `rnfs deny add ` + (once per file): a `sha256:` entry doesn't stop the file's blocks being served one by one, and denying its CID + does. +- The README says plainly that pipx and plain `pip install` resolve dependencies unpinned, not at the versions in + `packaging/requirements.lock`. +- The gateway treats a request without `Sec-Fetch-Site` as the user's own only if its `User-Agent` doesn't start + with `Mozilla/`. Browsers without Fetch Metadata (Safari before 16.4, Firefox before 90) are now treated like a + web page on every request: `?format=` downloads get 409, and their network fetches count against + `gateway_fetches`. curl, scripts and other tools keep the user's own access, and so does a browser's own + navigation (`Sec-Fetch-Site: none`). +- `rnfs name publish` in private mode warns that the name's announce (its sequence and target CID) is visible + outside the private network; only the content is served to members only. `POST /api/name/publish` then carries + the warning as an optional `warning` key, and `rnfs.Node`/`rnfs.Client` raise it as a `RuntimeWarning`; outside + private mode nothing changes. +- The specification documents that the API's proof keep-open lets one connection hold an API slot about twice as + long as one request, and the measured cost of grinding node IDs near a key (raising it needs a change to how + node IDs are derived, so it is deferred). +- `rnfs inbox` prints each message as number, time, address, trust, method, then the quoted display name (the + text format isn't a stable interface; scripts should use the API). Display names are also escaped for + backslashes, blank filler characters (U+115F, U+1160, U+3164, U+FFA0, U+2800), stacked or leading combining + marks, and names of only spaces. +- The specification documents that many LXMF identities can keep the strangers' bucket empty; contacts and other + trusted senders are unaffected. +- The request server checks a feature path's response (keepers' `/name`) as requesters do, and answers `False`, + logging an error, rather than send one they would refuse. Unexpected errors at five places that catch broadly + (the fetcher's `/blocks` worker, the lookup's routing-table callbacks, sending a request on a link, path + lookups, the peers listing) are logged as warnings with their traceback in the debug log (the last two at most + once a minute). +- Stopping the node is exception-safe: every step (features, announcer, request server, names, saves, closing + the state database, releasing an embedded node's destination) runs even if an earlier one fails, so a failing + step no longer leaves the database open or the `rnfs.dht` destination registered in a host program. Stopping + waits at most 5 s in all for the features' threads and the node's own (LXMF and jobs no longer add 2 s each), + and remote-pin fetches stop with the node (their pins are fetched at the next start). +- A pin name or contact whose save failed (a full disk, say) is written again by the node's next save, at the + latest at stop, rather than only at the next change. +- Gateway and other resolves with a `max_age` (60 s on the gateway) reuse a network refresh of the name for that + long, including one that found nothing, and simultaneous resolves of a name share one refresh: a page with many + `/name//…` assets whose owner is offline costs one path request and one keeper query, not one each. A name + republished moments ago may show its older record on the gateway for up to 60 s. CLI resolves always refresh. +- GitHub releases take their notes from the version's section of this file instead of generated notes, and the + release workflow refuses a tag whose section is still marked unreleased. The check that the Flatpak's wheels + match `packaging/requirements.lock` runs on every push and pull request, and the release builds (Flatpak, + Windows exe, arm64 .deb, Debian 12 smoke test) also run, without publishing, on pull requests that touch + packaging and weekly. The Flatpak build no longer copies local debris from a checkout. +- A block that doesn't match its CID raises `BlockMismatch`, and a damaged sealed file `SealError`: both are + `RnfsError`s and still `ValueError`s, so the API keeps answering 400 (and logs a `BlockMismatch` as a warning, + as it means a bug rather than a bad request). A shared-folder entry that fails with an unexpected `ValueError` + is logged and tried again after 10 minutes rather than skipped until the file changes. `dir rm` refuses a + target without a path in the node itself, before fetching anything. +- `rnfs jobs` (`--background`, `cancel`, `clear` and the progress line), `rnfs peers` and `rnfs stats` escape every + field the daemon sends, as the other feature commands already did. Internally, features declare the features + they use, and the node checks those and every feature's API routes and request paths when it loads them. +- 16-byte IDs written as hex (node IDs, names, identity hashes, LXMF addresses) are parsed in one place with the + same rules everywhere; the only input refused that was accepted before is a `bootstrap_nodes` entry with + whitespace between its hex digits, now skipped with the warning other malformed entries get. +- A DHT lookup run also ends after 80 `/find` queries, or after 9 answers in a row (3 × max(`alpha`, `beta`)) + that bring the closest nodes no nearer the key. Honest lookups end as before (in a simulated network of 2000 + nodes they complete within 12 queries); peers see fewer `/find` requests. +- The specification now says a requester that gives up on a `/blocks` response closes its link, and never + cancels the Resource (0.1.5 and earlier already do so). +- `rnfs inbox` shows the CID each name link was sent as ("sent as ``"), and marks an auto-fetch that + pinned the sent CID because the name didn't resolve ("auto-fetch pinned (sent CID; name unresolved)"; + `"sent_cid": true` in the inbox record). + +Faster: +- `cat`, gateway and range reads ask for the next 32-chunk window before handing the reader the current one, so + every provider stays busy across window boundaries: a 48 MiB read over 3 simulated 40 Mbit/s providers went + from 8.06 to 5.95 s (links 57% → 77% busy; 6.12 → 4.73 s without fsync). A reader that stops early has fetched + at most one extra window; closing it closes its links, except one another fetch is using (that request is left + to finish), and never cancels a Resource. +- `get`, `pin add` and `ls` ask for a block's children as soon as the block arrives instead of one tree level at a + time, and count `max_output_entries` per block (small gains in simulation: up to about 4%, one request fewer on + a single LoRa provider). +- The fetcher keeps a set beside its queue of blocks to fetch, so requeueing a refused or budget-cut request's + blocks no longer scans the queue: 22 ms → 0.01 ms per answer with 16 000 blocks pending (1 GiB), 242 → 0.015 + ms at 10 GiB. +- `open()` on `rnfs.Client` or `rnfs.Node` reads ahead while reads are sequential, doubling each refill from 64 + KiB up to 4 MiB; a seek that moves starts again with a read of only what it covers. Sequential 64 KiB reads: 33 + → 560 MB/s through `rnfs.Client`, 79 → 1219 MB/s on `rnfs.Node`; `read()` of a whole file 310 → 589 MB/s. +- An add makes one durable commit instead of two: the pin is committed durably first, and the name `pin ls` shows + for it is written after it without a sync of its own, so a power loss can lose at most the latest names, never + a pin. 4 KiB adds 12.9 → 10.5 ms, 1 MiB adds 18.9 → 16.4 ms on a local SSD. +- The shared folder re-reads only the files that changed inside a changed entry: a per-file cache keyed by + device, inode, size, modification time and execute bit reuses a file's CID while all its blocks are stored and + none is denied, and its remembered content hash still goes through the deny list. Rescanning a 5000-file, 1 GiB + folder after a 4 KiB change: 1.20 → 0.10 s (256 MiB: 0.33 → 0.10 s). The cache is kept in the state database (a + new `shared_folder_files` table, added by the upgrade), so it holds right after a restart too. +- `dir set` and `dir rm` no longer walk the whole tree to check its depth: subtree heights are remembered, so an + edit reads only the directories it rebuilds. On a 16 GiB tree, an edit after the first went from about 0.9 s to + 0.3 ms (30 edits: 27.5 → 0.89 s). +- Storage no longer walks complete DAGs again under its lock: eviction takes a DAG known to be complete as a + whole, `dag_bytes` sums it once, and `is_complete` remembers an incomplete DAG's missing block until it + arrives. With about 550 000 blocks an eviction pass holds the lock for about 240 ms instead of 900 ms; + `is_complete` on a 4 GiB file missing its first chunk went from 530 ms per call to 0.002 ms after the first. +- Expiring provider records no longer checks every record for a time in the future unless the clock went back + since a time was taken: one expiry pass at 100 000 records went from 16.6 to 8.7 ms, held under the store's + lock. +- `cat`, `info` and `GET /api/cat` remember which files aren't sealed (at most 4096; forgotten when the deny + list changes), so reading one again doesn't read its first bytes to check, and `GET /api/cat` opens a sealed + file once instead of twice: a 64 KiB read of a stored file 0.32 → 0.19 ms (through the API 0.47 → 0.21 ms). +- `pin ls` remembers each complete pin's type, name and whether it is sealed instead of reading its blocks again + at every call: 68 → 3.8 ms at 2000 pins after the first. +- The publisher's refresh after an add reads a cached set of pinned CIDs instead of copying every pin, and + gathers the keys of queued withdrawals once: the first refresh with 10 000 CIDs and withdrawals queued for 300 + nodes 71 → 14.5 ms; a refresh after a pin at 3000 pins 2.25 → 1.69 ms. +- Attached to a shared Reticulum instance, path lookups no longer cost an RPC per node each time (about 41 ms each + over TCP, which macOS and Windows use for it): each node's next-hop interface is kept for 15 s (hops and whether + there is a path are still read fresh), and `rnfs peers` and the start of a fetch read every interface they need + from one path-table RPC. A 1000-peer `rnfs peers` went from 41.4 s to 95 ms, and the path lookups of 20 open + links at the start of a fetch from 825 ms to one RPC (under 0.1 ms within 15 s of the last). +- Read-only identity checks no longer mark destinations as used in RNS (the peers listing, the DHT lookup's + known-identity check, private-mode routing admission, the link opener's polls while it waits for an identity): + on a shared instance each mark was an RPC. A link actually opened still marks its destination. +- The peers feature no longer registers a second `rnfs.dht` announce handler: the node's own records what `rnfs + peers` shows, which halves the threads RNS starts per announce (2 → 1; about 130 → 62 µs of dispatch each). +- Opening a node scans its block directory about twice as fast: block file names are checked for canonical CID + form without being encoded again (10.4 → 4.5 µs per block, about 6 s less per million blocks). +- Hearing a DHT announce no longer scans every remembered ban and dropped node: 201 → 0.9 µs with 3000 dropped + nodes (247 → 0.9 µs with 3000 bans as well). +- A background job nobody polls no longer keeps a rate sample for every second it runs (about 10 MB a day): it + keeps the last 5 minutes' (301 samples), and reports the same rate. +- An LXMF auto-fetch checks whether its sender already pins a target with one lookup instead of copying every + pin: 1.96 ms → 0.24 µs per link at 10 000 pins. +- An idle node no longer wakes every second: the LXMF loop polls only while a message is being sent or held or a + propagation-node download runs (a newly queued message is polled at once), and on Linux and macOS the daemon's + main thread waits for SIGINT/SIGTERM with no timeout (Windows still wakes each second to run signal + handlers): 120 fewer wakeups a minute on an idle daemon. + +Added: +- `rnfs inbox get --use-sent-cid`, and an optional `use_sent_cid` on `POST /api/lxmf/inbox/get`. +- `trusted` and `contact` fields in `GET /api/lxmf/inbox` records, and an `untrusted_dropped` counter in + `status` (messages from untrusted senders dropped by the rate limits since the start). +- `rnfs.BlockMismatch`, `rnfs.SealError` and `rnfs.NotRecipient`, to catch these errors by class. `ApiConnection` + has a typed method for every core API route (`dir_set`, `dir_rm`, `pin_remote`, `providers`, `ping`, + `name_import`, `name_export`, `name_rm`, `export`, `import_archive`, `gc`, `verify`). +- `tests/security/README.md`: an index from each security finding's audit ID to its regression tests, the code they + guard and the commits that fixed it, grouped by subsystem (a test checks that every security test is listed). +- CONTRIBUTING.md: running the tests and their options, lint, the dependency locks, adding a schema step or an + API route, and releasing. `specification.md` names the version it describes and that the wire format is + byte-identical since 0.1.3. Dependabot updates the workflows' pinned actions. Tests check that the supported + Python versions agree across CI, `pyproject.toml`, the lock scripts, `build-deb.sh` and `install.sh`, and that + this file's version is `rnfs.__version__`'s. +- The `follow_quota` setting: bytes a followed name's target may take (default 1 GiB; `0` stops fetching + followed names). +- `contrib/rnfs.service.d/hardening.conf`: an opt-in systemd drop-in with `ProtectSystem=full` and + `PrivateTmp=yes`, for systems with unprivileged user namespaces. The default unit sets neither, nor `UMask`. + ## 0.1.5 Performance work, a debug log, state kept in one SQLite database, deny lists, and a public API for apps. No diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ae8d018 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,210 @@ +# Contributing to rnfs + +[specification.md](specification.md) is the authority on behaviour, [design.md](design.md) describes how this +implementation works, and [CHANGELOG.md](CHANGELOG.md) records what each release changed. + +rnfs is the core: content, the DHT, names, pinning, the local API and the public library. Features for users +beyond that belong in companion apps built on `rnfs.Client` or `rnfs.Node` (README, "Building apps on rnfs"). + +Compatibility rules every change keeps: +- **The wire format doesn't change** without a deliberate decision: request paths, messages, CIDs, DAG bytes, + chunk boundaries, provider and name records and announce app_data stay byte-identical, so nodes of different + versions work together. `tests/test_wire.py` checks this against vectors made by 0.1.3 and 0.1.4. +- **The local HTTP API stays compatible**, status codes included. +- **State upgrades are automatic and one way.** A newer rnfs upgrades an `` an older one wrote; + downgrades need not work. + +## Setting up + +```sh +git clone https://github.com/afit21/rnfs && cd rnfs +python3 -m venv .venv && . .venv/bin/activate +pip install --require-hashes --only-binary=:all: -r packaging/requirements.lock # rns, lxmf and theirs +pip install --require-hashes --only-binary=:all: -r packaging/build-tools.lock # ruff, coverage, build, ... +pip install --no-deps --no-build-isolation -e . +``` + +The library and the CLI use the standard library besides RNS and LXMF; `rnfs.apiclient`, `rnfs.remote` +(rnfs.Client) and the modules they import never import RNS. Keep it that way. + +## Tests + +The suite uses `unittest` (pytest isn't needed) and runs on Linux, macOS and Windows, Python 3.11 to 3.14: + +```sh +python3 -m unittest discover tests # everything (a few minutes) +python3 -m unittest tests.test_storage tests.test_fetcher # some modules +python3 -m unittest discover -s tests/security -p "test_*.py" # the security regressions +``` + +`.github/workflows/tests.yml` holds the exact commands CI runs. Some tests are opt-in through environment +variables: + +| Variable | What it turns on | +|---|---| +| `RNFS_E2E=1` | Real daemons and real RNS over local TCP, driven through the CLI or the library. Run them with `RNFS_E2E=1 python3 -m unittest tests.test_e2e tests.test_client tests.test_embedded tests.security.test_h1_real_rns tests.security.test_m1_real_rns tests.security.test_h2_display_page_pins` (3 to 4 minutes). CI's end-to-end step lists every module gated on it, and `tests/test_packaging.py` fails if a new one is missing from that list. | +| `RNFS_FUZZ_EXAMPLES=N` | A deeper, randomized run of `tests/security/test_fuzz_parsers.py` (e.g. 3000 examples per parser). That module needs hypothesis, which isn't one of rnfs's dependencies: without it every test there is skipped, as in CI. | +| `RNFS_BENCH=1` | Prints the FastCDC chunking rate measured by `tests/test_chunker.py` (the test itself only checks a floor). | + +`tests/bench_block_writes.py` is a benchmark, not a test: run it by hand on the disks that matter (its docstring +says how, including on a Raspberry Pi SD card). Performance changes record their before and after numbers in +CHANGELOG.md. + +`tests/interop.py` is a manual cross-version check, not a test either: it runs this tree's daemons against daemons +of older releases over local TCP, in both directions, and prints a PASS/FAIL table (providers, get and cat, +transfers with and without `max_upload_rate` checked for refusals, names, keepers, LXMF). Run it before a release +with each supported old release extracted by `git archive`: + + mkdir -p /tmp/rnfs-0.1.5 && git archive v0.1.5 | tar -x -C /tmp/rnfs-0.1.5 + python3 tests/interop.py --old /tmp/rnfs-0.1.5 --old /tmp/rnfs-0.1.4 + +It takes about five minutes per release; logs are kept on failure or with `--keep`. + +Tests that guard compatibility, and must stay green: +- `tests/test_wire.py`: wire known-answer vectors (`tests/fixtures/wire`). +- `tests/test_upgrade_014.py`, `tests/test_upgrade_015.py`: the current code started on an `` exactly + as 0.1.4 and 0.1.5 left it (`tests/fixtures/rnfs-0.1.4`, `tests/fixtures/rnfs-0.1.5`) keeps every row. +- `tests/test_statedb.py`'s released-schema digest (below). +- `tests/test_rns_compat.py`: the RNS internals rnfs relies on. +- `tests/test_packaging.py`: the workflows, the locks, install.sh, build-deb.sh, the Flatpak manifest, the + supported Pythons, and the CHANGELOG's version against `rnfs.__version__`. + +Never weaken or delete a test to make a change pass. If a change deliberately alters behaviour a test encodes, +change the test in the same commit and say why in the commit message. + +### Security regression tests + +Every security fix comes with a regression test in `tests/security/test__.py` (for example +`test_cont1_deny_hash_amplification.py`), written first and seen failing before the fix. Rebuild the attack from +its description with the fakes and helpers in `tests/` and `tests/security/` (see the sibling tests), cite +function names rather than line numbers, and add the test to the index in `tests/security/README.md` (ID, +module, fix). A test that needs real RNS is gated on `RNFS_E2E` and added to CI's end-to-end step. + +## Lint + +ruff, at the version `packaging/build-tools.lock` pins, with the rules in `pyproject.toml`: + +```sh +ruff check . +``` + +A broad `except Exception` in `rnfs/` carries `# noqa: BLE001 - `, an import that +can't be at the top `# noqa: E402`, and RUF100 flags a `noqa` that suppresses nothing. The companion NomadNet +pages (`companion-nomadnet-sites/*/index.mu`) are linted as Python. CI also fails on merge-conflict markers. + +Style: match the surrounding code. Every function has a docstring, comments say why rather than what, and lines +are at most 120 columns. + +## Dependency and build locks + +Every install path (install.sh, build-deb.sh, the Flatpak, the PyInstaller builds, the tests) uses hash-locked +versions: + +| File | What | Regenerate with | +|---|---|---| +| `packaging/requirements.lock` | rnfs's runtime dependencies (rns, lxmf and theirs), every file PyPI has for each | `python3 -I packaging/update-lock.py [rns==X lxmf==Y ...]` (no arguments: the newest versions `pyproject.toml` allows) | +| `packaging/build-backend.lock`, `packaging/build-tools.lock` | setuptools; and build, twine, PyInstaller, ruff, coverage and their dependencies | `python3 -I packaging/update-lock.py --build pyinstaller==X ...` (run it on Linux with Python 3.12) | +| `packaging/flatpak/python3-deps.json`, `build-sources.json` | the Flatpak's wheels, from the locks | `python3 -I packaging/flatpak/generate-sources.py` | + +After changing a lock, regenerate the Flatpak sources and commit them together: tests.yml's flatpak-sources job +fails if regenerating changes them. The setuptools in `build-backend.lock` must fit `pyproject.toml`'s +`[build-system] requires`, and README's `pyinstaller==` the version `build-tools.lock` pins (both are checked). + +**Supported Pythons.** The list is in several places, and `tests/test_packaging.py` checks they agree: the +tests.yml matrix, `requires-python` in `pyproject.toml`, `BUILD_PYTHONS` and the `--python-version` of +`RESOLVE_ARGS` in `packaging/update-lock.py`, `RUNTIME_PYTHONS` in `packaging/flatpak/generate-sources.py`, +`EXTRA_PYTHONS` and `PYTHON_BELOW` in `packaging/build-deb.sh`, and the interpreters `install.sh`'s +`find_python` tries. + +## Adding a schema step + +Everything rnfs persists is in one SQLite database (`rnfs/statedb.py`). Its schema is the list of steps +`statedb.SCHEMA`; `PRAGMA user_version` counts those applied, and a database is upgraded step by step at start. + +- **Append only.** A store whose layout changes gets a new step (`ALTER TABLE`, or a new table and an + `INSERT ... SELECT`; a callable taking the connection for what SQL alone can't express), labelled with the + store. Never edit a released step: a database that release wrote already has it, so the edit would never reach + it. +- The released steps are pinned by a digest in `tests/test_statedb.py` (`RELEASED_0_1_5`, the first 16). Never + change it; when a release adds steps, pin those with a digest of their own. +- `tests/test_upgrade_015.py` starts the current code on `tests/fixtures/rnfs-0.1.5` (see its README): new + tables and rows are fine, but none of 0.1.5's rows may change or go. A step that rewrites old rows needs the + test to say what they become. +- The old JSON state files are imported once (`statedb.LEGACY`, `migrate`); a store that still reads one keeps + its `tests/test_migrate_*.py`. + +## Adding an API route + +The local API (`rnfs/api.py`, specification section 11) must stay compatible, status codes included. A new +route touches these places: + +1. `ApiHandler.ROUTES` (`(method, path)` to the handler's name) and its `api_*` method in `rnfs/api.py`; a + feature's route goes in that feature's `api_routes()` (`rnfs/features/`) instead. GET routes never change + pins, names or content. Errors map to statuses through the error types (`ValueError` 400, `NotFound` 404, + ...), not by hand. +2. The route tables in `rnfs/api.py`'s module docstring (core or feature table): `tests/test_api.py`'s + `RouteTableTest` checks that they list exactly the routes, and that each name in `ROUTES` is a method. +3. design.md's list of core routes ("Core API routes", under "Node facade") and, for new behaviour, + specification.md section 11. +4. The client: an `ApiConnection` method in `rnfs/apiclient.py`, and if it is part of the public library, the + same method on `rnfs.Node` and `rnfs.Client` (`rnfs/interface.py`, `rnfs/embedded.py`, `rnfs/remote.py`), + tested on both by `tests/test_public.py`. +5. The CLI command, if any (`rnfs/cli.py`, or the feature's own commands), through `ApiConnection` rather than + a hand-written path. +6. Tests of the handler (`tests/test_api.py` or the feature's test file), and a CHANGELOG line. + +## Commits + +One change per commit, with a plain summary line (ending with the plan or audit item ID in parentheses where +there is one) and a body saying what changed and why. Behaviour-neutral refactors go in their own commits, with +the suite green, before the change that relies on them. CHANGELOG.md gets a line for every user-visible change, +under the next version's `## X.Y.Z (unreleased)` heading. + +## Releasing + +A release is published by pushing a tag `vX.Y.Z`: `.github/workflows/release.yml` runs the whole tests workflow, +builds every package (wheel and sdist, `.deb` for amd64 and arm64, the x86_64 Flatpak, the Windows exe), +smoke-tests them, and (through the `release` environment, which may require a reviewer) publishes a GitHub Release with +`SHA256SUMS` and signed build provenance. The release notes are CHANGELOG.md's section for the version. + +The same builds run, without publishing, on pull requests that touch `packaging/**`, `pyproject.toml` or +`.github/**`, weekly, and when the workflow is started by hand, so check that they pass before tagging. + +Steps: +1. Go through the checklist below. +2. Set `__version__` in `rnfs/__init__.py` to X.Y.Z, change CHANGELOG.md's first heading from + `## X.Y.Z (unreleased)` to `## X.Y.Z`, and update README's release links and file names + (`tests/test_packaging.py` checks the three agree). +3. Commit, let CI pass on `main`, then push the tag: `git tag vX.Y.Z && git push origin vX.Y.Z`. The workflow + refuses a tag that isn't `v` + `__version__`, and a CHANGELOG whose section for it is still marked + unreleased. +4. If the `release` environment requires a reviewer, approve its deployment once the builds pass. +5. Open the next version's section: `## X.Y.(Z+1) (unreleased)` at the top of CHANGELOG.md (`__version__` stays + X.Y.Z until that release), and name that version in the note at the top of specification.md (checked too). + +### Release checklist + +- [ ] Every decision the release's plan needed is answered, and the maintainer-only actions (GitHub settings, + PyPI, benchmarks on real hardware) are done or deliberately left. +- [ ] Wire: `tests/test_wire.py`'s 0.1.3 and 0.1.4 vectors pass byte for byte. +- [ ] Interop: the new version against the previous releases' nodes (fixtures in `tests/fixtures/`, + `tests/test_rns_compat.py`). Fetch, provide, names and keepers behave the same, and a previous release's + fetcher sees no new refusals in an honest transfer. +- [ ] Upgrade: `tests/test_upgrade_015.py` (and `test_upgrade_014.py`) pass, including any table the release + adds, and the released-schema digest is unchanged. +- [ ] Every security fix has its `tests/security/test__*.py` regression test, and `tests/security/README.md` + lists it. +- [ ] CI runs every `RNFS_E2E` module. Lint passes. +- [ ] Before and after numbers for the performance changes (a desktop NVMe disk and a Raspberry Pi SD card), + recorded in CHANGELOG.md. +- [ ] specification.md updated for every change in behaviour, with its version and compatibility note at the top. +- [ ] design.md updated for every change in how things work. +- [ ] CHANGELOG.md's entry states whether the wire format or CIDs changed ("no wire-format or CID changes"), + has a **Security** section, and lists every user-visible change (changed defaults, new refusals, changed + formats, new settings). +- [ ] CHANGELOG's version equals `__version__` (`tests/test_packaging.py`). + +For 0.1.6 the plan (`0.1.6-plan.md`, "Release checklist") adds: the power-loss recovery path of item B is +tested (a leftover `.dirty`, a truncated recent block, an old block left alone); the numbers cover A1, A2, B, C1, +D1 and D3; and the CHANGELOG's user-visible changes include FEAT-5's API default, API-1's refusals, API-7, +SUP-3, SUP-5, FEAT-2's inbox format and `follow_quota`. diff --git a/README.md b/README.md index 45f3182..099f4ee 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Access any file on RNFS by it's **'CID'**. ***Example:*** The Communist Manifesto PDF: ```bagaibqabciqkq6rth6jhr2vkh7c2ip7p6bsmdxmslbia47rjbmwpcz6wwu6veya``` [![License: MIT](https://img.shields.io/badge/License-MIT-cyan.svg)](LICENSE) -[![Python 3.8+](https://img.shields.io/badge/Python-3.11+-blue.svg)](https://python.org) +[![Python 3.11+](https://img.shields.io/badge/Python-3.11+-blue.svg)](https://python.org) [![Reticulum](https://img.shields.io/badge/Reticulum-Mesh_Network-purple.svg)](https://reticulum.network/)
@@ -93,7 +93,7 @@ Different package options available at [latest release](https://github.com/afit2 **Windows:** -[`rnfs-0.1.5-windows-x64.exe`](https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-windows-x64.exe), saved as `rnfs.exe` +[`rnfs-0.1.6-windows-x64.exe`](https://github.com/afit21/rnfs/releases/download/v0.1.6/rnfs-0.1.6-windows-x64.exe), saved as `rnfs.exe` somewhere on your PATH ([how](#windows)). **Linux:** @@ -110,7 +110,7 @@ Or installers for ([Debian & Ubuntu](#debian-and-ubuntu)). [Flatpak](#other-lin **macOS:** -`pipx install https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-py3-none-any.whl` ([details](#macos)). +`pipx install https://github.com/afit21/rnfs/releases/download/v0.1.6/rnfs-0.1.6-py3-none-any.whl` ([details](#macos)). ## 2 - Start rnfs and leave it running in its own terminal: @@ -144,14 +144,14 @@ command. Your data in `~/.rnfs` is never touched by an update or uninstall. | System | Download | |---|---| -| Windows (64-bit) | [`rnfs-0.1.5-windows-x64.exe`](https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-windows-x64.exe) ([how](#windows)) | -| Debian 12+, Ubuntu 24.04+, Raspberry Pi OS (64-bit) | [`rnfs_0.1.5_amd64.deb`](https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs_0.1.5_amd64.deb) or [`rnfs_0.1.5_arm64.deb`](https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs_0.1.5_arm64.deb) ([how](#debian-and-ubuntu)) | -| Other Linux (x86_64) | [`rnfs-0.1.5-x86_64.flatpak`](https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-x86_64.flatpak) ([how](#other-linux-flatpak)), or the [install script](#install-script) | -| macOS, or any system with Python 3.11+ | [`rnfs-0.1.5-py3-none-any.whl`](https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-py3-none-any.whl) ([how](#macos)) | +| Windows (64-bit) | [`rnfs-0.1.6-windows-x64.exe`](https://github.com/afit21/rnfs/releases/download/v0.1.6/rnfs-0.1.6-windows-x64.exe) ([how](#windows)) | +| Debian 12+, Ubuntu 24.04+, Raspberry Pi OS (64-bit) | [`rnfs_0.1.6_amd64.deb`](https://github.com/afit21/rnfs/releases/download/v0.1.6/rnfs_0.1.6_amd64.deb) or [`rnfs_0.1.6_arm64.deb`](https://github.com/afit21/rnfs/releases/download/v0.1.6/rnfs_0.1.6_arm64.deb) ([how](#debian-and-ubuntu)) | +| Other Linux (x86_64) | [`rnfs-0.1.6-x86_64.flatpak`](https://github.com/afit21/rnfs/releases/download/v0.1.6/rnfs-0.1.6-x86_64.flatpak) ([how](#other-linux-flatpak)), or the [install script](#install-script) | +| macOS, or any system with Python 3.11+ | [`rnfs-0.1.6-py3-none-any.whl`](https://github.com/afit21/rnfs/releases/download/v0.1.6/rnfs-0.1.6-py3-none-any.whl) ([how](#macos)) | -These links are for 0.1.5; for a newer version, take the same files from the +These links are for 0.1.6; for a newer version, take the same files from the [latest release](https://github.com/afit21/rnfs/releases/latest), which also has the source archive -(`rnfs-0.1.5.tar.gz`) and `SHA256SUMS` ([checking a download](#checking-a-download)). +(`rnfs-0.1.6.tar.gz`) and `SHA256SUMS` ([checking a download](#checking-a-download)). ## Windows @@ -160,7 +160,7 @@ The exe is a single file with Python, Reticulum and LXMF inside. In PowerShell, ```powershell $bin = "$env:USERPROFILE\.local\bin" New-Item -ItemType Directory -Force $bin | Out-Null -Move-Item .\rnfs-0.1.5-windows-x64.exe "$bin\rnfs.exe" -Force +Move-Item .\rnfs-0.1.6-windows-x64.exe "$bin\rnfs.exe" -Force Unblock-File "$bin\rnfs.exe" [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", "User") + ";$bin", "User") # open a new terminal, then: @@ -181,8 +181,8 @@ Pick `amd64` or `arm64` to match `dpkg --print-architecture`. Ubuntu 22.04 has P use the [install script](#install-script) there. ```sh -curl -LO https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs_0.1.5_amd64.deb -sudo apt install ./rnfs_0.1.5_amd64.deb +curl -LO https://github.com/afit21/rnfs/releases/download/v0.1.6/rnfs_0.1.6_amd64.deb +sudo apt install ./rnfs_0.1.6_amd64.deb systemctl --user enable --now rnfs # run it as a service; logs: journalctl --user -u rnfs -f loginctl enable-linger $USER # keep it running while you're logged out ``` @@ -193,23 +193,25 @@ loginctl enable-linger $USER # keep it running while you're logged ou ## Other Linux (Flatpak) ```sh -curl -LO https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-x86_64.flatpak +curl -LO https://github.com/afit21/rnfs/releases/download/v0.1.6/rnfs-0.1.6-x86_64.flatpak flatpak remote-add --user --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo -flatpak install --user rnfs-0.1.5-x86_64.flatpak +flatpak install --user rnfs-0.1.6-x86_64.flatpak alias rnfs='flatpak run io.github.afit21.rnfs' # add to ~/.bashrc rnfs daemon ``` -The app can use the network, your home folder and all devices (for RNode and other serial interfaces; remove -that with `flatpak override --user --nodevice=all io.github.afit21.rnfs`). Other folders need -`flatpak override --user --filesystem=/mnt/data io.github.afit21.rnfs`. Update by installing the newer -`.flatpak`; uninstall with `flatpak uninstall --user io.github.afit21.rnfs`. +The app can use the network, `~/.rnfs` and `~/.reticulum`, and your Downloads folder: not the rest of your home +folder, nor any device. Grant more with `flatpak override --user --device=all io.github.afit21.rnfs` (RNode and +other serial interfaces) or `flatpak override --user --filesystem= io.github.afit21.rnfs` (another +folder, or `~/.config/reticulum` if your Reticulum config is there); `--reset` takes them back. Until 0.1.6 the app had your whole home folder and every +device: after updating, grant the ones you used again. Update by installing the newer `.flatpak`; uninstall +with `flatpak uninstall --user io.github.afit21.rnfs`. ## macOS ```sh brew install pipx && pipx ensurepath # then open a new terminal -pipx install https://github.com/afit21/rnfs/releases/download/v0.1.5/rnfs-0.1.5-py3-none-any.whl +pipx install https://github.com/afit21/rnfs/releases/download/v0.1.6/rnfs-0.1.6-py3-none-any.whl rnfs daemon ``` @@ -220,15 +222,19 @@ rnfs daemon https://raw.githubusercontent.com/afit21/rnfs/main/contrib/io.github.afit21.rnfs.plist launchctl load -w ~/Library/LaunchAgents/io.github.afit21.rnfs.plist ``` -- pipx installs the newest Reticulum and LXMF rnfs allows, not the exact tested versions the other packages - bundle. +- pipx (and plain `pip install`) resolves rnfs's dependencies unpinned, at the newest Reticulum, LXMF and + cryptography rnfs allows, not at the tested versions and hashes in `packaging/requirements.lock` that the + other packages and the install script use. ## Install script For any Linux with Python 3.11+ and venv (`sudo apt install python3 python3-venv` on Debian/Ubuntu). It installs the latest release into `~/.local/share/rnfs/venv`, with the dependency versions and hashes pinned in -the release, and checks the download against `SHA256SUMS` (and its signed provenance, if the GitHub CLI is -installed and logged in; `RNFS_REQUIRE_ATTESTATION=1` makes that required). +the release, and checks the download against `SHA256SUMS` and, if the GitHub CLI is installed, its signed +provenance for that release's tag. If `gh` is installed but not logged in, it stops: run `gh auth login`, or set +`RNFS_REQUIRE_ATTESTATION=0` to check the checksum only. Without `gh` it warns (`RNFS_REQUIRE_ATTESTATION=1` makes +it stop): `curl | sh` without `gh` trusts whoever can write the release's files, since `SHA256SUMS` comes from +the same place. ```sh curl -fsSL https://raw.githubusercontent.com/afit21/rnfs/main/install.sh | sh # install or update @@ -240,8 +246,11 @@ curl -fsSL https://raw.githubusercontent.com/afit21/rnfs/main/install.sh | sh -s - `--main` installs the main branch instead (no checksum to check; give it on every update). - `--system`, as root, installs for all users in `/opt/rnfs` and `/usr/local/bin/rnfs`. - `--python PATH` picks the Python; `--force` replaces another `rnfs` command (e.g. pipx's); - `RNFS_SOURCE=` installs something other than the latest release. -- With the service, `loginctl enable-linger $USER` keeps it running while you're logged out. + `RNFS_SOURCE=` installs something other than the latest release (nothing checks + it; a plain `http://` URL gets a warning). +- With the service, `loginctl enable-linger $USER` keeps it running while you're logged out. More sandboxing + (`ProtectSystem`, `PrivateTmp`) needs unprivileged user namespaces, so it is an optional drop-in, for this + unit or the `.deb`'s: see [`contrib/rnfs.service.d/hardening.conf`](contrib/rnfs.service.d/hardening.conf). ## Checking a download @@ -249,12 +258,12 @@ Each release's `SHA256SUMS` lists every file's checksum. With it next to your do ```sh sha256sum -c SHA256SUMS --ignore-missing # Linux; macOS: shasum -a 256 -gh attestation verify rnfs_0.1.5_amd64.deb --repo afit21/rnfs +gh attestation verify rnfs_0.1.6_amd64.deb --repo afit21/rnfs ``` A matching checksum shows the file is whole, not who made it. The signed build provenance, checked with the [GitHub CLI](https://cli.github.com/) (2.49 or newer, logged in), shows it was built by this repository's -release workflow. On Windows, `Get-FileHash .\rnfs-0.1.5-windows-x64.exe` gives the checksum. +release workflow. On Windows, `Get-FileHash .\rnfs-0.1.6-windows-x64.exe` gives the checksum. ## From source @@ -267,7 +276,9 @@ python3 -m unittest discover tests # the tests ``` `pipx install git+https://github.com/afit21/rnfs` installs the main branch without a checkout. The release -packages are built by `.github/workflows/release.yml`; the scripts it uses are in `packaging/`. +packages are built by `.github/workflows/release.yml`; the scripts it uses are in `packaging/`. To work on rnfs +itself (the tests and their options, lint, the locks, adding a schema step or an API route, releasing), see +[CONTRIBUTING.md](CONTRIBUTING.md). ## Reticulum and where things live @@ -434,6 +445,7 @@ lxmf_accept_from = [""] | `remote_pin_identities`, `remote_pin_quota` | `[]`, 1 GiB | Identities allowed to ask this node to pin (`pin remote add`), and the storage each may use | | `deny_lists` | `[]` | [Deny lists](#keeping-content-off-your-node) to apply | | `keep_max_names` | `100` | Names you keep or follow for others | +| `follow_quota` | 1 GiB | Storage each followed name's target may use; a larger one isn't fetched and the previous one stays pinned (up to twice this while it changes). `0` stops fetching followed names | | `lxmf_accept_from` | everyone | Only keep LXMF messages from these addresses (recommended if you only expect links from people you know) | | `lxmf_auto_fetch_from`, `lxmf_auto_fetch_quota` | `[]`, 1 GiB | Senders whose links are fetched and pinned automatically, and the storage each may use | | `lxmf_propagation_node` | none | Propagation node for offline delivery (`send --propagate`) and periodic downloads | @@ -444,7 +456,9 @@ lxmf_accept_from = [""] | `verbosity` | `0` | Extra logging: 1 per-query detail, 2 RNS debug output (`daemon -v`, `-vv`) | **Private mode.** `private = true` with a `members` list (identity hashes, from each -node's `rnfs status`) makes the node link to, serve and keep records for members only. +node's `rnfs status`) makes the node link to, serve and keep records for members only. A name's announce still +reaches everyone, so outsiders can see which CID it points at and when it changes (not fetch it), and +`rnfs name publish` warns of this in private mode. LXMF senders you've named (in `lxmf_accept_from`, `lxmf_auto_fetch_from`, or as an LXMF contact) are trusted. Anyone can make new LXMF addresses for free, so messages from everyone else only push out each other's, within @@ -479,7 +493,7 @@ Every command except `daemon` talks to the running daemon, so run them as the da | `rnfs contacts add NICK HEX` / `rm` / `ls` | Nicknames for names, node IDs and LXMF addresses | | `rnfs lxmf address` / `lxmf sync` | Your LXMF address / download waiting messages now | | `rnfs send ADDRESS TARGET [-m TEXT] [--propagate]` | Send an `rnfs:` link over LXMF | -| `rnfs inbox` / `inbox get N` / `inbox rm N` / `outbox` | Received links, fetching them, sent messages | +| `rnfs inbox` / `inbox get N` / `inbox rm N` / `outbox` | Received links (each line starts with the sender's address and whether it is `untrusted`, `trusted` or a contact; the display name the sender chose comes last, quoted), fetching them (a name link whose name can't be resolved needs `--use-sent-cid`), sent messages | `add` options: `--wrap` keeps the file's name in the link; `--only-hash` prints the CID without storing; `--no-pin` keeps it as evictable cache; `--chunker fastcdc` lets an edited version share most blocks; @@ -514,7 +528,7 @@ with rnfs.Node("~/.myapp/rnfs") as node: # or: rnfs.Client(), for the d added = node.add("photos/") # rnfs.Added: .cid, .size, .warnings for entry in node.ls(added.cid): # rnfs.Entry(name, kind, size, cid) print(entry.name, entry.size) - with node.open(f"{added.cid}/cat.jpg") as f: # seekable: each read fetches only the chunks it covers + with node.open(f"{added.cid}/cat.jpg") as f: # seekable: a read after a seek fetches only what it covers data = f.read(4096) site = node.names.create("site") site.publish(added.cid) @@ -545,7 +559,10 @@ To use a list someone maintains (a text file, one entry per line, `#` for commen A denied CID is never fetched, stored or served, whether asked for directly or found inside other content (the gateway answers 451). Cached copies are deleted and remote pins released; your own pins are kept but not served, and the log names them so you can remove them. A file hash is only known once a file is downloaded, -so it stops the file being handed out or added, not its blocks being fetched: deny the CID for that. +so it stops the file being handed out or added, not its blocks being fetched: deny the CID for that (the first +time rnfs refuses a file by its hash, the log names the command). While there are file hashes, a file that +repeats its content more than 16 times beyond 64 MiB can't be read or imported (451): it can't be checked +cheaply. # Example use cases diff --git a/companion-nomadnet-sites/display-mu-file/index.mu b/companion-nomadnet-sites/display-mu-file/index.mu index 0d330ac..f2b64f9 100755 --- a/companion-nomadnet-sites/display-mu-file/index.mu +++ b/companion-nomadnet-sites/display-mu-file/index.mu @@ -332,7 +332,7 @@ def main(): show(cid) except subprocess.TimeoutExpired: unavailable(cid, f"rnfs took longer than {TIMEOUT} s") - except Exception as error: # rnfs not installed, a state file that can't be written, odd CLI output... + except Exception as error: # noqa: BLE001 - rnfs missing, an unwritable state file, odd CLI output unavailable(cid, f"{type(error).__name__}: {error}") diff --git a/companion-nomadnet-sites/file-index-node/README.md b/companion-nomadnet-sites/file-index-node/README.md index f095cc5..8749b5e 100644 --- a/companion-nomadnet-sites/file-index-node/README.md +++ b/companion-nomadnet-sites/file-index-node/README.md @@ -41,6 +41,8 @@ directory, so none of it is served. | File | What it is | | --- | --- | | `sealed.json` | Which pinned files are sealed, by CID. A CID's content never changes, so the answers stay valid. You can delete it at any time. | +| `page.mu` | The page as last rendered, served for a minute (see below). Delete it to show a change (a new pin, `DESCRIPTIONS`, `HIDDEN`) at once. | +| `lock` | Held while a request renders the page. | | `page.log` | Errors, with details. Moved to `page.log.1` past 1 MB. | If the daemon can't be reached, visitors only see "The file list isn't available right now". The CLI's error @@ -48,3 +50,13 @@ message, which can contain local paths, goes to `page.log`. `RNFS_PAGE_DIR` overrides the directory when you run the page by hand. NomadNet runs pages with a minimal environment, so it doesn't pass that variable on. + +## How often it asks rnfs + +Rendering the page runs `rnfs pin ls`, and a sealed check for each file pin it hasn't checked before. So +that visitors can't keep the daemon busy by reloading the page, it renders at most once every 60 seconds +(`CACHE_SECONDS`), one request at a time (a lock, as the display page uses). Every visit in between gets the +page as last rendered, from `page.mu`. A visit that arrives while another request is rendering also gets +`page.mu`, however old, or "This page is busy" if there is none yet. The page is replaced atomically, so a +visitor never gets part of one. A failed render (the daemon isn't running) isn't kept, so the next visit +tries again. diff --git a/companion-nomadnet-sites/file-index-node/index.mu b/companion-nomadnet-sites/file-index-node/index.mu index 0d8dd61..90f9aab 100755 --- a/companion-nomadnet-sites/file-index-node/index.mu +++ b/companion-nomadnet-sites/file-index-node/index.mu @@ -7,7 +7,9 @@ Anyone can open the page, so it lists only what the operator chose to share and - never a sealed (encrypted) file, whose name is meant to stay private: the page checks each file pin's first bytes once (`rnfs cat --raw`, a local read) and remembers the answer in its data dir; - names and descriptions are escaped, so they show literally on one line and can't add Micron markup; -- visitors see a generic message if the list isn't available; the details go to a log in the data dir. +- visitors see a generic message if the list isn't available; the details go to a log in the data dir; +- the page is rendered at most once a minute, one request at a time (a lock file): visitors meanwhile get the + page as last rendered, kept in the data dir, so visiting often doesn't keep the daemon busy. See this page's README. """ import json @@ -17,13 +19,9 @@ import subprocess import time import unicodedata -# Descriptions shown under each file, keyed by CID (a CID never changes, unlike a file name). +# Descriptions shown under each file, keyed by CID (a CID never changes, unlike a file name). For example: +# "bagaibqabciqkq6rth6jhr2vkh7c2ip7p6bsmdxmslbia47rjbmwpcz6wwu6veya": "Marx and Engels, 1848. PDF.", DESCRIPTIONS = { - "bagaibqabciqkq6rth6jhr2vkh7c2ip7p6bsmdxmslbia47rjbmwpcz6wwu6veya": "Marx and Engels, 1848. PDF.", - "bagaibqabciqh5e4vsvwazopdfypoiflpfjvbfqionjko7cvxsmzmumsdoppzixy": "Alpine Linux 3.24.2 ISOs.", - "bagaibqabciqaiphoe7qtscjuqprvwpr2ifr3hgnvuk5rgeh53mpk4ntapwutr3a": "MeshChat v2.4.0 release files", - "bagaibqabciqospcwo756sqjp7yl3h2vukd3ijkvffy4nyh2umljg4yluwp2ce6a": "The Original SCP-106 image", - "bagaibqabciqd6s6yrwpp6eemv27sa7ud4l3r7rjcyu5oh4fgziopqa432htdxma": "The Reticulum source code taken from GitHub on 11/10/2026", } # CIDs to leave off the page. @@ -43,6 +41,15 @@ PAGE_DIR = os.environ.get("RNFS_PAGE_DIR") or os.path.join( SEALED_CACHE = os.path.join(PAGE_DIR, "sealed.json") # {cid: sealed?}, a CID's content never changes LOG = os.path.join(PAGE_DIR, "page.log") LOG_MAX = 1024 * 1024 # the log is moved to page.log.1 past this size +CACHE = os.path.join(PAGE_DIR, "page.mu") # the page as last rendered +LOCK = os.path.join(PAGE_DIR, "lock") # held while a request renders the page + +# The page is rendered (which runs `rnfs pin ls`) at most once per CACHE_SECONDS; visitors meanwhile get the +# cached page, so visiting it often costs the daemon nothing. +CACHE_SECONDS = 60 + +UNAVAILABLE = "The file list isn't available right now; try again later.\n" +BUSY = "This page is busy with another request; try again in a moment.\n" # How a sealed (encrypted) file starts (rnfs/content/seal.py). SEALED_MAGIC = b"rnfs-sealed\n" @@ -123,7 +130,7 @@ def is_sealed(pin): if cid not in _sealed: try: _sealed[cid] = rnfs("cat", cid, "--raw", "--length", str(len(SEALED_MAGIC))).startswith(SEALED_MAGIC) - except Exception as error: + except Exception as error: # noqa: BLE001 - any failure counts as sealed, so it isn't listed log(f"checking whether {cid} is sealed failed: {type(error).__name__}: {error}") return True try: @@ -162,7 +169,8 @@ def entry(pin): return "\n".join(lines) -print("""#!c=0 +# The top of the page, before the lists. +HEADER = """#!c=0 ~+ * + @@ -188,27 +196,99 @@ RNFS is a decentralized file system built for Reticulum & based on IPFS (InterPl Any of the following files or folders can be freely accessed using RNFS: `F888rnfs get -o `f -""") +""" + def section(title, pins): - """Print a heading and its entries, sorted by name; nothing if no entry is shown.""" + """The Micron text of a heading and its entries, sorted by name; "" if no entry is shown.""" entries = [entry(pin) for pin in sorted(pins, key=lambda pin: (display_name(pin) or "").lower())] entries = [text for text in entries if text] - if entries: - print(f">{title}") - print("\n\n".join(entries) + "\n") - return bool(entries) - - -try: - pins = load_pins() -except Exception as error: # the daemon isn't running, rnfs isn't installed, ... - log(f"listing pins failed: {type(error).__name__}: {error}") - print("The file list isn't available right now; try again later.") -else: + return f">{title}\n" + "\n\n".join(entries) + "\n\n" if entries else "" + + +def render(): + """The whole page, or None if the pin list can't be had (the details are logged).""" + try: + pins = load_pins() + except Exception as error: # noqa: BLE001 - the daemon isn't running, rnfs isn't installed, ... + log(f"listing pins failed: {type(error).__name__}: {error}") + return None folders = [pin for pin in pins if pin.get("type") == "directory"] files = [pin for pin in pins if pin.get("type") != "directory"] - listed = section("Folders", folders) - listed = section("Files", files) or listed - if not listed: - print("Nothing is shared yet.") + lists = section("Folders", folders) + section("Files", files) + return HEADER + "\n" + (lists or "Nothing is shared yet.\n") + + +def cached(max_age): + """The cached page if it was rendered less than max_age seconds ago (at any time if max_age is None), else + None. One dated in the future (the clock was set back) counts as old.""" + try: + with open(CACHE, encoding="utf-8") as f: + age = time.time() - os.fstat(f.fileno()).st_mtime + if max_age is not None and not 0 <= age < max_age: + return None + return f.read() + except (OSError, ValueError): # missing, unreadable, or not UTF-8 (UnicodeDecodeError) + return None + + +def save(page): + """Cache page, atomically: a request reading the cache meanwhile gets the old page or this one, whole. + Called with the lock held, so the temporary file is this request's alone.""" + try: + with open(CACHE + ".tmp", "w", encoding="utf-8") as f: + f.write(page) + os.replace(CACHE + ".tmp", CACHE) + except OSError as error: + log(f"saving {CACHE} failed: {error}") + + +def take_lock(): + """Lock the page's lock file so requests render one at a time; the open file (closing it unlocks), or + None if another request holds the lock. (The display page's take_lock.)""" + handle = open(LOCK, "a+") + try: + try: + import fcntl + except ImportError: # Windows + import msvcrt + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + else: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + handle.close() + return None + return handle + + +def page_now(): + """The page for a visit that found no fresh cached page: rendered (and cached) now, under the lock; while + another request holds the lock (it is rendering), the cached page of any age, or a busy message if there is + none; a generic message if the data dir or the pin list can't be used.""" + try: + os.makedirs(PAGE_DIR, mode=0o700, exist_ok=True) + lock = take_lock() + except OSError as error: + # No lock, no rendering: the page fails closed, as the display page does + log(f"can't use the page's data dir {PAGE_DIR}: {error}") + return HEADER + "\n" + UNAVAILABLE + if lock is None: + return cached(None) or HEADER + "\n" + BUSY + with lock: + page = cached(CACHE_SECONDS) # the request that held the lock before may have just rendered it + if page is None: + page = render() + if page is None: + # Not cached: the next visit tries again, as the daemon may be back by then + return HEADER + "\n" + UNAVAILABLE + save(page) + return page + + +def main(): + """Print the page: the cached one while it is fresh (CACHE_SECONDS), else page_now().""" + print(cached(CACHE_SECONDS) or page_now(), end="") + + +if __name__ == "__main__": + main() diff --git a/contrib/io.github.afit21.rnfs.plist b/contrib/io.github.afit21.rnfs.plist index 2c46492..22bf726 100644 --- a/contrib/io.github.afit21.rnfs.plist +++ b/contrib/io.github.afit21.rnfs.plist @@ -8,7 +8,7 @@ Stop it with `launchctl unload ~/Library/LaunchAgents/io.github.afit21.rnfs.plist`; launchd sends SIGTERM, which the daemon handles by shutting down cleanly. It assumes `pipx install`, which puts rnfs in - ~/.local/bin; adjust the path below (both places) otherwise. launchd doesn't expand ~, hence the sh -c. + ~/.local/bin; adjust the path below (all three places) otherwise. launchd doesn't expand ~, hence the sh -c. If rnfs isn't there (e.g. uninstalled), the job logs that and exits successfully, so launchd doesn't keep restarting it. daemon.log isn't rotated: empty it now and then with `: > ~/.rnfs/daemon.log`. --> diff --git a/contrib/rnfs.service b/contrib/rnfs.service index 7e9776f..6a6252c 100644 --- a/contrib/rnfs.service +++ b/contrib/rnfs.service @@ -39,6 +39,10 @@ RestrictSUIDSGID=yes # is left out: libffi closures, which cffi (under cryptography) can use, may need writable executable memory. SystemCallFilter=@system-service SystemCallArchitectures=native +# ProtectSystem and PrivateTmp are left out: in a user unit they need unprivileged user namespaces, which some +# systems turn off. Where that works, add them with the drop-in in contrib/rnfs.service.d/hardening.conf +# (https://github.com/afit21/rnfs/blob/main/contrib/rnfs.service.d/hardening.conf). UMask is left out too: what +# `rnfs get` writes gets its permissions from your umask. [Install] WantedBy=default.target diff --git a/contrib/rnfs.service.d/hardening.conf b/contrib/rnfs.service.d/hardening.conf new file mode 100644 index 0000000..f3d97b9 --- /dev/null +++ b/contrib/rnfs.service.d/hardening.conf @@ -0,0 +1,24 @@ +# Optional extra sandboxing for the rnfs systemd user unit (contrib/rnfs.service, or the unit install.sh or +# the .deb installs), as a drop-in: +# +# mkdir -p ~/.config/systemd/user/rnfs.service.d +# cp contrib/rnfs.service.d/hardening.conf ~/.config/systemd/user/rnfs.service.d/ +# systemctl --user daemon-reload && systemctl --user restart rnfs +# systemctl --user status rnfs # check that it started +# +# To undo it, delete the file, then daemon-reload and restart again. +# +# These aren't in the unit itself: in a user unit (run by your own systemd --user, not as root) they need +# unprivileged user namespaces, which some distributions and hardened kernels turn off, and the unit then fails +# to start. Also, the daemon is what reads the files `rnfs add` names and writes the ones `rnfs get` and +# `rnfs export` write, so with PrivateTmp a path under /tmp or /var/tmp is the daemon's own private one, not +# the /tmp your shell sees. +# +# UMask= is left out, here and in the unit: what `rnfs get` writes gets its permissions from your umask, as any +# command's output does. + +[Service] +# /usr, /boot, /efi and /etc read-only for the daemon and any program a PipeInterface starts +ProtectSystem=full +# Private /tmp and /var/tmp: other processes don't see the daemon's temporary files, nor it theirs +PrivateTmp=yes diff --git a/design.md b/design.md index 01fbf75..1a0b6a7 100644 --- a/design.md +++ b/design.md @@ -5,30 +5,47 @@ authority on behaviour; this file fixes module boundaries and the interfaces bet ## Shared foundation -- `rnfs/constants.py` — protocol constants (spec section 12), request paths, aspects, DHT modes. +- `rnfs/constants.py` — protocol constants (spec section 12), request paths, aspects, DHT modes, and + `MISSED_ANNOUNCES` (2): the announces a node or name may miss before the routing table, name resolves, follows + and deny lists count it gone or stale (`2 × announce_interval` in the spec). - `rnfs/config.py` — `Config`: every local setting (section 12), paths under ``, and the `effective_*` helpers for settings that depend on slow nodes or links. Loaded from `/rnfs.toml` plus command-line overrides (and `defaults` beneath the file: an embedded node's); `validate()` checks every core setting's type and range (`WHOLE_NUMBERS`, `RATES`, `DURATIONS`, `FLAGS`, `IDENTITY_LISTS`), and the daemon exits on an invalid one. Reticulum is started from `reticulum_dir`, by default RNS's own config directory, so the node attaches to `rnsd` - with the matching RPC key. + with the matching RPC key. `member_set` and `remote_pin_set` are `members` and `remote_pin_identities` as + frozensets of lowercase hex, normalised once there for the node, the link pool and the request server. +- `rnfs/ids.py` — stdlib only: the one parser of 16-byte IDs written as 32 hex digits (node IDs, names, identity + hashes, LXMF addresses). `parse_hex16(text, what, *, brackets=False, upper=True)` takes whitespace around the + digits, RNS's `` only where `brackets`, and nothing between them (`bytes.fromhex` alone would skip + whitespace there); `is_hex16` is its check and `normal_hex16` the lowercase form. `api.parse_hex_id`, + `names.parse_name`, `identity.parse_node_id`, `contacts.parse_hex`, `lxmf.parse_address` and `Config.validate` + use it. Persisted rows are checked against its patterns (`HEX16`, `HEX16_LOWER`) as they are, unstripped, so + what loaded before still loads. - `rnfs/errors.py` — `RnfsError` and subclasses `InvalidDag`, `NotFound`, `TooLarge`, `StorageFull`, `Busy`, `Denied`, `Cancelled` (a fetch stopped by its observer), `Closed` (an `rnfs.Node` or `rnfs.Client` used after `close()`). - `rnfs/content/cid.py` — `Cid` (binary form `.binary`, `.codec` `RAW`/`DAG`, `.key` DHT key, `.verify(block)`, `str()` written form, `Cid.parse`, `Cid.from_bytes`, `Cid.for_block`). Code passes `Cid` objects everywhere; - strings only at the user edge (CLI, API, gateway, logs) and binary only on the wire. + strings only at the user edge (CLI, API, gateway, logs) and binary only on the wire. `Cid.from_block_name` is the + block store's scan at startup: it accepts exactly what `parse` accepts, but checks the canonical form with one + regular expression and decodes with `int(..., 32)` instead of re-encoding, keeping the name as the written form. - `rnfs/log.py` — logging through RNS's logger. `log()` escapes non-printable characters in every message (remote names can't inject terminal escapes or fake lines); `cap_warnings` keeps a result's first `MAX_WARNINGS` (100) warnings plus a count line and `warnings_total`. With a debug sink set - (`set_debug_sink`), every message also goes to it, whatever RNS's loglevel. + (`set_debug_sink`), every message also goes to it, whatever RNS's loglevel. `ThrottledLog(interval, clock)` + logs one line about a kind of event at most every interval (60 s), with a count of those left out; `due()` is + the check alone. The response guards, the request rate limit (`RequestServer._over_rate`), `RnsPaths` and the + peers listing use it. An unexpected exception caught by a broad `except` (a `# noqa: BLE001` reason says why + it's caught) is logged at `LOG_WARNING` inside the `except`, so the debug log gets its traceback: among them the + fetcher's `/blocks` worker, the lookup's routing-table callbacks and `LinkPool.request`. - `rnfs/debuglog.py` — the debug log file (`debug_log`, `daemon --debug-log`), `/debug.log` (0600), started by the daemon before the node: version first, then every rnfs message (warnings and errors logged in an `except` block with their traceback), RNS's own log lines at Info or more severe (`RNS_LEVEL`; copied from `RNS.logdest = LOG_CALLBACK`, which still prints all of them), threads that die (`threading.excepthook`), a status line every 5 minutes (`status_line`: the node's status, open links, each interface up or down), and gateway requests, numbered (`tag`). Each start and every 16 MiB moves the file to `debug.log.1`. Lookups report their queries through `Lookup(report=)` (dht - stays free of RNS), links their path, close reason and each request's outcome, all at `LOG_DEBUG`. + stays free of RNS at import), links their path, close reason and each request's outcome, all at `LOG_DEBUG`. - `rnfs/fsutil.py` — the one platform switch (`IS_WINDOWS`), `O_BINARY`, and `replace`/`remove` that retry for about a second on Windows while another handle holds the file; `rename_new` (rename onto a path that must not exist; atomic through `renameat2(RENAME_NOREPLACE)` on Linux or a hard link, else check then rename); @@ -57,7 +74,16 @@ pin source `remote:`) are RNS identity hashes, and LXMF addresses (`lx Persistent state lives in the state database. Each store keeps its state in memory, which is all reads use, loads it once at startup (checking every row as the JSON loaders checked every record: a malformed row is logged, skipped and deleted) and then writes only what changed: a pin is one row inserted or deleted, batched -stores write the records changed since their last save. At every `connect`, `statedb.migrate` first imports +stores write the records changed since their last save. The shared parts are in statedb: `open_store(state_dir, +load)` connects and loads, releasing the reference if the load fails; `delete_rows` deletes the rows a load found +malformed; and a `RowStore(db, table, key_cols, *, durable, mode)` keeps the keys a store changed (`changed(key)`, +under the store's lock) and not yet written, and writes their rows (`row(key)`, None once gone) in one +transaction: `write_through` at once, under the store's lock, raising if it fails; `snapshot` at `save()`, taking +the rows under the lock and writing them outside it, one save at a time so an older snapshot never lands last. +Either way a failed write keeps the keys for the next one, and the change stays in memory. pin_names, contacts, +jobs and the publisher are built on it (the other stores move over later). `reconcile_pins` is the start-up +check of a feature's pins against its saved state (released if it was saved whole, recorded again if not). +At every `connect`, `statedb.migrate` first imports the JSON state files of 0.1.4 and earlier (`LEGACY`): each through its owner's importer, which checks every record as the old loader did, in one transaction that also records the file in the `migrated` table; then the file is renamed `.migrated` and the directory synced. A file the table already lists (a crash between @@ -73,15 +99,15 @@ locks internally. ``` rnfs.py, rnfs/__main__.py entry points (python3 rnfs.py, python3 -m rnfs); the installed command is rnfs.cli:main rnfs/ - constants.py config.py errors.py log.py shared foundation + constants.py config.py ids.py errors.py log.py shared foundation debuglog.py the debug log file, /debug.log (debug_log) fsutil.py cross-platform file operations: replace/remove retried on Windows, O_BINARY, IS_WINDOWS statedb.py the state database, /state/rnfs.db: schema, transactions, JSON state migration content/ no networking; uses RNS's bundled msgpack and RNS logging, so importing it loads RNS cid.py CIDs (section 4) dag.py DAG node encoding, strict decoding, names (section 5) - chunker.py fixed and FastCDC chunkers; chunker.using(name) picks one per context (section 5) - builder.py building trees: files, directories, wrap, only-hash (section 5) + chunker.py fixed and FastCDC chunkers (section 5) + builder.py building trees: files, directories, wrap, only-hash; BuildOptions (section 5) seal.py sealed files: SealingReader, open_sealed/SealedFile, recipients (section 5, Sealed files) walker.py Expect, reader checks, path lookup, byte ranges, listing, writing out (sections 5, 7) edit.py dir set / dir rm (section 11) @@ -95,7 +121,8 @@ rnfs/ lookup.py iterative lookups with alpha/beta, time budget, unknown-ID budget (section 6) network/ everything that talks to RNS protocol.py message encoding/validation, batch sizes (sections 3, 6, 7, 8) - rates.py rate estimates, slow links, slow node detection (section 2) + rates.py rate estimates, slow links, slow node detection (section 2); the /blocks size rules both + ends share: response_capacity, default_upload_cap (sections 3, 7) links.py LinkPool: shared unidentified links, identified one-shot links, idle close (section 3) guards.py guards rejecting oversized, compressed or request Resource advertisements (on every link in the daemon, on rnfs.dht links only in an embedded node) and, on rnfs.dht links, @@ -144,9 +171,11 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda ## Content layer interfaces `dag.py` -- `FILE, DIRECTORY, INDEX = 1, 2, 3`; `KIND_FILE, KIND_EXECUTABLE, KIND_DIRECTORY = 0, 1, 2`. +- `FILE, DIRECTORY, INDEX = 1, 2, 3`; `KIND_FILE, KIND_EXECUTABLE, KIND_DIRECTORY = 0, 1, 2`, and `KIND_NAMES` + (`file`, `executable`, `directory`: how `ls` shows them). - Frozen dataclasses `Child(cid, size)`, `Entry(name, cid, size, kind)`, `IndexChild(first_name, cid, size)`; - nodes `FileNode(children)`, `DirectoryNode(entries)`, `IndexNode(children)`. + nodes `FileNode(children)`, `DirectoryNode(entries)`, `IndexNode(children)`, each with `child_cids` (the CIDs it + lists, in order) and `names` (its entries' names or children's first names; none for a file node). - `encode(node) -> bytes`; `decode(block) -> node`, strict (types, shortest headers, re-encodes identically), raising `InvalidDag`. Nodes are immutable and decoding is a pure function of the bytes, so `decode` remembers the nodes of the blocks it decoded last (`MEMO_BLOCKS` 256, `MEMO_BYTES` 2 MiB of blocks, least recently used @@ -162,16 +191,22 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda - `expand(expect, block) -> list[Expect]` — check a verified block against its expectation (type, size, names, index bounds, depth) and return its children's expectations. Raises `InvalidDag`. Raw blocks have none. - `step(expect, block, name) -> Expect | None` — one path step in a directory or index node (for an index, the - child that would hold `name`; keep stepping until a directory entry is reached). + child that would hold `name`; keep stepping until a directory entry is reached). `index_child(node, name)` is + that index rule (the child with the greatest first name <= `name`), shared with `Storage`'s path walk. - `range_children(expect, block, offset, length) -> list[tuple[Expect, int, int]]` — children of a file node overlapping a byte range, with the range relative to each child. - `parse_target(text) -> tuple[Cid, list[str]]` for `[/name…]` (ValueError if malformed). - Local operations over a blocking `get(expect) -> bytes` (raise `NotFound` when absent): `step_name(expect, name, get) -> Expect | None` (one name, stepping through index nodes), `resolve(root, names, get) -> Expect`, `iter_dag(expect, get)` (each `(expect, block)`, each CID once, - depth-first), `read_range(expect, get, offset, length) -> Iterator[bytes]`, + depth-first), `read_range(expect, get, offset, length) -> Iterator[bytes]`, `leaf_bytes(expect, get, stop_at)` + (the bytes of a file's distinct chunks, from its file nodes only: the deny list's hash bound), `list_directory(expect, get, max_entries=None) -> Iterator[Entry]` (`TooLarge` past - `max_entries`), `write_out(expect, get, dest, max_entries) -> warnings`. + `max_entries`), `height(expect, get)` (DAG nodes on the longest path, checking every node; `dir set` and + `dir rm` check the edited tree's depth with it, and it remembers each subtree it measured by its expectation, + `HEIGHT_MEMO` 32768 least recently used, so an edit reads only the directories it rebuilt), `write_out(expect, get, dest, max_entries, check=None, created=None) -> warnings` + (`created` gets a directory output's `(st_dev, st_ino)`, for `check_unchanged(dest, identity)` before it is + renamed; a directory replaced while written raises `ValueError("output was changed while being written")`). `storage.py` — `Storage(blocks_dir, state_dir, max_storage)`, thread-safe: - blocks: `has(cid)`, `get(cid, verify=False) -> bytes | None` (with `verify`, a block no longer matching its @@ -182,7 +217,9 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda are. It is handed only blocks built from, or checked against, their CIDs. `BlockStore.sync()` syncs the blocks directory; `Storage` calls it before saving pins or cache records, so they never refer to block names a power loss could undo. - pins: `pin(cid, source)`, `pin_many(cids, source)` (one save for them all; archive import), - `unpin(cid, source) -> bool`, `pins() -> dict[Cid, set[str]]`, `pin_count()`, `is_complete(cid)`. + `unpin(cid, source) -> bool`, `pins() -> dict[Cid, set[str]]` (a copy), `pinned_cids() -> frozenset[Cid]` (the + same set until pins change: the publisher's refresh reads it after every add), `has_pin(cid, source)`, + `pin_count()`, `is_complete(cid)`. - cache: `record_fetch(root, names, target)` (keeps the path's nodes and the target's DAG), LRU touched on use. `cache_generation()` grows whenever records are removed or retargeted, so the node prunes its fetched targets only then. `changes()` grows with every block stored or deleted, pin or release, and record added, @@ -201,7 +238,8 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda - `evict() -> set[Cid]` enforce `max_storage` (LRU cached fetches); `gc() -> stats` evicts every cached fetch; both delete unreachable blocks and report targets no longer complete, so the node can withdraw them. Both first read, without the lock, the DAG nodes they will walk (`_warm`, filling the `_links` and `_steps` - memos; completeness via `dag_blocks`), then decide and delete under the lock from the pins, sessions and + memos, its path walk the one `_path_cids` makes under the lock, `_walk_path`; completeness via `dag_blocks`), + then decide and delete under the lock from the pins, sessions and records current then, so `get` (and `/blocks` serving) waits only for that in-memory walk and the deletions, and a block that became reachable meanwhile is kept. The size queries (`pinned_bytes`, `dag_bytes`, `source_usage`, `stat`) read their DAG nodes the same way first. `_links` memoises only stored DAG nodes: a @@ -216,6 +254,13 @@ packaging/ requirements.lock (hash-pinned runtime dependencies), upda reaches one, `evict()` and `gc()` delete nothing and log it once (`_held_back`) until `verify()` removes it or a fetch replaces it. Fetches treat such a block as not stored: deleted and fetched again. - `dag_blocks(cid) -> frozenset[Cid]` the blocks of a complete DAG (memoised), used for "what a node serves". + For an incomplete DAG it remembers the block it found missing (`_missing`, `MISSING_MEMO` 10 000, oldest + forgotten first): a CID's content never changes, so the DAG stays incomplete while that block isn't stored, + and `is_complete` is a lookup until it arrives. The walks under the lock take a DAG memoised as complete + whole (`_reach`: the memo's set, less what is already kept) rather than walking it, except one holding a path + node eviction kept without its other children, which is walked as before; `dag_bytes` of a complete DAG is + summed once (`_complete_bytes`). `_delete` forgets the complete memos a deleted block breaks, which is how + every deletion (eviction, gc, verify, `drop_denied`) keeps them exact. `denylist.py` — `parse_entry(text) -> (canonical, binary CID | None, digest | None)`, `parse_list(bytes) -> (binary CIDs, digests)` (ValueError naming the line; `MAX_LIST_BYTES` 4 MiB, `MAX_LIST_ENTRIES` 100 000). @@ -226,9 +271,10 @@ lookup), `denies(cid)`, `denies_hash(digest)`; the local list `entries()`, `add( accepted; `reload(read)` reads every one back from storage at start); `known_hash(cid)` / `remember_hash` (a file CID's content SHA-256, `MAX_KNOWN_HASHES` LRU, kept across restarts in `deny_hashes`: `save()`, called by the node's `_save_state`, writes the rows changed since the last one outside the lock, behind a save lock); `add_listener(fn)`, -called after each change outside its lock. `walker.write_out(..., check)` and `walker.iter_files` and -`builder.checking(check)` (a ContextVar: `build_file` hashes each file and passes the digest before the file's root -node is stored) are its hooks in the content layer; `archive.import_archive(..., check)` runs on the complete roots +called after each change outside its lock; `DENY_HASH_RATIO` and `DENY_HASH_SLACK` bound what `_check_file` hashes. +`walker.write_out(..., check)` and `walker.iter_files` and +`builder.BuildOptions(chunker, check)` (`build_file` hashes each file and passes the digest to check before the +file's root node is stored) are its hooks in the content layer; `archive.import_archive(..., check)` runs on the complete roots before they are pinned. ## DHT layer interfaces @@ -247,6 +293,9 @@ the next one back a further backoff, so concurrent lookups don't all retry it), min_age) -> int` (0 senior and healthy, 1 healthy, 2 stale, 3 absent: ranks keepers), `standings(min_age) -> dict` (every node's standing now: the keeper query's snapshot from before its lookup), `contains(node_id)`, `nodes()`, `node_count()`, `live_count()` (nodes `/find` replies would include). +Every `heard` forgets expired bans and dropped nodes' seniority, looking only at the front of each: both are kept +in expiry order (an entry set again is moved to the end), and their readers check expiry too, so an entry out of +order (after a clock step or a changed setting) only lingers in memory. `providers.py` — `ProviderStore(max_records, state_dir, ttl=PROVIDER_TTL, clock=time.time, rng=random, max_per_publisher=None, max_per_key=1000, own_id=None, seniority=0)` (`state_dir` holds the state database): `store(key, publisher) -> bool`, `withdraw(key, publisher)`, `provide(publisher, @@ -257,7 +306,9 @@ non-members), `for_reply(key, exclude=lambda id: False) -> list[bytes]`, `provid that (at least 10) before, the node's own (`own_id`) excepted. A publisher at its limit has its expired records swept at most once per batch, and not at all before `sweep_after` (the earliest its oldest record can expire, from its last sweep), so each further key of a refused `/provide` costs one check; a key keeps at most 1000 publishers: a newcomer to a full key evicts a random -one, never one of the `K/2` oldest records. A tenth of the store is a newcomer pool evicting by fair share: each +one, never one of the `K/2` oldest records. The node's own record (`own_id`) isn't counted toward the 1000 and is +never a victim, here or in the pool (`_most_pooled_by_others` steps past it when it alone holds the most), so +Sybil stores can't push it out of its own store; its records may exceed the pool rather than evict each other. A tenth of the store is a newcomer pool evicting by fair share: each publisher's pooled keys sit in an ordered dict (oldest pooled first), and publishers in lists by how many they hold (`_holders`, with each one's index, and `_most_pooled`), so a full pool picks one of the biggest holders at random in O(1); one already holding the most is refused. The newest publisher with records outside the pool comes @@ -265,13 +316,22 @@ from a heap of (first seen, arrival) with stale entries dropped lazily. First-se (seen for `seniority`) are kept for a TTL, at most `max_records` // 10, in departure order; those of younger publishers whose last record the pool evicted (`_evicted`) for a TTL too, at most `pool_size` // 10 (at least 10), forgetting the one first seen most recently when full (an O(n) scan, only on such evictions). Times later than now -(a clock corrected backwards) are clamped to now at load and at each `expire()`. +(a clock corrected backwards) are clamped to now at load and at the first `expire()` after the clock went backwards: +the store notes the latest clock reading any time it holds came from (`_latest`, via `_now()`), and an `expire()` +with the clock not behind it skips the clamping pass, which would change nothing (about half of a pass at 100 000 +records, held under the lock). Records are added only through +`_insert`, by `store` and by the load alike, so the indexes beside `_records` (each publisher's keys and count +outside the pool, the counts, `_pooled`, `_holders`, `_newest`) are kept in step in one place; +`_check_consistency()` recomputes them from `_records`, and test_providers runs it after every test. `lookup.py` — network access is injected: `FindQuery = Callable[[node_id, key], tuple[list[bytes], list[bytes]] | None]` returning (providers, nodes). `Lookup(key, start_nodes, query, own_id, alpha, beta, k, timeout, is_known, on_answer, on_failure, exclude, report, -unreliable, may_retry)` with `find_closest() -> list[bytes]` and `find_providers(on_provider, should_stop) -> -list[bytes]`. A run ranks the nodes `unreliable` named when it learnt of them (`doubtful`) after every other, for +unreliable, may_retry, max_queries)` with `find_closest() -> list[bytes]` and `find_providers(on_provider, should_stop) +-> list[bytes]`. Besides completion and the deadline, a run (`_iterate`) ends once it has sent `max_queries` +(`MAX_QUERIES`, 8 × K) queries and their answers are in ("query cap"), or after `STALL_WINDOWS` (3) × max(alpha, +beta) answers in a row that leave the bit length of the beta-th closest known distance (`_reach`) unchanged ("no +progress"). A run ranks the nodes `unreliable` named when it learnt of them (`doubtful`) after every other, for its queries, its termination and `find_closest`'s result; one that distance alone would rank within the query window is asked again when `may_retry` says so, as a query beyond `alpha` (`retrying`) that the run waits for only if the node comes to rank within the window by `rank` (it is then counted in flight). `is_known(id)` @@ -318,11 +378,20 @@ every request needing the token first proves the daemon on its own connection (` which the daemon keeps open for that request only, so a client used for a long time sends nothing to a program that took the port after the daemon stopped; the token is read again when a proof with it fails (a restarted daemon's). The proof also reads the daemon's version for `check_daemon`, the one place a version -check goes. `request()` maps failures to `DaemonNotRunning`, `ConnectionFailed`, `AuthFailed` (a failed proof, -or 403) and the API's statuses (`STATUS_ERRORS`, `error_for`); `Stream` reads a body, raising +check goes. `verify_daemon` checks `X-Rnfs-Proof` before anything else of the answer: without a valid proof a +`503` is `Busy` and every other status `AuthFailed`, both in the client's own words, so a program on the port +chooses neither the error nor its text; it reads at most `PROOF_READ_LIMIT` (64 KiB), and connecting and proving +take at most `PROOF_TIMEOUT` (15 s, or the client's timeout if shorter), after which the request gets the client's +own timeout. A daemon that closes the proved connection gets the request on a new one only if the version it proved +is below 0.1.5 (`KEEPS_PROOF_OPEN_SINCE`; such daemons close every connection), else `ConnectionFailed`: a new +connection could reach another listener. That fallback goes in 0.1.7. `request()` maps failures to `DaemonNotRunning`, `ConnectionFailed` (`_exchange`), `AuthFailed` (a +failed proof, or 403) and the API's statuses (`_raise_for`: `STATUS_ERRORS`, `error_for`, the one mapping); +`Stream` reads a body, raising `ConnectionFailed` if it ends short. Its methods (`status`, `add_path`, `add_stream`, `fetch`, `cat`, `info`, -`ls`, `stat`, `pin_*`, `name_*` (with `name_watch`), `submit_job`, `job`, `jobs`, `cancel_job`, `clear_jobs`, -`deny*`) send only the parameters given; `ApiError` is an `RnfsError`, and each status's error a subclass of the +`ls`, `stat`, `dir_set`, `dir_rm`, `export`, `import_archive`, `gc`, `verify`, `pin_*` (with `pin_remote`), +`providers`, `ping`, `name_*` (with `name_watch`, `name_import`, `name_export`, `name_rm`), `submit_job`, `job`, +`jobs`, `cancel_job`, `clear_jobs`, `deny*`) cover every core route (`ApiHandler.ROUTES`; checked by +test_apiclient), so `remote.py` writes no path itself, and send only the parameters given; `ApiError` is an `RnfsError`, and each status's error a subclass of the node's own too (`NotFound` of `errors.NotFound`, `BadRequest` of `ValueError`, ...). The CLI's `ApiClient` subclasses it, turning each `ApiError` into `fail(str(e))`, so its messages are the library's. @@ -333,9 +402,9 @@ subclasses it, turning each `ApiError` into `fail(str(e))`, so its messages are |---|---| | `add_path(path, wrap=False, only_hash=False, pin=True, to=None, to_self=False, seal_name=True, chunker=None)` | `{"cid", "size", "warnings": [str]}` (+ `"sealed"`); file or directory, pinned `user` (or cached with `pin=False`) | | `add_stream(stream, size=None, name=None, only_hash=False, …)` | as above; `name` wraps it as that entry (sealed: the sealed name) | -| `get(target, output, from_node=None, raw=False)` | `{"cid", "warnings"}` (+ `"warnings_total"` past 100; + `"sealed", "name"`, the name only if `safe_default_name` accepts it); fetches and writes the file or directory under `.rnfs-get-.partial` beside `output`, then renames it to `output` (`rename_new`), decrypting a sealed file unless `raw` | +| `get(target, output, from_node=None, raw=False)` | `{"cid", "warnings"}` (+ `"warnings_total"` past 100; + `"sealed", "name"`, the name only if `safe_default_name` accepts it); fetches and writes the file or directory under `.rnfs-get-.partial` beside `output`, then renames it to `output` (`rename_new`), decrypting a sealed file unless `raw`; a directory output is first checked to still be the one `write_out` made (`walker.check_unchanged`) and, on Windows, made to inherit its folder's access list (`winacl.inherit_from_parent`; if that fails it stays private, with a warning logged) | | `info(target, for_gateway=False, from_node=None, raw=False)` | `{"cid", "type": "file"\|"directory", "size", "executable"}` (+ `"sealed", "recipient", "stored_size", "name"`; the sealed name is None unless `safe_default_name` accepts it) | -| `stored_head(target, length)` | a file's first `length` bytes if stored, else None; never fetches (the user's own gateway `HEAD` and ranges not at 0) | +| `stored_head(target, length)` | a file's first `length` bytes if stored, else None; never fetches (the user's own gateway `HEAD` and ranges not at 0); `Denied` as a read: a denied CID met, or while there are file hashes a file that fails `_check_file` (one not wholly stored gives None) | | `read(target, offset=0, length=None, for_gateway=False, from_node=None, raw=False)` | iterator of bytes of a file (range fetched only; sealed files decrypted unless `raw` or `for_gateway`) | | `read_file(target, offset=0, length=None, from_node=None, raw=False)` | `(info, chunks)`: `info()` and `read()` in one fetch, resolving the target once; `chunks` reads `[offset, offset + length)` cut to the info's size only once iterated (a directory's raises `ValueError`), and closing it ends the fetch (`GET /api/cat`, which used to call `info` and `read`: two fetches) | | `block(cid, for_gateway=False)` | the CID's own block, verified (`cat --block`, `?format=raw`: the gateway calls it only for the user's own GET) | @@ -343,10 +412,10 @@ subclasses it, turning each `ApiError` into `fail(str(e))`, so its messages are | `ls(target, for_gateway=False)` | `[{"name", "kind": "file"\|"executable"\|"directory", "size", "cid"}]` | | `stat(target)` | `{"cid", "size", "blocks", "local_blocks", "local_bytes"}` | | `is_local(target)` | `bool`: everything needed is stored, so no network fetch | -| `dir_set(target, name, child_cid, executable=False)`, `dir_rm(target)` | `{"cid"}`, pinned `user` | +| `dir_set(target, name, child_cid, executable=False)`, `dir_rm(target)` | `{"cid"}`, pinned `user`; `dir_rm` refuses a target that names no entry (`edit.check_entry_target`: ValueError, before fetching anything), so the API and `rnfs.Node` don't check it themselves | | `pin_add(target)` | `{"cid"}` once complete | | `pin_rm(cid)` | `None`; NotFound if no `user` source | -| `pin_ls(cid=None)` | `[{"cid", "sources": [str], "complete": bool, "type": "file" \| "directory" \| None, "sealed": bool \| None, "name": str \| None}]`; `type` and `name` only when the root block is stored, `name` where the pin was added from (pin_names; for a sealed add, the local path or the name given, kept for the local user), its shared-folder entry, or a directory's single entry or entry count; `sealed` from the file's first bytes (`seal.MAGIC`), False for directories, None if the root or first chunk isn't stored. Callers that show pins to others must skip `sealed` entries' names and content | +| `pin_ls(cid=None)` | `[{"cid", "sources": [str], "complete": bool, "type": "file" \| "directory" \| None, "sealed": bool \| None, "name": str \| None}]`; `type` and `name` only when the root block is stored, `name` where the pin was added from (pin_names; for a sealed add, the local path or the name given, kept for the local user), its shared-folder entry, or a directory's single entry or entry count; `sealed` from the file's first bytes (`seal.MAGIC`), False for directories, None if the root or first chunk isn't stored. A complete pin's `type`, name from inside and `sealed` are read from its blocks once, then remembered while it stays complete (`_pin_details`, `PIN_MEMO` 16 384, forgotten when released). Callers that show pins to others must skip `sealed` entries' names and content | | `pin_remote(node_id, cid, pin)` | `bool` accepted | | `gc()` / `verify()` | `{"removed_blocks", "freed_bytes"}` / `{"checked", "bad": [cid]}` | | `providers(cid)` | `[node_id bytes]` | @@ -354,7 +423,7 @@ subclasses it, turning each `ApiError` into `fail(str(e))`, so its messages are | `name_create(alias)`, `name_import(alias, key_bytes)` | `{"alias", "name"}` | | `name_rm(alias)`, `name_export(alias)` | `None`, key bytes | | `name_list()` | `[{"alias", "name", "cid" or None, "sequence"}]` | -| `name_publish(alias, target)` | `{"name", "cid", "sequence"}` | +| `name_publish(alias, target)` | `{"name", "cid", "sequence"}`, plus `"warning"` in private mode (the name's announce reaches outsiders: `names.PRIVATE_NAME_WARNING`) | | `name_resolve(name, for_gateway=False)` | `{"name", "cid", "sequence", "age"}`; NotFound if unresolved | | `export(target, output)` / `import_archive(path)` | `{"roots", "blocks"}` / `{"roots", "incomplete", "blocks"}` | | `status()` | (API: without the token, only `{"version"}`) `{"version", "node_id", "identity_hash", "dht_mode", "slow", "private", "known_nodes", "records", "pins", "provided", "blocks", "used_bytes", "max_storage", "names"}` plus every feature's `status()` fields (traffic, jobs, LXMF); `identity_hash` is the RNS identity hash (None on offline test nodes) | @@ -376,9 +445,13 @@ nodes) and re-records the node's own record, and a round sends only withdrawals each node, so a withdrawal queued while the key was briefly unprovided never follows a store of it. `protocol.provide_batches` puts withdrawals before stores, and a key in both lists is only stored. -Errors: `ValueError` → 400, `NotFound` → 404, `TooLarge` → 413, `InvalidDag` → 422, `Busy` → 503, -`StorageFull` → 507, `Denied` → 451, other `RnfsError` → 500. Paths passed to `add_path`, `get`, `export` and `import_archive` -are absolute paths on the daemon's machine (the CLI resolves them); stdin is uploaded as a request body. +Errors (`errors.STATUSES`, the one table: `api.ERROR_STATUSES` is it, `apiclient.STATUS_ERRORS` and the gateway's +`ERROR_MESSAGES` follow it): `ValueError` → 400 (with `BlockMismatch`, `SealError` and `NotRecipient`, which +subclass it), `NotFound` → 404, `TooLarge` → 413, `InvalidDag` → 422, `Busy` → 503, `StorageFull` → 507, `Denied` +→ 451, other `RnfsError` → 500. The API logs a 500 as an error and a `BlockMismatch` (a block the node itself +handed the store not matching its CID: a bug, not a bad request) as a warning. Paths passed to `add_path`, `get`, +`export` and `import_archive` are absolute paths on the daemon's machine (the CLI resolves them); stdin is uploaded +as a request body. **Core API routes** (`rnfs/api.py` `ROUTES`): `GET /api/status` (open), `/api/info`, `/api/cat`, `/api/ls`, `/api/stat`, `/api/pin/ls`, `/api/providers`, `/api/ping`, `/api/name/list`, `/api/name/resolve`, @@ -396,18 +469,34 @@ prints the version and says why). The daemon writes a new random token to `/daemon.lock` (`fsutil.lock_exclusive`: `flock`, or `msvcrt.locking` of a byte past the text on Windows; the file holds its process ID) and keeps the lock until it exits, so a second daemon -on the same dir stops at once, naming the first (`lock_holder`). `_make_private_dirs` refuses to start (RnfsError) if the rnfs dir -belongs to another user or can't be made 0700. On Windows (`rnfs/winacl.py`), mode bits set no ACL, so a new +on the same dir stops at once, naming the first (`lock_holder`). A start runs in this order, in the daemon +(`cli.run_daemon`) and in `rnfs.Node` alike: the root (`node.make_private_root`), then the settings +(`Config.load`), then the subdirectories and the lock (`lock_rnfs_dir`: `_make_private_dirs(root)`, then +`lock_exclusive`), so each step relies only on what an earlier one secured. `make_private_root` refuses to start +(RnfsError) if the rnfs dir belongs to another user or can't be made 0700. `_make_private_dirs` (audit API-1) +opens each of `node.PRIVATE_SUBDIRS` (`state`, `blocks`, `storage`, `names`, `lxmf`) with `O_DIRECTORY | +O_NOFOLLOW`, creating it 0700 if missing: a symlink, a non-directory or one another user owns is refused with an +RnfsError naming `chown`, before Reticulum starts; a loose one is `fchmod`ed to 0700 with a notice, and if its old +mode let others write, one `lstat` pass (`_warn_foreign_entries`) warns about entries another user owns (a +planted raw block would be trusted by its length). `rnfs.toml`, and RNS's `config` when `reticulum_dir` is the +rnfs dir, are opened by `config.open_checked` (`O_NOFOLLOW | O_NONBLOCK`, so a symlink is refused and a FIFO +never waited on) and checked through the descriptor: a regular file, the user's own, and if group or other bits +are set `fchmod`ed to 0600 (`fsutil.check_owned_fd`, which the identity's check shares); at most +`MAX_SETTINGS_SIZE` (1 MiB) is read. What was changed goes to `config.warnings` (a plain attribute, not a field, +so `rnfs.toml` can't set it), never to `rnfs.log`, which imports RNS: the daemon prints them (`cli.warn`), +`rnfs.Node` logs them, the CLI client prints them and `rnfs.Client` (`apiclient.api_port`) raises them as +`RuntimeWarning`s. On Windows (`rnfs/winacl.py`), mode bits set no ACL, so a new rnfs dir and its subdirectories are created with a protected DACL allowing only the user and SYSTEM (inherited below it) in the CreateDirectoryW call itself (`winacl.make_private_dir`, a self-relative descriptor built by `private_descriptor`), so the parent's DACL never applies to them; a new root that already holds anything, and -a root that is a junction or symlink (`fsutil.is_link` on its lstat), are refused. An existing one that allows -anyone else is given the DACL, and one owned by another account is refused; the token, identity and name key +a root that is a junction or symlink (`fsutil.is_link` on its lstat), are refused. An existing root or +subdirectory that allows anyone else is given the DACL (`winacl.tighten(d, directory=True)`), one owned by +another account is refused, and so is a subdirectory that is a junction or symlink; the token, identity and name key files get the same DACL before they are renamed into place (`fsutil.write_atomic(private=True)`), and -`read_token` checks the opened token's DACL (Administrators tolerated). `rnfs.toml` (`Config.load`, -`winacl.secure_settings`) and the identity (`network/identity.py`) are checked through the handle they are -read from (`winacl.tighten(fd=...)`): another account's is refused, a loose one tightened with a warning, and -a failure is fatal. A `--dir` outside `%USERPROFILE%` is warned about. The advapi32 and kernel32 calls are +`read_token` checks the opened token's DACL (Administrators tolerated). `rnfs.toml` (`Config.load`) and the +identity (`network/identity.py`) are checked through the handle they are read from (`fsutil.check_owned_fd`, +which calls `winacl.tighten(fd=...)` there): another account's is refused, a loose one tightened with a warning, +and a failure is fatal. A `--dir` outside `%USERPROFILE%` is warned about. The advapi32 and kernel32 calls are behind `winacl.backend`, which tests replace with a fake. A response to a request with `X-Rnfs-Challenge: <32 hex>` carries `X-Rnfs-Proof: apiclient.api_proof(token, port, challenge)` (HMAC-SHA256 of `rnfs api proof `); `apiclient.ApiConnection.verify_daemon` checks it with `GET /api/status` on the connection @@ -415,8 +504,9 @@ that then carries the token: `ApiHandler.dispatch` keeps one open after a proof `keep-alive` (`keep_open`, once per connection: `proved`), giving the next request a new request deadline. Both HTTP servers (`TimedHandler`) time out slow requests, cap concurrent connections (API 32, gateway 64) and refuse HTTP/0.9 with a 400 carrying the usual headers (both add them in `end_headers`, so http.server's own error pages, 400, 414, 431 and 501, carry them too). Gateway errors raised by the node -(anything but `GatewayError` and `NotFound`: 400, 413, 422, 503, 507, 500) say only a fixed message per status -(`gateway.ERROR_MESSAGES`; the detail goes to the log); malformed `Range` headers are ignored. +(anything but `GatewayError` and `NotFound`: 400, 413, 422, 451, 503, 507, 500) say only a fixed message per status +(`gateway.ERROR_MESSAGES`, which has one for every status of `errors.STATUSES`; the detail goes to the log); +malformed `Range` headers are ignored. Neither server sends CORS headers. Gateway pages run sandboxed in an opaque origin, so they load their DAG's files as subresources but can't read any gateway response with `fetch()`; an earlier rule allowing a gateway @@ -477,7 +567,8 @@ and errors import neither RNS nor anything that does (tested, as the CLI's start `Node(rnfs_dir, *, reticulum=None, settings=None)` - `rnfs_dir` (str or path) is required: the app's own node directory (identity, blocks, state, optional - `rnfs.toml`), never guessed. Created private as the daemon's is (`_make_private_dirs`). + `rnfs.toml`), never guessed. Created private as the daemon's is, in the same order (`make_private_root`, + `Config.load`, whose `warnings` it logs, then `lock_rnfs_dir`). - `reticulum`: the running `RNS.Reticulum`, which must be `RNS.Reticulum.get_instance()` (RNS allows one per process; anything else is ValueError). None: the running instance if there is one, else the node starts one from the `reticulum_dir` setting, as the daemon does (so it attaches to `rnsd` if one runs). The node never @@ -503,7 +594,8 @@ and errors import neither RNS nor anything that does (tested, as the CLI's start `Client(rnfs_dir=None, *, port=None, token_file=None, timeout=None)`: as `apiclient.ApiConnection` (default `~/.rnfs/`, the `api_port` in its `rnfs.toml`, the token from `/api_token`, proved before it is sent). -Nothing is sent until the first call. Raises `SettingsError` if `rnfs.toml` can't be read. +Nothing is sent until the first call. Raises `SettingsError` if `rnfs.toml` can't be read or trusted; one made +private on the way is reported as a `RuntimeWarning` (`warnings.warn`), without loading RNS. Both: `close()` (idempotent; both end their event and job-watch threads, then a Node stops its features and threads, saves, closes the state database, releases its rnfs.dht destination and the lock; a Client's name @@ -581,8 +673,13 @@ Client the connection errors below). "Fetch errors" = `NotFound` (no such conten `read()` of the rest is one read. Reads go by the target as opened, root and path, since providers announce roots and the file's own CID may have none; an `rnfs:name/` URI is resolved once, at `open`, and a contact nick is expanded by the node (which reuses its -CID for 60 s). It is an `io.BufferedReader` of 64 KiB over `_FileReader`, whose every `readinto` is one `read` -(or one `GET /api/cat`) of `[position, position + its length)`, so only the chunks covering it are fetched; a sealed file reads its plaintext unless `raw`. Raises fetch errors as reads happen. Not +CID for 60 s). It is an `io.BufferedReader` of 64 KiB (`FILE_BUFFER`) over `_FileReader`, whose refill after +`open` or a seek is one `read` (or one `GET /api/cat`) of `[position, position + its length)`, so only the chunks +covering it are fetched. While reads are sequential (a refill starting where the last read ended), each refill +reads ahead twice as far as the last, up to `READ_AHEAD` (4 MiB), and keeps the rest for the next `readinto`s: a +sequential reader makes few requests and fetches beyond what it reads at most about what it has read already. A +seek that moves the position drops what was read ahead and starts again at 64 KiB. A sealed file reads its +plaintext unless `raw`. Raises fetch errors as reads happen. Not thread-safe, like any file object. `Job`, `Name` and every object are snapshots as of their last `refresh()`, `wait()` or callback. @@ -595,7 +692,9 @@ before submitting, as the CLI resolves a nick. Jobs are persisted, resumed at th LXMF). Not in the typed interface for 0.2: keep/follow, contacts, peers/stats, LXMF. **Errors.** One hierarchy in `rnfs.errors`, every class importable as `rnfs.`: `RnfsError` ⊃ -`NotFound`, `TooLarge`, `InvalidDag`, `StorageFull`, `Busy`, `Denied`, `Cancelled`, `Closed`, `ApiError`. +`NotFound`, `TooLarge`, `InvalidDag`, `StorageFull`, `Busy`, `Denied`, `Cancelled`, `Closed`, `ApiError`, and +`BlockMismatch` and `SealError` (a sealed file malformed or tampered with), which are `ValueError`s too, as is +`NotRecipient` (this node can't open a sealed file). `ApiError` (with `.status`) ⊃ the Client's own: `DaemonNotRunning`, `ConnectionFailed`, `TokenError`, `AuthFailed`, `SettingsError`, `ServerError` (500, or another 5xx), and the API statuses, each also a subclass of the node's class: `apiclient.NotFound(ApiError, errors.NotFound)` and so on, `BadRequest(ApiError, @@ -644,7 +743,7 @@ Rules, the same for both: |---|---| | A Node's events come from in-process hooks; a Client's from polling (jobs, 1 s) and the long poll (names) | Documented; the same callbacks and rules. A daemon older than the endpoint can't watch names (`NotFound` at `watch()`) | | Paths (`add`, `get`, `export`, `import_archive`, a job's output) are opened by the daemon for a Client | Same machine and user (the token requires it); made absolute on the app's side, as now | -| `open()` costs one API request per buffer refill for a Client | Accepted; the 64 KiB buffer bounds the requests | +| `open()` costs one API request per buffer refill for a Client | Accepted; the 64 KiB buffer bounds the requests, and sequential reads read ahead up to 4 MiB a request | | Settings, DHT mode, LXMF, lifecycle: a Node's own; the daemon's for a Client | Documented: a Client can't change them; `close()` only ends the Client | | No gateway or API in a Node | Not offered in 0.2 (decision 6) | | The API's `GET /api/info` has no `raw` | A Client's `info(raw=True)` (and `open(raw=True)`) takes a sealed file's `stored_size` as its size | @@ -712,9 +811,12 @@ in these, each tested (tests/test_embedded.py: the guards in both scopes, and a ## Content additions -- `content/chunker.py` — `chunks(stream, size)` yields chunks from the chunker chosen with the context - manager `chunker.using(name)` (a ContextVar, so the shared-folder scan on its own thread keeps fixed - chunks); `builder.build_file` takes its chunks from it. FastCDC computes gear hashes for 8 KiB at a time on +- `content/chunker.py` — `fixed_chunks` and `fastcdc_stream` cut a stream into chunks. Which one a build uses + is `builder.BuildOptions.chunker`, frozen with the deny check (`check`) and passed explicitly down through + `add_path`, `add_stream`, `build_open`, `build_file` and `SharedFolder.scan`: a ContextVar isn't copied to a + worker thread, so a build moved onto one would silently get other CIDs and skip the check. `chunker.using(name)` + and `builder.checking(check)` remain as wrappers setting them for the current context, which a build given no + options takes (`builder.current_options()`). FastCDC computes gear hashes for 8 KiB at a time on a big integer of 9-byte lanes (≈ 39 MB/s in CPython 3.12, against ≈ 15 MB/s for the byte-by-byte reference `cut_point_simple` it is tested against, and ≈ 2 GB/s for fixed chunking). - `content/seal.py` — the sealed-file format: `SealingReader` (file-like, one segment in memory), @@ -735,15 +837,24 @@ in these, each tested (tests/test_embedded.py: the guards in both scopes, and a creating the output, so nothing raised there leaves a file. `open_sealed` reports the stored (ciphertext) size as a job's total, which is what its done bytes count. - Cost: `get`, `cat` and `info` of a file read its first 12 bytes to detect sealing (`--raw` skips this), so - the first chunk is fetched first even for a `cat --offset` deep in a file. The gateway does so only for a + the first chunk is fetched first even for a `cat --offset` deep in a file. `info` and `read` (so `cat`) remember + the targets found not to be sealed (`RnfsNode._open_sealed`: by resolved expectation, `NOT_SEALED_MEMO` 4096, + least recently used forgotten first; emptied when the deny list's CIDs change, since that read meets them), so + reading one again doesn't read its first bytes; `read_file` (`GET /api/cat`) hands the `SealedFile` its info + opened to its read rather than opening it twice. The gateway does so only for a GET from byte 0 or a web page's request (`limited()`, so its type never depends on what is stored); the user's own HEAD, or range not at 0, uses the first bytes only if already stored (`stored_head`), else takes - the Content-Type from the name, or `application/octet-stream`. + the Content-Type from the name, or `application/octet-stream`. `stored_head` applies the deny list as a read + does (`node_content.denying` wraps its getter; with file hashes, `_check_file` over stored blocks once the file + is wholly stored), so such a HEAD of a denied file answers 451 like GET; `pin ls` (`_pin_sealed`) passes + `deny=False`. ## Mesh `mesh.py` (no RNS at module level): `PathInfo`, `provider_rank` and `RnsPaths` (hops and next-hop -interfaces from `RNS.Transport`, or through a shared instance's RPC with its interface stats cached for 30 s); +interfaces from `RNS.Transport`, or through a shared instance's RPC with its interface stats cached for 30 s +and each node's next-hop interface name for 15 s, `PATH_CACHE`, in an LRU of at most 1024 nodes; `infos()` fills +the names of many nodes from one `get_path_table` RPC, for `rnfs peers`); the airtime policy (`TokenBucket`, `AirtimePolicy`, interface matching, quiet-hours parsing, all with injected clocks); and `Traffic`, thread-safe counters per peer (LRU, at most 1000), per interface (at most 64) and in total, never persisted. @@ -768,12 +879,25 @@ and in total, never persisted. has passed, or that makes no progress for `stall_timeout`. `Cooldowns` then refuses `/blocks` from a link (and its identity, if identified) whose response was displaced or cancelled idle, for `blocks_cooldown`, cheaply and before any slot is taken. `_concluded` starts the same - cooldown for a response whose Resource the requester rejected (`REJECTED`) with nothing sent (`_sent_any`: - `Response.sent`, else the Resource's `get_progress()`). + cooldown for a response whose Resource the requester rejected or cancelled (both `REJECTED` on RNS 1.5.6) + before `NO_PROGRESS_FRACTION` (a quarter) of it was sent (`_made_progress`: the more of `Response.sent` and + the Resource's `get_progress()` × size; progress counts if either can't be read). A `FAILED` response (the + link closed, or RNS gave up) starts none, so a reader that gives up early must close its links and never + call `Resource.cancel()` on a response, or it would cool itself down at its providers. `Cooldowns` is flat: + every cooldown lasts `blocks_cooldown`, per link and per identity. The `Response` docstring records which + thread or lock owns each of its fields. A new `/blocks` arriving while the link's previous response has + sent all its parts waits, on `RequestServer.clock`, up to `_proof_wait(link)` (2 × RTT + 1 s, at most + `PROOF_GRACE`, 5 s) for its proof (`_previous_done`). That is the server's only use of `link.rtt`, which on an + incoming link is what the requester claims: anything but a finite, non-negative number counts as no RTT, and + any future use must go through it, capped. - `IncomingLinks.take_token` keeps a per-link token bucket (`REQUEST_BURST`, `REQUEST_RATE`) and a smaller one of refusals (`OVER_RATE_REFUSALS`, one a second) in `_LinkState`. `RequestServer._guarded` asks it first for every path: over the limit it answers `False` while refusals are left, else `None`, which RNS sends - nothing for; `_over_rate` counts them and logs at most once a minute. + nothing for; `_over_rate` counts them and logs at most once a minute. Every handler but `/blocks` runs its + work through `RequestServer._in_slot(link_id, answer)` after its own checks (so a refusal by those takes no + slot): it takes one of the link's `MAX_REQUESTS_PER_LINK` slots (`IncomingLinks.start_request`), answers + `False` if none is free, and frees it when `answer()` returns or raises. A `/blocks` slot outlives its handler + (until the response's Resource concludes), so `_handle_blocks` keeps its own. - `RequestServer.airtime` checks interface rules, quiet hours and an empty budget before taking any slot, caps the block budget, and charges the bucket as responses are sent. Each `Response` records its `cost` (packed size), what has been `charged` so far and its requester (identity hash, else link ID). `_admit` @@ -803,7 +927,8 @@ and in total, never persisted. `MAX_BLOCKS_CIDS` (26) entries of `bin` up to `MAX_BLOCK_SIZE` or `nil`, and nothing after it. A refused response is handed to `Link.handle_response` as `None`, which every parser refuses, so the request fails as malformed; refusals are logged at most once a minute. `guarded_receive` drops a response packet on an - incoming `rnfs.dht` link without decrypting it. Everything else goes to RNS's own methods unchanged. + incoming `rnfs.dht` link, and a request packet on one this node opened (`link.initiator`), without + decrypting it. Everything else goes to RNS's own methods unchanged. - `guards.limit_half_open(destination, limit)` replaces a destination's `incoming_link_request` (on the instance; RNS's `Destination.receive` calls it for each link request) with `HalfOpenLimit`, which, before RNS validates a request, closes (`Link.teardown()`) the oldest `PENDING`/`HANDSHAKE` link in @@ -814,22 +939,47 @@ and in total, never persisted. `rnfs/features/__init__.py` defines `Feature` and `FEATURE_MODULES` (in start order: peers, jobs, keepers, follow, contacts, lxmf, deny). The node builds every feature in its constructor (`node.feature(name)`), starts them after -its own threads and stops them, in reverse, before its own services. A feature may override: `start()`, `stop()`, +its own threads and stops them, in reverse, before its own services. A feature may override: `start()`, +`stop(deadline=None)` (asks its threads to end, then waits for them until `deadline`, a `time.monotonic()` value: +the node passes its stop budget's, so `STOP_TIMEOUT`, 5 s, bounds the features' threads and its own together; +`None` waits `features.STOP_WAIT`; the base's `_spawn(target, name)` starts a thread that `_join(deadline)` waits +for; the node calls a `stop()` with no `deadline` parameter as before, `features.stop_feature`), `save()` (called with the node's periodic saves), `close()` (releases the state database at shutdown, after `stop()` and the last save), `status()` (fields merged into `rnfs status`), `requests()` (new `rnfs.dht` paths; the request server applies private mode and the per-link limit, and turns `None` or an exception into `False`; a response must be a shape `protocol.response_refusal` accepts, True/False/None, a `/find` answer, or at most `MAX_BLOCKS_CIDS` bins -or nils, since nodes since 0.1.4 refuse any other), `api_routes()` (all token-protected) and `cli_commands()`. +or nils, since nodes since 0.1.4 refuse any other: the server checks each with `protocol.body_refusal` and answers +`False`, logged as an error, instead of one they'd refuse), `api_routes()` (all token-protected) and `cli_commands()`. Feature modules don't import RNS at module level (the CLI imports them to build its parser) and their constructors work on `RnfsNode.offline` test nodes. +A feature declares the features it uses: `requires` (it can't work without them; each must come before it in +`FEATURE_MODULES`) and `optional` (used when loaded). The node loads them through `features.checked_classes()`, +which raises `features.FeatureError` before making any if a required feature is missing or later, or two features +share a name; a feature reaches another only with `self.dependency(name)`, which raises `FeatureError` for a name it +didn't declare and returns `None` for an optional one not loaded. Today follow requires keepers; jobs, keepers, +follow and lxmf use contacts, contacts uses keepers and lxmf uses jobs, all optionally. What the others call on +contacts is the `features.ContactsApi` Protocol (`lookup`, `has`, `nick_of`, `resolve_id`, `expand_target`), which +types `node.feature("contacts")`, `self.dependency("contacts")` and `features.find(node, "contacts")`; +`tests/test_contacts.py` checks `ContactsFeature` against it, so a renamed method fails there instead of silently +turning off LXMF's contact trust. LXMF resolves a recipient's nick with `ContactsFeature.resolve_id(nick, "lxmf")`. + +The node checks the features' names too when it loads them, with one error style (`FeatureError`, "feature +: "), so nothing is shadowed or dropped silently: `checked_classes()` checks every API route +(`features.api_routes()`: GET or POST of a path matching `API_PATH`, under `/api/`, with a handler, defined neither +by `ApiHandler.ROUTES` nor by another feature), and once the features are made, `features.request_handlers()` +checks their request paths (`/` and a name, `REQUEST_PATH`, with a handler, neither one of rnfs's own, +`CORE_REQUEST_PATHS`, nor another feature's). `NodeBackend.feature_requests()` hands the request server the paths +checked then. A feature's constructor raising stops the node from being made (those made before it are closed); +its `start()` raising is logged and the others still start (`RnfsNode._start_features`). + | Feature | Module | What it does | Hooks in shared code | |---|---|---|---| -| `peers` | `peers.py` | `rnfs peers`, `rnfs stats`; its own `rnfs.dht` announce handler, so client-mode nodes' mode and last-heard time are known | reads `Traffic`, the routing table, links, paths | +| `peers` | `peers.py` | `rnfs peers`, `rnfs stats`; client-mode nodes' mode and last-heard time come from the node's `rnfs.dht` announce handler, which counts every announce in `Traffic` | reads `Traffic`, the routing table, links, paths | | `jobs` | `jobs.py` | `JobsFeature` (the job manager), `Job`, CLI progress formatting | `fetcher.observing(observer)`: a thread-local observer every `Fetch` picks up (`total`, `block`, `cancelled` checked at each step, raising `fetcher.Cancelled`) | | `keepers` | `keepers.py` | signed records (`SignedRecord`, `parse_signed`, `sign_record`, `from_announce_data`), `/name`, `KeeperClient`, `KeeperStore` | `Feature.requests`; `publisher.DhtKey` and `Publisher.add_key_source` (non-CID keys published like CIDs); `NameManager.keepers` (`published`, `heard`, `query`, `always_ask`, `local_record`), `NameManager.offer()`/`record()`; the 5-argument announce handler (packet hash) | | `follow` | `follow.py` | followed names: each check and fetch one at a time, `fetcher.open(pinned=True)`, pin inside the fetch's protection, old pin released after | keepers' `FOLLOW` reason | -| `contacts` | `contacts.py` | address book; `expand_target`, `lookup(nick, kind)`, `node_arg` | `RnfsNode._parse_target` (never for the gateway), `JobsFeature.submit` | +| `contacts` | `contacts.py` | address book; `expand_target`, `lookup(nick, kind)`, `resolve_id(text, kind)`, `node_arg` | `RnfsNode._parse_target` (never for the gateway), `JobsFeature.submit` | | `lxmf` | `lxmf.py` | pure parts (`Link`, `parse_uri`, `format_uri`, `find_links`, `encode_field`/`decode_field`); `MessageLog` (the inbox and outbox, in the state database); `LxmfFeature` (send with direct→propagated fallback, inbox, auto-fetch, inbox get, pin release); `LxmfNetwork`, made in `start()`, holds every RNS/LXMF object | `lxmf.validate_settings` from `Config.validate`; `LXMF_PREFIX` in `storage.check_source` | | `deny` | `deny.py` | `deny_lists` sources (`parse_source`, `validate_settings` from `Config.validate`): `check(source)` resolves a name (persisted), fetches the target pinned, checks it is a file within `MAX_LIST_BYTES`, parses it, pins it `deny` and `accept`s it, then releases unused `deny` pins; `run_due` on `rnfs-deny`, after the join (`CHECK_INTERVAL` for names, `retry_delay` backoff); sources no longer configured are forgotten at start | `node.deny` (core); `DENY` in `storage.check_source` | @@ -853,7 +1003,12 @@ retried first. A held message is dropped 30 s (`HOLD_PATH_WAIT`) after its sende arrival if later, with no identity known, and after 2 minutes in any case. A held message stays in the queue while it is retried, so it still counts as a duplicate and against the limit. The inbox (`MessageLog`) holds at most 500 records and 4 MiB as JSON, 50 per sender not in `lxmf_auto_fetch_from`, 16 KiB per record (links -beyond it dropped, then text cut), and never evicts a record holding auto-fetched pins for one that holds none; +beyond it dropped, then text cut), and never evicts a record holding auto-fetched pins for one that holds none, +nor for another sender's (when all others hold pins, the new record's sender's own goes, or it is refused). An +auto-fetch sender's record is added with `share=` 1/n of the record and byte limits (`_auto_share`, n senders in +`lxmf_auto_fetch_from`, at least one record and 16 KiB), counting its protected records; past it, `_own_victim` +drops its own unprotected records first, then those whose holdings (`holds=_held`: pinned CIDs, queued links) +another of its records also holds, then the oldest; `add(reserved=)` keeps trusted senders' records from being evicted except for each other's (untrusted ones go first), so free identities can't push them out, and untrusted senders' new records come from a per-sender allowance (`SENDER_BURST` 4, then one per `SENDER_INTERVAL` 15 min; `_sender_tokens`, the last @@ -868,7 +1023,12 @@ busiest one's newest place (that link is marked skipped). Links without the fiel `MAX_SCAN_CHARS` (16 384) characters of the content. `inbox get` (through the jobs feature) checks every output before starting any (two links with one output are refused), then submits a get job per link with its `from` and `fallback_dht`; once one has started, a later link's failure is reported in its result (`error`) rather -than raised. An output without `-o` is named with `fsutil.safe_default_name`. Sent links carry `from` only with +than raised. Names are resolved while planning: `_resolve(link, persist, allow_sent_cid)` returns +`(cid, names, fell_back)` and raises NotFound (naming `--use-sent-cid`) for an unresolvable name unless +`allow_sent_cid`, so without `use_sent_cid` nothing is written and no job submitted; with it, a link that fell +back to its sent CID gets a `warnings` entry in its result. Auto-fetch passes `allow_sent_cid=True` (its senders +are trusted) and records `"sent_cid": true` with such a pin; `_print_links` shows it, and a name link's sent CID. +An output without `-o` is named with `fsutil.safe_default_name`. Sent links carry `from` only with `lxmf_send_from_hint`. The inbox and outbox are rows of the `lxmf_messages` table, saved in batches (NORMAL). **Feature API routes** (all need the token): `POST /api/jobs`, `GET /api/jobs`, `GET /api/jobs/get?id=&wait=`, @@ -882,8 +1042,13 @@ takes, for a get, the JSON booleans `sealed_name` (write a sealed file this node name in the output's directory; the CLI sets it for `rnfs get ` without `-o`, so it no longer calls `GET /api/info` first) and `fallback_dht` (a get with `from` that fails, other than by cancellation, `TooLarge` or `StorageFull`, is retried once through the DHT). `POST /api/lxmf/send` and `/api/lxmf/sync` take -`nearest`; `GET /api/lxmf/address` returns `propagation_node` (the configured one only) and -`nearest_propagation_node`. Core routes gained options: `POST /api/add` `pin`, `to`, `to_self`, `seal_name`, +`nearest`; `POST /api/lxmf/inbox/get` takes `use_sent_cid` (without it, an unresolvable name link is 404); +`GET /api/lxmf/address` returns `propagation_node` (the configured one only) and `nearest_propagation_node`. +`GET /api/lxmf/inbox` adds to each record, computed when listed, `trusted` (`LxmfFeature._trusted_hex`) and +`contact` (`ContactsFeature.nick_of(address, "lxmf")`, the first nick naming it, or null); `rnfs inbox` prints the +address, then `contact `, `trusted` or `untrusted` (`?` without the fields), the method, and last the +display name quoted and escaped with `fsutil.printable_label`. +Core routes gained options: `POST /api/add` `pin`, `to`, `to_self`, `seal_name`, `chunker` (and the same as query parameters on `/api/add/stream`); `POST /api/get` `raw`; `GET /api/cat` `raw=1`, `block=1`; `GET /api/info` `from`. @@ -895,15 +1060,18 @@ name in the output's directory; the CLI sets it for `rnfs get ` withou | `rnfs-announcer`, `rnfs-response-monitor`, `rnfs-link-sweeper` | network | rnfs.dht announces, `/blocks` deadlines and displacement, idle links | | `rnfs-lookup`, `rnfs-fetch`, `rnfs-find-providers`, `rnfs-remote-pin`, `rnfs-publish-lookup` | per operation | lookups, fetch workers, provider search, remote-pin fetches (one thread per source with pins waiting), a publishing round's lookups (up to 4 at once, ended with the round; none on slow nodes) | | `rnfs-write` | storage | an add's, shared-folder import's or archive import's block writes (`Session.writer()`), up to 16 at once, ended with it | -| `rnfs-job-` | jobs | one per running job, at most `max_jobs`; `stop()` waits ≤ 2 s | +| `rnfs-job-` | jobs | one per running job, at most `max_jobs`; `stop()` waits for them until the node's stop deadline | | `rnfs-keepers`, `rnfs-keeper-query` | keepers | pending refreshes (≤ 64 queued), own-name sync, saves, re-asking every `announce_interval`; parallel `/name` queries | | `rnfs-follow` | follow | every check and fetch, one at a time | | `rnfs-deny` | deny | every deny-list source's check and fetch, one at a time, once joined | | `rnfs-events`, `rnfs-job-watch` | rnfs.Node | callbacks into the program, one at a time; polling watched jobs (both started on first use) | | `rnfs-client-events`, `rnfs-client-job-watch`, `rnfs-client-name-watch` | rnfs.Client (in the program) | callbacks; polling watched jobs; the `GET /api/name/watch` long poll | -| `rnfs-lxmf`, `rnfs-lxmf-fetch` | lxmf | announces, propagation-node downloads, polling outbound states each second; auto-fetch, one at a time, senders in turn (LXMF runs its own job thread) | +| `rnfs-lxmf`, `rnfs-lxmf-fetch` | lxmf | announces, propagation-node downloads, polling outbound states each second while a message is followed or held or a download runs (otherwise it sleeps until the next announce or download; `LxmfNetwork.send`, `_hold` and `sync` wake it); auto-fetch, one at a time, senders in turn (LXMF runs its own job thread) | -The API and gateway run one thread per connection, capped (32 and 64). +The API and gateway run one thread per connection, capped (32 and 64). The daemon's main thread waits for +SIGINT, SIGTERM or SIGBREAK (`cli.wait_until_stopping`): on POSIX a signal interrupts the wait, so it has no +timeout and wakes only for the debug log's status line; on Windows a wait isn't interrupted for signal handlers, +so it wakes each second. Locks are taken in this order, never the reverse: `names._lock` → `publisher._lock` → `node._lock` → `storage._lock` → the state database's lock (`statedb`, innermost: a store opens a transaction only once it holds every lock the @@ -929,7 +1097,10 @@ runs while a lock is held: deny list's CID set, the key sources' keys or the strategy. A refresh that changed none of them returns at once (about 1 µs, from 7 ms at 10 000 pins); the generation is read before computing, so a change made meanwhile makes the next refresh compute. When it does compute, it builds the set of blocks served again only when a - provided CID went or its DAG's blocks changed (CIDs only added are unioned in). + provided CID went or its DAG's blocks changed (CIDs only added are unioned in). It takes the pins as + `Storage.pinned_cids()`, not a copy of every pin's sources, and gathers the keys withdrawals are queued for once, + so an item newly provided searches the queues only when its key is among them (a mass re-pin, or the first + refresh after a start with withdrawals pending). - `Storage.evict` and `gc` decide what to delete, and delete it, under the storage lock (after reading the DAG nodes outside it); `verify` checks completeness and re-reads the blocks outside it. - Every pin and unpin commits its row durably (`synchronous=FULL`, after the blocks are synced) under the @@ -941,7 +1112,9 @@ runs while a lock is held: change returns, "NORMAL" ones are batched saves a power loss may lose the last of: - storage: `pins` (one row per source; each pin and release FULL, after the blocks are synced) and `cache` (cached fetches, LRU order in `used`, at most 10 000; NORMAL every 30 s and at stop, changed records only); -- `pin_names` (rnfs/pin_names.py: where each pin was added from, local only, pruned to the pin list; FULL); +- `pin_names` (rnfs/pin_names.py: where each pin was added from, local only, pruned to the pin list; NORMAL, + each change written at once, after the pin it names was committed FULL: a power loss can lose a name, never the + pin, and an add costs one durable commit, not two); - node: `fetched` (targets fetched for the user, for `roots`; with the cache records) and `remote_pins` (remote pins accepted and not yet fetched, in order; about once a second and at stop, rewritten whole); fetcher: `invalid` (invalid roots, oldest first; with the cache records); @@ -954,7 +1127,11 @@ change returns, "NORMAL" ones are batched saves a power loss may lose the last o `name_floor` (highest sequence of names resolved for the gateway or auto-fetch and when, in `used` order, at most 4 × `name_table_size`, or 8 × for names resolved within 7 days; NORMAL, saved by a resolve or record raising it, at most every 5 s, and by the re-announce loop when changed); -- `shared_folder` (FULL); `jobs` (record per job in creation order; FULL, including progress saves; results keep +- `shared_folder` (FULL) and `shared_folder_files` (the shared folder's per-file cache, `FolderFiles`: one row per + file per top-level entry, the `dump()`ed entry name and `[device, inode, size, mtime_ns, execute bit]` as text, + the CID, its height and the content SHA-256 if one was computed; a `RowStore` in SNAPSHOT mode, NORMAL with the + periodic saves and at stop, changed rows only: a power loss costs at most reading some files again); +- `jobs` (record per job in creation order; FULL, including progress saves; results keep at most 100 warnings plus `warnings_total`, older results capped at load); `keepers` (records re-verified at load; NORMAL); `follow` (FULL); `contacts` (FULL); `lxmf_messages` (inbox: 500 messages, 4 MiB, 50 per non-auto-fetch sender, 16 KiB per record; outbox 200, unfinished messages marked failed at a restart; NORMAL, @@ -966,6 +1143,14 @@ change returns, "NORMAL" ones are batched saves a power loss may lose the last o - `meta` (per-store values: follow's and each box's `clean`, set by every save, which the reconcile at start requires; each box's `next_id`) and `migrated` (JSON files imported). +The node lists its stores as `_setup` makes them (`RnfsNode._add_store`; the `node.Store` protocol: `save()` +writes what changed since the last save, raising `OSError` and keeping the change if the database fails, and +`close()` releases the store's reference to the state database). A store is batched (`_save_state`: every 30 s +and at gc) or saves on a schedule of its own (provider records, the publisher, the remote pins waiting, the +write-through stores); the features are stores too, batched. Stop saves every store in the order made, then +`_close_state` closes them in reverse, the node's own reference last; after a `_setup` that failed, only those it +made. A new store needs only its `_add_store` call. + Free text that may hold lone surrogates (file names, LXMF records, cache paths) is stored as JSON text (`statedb.dump`). Rows that fail the checks the JSON loaders applied are dropped at load with a warning and deleted. The JSON files of 0.1.4 and earlier, once imported, stay as `.migrated`. Elsewhere: @@ -973,7 +1158,19 @@ deleted. The JSON files of 0.1.4 and earlier, once imported, stay as `.mig delivered-message IDs, stamp costs), `api_token` (while the daemon runs), `daemon.lock` (locked while the daemon, or an embedded `rnfs.Node`, uses the dir: `node.lock_rnfs_dir`). Outputs being written live beside the output as `.rnfs-get-.partial` (`RnfsNode.get`) or `.rnfs-job-.partial` (get jobs) until renamed; a failed, cancelled or shut-down job removes its partial, -and a resumed job removes one left by a crash. +and a resumed job removes one left by a crash. A get job runs `RnfsNode.get` with its `.rnfs-job-.partial` as +the output, so it is written in a `.rnfs-get-.partial` beside that and renamed twice. +A directory partial is private while written (audit CONT-2), so nobody can swap one of its directories for a +link and have the DAG's files created elsewhere: on POSIX (`walker._FdOutput`, where `FD_WRITE`) it is made with +`mkdir`, opened `O_DIRECTORY | O_NOFOLLOW`, checked to be this user's, `fchmod`ed to 0700 (setgid kept, so +entries still get a shared directory's group) and must then be empty; entries are created relative to their +parent's descriptor (`mkdir(dir_fd=)`, files `O_CREAT | O_EXCL | O_NOFOLLOW`), and a directory is reopened from +the root, one component at a time, checked against the device and inode recorded when it was made. The stack +of directories still to write holds those names and identities, not descriptors, so only a few are open +however many directories a DAG has. The root gets its original mode back when complete. On Windows +(`walker._PathOutput`) the root is made with `winacl.make_private_dir`, entries by path, and each file's final +path must be inside the root; `RnfsNode.get` then restores inheritance (`winacl.inherit_from_parent`) before +the rename. ## Tests @@ -984,7 +1181,10 @@ feature. The public API has `test_public.py` (every interface test run on an emb ApiServer, the name-watch endpoint, the objects, `import rnfs`'s weight) and `test_embedded.py` (the guards' scopes, the dir lock, and with `RNFS_E2E=1` a host program over real RNS beside an embedded node). The state database has `test_statedb.py`, each store's migration a `test_migrate_*.py`, and `test_upgrade_014.py` loads `tests/fixtures/rnfs-0.1.4/`, a node directory 0.1.4 wrote (its `generate.py` -writes it again). CI runs them on Linux, macOS and Windows; `packaging/smoke_test.py` checks a built package +writes it again). Test doubles several modules share live in `tests/fakes/` (`FakeClient`, the `PeerClient` +stand-in, and the quiet-log fixture); `test_fake_contract.py` checks the fakes against what they stand in for +(`FakeNode` against `RnfsNode`, `FakeClient` and the node tests' `NetClient` against `PeerClient`, `FakeBackend` +against `server.Backend`), so production code needn't look up with `getattr` a method a fake might lack. CI runs them on Linux, macOS and Windows; `packaging/smoke_test.py` checks a built package (`--version`, `--help`, daemon and status, an add, and that the state database holds the pin and imported a 0.1.4 state file). @@ -1021,12 +1221,13 @@ Everything in the specification is implemented, with these simplifications and a - `max_download_size` is checked against the target's verified size, or the range's size, before anything below the target is fetched, and the DAG-node bytes a fetch downloads count against it too, so directories of empty files can't get around it. Listing and writing out also stop fetching at `max_output_entries`. -- **DAG byte bound** (`fetcher.dag_bound`, `Fetch.dag_limit`). `fetch_dag` counts the bytes of the distinct +- **DAG byte bound** (`fetcher.dag_bound`, `Fetch.dag_limit`; the bookkeeping is `fetcher._DagBudget`, pure, + the fetch's own in `Fetch.budget`). `fetch_dag` counts the bytes of the distinct blocks of its target's DAG, stored or downloaded, against `dag_bound(size)` = size + `DAG_GRANULE_OVERHEAD` (64) per `DAG_GRANULE` (4 KiB, `SmallLeaves`' threshold) of the declared size + `DAG_SLACK` (1 KiB), plus what the counted directory and index nodes bring (`node_credit`): a directory node its own bytes and `DAG_ENTRY_OVERHEAD` (16) per entry, an index node its own bytes and 16 per child less `INDEX_NODE_COST` - (64), never below 0; and `max_dag_bytes` (a remote pin's or auto-fetch's quota) if lower. Over it is + (64), never below 0; and `max_dag_bytes` (a remote pin's, auto-fetch's or follow's quota) if lower. Over it is `TooLarge`, not `InvalidDag`. Every caller gets it, since it lives in `fetch_dag`: `get`, `pin_add`, `dir_set`/`dir_rm`, `export`, `export_stream`, follow, remote pins and LXMF auto-fetches. A target whose DAG is stored complete already isn't walked: it is checked whole against the deny list and, with @@ -1054,19 +1255,20 @@ Everything in the specification is implemented, with these simplifications and a `max_output_entries` entries, so a pin of an honest directory of any size is allowed (`get` and `export` still stop at `max_output_entries` entries); a directory's blocks are then about what an honest directory of as many entries takes, and `max_download_size` and quotas bound them. To keep the overshoot to about - one block, `_take_batch` asks only for what fits the room left after the requests in flight (`_room`, - `_reserve`): a raw chunk's declared size, anything else `MAX_BLOCK_SIZE`, except that under the + one block, `_take_batch` asks only for what fits the room left after the requests in flight + (`_DagBudget.room`, `reserve`): a raw chunk's declared size, anything else `MAX_BLOCK_SIZE`, except that under the size-derived bound (not under a quota, which entries don't raise) a directory or index node takes nothing, since it costs at most 64 bytes (a request of `MAX_BATCH` of them overshoots by at most 2 KiB); and always one CID when nothing is in flight. Honest fetches are not held back by this: the content still to come leaves room for many file nodes, and directory nodes cost nothing. - **Listings** (`fetch_listing`: `ls`, the gateway's directory pages) fetch only directory and index nodes, - and are bounded as `fetch_dag` is for a declared size of 0 (`_declared` = 0): `DAG_SLACK` plus the nodes' + and are bounded as `fetch_dag` is for a declared size of 0 (`declare(0)`): `DAG_SLACK` plus the nodes' credit. It used to count index children per level only, so 40 entries below chains of single-child index nodes down to `MAX_DEPTH` took 751 KB; now a single-child index node costs 48 bytes, and those chains are refused after 24 KB. - **Byte-range reads** (`read_range`: `cat`, the gateway's file responses, `/api/cat`, a sealed file's - header and its decryption by `get`, `cat` and `info`, which never run `fetch_dag`) count the bytes of the + header and its decryption by `get`, `cat` and `info`, which run `fetch_dag` only to check a file against the deny + list's file hashes) count the bytes of the distinct file nodes each call meets, stored or downloaded, against `range_bound(size, length)` = `dag_bound(length)` + 2 × `file_levels(size)` × `FILE_NODE_MAX` (13,061 bytes: a header of 5 and 256 children of 51). Nodes wholly inside the range cost no more than in a whole file of `length` bytes; on each @@ -1075,9 +1277,10 @@ Everything in the specification is implemented, with these simplifications and a 1 MiB, two up to 256 MiB, seven for any size. Raw chunks aren't counted: they hold the range and at most two chunks more. A file node padded with empty children made `cat` of a file declaring 1 byte download 7.3 MB (and a sealed `get` of 12 KB 29 MB); now the read stops at about 27 KB plus one block, and at most - about 184 KB plus one block for any declared size. The budget (`_RangeBudget`) is set only while - `_range_leaves` ensures a node's children, since another read of the same fetch may run between a - generator's steps; `_reserve` counts a file node not yet received as `MAX_BLOCK_SIZE` against it, so a + about 184 KB plus one block for any declared size. Each call's budget (`_DagBudget.for_range`, which + counts every block against the fetch's own budget too) is passed explicitly to the `ensure` of a node's + children by `_range_leaves`, never kept on the fetch, since another read of the same fetch may run between a + generator's steps; `reserve` counts a file node not yet received as `MAX_BLOCK_SIZE` against it, so a small honest range ending inside two nodes of a level asks for them in two requests instead of one. - `max_storage` is soft. An unpinned fetch evicts other cached fetches (LRU) to make room for each response and goes above `max_storage` when nothing is evictable; when its session ends the node evicts again, which @@ -1099,10 +1302,23 @@ Everything in the specification is implemented, with these simplifications and a exits starts a new one; if `Thread.start()` fails the pin is given back and refused. The pins waiting are saved to the `remote_pins` table by maintenance (when changed; rewritten whole, a few rows) and at stop, and `_resume_remote_pins` queues them again at start (skipping those already pinned or whose identity left `remote_pin_identities`); workers - wait for the first join before fetching, and a fetch cut short by stopping stays queued. -- Byte-range reads (`cat`, gateway) walk the file lazily and fetch 32 chunks at a time ahead of what has been - written out. A read may meet at most `max_output_entries` pieces much shorter than `CHUNK_SIZE`, which bounds - files built from repeated tiny or empty subtrees without limiting honest large files. + wait for the first join before fetching, and a fetch cut short by stopping stays queued. Stop cancels their + fetches (an observer bound to `stop_event`, so a fetch ends at its next step) and waits for the workers + (`_remote_workers` maps each source to its thread) within its budget. +- Byte-range reads (`cat`, gateway) walk the file lazily and fetch 32 chunks (`WINDOW`) at a time ahead of what + has been written out. The next window is asked for before the current one is handed to the reader, so every + provider stays busy across window boundaries (a block that arrives after the call that asked for it is stored, + not dropped). A reader that stops early has fetched at most one window more than it read; when its fetch ends, + the links it still has requests on are closed (`PeerClient.close_unless_used`) unless another fetch is using one, + whose request is then left to finish: never a `Resource.cancel()`, which a 0.1.6 server cools down (NET-1). A + read may meet at most `max_output_entries` pieces much shorter than `CHUNK_SIZE`, which bounds files built from + repeated tiny or empty subtrees without limiting honest large files. +- `fetch_dag` and `fetch_listing` are one `ensure()` over a `_Walk`: a block's children are queued as soon as it is + stored or downloaded, not one tree level at a time, and `max_entries` is counted per block. A parent is stored + before its children are asked for, so an interrupted fetch resumes from what it stored, as before. +- End game (NET-4): when nothing is left to ask for and a provider is idle, a batch in flight longer than + `deadline_slack` is asked of it too (once, not on slow nodes, within `max_parallel`); the first verified copy + wins, and the slower request's link is closed as above, without counting as a stall. - Archive import pins only roots whose every block was in the archive, and reads at most 1024 roots. It verifies every block, including ones already stored, and stops with `StorageFull` like a fetch when a block not yet stored would leave less than the disk reserve free (`Storage.check_disk`). @@ -1111,15 +1327,42 @@ Everything in the specification is implemented, with these simplifications and a `Fetch._pace` waits as long again as the request took before asking that provider again, and a refusal within 4 × that time of the request being sent isn't counted towards `max_failures` (retried after an eighth of it, at least `busy_delay`). A `Fetch` exiting with an error (or cancelled) closes the links of its requests still - in flight. In private mode, incoming links not identified as a member's are closed first to make room. -- A shared-folder entry that fails to import because of its content (an invalid name, the deny list, too large) - isn't tried again until it changes; one that fails for a reason that may pass (`OSError`, including a pin the - state database couldn't save, or `StorageFull`) is tried again `RETRY_AFTER` (600 s) later, and an unchanged - entry whose pin fails is logged without ending the scan. + in flight. It never cancels a response's Resource (`Resource.cancel()`), and nothing in the fetcher may: on + RNS 1.5.6 that reaches the provider as a rejection, which before a quarter of the response is sent cools the + requester down there (`RequestServer._concluded`), while a closed link doesn't. In private mode, incoming + links not identified as a member's are closed first to make room. +- A shared-folder entry that fails to import because of its content (an `RnfsError`: the deny list, too large or + too deep) isn't tried again until it changes; one that fails for a reason that may pass (`OSError`, including a + pin the state database couldn't save, or `StorageFull`) is tried again `RETRY_AFTER` (600 s) later, as is one + failing with a `ValueError` (a bug, such as a `BlockMismatch`: invalid names and special files inside an entry + are skipped, so the content doesn't cause one), logged with its type; an unchanged entry whose pin fails is + logged without ending the scan. +- Inside an entry whose signature changed, the scan reads only the files that changed (audit D1): + `SharedFolder.files` (a `FolderFiles`) maps each file's `builder.file_state` (device, inode, size, `mtime_ns`, + execute bit; the open file's `fstat`, taken before it is read) to the `Built` it gave, grouped by top-level + entry so an entry imported again keeps only what it holds now and one gone takes its files along; a file moved + to another entry is found too. `build_open(..., cache=)` asks it (`builder.FileCache`) before reading each + regular file: a known file stands in only while `is_complete` holds for its CID (protected in the import's + session first) and no block of it is denied (`denied_roots`; else it is read again and its put refuses the + import, as before), and its remembered content hash still goes to `BuildOptions.check`; one remembered without + a hash (built while the deny list had none) is read again when there is a check. A file whose state changed + while it was read isn't remembered. Only fixed-chunker builds use it. It sits behind CONT-3's target check: a + refused folder is scanned as empty, which empties it. It is a store of its own (`shared_folder_files`, saved + with the node's batched stores, `RnfsNode._add_store`), so a restart doesn't read the folder's files again. - The shared-folder scan and `add` walk directories by file descriptor (`O_NOFOLLOW`, `O_NONBLOCK`, type and inode checks), so entries swapped for symlinks, FIFOs or devices mid-scan are skipped. The scan (not `add`) skips regular files whose opened descriptor has `st_nlink > 1` (`builder.hard_links`, `build_open`'s `skip_hard_links`); such a file's signature is marked, so it is imported once its other links are gone. +- Each scan resolves the shared folder it opened (`shared_folder.opened_path`: `fsutil.final_path` of its + descriptor, the path walk's checked root, or else its real path with the same device and inode) and refuses one + that is the rnfs dir (`SharedFolder`'s `rnfs_dir`, `config.root` in the node), a directory holding it, or inside + one of `PRIVATE_SUBDIRS` (`storage`, `names`, `state`, `blocks`, `lxmf`, each resolved): a warning while it + stays refused, and the scan treats it as empty (`current = {}`): nothing imported, every `folder` pin released + and the shared_folder table emptied, so pins a 0.1.5 node made from such a folder go at the first scan after the + upgrade. The walk then reads that descriptor, so a link retargeted after + the check can't redirect it. On POSIX, the first directory above it owned by another user (not root) or writable + by group or others (unless sticky and what is below it is the user's or root's) is warned about once + (`loose_ancestor`). - The identity file (`network/identity.py`) is opened with `O_NOFOLLOW | O_NONBLOCK` after an lstat that must show a regular file, not a link; the descriptor must be that same regular file (device and inode), owned by the user, and is tightened through `fchmod` if loose. The key is read from it (`RNS.Identity.from_bytes`), @@ -1162,8 +1405,14 @@ Everything in the specification is implemented, with these simplifications and a an answer reaches the highest known sequence (`NameManager.record` or the kept record) from two keepers (two established ones when no sequence is known), at most `keeper_rounds()` batches (3; 2 on slow nodes, where one current answer is enough once a sequence is known). On a slow node that is still at most two - `/name`; a fast node with no known sequence and no established keepers asks all 3 batches (up to 9), and the - lookup runs to completion unless two established nodes list keepers. + `/name`; a fast node with no known sequence and no established keepers asks all 3 batches (up to 9). The + query's state is a `_KeeperQuery` (`search`, `next_batch`, `ask` on a thread per keeper, `settled`, `resume`). + Its search stops (`should_stop`) once enough candidates are found, or once some are and `lookup_timeout` / + `SEARCH_SHARE` (5) has passed; candidates aren't asked while it runs. Each batch is joined for + max(`Lookup.remaining()`, `name_wait(config)`), the latter 87.5 s at the defaults. When the candidates run out + unsettled with batches left, `resume` runs `find_providers` again, but only if the last search was stopped + early (`cut_short`) and some of its candidates were asked, so resumes stay within the batch bound and the + lookup's deadline. - **Names.** A key imported, or found without a `names_published` row, is marked `imported`: its first publish offers kept records, makes a path request and queries keepers before choosing a sequence. `publish()` pins the new target, saves its row (the commit point; a failed commit rolls the record back and releases the new pin) and only @@ -1180,13 +1429,19 @@ Everything in the specification is implemented, with these simplifications and a (`_heard_only`, its own LRU, so finding one to evict is O(1)); an announce with no waiting resolve, of a name neither kept nor watched, may only evict one of those (`watch()` takes a name out of `_heard_only`). A name resolved for the user, or published here, leaves the table and the floor (`_hold`). A resolve with `persist=False` - (gateway, LXMF auto-fetch) raises the name's floor (`_floor`: name -> (sequence, wall time last resolved), in - resolve order), which `_offer` checks. A raise (or a new name) saves the floor at once if the last save was - `FLOOR_SAVE_INTERVAL` (5 s) ago, else from a one-shot `threading.Timer`; a raise by a record heard for a name in - the table (`_store`: an announce, a keeper's record, or one a resolve's refresh heard) is always the timer's, - started at once when a save is due, since its callers hold the lock; renewing only the time marks it for the next - save after an hour. `_trim_floor` drops the least recently resolved past `FLOOR_FACTOR` × `name_table_size`, but - stops at one resolved within `FLOOR_KEEP_AGE` (7 days) until the floor holds `FLOOR_HARD_FACTOR` (2) times that. + (gateway, LXMF auto-fetch) raises the name's floor (`_floor`, a `_NameFloor`: name -> (sequence, wall time last + resolved, used), in resolve order, `used` being its row's place; it owns its trim, load, save snapshot and timer, and + reads as name -> (sequence, resolved)), which `_offer` checks. A raise (or a new name) saves the floor at once if the + last save was `FLOOR_SAVE_INTERVAL` (5 s) ago, else from a one-shot `threading.Timer`; a raise by a record heard for a + name in the table (`_store`: an announce, a keeper's record, or one a resolve's refresh heard) is always the timer's, + started at once when a save is due, since its callers hold the lock; renewing only the time marks it for the next save + after an hour. `_NameFloor.trim` drops the least recently resolved past `FLOOR_FACTOR` × `name_table_size`, but stops + at one resolved within `FLOOR_KEEP_AGE` (7 days) until the floor holds `FLOOR_HARD_FACTOR` (2) times that. A resolve's + network refresh (`_shared_refresh` around `_refresh_name`) is remembered per name (`_refreshed`: when it ended, + monotonic, and whether it left a record; at most `REFRESHED_SIZE`, least recent dropped), so a resolve within its + `max_age` skips it unless that record has left the table or the refresh left a suspected replay (`_absorb_recalled`); + resolves of a name being refreshed wait on the leader's `Event` in `_refreshing`, then each runs only the local step + (persist, floor). - **Deny lists.** `RnfsNode._setup` makes the `DenyList` right after storage, gives it to storage (`storage.deny`), and `reload`s the accepted lists from their pinned blocks (`_read_deny_list`), so the publisher's first refresh already leaves out denied content. Fetches check every expectation in `_ensure` and the root in `Fetcher.open`, @@ -1196,15 +1451,26 @@ Everything in the specification is implemented, with these simplifications and a refreshes the publisher, `drop_denied`s the cache, releases `remote:` pins of `denied_roots` and collects, and warns about the other pins it keeps. File hashes: `_hash_check` (for `builder.checking`, in adds, the shared folder and imports, under `_refusing`, which collects what a refused add or import stored) and - `_check_file(expect, read)` (get's write-out check, a sealed get, `read` before its first byte, - `node_content.block` for a raw block and `export_stream` for every file before `start`; a raw file's digest is its - CID's, others are hashed once and remembered, across restarts: `deny_hashes`). A failed `_apply_deny` leaves + `_check_file(expect, get)` (get's write-out check, a sealed get, `read` before its first byte, + `node_content.block` for a raw block, `export_stream` for every file before `start` and `import_archive` for the + complete roots; a raw file's digest is its CID's, others are hashed once and remembered, across restarts: + `deny_hashes`; a refusal by hash logs a NOTICE suggesting `rnfs deny add `, since the file's blocks are still + served, once per file CID in `_hash_noticed`, cleared at `HASH_NOTICES` (1024)). `get` serves the file's whole DAG, stored and checked: `_check_fetched` (a sealed get, `read`) + runs `fetch_dag` first unless the hash is known, which also brings `max_download_size` and `check_disk`. Before + hashing, a file declaring more than `DENY_HASH_RATIO` (16) × `walker.leaf_bytes` + `DENY_HASH_SLACK` (64 MiB) is + refused as `Denied`, so `_refusing` collects what a refused import stored: hashing reads every occurrence of + repeated content, and six blocks (one chunk, five nodes each listing the one below 256 times) declare 2^56 bytes. + `leaf_bytes(expect, get, stop_at)` reads each distinct file node once and adds each distinct raw child's size + once, never reading a raw block (a size over `MAX_BLOCK_SIZE` is `InvalidDag`), and stops at the bytes needed, + so the check costs the DAG's nodes at most and the hash at most 16 × the distinct chunks plus 64 MiB. A failed `_apply_deny` leaves `_deny_applied` as it was, so maintenance tries again after `DENY_RETRY` (60 s). `remote_pin` refuses a denied CID; jobs don't fall back to the DHT on `Denied`, and LXMF auto-fetch records it as refused. - **Follow** isn't built on jobs (job pins use source `user`); `pin add --follow` doesn't exist. Checks use the record held unless nothing was heard for 2 × `announce_interval`; a failed check (any exception) is retried after 300 s, doubling up to `announce_interval`, and a failed fetch is retried with the record held - (no path request). + (no path request). The fetch checks the resolved target's size against `follow_quota` and sets it as + `max_dag_bytes` (as remote pins do with their quota, but per name rather than per source: the old target's + pin isn't counted, so a swap peaks at twice the quota); 0 refuses before opening the fetch. - **Contacts.** Every `` argument (`name resolve|keep|unkeep|follow|unfollow`) takes a name contact's nick; the gateway never expands nicks. A nick at the start of a target reuses the CID resolved for it for 60 s (`EXPAND_AGE`) from the time of the resolve, not from when the record was heard. @@ -1231,7 +1497,8 @@ Everything in the specification is implemented, with these simplifications and a are built by `.github/workflows/release.yml`. Only a pushed `vX.Y.Z` tag matching `rnfs/__init__.py`'s `__version__` publishes them, once the whole of `tests.yml` (which release.yml calls) has passed, with `SHA256SUMS` and signed build provenance (`gh attestation verify`), never replacing files already attached; - a manual run only builds. The Windows exe isn't code-signed, and unpacks itself on each start. + a manual run only builds, as do the runs for a pull request touching `packaging/**`, `pyproject.toml` or + `.github/**` and the weekly scheduled run. The Windows exe isn't code-signed, and unpacks itself on each start. - `lxmf>=1.2.0,<1.3` is a regular dependency next to `rns>=1.5.6,<1.6` (`pyproject.toml`), so every install includes LXMF; the PyInstaller spec fails the build without it. The upper bounds keep unpinned installs off RNS and LXMF releases rnfs hasn't been tested with, since it relies on some of their internals. @@ -1240,8 +1507,8 @@ Everything in the specification is implemented, with these simplifications and a `python3 -I packaging/update-lock.py [pkg==ver …]` regenerates it (no arguments: the newest versions `pyproject.toml` allows), then `packaging/flatpak/generate-sources.py` regenerates the Flatpak's `python3-deps.json` from it; both are committed. `install.sh`, `build-deb.sh`, the tests and the release's - Windows exe install from it with `pip --require-hashes --only-binary=:all:` (then `pip check`), and the - release's flatpak-sources step fails if regenerating changes the committed files. The sdist includes the + Windows exe install from it with `pip --require-hashes --only-binary=:all:` (then `pip check`), and + `tests.yml`'s flatpak-sources job fails if regenerating changes the committed files. The sdist includes the lock. - `update-lock.py --build pyinstaller==` writes `build-backend.lock` (setuptools, within `pyproject.toml`'s `<85` bound) and `build-tools.lock` (build, twine, PyInstaller, setuptools, ruff and @@ -1249,9 +1516,13 @@ Everything in the specification is implemented, with these simplifications and a without build isolation, so nothing unpinned is fetched for a build; the Flatpak's `build-sources.json` comes from `build-backend.lock`. `build-deb.sh` fails if a supported Python has no cffi backend, caps `python3` below 3.15, and writes a DEP-5 `copyright` and `md5sums`. - - `install.sh` installs the latest release's source archive (`archive/refs/tags/.zip`, the tag taken - from where `releases/latest` redirects, so no GitHub API), the main branch's with `--main`, or - `RNFS_SOURCE`. Since the script on `main` installs older release trees, it must keep working with every + - `install.sh` installs the latest release's sdist (`rnfs-X.Y.Z.tar.gz`, the tag taken from where + `releases/latest` redirects, so no GitHub API), the main branch's archive with `--main`, or `RNFS_SOURCE` + (with a warning for a plain `http://` URL). A release's sdist must match the release's `SHA256SUMS` and, + where the GitHub CLI (2.49 or newer) can check it, its signed build provenance for that tag + (`--source-ref=refs/tags/` and `--deny-self-hosted-runners` when this `gh` has them); a `gh` with the + attestation command that isn't logged in stops the install unless `RNFS_REQUIRE_ATTESTATION=0`, and without + `gh` it warns. Only a wheel of the tag's version is installed from it. Since the script on `main` installs older release trees, it must keep working with every release's layout from 0.1.1 on (`packaging/requirements.lock`, `packaging/build-backend.lock`). It unpacks an archive source itself (with Python, refusing absolute or `..` entries) to find its locks; a `.whl` or other requirement has none, so it warns and installs dependencies unpinned. It installs diff --git a/examples/rnfs_share.py b/examples/rnfs_share.py index cb8c685..9f6f16c 100644 --- a/examples/rnfs_share.py +++ b/examples/rnfs_share.py @@ -15,7 +15,8 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # run from a source checkout -import rnfs # noqa: E402 +import rnfs +from rnfs.fsutil import printable # names come from DAGs other people made: escape them for the terminal def share(node: rnfs.Rnfs, path: str) -> str: @@ -23,19 +24,19 @@ def share(node: rnfs.Rnfs, path: str) -> str: added = node.add(path) print(f"added {added.cid} ({added.size} bytes)") for warning in added.warnings: - print(f" warning: {warning}") + print(f" warning: {printable(warning)}") target = str(added.cid) if os.path.isdir(path): entries = node.ls(added.cid) for entry in entries: - print(f" {entry.kind:10} {entry.size:>12} {entry.name}") + print(f" {entry.kind:10} {entry.size:>12} {printable(entry.name)}") files = [entry for entry in entries if not entry.is_dir] target = f"{added.cid}/{files[0].name}" if files else None if target is not None: with node.open(target) as f: f.seek(f.size // 2) # only the chunks this read covers are fetched data = f.read(4096) - print(f"read {len(data)} bytes from the middle of {target}") + print(f"read {len(data)} bytes from the middle of {printable(target)}") mine = {name.alias: name for name in node.names.list()} name = mine.get("example") or node.names.create("example") @@ -43,12 +44,12 @@ def share(node: rnfs.Rnfs, path: str) -> str: def on_record(record: rnfs.Record): """Called on the node's event thread when a newer record of the name is heard (here: our publish).""" - print(f"heard rnfs:name/{record.name.hex()} -> {record.cid} (sequence {record.sequence})") + print(printable(f"heard rnfs:name/{record.name.hex()} -> {record.cid} (sequence {record.sequence})")) heard.set() with node.names.watch(name, on_record): record = name.publish(added.cid) - print(f"published rnfs:name/{name.hex} -> {record.cid}") + print(printable(f"published rnfs:name/{name.hex} -> {record.cid}")) heard.wait(10) output = os.path.join(tempfile.mkdtemp(), os.path.basename(os.path.normpath(path)) or "copy") @@ -82,7 +83,7 @@ def main(): with instance as node: share(node, args.path) except rnfs.RnfsError as e: # every rnfs error, from either class - sys.exit(f"rnfs_share: {e}") + sys.exit(f"rnfs_share: {printable(e)}") if __name__ == "__main__": diff --git a/install.sh b/install.sh index 5372b74..4176b4e 100755 --- a/install.sh +++ b/install.sh @@ -28,13 +28,16 @@ # them are then fetched unpinned, and a C compiler and Python headers are needed # RNFS_REQUIRE_ATTESTATION=1 fail unless the release's signed provenance could be checked (needs # the GitHub CLI, gh 2.49 or newer, logged in) +# RNFS_REQUIRE_ATTESTATION=0 install with only the SHA256SUMS check when gh is installed but not +# logged in (which otherwise stops the install) # # The latest release is installed from its source archive (rnfs-X.Y.Z.tar.gz, a release file, not the archive # GitHub makes of the tag), which must match the release's SHA256SUMS. Where the GitHub CLI (gh 2.49 or newer) # is installed and logged in, its signed build provenance is checked too (gh attestation verify, made by -# release.yml for this repository), and a failed check stops the install. Without gh the install goes ahead with -# a warning: SHA256SUMS shows the download is whole, but whoever could replace the release's files could -# replace it too. +# release.yml for this repository from this release's tag), and a failed check stops the install. Where gh is +# installed but not logged in, the install stops (unless RNFS_REQUIRE_ATTESTATION=0). Without gh the install +# goes ahead with a warning: SHA256SUMS shows the download is whole, but whoever could replace the release's +# files could replace it too. # # Dependencies (rns, lxmf, cryptography, ...) are installed as wheels at the versions and sha256 hashes # pinned in the source's packaging/requirements.lock (pip --require-hashes --only-binary=:all:), and rnfs @@ -273,6 +276,10 @@ RestrictSUIDSGID=yes # is left out: libffi closures, which cffi (under cryptography) can use, may need writable executable memory. SystemCallFilter=@system-service SystemCallArchitectures=native +# ProtectSystem and PrivateTmp are left out: in a user unit they need unprivileged user namespaces, which some +# systems turn off. Where that works, add them with the drop-in in contrib/rnfs.service.d/hardening.conf +# (https://github.com/afit21/rnfs/blob/main/contrib/rnfs.service.d/hardening.conf). UMask is left out too: what +# `rnfs get` writes gets its permissions from your umask. [Install] WantedBy=default.target @@ -313,6 +320,13 @@ the main branch instead with --main." fi } +warn_plain_http() { # warn when SOURCE (from RNFS_SOURCE) is, or names, a plain http:// URL + case "$SOURCE" in + *[Hh][Tt][Tt][Pp]://*) warn "$SOURCE is fetched over plain http://, so anyone on the network path can replace +what gets installed (no checksum or provenance is checked for RNFS_SOURCE). Use an https:// URL or a local file." ;; + esac +} + download() { # download URL FILE: save URL's content as FILE "$PYTHON" -I - "$1" "$2" <<'EOF' import shutil, sys, urllib.request @@ -347,14 +361,30 @@ if digest.hexdigest() != expected[0]: EOF # SHA256SUMS is a release file like any other: whoever can replace the sdist can replace it too. The signed # provenance can't be made outside this repository's release workflow - if gh_can_verify; then - # Older gh versions can only name the repository, not the workflow - signer="" - if gh attestation verify --help 2>&1 | grep -q -e '--signer-workflow'; then - signer="--signer-workflow=$REPO/.github/workflows/release.yml" + if gh_has_attestation; then + if ! gh auth status >/dev/null 2>&1; then + # This gh could check the provenance once logged in, so a missing login stops the install rather than + # quietly skipping the check (root's gh, for a --system install, is rarely logged in) + [ "${RNFS_REQUIRE_ATTESTATION:-}" = 0 ] || die "the GitHub CLI (gh) is installed but not logged in, so +it can't check the signed build provenance of $(basename -- "$1"), which wasn't installed. Log in with +gh auth login (as the user running this script) and run this again, or set RNFS_REQUIRE_ATTESTATION=0 to +install with only the SHA256SUMS check: + curl -fsSL .../install.sh | RNFS_REQUIRE_ATTESTATION=0 sh (or: | sudo RNFS_REQUIRE_ATTESTATION=0 sh -s -- --system)" + warn "RNFS_REQUIRE_ATTESTATION=0: checked $(basename -- "$1") against the release's SHA256SUMS only, not its +signed provenance (gh isn't logged in). That shows the download is whole, not that the release workflow built it." + return 0 fi - # shellcheck disable=SC2086 # signer is one word or nothing - if ! gh attestation verify "$1" --repo "$REPO" $signer >"$1.attestation" 2>&1; then + # Older gh versions can only name the repository, not the workflow or the tag (gh_help, read by + # gh_has_attestation, lists the options this one has) + options="" + case "$gh_help" in + *--signer-workflow*) options="--signer-workflow=$REPO/.github/workflows/release.yml" ;; + esac + # A build of this tag, not an older release's attested sdist renamed to this release's file name + case "$gh_help" in *--source-ref*) options="$options --source-ref=refs/tags/$RELEASE_TAG" ;; esac + case "$gh_help" in *--deny-self-hosted-runners*) options="$options --deny-self-hosted-runners" ;; esac + # shellcheck disable=SC2086 # options are words without spaces (REPO and RELEASE_TAG have none) + if ! gh attestation verify "$1" --repo "$REPO" $options >"$1.attestation" 2>&1; then cat "$1.attestation" >&2 die "gh attestation verify found no valid signed provenance for $(basename -- "$1") (above), so it wasn't installed: it must have been built by $REPO's release workflow. Please report this at $REPO_URL/issues." @@ -370,8 +400,8 @@ signed provenance is checked too. Set RNFS_REQUIRE_ATTESTATION=1 to refuse to in fi } -gh_can_verify() { # the GitHub CLI is here, logged in, and new enough to have gh attestation (2.49) - command -v gh >/dev/null 2>&1 && gh attestation verify --help >/dev/null 2>&1 && gh auth status >/dev/null 2>&1 +gh_has_attestation() { # the GitHub CLI is here and new enough to have gh attestation (2.49); sets gh_help + command -v gh >/dev/null 2>&1 && gh_help=$(gh attestation verify --help 2>&1) } fetch_release() { # fetch_release WORKDIR: download the release's sdist and SHA256SUMS, check them, and install that file @@ -476,6 +506,17 @@ build_wheel() { # build_wheel WORKDIR: build (or fetch) the rnfs wheel from bui fi } +find_wheel() { # find_wheel WORKDIR: print the rnfs wheel built in WORKDIR (a release's: of its version), or nothing + # An older release's sdist renamed to this release's file name passes SHA256SUMS and, with a gh too old for + # --source-ref, the provenance check too; but it builds a wheel of its own, older version + pattern="rnfs-*.whl" + if [ -n "$RELEASE_TAG" ]; then pattern="rnfs-${RELEASE_TAG#v}-*.whl"; fi + # shellcheck disable=SC2086 # pattern is a glob (RELEASE_TAG has no glob characters) + for found in "$1"/$pattern; do + if [ -f "$found" ]; then printf '%s\n' "$found"; return 0; fi + done +} + install_locked() { # install_locked LOCK WORKDIR: rnfs's dependencies, as LOCK pins them (versions and hashes) # Wheels only: building an sdist would fetch its build tools (setuptools, ...) unpinned only_binary=:all: @@ -506,6 +547,7 @@ do_install() { check_launcher check_unit choose_source + warn_plain_http updating=no if venv_ok; then say "Updating rnfs in $VENV" @@ -531,11 +573,8 @@ do_install() { if [ -n "$RELEASE_TAG" ]; then fetch_release "$wheels"; fi fetch_source "$wheels" build_wheel "$wheels" - wheel="" - for found in "$wheels"/rnfs-*.whl; do - if [ -f "$found" ]; then wheel=$found; break; fi - done - [ -n "$wheel" ] || die "building $SOURCE didn't produce an rnfs wheel" + wheel=$(find_wheel "$wheels") + [ -n "$wheel" ] || die "building $SOURCE didn't produce an rnfs wheel${RELEASE_TAG:+ of version ${RELEASE_TAG#v}}" if [ -n "$lock" ]; then install_locked "$lock" "$wheels" else diff --git a/maintainability-audit0.1.5.md b/maintainability-audit0.1.5.md new file mode 100644 index 0000000..0f8ba2f --- /dev/null +++ b/maintainability-audit0.1.5.md @@ -0,0 +1,583 @@ +# 0.1.5 maintainability audit + +This audit covers rnfs at v0.1.5 (`dev` @ 5bd3075, the same tree as the tag). It comes from six read-only reviews on +2026-10-17: content; fetcher and network; DHT and names; node, API and CLI; features; tests, packaging and docs. A +mechanical pass (function size, complexity, broad excepts, churn) was also run. All line references were checked +against the tree. A sample of each review's claims was re-checked by hand before writing this up. + +It is a companion to [0.1.6-plan.md](0.1.6-plan.md). That plan is about speed and this audit is about the cost of +change, so items already in the plan are not repeated. Where a refactor here makes a plan item easier or safer, it +says so (see [Order of work](#order-of-work)). + +**Rules for every item** (the same as the plan's): +- No wire changes: request paths, messages, CIDs, DAG bytes, chunk boundaries, and provider and name records stay + byte-identical. +- The local HTTP API stays compatible, including status codes. +- Local state may change only with an automatic one-way upgrade. +- rnfs is the core, and user-facing features belong in companion apps. + +**Sizes:** S is under a day, M is a few days, L is a week or more. **Risk** is the chance of a regression if the +change is made with the existing tests. + +--- + +## Summary + +The codebase is in good shape for its size. The docs are unusually accurate: ten spot checks of the README, the spec +and design.md against the code all matched. Each wire format has its encoder and strict decoder side by side. Clocks, +networks and RNG are injected through most of the core. The test suite is large (2001 tests) and the security tests +are thorough. + +The maintainability risk is concentrated in three places: + +1. **Seven files over 1,000 lines hold most of the churn.** `node.py`, `server.py`, `lxmf.py`, `names.py`, + `storage.py`, `fetcher.py` and `cli.py` are also the files changed most often this year. Each one has more than + one job and shares state across those jobs by hand. +2. **The same patterns are copied instead of shared.** Examples are the "dirty rows, snapshot, save, restore on + failure" store (about 11 copies), 32-hex ID parsing (about 8), DAG path walking (4), msgpack header tables (3), + schedulers, log throttles and token buckets. Several copies already differ. +3. **Some guarantees rest on conventions nothing checks.** Released schema steps are never edited, and four + real-RNS test classes are assumed to run in CI but don't. Wire bytes are assumed unchanged, but the protocol + tests only round-trip. Shutdown is assumed to release everything, and the plugin boundary is assumed to hold. + Each of these will break silently. + +The cheapest wins are in the tests and CI (TS1–TS3). Those make every later refactor safer, so they come first. + +### Numbers +| Measure | Value | +|---|---| +| Source | 28,013 lines in 61 modules under `rnfs/` | +| Tests | 37,794 lines; 2001 tests, 38 skipped; 3 m 29 s wall but only ~34 s CPU (Python 3.12) | +| Functions over 60 / 100 lines | 14 / 6 (of 1998). Longest: `keepers.ask_keepers` 138, `cli.build_parser` 138, `publisher.publish_due` 127, `lookup._iterate` 125, `cli.run_daemon` 123, `server._handle_blocks` 111 | +| Largest classes | `RnfsNode` 88 methods / 1336 lines; `Storage` 59 / 891; `NameManager` 52 / 963; `ApiHandler` 48; `Fetch` 46; `LxmfFeature` 43 | +| Broad `except Exception` / bare | 153 in `rnfs/`; 111 have no `noqa` rationale (the BLE001 rule isn't enabled, so the 43 `noqa: BLE001` comments aren't checked either) | +| Function-level imports | 172, mostly in features and cli (some to keep RNS off the CLI path, some to dodge import cycles) | +| TODO / FIXME | none | +| Coverage measurement | none | + +--- + +## Cross-cutting themes + +These came up in two or more reviews, so they're best fixed once rather than area by area. + +### XC1. The core/feature boundary leaks in both directions (L, med) +- **Core imports features:** + - [api.py:150](rnfs/api.py#L150) imports `contacts.is_nick` and [api.py:939](rnfs/api.py#L939) imports `keepers.parse_name_arg`. + - [node.py:1047](rnfs/node.py#L1047) calls `features.get("contacts")`. + - [embedded.py:259-282](rnfs/embedded.py#L259-L282) uses keepers and jobs. + - The core `get`, `pin` and `export` commands in cli.py run on `jobs.run_transfer` ([cli.py:377](rnfs/cli.py#L377), 392, 457, 491, 528, 607), so they can't work without the jobs feature. +- **Core hard-codes feature details:** + - Config holds the `lxmf_*` settings and calls `lxmf.validate_settings` and `deny.validate_settings` by name ([config.py:122-130](rnfs/config.py#L122-L130), [210-223](rnfs/config.py#L210-L223)). + - `storage.check_source` hard-codes the `"follow:"` and `"lxmf:"` pin sources ([storage.py:68-83](rnfs/content/storage.py#L68-L83)). The `"name:"` and `"follow:"` prefixes are defined again in names.py and follow.py. + - statedb's `SCHEMA` and `LEGACY` hold every feature's tables and importers ([statedb.py:105-121](rnfs/statedb.py#L105-L121), [464-469](rnfs/statedb.py#L464-L469)). +- **keepers is core protocol under a feature label:** + - It serves the `/name` wire path, and names.py calls back into it. + - It reaches node privates (`node._lookup`, `node._is_known` at [keepers.py:730](rnfs/features/keepers.py#L730) and [772](rnfs/features/keepers.py#L772)). + - It wires itself in with `node.names.keepers = self` ([keepers.py:446](rnfs/features/keepers.py#L446)). +- **Why it matters:** [features/__init__.py](rnfs/features/__init__.py) promises that a new feature doesn't mean editing the node, the API and the CLI. In practice a new stateful feature, or moving lxmf or contacts out to a companion app, means editing config.py, statedb.py, storage.py, api.py and cli.py. +- **Change:** + 1. Classify explicitly. Move core-like code out of `features/`: the keepers record, store and query logic (into e.g. `rnfs/names_keepers.py`), and `run_transfer` and `StreamProgress` (into a CLI module). Keep `features/` for app-like code. + 2. Replace named imports with hooks: `Feature.validate_settings(config)` called in a loop, a target-expander hook in place of `features.get("contacts")`, and feature-declared schema entries. Existing tables and `LEGACY` strings stay as they are, so the state is unchanged. + 3. Make `lookup()` and `is_known()` public on the node, and replace `names.keepers = …` with a `set_keepers()` method plus a `typing.Protocol` that a test checks the real feature against. +- **Don't** let features register pin-source prefixes at run time. `Storage._load` durably deletes rows that fail `check_source` ([storage.py:293-296](rnfs/content/storage.py#L293-L296)), so a node started without a feature would erase that feature's pins. Keep one static `{prefix: validator}` table in storage, holding every prefix constant, and have names, follow and lxmf import from it. + +### XC2. The persisted-store pattern is copied about 11 times, and the copies already differ (M, low–med) +- **Where:** the same structure (connect, load and validate rows, delete bad ones, keep dirty sets, snapshot under the lock, write outside it, restore the dirty sets on failure) is written separately in: + - providers ([providers.py:732-780](rnfs/dht/providers.py#L732-L780)), publisher ([publisher.py:338-371](rnfs/publisher.py#L338-L371)), denylist ([denylist.py:344-366](rnfs/denylist.py#L344-L366)), names (two different ways, [names.py:1288-1346](rnfs/names.py#L1288-L1346)) and pin_names ([pin_names.py:99-108](rnfs/pin_names.py#L99-L108)); + - the features: lxmf `MessageLog`, jobs, keepers, follow and contacts ([lxmf.py:641-662](rnfs/features/lxmf.py#L641-L662), [jobs.py:350-379](rnfs/features/jobs.py#L350-L379), [keepers.py:333-350](rnfs/features/keepers.py#L333-L350), [follow.py:164-180](rnfs/features/follow.py#L164-L180), [contacts.py:148-157](rnfs/features/contacts.py#L148-L157)); + - the node's remote-pin queue and fetched-targets store. +- **How the copies differ:** + - Some catch `BaseException` and re-raise, names catches only `OSError` and logs, and pin_names writes under the lock. + - jobs, follow and contacts save durably; keepers and lxmf don't. + - contacts keeps the in-memory change *and* raises, while the others re-queue it. + - The "clean meta plus orphan-pin reconcile" algorithm is written twice ([lxmf.py:849-877](rnfs/features/lxmf.py#L849-L877), [follow.py:125-162](rnfs/features/follow.py#L125-L162)). +- **Scenario:** a fix to the "an older snapshot must never land last" ordering lands in one store and not the other ten. Or a new store copies whichever version happens to be nearest. +- **Change:** a small `statedb.RowStore` helper that owns the save lock, the dirty keys and restore-on-failure, takes `snapshot()` and `write(tx, rows)` callbacks, and has an explicit `durable` flag. Add an `open_store(state_dir, load)` helper and a `reconcile_pins(prefix, recorded, clean, adopt, release)` helper. Table layouts don't change. Move stores over one at a time, starting with the features. + +### XC3. 32-hex ID parsing exists about 8 times with different rules (S–M, low) +- **Where:** + - The private `config._HEX_ID` ([config.py:43](rnfs/config.py#L43)) is imported by api.py, names.py and interface.py. + - jobs.py:63, contacts.py:34, storage.py:60 and lxmf.py:106 have their own regexes, some lowercase-only. + - `identity.parse_node_id` uses bare `bytes.fromhex` ([identity.py:148-153](rnfs/network/identity.py#L148-L153)), which accepts whitespace between digits. names.py and api.py carry comments guarding against exactly that. + - `api.parse_hex_id` rejects `` IDs, while `parse_node_id` and `parse_name` accept them. + - The member set is normalised separately in [node.py:397](rnfs/node.py#L397) and [links.py:206](rnfs/network/links.py#L206). The remote-pin identity set is normalised separately in node.py:1413 and server.py:684. +- **Scenario:** a case, bracket or whitespace fix goes into one parser and the others keep accepting different input. In private mode, the node and the link pool could disagree about who is a member. +- **Change:** one stdlib-only `rnfs/ids.py` with `parse_hex16(text, what, *, brackets=…)`, plus `Config` properties that hold the normalised member and remote-pin sets as frozensets. Narrowing `parse_node_id` only affects odd `bootstrap_nodes` values, which are already warned about and skipped. + +### XC4. Errors: ValueError means two things, the status mapping is in four tables, and broad excepts hide bugs (M, med) +- **ValueError is overloaded:** + - The API maps ValueError to 400 ([api.py:103](rnfs/api.py#L103)) and logs only 500s ([api.py:772-774](rnfs/api.py#L772-L774)). + - Content code also raises ValueError for internal invariants, such as a block/CID mismatch on the add path ([blockstore.py:202-204](rnfs/content/blockstore.py#L202-L204), [storage.py:380](rnfs/content/storage.py#L380)). A bug there becomes a silent "bad request". + - `SealError` and `NotRecipient` subclass ValueError ([seal.py:59-64](rnfs/content/seal.py#L59-L64)), so a tampered sealed file is a 400 while a damaged DAG is a 422. Neither is exported, so programs can only catch ValueError. + - `SharedFolder._import` treats any ValueError as a permanent content fault ([shared_folder.py:251-254](rnfs/content/shared_folder.py#L251-L254)), so a bug there silently parks entries. +- **Four status tables:** `api.ERROR_STATUSES`, `apiclient.STATUS_ERRORS`, `gateway.ERROR_MESSAGES` and the hand-listed cases in test_apiclient.py:406-410. A new error class needs four coordinated edits. +- **Bugs reported as network failures:** + - [fetcher.py:963-964](rnfs/fetcher.py#L963-L964) turns *any* exception from `client.blocks`, including a TypeError, into `BlocksResult(FAILED)`. It is logged at VERBOSE outside the `except`, so the debug log never gets the traceback, and the user sees "no provider could supply X". + - [lookup.py:352-355](rnfs/dht/lookup.py#L352-L355) is `except Exception: pass` around the routing table's `on_answer` and `on_failure`. If private mode's `admit` raises, that node is never added and nothing is logged. + - The same pattern appears in links.py:487-489, mesh.py:99-100 and peers.py:75/87/125/136. +- **Change:** + 1. Add specific ValueError subclasses (`BlockMismatch`, and make `SealError` an `RnfsError` too), keeping each one's current HTTP status, and export them. Have the shared folder treat only content-caused errors as permanent. + 2. Keep one status table in errors.py (stdlib-only, so apiclient can import it) and derive the other three from it. + 3. Log unexpected exceptions at WARNING *inside* the `except`, so the debug log records the traceback. Narrow the catch where the expected failures are known. + 4. Enable ruff BLE001 (see TS10) so each broad except carries a reason. + +### XC5. Shutdown is not exception-safe, and threads outlive `stop()` (M, med) +- **Where:** + - [node.py:492-528](rnfs/node.py#L492-L528): `announcer.stop`, `announce_handler.deregister` and `server.stop` are not guarded. If one raises in an embedded node, `_close_state` and `_release_destination` are skipped. The destination stays registered and the database stays open, so the next `rnfs.Node` in the process fails. + - The stores are named in three hand-written lists, in `_close_state` ([node.py:544](rnfs/node.py#L544)), `_save_state` and `stop`, plus a fourth schedule in `maintain`. A store added to `_setup` but missed in the `_close_state` tuple keeps its statedb reference. The reference-counted registry ([statedb.py:374-383](rnfs/statedb.py#L374-L383)) then reuses the old connection on the next start. + - The remote-pin workers ([node.py:1348](rnfs/node.py#L1348)) aren't in `self.threads`, and their fetches don't watch `stop_event`. + - follow, deny and keepers `stop()` only set an event and never join. lxmf and jobs join with their own hard-coded 2 s ([lxmf.py:946](rnfs/features/lxmf.py#L946), [jobs.py:57](rnfs/features/jobs.py#L57)), and the node's 5 s budget is never passed down. +- **Scenario:** follow's thread is still inside `check()` when the node closes the state, so its `_save` raises StateDbError after `storage.pin` already ran. Correctness then rests on the next start's reconcile, and shutdown logs spurious errors. +- **Change:** + - Register stores as `_setup` builds them (`self._stores.append(x)`), behind a `save()` / `close()` protocol, and loop over that list. + - Write `stop()` as an ordered list of individually guarded steps. + - Change the feature contract to `Feature.stop(deadline)`, with base-class `_spawn` and `_join(deadline)` helpers. + - Track the remote-pin workers and run their fetches under an observer bound to `stop_event`. + +### XC6. Clocks are injected, but waits are real (M, low) +- **Where:** + - `server._previous_done` uses `time.monotonic()` in spite of `self.clock` ([server.py:1044](rnfs/network/server.py#L1044), 1054). + - `PeerLink.acquire`, `_open` and `_recall` use `time.monotonic` ([links.py:152-339](rnfs/network/links.py#L152-L339)), while `LinkPool.request` compares against `self.clock()`. + - `Lookup._iterate` blocks on a real `results.get(timeout=…)` ([lookup.py:292](rnfs/dht/lookup.py#L292)), and `fetcher._download` does the same. + - `NameManager._refresh` waits on a real `time.monotonic()` for `path_timeout` ([names.py:858-866](rnfs/names.py#L858-L866)). That alone is 7 of the 13 seconds test_names takes. +- **Effect:** + - About 84% of the suite's wall time is idle. + - 27 tests assert wall-clock bounds (`assertLess(elapsed, …)`). One already had to be widened for CI (2f2b629), and `test_fetcher.py:755` leaves a thread sleeping 5 s after the test ends. +- **Change:** use the injected clock everywhere and add an injectable waiter or thread starter where code blocks, following `test_pacing.py`, which already runs in virtual time. Drop wall-clock upper bounds where a counter already proves the point, e.g. [test_publisher.py:618](tests/test_publisher.py#L618), where line 614's `running.peak == PUBLISH_LOOKUPS` makes the timing check redundant. + +### XC7. Test fakes: shared but kept in test modules, duplicated and unchecked (M, low) +- **Where:** + - [tests/support.py](tests/support.py) is 117 lines. The real shared doubles live in big test modules and are imported across the suite: `from tests.test_names` 36 times, `test_node` 27, `test_api` 13 and `test_server` 11. The security helpers import from test modules too. + - Duplicate fakes: FakeDestination (3 copies), FakeLink (`test_server.py:265`, `test_links.py:36`), FakeClient (`test_fetcher.py:46`, `test_publisher.py:39`), FakeLookup and FakeNetwork. + - Production code bends to incomplete fakes. [fetcher.py:927-930](rnfs/fetcher.py#L927-L930) does `getattr(self.client, "link_mdu", None)` because "test fakes may not know links", and `server.Backend` marks two methods "optional (looked up with getattr)" because `FakeBackend` lacks them. So tests run a different code path from production. + - `test_fake_contract.py` checks only FakeNode against RnfsNode. + - `LxmfNetwork` tests build the object with `__new__` and hand-assign up to 12 private attributes ([test_lxmf.py:929-934](tests/test_lxmf.py#L929-L934), 1323-1333), so any new field breaks them. + - The same `setUpModule` / `tearDownModule` log-quieting pair is repeated in 45 files. +- **Change:** + - Add a `tests/fakes/` package (network, node, client, rns_link, lxmf) and one shared quiet-log fixture. + - Make `link_mdu`, `served_block_size` and `feature_requests` required, and add them to the fakes. + - Extend the contract test to FakeClient↔PeerClient, FakeBackend↔Backend and FakeLink↔the RNS.Link surface used. + +### XC8. Small rules restated locally +Each of these is S and low risk. Fix them as the surrounding code is touched. +- **"Two missed announces":** `routing._silence_limit`, `names.ANNOUNCE_VALIDITY = 2` and `follow.STALE_ANNOUNCES = 2` (and deny's copy). Use one constant in constants.py. +- **60 s log throttles:** written three times ([server.py:820-830](rnfs/network/server.py#L820-L830), [guards.py:182-201](rnfs/network/guards.py#L182-L201), [guards.py:388-393](rnfs/network/guards.py#L388-L393)). Move `guards._ThrottledLog` into `rnfs.log` and reuse it. +- **Token buckets:** lxmf hand-rolls three ([lxmf.py:1218-1240](rnfs/features/lxmf.py#L1218-L1240), 2132-2150), while `mesh.TokenBucket` already has an injectable clock and locking. +- **Formatting helpers:** `human_size` is written three times and they disagree ("1.5 MB", "1.50 MB", and "?" vs "-" for None), and the timestamp formats differ. Use one no-RNS `rnfs/fmt.py`. +- **Block validity check:** `len(block) <= MAX_BLOCK_SIZE and cid.verify(block)` appears five times in blockstore.py and storage.py. Add a `blockstore.matches(cid, block)` helper. +- **`chunker.MAX_SIZE`:** it is commented "= MAX_BLOCK_SIZE" but not derived from it, it decides fastcdc CIDs, and it shares its name with the unrelated `dag.MAX_SIZE`. Rename it `FASTCDC_MAX` and assert `<= MAX_BLOCK_SIZE`. +- **"file" / "directory" literals:** compared as plain strings in edit.py, node.py, fetcher.py, deny.py and node_content.py, although walker.py defines `FILE_TYPE` / `DIRECTORY_TYPE`. `KIND_NAMES` lives in node.py, not next to `dag.KIND_*`. + +--- + +## Tests, CI and docs (TS) + +### TS1. Four real-RNS test classes never run in CI (S, low): do first +- **Where:** [tests.yml:41-44](.github/workflows/tests.yml#L41-L44) sets `RNFS_E2E=1` only for `tests.test_e2e tests.test_client`. These classes are gated on the same variable and never selected: + - [test_embedded.py:464](tests/test_embedded.py#L464) `HostProgramTest`, the only real-RNS test of `rnfs.Node`; + - [security/test_h1_real_rns.py:37](tests/security/test_h1_real_rns.py#L37); + - [security/test_m1_real_rns.py:45](tests/security/test_m1_real_rns.py#L45); + - [security/test_h2_display_page_pins.py:306](tests/security/test_h2_display_page_pins.py#L306). +- **Why:** these are security regressions plus the embedded mode, and design.md:982-985 implies CI runs them. +- **Change:** run `python -m unittest discover tests` with `RNFS_E2E=1` in the E2E step. Or list every gated module and add a test that each `RNFS_E2E` gate's module appears in tests.yml. + +### TS2. Nothing protects upgrades from 0.1.5 state (S–M, low): do before any 0.1.6 schema change +- **Where:** + - `tests/fixtures/` holds only `rnfs-0.1.4/`, which is JSON. There is no `rnfs.db` written by 0.1.5, the first SQLite release. + - "Steps are only ever appended" ([statedb.py:43-44](rnfs/statedb.py#L43-L44)) is just a comment. [test_statedb.py:94-102](tests/test_statedb.py#L94-L102) upgrades from the *current* code's `SCHEMA[:2]`, so editing a released step in place passes. + - Fresh-DB tests can't see that kind of edit. Upgraded databases miss the change, and every test still passes. + - Two branches that each append a step collide on the same index with no error. + - Schema steps can only be SQL, so the first migration that needs Python has no hook. +- **Change:** + - Add a `tests/fixtures/rnfs-0.1.5/` fixture made by v0.1.5's own code (the way `generate.py` made 0.1.4's) and a `test_upgrade_015.py`. + - Add a golden test that pins a digest of `SCHEMA[:16]` as "released, never edit". + - Allow `(label, callable(tx))` steps. +- This also gives the plan's release-checklist item "a 0.1.5 `` starts cleanly" something to run against. + +### TS3. No known-answer tests for the wire format (M, low): do before the plan's fetcher work +- **Where:** [test_protocol.py:79-203](tests/test_protocol.py#L79-L203) only encodes then decodes with the same code. The only literal vectors are for DAG and archive bytes, chunker CIDs and *malformed* name records. +- **Scenario:** a refactor changes a msgpack int width, a list/tuple choice or field order. It still round-trips, every e2e node runs the same code, and interop with 0.1.3 breaks silently. +- **Change:** add `tests/fixtures/wire/*.hex` covering valid `/find`, `/provide`, `/blocks`, `/pin` and `/name` requests and responses, announce app_data, name records and LXMF fields. Generate them once from v0.1.3 and v0.1.4. Assert both `decode(vector) == value` and `encode(value) == vector`. + +### TS4. No coverage measurement (S, low) +- Coverage is not in `packaging/build-tools.lock` and no CI step runs it. +- Modules with no dedicated test file, 3,240 lines in all, so their coverage is unknown: `features/keepers.py` (969), `winacl.py` (463), `features/follow.py` (442), `network/guards.py` (409), `features/deny.py` (309), `features/peers.py` (260), `node_content.py` (255) and `network/announces.py` (133). +- **Change:** lock `coverage` and add one Linux job that runs `coverage run -m unittest discover tests` with `RNFS_E2E=1`. Report only at first, with no threshold. + +### TS5. Several build paths run only when a tag is pushed (S, low) +- **Where:** the Flatpak build and its sources-match check ([release.yml:121-139](.github/workflows/release.yml#L121-L139)), the Windows exe, the arm64 .deb and the Debian 12 smoke test run only in release.yml. +- **Scenario:** a lock bump leaves the Flatpak sources stale. It's caught only after `vX.Y.Z` is pushed, and because [release.yml:37](.github/workflows/release.yml#L37) requires the tag to equal `__version__`, the fix means moving the tag or bumping the version. +- **Change:** move the cheap Flatpak sources diff into tests.yml. Run the release build jobs on PRs that touch `packaging/**`, `pyproject.toml` or `.github/**`, and weekly. + +### TS6. No developer or release guide (S, low) +- **What exists:** + - How to run the tests is one sentence (design.md:980). + - `RNFS_FUZZ_EXAMPLES`, `RNFS_BENCH` and the full `RNFS_E2E` list are documented only in module docstrings. + - The release checklist exists only in the untracked 0.1.6-plan.md. + - Adding an API route touches about six places (see ND4) and is written down nowhere. + - Nothing checks that CHANGELOG's first `## X.Y.Z` equals `__version__`. + - Release notes use `--generate-notes` ([release.yml:264](.github/workflows/release.yml#L264)) rather than the CHANGELOG. +- **Change:** a CONTRIBUTING.md (or a "Development" section) covering tests, lint, locks, releasing, adding a schema step and adding an API route. Add a CHANGELOG == `__version__` test, and feed that CHANGELOG section into the release notes. + +### TS7. The supported-Python list is in 7+ places with no cross-check (S, low) +- **Where:** `pyproject.toml:12`, `tests.yml:21`, `packaging/update-lock.py:42,60`, `flatpak/generate-sources.py:28`, `build-deb.sh:33,36` and the interpreter names `install.sh:154` tries. README.md:340 also hard-codes `pyinstaller==6.22.3`. +- **Scenario:** adding 3.15 but missing `BUILD_PYTHONS` leaves build-tools.lock without wheels for the new CI cell. +- **Change:** a `test_packaging` test that checks these lists agree with the tests.yml matrix. + +### TS8. Pinned actions and the install sequence are repeated, and nothing updates them (S, low) +- `actions/checkout` is pinned 9 times and `setup-python` 7 times, each with an "update together" comment. +- The locked install sequence (requirements.lock, then the build lock, then `pip install --no-deps --no-build-isolation .`, then `pip check`) is pasted 4 times. +- There is no `.github/dependabot.yml`, although [test_rns_compat.py:5](tests/test_rns_compat.py#L5) names Dependabot as what would trigger it. +- **Change:** Dependabot for `github-actions`, which handles SHA pins with version comments; a composite `.github/actions/install-locked`; and a scheduled job that reports newer rns or lxmf releases within the pyproject bounds. + +### TS9. Security tests are organised by external audit IDs (M, low) +- The 73 `tests/security/test_*.py` files are named by IDs (`m3r2`, `new6`, …) that appear nowhere else in the repo. Finding every test for `keepers.py` means searching five files. +- Seven docstrings cite `file:line` locations that have since moved (e.g. `test_l10_index_page_injection.py:2`, `test_l14_page_local_paths.py:3`). +- Small files overlap the main ones: `test_pin_false.py` (4 tests), `test_get_partial.py`, `test_server_outcomes.py` and `test_migrate_small.py`, which is grouped by file size rather than by store. +- **Change:** group the security tests by subsystem, one class per finding with its ID in the docstring, or keep the files and add an ID → module → fix-commit index. Cite function names, not line numbers. + +### TS10. Lint rules don't cover the `noqa` comments in the code (S, low–med) +- [pyproject.toml:41](pyproject.toml#L41) selects F, E9, B006, B008 and B904. The code carries 43 `noqa: BLE001` and 30 `noqa: E402` comments for rules that aren't enabled, so nothing checks them. There is also a `noqa: B902`, a flake8 code ruff doesn't implement. +- The companion pages (`companion-nomadnet-sites/*/index.mu`, 554 lines of Python) are never linted. +- **Change:** enable BLE001 (start with per-file ignores and triage the 111 sites), E402, RUF100 (flags stale `noqa`) and the rest of B. Add `extend-include` for the `.mu` scripts. + +### TS11. Stale planning docs, and no reference for the local API (S, low) +- [roadmap.md](roadmap.md) lists 0.1.5's deny-list and SQLite work as not done. It puts "Client Library" under 0.1.6, but it shipped in 0.1.5 (CHANGELOG.md:57), and "change notifications" shipped as `/api/name/watch`. +- The local API must stay compatible, but its routes exist only as a prose list (design.md:383-389), with no request or response shapes. +- [specification.md](specification.md) doesn't say which version it describes, nor that the wire format is byte-identical since 0.1.3. +- **Change:** update roadmap.md. Generate a route table from `api.ROUTES` plus each feature's `api_routes()`, with a test that the documented list matches. Add a version and compatibility note to the spec. + +### TS12. Smaller hygiene items (S, low) +- The Flatpak manifest copies `../..` with a skip list that doesn't skip `*.egg-info`, `.ruff_cache` or `.venv`, so a local flatpak-builder run picks up dev debris. +- `companion-nomadnet-sites/file-index-node/index.mu:21-26` ships the maintainer's own CIDs as defaults. +- The plist comment says "adjust the path (both places)", but the path appears three times. +- `rnfsgifcompressed.gif` (427 KB) is tracked at the root, so every clone carries it. It could move to a release asset or `docs/`. +- `.gitignore` is correct: `__pycache__`, `rnfs.egg-info` and `.ruff_cache` are ignored, not tracked. + +--- + +## Content layer (CT): `rnfs/content/`, `objects.py` + +### CT1. Storage has its own DAG walking and path stepping, written twice inside itself (M, med): do before plan D3 +- **Where:** + - The index step rule ("the child with the greatest first_name <= name") is in [walker.py:162-169](rnfs/content/walker.py#L162-L169) and again in [storage.py:609-611](rnfs/content/storage.py#L609-L611). + - The path walk over `_step_in` is written twice with different stop conditions: `_path_cids` under the lock ([storage.py:647-670](rnfs/content/storage.py#L647-L670)) and `_warm` without it ([712-724](rnfs/content/storage.py#L712-L724)). + - Child listing is done separately in `Storage._listed`, `walker.children`, `dag._check_node` and `walker.check_node`. + - `Storage.dag_blocks` ([storage.py:841-859](rnfs/content/storage.py#L841-L859)) re-does `walker.iter_dags`' walk, and `Storage.stat` is another copy of `_reach`. + - `MISSING` means both "the block isn't stored" and "no memo entry". +- **Scenario:** + - D3 changes `_reach` and `dag_blocks`, and any change to path semantics has to land in four places. + - If `_warm` and `_path_cids` drift apart, warming silently stops covering what the locked walk reads, and disk reads under the lock come back (the problem `security/test_new5_storage_lock_walks.py` guards against). + - `archive_size` ([node_content.py:213-214](rnfs/node_content.py#L213-L214)) assumes `dag_blocks` returns exactly what `iter_dags` writes. +- **Change:** add `names` and `child_cids` properties to the `dag` node classes. Put one `step_in(node, name)` in walker.py, used by both modules. Write one path-walk generator over a "node or unavailable" callback, shared by `_path_cids` and `_warm`. Use a separate `_NO_MEMO` sentinel. + +### CT2. A CID-deciding option and the deny check travel as ContextVars (M, low–med) +- **Where:** `chunker._active` ([chunker.py:56](rnfs/content/chunker.py#L56)) decides fastcdc CIDs, and `builder._check` ([builder.py:120](rnfs/content/builder.py#L120)) is the deny list's file-hash check. Both are set two layers up (node.py:750, 979, 1003, 1030), and `build_file` takes neither as a parameter. +- **Scenario:** `ThreadPoolExecutor` doesn't copy contextvars. Moving directory hashing onto worker threads, a natural speed-up, would silently do two things: + - fall back to the fixed chunker, so `--chunker fastcdc` adds get different CIDs; + - skip the deny check. + + A new `SharedFolder.scan()` caller not wrapped in `builder.checking` would also skip the check. +- **Change:** pass an explicit `BuildOptions(chunker, check)` down through `build_file`, `build_open`, `add_path` and `add_stream`. Keep `using()` and `checking()` as thin wrappers during the move. + +### CT3. The rule for which local files are included is written three times (M, med) +- **Where:** `builder._build_tree` ([builder.py:472-503](rnfs/content/builder.py#L472-L503)), `shared_folder._scan_item` ([shared_folder.py:196-222](rnfs/content/shared_folder.py#L196-L222)) and `open_signature`'s `add` ([shared_folder.py:67-89](rnfs/content/shared_folder.py#L67-L89)). +- **They already differ:** + - The depth bound is `depth > MAX_DEPTH` in the builder but `MAX_DEPTH + 1` in the signature. + - The hard-link count comes from the listed stat in one and from `fstat` in the other. +- **Scenario:** a new skip rule added to the builder doesn't trigger a re-import when that attribute changes, unless the signature is updated too. The `"linked"` marker exists only to patch this coupling. +- **Change:** one walk generator in the fs layer that applies the policy and yields `(relative, st, handle | None, skip_reason)`. The builder, the signature and the top-level scan all consume it. + +### CT4. Cache and pin bookkeeping is spread across five removal sites (M, low–med) +- **Where:** cache records are removed by `_cap_cache`, `_evict_to`, `drop_denied` and `gc`, and retargeted by `record_fetch`. Each updates a different subset of `_cache_dirty`, `_garbage` and `_cache_generation` ([storage.py:494-513](rnfs/content/storage.py#L494-L513), 820-824, 909-915, 1033-1036). + - The cache row key is built in four places. + - Records are validated separately by `import_*` and `_load`. + - The "what stopped being complete" sequence is copied in `evict`, `gc`, `drop_denied` and `verify`. +- **Scenario:** + - A new removal path that forgets `_cache_generation += 1` leaves `_prune_fetched` unrun. + - One that forgets `_cache_dirty` resurrects the row at restart. + - The D3 memo change has to update every copy of the completeness sequence. +- **Change:** `_drop_records(keys)` and `_set_record(key, target)` helpers that do all the bookkeeping; `_cache_row` / `_parse_cache_row`; shared validators; and `_newly_incomplete(targets, action)`. Later, split a `CacheRecords` class out of the ~900-line `Storage`. + +### CT5. msgpack is hand-parsed in three modules (M, med) +- **Where:** + - [dag.py:155-236](rnfs/content/dag.py#L155-L236) parses over bytes. + - [archive.py:56-138](rnfs/content/archive.py#L56-L138) parses over a stream. + - [network/protocol.py:277-310](rnfs/network/protocol.py#L277-L310) parses responses. + - `archive_size` ([archive.py:47-53](rnfs/content/archive.py#L47-L53)) hand-computes header lengths apart from the encoder. +- **Change:** a small `rnfs/msgpack_headers.py` holding the type tables plus `bin_header_size(n)` and `array_header_size(n)`. Share only the tables; leave `dag._decode`'s validation alone. Add a test that `archive_size == len(export bytes)` at the 255/256 and 65535/65536 boundaries. + +### CT6. Smaller content items +- **Untyped results:** operation results are untyped dicts that are re-mapped by hand at three layers (`Storage.gc`, `verify` and `stat`; `node.py:836-849`; `objects.py`). Use internal frozen dataclasses and keep the JSON form unchanged. (M, low) +- **builder.py has two jobs:** about 270 lines of it ([builder.py:238-510](rnfs/content/builder.py#L238-L510)) are a secure local-filesystem layer. Move it to `content/localfs.py`. Two unrelated functions are both called `list_directory` (builder's and walker's). (S–M, low) +- **Tests reach into internals:** test_storage patches `_reach`, `_children`, `_complete_sets` and others. D3 will break those tests or quietly make them test nothing. The security tests call CPython's private `RLock._is_owned()`. Add an injectable block-read hook and a lock wrapper with `held_by_current_thread()`. (S, low) +- **Wrong docstrings:** "No RNS dependency" is false for builder, walker, storage, dag, edit, shared_folder and archive, which load RNS through `RNS.vendor.umsgpack` or `rnfs.log`. Only cid, blockstore and chunker are RNS-free. Fix the docstrings and add a test that those three don't import RNS. (S, low) + +--- + +## Fetcher and network (NW): `fetcher.py`, `network/`, `mesh.py` + +### NW1. `Fetch` does too many jobs (L, med): pair with plan A1–A4 +- **Where:** [fetcher.py:357-1121](rnfs/fetcher.py#L357-L1121), 46 methods. One class handles: + - tree walking; + - DAG byte-budget bookkeeping; + - provider scheduling and failure rules; + - lookups; + - storage. +- **State is passed around loosely:** + - Per-download state goes around as 5–6 parameters (`_handle` takes 9). `in_flight` is both a parameter and `self._in_flight` (:806). + - The range budget reaches `_count_dag` and `_room` through `self._range`, set and cleared around `ensure()` (:644-648). + - `room` is a positional `(bound, quota, budget)` tuple whose parts may be None. + - Events are positional string tuples (`("result", node_id, cids, result, sent_at, rate)`). + - The module docstring says only the fetch thread changes state, but `progress()` writes `_progress_at` from request threads (:951-953). +- **Scenario:** a new limit has to be added to the tuple, `_reserve`, `_fits`, `_count_dag` and `dag_limit` together. A new read path that forgets `_range` silently skips the range budget. A1/A4 pipelining will add more state to this same class. +- **Change:** extract a pure `_DagBudget` (absorbing `_RangeBudget`, passed to `ensure` explicitly), a `_Providers` policy object (failure, busy and pacing rules and ranking, with a clock only) and a `_Download` state object. Use NamedTuple events. The tests already call about 15 of these private methods, which shows where the seams are. + +### NW2. `Response` is shared mutable state with three locking styles (M, high: security-relevant) +- **Where:** `Response` ([server.py:233-319](rnfs/network/server.py#L233-L319)) has about 25 mutable fields, touched by RNS request threads, the RNS packet thread, the monitor thread and link-closed callbacks. + - `idle` is set under `ResponsePools.lock` in some places and with no lock in `_check_progress` (:1350). + - `finished`, `settled`, `charged` and `concluded` use `RequestServer.lock`. + - `size`, `deadline`, `due`, `grace`, `untracked` and `resource` use no lock. + - `watched` is changed without the lock, discarded under it, and read without it. +- **Why:** correctness rests on idempotence flags and the order fields are assigned in, and neither is written down. The next person to touch upload pacing has to reconstruct the model from scratch. +- **Change:** first, document which thread or lock owns each field. Then give `Response` its own lock and move its state changes (attach, conclude, finish, settle) into methods on it. + +### NW3. `_handle_blocks` is 111 lines with about 9 return points (M, med) +- **Where:** [server.py:895-1005](rnfs/network/server.py#L895-L1005). The steps run in this order: cooldown, airtime, slot, pacing, grace, pool reserve, pace wait, budget, select and encode, admit, pacer, traffic, handover. One `handed_over` flag and a `finally` free the slots exactly once. +- **Change:** split it into `_admit_request`, `_reserve_slot`, `_build_body` and `_hand_over`, each returning a refusal or a value, and keep the single `finally` owner. + +### NW4. Per-request slot bookkeeping is copied into four handlers (S, low) +- **Where:** `start_request` / `try` / `finally: finish_request` is written out at [server.py:841-849](rnfs/network/server.py#L841-L849), 857-867, 873-879 and 887-893. +- **Scenario:** a new handler that forgets the `finally` leaks one of a link's two slots (`MAX_REQUESTS_PER_LINK`), and that link is then refused forever. +- **Change:** a `_with_slot(link_id, fn)` helper, or have `_guarded` count the slot for every path other than /blocks. + +### NW5. The /blocks size rules are copied between client and server (S, low) +- **Where:** `server.blocks_budget`, `rates.requester_max_bytes`, [fetcher.py:923-925](rnfs/fetcher.py#L923-L925) and `protocol.max_response_size`. `fetcher._pace` also hard-codes the server's default cap of half the rate estimate, copied from `config.effective_max_upload_rate`. +- **Scenario:** changing the server's cap leaves the client's batching estimates silently wrong. The wire is unaffected either way. +- **Change:** shared `rates.response_capacity()` and `default_upload_cap()` functions, called by both ends. + +### NW6. Smaller network items +- **`LinkPool.get` timeout:** a waiting thread gives up after the magic `path_timeout + 120` ([links.py:237](rnfs/network/links.py#L237)). The opener can take two path timeouts plus establishment plus grace, so with a large LoRa `path_timeout` waiting threads report "unreachable" while the opener is still connecting. Add one `open_budget()` and one path-wait helper; the poll loop is also duplicated in `announces.read_mode`. (S, low) +- **Feature response shapes:** these are enforced only by a docstring. `_feature_handler` returns whatever the handler gives, and a feature that returns a dict would be refused by every 0.1.4+ peer. Run `protocol.response_refusal` on the server's own feature responses, and list `/name` in protocol.py's table. (S, low) +- **RNS internals:** these are read directly at many call sites. For example, `getattr(link, "attached_interface", None)` appears six times in server.py, client.py and guards.py. A thin accessor module registered in `test_rns_compat.INTERNALS` would turn an RNS rename into a one-place edit. (M, low) +- **Dead code that is a trap:** `AirtimePolicy.spend` and `TokenBucket.take` ([mesh.py:241-248](rnfs/mesh.py#L241-L248), 311-313) are used only by tests. They predate the reserve/charge/refund model and bypass its commitment bookkeeping. Delete them and move their tests onto `reserve` / `charge`. (S, low) + +--- + +## DHT, names and publisher (DH) + +### DH1. names.py is five modules in one, and the floor is a hand-synced bundle (L, med): do the floor part before plan E1 +- **Where:** [names.py](rnfs/names.py) holds the record codec (76-167), the legacy JSON import (172-292), `RnsNetwork` (298-357), the announce handler and `NameManager` (384-1346). `NameManager.__init__` sets about 35 attributes, 9 of them for the floor (422-432). +- **The floor dicts:** `_floor` and `_floor_used` must keep the same keys, and four sites maintain that by hand. `_save_floor` indexes `_floor_used[name]` for every floor name (:1300), so one missed update is a KeyError on the floor timer's thread. +- **Duplicated refresh:** `publish` and `resolve` each have their own "refresh, then ask the keepers" block (634-641, 814-824). E1 will add caching to `resolve` and leave `publish`'s copy behind. +- **Lock scope:** + - `_offer` does a durable save and calls listeners while holding the names lock, on RNS's announce thread (994-1002). + - Public methods re-enter under the RLock. + - The lock order is written only in comments. +- **Change:** + 1. Extract `_refresh_name(name, ask_keepers)` *before* E1. + 2. Extract a `_NameFloor` class with one value per name, `(sequence, resolved, used)`, which owns trim, load, save and the timer, and a `_RecordTable` class. + 3. Move the codec, `parse_name` and `check_alias` to `name_record.py`, and the JSON importers to `rnfs/legacy/`. Keep the `rnfs.names` re-exports, because statedb's LEGACY strings point at them. + 4. Queue listener calls under the lock and make them after releasing it. Write the lock order down in one place. + +### DH2. ProviderStore keeps about 10 indexes in step by hand, with no consistency check (M, low) +- **Where:** + - [providers.py:214-236](rnfs/dht/providers.py#L214-L236) holds `_records`, `entry.keys`, `entry.main`, `_count`, `_main_count`, `_pool_count`, `_pooled`, `_holders`, `_holder_index`, `_most_pooled` and the `_newest` heap. They change in five methods. + - `_load` (787-870) repeats `store`'s insert and an inline copy of `_add_main`. + - Tests check only `record_count()` and `publisher_count()`. +- **Scenario:** plan item S5, or a new limit, adds a counter in `_add_main` but not in `_load`. The counts drift after a restart and only show up later as wrong eviction choices. +- **Change:** one `_insert(key, publisher, record)` used by `store` and `_load`. Add a `_check_consistency()` that recomputes every index from `_records`, call it in test_providers' `tearDown`, and add one randomized store/withdraw/expire/reload sequence test. + +### DH3. `Publisher.publish_due` is 127 lines in five phases under four lock windows (M, med) +- **Where:** [publisher.py:570-696](rnfs/publisher.py#L570-L696). About 8 locals cross the phases, and the "still provided?" check is repeated after each re-lock. +- **Wrong type hints:** `_provided: frozenset[Cid]` and `_published: dict[Cid, …]` also hold `DhtKey`s, which is why `provided()` has to filter. A `cid.binary` added in `refresh` would break only on nodes running keepers. +- **Change:** a `_Round` dataclass with `_plan_round`, `_send_to` and `_settle_round` methods; test_publisher already covers the behaviour. Add a `ProvidedItem = Cid | DhtKey` alias. + +### DH4. `Lookup._iterate` is 125 lines over nine interlocking local sets (M, med) +- **Where:** + - [lookup.py:211-335](rnfs/dht/lookup.py#L211-L335). Retries move in and out of `in_flight` and launch on top of alpha, so the comment "at most alpha run at a time" (:359) is wrong. + - The constructor takes 17 parameters, four of which are callables from one `RoutingTable` ([node.py:582-587](rnfs/node.py#L582-L587)). +- **Change:** a `_Run` object with `fill()`, `retry_due()`, `merge(reply)` and `done()`, and a `NodeHealth` Protocol in place of the four callables. Also fix the swallowed callback exceptions (XC4). + +### DH5. Smaller DHT and names items +- **State-row validators** are duplicated with slightly different rules, all S and low: + - Time: `providers._number` accepts ±inf, `publisher._time` rejects it, and `names._held_row` checks only the type. + - Integers: denylist.py:160 accepts `bool` as an int. + - **Change:** shared `finite_time`, `int_column` and `id_column` helpers in statedb. Keep the providers' inf behaviour explicitly if it matters. +- **Legacy importers** sit next to live store code in providers, names, publisher and pin_names. `providers.read_records` re-implements `statedb.read_json`, and the `shape` parameter is unused. Move them to `rnfs/legacy/`; test_upgrade_014 already verifies every LEGACY string. (M, low) +- **The `names.keepers` collaborator contract** is a comment mirrored by `FakeKeepers`, so renaming `KeepersFeature.always_ask` passes test_names. This is covered by XC1 step 3. + +--- + +## Node, API and CLI (ND) + +### ND1. `RnfsNode` does too many jobs; split out the remote-pin queue and fetched targets (M–L, med) +- **Where:** [node.py:303-1639](rnfs/node.py#L303-L1639), 88 methods and 48 attributes. Besides wiring and the facades, it runs the DHT join, private-mode membership, the maintenance scheduler and deny-list application. It also owns two persisted stores: + - a remote-pin queue with its own workers and quota checks (89-109, 446-451, 1296-1502; about 230 lines); + - the fetched-targets store (75-86, 434-445, 872-922). +- **Why:** both stores share `self._lock` ([node.py:433](rnfs/node.py#L433)) with nothing else in common, and storage locks are taken under it. node.py is the most-changed file in the repo. +- **Change:** a `RemotePinQueue` class (state, workers, save and load, importer, `close()`) and a `FetchedTargets` store, each owning its lock and table, the way `PinNames` and `SharedFolder` already do. Once XC5's store registry exists, both plug into it. + +### ND2. node_content.py and gateway_formats.py are free-function "mixins" with import cycles (S–M, low) +- **node_content.py:** + - Its functions take the node and reach into its privates (`_note_fetched`, `_fetching`, `_check_file`). + - It lazily imports from rnfs.node, and `rnfs.api.parse_bool`, while api.py imports it. + - `add_options` ([node_content.py:222-255](rnfs/node_content.py#L222-L255)) is HTTP parameter parsing living in the node layer. +- **gateway_formats.py** lazily imports gateway.py for the same circular reason. +- **Scenario:** renaming `_fetching` breaks a module that a grep for `self._fetching` won't find. +- **Change:** fold the node-side functions into a `ContentOps` component the node owns, move `add_options` into api.py and the output-file helpers into fsutil, and pass gateway_formats what it needs. + +### ND3. The facade doesn't check its own inputs (S, low) +- **Where:** + - The `dir_rm` rule exists three times with three messages ([api.py:1000](rnfs/api.py#L1000), [embedded.py:189-190](rnfs/embedded.py#L189-L190), edit.py:69-70). edit.py's copy only runs after the whole DAG is fetched. + - Name nicknames are resolved by both callers before `node.name_resolve`. + - The "pin is a bool" check exists only in the API. +- **Change:** do these checks in `RnfsNode`'s facade methods, before any fetch, and drop the copies. A third caller then behaves the same. + +### ND4. The API surface is written out in 6+ places and the client covers only part of it (M, low) +- **Route table:** `ApiHandler.ROUTES` ([api.py:611-640](rnfs/api.py#L611-L640)) dispatches by name through `getattr`, so a typo fails only when that route is requested. The routes are restated in api.py's docstring and design.md, which match today but aren't checked. +- **Stale handler docstrings:** they list old parameter sets (`api_add` :965, `api_add_stream` :972, `api_get` :981). +- **Partial client:** `ApiConnection` has no typed methods for dir set/rm, remote pin, gc, verify, providers, ping, name rm/export/import, export or import. So remote.py, cli.py (18 sites) and deny.py hand-write paths and parameters. +- **Add options are encoded three times:** in `cli.add_options`, in `apiclient.add_path` / `add_stream`, and in `node_content.add_options` on the server, each handling the JSON-vs-query differences its own way. +- **Scenario:** a new `/api/add` option needs edits in 5+ files, and the CLI's copy is the one most likely to drift. +- **Change:** complete `ApiConnection` and route cli.py, remote.py and the feature CLIs through it. Add a test that every ROUTES name resolves and that the docstring table equals ROUTES. Delete the parameter lists from per-handler docstrings. + +### ND5. statedb steps can't be protected or written in Python (S, low) +See TS2. The fix belongs in [statedb.py](rnfs/statedb.py): a golden digest test of the released steps, and `(label, callable(tx))` steps. Optionally record the version that added each step. + +### ND6. Smaller node, API and CLI items +- **Missing attributes in `offline()` nodes:** the offline/test node built by `cls.__new__` ([node.py:367-379](rnfs/node.py#L367-L379)) never sets `identity`, `destination` or `paths`. So `getattr(node, "identity", None)` appears in node.py and node_content.py, and the features use `reticulum is None` as an implicit "offline" flag. Have `offline()` set every attribute explicitly and expose `node.online`. (S, low) +- **Config defaults:** + - The jobs default lives in jobs.py rather than an `effective_*` helper. + - `"~/.rnfs/"` appears twice in config.py, and cli.py:664 hard-codes "default 8789". + - Unknown override keys are checked in embedded.py rather than in `Config.load`. + - Use constants and one check. (S, low) +- **`winacl` and RNS:** winacl.py says "No RNS dependency, so the CLI can use it", but it lazily imports rnfs.log, which imports RNS ([winacl.py:388](rnfs/winacl.py#L388), 422). So on Windows a loose `rnfs.toml` loads RNS inside the CLI and `rnfs.Client`. Return the warnings to the caller instead. (S, low) +- **api.py doubles as the HTTP framework** for the gateway (`LocalServer`, `TimedHandler`, `host_allowed`, …), and keeps test-only aliases (`api.loggable`, `cli.printable`, `cli.read_token`). Split out `rnfs/httpserver.py`. cli.py:8 calls `rnfs.apiclient` "the library companion apps use", which contradicts design.md:593: `rnfs.Client` is the public library. (S, low) +- **`cli.build_parser` and `cli.run_daemon`** are 138 and 123 lines. They are low priority, because they're flat, but `run_daemon` is the place to make startup steps individually guarded, like XC5's `stop()`. (S, low) + +--- + +## Features (FT): `rnfs/features/` + +### FT1. lxmf.py (2188 lines) is seven modules, and its constructor runs before it is fully built (M, low): fix the constructor order now +- **Seams:** + - URI and field codec, which is the wire format (182-415); + - `MessageLog` persistence, the 0.1.4 import and validators (417-698); + - `_RoundRobin` (746-799); + - `LxmfFeature` (804-1569); + - API handlers (1574-1647); + - CLI rendering (1649-1780); + - `LxmfNetwork`, the RNS/LXMF glue that monkey-patches LXMRouter (1782-2188). +- **Constructor order:** `LxmfFeature.__init__` calls `_release_unrecorded()` at [lxmf.py:832](rnfs/features/lxmf.py#L832), which can go on to `_record_kept()` and the inbox. That happens *before* `_resend`, `_auto_queue`, `clock`, the rate-limit state, `_stop` and `_threads` are assigned (836-847). It works today only because that path doesn't touch them. A change there would raise AttributeError only on the rarely tested unclean-inbox path. +- **Change:** now, assign every attribute before calling any method (S). Later, make lxmf a package: + - `codec.py`, the frozen wire format with its own known-answer tests (TS3); + - `store.py`; + - `feature.py`; + - `network.py`; + - `cli.py`; + - an `__init__` that re-exports `FEATURE` and the names statedb's LEGACY strings use. + + Replace the positional tuples (`_resend`, `_outbound`, `_held`, whose source is re-sliced as `p[ID_LENGTH:2*ID_LENGTH]` eight times) with small dataclasses, so the tests no longer need `__new__`. + +### FT2. keepers: `ask_keepers` is 138 lines of closures over shared mutable state (M, med) +- **Where:** + - [keepers.py:702-839](rnfs/features/keepers.py#L702-L839) has six nested closures sharing `found`, `unknown`, `used`, `results` and `answered`. It also starts unjoined per-keeper threads (:816). + - `name_subcommands` (908-914) uses argparse's private `_actions` and `_SubParsersAction`, and follow.py depends on it. +- **Change:** a small `_KeeperQuery` class (`next_batch`, `ask`, `settled`), and have the CLI's `command` helper hand out named subparsers. Moving the record, store and query logic into core is XC1. + +### FT3. jobs: the state machine is implicit, and a kind is dispatched in five places (M, med) +- **Where:** + - Transitions happen in `Job.begin_run`, `_load`, `_run`, `_finish` and `cancel`. `cancel` repeats `_finish`'s release, `ended.set`, cap and save sequence inline ([jobs.py:643-655](rnfs/features/jobs.py#L643-L655)). + - `job.state` is guarded by two different locks. + - Adding a kind touches `KINDS`, `check_job_params`, `_execute`, `api_start` and `print_result`. + - About 275 lines of CLI rendering, and `StreamProgress` (used by `rnfs cat`), live in the same file. +- **Change:** one `_transition(job, state, …)` with an allowed-transitions table under one lock, a per-kind spec (validator, executor, printer), and the formatting moved to a CLI module. + +### FT4. Features depend on each other through string lookups and duck typing (S–M, low) +- **Where:** + - `getattr(self.node, "features", {}).get(...)` plus `hasattr(x, "lookup")` in lxmf (4×) and jobs. + - follow requires keepers through a `KeyError`, while names treats keepers as optional. + - `lxmf._recipient` duplicates `contacts.resolve_id` down to the error wording, and `lxmf._contact_target` is a variant of `contacts.expand_target`. +- **Scenario:** renaming `contacts.lookup` silently disables LXMF contact trust and sending by nickname, because `hasattr` returns False and nothing fails. +- **Change:** declare dependencies (`requires = ("contacts",)`, `optional = (...)`) and check them at load time. Call `contacts.resolve_id` instead of copying it. + +### FT5. Smaller feature items +- **Copy-pasted schedulers:** follow and deny have the same `run_due` and `_loop` code, and duplicate `STALE_ANNOUNCES`, `FAR_FUTURE` and a magic `min(delay, 60.0)` ([follow.py:335-377](rnfs/features/follow.py#L335-L377), [deny.py:190-231](rnfs/features/deny.py#L190-L231)). Add a `_common.DueScheduler`. (S, low) +- **Name collisions are handled three ways:** + - Duplicate feature API routes raise, but a core route silently shadows a feature route ([api.py:741-742](rnfs/api.py#L741-L742)). + - A feature request path that collides with a core path is silently dropped, and two features with the same path silently overwrite each other ([server.py:704-706](rnfs/network/server.py#L704-L706), [node.py:299-300](rnfs/node.py#L299-L300)). + + Validate all three namespaces at load time with one error style. Add lifecycle tests with `DummyFeature` (`requests()`, save/close order, `start()` failure isolation versus `__init__` failure aborting the node). (S, low) +- **Uneven escaping:** daemon-supplied fields are printed raw in jobs.py (962, 1017-1028) and peers.py (216, 224-228), although test_feature_output's docstring says feature commands escape what the daemon sends. Extend that test to jobs, deny, lxmf and `format_stats`. (S, low) +- **Dead or stale code:** + - `_RoundRobin.clear` is never called. + - lxmf.py:1336 has a redundant import. + - The lxmf docstring calls LXMF optional, but pyproject requires it. + - `_api_sync` calls `_require_network()` directly, unlike every other handler. + - jobs.py:313 does `getattr(config, "max_jobs", None)` on a field Config declares. + - contacts uses a `@staticmethod _log` where every other module uses a module-level one. + + (S, low) + +--- + +## Order of work + +These phases fit around the 0.1.6 plan's. + +1. **Safety nets first (S, low risk, about a week in total).** These make every later refactor safer, and none changes behaviour: + - TS1: run every `RNFS_E2E` test in CI. + - TS2 / ND5: the 0.1.5 state fixture and the released-schema digest. Do this before the plan's D1 adds a table. + - TS3: wire known-answer vectors. Do this before the plan's A1/A2/A4 fetcher work. + - TS4: report-only coverage. + - TS10: enable BLE001, E402 and RUF100 with per-file ignores. +2. **Quick correctness-adjacent fixes (S):** + - FT1: the constructor order. + - XC4 step 3: log swallowed exceptions in lookup and fetcher. + - NW4: the slot helper. + - XC5: guarded `stop()` steps and the store registry. + - XC6: drop the redundant wall-clock bound in test_publisher. +3. **Refactors that the 0.1.6 plan items touch.** Do each just before, or as part of, the plan item: + + | Before plan item | Do | + |---|---| + | D3 (storage walks under the lock) | CT1, CT4's `_newly_incomplete`, CT6's test hooks | + | E1 (gateway name-resolve cache) | DH1 step 1 (`_refresh_name`) and the `_NameFloor` extraction | + | A1, A2, A4 (fetch pipelining) | NW1's `_DagBudget`, typed events and the `progress()` thread fix | + | B (block write durability) | XC8's `blockstore.matches` helper | + | S5 (provider store) | DH2's `_insert` and consistency check | +4. **Shared helpers (M), one at a time, with no behaviour change:** + - XC3: `rnfs/ids.py`. + - XC2: `statedb.RowStore`, starting with the features. + - XC7: the `tests/fakes/` package. + - XC4 steps 1–2: error types and one status table. + - CT5: msgpack headers. + - NW5: shared size rules. +5. **Structural splits (L), after 0.1.6, each in its own release:** + - ND1: remote pins and fetched targets out of `RnfsNode`. + - FT1: lxmf as a package. + - DH1: the rest of names.py. + - NW2 / NW3: the `Response` lock and `_handle_blocks`. + - XC1: the feature boundary. This is the one that decides how easily features can move to companion apps. + +## Looked at and not recommended +- **Removing the 0.1.1 temp-file cleanup** ([names.py:1182-1197](rnfs/names.py#L1182-L1197)). It is old, but those files can hold a private key, and an install upgraded straight from 0.1.1 would keep them. It's a few lines, so keep it. +- **Runtime registration of pin-source prefixes:** see XC1. It would let a node started without a feature erase that feature's pins. +- **Merging `embedded._Backend` and `remote._Backend`.** They have 39 parallel methods each, but that is deliberate (in-process versus HTTP behind `interface.Rnfs`), and test_public runs the whole public interface over both. +- **Splitting `cli.build_parser`:** it is long but flat, a declarative table that is easy to read. +- **Restructuring `dag._decode`'s validation** while sharing msgpack tables (CT5): the plan already marks it as needing fuzzing first. + +## What is already done well +- **Wire formats:** each has its encoder and strict decoder side by side, documented against its spec section. Decoders re-encode to reject non-canonical input. `protocol.response_refusal` checks a response's header only. +- **Injected dependencies:** clocks, networks, RNG and a `put` callback are injected through most of the core. routing, lookup, providers, cid, blockstore and chunker have no RNS dependency, so most behaviour is unit-tested without Reticulum. +- **Drift guards:** + - `test_fake_contract.py` checks FakeNode against RnfsNode. + - `test_rns_compat.py`'s `INTERNALS` lists every RNS and LXMF internal rnfs relies on. + - `test_packaging.py` checks the locks, the README release links and that install.sh agrees with `contrib/rnfs.service`. + - The 0.1.4 upgrade fixture is written by 0.1.4's own code. +- **statedb:** upgrades are one-way, a newer schema refuses to start, durability is chosen per transaction and migrations are crash-safe. +- **Policy classes:** the server's (`ResponsePools`, `IncomingLinks`, `Cooldowns`, `UploadPacer`, `TokenBucket`, `AirtimePolicy`) are pure and take injected clocks. Request dispatch is a handler table, not an if/elif chain. +- **Docs and comments:** they match the code. Comments explain *why*, including the attack each rule defends against. +- **Supply chain:** builds are hash-locked, actions are SHA-pinned, and releases are signed with provenance. diff --git a/packaging/build-backend.lock b/packaging/build-backend.lock index cfd08af..6ef2b17 100644 --- a/packaging/build-backend.lock +++ b/packaging/build-backend.lock @@ -1,6 +1,6 @@ # rnfs's build backend (setuptools), with the sha256 of every file PyPI has for it. Generated by # packaging/update-lock.py; don't edit by hand: -# python3 -I packaging/update-lock.py --build pyinstaller==6.22.3 +# python3 -I packaging/update-lock.py --build setuptools==84.0.0 build==1.6.1 twine==7.0.0 pyinstaller==6.22.3 ruff==0.17.0 more-itertools==11.1.0 # Install with: pip install --require-hashes --only-binary=:all: -r packaging/build-backend.lock # then build with: pip wheel --no-build-isolation --no-deps . setuptools==84.0.0 \ diff --git a/packaging/build-deb.sh b/packaging/build-deb.sh index d9bcbac..d8817ac 100755 --- a/packaging/build-deb.sh +++ b/packaging/build-deb.sh @@ -197,7 +197,8 @@ cat > "$ROOT/DEBIAN/postinst" <<'EOF' set -e if [ "$1" = configure ]; then # Byte-compile for the installed python3, so starting rnfs doesn't recompile on every run - python3 -m compileall -q -j 0 /opt/rnfs/lib >/dev/null 2>&1 || true + # -I: as root, so no module from the current directory or PYTHON* variables may be used + python3 -I -m compileall -q -j 0 /opt/rnfs/lib >/dev/null 2>&1 || true fi exit 0 EOF diff --git a/packaging/build-tools.lock b/packaging/build-tools.lock index c379c6c..8da841d 100644 --- a/packaging/build-tools.lock +++ b/packaging/build-tools.lock @@ -1,7 +1,7 @@ -# The tools the workflows build, check and lint with (build, twine, PyInstaller, setuptools, ruff) and their -# dependencies, for CPython 3.11, 3.12, 3.13, 3.14 on Linux x86_64 and Windows x64, with the sha256 of every -# file PyPI has for each. Generated by packaging/update-lock.py; don't edit by hand: -# python3 -I packaging/update-lock.py --build pyinstaller==6.22.3 +# The tools the workflows build, check, lint and measure coverage with (build, twine, PyInstaller, setuptools, +# ruff, coverage) and their dependencies, for CPython 3.11, 3.12, 3.13, 3.14 on Linux x86_64 and Windows x64, with the +# sha256 of every file PyPI has for each. Generated by packaging/update-lock.py; don't edit by hand: +# python3 -I packaging/update-lock.py --build setuptools==84.0.0 build==1.6.1 twine==7.0.0 pyinstaller==6.22.3 ruff==0.17.0 more-itertools==11.1.0 # Install with: pip install --require-hashes --only-binary=:all: -r packaging/build-tools.lock altgraph==0.17.5 \ --hash=sha256:c87b395dd12fabde9c99573a9749d67da8d29ef9de0125c7f536699b4a9bc9e7 \ @@ -292,6 +292,128 @@ charset-normalizer==3.5.2 \ colorama==0.4.6 ; sys_platform == "win32" \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 +coverage==7.16.2 \ + --hash=sha256:00d3eb96e9988c45f50cccd1f1496571ac5c1f91386ac02c4d55516eeda19a24 \ + --hash=sha256:01c6908bc613b420c26c818fe948e1b97dfd041a53c98b01c63bd8321f5c9aae \ + --hash=sha256:066429634299e14dd2d511e1e85f8f9cecc500781f6b41907c0dd6f1baea7e63 \ + --hash=sha256:0993d0e90858c03943d3cb152e068a20dd4707924deec84dd2230261baae3b1b \ + --hash=sha256:0dcbcfcc059117284c603ff8cb61a65872512882f84a8cf0339241f7f7c2f148 \ + --hash=sha256:0fd7a86fdda7cb6d616d178654bd0ad6bc0f3f33c2e478aa598500a1a9e34eda \ + --hash=sha256:11d28e9123a9156cb405d8d27b44256c9a58fb5decc2073a8f17862057e3aa0f \ + --hash=sha256:11e597173af1dc33d5f8a7332ada544199269a223af1ee1770ddd5e245ad0fe8 \ + --hash=sha256:126d1af8804d7224421fe991ff65d3ce649081560df7a98b1a5ffff07f9923bd \ + --hash=sha256:14253fc7bb15749b849795a06f5d3b6d8bc3fb8a4b5ddc341faf7a89dce205fc \ + --hash=sha256:152877cdc8a07264882cfcd503ba56a3ef6cba56a70e8c70f6eb8ffd7384789a \ + --hash=sha256:17228fbca0f22976f797be94e975dcd237799c657d49551c7de1e0654d1202e9 \ + --hash=sha256:191803c4996b499fcd78c2ad5e5f767dcc53cb4dc6de6d6a741b443a1821ef02 \ + --hash=sha256:1a37c6e478cf687e1aa30a593d19c92c02fad9d122b51ab73f51b8dc7a0c0fc9 \ + --hash=sha256:1c569a9fd25505f1cd6bea90588818f90373ce90e2632e2cacf19ddbd6e14fdb \ + --hash=sha256:1d56e4d21c56d2046447733f8b118409597db48c01efe898ee9ac24e858ec2d6 \ + --hash=sha256:1d5d0e3b660506fb84f995814e3118a21efdc0c8eb80127da1be627d90093c17 \ + --hash=sha256:1f15254427c9b33eedac4f198eaf9e356eb4f6214551afb43da6194a2c088ad7 \ + --hash=sha256:218d742afca2b5ad5ca759e93eddedfbcc6eadf8322f080dcefc40b7bd4e2d48 \ + --hash=sha256:22957cef43ce038641de78ba995de7568d2d6a37c6ddbf7fa0fd7d1ae2344d91 \ + --hash=sha256:23219888477edd736b6fcaec1272d47d93b926e999641ffea7e53a1738e70b2b \ + --hash=sha256:251aed777c47c77aba047096d4542889db089227655711dfc2b9c54ef0e15e35 \ + --hash=sha256:28ff850182a67d117990fa2ce5ea1032836d8c9630dae867e8bdd3bff4533b79 \ + --hash=sha256:29309ccc86b7f33df7db12813c299f215bbbc470ed6292d0bedd63ffae1ebf64 \ + --hash=sha256:2aca0bdfa9e91621d5b09d815357bf63def4fc0e9cb66da67bf2cf93f3b1a6f5 \ + --hash=sha256:30c1b65d529e46569899fadca59e4a87c1faf2886923f1307ba61e654d4f3c20 \ + --hash=sha256:35f37886699cb9abd29958247d718628d5bc6f39e623dff66a09e546c42a7e03 \ + --hash=sha256:382d3346d56b0eec1b793d53a4c88799c8053f516aa3a8d7c44315696954bacf \ + --hash=sha256:396bb16e04ce04efbb3df91456ae4e3da918e69ecdf67fb711b0a0fdf35ccce0 \ + --hash=sha256:3e7f99698ba3a7d13988bdd984b7ebf13af4dbe2166dc8502eef90d77603b0a4 \ + --hash=sha256:3e861f1071dcc2fec1e88bef0920f6b1eaa66a143555b4f8ab79ba2b0f30ef55 \ + --hash=sha256:3f43bac1856ba269b905302778d4df433d6006489a192174ad77ac528e395032 \ + --hash=sha256:40c0f00899fe6181ae7f434ceb200e51f5ee4b8ed10e3b5f0b605f0cae15da87 \ + --hash=sha256:414c26dfdb96aac2d570a54e03008f001e32eb2d413705365503648c6bd361d8 \ + --hash=sha256:4358b9c8c0125b460407f3017c6cce8156e904b32772c5630d27112f52bdbfe5 \ + --hash=sha256:444889f7f66b74e4455c0a97e0e166dd41177f1dca8c0239a47cff25e05ba7e1 \ + --hash=sha256:44f21e407b278efdfc1ee5e481e00518bd1d500310a30a5fbf2bcbedfef4aaf0 \ + --hash=sha256:4cc4f73aa3fabc36e32046d6cd2971405948d8a903636508a3d3b2f9128b3a95 \ + --hash=sha256:4dbbd1155ca46e6e0b6b89d204428c56ef6a459af21333f365d135a2820e5a09 \ + --hash=sha256:4ee546b9e4872ffa194bf07ac87bfa1202ebb824d0795dc1ef22f175545ca90a \ + --hash=sha256:5139009b5efd2194fc168ee9362f0e191ba612ef5d29242f9269c22f9b8f80c7 \ + --hash=sha256:5375ebd99038021b35e99dc88255022912c06565d316212f4a576e4b08d30f5d \ + --hash=sha256:5397e21a90dde0e9c6896b77ded8f0be26b66f8b22b33aed41f6043ed95d55e6 \ + --hash=sha256:57ff3783f99d75a1e81dd56a9737eb5665e6736a5d93258ba596b6dcad8fd05b \ + --hash=sha256:58d4a54c6ea672afef66d49be922a2c69826c5ae1a42a9cd94f0c9c2bacdf800 \ + --hash=sha256:59c3926585e1cd1f2190f4b2ac9014de1bbeaf0d5d0587b0dc6b0aa90d17896a \ + --hash=sha256:5a27b731c171e43dc8b5f32b76a5051dde2ec9b9366c87028f08a7088ebc2c7b \ + --hash=sha256:5b3146d2317c75f70df2509066d979dadd941f7021cdf9b5db4bcd8568258e25 \ + --hash=sha256:5dca0bb66b4c3d624ba047887bf70270030c150692d543cb501293dc38a9f4b5 \ + --hash=sha256:611a44e5229a59d7483ce830160e1a0e85f700562c7a5651c7c63fb8f4eb528c \ + --hash=sha256:648352b94507179d82637292e7ae8802508d95f78e2f00a705a50b6c48011681 \ + --hash=sha256:6a75180829efb8ae62b4aded25be6ddca1c888d138d2d82e21d93bfbd88f41cb \ + --hash=sha256:705e5af11d34647efdc170c7840b6857c81cf74be96419a553f237e68e62cb72 \ + --hash=sha256:723dcdab91357159b722935b500ee8abc0a66c8c432e1e9fabf4cc7598952de8 \ + --hash=sha256:724bd0f1e81856b35e59fc98cf7b4e544a3cb662e4e0864dca73d4326ee9d808 \ + --hash=sha256:732d950e51f3ba4fb6209c73250f3e8924fefca42953ee04a9e65d8c02414d7d \ + --hash=sha256:736fde09ea39646d11f8e3b76bd3425c075aa4dd45f24891970bb77c14ff20f5 \ + --hash=sha256:7a076277ca9f5750cc230f0f578ebd2620cec60255b25707361699fef6fb465c \ + --hash=sha256:7b3bce4a0d05401d70b7d0d5ca783e686bc9d30e81dbd7d980d532609bf809e4 \ + --hash=sha256:7b451c68218c150f616bc9649783ec8de76a59792c759b43aa0c9c0466a465e4 \ + --hash=sha256:7d0732c83746bc24123c581a85d9dd96b70ddb538c9076020aa1a041790361e9 \ + --hash=sha256:7ed238d227e23cc300c3d464babdaf9f6ddc740aa1b15a77ae96136e6a7c4516 \ + --hash=sha256:80d3f7b48d43ee8fc5e8707a8adb43d743a5a1a85256c25a24f9d6d0e2238fa6 \ + --hash=sha256:80e9fdb4c3d926b6ba721d4bf7435bdb869c3527ae7803290361d0ab73db13b6 \ + --hash=sha256:848893e1d361448c113dc2f0913503522a6f7be231d0e38333d2a22d9698a011 \ + --hash=sha256:893ea9cf86cb8d2546812ac93d973aaf2ee1fb45110a873b014214fd23e3725e \ + --hash=sha256:8afd9bf35cc6a1f22eb3634808fa8e0b91902459c5721ef2e4461dfe771d7f08 \ + --hash=sha256:8be099e979fc42559328a21828281b4578304191ae46ed4e80a407048a82eee6 \ + --hash=sha256:8e209591f7c41ae4a9171335cf6156afda0b21de73b02f73f5aa95b2d5fbb08d \ + --hash=sha256:8fc15cc8d0d06e873c00ef18e1372d605f9aaf3de27d8c24e50782e75bc8b843 \ + --hash=sha256:9174f0af24e5eff248b9dbfe76ec5275a3d19d37edbc2810543f12cf97347a34 \ + --hash=sha256:921415102a90637fcc2e3f169f61dad7699ecf690e8639fc21b813acbedc0967 \ + --hash=sha256:967d72c835d7a8cf0af99ec813a2d06e3db6df706402f1fe85b31b437645f495 \ + --hash=sha256:98d9c97f51b334b0adce7b964442a9af33c1a00c6ac856984cc5dc8d18f81c75 \ + --hash=sha256:99704f73721e23859112072d522076e11c31744fc96b5652e5dd2018aa4359f7 \ + --hash=sha256:9a75a4704ff640e46170042eec1f984385a121227c505d5a16ad8e495f452541 \ + --hash=sha256:9acc7f7ec4a1b5f89bd929fde5b8a714f6fafdc6cc18725413d510aa082b47ad \ + --hash=sha256:9c6afdd69218202bc1758c9a14b86b8cf1084f37ed2ca143e567a103772b16d1 \ + --hash=sha256:9cdf19874e0d247f32f03609200370343c3c7aa260b191d8c2bb251d36198283 \ + --hash=sha256:9e1d0ced76318bab499693ff25f64faa343415187cb2e4d7befdfdd391a1cf6a \ + --hash=sha256:9fd670ac43b709c575aefc25bf52d8a598a3bc5017bddfd0a179152ab06a2deb \ + --hash=sha256:a0f2285329dac10ab08f79cb11f5692c497018e6c7c511f95e6fd63a70b8f831 \ + --hash=sha256:a2fac6895eb299a2e52d7bbb8fb3903502b9da8d3f5309ceb16ec40c646b58ee \ + --hash=sha256:a336eec40e3520d369b8a6cdabb4f596e69a8b42927ca074aa1452fed943238a \ + --hash=sha256:a4624f80732f6b427ac58f1f59c577a0994a12e8174b5af6a027b4b58795d4c3 \ + --hash=sha256:a56ac4fa5a75c7e182e8f62600cfb4aff43c5ed7356a034f3557659c3bec1d90 \ + --hash=sha256:a678c0b6b22086ec2427359d22e37445d4a792f5fdbbc744112c7dade65cad02 \ + --hash=sha256:a740ea6f083c6db7b926534d159508f80ba275ab35e722522de0d18d0f56e55f \ + --hash=sha256:a90700f743e29aa3d75a6ff5f01953176a889c00e526194bc4d281731b88d99d \ + --hash=sha256:a9a638be322a8d76a41cdb17781c7f82aaee6a66493d8ffb7e2c09ee22423d99 \ + --hash=sha256:a9cd3de0a5bfe7b0e21ee10e1a14e3d61bf52efc88217ab1d95d6ace6970bd46 \ + --hash=sha256:aa62c85046473959c13ba9edca9dc90a77d5c1095b1ba313556314d77fe5b036 \ + --hash=sha256:aba5c63b7afdc749cc9eae943d5b868cba2b261a176378fa1c5a30bc8bc89982 \ + --hash=sha256:ac0f3b379c94acc2f7dce5f5f0b24d44fa1cc6a509717ef83dfee07450c2117c \ + --hash=sha256:af2a2a8c7c74de0559e0c368d94c8def9e16c58faaee33a0bf081057c4227e3b \ + --hash=sha256:af98ad5ed9d6daaca956201e00bb429a7eb2b080426686f70a20353e0f9839f5 \ + --hash=sha256:afdf43b72ef3876c1fe66423b91466e37877c9e81e8cec70542b7e8525b9d1b7 \ + --hash=sha256:b88841e654f09732804809e435b3e005a929ffd9998b872b7b213957b8759cb8 \ + --hash=sha256:bb2fc905bbf4e6b7f40806ea79e31515abf6349594cdf0adf27c4215f0463204 \ + --hash=sha256:bb4ffe96aa663cee727659db5a2afeb38c95f8677b747d447b90d6d4874ea2c5 \ + --hash=sha256:bc0b0ac781d489304b741269857f1f8338b7a26b1b89c06c0344658001ec0035 \ + --hash=sha256:bf1bd822ec4e387ed245bed0d71151582cf7be9e5309bc4145eefe36083d5878 \ + --hash=sha256:c19cd6d025c1673f22afcd22c7df8a662d779e05d8e3fa6820c22afb895b0206 \ + --hash=sha256:c3305c38a2fa21a4254f2ace7dd9ef5fc569c9a558b66e7017650b3d637fb95e \ + --hash=sha256:c85d54e7e8a2ca932fe8399301af9b8d5907ea2a455ffaff6e7d1208db83b943 \ + --hash=sha256:ca64d9f1f384f151b9511bec01126072acd2f313439f8ed015a22d8790aab6fa \ + --hash=sha256:cce2bc991293f15cc4084ca116827b5900c5f34e1a54dfe83f10ab5c43162eb7 \ + --hash=sha256:d6276d78f6fca7d0ac066d5da4165c5acd07829e8305c2cb900b738fb3a75a72 \ + --hash=sha256:d93db87adb6b1c1b408dce4763314b55d76a9f589e96783a84ac9e7689e48bdf \ + --hash=sha256:db5f8394e17f877a625b257f2ba0ce8e728a499c2c1579ad66220272cd3df510 \ + --hash=sha256:db76506aa5416081f3e8974ae0f7965c58ada0bb0ef7339ac86099588dbb20d3 \ + --hash=sha256:dba2edfb054f6d4a08df9d1637c39a5aa3865bca6617c13c86be21e45658a59c \ + --hash=sha256:dcf4bc2aab4e16b1c4c0c2005918f23a7dd5d7821ddae82caed9e3342dc2fcce \ + --hash=sha256:e1fa594c887365b69745f25a416806e61085dd07b94c9eae68a6e20730629b23 \ + --hash=sha256:e6c52d3307824ff93b39efd99e4185d557db40bd841452abfb32e5d9151ca162 \ + --hash=sha256:eb57acff4a74246ae513c142d4b36e18c389c3aed8661914a53f7cd0071031b2 \ + --hash=sha256:f80bd9f9633eafc73d0a913ba2645c96ba58bba1befc30590f7c0fbfde59d865 \ + --hash=sha256:f8475460aa33ee28ac896ab1156d0bb3b6c639f7f8383c2677d3359eb35f8205 \ + --hash=sha256:fb2bde05838fffae1a1bf75e5d411a6cac3e4e9bb97e6640fed8cd47888b33f0 \ + --hash=sha256:fb9d92ecfe2d5b494367c67f7446f8b75b68d8d0c8cf3bc3e6997478be25d9e2 \ + --hash=sha256:fd3d72233eb8b48acc94fa57d44e2d32ce8e7abed02882ccb6d855ccc4ed33ec cryptography==50.0.2 ; sys_platform == "linux" \ --hash=sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602 \ --hash=sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2 \ diff --git a/packaging/flatpak/generate-sources.py b/packaging/flatpak/generate-sources.py index 298f144..914bbbf 100755 --- a/packaging/flatpak/generate-sources.py +++ b/packaging/flatpak/generate-sources.py @@ -10,7 +10,7 @@ The requirements come from pyproject.toml (project.dependencies, plus the optional dependencies except development extras) at the exact versions in packaging/requirements.lock, and every wheel's sha256 must be one the lock lists, so the output only changes when the lock (or packaging/build-backend.lock, for -build-sources.json) does. The release +build-sources.json) does. The tests workflow re-runs this and fails if the result differs from what's committed. Needs pip >= 23 (for `--dry-run --report`) and network access. """ diff --git a/packaging/flatpak/io.github.afit21.rnfs.yml b/packaging/flatpak/io.github.afit21.rnfs.yml index 17a1184..306b42e 100644 --- a/packaging/flatpak/io.github.afit21.rnfs.yml +++ b/packaging/flatpak/io.github.afit21.rnfs.yml @@ -14,13 +14,18 @@ finish-args: # Reticulum's TCP/UDP/Auto interfaces, the local API (127.0.0.1:8788) and the gateway (127.0.0.1:8789). # The host's network namespace is shared, so a host rnsd's shared instance and host `rnfs` clients work too - --share=network - # rnfs reads files you add and writes files you get anywhere in your home, and keeps its state in - # ~/.rnfs and Reticulum's in ~/.reticulum, the same directories a pip-installed rnfs uses - - --filesystem=home - # Serial ports (/dev/ttyUSB*, /dev/ttyACM*) for RNode, KISS and serial interfaces. Flatpak has no - # serial-only permission, so this grants every device; remove it (flatpak override --user - # --nodevice=all io.github.afit21.rnfs) if you only use network interfaces - - --device=all + # rnfs keeps its state in ~/.rnfs and Reticulum's in ~/.reticulum, the same directories a pip-installed rnfs + # uses, and reads files you add and writes files you get in Downloads. Not the whole home: rnfs parses + # untrusted network input, and a compromised daemon that can write ~/.bashrc has left the sandbox. Grant + # other folders yourself, e.g. a Reticulum config in ~/.config/reticulum or a data disk: + # flatpak override --user --filesystem=~/.config/reticulum io.github.afit21.rnfs + # flatpak override --user --filesystem=/mnt/data io.github.afit21.rnfs + - --filesystem=~/.rnfs:create + - --filesystem=~/.reticulum:create + - --filesystem=xdg-download + # No devices by default. Serial ports (/dev/ttyUSB*, /dev/ttyACM*) for RNode, KISS and serial interfaces need + # every device, since Flatpak has no serial-only permission: + # flatpak override --user --device=all io.github.afit21.rnfs modules: # rns, lxmf, cryptography, ... as pinned PyPI wheels (generated by generate-sources.py) - python3-deps.json @@ -32,6 +37,8 @@ modules: - pip3 install --verbose --no-index --find-links="file://${PWD}" --prefix=${FLATPAK_DEST} --no-deps . - ${FLATPAK_DEST}/bin/rnfs --version sources: + # The checkout, without what isn't needed to build rnfs or is left by local builds, tests and lint (each + # .gitignore entry that isn't a pattern). Entries are paths from the top of the checkout, not patterns - type: dir path: ../.. skip: @@ -44,4 +51,11 @@ modules: - tests - packaging - __pycache__ + - rnfs.egg-info + - .venv + - venv + - .ruff_cache + - .hypothesis + - .coverage + - htmlcov - build-sources.json diff --git a/packaging/update-lock.py b/packaging/update-lock.py index 37bcc23..53aa21c 100644 --- a/packaging/update-lock.py +++ b/packaging/update-lock.py @@ -14,8 +14,8 @@ build-backend.lock just the build backend (setuptools), for `pip wheel --no-build-isolation` in install.sh, build-deb.sh and the tests. Its version must satisfy pyproject.toml's [build-system] requires (this script checks); raise that bound when updating it. - build-tools.lock what the workflows build, check and lint with: build, twine, PyInstaller, setuptools, - ruff and their dependencies, for Linux and Windows (platform-only dependencies carry a + build-tools.lock what the workflows build, check, lint and measure coverage with: build, twine, + PyInstaller, setuptools, ruff, coverage and their dependencies, for Linux and Windows (platform-only dependencies carry a sys_platform marker). Packages also in requirements.lock are pinned to its versions, so both locks can go into one environment (the PyInstaller one). @@ -51,7 +51,7 @@ # The build tools, resolved on this machine (pip evaluates environment markers for the running Python, so run # it on Linux with Python 3.12, the release workflow's), then completed with the dependencies other # platforms' markers select. Every file must have a wheel for each of these environments. -BUILD_TOOLS = ["setuptools", "build", "twine", "pyinstaller", "ruff"] +BUILD_TOOLS = ["setuptools", "build", "twine", "pyinstaller", "ruff", "coverage"] BUILD_PLATFORMS = { # sys_platform: (marker values, wheel platform tags accepted) "linux": ({"os_name": "posix", "platform_system": "Linux", "platform_machine": "x86_64"}, ("manylinux", "linux_x86_64")), @@ -69,9 +69,9 @@ # then build with: pip wheel --no-build-isolation --no-deps . """ TOOLS_HEADER = """\ -# The tools the workflows build, check and lint with (build, twine, PyInstaller, setuptools, ruff) and their -# dependencies, for CPython {pythons} on Linux x86_64 and Windows x64, with the sha256 of every -# file PyPI has for each. Generated by packaging/update-lock.py; don't edit by hand: +# The tools the workflows build, check, lint and measure coverage with (build, twine, PyInstaller, setuptools, +# ruff, coverage) and their dependencies, for CPython {pythons} on Linux x86_64 and Windows x64, with the +# sha256 of every file PyPI has for each. Generated by packaging/update-lock.py; don't edit by hand: # python3 -I packaging/update-lock.py --build {pins} # Install with: pip install --require-hashes --only-binary=:all: -r packaging/build-tools.lock """ diff --git a/pyproject.toml b/pyproject.toml index 4771e88..5418c34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,15 @@ version = {attr = "rnfs.__init__.__version__"} # not "rnfs": setuptools would f [tool.ruff] target-version = "py311" extend-exclude = ["packaging/flatpak"] +# The companion NomadNet pages are Python scripts named index.mu +extension = { mu = "python" } +extend-include = ["companion-nomadnet-sites/*/index.mu"] [tool.ruff.lint] -select = ["F", "E9", "B006", "B008", "B904"] # pyflakes, syntax errors, mutable/call defaults, raise-from +# pyflakes, syntax errors, mutable/call defaults, raise-from; broad excepts (each carries a noqa with its reason), +# imports not at the top (each carries a noqa), and noqa comments that suppress nothing +select = ["F", "E9", "B006", "B008", "B904", "BLE001", "E402", "RUF100"] + +[tool.ruff.lint.per-file-ignores] +# Tests catch broadly to report or clean up (threads, fakes, subprocesses) +"tests/**" = ["BLE001"] diff --git a/rnfs/__init__.py b/rnfs/__init__.py index ebd7e97..bbdf541 100644 --- a/rnfs/__init__.py +++ b/rnfs/__init__.py @@ -20,7 +20,7 @@ interface.py, objects.py, embedded.py, remote.py: the public API (rnfs.Node, rnfs.Client) """ -__version__ = "0.1.5" +__version__ = "0.1.6" API_STATUS = "provisional" # the public API (rnfs.Node, rnfs.Client) may change until rnfs 0.2 # Public name -> the module defining it, imported on first use (PEP 562) @@ -31,7 +31,8 @@ "Status", "Collected", "Verified", "Exported", "Imported", "DenyState")}, **{name: "rnfs.interface" for name in ("Rnfs", "Name", "Names", "Job", "Jobs", "Deny", "Watch", "File")}, **{name: "rnfs.errors" for name in ("RnfsError", "NotFound", "TooLarge", "InvalidDag", "StorageFull", "Busy", - "Denied", "Cancelled", "Closed")}, + "Denied", "Cancelled", "Closed", "BlockMismatch", "SealError", + "NotRecipient")}, **{name: "rnfs.apiclient" for name in ("ApiError", "DaemonNotRunning", "ConnectionFailed", "TokenError", "AuthFailed", "SettingsError", "BadRequest", "ServerError")}, } diff --git a/rnfs/api.py b/rnfs/api.py index 6e28c0b..c51753e 100644 --- a/rnfs/api.py +++ b/rnfs/api.py @@ -34,6 +34,40 @@ | POST | /api/export | JSON `target`, `output` | `export(target, output)` | | POST | /api/import | JSON `path` | `import_archive(path)` | +The features (rnfs.features, each's `api_routes()`) add these; `name` is a name or a name contact's nick: + +| Method | Path | Parameters | Response | +|---|---|---|---| +| GET | /api/jobs | | `{"jobs": [job]}`, oldest first | +| POST | /api/jobs | JSON `kind` (get, pin, export), `target`, `output`, `from`, `sealed_name`, `fallback_dht` | the job queued | +| GET | /api/jobs/get | `id`, `wait` (≤ 30 s) | the job, once it ends or after `wait` | +| POST | /api/jobs/cancel | JSON `id` | the job | +| POST | /api/jobs/clear | JSON `id` (optional) | `{"removed": count}` of the finished jobs forgotten | +| GET | /api/peers | | `{"peers": [...]}` | +| GET | /api/stats | | traffic totals, refusals and per-interface bytes | +| POST | /api/name/keep | JSON `name` | the kept name | +| POST | /api/name/unkeep | JSON `name` | `{}` | +| GET | /api/name/kept | | `{"kept": [{"name", "reasons", "sequence", "cid", "emitted"}]}` | +| POST | /api/name/follow | JSON `name` | the followed name | +| POST | /api/name/unfollow | JSON `name` | `{}` | +| GET | /api/name/following | | `{"following": [{"name", "pinned", "latest", "sequence", "state"}]}` | +| GET | /api/contacts | | `{"contacts": [{"nick", "kind", "id"}]}` | +| GET | /api/contacts/get | `nick` | the contact | +| POST | /api/contacts/add | JSON `nick`, `id`, `kind` (optional) | the contact | +| POST | /api/contacts/rm | JSON `nick` | `{}` | +| GET | /api/lxmf/address | | `{"address", "display_name", "propagation_node", "nearest_propagation_node"}` | +| POST | /api/lxmf/send | JSON `to`, `target`, `message`, `propagate`, `nearest` | `{"id", "hash", "to", "method"}` | +| GET | /api/lxmf/inbox | | `{"messages": [...]}` | +| POST | /api/lxmf/inbox/get | JSON `id`, `link`, `output` or `directory`, `use_sent_cid` | `{"results": [...]}` | +| POST | /api/lxmf/inbox/rm | JSON `id`, or `all` | `{"removed": count}` | +| GET | /api/lxmf/outbox | | `{"messages": [...]}` | +| POST | /api/lxmf/sync | JSON `nearest` | `{"propagation_node", "started"}` | +| GET | /api/deny | | `{"entries": [local entry], "lists": [{"source", "cid", "entries", "state"}]}` | +| POST | /api/deny/add | JSON `entries` | `{"added": [entries new to the local list]}` | +| POST | /api/deny/rm | JSON `entries` | `{"removed": [...]}` (404 if one isn't on the local list) | + +Both tables are checked against ApiHandler.ROUTES and the features' routes (tests/test_api.py). + GET parameters are in the query string. Request bodies may use Content-Length or chunked encoding. Watching names (rnfs.Client's names.watch): `GET /api/name/watch` is a long poll, level-triggered so a client loses @@ -57,8 +91,10 @@ (rnfs.apiclient, which the CLI uses) makes whatever listens on the port prove this (with a status request), so another local user listening there while the daemon is down learns neither the token nor a name key being imported. -- Errors map to statuses as design.md says: ValueError 400, NotFound 404, TooLarge 413, Denied 451 (the deny - list), InvalidDag 422, Busy 503, StorageFull 507, other errors 500. +- Errors map to statuses as design.md says (rnfs.errors.STATUSES): ValueError 400, NotFound 404, TooLarge 413, + Denied 451 (the deny list), InvalidDag 422, Busy 503, StorageFull 507, other errors 500. A 500 is logged as an + error, and a BlockMismatch (a 400, but a bug: the node handed the store a block that doesn't match) as a + warning, with its traceback in the debug log. Connections (both this API and the gateway, through LocalServer): - The request line, headers and any JSON body must arrive within request_timeout of connecting, else the @@ -80,13 +116,12 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import parse_qs, urlsplit -from rnfs import __version__, fsutil +from rnfs import __version__, fsutil, ids from rnfs import node_content # --- content: add options (pin, to, chunker) --- from rnfs.apiclient import API_HEADER, CHALLENGE_HEADER, PROOF_HEADER, api_proof -from rnfs.config import _HEX_ID # exactly 32 hex digits, as names.parse_name and identity lists are checked from rnfs.content.cid import Cid -from rnfs.errors import Busy, Denied, InvalidDag, NotFound, StorageFull, TooLarge -from rnfs.log import LOG_ERROR, log +from rnfs.errors import STATUSES, BlockMismatch, Busy +from rnfs.log import LOG_ERROR, LOG_WARNING, log MAX_JSON_BODY = 1024 * 1024 # JSON parameters are tiny; this only stops a runaway body filling memory MAX_KEY_BODY = 64 * 1024 # an RNS identity file is 64 bytes @@ -99,9 +134,8 @@ WATCH_GRACE = 60.0 # seconds a name stays watched after the last call naming it ends MAX_WATCHED = 1024 # names watched for API clients at most; the soonest to end go first -# Exception class → HTTP status, checked in order (design.md); anything else is 500 -ERROR_STATUSES = [(ValueError, 400), (NotFound, 404), (TooLarge, 413), (Denied, 451), (InvalidDag, 422), (Busy, 503), - (StorageFull, 507)] +# Exception class → HTTP status, checked in order (design.md); anything else is 500. The one table, rnfs.errors' +ERROR_STATUSES = STATUSES _feature_routes: dict | None = None @@ -162,11 +196,9 @@ def parse_cid(text) -> str: def parse_hex_id(text, what: str = "node ID") -> bytes: - """A 16-byte node ID or name written as 32 hex characters (spaces around allowed). Raises ValueError.""" - # Hex digits only: bytes.fromhex alone would skip whitespace between them (as names.parse_name checks) - if not isinstance(text, str) or not _HEX_ID.fullmatch(text.strip()): - raise ValueError(f"invalid {what} {text!r}: must be 32 hex characters") - return bytes.fromhex(text.strip()) + """A 16-byte node ID or name written as 32 hex characters (spaces around allowed, no ). + Raises ValueError.""" + return ids.parse_hex16(text, what) def parse_path(text) -> str: @@ -607,7 +639,8 @@ class ApiHandler(TimedHandler): server_version = "rnfs" sys_version = "" - # (method, path) → handler method name. GET routes never change pins, names or content + # (method, path) → handler method name, as the module docstring's table lists them (test_api checks both). + # GET routes never change pins, names or content ROUTES = { ("GET", "/api/status"): "api_status", ("GET", "/api/info"): "api_info", @@ -772,6 +805,9 @@ def dispatch(self): status = error_status(e) if status == 500: log(f"API {self.command} {loggable(url.path)} failed: {e!r}", LOG_ERROR) + elif isinstance(e, BlockMismatch): + # Answered 400 as a ValueError, as before, but the node's own fault: logged, with its traceback + log(f"API {self.command} {loggable(url.path)} failed: {e!r}", LOG_WARNING) message = str(e) or type(e).__name__ self.respond_error(status, message, {"Retry-After": "10"} if status == 503 else None) @@ -962,14 +998,15 @@ def api_name_watch(self): # POST endpoints def api_add(self): - """POST /api/add {"path", "wrap", "only_hash"}: import a file or directory on this machine.""" + """POST /api/add: import a file or directory on this machine (parameters: the module docstring's table).""" body = self.json_body() self.send_json(200, self.node.add_path(parse_path(body.get("path")), wrap=parse_bool(body.get("wrap")), only_hash=parse_bool(body.get("only_hash")), **node_content.add_options(body.get))) # pin, to, chunker… def api_add_stream(self): - """POST /api/add/stream?name=&only_hash=: import the request body, wrapped as name if given.""" + """POST /api/add/stream: import the request body, wrapped as name if given (parameters: the module + docstring's table).""" reader = self.stream_reader() size = None if reader.chunked else reader.remaining name = self.param("name", False) or None @@ -978,7 +1015,7 @@ def api_add_stream(self): self.send_json(200, self.node.add_stream(reader, size=size, name=name, only_hash=only_hash, **options)) def api_get(self): - """POST /api/get {"target", "output", "from"}: fetch and write out to a path on this machine.""" + """POST /api/get: fetch and write out to a path on this machine (parameters: the module docstring's table).""" body = self.json_body() from_node = parse_hex_id(body["from"]) if body.get("from") else None raw = {"raw": True} if parse_bool(body.get("raw")) else {} # sealed files: the ciphertext @@ -995,11 +1032,8 @@ def api_dir_set(self): executable=parse_bool(body.get("executable")))) def api_dir_rm(self): - """POST /api/dir/rm {"target"}: a new directory without the target's entry.""" - target = parse_target(self.json_body().get("target")) - if "/" not in target.rstrip("/"): - raise ValueError("dir rm needs /: the entry to remove") - self.send_json(200, self.node.dir_rm(target)) + """POST /api/dir/rm {"target"}: a new directory without the target's entry (the node checks it names one).""" + self.send_json(200, self.node.dir_rm(parse_target(self.json_body().get("target")))) def api_pin_add(self): """POST /api/pin/add {"target"}: fetch and pin with source user; answers once complete.""" diff --git a/rnfs/apiclient.py b/rnfs/apiclient.py index db436cf..6fdbd77 100644 --- a/rnfs/apiclient.py +++ b/rnfs/apiclient.py @@ -9,29 +9,35 @@ when a proof with it fails, and sends it as `X-Rnfs-Api` with every request but the open status (version()). - Before sending the token, the client makes whatever listens on the API port prove it knows it (api_proof, verify_daemon), since while the daemon is down any local user could listen there. Nothing else is sent to a - listener that fails: neither the token nor a request's body. The proof is made on the connection that then - carries the request (the daemon keeps it open for that one request), for every request, so a client used - for a long time sends nothing to a program listening on the port after the daemon stopped. + listener that fails: neither the token nor a request's body. Nor is anything it answers believed: whatever its + status and text, the client raises AuthFailed (Busy for a 503, as a full daemon answers) in its own words. + Connecting and proving take at most PROOF_TIMEOUT seconds, reading at most PROOF_READ_LIMIT bytes. The proof + is made on the connection that then carries the request (the daemon keeps it open for that one request), for + every request, so a client used for a long time sends nothing to a program listening on the port after the + daemon stopped. Only a daemon proved older than 0.1.5, which closes every connection, gets the request on a + new one (KEEPS_PROOF_OPEN_SINCE, until 0.1.7). - That proof request is also where the daemon's version is read (Client.daemon_version), and check_daemon is the one place to refuse a daemon by it; every version is accepted for now. - Every failure raises an ApiError subclass whose message says what went wrong, as the CLI prints it: no daemon (DaemonNotRunning), the connection failing (ConnectionFailed), the token (TokenError), the proof or the token - refused (AuthFailed), the settings (SettingsError), and the API's own error statuses (STATUS_ERRORS: 400 - BadRequest, 404 NotFound, 413 TooLarge, 422 InvalidDag, 451 Denied, 503 Busy, 507 StorageFull, other 5xx - ServerError). ApiError is an rnfs.errors.RnfsError, and each status's error also a subclass of the node's own - (NotFound of rnfs.errors.NotFound, BadRequest of ValueError, ...), so one except clause catches what rnfs.Node - and rnfs.Client raise alike. -- Each request opens its own connection (its proof, then the request), with no timeout unless one is given - (fetches can take hours), so an ApiConnection may be used from several threads. Paths are on the daemon's - machine, which is this one: they are made absolute here. + refused (AuthFailed), the settings (SettingsError), and the API's own error statuses (STATUS_ERRORS, from + rnfs.errors.STATUSES: 400 BadRequest, 404 NotFound, 413 TooLarge, 422 InvalidDag, 451 Denied, 503 Busy, 507 + StorageFull, other 5xx ServerError). ApiError is an rnfs.errors.RnfsError, and each status's error also a + subclass of the node's own (NotFound of rnfs.errors.NotFound, BadRequest of ValueError, ...), so one except + clause catches what rnfs.Node and rnfs.Client raise alike. +- Each request opens its own connection (its proof, then the request), with no timeout for the request unless + one is given (fetches can take hours), so an ApiConnection may be used from several threads. Paths are on the + daemon's machine, which is this one: they are made absolute here. """ import hashlib import hmac import http.client import json import os +import re import secrets import stat +import warnings from urllib.parse import urlencode from rnfs import errors, fsutil, winacl @@ -43,7 +49,12 @@ # Before sending the token, a client makes whatever listens on the API port prove it knows it (api_proof) CHALLENGE_HEADER = "X-Rnfs-Challenge" PROOF_HEADER = "X-Rnfs-Proof" +PROOF_TIMEOUT = 15 # seconds to connect and prove (at most the client's timeout): a listener mustn't hang a client +PROOF_READ_LIMIT = 64 * 1024 # the most read of a proof's response (the daemon's is a few dozen bytes) COPY_SIZE = 64 * 1024 # what a Stream reads at a time when iterated +# A daemon before 0.1.5 closes the connection it proved itself on; only one proved that old gets the request on a +# new connection (the fallback is to be dropped in 0.1.7) +KEEPS_PROOF_OPEN_SINCE = (0, 1, 5) class ApiError(errors.RnfsError): @@ -108,9 +119,22 @@ class ServerError(ApiError): """500 or another 5xx: the daemon failed (see its log).""" +# The client's error for each of the node's (errors.STATUSES), a subclass of both ApiError and the node's class +_STATUS_CLASSES = (BadRequest, NotFound, TooLarge, InvalidDag, Denied, Busy, StorageFull) + + +def _status_errors() -> dict[int, type[ApiError]]: + """Each API error status's ApiError subclass: errors.STATUSES' statuses, each to the client class of its node + error, and 403 (the token refused) to AuthFailed. A status added there without a class here fails here, at + import.""" + table: dict[int, type[ApiError]] = {403: AuthFailed} + for error_class, status in errors.STATUSES: + table[status] = next(client for client in _STATUS_CLASSES if issubclass(client, error_class)) + return table + + # The API's error statuses (design.md, "Node facade"); other 4xx raise ApiError, other 5xx ServerError -STATUS_ERRORS = {400: BadRequest, 403: AuthFailed, 404: NotFound, 413: TooLarge, 422: InvalidDag, 451: Denied, - 503: Busy, 507: StorageFull} +STATUS_ERRORS = _status_errors() def error_for(status: int, message: str) -> ApiError: @@ -119,19 +143,34 @@ def error_for(status: int, message: str) -> ApiError: return error_class(message, status) +def version_numbers(version) -> tuple[int, ...] | None: + """The numbers a version string starts with ("0.1.4.post1": (0, 1, 4)); None if it isn't one that does.""" + match = re.match(r"\d+(?:\.\d+)*", version) if isinstance(version, str) else None + return tuple(int(number) for number in match.group().split(".")) if match else None + + def token_path(rnfs_dir: str = DEFAULT_DIR) -> str: """Where the daemon using rnfs_dir keeps its API token.""" return os.path.join(os.path.expanduser(rnfs_dir), TOKEN_FILE) -def api_port(rnfs_dir: str = DEFAULT_DIR) -> int: - """The port the daemon using rnfs_dir listens on: api_port in its rnfs.toml, else DEFAULT_API_PORT. Raises - SettingsError if rnfs.toml can't be read.""" +def load_settings(rnfs_dir: str = DEFAULT_DIR) -> Config: + """The settings of the daemon using rnfs_dir (Config.load), with what was changed to trust them in its + warnings, for the caller to show. Raises SettingsError if rnfs.toml can't be read or trusted.""" try: - port = Config.load(rnfs_dir).api_port + return Config.load(rnfs_dir) except (ValueError, TypeError, OSError) as e: # TypeError: a badly typed setting validate() missed raise SettingsError(f"can't read the settings in {rnfs_dir}: {e}") from None - return port if port is not None else DEFAULT_API_PORT + + +def api_port(rnfs_dir: str = DEFAULT_DIR) -> int: + """The port the daemon using rnfs_dir listens on: api_port in its rnfs.toml, else DEFAULT_API_PORT. A + settings file made private on the way (others could access it) is reported as a RuntimeWarning. Raises + SettingsError if rnfs.toml can't be read.""" + config = load_settings(rnfs_dir) + for message in config.warnings: + warnings.warn(message, RuntimeWarning, stacklevel=2) + return config.api_port if config.api_port is not None else DEFAULT_API_PORT def api_proof(token: str, port: int, challenge: str) -> str: @@ -235,7 +274,7 @@ class ApiConnection: its rnfs.toml). token_file defaults to /api_token and is read on the first request needing it. request() and call() are the whole protocol; get() and post() call any endpoint (features' too); the other - methods are the main operations, jobs, the deny list and name watching, returning what the API answers + methods are every core route (api.ApiHandler.ROUTES), jobs and the deny list, returning what the API answers (design.md, "Node facade"). Every failure raises an ApiError subclass. """ @@ -271,36 +310,57 @@ def request(self, method: str, path: str, params: dict | None = None, json_body: headers["Content-Type"] = "application/octet-stream" if length is not None: headers["Content-Length"] = str(length) - connection = self._connect() if authenticated: - connection, headers[API_HEADER] = self._prove(connection) - return self._send(connection, method, url, body, headers) + connection, headers[API_HEADER] = self._prove(self._connect(self._proof_timeout())) + # Proved: the request itself may take as long as the client allows + connection.timeout = self.timeout + if connection.sock is not None: + connection.sock.settimeout(self.timeout) + else: + connection = self._connect() + response = self._exchange(connection, method, url, body, headers) + try: + return self._raise_for(response) + except ApiError: + connection.close() + raise + + def _connect(self, timeout: float | None = None) -> http.client.HTTPConnection: + """A new connection to the API port (opened by its first request), with timeout (default: the client's).""" + return http.client.HTTPConnection("127.0.0.1", self.port, timeout=self.timeout if timeout is None else timeout) - def _connect(self) -> http.client.HTTPConnection: - """A new connection to the API port (opened by its first request).""" - return http.client.HTTPConnection("127.0.0.1", self.port, timeout=self.timeout) + def _proof_timeout(self) -> float: + """The timeout for connecting and proving: PROOF_TIMEOUT, or the client's own if shorter.""" + return PROOF_TIMEOUT if self.timeout is None else min(self.timeout, PROOF_TIMEOUT) - def _send(self, connection: http.client.HTTPConnection, method: str, url: str, body, - headers: dict) -> http.client.HTTPResponse: - """Send one request on connection and return the successful response; errors raise ApiError.""" + def _exchange(self, connection: http.client.HTTPConnection, method: str, url: str, body, + headers: dict) -> http.client.HTTPResponse: + """Send one request on connection and return its response, whatever its status; a connection that fails + raises DaemonNotRunning or ConnectionFailed.""" try: connection.request(method, url, body=body, headers=headers) - response = connection.getresponse() - except ConnectionRefusedError: - raise DaemonNotRunning(f"no daemon is running on API port {self.port}. Start one with: rnfs daemon") \ - from None - except (http.client.HTTPException, ConnectionError) as e: - raise ConnectionFailed(f"the daemon dropped the connection ({e!r}); see its log") from None - except OSError as e: + return connection.getresponse() + except (http.client.HTTPException, OSError) as e: + connection.close() + if isinstance(e, ConnectionRefusedError): + raise DaemonNotRunning(f"no daemon is running on API port {self.port}. Start one with: rnfs daemon") \ + from None + if isinstance(e, (http.client.HTTPException, ConnectionError)): + raise ConnectionFailed(f"the daemon dropped the connection ({e!r}); see its log") from None raise ConnectionFailed(f"can't reach the daemon on API port {self.port}: {e}") from None - if response.status >= 400: + + def _raise_for(self, response: http.client.HTTPResponse, data: bytes | None = None) -> http.client.HTTPResponse: + """response if it succeeded; else raise the ApiError for its status (error_for, the one mapping), with the + message of its body (data, if read already). Only for a response of the daemon, or of the open status.""" + if response.status < 400: + return response + if data is None: data = read_all(response) - try: - message = json.loads(data)["error"] - except (ValueError, KeyError, TypeError): - message = f"HTTP {response.status} {response.reason}" - raise error_for(response.status, str(message)) - return response + try: + message = json.loads(data)["error"] + except (ValueError, KeyError, TypeError): + message = f"HTTP {response.status} {response.reason}" + raise error_for(response.status, str(message)) def _prove(self, connection: http.client.HTTPConnection) -> tuple[http.client.HTTPConnection, str]: """Make what listens on the API port prove, on connection, that it knows the token, and return the @@ -308,44 +368,81 @@ def _prove(self, connection: http.client.HTTPConnection) -> tuple[http.client.HT open for one more request, so the token and the request's body can only reach it: a proof made once, on another connection, would hold no longer than the daemon, and any local user can listen on its port once it stops. The token read is kept, and read again if the proof fails (a daemon restarted since - wrote a new one).""" + wrote a new one). A daemon that closes the connection anyway gets the request on a new one only if it + proved a version before 0.1.5 (whose daemons close every connection); for any other, ConnectionFailed.""" token = self.token or read_token(self.token_file) try: - self.verify_daemon(token, connection) + version = self.verify_daemon(token, connection) except AuthFailed: fresh = read_token(self.token_file) if fresh == token: raise connection.close() - connection = self._connect() - self.verify_daemon(fresh, connection) + connection = self._connect(self._proof_timeout()) + version = self.verify_daemon(fresh, connection) token = fresh self.token = token if connection.sock is None: - # A daemon before 0.1.5 closes every connection: the request goes on a new one, as its clients sent it + # A new connection could reach another listener (the daemon may stop in between), so only a daemon too + # old to keep it open gets the request on one, as its clients sent it + numbers = version_numbers(version) + if numbers is None or numbers >= KEEPS_PROOF_OPEN_SINCE: + raise ConnectionFailed(f"the daemon on API port {self.port} closed the connection it proved itself " + f"on, so the request wasn't sent; see its log") connection = self._connect() else: connection.auto_open = 0 # never reconnect: a new connection could reach another listener return connection, token - def verify_daemon(self, token: str, connection: http.client.HTTPConnection | None = None): + def verify_daemon(self, token: str, connection: http.client.HTTPConnection | None = None) -> str | None: """Raise AuthFailed unless whatever listens on the API port (on connection, else a new one) proves it - knows token, before the token (or a name key) is sent to it. Then check_daemon() with its version.""" + knows token, before the token (or a name key) is sent to it. Then check_daemon() with its version, which + is returned (None if it didn't say) and kept in daemon_version. + + The proof is checked before anything else of the answer is believed: a listener that can't prove itself + chooses neither the error (a 503 is Busy, as a daemon full of connections answers before reading the + request; every other status AuthFailed) nor its text. A proved daemon's error status raises its ApiError.""" challenge = secrets.token_hex(16) headers = {"Host": f"127.0.0.1:{self.port}", CHALLENGE_HEADER: challenge, "Connection": "keep-alive"} - response = self._send(connection or self._connect(), "GET", "/api/status", None, headers) - data = read_all(response) + connection = connection or self._connect(self._proof_timeout()) + response = self._exchange(connection, "GET", "/api/status", None, headers) proof = (response.getheader(PROOF_HEADER) or "").strip().encode("ascii", "replace") if not hmac.compare_digest(proof, api_proof(token, self.port, challenge).encode()): + response.close() + connection.close() + if response.status == 503: + raise Busy(f"the daemon on API port {self.port} is too busy (or has too many connections); try " + f"again shortly", 503) raise AuthFailed(f"what listens on API port {self.port} couldn't prove it knows the API token in " f"{self.token_file}, so nothing was sent to it: it may be another program, or a " f"daemon using another --dir (or one started before an upgrade: restart it)") + try: + data = response.read(PROOF_READ_LIMIT) + # Anything past PROOF_READ_LIMIT: no daemon's status, and the connection is unusable for the request + complete = response.isclosed() or not response.read(1) + except (http.client.HTTPException, OSError): # IncompleteRead, a timeout, a reset + complete = False + if not complete: + response.close() + connection.close() + raise ConnectionFailed("the daemon sent an invalid or incomplete response; see its log") + try: + self._raise_for(response, data) + except ApiError: + connection.close() + raise try: version = json.loads(data).get("version") except (ValueError, AttributeError): version = None - self.daemon_version = version if isinstance(version, str) else None - self.check_daemon(self.daemon_version) + version = version if isinstance(version, str) else None + self.daemon_version = version + try: + self.check_daemon(version) + except ApiError: + connection.close() + raise + return version def check_daemon(self, version: str | None): """The one place to refuse a daemon by its version (None if it didn't say), before the token is sent; @@ -431,6 +528,32 @@ def stat(self, target: str) -> dict: """{"cid", "size", "blocks", "local_blocks", "local_bytes"} of a target.""" return self.get("/api/stat", target=target) + def dir_set(self, target: str, name: str, cid: str, executable: bool = False) -> dict: + """A new directory: the directory target with entry name set to cid (a file, executable if asked, or a + directory), pinned user: {"cid"}.""" + return self.post("/api/dir/set", target=target, name=name, cid=cid, executable=executable) + + def dir_rm(self, target: str) -> dict: + """A new directory without the entry target (/) names, pinned user: {"cid"}.""" + return self.post("/api/dir/rm", target=target) + + def export(self, target: str, output: str) -> dict: + """Write a target's DAG as an archive to output (a path on this machine, which must not exist): + {"roots", "blocks", ...}.""" + return self.post("/api/export", target=target, output=os.path.abspath(os.path.expanduser(output))) + + def import_archive(self, path: str) -> dict: + """Import an archive on this machine: {"roots", "incomplete", ...}.""" + return self.post("/api/import", path=os.path.abspath(os.path.expanduser(path))) + + def gc(self) -> dict: + """Remove blocks no pin needs: {"removed_blocks", "freed_bytes"}.""" + return self.post("/api/gc") + + def verify(self) -> dict: + """Check every stored block against its CID: {"checked", "bad", ...}.""" + return self.post("/api/verify") + # Pins def pin_add(self, target: str) -> dict: @@ -445,6 +568,20 @@ def pin_ls(self, cid: str | None = None) -> list[dict]: """Every pin (or cid's): [{"cid", "sources", "complete", "type", "sealed", "name"}].""" return self.get("/api/pin/ls", cid=cid)["pins"] + def pin_remote(self, node: str, cid: str, pin: bool = True) -> bool: + """Ask node (hex) to pin cid for this node, or (pin=False) to release that pin; True if it accepted.""" + return self.post("/api/pin/remote", node=node, cid=cid, pin=pin)["accepted"] + + # The network + + def providers(self, cid: str) -> list[str]: + """The nodes (hex) providing cid: the records held, then what a provider lookup finds.""" + return self.get("/api/providers", cid=cid)["providers"] + + def ping(self, node: str) -> float | None: + """The round-trip time in seconds of a new link to node (hex), or None if it is unreachable.""" + return self.get("/api/ping", node=node)["rtt"] + # Names def name_list(self) -> list[dict]: @@ -455,8 +592,22 @@ def name_create(self, alias: str) -> dict: """Create a name key: {"alias", "name"}.""" return self.post("/api/name/create", alias=alias) + def name_import(self, alias: str, key: bytes) -> dict: + """Import a name key exported elsewhere (its bytes) as alias: {"alias", "name"}.""" + body = bytes(key) + return self.call("POST", "/api/name/import", params={"alias": alias}, body=body, length=len(body)) + + def name_export(self, alias: str) -> bytes: + """A name's key, to import elsewhere: whoever has it can publish the name.""" + return read_all(self.request("POST", "/api/name/export", json_body={"alias": alias})) + + def name_rm(self, alias: str): + """Delete a name's key (NotFound if there is no such alias).""" + self.post("/api/name/rm", alias=alias) + def name_publish(self, alias: str, target: str) -> dict: - """Point a name of this node at a target: {"name", "cid", "sequence"}.""" + """Point a name of this node at a target: {"name", "cid", "sequence"}, and "warning" in private mode (the + name's announce is visible outside the private network).""" return self.post("/api/name/publish", alias=alias, target=target) def name_resolve(self, name: str) -> dict: diff --git a/rnfs/cli.py b/rnfs/cli.py index b5d4b25..ff66e05 100644 --- a/rnfs/cli.py +++ b/rnfs/cli.py @@ -18,6 +18,7 @@ import os import stat import sys +import time from rnfs import __version__, apiclient, fsutil from rnfs.apiclient import CHALLENGE_HEADER, PROOF_HEADER, TOKEN_FILE, api_proof, token_path # noqa: F401 @@ -134,9 +135,12 @@ def client(args) -> ApiClient: port = args.api_port if port is None: try: - port = apiclient.api_port(args.dir) + config = apiclient.load_settings(args.dir) except apiclient.ApiError as e: fail(str(e)) + for warning in config.warnings: + warn(warning) + port = config.api_port return ApiClient(port, token_path(args.dir)) @@ -165,13 +169,30 @@ def print_warnings(result: dict): # Daemon +def wait_until_stopping(stopping, interval: float = 0.0, task=None): + """Return once stopping is set (by a signal handler), calling task every interval seconds meanwhile, if given. + + Python runs signal handlers in the main thread. On POSIX a signal interrupts the wait, so this sleeps until the + task is due, or without one until stopped: an idle daemon's main thread doesn't wake. On Windows a wait isn't + interrupted for them, so it wakes each second there.""" + due = time.monotonic() + interval if task is not None else None + while True: + timeout = None if due is None else max(0.0, due - time.monotonic()) + if fsutil.IS_WINDOWS: + timeout = 1.0 if timeout is None else min(timeout, 1.0) + if stopping.wait(timeout): + return + if due is not None and time.monotonic() >= due: + due = time.monotonic() + interval + task() + + def run_daemon(args): """Run a node with its API and gateway until SIGINT, SIGTERM or SIGBREAK, then shut everything down cleanly.""" # Imported here so client commands don't pay for loading RNS import faulthandler import signal import threading - import time import traceback import RNS @@ -181,8 +202,15 @@ def run_daemon(args): from rnfs.errors import RnfsError from rnfs.gateway import GatewayServer from rnfs.log import LOG_ERROR, LOG_NOTICE - from rnfs.node import DirInUse, RnfsNode, lock_rnfs_dir + from rnfs.node import DirInUse, RnfsNode, lock_rnfs_dir, make_private_root + # The rnfs dir is made private first, then the settings in it are checked and read, then its subdirectories + # are made private and it is locked (lock_rnfs_dir): nobody else can change what each step relies on + root = os.path.expanduser(args.dir) + try: + make_private_root(root) + except (RnfsError, OSError) as e: + fail(f"can't use the rnfs dir {root}: {e}") try: config = Config.load(args.dir, reticulum_dir=args.rnsconfig, api_port=args.api_port, gateway_port=args.gateway_port, @@ -193,6 +221,8 @@ def run_daemon(args): debug_log=args.debug_log) except (ValueError, TypeError, OSError) as e: fail(f"invalid settings: {e}") + for warning in config.warnings: + warn(warning) # Before anything uses the rnfs dir (made private first, as the node would make it): a second daemon on it # would load and save the same state, announce the same node ID and rotate the first one's debug log @@ -267,15 +297,15 @@ def abort(message: str): for name in ("SIGINT", "SIGTERM", "SIGBREAK"): if hasattr(signal, name): signal.signal(getattr(signal, name), lambda *_: stopping.set()) - # Python runs handlers in the main thread, and on Windows a wait isn't interrupted for them: wake each second - status_due = time.monotonic() + debuglog.STATUS_INTERVAL - while not stopping.wait(1): - if debug is not None and time.monotonic() >= status_due: - status_due = time.monotonic() + debuglog.STATUS_INTERVAL - try: - debug.note(debuglog.status_line(node), LOG_NOTICE) - except Exception as e: # noqa: BLE001 - a status line mustn't stop the daemon - debug.note(f"status line failed: {e!r}", LOG_ERROR) + + def status(): + """Write the debug log's periodic status line.""" + try: + debug.note(debuglog.status_line(node), LOG_NOTICE) + except Exception as e: # noqa: BLE001 - a status line mustn't stop the daemon + debug.note(f"status line failed: {e!r}", LOG_ERROR) + + wait_until_stopping(stopping, debuglog.STATUS_INTERVAL, status if debug is not None else None) for server in servers: server.stop() @@ -591,6 +621,8 @@ def run_name_publish(args): """Point a name at a target.""" result = client(args).name_publish(args.alias, args.target) print(printable(f"published {result['cid']} as {result['name']} (sequence {result['sequence']})")) + if result.get("warning"): # private mode: the name's announce still reaches outsiders + warn(result["warning"]) def run_name_resolve(args): diff --git a/rnfs/config.py b/rnfs/config.py index 884e718..01788ef 100644 --- a/rnfs/config.py +++ b/rnfs/config.py @@ -4,18 +4,20 @@ sizes in bytes and rates in bits/s. A setting left as None takes a default that depends on whether the node (or link) is slow; use the `effective_*` helpers rather than reading those fields directly. """ +import errno import math import os -import re +import stat import tomllib from dataclasses import dataclass, field, fields -from rnfs import winacl -from rnfs.constants import DAY, GIB, HOUR, KIB, PROVIDER_TTL +from rnfs import fsutil, ids +from rnfs.constants import DAY, GIB, HOUR, KIB, MIB, PROVIDER_TTL DEFAULT_API_PORT = 8788 DEFAULT_GATEWAY_PORT = 8789 SETTINGS_FILE = "rnfs.toml" +MAX_SETTINGS_SIZE = 1 * MIB # bytes of rnfs.toml read; a larger file is refused # In the rnfs dir, locked by the daemon or embedded node using it (node.lock_rnfs_dir), so a second can't start LOCK_FILE = "daemon.lock" @@ -40,7 +42,7 @@ MAX_DURATION = 30 * DAY FLAGS = ("private", "gateway_scripts", "default_bootstrap", "debug_log") IDENTITY_LISTS = ("members", "remote_pin_identities") # RNS identity hashes: 32 hex digits, optionally -_HEX_ID = re.compile(r"[0-9a-fA-F]{32}") +_HEX_ID = ids.HEX16 # the name other modules used before rnfs.ids def _is_number(value) -> bool: @@ -50,7 +52,13 @@ def _is_number(value) -> bool: @dataclass class Config: - """Settings for an RnfsNode.""" + """Settings for an RnfsNode. + + warnings: what Config.load changed to trust the settings, for the caller to show (a plain attribute, not a + field, so rnfs.toml can't set it). + """ + + warnings = () # unannotated, so not a dataclass field; Config.load sets a list rnfs_dir: str = "~/.rnfs/" # identity, blocks, pins, names and shared folder live here # Reticulum config directory. None uses RNS's own default (/etc/reticulum, ~/.config/reticulum or @@ -118,6 +126,9 @@ class Config: # (at least every 3 days) and, if followed, a pin: 100 keeps a slow channel's republish airtime small keep_max_names: int = 100 keeper_query: str = "fallback" # "fallback": ask keepers when a path request heard nothing; "always" + # Bytes a followed name's new target may hold (its size and its DAG's blocks); 0: no follow fetches. A swap + # briefly holds the old target and the new one, so up to twice this per followed name + follow_quota: int = 1 * GIB # --- content (pin=false, sealing, chunker, gateway formats) --- # --- lxmf (rnfs/features/lxmf.py) --- lxmf_enabled: bool | None = None # None: on if the LXMF package is installed @@ -166,6 +177,19 @@ def state_dir(self) -> str: """rnfs's own persistent state: pins, cache records, provider records, sequences.""" return os.path.join(self.root, "state") + # Identity lists, each normalised once here for its users: two sets differently normalised could disagree + # about who is a member + + @property + def member_set(self) -> frozenset[str]: + """members as lowercase hex (as identity.hash.hex() writes them), for private mode's membership checks.""" + return frozenset(ids.normal_hex16(value) for value in self.members) + + @property + def remote_pin_set(self) -> frozenset[str]: + """remote_pin_identities as lowercase hex (as identity.hash.hex() writes them): who may send /pin.""" + return frozenset(ids.normal_hex16(value) for value in self.remote_pin_identities) + # Values that depend on whether the node is slow (section 2) def effective_alpha(self, slow_node: bool) -> int: @@ -184,12 +208,13 @@ def effective_max_parallel_providers(self, slow_node: bool) -> int: return 1 if slow_node else 3 def effective_max_upload_rate(self, slow_link: bool, rate_estimate: float | None) -> float | None: - """The upload cap for a link in bits/s, or None for no cap.""" + """The upload cap for a link in bits/s, or None for no cap: max_upload_rate if set, else the default + (rates.default_upload_cap: a slow link's rate estimate halved).""" if self.max_upload_rate is not None: return self.max_upload_rate - if slow_link and rate_estimate is not None: - return rate_estimate / 2 - return None + # Here: rnfs.network.rates imports this module, and RNS (through rnfs.log), which settings alone don't need + from rnfs.network import rates + return rates.default_upload_cap(slow_link, rate_estimate) def validate(self): """Raise ValueError for settings the spec forbids, or of the wrong type or out of range.""" @@ -215,6 +240,8 @@ def validate(self): raise ValueError("keep_max_names must be a whole number, not negative") if self.keeper_query not in ("fallback", "always"): raise ValueError("keeper_query must be fallback or always") + if type(self.follow_quota) is not int or self.follow_quota < 0: + raise ValueError("follow_quota must be a whole number of bytes, not negative") # lxmf from rnfs.features import lxmf # here, like mesh: only needed once settings are checked lxmf.validate_settings(self) @@ -257,8 +284,7 @@ def _validate_core_types(self): raise ValueError(f"{name} must be true or false") for name in IDENTITY_LISTS: values = getattr(self, name) - if not isinstance(values, list) or not all( - isinstance(value, str) and _HEX_ID.fullmatch(value.strip().strip("<>")) for value in values): + if not isinstance(values, list) or not all(ids.is_hex16(value, brackets=True) for value in values): raise ValueError(f"{name} must be a list of RNS identity hashes (32 hex characters each)") # Node IDs that don't parse are skipped with a warning when joining, but a string isn't a list of them if not isinstance(self.bootstrap_nodes, list) or not all(isinstance(v, str) for v in self.bootstrap_nodes): @@ -269,17 +295,30 @@ def _validate_core_types(self): @classmethod def load(cls, rnfs_dir: str = "~/.rnfs/", defaults: dict | None = None, **overrides) -> "Config": """Settings from `/rnfs.toml` if it exists, then overrides (None values ignored), validated. - defaults (an embedded node's) apply where neither sets a value.""" + defaults (an embedded node's) apply where neither sets a value. + + rnfs.toml, and RNS's `config` when reticulum_dir is the rnfs dir, must be regular files of this user's; + one others may access is made private (open_checked). What was changed is in the result's warnings, for + the caller to show: this module doesn't log, as rnfs.log imports RNS, which the CLI and rnfs.Client never + load. Raises PermissionError if either file can't be trusted, ValueError for settings that aren't valid + (also a settings file over MAX_SETTINGS_SIZE), OSError if the file can't be read. + """ values: dict = dict(defaults or {}) - path = os.path.join(os.path.expanduser(rnfs_dir), SETTINGS_FILE) - if os.path.isfile(path): - with open(path, "rb") as f: - # On Windows, owned by and private to this user: its settings pick programs rnfs runs - try: - winacl.secure_settings(path, f.fileno()) - except OSError as e: - raise PermissionError(f"can't trust the settings file: {e}") from None - values.update(tomllib.load(f)) + warnings: list[str] = [] + root = os.path.expanduser(rnfs_dir) + path = os.path.join(root, SETTINGS_FILE) + # Its settings choose the Reticulum config, and with it programs rnfs starts, and the API port the CLI + # sends its token to: whoever could change it could choose them + opened = open_checked(path, "the settings file") + if opened is not None: + fd, warning = opened + if warning is not None: + warnings.append(f"{warning}. Check that its settings are yours") + with os.fdopen(fd, "rb") as f: + data = f.read(MAX_SETTINGS_SIZE + 1) + if len(data) > MAX_SETTINGS_SIZE: + raise ValueError(f"{path} is larger than {MAX_SETTINGS_SIZE // MIB} MiB") + values.update(tomllib.loads(data.decode("utf-8"))) known = {f.name for f in fields(cls)} unknown = values.keys() - known if unknown: @@ -291,4 +330,57 @@ def load(cls, rnfs_dir: str = "~/.rnfs/", defaults: dict | None = None, **overri values["rnfs_dir"] = rnfs_dir config = cls(**values) config.validate() + reticulum_root = config.reticulum_root + if reticulum_root is not None and os.path.realpath(reticulum_root) == os.path.realpath(root): + # Earlier versions' layout: RNS reads its config (interfaces, and their commands) from the rnfs dir + opened = open_checked(os.path.join(root, "config"), "the Reticulum config") + if opened is not None: + fd, warning = opened + os.close(fd) + if warning is not None: + warnings.append(f"{warning}. Check that its interfaces are yours") + config.warnings = warnings return config + + +def open_checked(path: str, what: str) -> tuple[int, str | None] | None: + """Open the file at path for reading, once checked through its descriptor: (the descriptor, a warning if it + was made private) or None if nothing is at path. what (e.g. "the settings file") names it in messages. + + It must be a regular file (a FIFO would block the start, a device be read without end), not a symlink + (whoever can write to the directory could point it at a file of theirs), and the user's own; one others may + access is made private (fsutil.check_owned_fd). Opened with O_NOFOLLOW and O_NONBLOCK, so neither a symlink + nor a FIFO is followed or waited on. Raises PermissionError if it can't be trusted, OSError if it can't be + opened. + """ + try: + # On Windows, which has no O_NOFOLLOW, a symlink or junction is found here (O_NOFOLLOW catches a swap) + listed = os.lstat(path) + except FileNotFoundError: + return None + if fsutil.is_link(listed): + raise PermissionError(f"{what} {path} is a symlink; rnfs only reads a regular file there (put the file " + f"itself there)") + try: + fd = os.open(path, os.O_RDONLY | fsutil.O_NOFOLLOW | fsutil.O_BINARY | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_CLOEXEC", 0)) + except FileNotFoundError: + return None # removed since the lstat + except OSError as e: + if e.errno in (errno.ELOOP, errno.EMLINK): # O_NOFOLLOW's answer for a symlink (EMLINK on FreeBSD) + raise PermissionError(f"{what} {path} is a symlink; rnfs only reads a regular file there") from None + raise + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode): + raise PermissionError(f"{what} {path} is not a regular file; rnfs only reads a regular file there") + try: + warning = fsutil.check_owned_fd(path, fd, info, what) + except OSError as e: + if not fsutil.IS_WINDOWS: + raise # its message says what and why + raise PermissionError(f"can't trust {what}: {e}") from None + except BaseException: + os.close(fd) + raise + return fd, warning diff --git a/rnfs/constants.py b/rnfs/constants.py index 0269ea2..3ea3ec7 100644 --- a/rnfs/constants.py +++ b/rnfs/constants.py @@ -24,6 +24,10 @@ BUCKET_COUNT = 128 RESPONSE_OVERHEAD = 1 * KIB # room left for msgpack framing in a /blocks response (section 7) MAX_PROVIDE_KEYS = 55 # keys in one /provide, both lists: the most that fit MAX_REQUEST_SIZE (section 4) +# Announces a node or name may miss, at announce_interval each, before it is presumed gone or its record stale: a +# routing table entry silent this long is dropped (section 6), an announce shows its record current for this long +# after it was emitted, and follows and deny lists refresh a record not heard this long (section 9) +MISSED_ANNOUNCES = 2 # Local, not protocol: a file being written out (.rnfs-get-.partial, .rnfs-job-.partial), then renamed PARTIAL_SUFFIX = ".partial" diff --git a/rnfs/content/blockstore.py b/rnfs/content/blockstore.py index 3f37261..ef89c68 100644 --- a/rnfs/content/blockstore.py +++ b/rnfs/content/blockstore.py @@ -17,10 +17,16 @@ from rnfs import fsutil from rnfs.constants import MAX_BLOCK_SIZE from rnfs.content.cid import Cid +from rnfs.errors import BlockMismatch PARTIAL_PREFIX = fsutil.PARTIAL_PREFIX # also the prefix of every other atomic write's temporary file +def matches(cid: Cid, block: bytes) -> bool: + """True if block is a valid block for cid: at most MAX_BLOCK_SIZE bytes and hashing to cid's digest.""" + return len(block) <= MAX_BLOCK_SIZE and cid.verify(block) # the size first, so an oversized block isn't hashed + + class BlockStore: """A directory of blocks named by their CIDs.""" @@ -57,11 +63,12 @@ def size(self, cid: Cid) -> int | None: return None def put(self, cid: Cid, block: bytes): - """Store block under cid unless already stored. Raises ValueError, storing nothing, if it doesn't match.""" - if len(block) > MAX_BLOCK_SIZE: - raise ValueError(f"block for {cid} exceeds MAX_BLOCK_SIZE") - if not cid.verify(block): - raise ValueError(f"block does not match {cid}") + """Store block under cid unless already stored. Raises BlockMismatch (a ValueError), storing nothing, if + it doesn't match.""" + if not matches(cid, block): + if len(block) > MAX_BLOCK_SIZE: + raise BlockMismatch(f"block for {cid} exceeds MAX_BLOCK_SIZE") + raise BlockMismatch(f"block does not match {cid}") path = self.path(cid) if os.path.isfile(path): return # content-addressed, so the same bytes; replacing them would fail on Windows while read @@ -106,7 +113,7 @@ def delete(self, cid: Cid) -> int | None: def verify(self, cid: Cid) -> bool: """True if the stored block exists and matches its CID.""" block = self.get(cid) - return block is not None and len(block) <= MAX_BLOCK_SIZE and cid.verify(block) + return block is not None and matches(cid, block) def scan(self) -> Iterator[tuple[Cid, int]]: """(CID, size) of every stored block. Files whose names aren't canonical CIDs are ignored.""" @@ -115,7 +122,7 @@ def scan(self) -> Iterator[tuple[Cid, int]]: if entry.name.startswith(PARTIAL_PREFIX): continue try: - cid = Cid.parse(entry.name) + cid = Cid.from_block_name(entry.name) if entry.is_file(follow_symlinks=False): yield cid, entry.stat(follow_symlinks=False).st_size except (ValueError, OSError): diff --git a/rnfs/content/builder.py b/rnfs/content/builder.py index 85d9e63..0895c11 100644 --- a/rnfs/content/builder.py +++ b/rnfs/content/builder.py @@ -12,9 +12,12 @@ directory is [2, []]. Builders hand every block to a `put(cid, block)` callback, so they don't depend on Storage; a no-op -callback (NO_PUT) gives only-hash mode. Files are streamed, one chunk in memory at a time. Within -`with checking(check):` each file's whole-content SHA-256 is passed to check (the deny list's file hashes), -which may raise to stop the build before the file's root node is stored. +callback (NO_PUT) gives only-hash mode. Files are streamed, one chunk in memory at a time. A build's +BuildOptions are passed explicitly: the chunker, which decides files' CIDs, and check, which is given each +file's whole-content SHA-256 (the deny list's file hashes) and may raise to stop the build before the file's +root node is stored. Explicit, because a ContextVar isn't copied to a worker thread: a build moved onto one would +silently fall back to the fixed chunker and skip the check (maintainability audit CT2). Without options, a build +takes them from chunker.using() and checking(), the wrappers that set them for the current context. """ import errno import hashlib @@ -23,8 +26,8 @@ import stat from contextlib import contextmanager from contextvars import ContextVar -from dataclasses import dataclass -from typing import BinaryIO, Callable, Iterable, Iterator +from dataclasses import dataclass, field +from typing import BinaryIO, Callable, Iterable, Iterator, Protocol from rnfs import constants, fsutil from rnfs.content import chunker, dag @@ -47,6 +50,8 @@ class Built: size: int # content bytes below the root kind: int # the kind a directory entry for it would have: dag.KIND_FILE, KIND_EXECUTABLE or KIND_DIRECTORY height: int # DAG nodes on the longest path from the root down, all types counted; 0 for a raw block + # A file's content SHA-256, when the build computed it (its BuildOptions had a check); None otherwise + digest: bytes | None = field(default=None, compare=False) class _Levels: @@ -116,14 +121,34 @@ def _put_node(node: dag.Node, put: Put) -> Cid: return cid +@dataclass(frozen=True) +class BuildOptions: + """What a build needs beyond its input, passed down explicitly so it holds on any thread. + + chunker: the name of the chunker cutting files (chunker.FIXED, the default, or FASTCDC); it decides their CIDs. + check: called with each file's content SHA-256 once it is read, before the file's root node is stored; it may + raise to stop the build (the deny list's file hashes). None checks nothing, and hashes nothing for it. + Raises ValueError for an unknown chunker. + """ + + chunker: str = chunker.FIXED + check: Callable[[bytes], None] | None = None + + def __post_init__(self): + """Check the chunker's name (None is the default).""" + object.__setattr__(self, "chunker", chunker.check(self.chunker or chunker.FIXED)) + + # What build_file passes each file's content SHA-256 to in this thread/context (checking), if anything _check: ContextVar[Callable[[bytes], None] | None] = ContextVar("rnfs_file_check", default=None) @contextmanager def checking(check: Callable[[bytes], None] | None) -> Iterator[None]: - """Within the block, build_file calls check(digest) with each file's content SHA-256 once it is read, before - the file's root node is stored; check may raise to stop the build. None checks nothing.""" + """Within the block, a build given no BuildOptions calls check(digest) with each file's content SHA-256 once it + is read, before the file's root node is stored; check may raise to stop the build. None checks nothing. + + A wrapper kept for callers that don't pass BuildOptions yet: it holds only in this thread/context.""" token = _check.set(check) try: yield @@ -131,8 +156,28 @@ def checking(check: Callable[[bytes], None] | None) -> Iterator[None]: _check.reset(token) -def build_file(stream: BinaryIO, put: Put, size: int | None = None, kind: int = dag.KIND_FILE) -> Built: - """Build a file from everything read from stream, or exactly size bytes of it (ValueError if it ends early).""" +def current_options() -> BuildOptions: + """The BuildOptions that chunker.using() and checking() set for this thread/context (the defaults if neither).""" + return BuildOptions(chunker.active(), _check.get()) + + +def _resolve(options: BuildOptions | None) -> BuildOptions: + """options, or those of this context (current_options) if None.""" + return options if options is not None else current_options() + + +def _chunks(stream: BinaryIO, size: int | None, name: str) -> Iterator[bytes]: + """A file's chunks by the chunker called name (as chunker.chunks gives them for the active one).""" + if name == chunker.FASTCDC: + return chunker.fastcdc_stream(stream, size) + return chunker.fixed_chunks(stream, size) + + +def build_file(stream: BinaryIO, put: Put, size: int | None = None, kind: int = dag.KIND_FILE, + options: BuildOptions | None = None) -> Built: + """Build a file from everything read from stream, or exactly size bytes of it (ValueError if it ends early). + + options: the chunker and check (by default those of this context, current_options).""" def make_group(level: int, pairs: list[tuple]) -> tuple: """A type 1 node over a group of chunks or file nodes.""" @@ -140,33 +185,36 @@ def make_group(level: int, pairs: list[tuple]) -> tuple: return Child(_put_node(node, put), node.size), _height(pairs) levels = _Levels(make_group, constants.MAX_CHILDREN) - check = _check.get() + options = _resolve(options) + check = options.check digest = hashlib.sha256() if check is not None else None - # Fixed CHUNK_SIZE pieces unless `add --chunker fastcdc` selected content-defined chunks (chunker.using) - for chunk in chunker.chunks(stream, size): + # Fixed CHUNK_SIZE pieces unless `add --chunker fastcdc` selected content-defined chunks + for chunk in _chunks(stream, size, options.chunker): cid = Cid.for_block(RAW, chunk) if digest is not None: digest.update(chunk) put(cid, chunk) levels.add(Child(cid, len(chunk)), 0) + content_hash = None if check is not None: - check(digest.digest()) + content_hash = digest.digest() + check(content_hash) level, pairs = levels.finish() if level == 0 and not pairs: put(EMPTY_FILE, b"") - return Built(EMPTY_FILE, 0, kind, 0) + return Built(EMPTY_FILE, 0, kind, 0, content_hash) if level == 0 and len(pairs) == 1: chunk_child = pairs[0][0] - return Built(chunk_child.cid, chunk_child.size, kind, 0) + return Built(chunk_child.cid, chunk_child.size, kind, 0, content_hash) child, height = make_group(level, pairs) _check_depth(height) - return Built(child.cid, child.size, kind, height) + return Built(child.cid, child.size, kind, height, content_hash) -def build_bytes(data: bytes, put: Put, kind: int = dag.KIND_FILE) -> Built: - """Build a file from bytes in memory.""" - return build_file(io.BytesIO(data), put, len(data), kind) +def build_bytes(data: bytes, put: Put, kind: int = dag.KIND_FILE, options: BuildOptions | None = None) -> Built: + """Build a file from bytes in memory (options as for build_file).""" + return build_file(io.BytesIO(data), put, len(data), kind, options) def build_directory(entries: Iterable[Entry], put: Put, child_height: int = 0) -> Built: @@ -422,8 +470,8 @@ def close(handle: Handle): os.close(handle) -def build_path(path: str, put: Put, warnings: list[str]) -> Built: - """Build the file or directory at path, appending to warnings what was skipped. +def build_path(path: str, put: Put, warnings: list[str], options: BuildOptions | None = None) -> Built: + """Build the file or directory at path, appending to warnings what was skipped (options as for build_file). The path itself is followed if it is a symlink (the user named it); inside a directory, hidden files are included and symlinks, devices, sockets, FIFOs and invalid names are skipped with a warning. @@ -431,11 +479,32 @@ def build_path(path: str, put: Put, warnings: list[str]) -> Built: """ handle, st = open_path(path) try: - return build_open(handle, st, path, put, warnings) + return build_open(handle, st, path, put, warnings, options=options) finally: close(handle) +FileState = tuple[int, int, int, int, bool] + + +def file_state(st: os.stat_result) -> FileState: + """What a FileCache knows a regular file by: (device, inode, size, mtime in ns, execute bit). The trust is the + shared folder's signature's: a file rewritten with the same size and modification time looks unchanged.""" + return st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns, is_executable(st.st_mode) + + +class FileCache(Protocol): + """What build_open asks about each regular file before reading it, and tells after reading one (the shared + folder's per-file cache, audit D1). st is the open file's stat, taken before it is read.""" + + def lookup(self, st: os.stat_result, kind: int) -> Built | None: + """The Built of the file whose stat is st, from an earlier build, if it may stand for the file now: its + blocks all stored, and protected for this build. None to read the file.""" + + def remember(self, st: os.stat_result, built: Built): + """The file whose stat was st, unchanged while it was read, was built as built.""" + + def hard_links(handle: Handle) -> int: """How many hard links the open regular file handle has beyond one (0 for a directory).""" if not isinstance(handle, int): @@ -445,23 +514,49 @@ def hard_links(handle: Handle) -> int: def build_open(handle: Handle, st: os.stat_result, path: str, put: Put, warnings: list[str], - depth: int = 1, skip_hard_links: bool = False) -> Built: + depth: int = 1, skip_hard_links: bool = False, options: BuildOptions | None = None, + cache: FileCache | None = None) -> Built: """Build the regular file or directory open as handle (from open_path or open_child), whose stat is st. path is only for messages (and for directories without FD_WALK); depth as for _build_tree. With skip_hard_links, files below a directory that have other hard links are skipped with a warning (the shared - folder: someone who may write to it could link a file of yours there that you never put in it). + folder: someone who may write to it could link a file of yours there that you never put in it). options as + for build_file, taken from this context once, here, if not given: the whole tree is built with them. cache, + if given, may stand in for reading each regular file (_build_cached). """ + options = _resolve(options) if stat.S_ISDIR(st.st_mode): - return _build_tree(handle, path, put, warnings, depth, skip_hard_links) + return _build_tree(handle, path, put, warnings, depth, skip_hard_links, options, cache) kind = dag.KIND_EXECUTABLE if is_executable(st.st_mode) else dag.KIND_FILE + if cache is not None: + return _build_cached(handle, put, kind, options, cache) # Reading a regular file never blocks, so O_NONBLOCK can stay; the caller keeps the descriptor with os.fdopen(handle, "rb", closefd=False) as f: - return build_file(f, put, kind=kind) + return build_file(f, put, kind=kind, options=options) + + +def _build_cached(handle: Handle, put: Put, kind: int, options: BuildOptions, cache: FileCache) -> Built: + """build_open of the regular file open as handle, through cache: a file it knows isn't read again. + + Its remembered content hash still goes to options.check, so a hash denied since is refused as if the file + were read; a file remembered without one (built when there was no check) is read again when there is a check. + """ + before = os.fstat(handle) + known = cache.lookup(before, kind) + if known is not None and (options.check is None or known.digest is not None): + if options.check is not None: + options.check(known.digest) + return known + with os.fdopen(handle, "rb", closefd=False) as f: + built = build_file(f, put, kind=kind, options=options) + # Changed while being read, what was read may be neither version: not remembered, so it is read again next time + if built.size == before.st_size and file_state(os.fstat(handle)) == file_state(before): + cache.remember(before, built) + return built def _build_tree(directory: Handle, path: str, put: Put, warnings: list[str], depth: int, - skip_hard_links: bool = False) -> Built: + skip_hard_links: bool, options: BuildOptions, cache: FileCache | None = None) -> Built: """Build the open directory at path, which sits depth directories from the root (1 for the root).""" # Every directory level adds at least one DAG node, so this bounds the recursion before it gets deep if depth > constants.MAX_DEPTH: @@ -491,7 +586,7 @@ def _build_tree(directory: Handle, path: str, put: Put, warnings: list[str], dep if skip_hard_links and (links := hard_links(handle)): warnings.append(f"skipped {shown}: {hard_link_reason(links)}") continue - child = build_open(handle, st, child_path, put, warnings, depth + 1, skip_hard_links) + child = build_open(handle, st, child_path, put, warnings, depth + 1, skip_hard_links, options, cache) except Changed: # A directory walked by path, swapped when it was listed (see list_directory) warnings.append(f"skipped {shown}: it changed while being added") @@ -509,13 +604,15 @@ def hard_link_reason(links: int) -> str: f"not be meant for sharing") -def add_path(path: str, put: Put, wrap_it: bool = False) -> tuple[Built, list[str]]: - """Build the file or directory at path, optionally wrapped in a directory under its own name. +def add_path(path: str, put: Put, wrap_it: bool = False, + options: BuildOptions | None = None) -> tuple[Built, list[str]]: + """Build the file or directory at path, optionally wrapped in a directory under its own name (options as for + build_file). Returns the root and the warnings for what was skipped. """ warnings: list[str] = [] - built = build_path(path, put, warnings) + built = build_path(path, put, warnings, options) if wrap_it: name = os.path.basename(os.path.normpath(os.path.abspath(path))) built = wrap(name, built, put) @@ -523,12 +620,13 @@ def add_path(path: str, put: Put, wrap_it: bool = False) -> tuple[Built, list[st def add_stream(stream: BinaryIO, put: Put, size: int | None = None, name: str | None = None, - executable: bool = False) -> Built: - """Build a file from a stream, wrapped as the entry name in a directory if name is given.""" + executable: bool = False, options: BuildOptions | None = None) -> Built: + """Build a file from a stream, wrapped as the entry name in a directory if name is given (options as for + build_file).""" kind = dag.KIND_EXECUTABLE if executable else dag.KIND_FILE if name is not None: dag.check_name(name) # before reading a possibly large stream - built = build_file(stream, put, size, kind) + built = build_file(stream, put, size, kind, options) if name is not None: built = wrap(name, built, put) return built diff --git a/rnfs/content/chunker.py b/rnfs/content/chunker.py index 99960b1..e7b8e87 100644 --- a/rnfs/content/chunker.py +++ b/rnfs/content/chunker.py @@ -6,13 +6,13 @@ FastCDC (Xia et al., "FastCDC: a Fast and Efficient Content-Defined Chunking Approach", 2016), with normalized chunking, as fixed by the specification: -- MIN_SIZE 16 KiB, AVG_SIZE 64 KiB, MAX_SIZE 128 KiB (= MAX_BLOCK_SIZE). +- MIN_SIZE 16 KiB, AVG_SIZE 64 KiB, FASTCDC_MAX 128 KiB (= MAX_BLOCK_SIZE; the specification's MAX_SIZE). - GEAR[i] (i = 0..255) is the first 8 bytes, big-endian, of SHA-256("rnfs-fastcdc-gear" || byte i). - Gear hash: h = 0 at offset MIN_SIZE of each chunk, then for every byte b: h = ((h << 1) + GEAR[b]) mod 2^64. -- At chunk offset i (0-based, MIN_SIZE <= i < MAX_SIZE), after hashing byte i: cut after it (the chunk is +- At chunk offset i (0-based, MIN_SIZE <= i < FASTCDC_MAX), after hashing byte i: cut after it (the chunk is bytes 0..i) if h & MASK_S == 0 while i < AVG_SIZE, or h & MASK_L == 0 from AVG_SIZE on. MASK_S is the top 18 bits of 64 (harder to meet before the average), MASK_L the top 14 bits (easier after it). -- A chunk with no cut point ends at MAX_SIZE; data left of at most MIN_SIZE bytes is the last chunk. +- A chunk with no cut point ends at FASTCDC_MAX; data left of at most MIN_SIZE bytes is the last chunk. Speed: a byte-by-byte loop in CPython manages about 15 MB/s. The hash after byte i only depends on the 64 bytes before it: h_i = sum over k < 64 of GEAR[b_(i-k)] << k (mod 2^64), with nothing before the chunk's @@ -36,7 +36,10 @@ MIN_SIZE = 16 * constants.KIB AVG_SIZE = 64 * constants.KIB -MAX_SIZE = 128 * constants.KIB +# The largest FastCDC chunk. It decides fastcdc CIDs, so it never changes, and a chunk is one block +FASTCDC_MAX = 128 * constants.KIB +assert FASTCDC_MAX <= constants.MAX_BLOCK_SIZE +MAX_SIZE = FASTCDC_MAX # its name before 0.1.6 (not dag.MAX_SIZE, which is unrelated) GEAR_SEED = b"rnfs-fastcdc-gear" GEAR = tuple(int.from_bytes(hashlib.sha256(GEAR_SEED + bytes([i])).digest()[:8], "big") for i in range(256)) MASK_S = ((1 << 18) - 1) << 46 @@ -124,20 +127,20 @@ def fixed_chunks(stream: BinaryIO, size: int | None = None) -> Iterator[bytes]: def fastcdc_stream(stream: BinaryIO, size: int | None = None) -> Iterator[bytes]: - """FastCDC chunks of a stream, holding at most READ_SIZE + MAX_SIZE bytes.""" + """FastCDC chunks of a stream, holding at most READ_SIZE + FASTCDC_MAX bytes.""" remaining = size buffer = b"" start = 0 eof = False while True: - if not eof and len(buffer) - start < MAX_SIZE: + if not eof and len(buffer) - start < FASTCDC_MAX: data, remaining = read_limited(stream, READ_SIZE, remaining) eof = len(data) < READ_SIZE or remaining == 0 buffer = buffer[start:] + data start = 0 if start >= len(buffer): return - end = cut_point(buffer, start, len(buffer)) if eof or len(buffer) - start >= MAX_SIZE else None + end = cut_point(buffer, start, len(buffer)) if eof or len(buffer) - start >= FASTCDC_MAX else None if end is None: continue yield buffer[start:end] @@ -158,11 +161,11 @@ def fastcdc_cuts(data: bytes) -> list[int]: def cut_point(data: bytes, start: int, end: int) -> int: """Where the chunk starting at data[start] ends, given that the data (as far as known) ends at end. - The caller must provide at least MAX_SIZE bytes from start unless end is the end of the file. + The caller must provide at least FASTCDC_MAX bytes from start unless end is the end of the file. """ if end - start <= MIN_SIZE: return end - limit = min(end, start + MAX_SIZE) + limit = min(end, start + FASTCDC_MAX) normal = min(limit, start + AVG_SIZE) base = start + MIN_SIZE cut = _scan(data, base, normal, base, False) @@ -175,7 +178,7 @@ def cut_point_simple(data: bytes, start: int, end: int) -> int: """cut_point by the byte-by-byte definition: the reference the fast version is tested against.""" if end - start <= MIN_SIZE: return end - limit = min(end, start + MAX_SIZE) + limit = min(end, start + FASTCDC_MAX) normal = min(limit, start + AVG_SIZE) h = 0 for i in range(start + MIN_SIZE, limit): diff --git a/rnfs/content/cid.py b/rnfs/content/cid.py index d9d5101..0a6e2b1 100644 --- a/rnfs/content/cid.py +++ b/rnfs/content/cid.py @@ -10,6 +10,7 @@ """ import base64 import hashlib +import re from rnfs.constants import ID_LENGTH @@ -21,12 +22,30 @@ DAG: bytes([0x01, 0x80, 0x80, 0xC0, 0x01, 0x12, 0x20]), # codec 0x300000, multicodec private use } _DIGEST_LENGTH = 32 +_BASE32 = "abcdefghijklmnopqrstuvwxyz234567" # RFC 4648, lowercase as written +_TO_INT_DIGITS = str.maketrans(_BASE32, "0123456789abcdefghijklmnopqrstuv") # the same values as int(..., 32) reads + + +def _written_form(prefix: bytes) -> tuple[int, int, str]: + """(length, spare bits, regular expression) of the canonical written form of the CIDs with prefix: "b", then + lowercase base32 digits, the final one's spare bits (past the binary's last) zero, as encoding leaves them.""" + bits = (len(prefix) + _DIGEST_LENGTH) * 8 + digits = -(-bits // 5) + spare_bits = digits * 5 - bits + last = "".join(c for value, c in enumerate(_BASE32) if value % (1 << spare_bits) == 0) + return 1 + digits, spare_bits, f"b[a-z2-7]{{{digits - 1}}}[{last}]" + + +# For from_block_name(): every canonical written form (but for the prefix), and written length -> (prefix, spare bits) +_FORMS = {prefix: _written_form(prefix) for prefix in _PREFIXES.values()} +_CANONICAL = re.compile("|".join(pattern for _, _, pattern in _FORMS.values())) +_WRITTEN = {length: (prefix, spare_bits) for prefix, (length, spare_bits, _) in _FORMS.items()} class Cid: """A valid CID. Immutable and hashable; compare and use as dict keys freely. - Build one with Cid.for_block(codec, block), Cid.from_bytes(binary) or Cid.parse(text). + Build one with Cid.for_block(codec, block), Cid.from_bytes(binary) or Cid.parse(text) (or from_block_name). Every constructor raises ValueError for anything that isn't a valid CID. """ @@ -71,6 +90,23 @@ def parse(cls, text) -> "Cid": raise ValueError(f"invalid CID {text!r}: not in canonical form (lowercase, unpadded)") return cid + @classmethod + def from_block_name(cls, name: str) -> "Cid": + """The CID whose written form is name, a file name read from the block directory. Accepts exactly what + parse() accepts and raises ValueError for the rest, but checks the canonical form without re-encoding, + and keeps name as str(): opening a store scans every block's name. API and wire input use parse().""" + if not isinstance(name, str) or not _CANONICAL.fullmatch(name): + raise ValueError(f"invalid CID {name!r}: not in canonical form") + prefix, spare_bits = _WRITTEN[len(name)] + # The digits are known to be base32 here, so int() decodes them (far faster than b32decode) + value = int(name[1:].translate(_TO_INT_DIGITS), 32) >> spare_bits + binary = value.to_bytes(len(prefix) + _DIGEST_LENGTH, "big") + if not binary.startswith(prefix): + raise ValueError(f"invalid CID {name!r}: not a raw or rnfs-dag CID") + cid = cls(binary) + cid._text = name + return cid + @property def digest(self) -> bytes: """The block's SHA-256.""" diff --git a/rnfs/content/dag.py b/rnfs/content/dag.py index 045691c..f86f0be 100644 --- a/rnfs/content/dag.py +++ b/rnfs/content/dag.py @@ -28,6 +28,7 @@ FILE, DIRECTORY, INDEX = 1, 2, 3 KIND_FILE, KIND_EXECUTABLE, KIND_DIRECTORY = 0, 1, 2 KINDS = (KIND_FILE, KIND_EXECUTABLE, KIND_DIRECTORY) +KIND_NAMES = {KIND_FILE: "file", KIND_EXECUTABLE: "executable", KIND_DIRECTORY: "directory"} # as `ls` shows them MAX_SIZE = 2**64 - 1 # a size above this makes a DAG invalid MAX_NAME_BYTES = 255 @@ -79,6 +80,16 @@ def size(self) -> int: """Content bytes below this node: the sum of its children's sizes.""" return sum(child.size for child in self.children) + @property + def child_cids(self) -> tuple[Cid, ...]: + """The CIDs this node lists, in order.""" + return tuple(child.cid for child in self.children) + + @property + def names(self) -> tuple[str, ...]: + """The names this node lists: none, for a file node.""" + return () + @dataclass(frozen=True) class DirectoryNode: @@ -95,6 +106,16 @@ def size(self) -> int: """Content bytes below this node: the sum of its entries' sizes.""" return sum(entry.size for entry in self.entries) + @property + def child_cids(self) -> tuple[Cid, ...]: + """The CIDs this node lists (its entries'), in order.""" + return tuple(entry.cid for entry in self.entries) + + @property + def names(self) -> tuple[str, ...]: + """Its entries' names, in order.""" + return tuple(entry.name for entry in self.entries) + @dataclass(frozen=True) class IndexNode: @@ -111,6 +132,16 @@ def size(self) -> int: """Content bytes below this node: the sum of its children's sizes.""" return sum(child.size for child in self.children) + @property + def child_cids(self) -> tuple[Cid, ...]: + """The CIDs this node lists, in order.""" + return tuple(child.cid for child in self.children) + + @property + def names(self) -> tuple[str, ...]: + """Its children's first names, in order.""" + return tuple(child.first_name for child in self.children) + Node = FileNode | DirectoryNode | IndexNode @@ -343,14 +374,10 @@ def _decode(block: bytes) -> Node: def _check_node(node: Node): """The checks a decoded node must pass on its own. Raises InvalidDag.""" if isinstance(node, DirectoryNode): - names = [entry.name for entry in node.entries] for entry in node.entries: if entry.kind not in KINDS: raise InvalidDag(f"entry {entry.name!r} has unknown kind {entry.kind}") - elif isinstance(node, IndexNode): - names = [child.first_name for child in node.children] - else: - names = [] + names = node.names for name in names: try: check_name(name) diff --git a/rnfs/content/edit.py b/rnfs/content/edit.py index a42d0a0..83e229b 100644 --- a/rnfs/content/edit.py +++ b/rnfs/content/edit.py @@ -34,7 +34,9 @@ def _rebuild(chain: list[walker.Expect], names: list[str], entries: list[Entry], siblings = [entry for entry in walker.list_directory(parent, get) if entry.name != name] siblings.append(Entry(name, built.cid, built.size, dag.KIND_DIRECTORY)) built = builder.build_directory(siblings, put) - # The builder can't see the depth of subtrees it didn't build, so check the finished tree + # The builder can't see the depth of subtrees it didn't build, so check the finished tree: walker.height + # remembers the subtrees it measured before, so once a tree has been measured an edit of it reads only the + # directories it rebuilt (rather than every node: building a directory one entry at a time was O(n²)) if walker.height(walker.root_expect(built.cid), get) > constants.MAX_DEPTH: raise TooLarge(f"the edited tree would be deeper than MAX_DEPTH ({constants.MAX_DEPTH})") return built.cid @@ -51,7 +53,7 @@ def dir_set(root: Cid, names: list[str], name: str, child: Cid, executable: bool dag.check_name(name) child_expect = walker.root_expect(child) child_block = get(child_expect) - if walker.kind_of(child_expect, child_block) == "directory": + if walker.kind_of(child_expect, child_block) == walker.DIRECTORY_TYPE: if executable: raise ValueError("a directory can't be executable") kind = dag.KIND_DIRECTORY @@ -64,10 +66,21 @@ def dir_set(root: Cid, names: list[str], name: str, child: Cid, executable: bool return _rebuild(chain, names, entries, get, put) +NEEDS_ENTRY = "dir rm needs /: the entry to remove" + + +def check_entry_target(target) -> None: + """Raise ValueError unless target names an entry inside a directory, `/` (dir rm's target: a root + can't be removed). The node checks this before fetching anything; the path itself is checked as it is + parsed.""" + if not isinstance(target, str) or "/" not in target.rstrip("/"): + raise ValueError(NEEDS_ENTRY) + + def dir_rm(root: Cid, names: list[str], get: walker.Get, put: Put) -> Cid: """The new root after removing the entry /. Raises NotFound if it doesn't exist.""" if not names: - raise ValueError("dir rm needs a path: /...") + raise ValueError(NEEDS_ENTRY) chain = _directory_chain(root, names[:-1], get) entries = list(walker.list_directory(chain[-1], get)) remaining = [e for e in entries if e.name != names[-1]] diff --git a/rnfs/content/seal.py b/rnfs/content/seal.py index 6710a0f..7bef703 100644 --- a/rnfs/content/seal.py +++ b/rnfs/content/seal.py @@ -35,6 +35,7 @@ from rnfs.constants import KIB from rnfs.content import chunker, dag +from rnfs.errors import NotRecipient, SealError # defined there, so rnfs. exports them without RNS MAGIC = b"rnfs-sealed\n" VERSION = 1 @@ -56,14 +57,6 @@ RawReader = Callable[[int, int], Iterator[bytes]] # (offset, length) -> the stored bytes of that range -class SealError(ValueError): - """A sealed file is malformed or has been tampered with.""" - - -class NotRecipient(ValueError): - """This node's identity isn't among a sealed file's recipients.""" - - def is_sealed(head: bytes) -> bool: """True if a file's first bytes carry the sealed-file magic.""" return head.startswith(MAGIC) @@ -373,7 +366,7 @@ def _decode_header(header: bytes) -> tuple[dict[bytes, bytes], bytes | None]: """{identity hash: wrapped key} and the sealed name of a strictly encoded header. Raises SealError.""" try: value = umsgpack.unpackb(header) - except Exception: + except Exception: # noqa: BLE001 - umsgpack raises several types for hostile bytes raise SealError("sealed file header isn't valid msgpack") from None if not (isinstance(value, list) and len(value) == 3): raise SealError("sealed file header must be [segment_size, recipients, name]") diff --git a/rnfs/content/shared_folder.py b/rnfs/content/shared_folder.py index 39321d7..35d685d 100644 --- a/rnfs/content/shared_folder.py +++ b/rnfs/content/shared_folder.py @@ -4,13 +4,28 @@ pinned with source "folder"; a changed or removed entry releases its old pin. Entries are recognised as unchanged by a signature of path, size, modification time and execute bit (recursively for directories), remembered across restarts (the state database's shared_folder table, rnfs.statedb: after a scan that changed -any, the entries changed are written durably), so unchanged entries aren't hashed again. +any, the entries changed are written durably), so unchanged entries aren't hashed again. Inside an entry that +changed, a regular file is not read again either if it was imported before and its device, inode, size, +modification time and execute bit are as they were (FolderFiles, audit D1): its CID is reused while every block of +it is still stored and none is denied, and its remembered content hash still goes through the deny list's check. +What the files were built as is kept in the shared_folder_files table, saved with the node's periodic saves and at +stop, not durably: a power loss costs at most reading some files again. -An entry that fails to import because of its content (an invalid name, the deny list, too large) isn't tried -again until it changes; one that fails for a reason that may pass (a full or failing disk, a pin that couldn't be -saved, a file unreadable for now) is tried again RETRY_AFTER later. A failure to pin an unchanged entry again is -logged and the scan goes on. +An entry that fails to import because of its content (the deny list, too large or too deep: an RnfsError) isn't +tried again until it changes; one that fails for a reason that may pass (a full or failing disk, a pin that couldn't +be saved, a file unreadable for now) is tried again RETRY_AFTER later, and so is one failing with a ValueError, +which the content doesn't cause (invalid names and special files inside an entry are skipped): a bug, such as a +BlockMismatch, logged with its traceback in the debug log rather than leaving the entry unshared until it changes. +A failure to pin an unchanged entry again is logged and the scan goes on. + +The folder itself may be a symlink (the owner's setting), but it must not lead into the rnfs dir: a folder that is +the rnfs dir, a directory holding it, or one inside the subdirectories holding its keys and state (PRIVATE_SUBDIRS) +is refused, with a warning: it counts as empty, so nothing in it is imported and every folder pin is released +(audit CONT-3). The check is made on the folder as opened, which the scan then walks, so a link retargeted after +it can't redirect the scan. A folder below a directory other users can change, who could retarget it, is scanned +with a warning. """ +import errno import hashlib import json import os @@ -19,11 +34,11 @@ import time from collections.abc import Callable -from rnfs import constants, statedb -from rnfs.content import builder -from rnfs.content.builder import display_path +from rnfs import constants, fsutil, statedb +from rnfs.content import builder, chunker, dag +from rnfs.content.builder import Built, FileState, display_path from rnfs.content.cid import Cid -from rnfs.content.storage import FOLDER, Storage +from rnfs.content.storage import FOLDER, Session, Storage from rnfs.errors import RnfsError, StorageFull from rnfs.log import LOG_INFO, LOG_WARNING, log @@ -93,16 +108,236 @@ def add(item: builder.Handle | None, item_path: str, relative: str, st: os.stat_ RETRY_AFTER = 600.0 # seconds before an entry whose import failed for a reason that may pass is tried again +# The rnfs dir's subdirectories holding its keys (storage/identity, names/) and state: never scanned as the folder +PRIVATE_SUBDIRS = ("storage", "names", "state", "blocks", "lxmf") + + +def opened_path(handle: builder.Handle, st: os.stat_result, path: str) -> str: + """The real path of the directory open as handle (from builder.open_path(path), whose stat is st): what the + walk of handle reads, wherever path leads by now. Raises builder.Changed if path no longer leads to it where + the system can't name an open directory, OSError if it can't be resolved. + """ + if not isinstance(handle, int): + return handle.root # a path walk's root: its real path, checked against st as it was opened + final = fsutil.final_path(handle) + if final is not None: + return final + real = os.path.realpath(path) + found = os.stat(real) + if (found.st_dev, found.st_ino) != (st.st_dev, st.st_ino): + raise builder.Changed(errno.ESTALE, "changed while being read", display_path(path)) + return real + + +def refused_reason(real: str, rnfs_dir: str) -> str | None: + """Why a shared folder whose real path is real may not be scanned, or None: it is rnfs_dir, a directory + holding it, or inside one of its PRIVATE_SUBDIRS (each resolved, as one may be a link to another disk).""" + root = os.path.realpath(rnfs_dir) + if fsutil.is_below(root, real): + what = "the rnfs dir" if os.path.normcase(real) == os.path.normcase(root) else "a directory holding the rnfs dir" + return f"it is {what}, {root}" + for name in PRIVATE_SUBDIRS: + private = os.path.realpath(os.path.join(root, name)) + if fsutil.is_below(real, private): + return f"it is inside {private}, where rnfs keeps its keys and state" + return None + + +def loose_ancestor(real: str) -> str | None: + """The first directory above real (a real path) that other users could change, so as to swap what real is + for something else, or None. POSIX only (None on Windows): one owned by another user than this one or root, + or writable by group or others, unless it is sticky (as /tmp) and what is below it there is this user's or + root's. Best effort: a directory that can't be read is passed over.""" + if fsutil.IS_WINDOWS: + return None + trusted = (0, os.geteuid()) + child, parent = real, os.path.dirname(real) + while parent != child: + try: + st, below = os.lstat(parent), os.lstat(child) + except OSError: + return None + sticky_safe = st.st_mode & stat.S_ISVTX and below.st_uid in trusted + if st.st_uid not in trusted or (st.st_mode & 0o022 and not sticky_safe): + return parent + child, parent = parent, os.path.dirname(parent) + return None + + +FILES_TABLE = "shared_folder_files" + + +def _parse_state(value) -> FileState: + """A builder.file_state saved as a JSON list. Raises ValueError (or TypeError) if it isn't one.""" + if not (isinstance(value, list) and len(value) == 5 and isinstance(value[4], bool) + and all(isinstance(n, int) and not isinstance(n, bool) for n in value[:4]) and value[2] >= 0): + raise ValueError(f"not a file state: {value!r}") + return value[0], value[1], value[2], value[3], value[4] + + +def _record(built: Built | None) -> tuple | None: + """What a FolderFiles row holds of built (its size and kind come from the state), to tell a change.""" + return None if built is None else (built.cid, built.height, built.digest) + + +class FolderFiles: + """The shared folder's per-file cache (audit D1): what each regular file imported from the folder was built as, + by its builder.file_state, so the files of a changed entry that didn't change aren't read again. + + Kept per top-level entry: an entry imported again keeps only the files it holds now, and an entry gone takes its + files with it. A file is found by its state whatever entry holds it, so one moved to another entry is reused + too. Kept in state_dir's state database (FILES_TABLE; only in memory without one), as a node.Store: save() + writes the rows changed since the last save (not durably), close() releases the database. + """ + + def __init__(self, state_dir: str | None = None): + """The cache saved in state_dir (empty without one); a row that doesn't parse is logged and deleted.""" + self._lock = threading.Lock() # held briefly, never during a build + self._entries: dict[str, dict[FileState, Built]] = {} # entry name -> its files' states and what they built + self._index: dict[FileState, Built] = {} # every entry's files, by state + self._stale = False # _index may hold files no entry holds any more (retain rebuilds it) + self._rows: statedb.RowStore | None = None + self._db = statedb.open_store(os.path.expanduser(state_dir), self._load) if state_dir else None + + def _load(self, db: statedb.StateDb): + """Read FILES_TABLE.""" + # Saved with the node's batched saves and at stop, outside the lock, not durably: what a power loss takes + # is read again + self._rows = statedb.RowStore(db, FILES_TABLE, ("name", "state"), durable=False, mode=statedb.SNAPSHOT, + lock=self._lock, row=self._row, key_values=self._key_values) + bad = [] + for name_text, state_text, binary, height, digest in db.query( + f"SELECT name, state, cid, height, digest FROM {FILES_TABLE}"): + try: + name, state = statedb.load(name_text), _parse_state(statedb.load(state_text)) + if not isinstance(name, str): + raise TypeError("not a string") + if not (isinstance(height, int) and 0 <= height <= constants.MAX_DEPTH): + raise ValueError(f"height {height!r}") + if digest is not None and not (isinstance(digest, bytes) and len(digest) == 32): + raise ValueError("not a SHA-256 digest") + kind = dag.KIND_EXECUTABLE if state[4] else dag.KIND_FILE + built = Built(Cid.from_bytes(binary), state[2], kind, height, digest) + except (ValueError, TypeError): + log(f"shared folder: ignoring invalid saved file {name_text!r} {state_text!r}", LOG_WARNING) + bad.append((name_text, state_text)) + continue + self._entries.setdefault(name, {})[state] = built + self._index[state] = built + statedb.delete_rows(db, FILES_TABLE, ("name", "state"), bad) + + @staticmethod + def _key_values(key: tuple[str, FileState]) -> tuple[str, str]: + """The name and state columns of a row's key (entry name, state).""" + name, state = key + return statedb.dump(name), statedb.dump(list(state)) + + def _row(self, key: tuple[str, FileState]) -> tuple | None: + """The row of key (entry name, state), or None if it is gone (lock held).""" + name, state = key + built = self._entries.get(name, {}).get(state) + if built is None: + return None + return (*self._key_values(key), built.cid.binary, built.height, built.digest) + + def save(self): + """Write the files changed since the last save. Raises OSError (StateDbError) if the database fails; they + are written by the next save.""" + if self._rows is not None: + self._rows.save() + + def close(self): + """Release the state database (idempotent).""" + if self._db is not None: + self._db.close() + + def get(self, state: FileState) -> Built | None: + """What the file in state was built as, if known.""" + with self._lock: + return self._index.get(state) + + def update(self, name: str, found: dict[FileState, Built], complete: bool): + """Record what an import of entry name found: every file it holds if complete (the import succeeded), else + those the import got to, added to what was known (it is tried again, and they are still what they were).""" + with self._lock: + old = self._entries.get(name, {}) + self._set(name, dict(found) if complete else {**old, **found}) + self._index.update(found) + + def retain(self, names: set[str]): + """Forget the files of the entries not in names (gone from the folder, or a folder refused), at the end of a + scan.""" + with self._lock: + for name in [name for name in self._entries if name not in names]: + self._set(name, {}) + if self._stale: + self._index = {state: built for files in self._entries.values() for state, built in files.items()} + self._stale = False + + def _set(self, name: str, files: dict[FileState, Built]): + """Make files entry name's (none: forget it), noting the rows that change. Lock held.""" + old = self._entries.get(name, {}) + if old.keys() - files.keys(): + self._stale = True + if files: + self._entries[name] = files + else: + self._entries.pop(name, None) + if self._rows is not None: + self._rows.changed(*((name, state) for state in old.keys() | files.keys() + if _record(old.get(state)) != _record(files.get(state)))) + + +class _EntryFiles: + """One import's builder.FileCache, over the folder's FolderFiles. + + A file known by its state stands for reading it only while every block of it is stored (protected first, in the + import's session, so a collection can't take them before the pin) and none is denied: such a file is read again, + and the deny list refuses its blocks' put, failing the import as before. found collects what the import's files + were built as, for FolderFiles.update. + """ + + def __init__(self, files: FolderFiles, storage: Storage, session: Session): + """A cache for one import storing through session.""" + self.files = files + self.storage = storage + self.session = session + self.found: dict[FileState, Built] = {} + + def lookup(self, st: os.stat_result, kind: int) -> Built | None: + """builder.FileCache.lookup.""" + state = builder.file_state(st) + known = self.files.get(state) + if known is None or known.kind != kind: + return None + self.session.add(known.cid) + if not self.storage.is_complete(known.cid): + return None # a block was evicted, collected or deleted as damaged: read again, which stores it + denied = self.storage.deny.cids + if denied and self.storage.denied_roots([known.cid], denied): + return None + self.found[state] = known + return known + + def remember(self, st: os.stat_result, built: Built): + """builder.FileCache.remember (a file whose execute bit and kind disagree, changed between listing and + opening, isn't remembered).""" + state = builder.file_state(st) + if built.kind == (dag.KIND_EXECUTABLE if state[4] else dag.KIND_FILE): + self.found[state] = built class SharedFolder: """Keeps the shared folder's entries imported and pinned with source "folder".""" def __init__(self, directory: str, storage: Storage, state_dir: str | None = None, - clock: Callable[[], float] = time.monotonic): + clock: Callable[[], float] = time.monotonic, rnfs_dir: str | None = None): """Track directory (created if missing at the first scan); the signature cache lives in state_dir's - state database (not kept without one). clock times the retries of failed imports.""" + state database (not kept without one). clock times the retries of failed imports. rnfs_dir is the rnfs + dir the folder must not lead into (by default directory's parent: the folder is /files).""" self.directory = os.path.expanduser(directory) + self.rnfs_dir = (os.path.expanduser(rnfs_dir) if rnfs_dir is not None + else os.path.dirname(os.path.normpath(os.path.abspath(self.directory)))) self.storage = storage self.clock = clock self._db = statedb.connect(os.path.expanduser(state_dir)) if state_dir else None @@ -113,6 +348,8 @@ def __init__(self, directory: str, storage: Storage, state_dir: str | None = Non self._failed: dict[str, tuple[str, float | None]] = {} self._warned: set[str] = set() # skipped entries already warned about try: + # What the files inside entries were built as (D1): a store of its own, which the node saves + self.files = FolderFiles(state_dir) self._load() except BaseException: self.close() @@ -156,12 +393,20 @@ def _save(self): self._saved = dict(self._known) def close(self): - """Release the state database (the node calls this once it has saved everything at shutdown).""" + """Release the state database, the per-file cache's too (the node calls this once it has saved everything + at shutdown; the cache's changes are saved by files.save(), not here).""" + files = getattr(self, "files", None) # not made if __init__ failed first + if files is not None: + files.close() if self._db is not None: self._db.close() - def scan(self) -> set[Cid]: - """Import new and changed entries, release the pins of changed and removed ones; returns the folder's pins.""" + def scan(self, options: builder.BuildOptions | None = None) -> set[Cid]: + """Import new and changed entries, release the pins of changed and removed ones; returns the folder's pins. + + options: the chunker and the deny list's file-hash check every import is built with (builder.BuildOptions; + by default those of this context, builder.current_options).""" + options = options if options is not None else builder.current_options() with self._lock: os.makedirs(self.directory, exist_ok=True) current: dict[str, tuple[str, Cid]] = {} @@ -170,30 +415,52 @@ def scan(self) -> set[Cid]: try: if not stat.S_ISDIR(st.st_mode): raise NotADirectoryError(f"{display_path(self.directory)} is not a directory") - items = builder.list_directory(folder) + # A refused folder counts as empty: every folder pin is released, so a node whose folder already + # led into the rnfs dir (as 0.1.5 allowed) stops providing what it imported from there + items = builder.list_directory(folder) if self._check_target(folder, st) else [] for item in items: - result = self._scan_item(folder, item) + result = self._scan_item(folder, item, options) if result is not None: current[item.name] = result finally: builder.close(folder) + names = {item.name for item in items} + self.files.retain(names) pins = {cid for _, cid in current.values()} for cid in self.storage.pinned_by(FOLDER) - pins: self.storage.unpin(cid, FOLDER) if current != self._known: self._known = current self._save() - names = {item.name for item in items} self._failed = {name: sig for name, sig in self._failed.items() if name in names} return pins + def _check_target(self, folder: builder.Handle, st: os.stat_result) -> bool: + """False, with a warning while it stays so, if the folder open as folder (whose stat is st) may not be + scanned (refused_reason; scan then treats it as empty); a warning, once per directory, about a loose ancestor (loose_ancestor).""" + real = opened_path(folder, st, self.directory) + reason = refused_reason(real, self.rnfs_dir) + if reason is not None: + self._warn_once("target", f"not scanning {display_path(self.directory)}: {reason}. Its pins with " + f"source folder were released; point it at another directory") + return False + self._warned.discard("target") # so a folder refused again warns again + loose = loose_ancestor(real) + if loose is not None: + self._warn_once(f"loose:{loose}", f"other users can change {display_path(loose)}, above " + f"{display_path(real)}: they could replace the folder with another " + f"directory, to have it shared. Keep every directory above it yours") + return True + def _warn_once(self, key: str, message: str): - """Log a warning about a skipped entry, once per entry name and reason while the daemon runs.""" + """Log a warning about a skipped entry (or the folder), once per entry name and reason while the daemon + runs.""" if key not in self._warned: self._warned.add(key) log(f"shared folder: {message}", LOG_WARNING) - def _scan_item(self, folder: builder.Handle, item: os.DirEntry) -> tuple[str, Cid] | None: + def _scan_item(self, folder: builder.Handle, item: os.DirEntry, + options: builder.BuildOptions) -> tuple[str, Cid] | None: """(signature, CID) of a top-level entry, importing and pinning it if new or changed; None if skipped.""" path = os.path.join(self.directory, item.name) shown = display_path(path) @@ -217,12 +484,13 @@ def _scan_item(self, folder: builder.Handle, item: os.DirEntry) -> tuple[str, Ci if links: self._warn_once(f"hardlink:{item.name}", f"skipped {shown}: {builder.hard_link_reason(links)}") return None - return self._import(item.name, handle, st, path) + return self._import(item.name, handle, st, path, options) finally: builder.close(handle) - def _import(self, name: str, handle: builder.Handle, st: os.stat_result, path: str) -> tuple[str, Cid] | None: - """_scan_item for an entry open as handle.""" + def _import(self, name: str, handle: builder.Handle, st: os.stat_result, path: str, + options: builder.BuildOptions) -> tuple[str, Cid] | None: + """_scan_item for an entry open as handle, built with options.""" shown = display_path(path) try: sig = open_signature(handle, st, path) @@ -239,20 +507,35 @@ def _import(self, name: str, handle: builder.Handle, st: os.stat_result, path: s if failed is not None and failed[0] == sig and (failed[1] is None or self.clock() < failed[1]): return None warnings: list[str] = [] + files: _EntryFiles | None = None try: with self.storage.protect() as session: + # What files were built as is reused only for the chunker it was built with (the folder's: fixed) + if options.chunker == chunker.FIXED: + files = _EntryFiles(self.files, self.storage, session) with session.writer() as put: - built = builder.build_open(handle, st, path, put, warnings, skip_hard_links=True) + built = builder.build_open(handle, st, path, put, warnings, skip_hard_links=True, options=options, + cache=files) self.storage.pin(built.cid, FOLDER) - except (OSError, StorageFull) as e: # may pass (a full or failing disk, a pin not saved): tried again later + except (OSError, StorageFull, ValueError) as e: + # May pass (a full or failing disk, a pin not saved): tried again later. So is a ValueError (a + # BlockMismatch too), which the content doesn't cause (see the module docstring): a bug, shown by type self._failed[name] = (sig, self.clock() + RETRY_AFTER) - log(f"shared folder: couldn't import {shown}: {e}", LOG_WARNING) + log(f"shared folder: couldn't import {shown}: {repr(e) if isinstance(e, ValueError) else e}", LOG_WARNING) + self._keep_files(name, files, complete=False) return None - except (ValueError, RnfsError) as e: # the content's: not until it changes + except RnfsError as e: # the content's: not until it changes self._failed[name] = (sig, None) log(f"shared folder: couldn't import {shown}: {e}", LOG_WARNING) + self._keep_files(name, files, complete=False) return None + self._keep_files(name, files, complete=True) for warning in warnings: log(f"shared folder: {warning}", LOG_WARNING) log(f"shared folder: imported {shown} as {built.cid}", LOG_INFO) return sig, built.cid + + def _keep_files(self, name: str, files: _EntryFiles | None, complete: bool): + """Record in self.files what an import of entry name found its files built as (FolderFiles.update).""" + if files is not None: + self.files.update(name, files.found, complete) diff --git a/rnfs/content/storage.py b/rnfs/content/storage.py index 1fbc23b..791a682 100644 --- a/rnfs/content/storage.py +++ b/rnfs/content/storage.py @@ -20,11 +20,10 @@ at shutdown, writing only the records changed since the last save. Thread-safe: one lock guards the indexes; long walks run outside it: completeness walks are discarded if blocks were deleted meanwhile, and eviction, gc and the size queries read the DAG nodes they will walk beforehand (_warm), deciding what to delete -(or counting) under the lock. +(or counting) under the lock, where a DAG memoised as complete is taken whole rather than walked again. """ import itertools import os -import re import shutil import threading from collections import OrderedDict @@ -32,13 +31,13 @@ from contextlib import contextmanager from typing import Callable, Iterator -from rnfs import denylist, statedb -from rnfs.constants import MAX_BLOCK_SIZE, MIB -from rnfs.content import dag, walker +from rnfs import denylist, ids, statedb +from rnfs.constants import MIB +from rnfs.content import blockstore, dag, walker from rnfs.content.blockstore import BlockStore from rnfs.content.cid import Cid from rnfs.content.dag import DirectoryNode, IndexNode -from rnfs.errors import Denied, InvalidDag, NotFound, StorageFull +from rnfs.errors import BlockMismatch, Denied, InvalidDag, NotFound, StorageFull from rnfs.log import LOG_ERROR, LOG_VERBOSE, log, plural, size USER, FOLDER = "user", "folder" @@ -46,6 +45,7 @@ NAME_PREFIX, REMOTE_PREFIX = "name:", "remote:" LXMF_PREFIX = "lxmf:" MAX_CACHE_RECORDS = 10_000 # cached fetches remembered; past this the least recently used record is dropped +MISSING_MEMO = 10_000 # incomplete DAGs whose missing block dag_blocks remembers; past this the oldest is forgotten EVICT_MARGIN = 32 # eviction frees max_storage / EVICT_MARGIN more than it must, so it runs rarely DISK_RESERVE_MIN = 256 * MIB # free disk space fetches always leave on the blocks' filesystem... DISK_RESERVE_FRACTION = 0.02 # ...or this fraction of the filesystem, if larger @@ -56,8 +56,11 @@ DAMAGED = object() # Storage._node's answer for a stored DAG node that is unreadable or no longer matches its CID MISSING = object() # Storage._read_node's answer for a block that isn't stored (or was deleted meanwhile) +_NO_MEMO = object() # a memo's answer for a key it has no entry for (None is a memoised answer) -_IDENTITY_HASH = re.compile(r"[0-9a-f]{32}") +# The ID in a remote:, follow: or lxmf: pin source, as bytes.hex() writes it: exactly what loaded before rnfs.ids, as +# a pin row that fails is deleted +_IDENTITY_HASH = ids.HEX16_LOWER def disk_reserve(total: int) -> int: @@ -116,27 +119,69 @@ def import_pins(value: dict, tx) -> int: return count +CacheKey = tuple[Cid, tuple[str, ...]] # a cached fetch's root and path names + + +def _cache_names(names) -> tuple[str, ...]: + """A cache record's path names, checked: a list of valid names. Raises TypeError or ValueError.""" + if not isinstance(names, list): + raise TypeError("names aren't a list") + for name in names: + dag.check_name(name) + return tuple(names) + + +def _cache_row(key: CacheKey) -> tuple[bytes, str]: + """The cache table's key columns (root, path) for a record's key.""" + root, names = key + return root.binary, statedb.dump(list(names)) + + +def _parse_cache_row(root: bytes, path: str) -> CacheKey: + """A record's key from the cache table's key columns, checked. Raises TypeError or ValueError.""" + return Cid.from_bytes(root), _cache_names(statedb.load(path)) + + def import_cache(value: dict, tx) -> int: """Import cache.json (statedb.migrate): its records, least recently used first, replace the cache table, each checked as rnfs 0.1.4 loaded it. Returns the number of records imported.""" tx.execute("DELETE FROM cache") - records: OrderedDict[tuple[Cid, tuple[str, ...]], Cid] = OrderedDict() + records: OrderedDict[CacheKey, Cid] = OrderedDict() for record in value.get("fetches", []): try: root, names, target = record - if not isinstance(names, list): - raise TypeError("names aren't a list") - for name in names: - dag.check_name(name) - records[(Cid.parse(root), tuple(names))] = Cid.parse(target) # a repeated key keeps its first place + names = _cache_names(names) + records[(Cid.parse(root), names)] = Cid.parse(target) # a repeated key keeps its first place except (ValueError, TypeError): log(f"storage: ignoring invalid cache record {record!r}", LOG_ERROR) tx.executemany("INSERT INTO cache (root, path, target, used) VALUES (?, ?, ?, ?)", - [(root.binary, statedb.dump(list(names)), target.binary, used) - for used, ((root, names), target) in enumerate(records.items())]) + [(*_cache_row(key), target.binary, used) for used, (key, target) in enumerate(records.items())]) return len(records) +def _walk_path(root: Cid, names: tuple[str, ...], step: Callable[[Cid, str], tuple[Cid, bool] | None]) \ + -> Iterator[Cid]: + """The DAG nodes a walk along / passes through (not the target it reaches), in order: the walk + that keeps a cached fetch's path, written once for Storage._path_cids (under the lock) and Storage._warm + (without it), so warming covers exactly what the locked walk reads. + + step(cid, name) is where name leads from the node cid (Storage._step_in's answer), or None if the walk ends + there: no such name, not a directory, or cid can't be read. The walk also ends at a raw block, unyielded. + """ + current = root + for name in names: + while True: + if current.is_raw: + return + yield current + answer = step(current, name) + if answer is None: + return + current, through_directory = answer + if through_directory: + break # the next name + + class Session: """A protection session: blocks it stores or lists are kept as if pinned until it ends. @@ -154,7 +199,7 @@ def add(self, cid: Cid): self.cids.add(cid) def put(self, cid: Cid, block: bytes): - """Store a block and protect it. Raises ValueError if it doesn't match its CID.""" + """Store a block and protect it. Raises BlockMismatch (a ValueError) if it doesn't match its CID.""" self.add(cid) # first, so an eviction between storing and protecting can't take it self._storage._put(cid, block, protected=True) @@ -199,7 +244,7 @@ def _write(self, cid: Cid, block: bytes): """Store one block on a writer thread, keeping the first error.""" try: self._session._storage._put(cid, block, protected=True) - except BaseException as e: + except BaseException as e: # noqa: BLE001 - kept and raised on the caller's thread if self._error is None: self._error = e finally: @@ -232,17 +277,22 @@ def __init__(self, blocks_dir: str, state_dir: str, max_storage: int): self._sizes: dict[Cid, int] = dict(self.blocks.scan()) self._used = sum(self._sizes.values()) self._pins: dict[Cid, set[str]] = {} - self._cache: OrderedDict[tuple[Cid, tuple[str, ...]], Cid] = OrderedDict() # oldest first + self._pinned: frozenset[Cid] | None = None # pinned_cids(), built when asked; None again once pins change + self._cache: OrderedDict[CacheKey, Cid] = OrderedDict() # oldest first self._sessions: list[Session] = [] self._links: dict[Cid, tuple[Cid, ...]] = {} # children of stored DAG nodes; CIDs never change content # Where a path step leads from a stored DAG node: node -> {name: _step_in's answer}; memoised like _links self._steps: dict[Cid, dict[str, tuple[Cid, bool] | None]] = {} self._complete: dict[Cid, frozenset[Cid]] = {} # dag_blocks memo, complete DAGs only + self._complete_bytes: dict[Cid, int] = {} # dag_bytes memo: the bytes of _complete's DAGs, as asked for + # dag_blocks memo for incomplete DAGs: root -> a block of its DAG found missing. A CID's content never + # changes, so the DAG is incomplete for as long as that block isn't stored (checked at each lookup) + self._missing: dict[Cid, Cid] = {} self._generation = 0 # incremented whenever blocks are deleted, invalidating walks in progress self._cache_generation = 0 # incremented whenever cache records are removed or retargeted self._changes = 0 # incremented with each block stored, pin, release and new cache record (changes()) # Cache records changed (or removed) since the last save(); every one is among the most recently used - self._cache_dirty: set[tuple[Cid, tuple[str, ...]]] = set() + self._cache_dirty: set[CacheKey] = set() self._cache_used = 0 # the cache table's next used value: larger than every one saved # False only when nothing has happened since the last collection that could leave a block unreached, # so eviction over max_storage with nothing to evict needn't walk everything again @@ -273,21 +323,14 @@ def _load(self): bad_cache = [] for root, path, target, used in self._db.query("SELECT root, path, target, used FROM cache ORDER BY used"): try: - names = statedb.load(path) - if not isinstance(names, list): - raise TypeError("names aren't a list") - for name in names: - dag.check_name(name) - key = (Cid.from_bytes(root), tuple(names)) - self._cache[key] = Cid.from_bytes(target) + self._cache[_parse_cache_row(root, path)] = Cid.from_bytes(target) except (ValueError, TypeError): log(f"storage: ignoring invalid cache record {root!r} {path!r}", LOG_ERROR) bad_cache.append((root, path)) continue if isinstance(used, int): self._cache_used = max(self._cache_used, used + 1) - capped = [(root.binary, statedb.dump(list(names))) for root, names in - list(self._cache)[:max(0, len(self._cache) - MAX_CACHE_RECORDS)]] + capped = [_cache_row(key) for key in list(self._cache)[:max(0, len(self._cache) - MAX_CACHE_RECORDS)]] self._cap_cache() self._cache_dirty.clear() if bad_pins or bad_cache or capped: @@ -314,11 +357,10 @@ def save(self): # record to the end), so those still cached are the last ones, in order kept = sum(1 for key in dirty if key in self._cache) changed = list(itertools.islice(reversed(self._cache.items()), kept))[::-1] - rows = [(root.binary, statedb.dump(list(names)), target.binary, self._cache_used + n) - for n, ((root, names), target) in enumerate(changed)] + rows = [(*_cache_row(key), target.binary, self._cache_used + n) + for n, (key, target) in enumerate(changed)] self._cache_used += len(rows) - gone = [(root.binary, statedb.dump(list(names))) for root, names in dirty if - (root, names) not in self._cache] + gone = [_cache_row(key) for key in dirty if key not in self._cache] try: self.blocks.sync() with self._db.transaction() as tx: @@ -347,7 +389,7 @@ def get(self, cid: Cid, verify: bool = False) -> bytes | None: if cid not in self._sizes: return None block = self.blocks.get(cid) - if block is None or (verify and not (len(block) <= MAX_BLOCK_SIZE and cid.verify(block))): + if block is None or (verify and not blockstore.matches(cid, block)): with self._lock: if cid in self._sizes and not self.blocks.verify(cid): log(f"storage: block {cid} is damaged or missing; removing it", LOG_ERROR) @@ -361,8 +403,8 @@ def block_size(self, cid: Cid) -> int | None: return self._sizes.get(cid) def put(self, cid: Cid, block: bytes): - """Store a block after verifying it (atomic, durable). Raises ValueError if it doesn't match or is too - large, even when the block is already stored, and Denied if the deny list denies it. + """Store a block after verifying it (atomic, durable). Raises BlockMismatch (a ValueError) if it doesn't + match or is too large, even when the block is already stored, and Denied if the deny list denies it. A block stored outside a protection session is cache that no record reaches: the first to go. """ @@ -376,8 +418,8 @@ def _put(self, cid: Cid, block: bytes, protected: bool): stored = cid in self._sizes if stored: # Verified all the same, so callers (archive import) never count unchecked bytes as supplied - if len(block) > MAX_BLOCK_SIZE or not cid.verify(block): - raise ValueError(f"block does not match {cid}") + if not blockstore.matches(cid, block): + raise BlockMismatch(f"block does not match {cid}") return self.blocks.put(cid, block) # verifies; outside the lock, since it writes to disk with self._lock: @@ -430,6 +472,7 @@ def pin_many(self, cids, source: str): if not added: return self._changes += 1 + self._pinned = None try: self._save_pins(source, added=added) except BaseException: @@ -439,6 +482,7 @@ def pin_many(self, cids, source: str): sources.discard(source) if not sources: del self._pins[cid] + self._pinned = None raise def unpin(self, cid: Cid, source: str) -> bool: @@ -452,10 +496,12 @@ def unpin(self, cid: Cid, source: str) -> bool: if not sources: del self._pins[cid] self._maybe_garbage = True + self._pinned = None try: self._save_pins(source, removed=[cid]) except BaseException: self._pins.setdefault(cid, set()).add(source) # still pinned, as the database says + self._pinned = None raise return True @@ -464,6 +510,14 @@ def pins(self) -> dict[Cid, set[str]]: with self._lock: return {cid: set(sources) for cid, sources in self._pins.items()} + def pinned_cids(self) -> frozenset[Cid]: + """Every pinned CID: the same frozenset until pins change, so asking after every add costs nothing where + pins() would copy every pin and its sources.""" + with self._lock: + if self._pinned is None: + self._pinned = frozenset(self._pins) + return self._pinned + def pinned_by(self, source: str) -> set[Cid]: """CIDs held by source.""" with self._lock: @@ -474,6 +528,12 @@ def is_pinned(self, cid: Cid) -> bool: with self._lock: return cid in self._pins + def has_pin(self, cid: Cid, source: str) -> bool: + """True if source pins cid (a lookup, where pins() would copy every pin).""" + with self._lock: + sources = self._pins.get(cid) + return sources is not None and source in sources + def pin_count(self) -> int: """Number of pinned CIDs.""" with self._lock: @@ -486,31 +546,41 @@ def record_fetch(self, root: Cid, names: list[str] | tuple[str, ...], target: Ci Returns the targets of records dropped to stay within MAX_CACHE_RECORDS, no longer cached. """ - key = (root, tuple(names)) with self._lock: - previous = self._cache.get(key) - if previous is None: - self._changes += 1 - elif previous != target: - self._maybe_garbage = True - self._cache_generation += 1 - self._cache[key] = target - self._cache.move_to_end(key) - self._cache_dirty.add(key) + self._set_record((root, tuple(names)), target) dropped = self._cap_cache() return dropped - set(self._cache.values()) - def _cap_cache(self) -> set[Cid]: - """Drop the least recently used records past MAX_CACHE_RECORDS; their targets (lock held).""" - dropped = set() - while len(self._cache) > MAX_CACHE_RECORDS: - key, target = self._cache.popitem(last=False) - dropped.add(target) + def _set_record(self, key: CacheKey, target: Cid): + """Record key -> target as the most recently used, with the bookkeeping a new or retargeted record needs: + a retargeted one may leave its old target's blocks unreached (lock held).""" + previous = self._cache.get(key) + if previous is None: + self._changes += 1 + elif previous != target: + self._maybe_garbage = True + self._cache_generation += 1 + self._cache[key] = target + self._cache.move_to_end(key) + self._cache_dirty.add(key) + + def _drop_records(self, keys) -> list[Cid]: + """Remove the records of keys, with the bookkeeping every removal needs: the save deletes their rows + (_cache_dirty), _prune_fetched runs (_cache_generation), and the next collection looks for what they + left unreached (_maybe_garbage); their targets, in order, a target once per record (lock held).""" + targets = [] + for key in keys: + targets.append(self._cache.pop(key)) self._cache_dirty.add(key) - if dropped: + if targets: self._maybe_garbage = True self._cache_generation += 1 - return dropped + return targets + + def _cap_cache(self) -> set[Cid]: + """Drop the least recently used records past MAX_CACHE_RECORDS; their targets (lock held).""" + excess = max(0, len(self._cache) - MAX_CACHE_RECORDS) + return set(self._drop_records(list(itertools.islice(self._cache, excess)))) def touch(self, root: Cid, names: list[str] | tuple[str, ...]) -> bool: """Mark a cached fetch as used now; False if there is no such record.""" @@ -583,7 +653,7 @@ def _read_node(self, cid: Cid): return DAMAGED if block is None: return MISSING - if len(block) > MAX_BLOCK_SIZE or not cid.verify(block): + if not blockstore.matches(cid, block): return DAMAGED try: return dag.decode(block) @@ -593,22 +663,19 @@ def _read_node(self, cid: Cid): @staticmethod def _listed(node) -> tuple[Cid, ...]: """The CIDs a decoded DAG node lists; none for None (raw, missing or invalid).""" - if node is None: - return () - if isinstance(node, DirectoryNode): - return tuple(entry.cid for entry in node.entries) - return tuple(child.cid for child in node.children) + return () if node is None else node.child_cids @staticmethod def _step_in(node, name: str) -> tuple[Cid, bool] | None: """Where the path step name leads from a decoded node: (entry, True) through a directory, (child, False) - to the index node child that may hold it, or None (no such entry, or not a directory).""" + to the index node child that may hold it (walker.index_child), or None (no such entry, or not a + directory).""" if isinstance(node, DirectoryNode): entry = next((e for e in node.entries if e.name == name), None) return None if entry is None else (entry.cid, True) if isinstance(node, IndexNode): - candidates = [c for c in node.children if c.first_name <= name] - return (candidates[-1].cid, False) if candidates else None + position = walker.index_child(node, name) + return None if position is None else (node.children[position].cid, False) return None def _children(self, cid: Cid) -> tuple[Cid, ...] | None: @@ -625,15 +692,25 @@ def _children(self, cid: Cid) -> tuple[Cid, ...] | None: return listed def _reach(self, roots, known: set[Cid] | frozenset[Cid] = frozenset(), - damaged: set[Cid] | None = None) -> set[Cid]: + damaged: set[Cid] | None = None, partial: set[Cid] | frozenset[Cid] = frozenset()) -> set[Cid]: """Stored blocks reachable from roots through stored DAG nodes, not passing through known; damaged - nodes reached are added to damaged (lock held).""" + nodes reached are added to damaged (lock held). + + A DAG memoised as complete (_complete) is taken whole, less known, rather than walked: every block of it + is stored, and the walk would reach exactly those not in known, since known holds the whole DAG below + each of its blocks, except partial ones: blocks of known whose stored children may not all be in known + (a kept record's path nodes, in _evict_to). A complete DAG holding one of those is walked as before. + """ reached: set[Cid] = set() stack = [cid for cid in roots if cid in self._sizes and cid not in known] while stack: cid = stack.pop() if cid in reached: continue + blocks = self._complete.get(cid) + if blocks is not None and partial.isdisjoint(blocks): + reached |= blocks - known if known else blocks + continue reached.add(cid) listed = self._children(cid) if listed is None: @@ -647,27 +724,22 @@ def _reach(self, roots, known: set[Cid] | frozenset[Cid] = frozenset(), def _path_cids(self, root: Cid, names: tuple[str, ...], damaged: set[Cid] | None = None) -> set[Cid]: """The stored nodes along / (not their other children), as far as they're stored; damaged nodes met are added to damaged (lock held).""" - found: set[Cid] = set() - current = root - for name in names: - while True: - if current.is_raw or current not in self._sizes: - return found - found.add(current) - step = self._steps.get(current, {}).get(name, MISSING) - if step is MISSING: - node = self._node(current) - if node is DAMAGED: - if damaged is not None: - damaged.add(current) - return found # not memoised, as in _children - step = self._steps.setdefault(current, {})[name] = self._step_in(node, name) - if step is None: - return found - current, through_directory = step - if through_directory: - break # the next name - return found + + def step(cid: Cid, name: str) -> tuple[Cid, bool] | None: + """_step_in's answer at cid, memoised (_steps); None if cid isn't stored, or is damaged.""" + if cid not in self._sizes: + return None + answer = self._steps.get(cid, {}).get(name, _NO_MEMO) + if answer is _NO_MEMO: + node = self._node(cid) + if node is DAMAGED: + if damaged is not None: + damaged.add(cid) + return None # not memoised, as in _children + answer = self._steps.setdefault(cid, {})[name] = self._step_in(node, name) + return answer + + return {cid for cid in _walk_path(root, names, step) if cid in self._sizes} def _warm(self, roots: set[Cid], paths: list[tuple[Cid, tuple[str, ...]]]): """Read and decode, without the lock, the DAG nodes a collection will walk from roots and along paths, @@ -699,29 +771,20 @@ def _warm(self, roots: set[Cid], paths: list[tuple[Cid, tuple[str, ...]]]): listed = found_links[cid] = self._listed(node) stack.extend(child for child in listed if child not in seen and not child.is_raw) - def step(cid: Cid, name: str): - """_step_in's answer for cid, memoised or read; MISSING if cid can't be read.""" - known = steps.get(cid, {}).get(name, found_steps.get((cid, name), MISSING)) - if known is MISSING: + def step(cid: Cid, name: str) -> tuple[Cid, bool] | None: + """_step_in's answer at cid, memoised or read; None if cid can't be read (the walk under the lock + reads it).""" + known = steps.get(cid, {}).get(name, found_steps.get((cid, name), _NO_MEMO)) + if known is _NO_MEMO: node = self._read_node(cid) if node is MISSING or node is DAMAGED: - return MISSING + return None known = found_steps[(cid, name)] = self._step_in(node, name) return known for root, names in paths: - current, ended = root, False - for name in names: - while not ended: - result = None if current.is_raw else step(current, name) - if result is None or result is MISSING: - ended = True # the path ends here, or the walk under the lock reads it - else: - current, through_directory = result - if through_directory: - break # the next name - if ended: - break + for _ in _walk_path(root, names, step): + pass # step memoises as the walk goes: the same walk _path_cids makes under the lock with self._lock: for cid, listed in found_links.items(): if cid in self._sizes: @@ -777,6 +840,7 @@ def _delete(self, cids: set[Cid]) -> tuple[set[Cid], int]: self._used -= freed self._generation += 1 self._complete = {root: blocks for root, blocks in self._complete.items() if not blocks & deleted} + self._complete_bytes = {root: n for root, n in self._complete_bytes.items() if root in self._complete} if deleted or deferred: # A deleted DAG node no longer reaches its children; a deferred block is still garbage self._maybe_garbage = True @@ -804,25 +868,29 @@ def _evict_to(self, limit: int) -> tuple[set[Cid], int, int]: return set(), 0, 0 # only pins and sessions hold blocks: there is nothing to evict damaged: set[Cid] = set() live = self._reach(self._roots(), damaged=damaged) - kept = sum(self._sizes[cid] for cid in live) + # What pins and sessions don't reach, rather than what they do: the bytes kept follow from it with one + # pass over the stored blocks, and the blocks to delete are what is left of it once records are kept + dead = self._sizes.keys() - live + kept = self._used - sum(map(self._sizes.__getitem__, dead)) + partial: set[Cid] = set() # path nodes kept without the rest of their DAG: see _reach records = list(self._cache.items()) # oldest first keep = 0 for (root, names), target in reversed(records): - more = self._reach({target}, live, damaged) | (self._path_cids(root, names, damaged) - live) - more_bytes = sum(self._sizes[cid] for cid in more) + reached = self._reach({target}, live, damaged, partial) + path = self._path_cids(root, names, damaged) - live + more = reached | path + more_bytes = sum(map(self._sizes.__getitem__, more)) if kept + more_bytes > limit: break live |= more + dead -= more + partial |= path - reached kept += more_bytes keep += 1 if self._held_back(damaged): return set(), 0, 0 evicted = len(records) - keep - for _ in range(evicted): - self._cache_dirty.add(self._cache.popitem(last=False)[0]) - if evicted: - self._cache_generation += 1 - dead = {cid for cid in self._sizes if cid not in live} + self._drop_records([key for key, _ in records[:evicted]]) deleted, freed = self._delete(dead) self._maybe_garbage = deleted != dead # anything deferred is collected on a later pass return deleted, freed, evicted @@ -832,11 +900,17 @@ def _evict_to(self, limit: int) -> tuple[set[Cid], int, int]: def dag_blocks(self, cid: Cid) -> frozenset[Cid]: """The blocks of cid's DAG, which must be completely stored and valid (memoised). - Raises NotFound if a block is missing, InvalidDag if the DAG breaks a reader check. + Raises NotFound if a block is missing, InvalidDag if the DAG breaks a reader check. A missing block is + remembered (_missing), so asking again while it is still missing walks nothing. """ with self._lock: if cid in self._complete: return self._complete[cid] + missing = self._missing.get(cid) + if missing is not None: + if missing not in self._sizes: + raise NotFound(f"block {missing} is not stored") + del self._missing[cid] # stored since: walk again generation = self._generation found: set[Cid] = set() seen: set[walker.Expect] = set() @@ -849,12 +923,12 @@ def dag_blocks(self, cid: Cid) -> frozenset[Cid]: if expect.cid.is_raw: length = self.block_size(expect.cid) if length is None: - raise NotFound(f"block {expect.cid} is not stored") + self._lacks(cid, expect.cid) walker.check_raw(expect, length) # stored blocks were verified, so their length is enough else: block = self.get(expect.cid) if block is None: - raise NotFound(f"block {expect.cid} is not stored") + self._lacks(cid, expect.cid) stack.extend(walker.expand(expect, block)) found.add(expect.cid) result = frozenset(found) @@ -862,10 +936,22 @@ def dag_blocks(self, cid: Cid) -> frozenset[Cid]: # A deletion during the walk may have removed a block already counted if self._generation == generation: self._complete[cid] = result - elif not all(block in self._sizes for block in result): - raise NotFound(f"a block of {cid} was deleted") + else: + gone = next((block for block in result if block not in self._sizes), None) + if gone is not None: + self._lacks(cid, gone, f"a block of {cid} was deleted") return result + def _lacks(self, root: Cid, block: Cid, message: str | None = None): + """Remember that block, part of root's DAG, isn't stored (_missing), and raise NotFound (with message, + if given).""" + with self._lock: + if block not in self._sizes: # else stored meanwhile: nothing to remember + if root not in self._missing and len(self._missing) >= MISSING_MEMO: + del self._missing[next(iter(self._missing))] # the oldest + self._missing[root] = block + raise NotFound(message or f"block {block} is not stored") + def collect(self) -> int: """Delete every block nothing reaches (what an add or import refused left behind); how many.""" with self._lock: @@ -898,25 +984,23 @@ def drop_denied(self, denied: frozenset[bytes]) -> dict: targets = self._targets() if not denied or not records: return {"uncached": set(), "removed_blocks": 0, "freed_bytes": 0, "incomplete": set()} - before = self._complete_sets(targets) - self._warm(set(records.values()) | {root for root, _ in records}, [key for key in records if key[1]]) - with self._lock: - before = self._with_memoised(before) - dropped = [key for key, target in self._cache.items() - if any(cid.binary in denied - for cid in {key[0], target} | self._path_cids(*key) | self._reach([target]))] - uncached = set() - for key in dropped: - uncached.add(self._cache.pop(key)) - self._cache_dirty.add(key) - uncached -= set(self._cache.values()) - if dropped: - self._cache_generation += 1 - self._maybe_garbage = True - deleted, freed = self._collect() if dropped else (set(), 0) - incomplete = {cid for cid, blocks in before.items() if blocks & deleted} - return {"uncached": uncached, "removed_blocks": len(deleted), "freed_bytes": freed, - "incomplete": incomplete} + stats = {} + + def act(before: dict[Cid, frozenset[Cid]]) -> set[Cid]: + """Drop the records reaching a denied block and collect; the blocks deleted.""" + self._warm(set(records.values()) | {root for root, _ in records}, [key for key in records if key[1]]) + with self._lock: + self._add_memoised(before) + dropped = [key for key, target in self._cache.items() + if any(cid.binary in denied + for cid in {key[0], target} | self._path_cids(*key) | self._reach([target]))] + uncached = set(self._drop_records(dropped)) - set(self._cache.values()) + deleted, freed = self._collect() if dropped else (set(), 0) + stats.update(uncached=uncached, removed_blocks=len(deleted), freed_bytes=freed) + return deleted + + stats["incomplete"] = self._newly_incomplete(targets, act) + return stats def is_complete(self, cid: Cid) -> bool: """True if every block of cid's DAG is stored and the DAG is valid.""" @@ -926,6 +1010,18 @@ def is_complete(self, cid: Cid) -> bool: except (NotFound, InvalidDag): return False + def _newly_incomplete(self, targets: set[Cid], act: Callable[[dict[Cid, frozenset[Cid]]], set[Cid]]) -> set[Cid]: + """Those of targets complete before act and not after: the one sequence for every operation that deletes + blocks a pin or cached fetch may need (eviction, gc, drop_denied, verify), so the node can withdraw them. + + Which targets are complete is found first, without the lock (dag_blocks walks their DAGs, reading blocks); + then act(before) runs, deletes blocks and returns those it deleted. An act that deletes under one hold of + the lock adds the targets memoised as complete by then (_add_memoised(before)) just before deleting. + """ + before = self._complete_sets(targets) + deleted = act(before) + return {cid for cid, blocks in before.items() if blocks & deleted} + def _complete_sets(self, cids) -> dict[Cid, frozenset[Cid]]: """dag_blocks for those of cids that are complete.""" result = {} @@ -985,19 +1081,22 @@ def wanted() -> bool: targets = self._targets() roots = self._roots() | set(self._cache.values()) paths = [key for key in self._cache if key[1]] - # The long part, reading DAG nodes from disk, runs without the lock (_warm) - before = self._complete_sets(targets) - self._warm(roots, paths) - with self._lock: - if not wanted(): - return set() # another eviction did it meanwhile - before = self._with_memoised(before) - limit = self.max_storage - needed - self.max_storage // EVICT_MARGIN - deleted, freed, evicted = self._evict_to(max(0, limit)) - if deleted: - log(f"storage: evicted {plural(evicted, 'cached fetch')}, freeing {size(freed)} " - f"in {plural(len(deleted), 'block')}", LOG_VERBOSE) - return {cid for cid, blocks in before.items() if blocks & deleted} + + def act(before: dict[Cid, frozenset[Cid]]) -> set[Cid]: + """Evict; the blocks deleted.""" + self._warm(roots, paths) # the long part, reading DAG nodes from disk, runs without the lock + with self._lock: + if not wanted(): + return set() # another eviction did it meanwhile + self._add_memoised(before) + limit = self.max_storage - needed - self.max_storage // EVICT_MARGIN + deleted, freed, evicted = self._evict_to(max(0, limit)) + if deleted: + log(f"storage: evicted {plural(evicted, 'cached fetch')}, freeing {size(freed)} " + f"in {plural(len(deleted), 'block')}", LOG_VERBOSE) + return deleted + + return self._newly_incomplete(targets, act) def _roots(self) -> set[Cid]: """What pins and sessions hold (lock held).""" @@ -1006,14 +1105,12 @@ def _roots(self) -> set[Cid]: roots |= session.cids return roots - def _with_memoised(self, before: dict[Cid, frozenset[Cid]]) -> dict[Cid, frozenset[Cid]]: - """before (complete DAGs found without the lock) plus the targets memoised as complete now, which a + def _add_memoised(self, before: dict[Cid, frozenset[Cid]]): + """Add to before (complete DAGs found without the lock) the targets memoised as complete now, which a deletion about to happen may make incomplete (lock held).""" - result = dict(before) for cid in self._targets(): if cid in self._complete: - result[cid] = self._complete[cid] - return result + before[cid] = self._complete[cid] def gc(self) -> dict: """Evict every cached fetch and delete every block no pin or session reaches. @@ -1024,20 +1121,22 @@ def gc(self) -> dict: with self._lock: targets = self._targets() roots = self._roots() - # The long part without the lock, as in evict(); every cached fetch goes, so only pins and sessions count - before = self._complete_sets(targets) - self._warm(roots, []) - with self._lock: - before = self._with_memoised(before) - uncached = set(self._cache.values()) - if self._cache: - self._cache_dirty.update(self._cache) - self._cache.clear() - self._cache_generation += 1 - deleted, freed = self._collect() - incomplete = {cid for cid, blocks in before.items() if blocks & deleted} - return {"removed_blocks": len(deleted), "freed_bytes": freed, "incomplete": incomplete, - "uncached": uncached} + stats = {} + + def act(before: dict[Cid, frozenset[Cid]]) -> set[Cid]: + """Drop every record and collect; the blocks deleted.""" + # The long part without the lock, as in evict(); every cached fetch goes, so only pins and sessions count + self._warm(roots, []) + with self._lock: + self._add_memoised(before) + stats["uncached"] = set(self._drop_records(list(self._cache))) + deleted, freed = self._collect() + stats.update(removed_blocks=len(deleted), freed_bytes=freed) + return deleted + + incomplete = self._newly_incomplete(targets, act) + return {"removed_blocks": stats["removed_blocks"], "freed_bytes": stats["freed_bytes"], + "incomplete": incomplete, "uncached": stats["uncached"]} def verify(self) -> dict: """Re-check every stored block against its CID, deleting bad ones. @@ -1047,21 +1146,24 @@ def verify(self) -> dict: with self._lock: cids = list(self._sizes) targets = self._targets() - before = self._complete_sets(targets) # without the lock, as in evict(); dag_blocks checks for deletions bad = [] - for cid in cids: - if self.blocks.verify(cid): - continue - with self._lock: - # Checked again under the lock: it may have been deleted (or rewritten) meanwhile - if cid in self._sizes and not self.blocks.verify(cid): - self._delete({cid}) # reported either way; one Windows defers stays until a later try - bad.append(cid) + + def act(before: dict[Cid, frozenset[Cid]]) -> set[Cid]: + """Re-check every block, deleting bad ones, each under its own hold of the lock; the bad ones.""" + for cid in cids: + if self.blocks.verify(cid): + continue + with self._lock: + # Checked again under the lock: it may have been deleted (or rewritten) meanwhile + if cid in self._sizes and not self.blocks.verify(cid): + self._delete({cid}) # reported either way; one Windows defers stays until a later try + bad.append(cid) + return set(bad) + + incomplete = self._newly_incomplete(targets, act) # completeness without the lock, as in evict() if bad: log(f"storage: verify removed {plural(len(bad), 'damaged block')}", LOG_ERROR) - bad_set = set(bad) - return {"checked": len(cids), "bad": bad, - "incomplete": {cid for cid, blocks in before.items() if blocks & bad_set}} + return {"checked": len(cids), "bad": bad, "incomplete": incomplete} # Sizes, for stat and the remote-pin quota @@ -1080,6 +1182,12 @@ def content_size(self, cid: Cid) -> int | None: def dag_bytes(self, cid: Cid) -> int: """Bytes of the stored blocks of cid's DAG (each block once), as far as its stored nodes reach.""" + with self._lock: + # A DAG memoised as complete is all stored: its total, summed once (_delete forgets it with the DAG) + if cid in self._complete: + if cid not in self._complete_bytes: + self._complete_bytes[cid] = sum(map(self._sizes.__getitem__, self._complete[cid])) + return self._complete_bytes[cid] self._warm({cid}, []) # the DAG nodes are read without the lock, as in gc() with self._lock: return sum(self._sizes[block] for block in self._reach({cid})) diff --git a/rnfs/content/walker.py b/rnfs/content/walker.py index 09970c8..6374ee8 100644 --- a/rnfs/content/walker.py +++ b/rnfs/content/walker.py @@ -11,11 +11,14 @@ (the same CID listed the same way at the same depth) are checked once, which is what keeps DAGs that repeat subtrees cheap to walk. """ +import bisect import errno import os import stat +import threading +from collections import OrderedDict from dataclasses import dataclass, replace -from typing import Callable, Iterator +from typing import BinaryIO, Callable, Iterator from rnfs import constants, fsutil from rnfs.content import dag @@ -31,6 +34,9 @@ ANY_TYPES = frozenset({RAW_TYPE, FILE_TYPE, DIRECTORY_TYPE, INDEX_TYPE}) FILE_TYPES = frozenset({RAW_TYPE, FILE_TYPE}) # an entry of kind 0 or 1, or a file node's child DIRECTORY_TYPES = frozenset({DIRECTORY_TYPE, INDEX_TYPE}) # an entry of kind 2, or an index's child +# height()'s memo of checked subtrees' heights, kept across calls (least recently used forgotten first): every +# entry of a directory of tens of thousands, in about 11 MB when full +HEIGHT_MEMO = 32768 @dataclass(frozen=True) @@ -94,12 +100,7 @@ def check_node(expect: Expect, node: dag.Node): raise InvalidDag(f"{expect.cid} is deeper than MAX_DEPTH ({constants.MAX_DEPTH})") if expect.size is not None and node.size != expect.size: raise InvalidDag(f"{expect.cid} holds {node.size} bytes, but its parent says {expect.size}") - if isinstance(node, DirectoryNode): - names = [entry.name for entry in node.entries] - elif isinstance(node, IndexNode): - names = [child.first_name for child in node.children] - else: - names = None + names = node.names if expect.first_name is not None: if not names or names[0] != expect.first_name: raise InvalidDag(f"{expect.cid}: first name below an index child isn't its first_name") @@ -144,6 +145,14 @@ def is_index_step(expect: Expect) -> bool: return expect.first_name is not None +def index_child(node: IndexNode, name: str) -> int | None: + """The position of the child of an index node that would hold name, if any is: the one with the greatest + first_name <= name (first names are strictly increasing in a decoded node); None if name sorts before them + all. The one rule for stepping through an index, here and in Storage.""" + position = bisect.bisect_right(node.names, name) - 1 + return None if position < 0 else position + + def step(expect: Expect, block: bytes, name: str) -> Expect | None: """One path step: the entry called name in a directory node, or for an index the child that would hold it. @@ -159,14 +168,8 @@ def step(expect: Expect, block: bytes, name: str) -> Expect | None: if entry.name == name: return child return None - # The name is in the child with the greatest first_name <= name, if anywhere - found = None - for index_child, child in zip(node.children, child_expects): - if index_child.first_name <= name: - found = child - else: - break - return found + position = index_child(node, name) + return None if position is None else child_expects[position] def range_children(expect: Expect, block: bytes, offset: int, length: int) -> list[tuple[Expect, int, int]]: @@ -225,9 +228,9 @@ def parse_target(text: str) -> tuple[Cid, list[str]]: def kind_of(expect: Expect, block: bytes) -> str: - """"file" or "directory": what the checked block is the root of.""" + """FILE_TYPE ("file") or DIRECTORY_TYPE ("directory"): what the checked block is the root of.""" node = decode_checked(expect, block) - return "directory" if isinstance(node, (DirectoryNode, IndexNode)) else "file" + return DIRECTORY_TYPE if isinstance(node, (DirectoryNode, IndexNode)) else FILE_TYPE def size_of(expect: Expect, block: bytes) -> int: @@ -339,6 +342,44 @@ def read_range(expect: Expect, get: Get, offset: int = 0, length: int | None = N stack.extend((child, o, n, None) for child, o, n in reversed(parts)) +def leaf_bytes(expect: Expect, get: Get, stop_at: int | None = None) -> int: + """Bytes of the distinct raw chunks of the file expect names: each distinct raw CID's size once, as the file + node listing it first gives it. What hashing the file costs beyond this is repetition (the deny list's bound). + + Only file nodes are read, each distinct one once; raw blocks never are (but a one-block file, whose length + this is), so get must serve a DAG whose chunks are all stored and checked (fetch_dag's, a complete root's) for + the count to be what they hold. Stops once + stop_at bytes are counted, returning at least stop_at. Raises ValueError for a directory, InvalidDag for a + node that breaks a reader check or a chunk said to be larger than MAX_BLOCK_SIZE. + """ + if expect.cid.is_raw: + return size_of(expect, get(expect)) + counted: set[Cid] = set() # raw chunks counted + nodes: set[Cid] = {expect.cid} # file nodes met + total = 0 + stack = [expect] + while stack: + current = stack.pop() + check_cid(current) + node = decode_checked(current, get(current)) + if not isinstance(node, FileNode): + raise ValueError(f"{expect.cid} is a directory") + for child in children(current, node): + if child.cid.is_raw: + if child.cid not in counted: + # Its block isn't read, so its parent's claim is what counts: the cap keeps a claim from + # counting more than any chunk can hold (reading it would find the DAG invalid anyway) + check_raw(child, child.size) + counted.add(child.cid) + total += child.size + if stop_at is not None and total >= stop_at: + return total + elif child.cid not in nodes: + nodes.add(child.cid) + stack.append(child) + return total + + def _directory_entries(expect: Expect, get: Get, memo: dict | None = None) -> Iterator[tuple[Entry, Expect]]: """Every entry of a directory (through its index nodes) in name order, with the entry's expectation. @@ -403,11 +444,46 @@ def _count(expect: Expect, get: Get, memo: dict[Expect, int], limit: int) -> int return total +class _Heights: + """height()'s memo across calls: the height of a subtree that passed every check, by the expectation it was + checked under (depth set to 1), so `dir set` and `dir rm` don't walk a whole tree again for each edit. + + Exact forever: a CID's content never changes, and the checks a subtree passes and its height depend only on + its blocks and that expectation. Least recently used forgotten first, past HEIGHT_MEMO entries. + """ + + def __init__(self): + """Empty.""" + self._lock = threading.Lock() + self._heights: OrderedDict[Expect, int] = OrderedDict() + + def get(self, expect: Expect) -> int | None: + """The height remembered for expect, if any.""" + with self._lock: + known = self._heights.get(expect) + if known is not None: + self._heights.move_to_end(expect) + return known + + def update(self, heights: dict[Expect, int]): + """Remember heights, forgetting the least recently used past HEIGHT_MEMO.""" + with self._lock: + for expect, known in heights.items(): + self._heights[expect] = known + self._heights.move_to_end(expect) + while len(self._heights) > HEIGHT_MEMO: + self._heights.popitem(last=False) + + +_heights = _Heights() + + def height(expect: Expect, get: Get) -> int: """DAG nodes on the longest path from expect down (0 for a raw block), checking every DAG node met. Depth isn't checked against MAX_DEPTH, since measuring it is the point; the walk stops once the - height is known to exceed MAX_DEPTH, returning MAX_DEPTH + 1. + height is known to exceed MAX_DEPTH, returning MAX_DEPTH + 1. A subtree measured (and checked) before under + the same expectation isn't walked again (_Heights), so a tree edited in one place costs only the new nodes. """ limit = constants.MAX_DEPTH memo: dict[Expect, int] = {} @@ -424,15 +500,39 @@ def visit(current: Expect, level: int) -> int: return 1 current = replace(current, depth=1) # every node is checked as if at the top, so depth can't trip if current not in memo: - child_expects = expand(current, get(current)) - memo[current] = 1 + max((visit(child, level + 1) for child in child_expects), default=0) + known = _heights.get(current) + if known is None: + child_expects = expand(current, get(current)) + known = 1 + max((visit(child, level + 1) for child in child_expects), default=0) + memo[current] = known return memo[current] result = visit(expect, 1) - return limit + 1 if truncated else result + if truncated: + return limit + 1 # nothing kept: heights found below a cut-off walk may be short + _heights.update(memo) # every one exact, the walk having reached the bottom of each + # A remembered subtree can make the tree too deep without the walk going below limit + return min(result, limit + 1) -# Writing out +# Writing out. A directory output must stay where it was made while it is written: anyone who could rename inside +# it could swap one of its directories for a link or junction between the directory's mkdir and its children's +# writes, and have the DAG's files (execute bit included) created where they chose (audit CONT-2). So where the +# platform allows (FD_WRITE: Linux, macOS) everything is created relative to its parent's descriptor, below a root +# that is private while written; on Windows the root gets a private DACL and every file's final path is checked. +# Either way, the caller checks the root is still the one made (check_unchanged) before renaming it. + +_O_DIRECTORY = getattr(os, "O_DIRECTORY", 0) +_O_CLOEXEC = getattr(os, "O_CLOEXEC", 0) +# Platforms without these write directory outputs by path, as Windows does +FD_WRITE = (os.open in os.supports_dir_fd and os.mkdir in os.supports_dir_fd and os.listdir in os.supports_fd + and hasattr(os, "O_NOFOLLOW") and hasattr(os, "O_DIRECTORY") and hasattr(os, "fchmod")) +# What opening a directory with O_NOFOLLOW | O_DIRECTORY reports for a symlink (ELOOP; EMLINK on FreeBSD), a +# non-directory or a missing one +_SWAPPED_ERRNOS = frozenset({errno.ELOOP, errno.EMLINK, errno.ENOTDIR, errno.ENOENT}) +CHANGED = "output was changed while being written" # the ValueError's message when one was replaced + +Identity = tuple[int, int] # a file's (st_dev, st_ino) def unsafe_name_reason(name: str) -> str | None: @@ -462,16 +562,18 @@ def iter_files(expect: Expect, get: Get) -> Iterator[Expect]: def write_out(expect: Expect, get: Get, dest: str, max_entries: int, - check: Callable[[Expect], None] | None = None) -> list[str]: + check: Callable[[Expect], None] | None = None, created: list[Identity] | None = None) -> list[str]: """Write the file or directory expect names to dest, which must not exist yet. Returns warnings. check(file expect), if given, runs before each file is created and may raise to stop the write-out (the - deny list's file hashes); what was written stays for the caller to remove. + deny list's file hashes); what was written stays for the caller to remove. created, if given, gets a + directory output's identity (st_dev, st_ino) once dest is made, for check_unchanged before dest is renamed. Entries whose names the local filesystem would read as a path, or can't represent (it refuses to create them), are skipped and reported, as are names that collide on this filesystem (e.g. case-insensitive ones). Kind 1 entries get the execute bit, except on Windows, which has none. Raises TooLarge after max_entries - entries, counting every occurrence, or after max_entries small chunks (SmallLeaves). + entries, counting every occurrence, or after max_entries small chunks (SmallLeaves); ValueError (CHANGED) + if a directory of the output was replaced while it was written (_FdOutput, _PathOutput). """ if os.path.lexists(dest): raise ValueError(f"{dest} already exists") @@ -482,6 +584,7 @@ def write_out(expect: Expect, get: Get, dest: str, max_entries: int, block = get(expect) node = decode_checked(expect, block) if not isinstance(node, (DirectoryNode, IndexNode)): + # Straight into the directory the user chose: there is nothing below it to swap if check is not None: check(expect) with open(dest, "xb") as out: @@ -489,49 +592,232 @@ def write_out(expect: Expect, get: Get, dest: str, max_entries: int, return warnings memo[expect] = node - os.mkdir(dest) - stack: list[tuple[Expect, str, str]] = [(expect, dest, "")] # (directory, where, path shown in warnings) - while stack: - directory, where, shown = stack.pop() - subdirectories = [] - for entry, child in _directory_entries(directory, get, memo): - count += 1 - if count > max_entries: - raise TooLarge(f"output has more than max_output_entries ({max_entries}) entries") - entry_shown = f"{shown}/{entry.name}" if shown else entry.name - reason = unsafe_name_reason(entry.name) - if reason is not None: - warnings.append(f"skipped {entry_shown!r}: {reason}") - continue - path = os.path.join(where, entry.name) - if check is not None and entry.kind != dag.KIND_DIRECTORY: - check(child) + output = _FdOutput(dest) if FD_WRITE and not fsutil.IS_WINDOWS else _PathOutput(dest) + try: + if created is not None: + created.append(output.identity) + # (directory, its location in the output, path shown in warnings): names and identities rather than open + # descriptors, so however many directories a DAG has, only a few descriptors are open at once + stack: list[tuple[Expect, object, str]] = [(expect, output.root, "")] + while stack: + directory, location, shown = stack.pop() + subdirectories = [] + parent = output.enter(location) try: - if entry.kind == dag.KIND_DIRECTORY: - os.mkdir(path) - else: - out = open(path, "xb") - except FileExistsError: - # Everything here was created by this call, so this is another name the filesystem folded - warnings.append(f"skipped {entry_shown!r}: it collides with another name on this filesystem") - continue - except OSError as e: - if e.errno not in _NAME_ERRORS: - raise - # Allowed on most filesystems, but not this one (e.g. ':' or '?' on FAT, long names on eCryptfs) - warnings.append(f"skipped {entry_shown!r}: this filesystem can't represent it ({e.strerror})") - continue - if entry.kind == dag.KIND_DIRECTORY: - subdirectories.append((child, path, entry_shown)) - else: - with out: - _write_file(child, get, out, small) - stack.extend(reversed(subdirectories)) + for entry, child in _directory_entries(directory, get, memo): + count += 1 + if count > max_entries: + raise TooLarge(f"output has more than max_output_entries ({max_entries}) entries") + entry_shown = f"{shown}/{entry.name}" if shown else entry.name + reason = unsafe_name_reason(entry.name) + if reason is not None: + warnings.append(f"skipped {entry_shown!r}: {reason}") + continue + if check is not None and entry.kind != dag.KIND_DIRECTORY: + check(child) + try: + if entry.kind == dag.KIND_DIRECTORY: + below = output.make_directory(parent, location, entry.name) + else: + out = output.create_file(parent, entry.name) + except FileExistsError: + # Everything here was created by this call, so this is another name the filesystem folded + warnings.append(f"skipped {entry_shown!r}: it collides with another name on this filesystem") + continue + except OSError as e: + if e.errno not in _NAME_ERRORS: + raise + # Allowed on most filesystems, but not this one (e.g. ':' or '?' on FAT, long names on + # eCryptfs) + warnings.append(f"skipped {entry_shown!r}: this filesystem can't represent it ({e.strerror})") + continue + if entry.kind == dag.KIND_DIRECTORY: + subdirectories.append((child, below, entry_shown)) + else: + with out: + _write_file(child, get, out, small) + finally: + output.leave(parent) + stack.extend(reversed(subdirectories)) + output.finish() + finally: + output.close() for warning in warnings: log(f"get: {warning}", LOG_WARNING) return warnings +def check_unchanged(path: str, identity: Identity): + """Raise ValueError (CHANGED) unless path is still the directory write_out made there with identity: not + moved away and replaced by a link or another directory (by anyone who can write beside it).""" + try: + st = os.lstat(path) + except FileNotFoundError: + raise ValueError(f"{CHANGED}: {path} is gone") from None + if fsutil.is_link(st) or (st.st_dev, st.st_ino) != identity: + raise ValueError(f"{CHANGED}: {path} was replaced") + + +class _FdOutput: + """A directory output written through descriptors (FD_WRITE): its root is held open, and private to this + user (0700) while it is written. + + A location is the (name, identity) of each directory from the root down to it, the root's being (). Each + directory is created with mkdir(dir_fd=) and reopened without following links to record its identity; to + write its entries it is reopened from the root, one component at a time, each checked against its recorded + identity, so a directory swapped for a link or for another directory is never written into. + """ + + def __init__(self, dest: str): + """Create dest and open it, then make it private and check that nobody got in first. + + The mode is set through the descriptor, not by the umask (which is process-wide), and keeps setgid, so + what is created below it still gets a shared directory's group. Raises ValueError (CHANGED) if dest was + replaced, isn't this user's or already holds an entry, OSError if it can't be made. + """ + os.mkdir(dest) + self.fd = _open_directory(dest) + try: + st = os.fstat(self.fd) + if st.st_uid != os.geteuid(): + raise ValueError(f"{CHANGED}: {dest} belongs to someone else") + self.mode = stat.S_IMODE(st.st_mode) # what mkdir gave it, given back by finish() + os.fchmod(self.fd, 0o700 | (self.mode & stat.S_ISGID)) + # Until the chmod, anyone the parent's mode lets in could create an entry in it; from now on nobody + # else can, so an empty root stays this write-out's alone + planted = os.listdir(self.fd) + if planted: + raise ValueError(f"{CHANGED}: {dest} held {fsutil.printable(planted[0])!r} before anything was " + f"written") + except BaseException: + os.close(self.fd) + raise + self.identity: Identity = (st.st_dev, st.st_ino) + self.root: tuple = () + + def enter(self, location: tuple) -> int: + """A descriptor for the directory at location, reopened from the root; give it back with leave(). Raises + ValueError (CHANGED) if a directory on the way isn't the one made there.""" + fd = self.fd + try: + for name, identity in location: + below = _open_directory(name, fd) + if fd != self.fd: + os.close(fd) + fd = below + st = os.fstat(fd) + if (st.st_dev, st.st_ino) != identity: + raise ValueError(CHANGED) + except BaseException: + if fd != self.fd: + os.close(fd) + raise + return fd + + def leave(self, fd: int): + """Close a descriptor from enter() (the root's stays open).""" + if fd != self.fd: + os.close(fd) + + def make_directory(self, parent: int, location: tuple, name: str) -> tuple: + """Create the directory name in parent (which is at location); its own location. Raises OSError + (FileExistsError if name is taken), ValueError (CHANGED) if it was replaced at once.""" + os.mkdir(name, dir_fd=parent) + fd = _open_directory(name, parent) + try: + st = os.fstat(fd) + finally: + os.close(fd) + return (*location, (name, (st.st_dev, st.st_ino))) + + def create_file(self, parent: int, name: str) -> BinaryIO: + """Create the file name in parent, open for writing: never through a link, nor over anything there. + Raises OSError (FileExistsError if name is taken).""" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | fsutil.O_NOFOLLOW | _O_CLOEXEC + return os.fdopen(os.open(name, flags, 0o666, dir_fd=parent), "wb") + + def finish(self): + """The write-out is complete: give the root back the mode mkdir gave it.""" + os.fchmod(self.fd, self.mode) + + def close(self): + """Close the root's descriptor.""" + os.close(self.fd) + + +class _PathOutput: + """A directory output written by path: on Windows, which can't open a directory as a descriptor, and where + the platform lacks FD_WRITE. A location is a directory's path. + + On Windows the root is created with a protected DACL for this user and SYSTEM only (winacl.make_private_dir), + which everything below it inherits, so nobody else can rename inside it. In case a directory was swapped for + a junction all the same, each file's final path, from the open file, must be inside the root before a byte + is written. Such a file was then created (empty) where the junction led; it isn't removed, since removing it + by path could be redirected too. + """ + + def __init__(self, dest: str): + """Create dest (private, on Windows). Raises ValueError (CHANGED) if it was replaced at once or already + holds an entry, OSError if it can't be made.""" + if fsutil.IS_WINDOWS: + from rnfs import winacl + if not winacl.make_private_dir(dest): + raise FileExistsError(errno.EEXIST, "it already exists", dest) + else: + os.mkdir(dest) + st = os.lstat(dest) + if fsutil.is_link(st) or not stat.S_ISDIR(st.st_mode): + raise ValueError(f"{CHANGED}: {dest} was replaced") + if os.listdir(dest): + raise ValueError(f"{CHANGED}: {dest} held an entry before anything was written") + self.identity: Identity = (st.st_dev, st.st_ino) + self.real_root = os.path.realpath(dest) + self.root = dest + + def enter(self, location: str) -> str: + """The directory's path: nothing to open.""" + return location + + def leave(self, path: str): + """Nothing to close.""" + + def make_directory(self, parent: str, location: str, name: str) -> str: + """Create the directory name in parent; its path. Raises OSError (FileExistsError if name is taken).""" + path = os.path.join(parent, name) + os.mkdir(path) + return path + + def create_file(self, parent: str, name: str) -> BinaryIO: + """Create the file name in parent, open for writing, and check it is inside the root. Raises OSError + (FileExistsError if name is taken), ValueError (CHANGED) if it isn't inside the root.""" + out = open(os.path.join(parent, name), "xb") + try: + final = fsutil.final_path(out.fileno()) + if final is not None and not fsutil.is_below(final, self.real_root): + raise ValueError(f"{CHANGED}: a directory in it was replaced by a link") + except BaseException: + out.close() + raise + return out + + def finish(self): + """Nothing to give back: on Windows, the caller makes the output inherit its folder's access list.""" + + def close(self): + """Nothing to close.""" + + +def _open_directory(name: str, dir_fd: int | None = None) -> int: + """Open the directory name (in dir_fd) without following a link. Raises ValueError (CHANGED) if it isn't one + (a link, a file, or gone), OSError for any other failure.""" + try: + return os.open(name, os.O_RDONLY | _O_DIRECTORY | fsutil.O_NOFOLLOW | _O_CLOEXEC, dir_fd=dir_fd) + except OSError as e: + if e.errno in _SWAPPED_ERRNOS: + raise ValueError(CHANGED) from None + raise + + def _write_file(expect: Expect, get: Get, out, small: SmallLeaves, block: bytes | None = None): """Write the file expect names to out, a file just created, with the execute bit for kind 1.""" if block is not None and expect.cid.is_raw: diff --git a/rnfs/denylist.py b/rnfs/denylist.py index f617358..5aee801 100644 --- a/rnfs/denylist.py +++ b/rnfs/denylist.py @@ -15,6 +15,10 @@ parse_list refuses one over MAX_LIST_BYTES or MAX_LIST_ENTRIES, not UTF-8, or with a line that isn't an entry, and its source keeps the version it had. Listeners (the node) are told after every change, outside the lock. +Hashing reads every occurrence of repeated content, so a few small blocks can declare a file of 2^56 bytes. While +there are file hashes, a file declaring more than DENY_HASH_RATIO times the bytes of its distinct chunks, plus +DENY_HASH_SLACK, is refused (Denied) unhashed: it can't be checked (the node's _check_file). + So a file isn't read and hashed again before each send, the SHA-256 of the files checked is remembered by CID (at most MAX_KNOWN_HASHES, least recently used forgotten first). A CID's content never changes, so a hash remembered never goes stale, and they are kept across restarts in the deny_hashes table: memory is what checks @@ -28,7 +32,7 @@ from collections.abc import Callable from rnfs import statedb -from rnfs.constants import KIB +from rnfs.constants import KIB, MIB from rnfs.content.cid import Cid from rnfs.errors import NotFound from rnfs.log import LOG_WARNING, log, plural @@ -39,6 +43,10 @@ MAX_LIST_ENTRIES = 100_000 # entries in one list MAX_LOCAL_ENTRIES = 100_000 # entries in the local list MAX_KNOWN_HASHES = 10_000 # files whose content hash is remembered (a CID's content never changes) +# A file is hashed only if it declares at most DENY_HASH_RATIO × the bytes of its distinct chunks + DENY_HASH_SLACK: +# hashing then costs at most 16 times what its blocks cost to store, and any file of up to 64 MiB is hashed +DENY_HASH_RATIO = 16 +DENY_HASH_SLACK = 64 * MIB _HEX_DIGEST = re.compile(r"[0-9a-f]{64}") diff --git a/rnfs/dht/lookup.py b/rnfs/dht/lookup.py index cd74cf0..e5ec692 100644 --- a/rnfs/dht/lookup.py +++ b/rnfs/dht/lookup.py @@ -1,5 +1,6 @@ """Iterative DHT lookups (specification section 6). No RNS dependency: network access, identity knowledge -and time are injected, so lookups run over in-memory networks in tests. +and time are injected, so lookups run over in-memory networks in tests. (Only a callback raising, a bug, is +logged through rnfs.log, which loads RNS, imported then.) A lookup keeps up to `alpha` /find queries in flight, always to the closest known nodes not yet queried, and merges the nodes each reply returns. It is complete once the `beta` closest nodes it knows of have all @@ -7,17 +8,23 @@ `lookup_timeout` budget. Queries are blocking calls run on worker threads, so a slow or hung node never holds the caller past the deadline. +Nodes near the key can keep answering with ever closer IDs (made up, or ground for the key), so the beta closest +never all answer. A run therefore also ends, incomplete, once it has sent `max_queries` queries and their answers +are in, or after STALL_WINDOWS × max(alpha, beta) answers in a row that leave the bit length of the beta-th +closest known distance unchanged. Over a simulated network of 2000 honest nodes, lookups complete within both. + Nodes whose recent queries all failed (the routing table's `unreliable`) rank after every other node, so a lookup asks and waits for them only when it has too few others. One whose retry is due (`may_retry`) is asked again on top of the `alpha` queries if it would otherwise rank among the closest, and nothing waits for that answer: a node that never establishes a link costs a lookup no time, yet one that comes back is found. """ +import heapq import queue import threading import time from collections.abc import Callable, Iterable -from rnfs.constants import K +from rnfs.constants import ID_LENGTH, K from rnfs.dht.routing import distance, is_id # query(node_id, key) -> (provider IDs, node IDs), or None if the node didn't answer, refused, or replied @@ -26,6 +33,9 @@ NodeCallback = Callable[[bytes], object] # return value ignored _POLL_INTERVAL = 0.5 # seconds between should_stop() checks while waiting for replies +MAX_QUERIES = 8 * K # queries a run sends at most (section 6) +STALL_WINDOWS = 3 # a run ends after this many query windows of answers in a row that bring the key no nearer +_FAR = ID_LENGTH * 8 + 1 # _reach with fewer than beta nodes known: farther than any distance _STOP = object() # queued by stop() to wake the waiting caller @@ -67,6 +77,7 @@ class Lookup: LOG_DEBUG, for the debug log); called from the running thread and the query threads. - `unreliable(id)`: nodes to avoid, checked when a run learns of them; `may_retry(id)`: True if an unreliable node is due to be asked again, claiming that retry (both the routing table's). + - `max_queries`: queries one run sends at most, retries included. A Lookup may be run again (find_providers after every provider failed): each run re-queries the nodes, so /find replies give a fresh random half of providers, within the same deadline and unknown-ID budget. @@ -79,7 +90,7 @@ def __init__(self, key: bytes, start_nodes: Iterable[bytes], query: FindQuery, o on_failure: NodeCallback = _ignore, exclude: Callable[[bytes], bool] = _never, clock: Callable[[], float] = time.monotonic, poll_interval: float = _POLL_INTERVAL, report: Callable[[str], None] = _quiet, unreliable: Callable[[bytes], bool] = _never, - may_retry: Callable[[bytes], bool] = _never): + may_retry: Callable[[bytes], bool] = _never, max_queries: int = MAX_QUERIES): """A lookup for key; nothing is sent until find_closest() or find_providers() runs it.""" self.key = key self.query = query @@ -97,6 +108,7 @@ def __init__(self, key: bytes, start_nodes: Iterable[bytes], query: FindQuery, o self.report = report self.unreliable = unreliable self.may_retry = may_retry + self.max_queries = max(1, max_queries) self.deadline: float | None = None self._failed: set[bytes] = set() # didn't answer: dropped for the rest of the lookup self._answered: set[bytes] = set() # answered in any run @@ -190,6 +202,13 @@ def _distance(self, node_id: bytes) -> int: """node_id's distance to the key.""" return distance(node_id, self.key) + def _reach(self, known: dict[bytes, int]) -> int: + """The bit length of the beta-th closest known node's distance to the key (_FAR if fewer are known): + what an answer must shorten to count as progress.""" + if len(known) < self.beta: + return _FAR + return heapq.nsmallest(self.beta, known.values())[-1].bit_length() + def _usable(self, node_id) -> bool: """True if node_id may be queried in this lookup.""" return (is_id(node_id) and node_id != self.own_id and node_id not in self._failed @@ -226,7 +245,11 @@ def _iterate(self, results: queue.Queue, on_provider: NodeCallback | None, found: list[bytes] = [] in_flight = 0 retried = 0 + sent = 0 # queries sent by this run, retries included: at most max_queries window = max(self.alpha, self.beta) + # Answers in a row that didn't bring the beta-th closest known node a bit nearer the key + reach = self._reach(known) + stalled = 0 self._last_known = known # live: find_closest reads them once the run ends self._last_queried = queried self._last_doubtful = doubtful @@ -249,6 +272,9 @@ def rank(node_id: bytes) -> tuple[bool, int]: if ranked and all(node_id in answered for node_id in ranked[:self.beta]): why = "complete" break # the beta closest known nodes have all answered: complete + if stalled >= STALL_WINDOWS * window: + why = "no progress" + break # nodes keep answering, but the closest known come no nearer for node_id in ranked[:window]: if node_id in retrying: # a retry the lookup now needs: waited for like any query retrying.discard(node_id) @@ -257,7 +283,7 @@ def rank(node_id: bytes) -> tuple[bool, int]: # Keep up to alpha queries in flight, to the closest nodes not yet queried. Only nodes ranked # within the closest max(alpha, beta) known can affect completion, so farther ones aren't # queried while nearer queries are pending: on slow networks every query costs airtime. - while in_flight < self.alpha: + while in_flight < self.alpha and sent < self.max_queries: ranked = sorted(known, key=rank)[:window] node_id = next((n for n in ranked if n not in queried), None) if node_id is None: @@ -267,20 +293,25 @@ def rank(node_id: bytes) -> tuple[bool, int]: del known[node_id] # over the unknown-ID budget: dropped, but not a failure continue self._send(node_id, results) + sent += 1 in_flight += 1 # An unreliable node that distance alone would rank among them is asked again once its retry is # due: only an answer shows it is back. On top of alpha, since nothing waits for it for node_id in sorted(known, key=known.__getitem__)[:window]: - if node_id not in doubtful or node_id in queried or node_id in passed: + if node_id not in doubtful or node_id in queried or node_id in passed or sent >= self.max_queries: continue if self.is_known(node_id) and self.may_retry(node_id): queried.add(node_id) retrying.add(node_id) retried += 1 + sent += 1 self._send(node_id, results) else: passed.add(node_id) if in_flight == 0: + if sent >= self.max_queries: + why = "query cap" + break # sent all it may, and every answer waited for is in why = "no node left to ask" break # every known node was queried and nothing new turned up @@ -320,6 +351,11 @@ def rank(node_id: bytes) -> tuple[bool, int]: known[new_id] = self._distance(new_id) if self.unreliable(new_id): doubtful.add(new_id) + nearer = self._reach(known) + if nearer < reach: + reach, stalled = nearer, 0 + else: + stalled += 1 if on_provider is not None: for provider in providers: if provider in self._reported or provider == self.own_id or self.exclude(provider): @@ -341,7 +377,7 @@ def work(): started = self.clock() try: reply = self._clean(self.query(node_id, self.key)) - except Exception as e: + except Exception as e: # noqa: BLE001 - one broken node must not abort the lookup reply = None # one broken node must not abort the lookup self.report(f"lookup for {self.key.hex()}: asking {_name(node_id)} raised {e!r}") else: @@ -351,12 +387,16 @@ def work(): else f"answered in {took:.1f}s: {len(reply[0])} providers, {len(reply[1])} nodes")) try: (self.on_failure if reply is None else self.on_answer)(node_id) - except Exception: - pass + except Exception as e: # noqa: BLE001 - a caller's callback must not end the lookup; logged + # A bug in the routing table (private mode's admit, say): logged inside the except, so the debug + # log has its traceback. rnfs.log loads RNS, which this module doesn't at import + from rnfs.log import LOG_WARNING, log + log(f"lookup for {self.key.hex()}: the callback for {_name(node_id)} raised {e!r}", LOG_WARNING) results.put((node_id, reply)) # Daemon threads rather than a ThreadPoolExecutor: a query hung past the deadline is abandoned - # and must not delay process exit. At most alpha run at a time per run. + # and must not delay process exit. A run waits for at most alpha at a time; retries of unreliable nodes + # run on top of those (each node once per run, every query within max_queries). threading.Thread(target=work, name="rnfs-lookup", daemon=True).start() def _clean(self, reply) -> tuple[list[bytes], list[bytes]] | None: diff --git a/rnfs/dht/providers.py b/rnfs/dht/providers.py index 6b6b57b..97a8ad4 100644 --- a/rnfs/dht/providers.py +++ b/rnfs/dht/providers.py @@ -8,24 +8,26 @@ one seen most recently) stores into the pool instead. A full pool evicts a record of the publisher holding the most pooled records (one of them at random, if several hold as many), never the storer's own, so a new publisher can always store something and a flood of identities evens out the pool rather than emptying it of -everyone else's records. This node's own records are always accepted. +everyone else's records. This node's own records are always accepted and never evicted: they are what the node +provides, and its /find replies list it as a provider from them. Local limits, not part of the protocol, bound what one identity can take: a publisher holds at most max_records // PUBLISHER_SHARE records once it has been seen for `seniority`, and a tenth of that before (so one long-lived identity can't fill the store and lock everyone else out, and a fresh one gets little; this -node's own records are exempt); and a key has at most MAX_PUBLISHERS_PER_KEY publishers (so /find replies -stay cheap however many identities advertise one key). A newcomer to a full key evicts one of its publishers -at random, never one of the K/2 oldest records: refusing newcomers would let early Sybil identities hold the -key, and the random half of /find replies relies on newcomers getting stored. A publisher's first-seen time -is kept for a TTL after its last record goes, so one that was away for a while isn't new again; that of a -newcomer whose last record was evicted from the pool is kept too, in a smaller memory that forgets the most -recently first seen first, so a flood can't keep a new publisher new forever. +node's own records are exempt); and a key has at most MAX_PUBLISHERS_PER_KEY publishers besides this node (so +/find replies stay cheap however many identities advertise one key). A newcomer to a full key evicts one of its +publishers at random, never one of the K/2 oldest records nor this node's own: refusing newcomers would let early +Sybil identities hold the key, and the random half of /find replies relies on newcomers getting stored. A +publisher's first-seen time is kept for a TTL after its last record goes, so one that was away for a while isn't +new again; that of a newcomer whose last record was evicted from the pool is kept too, in a smaller memory that +forgets the most recently first seen first, so a flood can't keep a new publisher new forever. Records, first-seen times and departures are persisted in the state database (rnfs.statedb: the provider_records, provider_publishers, provider_departed and provider_evicted tables) by save(), which the node calls periodically and at shutdown, writing only the rows changed since the last save; they are read from it only at startup. Times are wall clock and persisted, so a time in the future (the clock ran ahead, then was -corrected) is clamped to now on load and at each expire(); otherwise such records would outlive their TTL. +corrected) is clamped to now on load and at the first expire() after the clock went backwards; otherwise such +records would outlive their TTL. """ import heapq import itertools @@ -196,7 +198,8 @@ def __init__(self, max_records: int, state_dir: str | None, ttl: float = PROVIDE max_per_publisher defaults to max(MIN_PER_PUBLISHER, max_records // PUBLISHER_SHARE); a publisher seen for less than `seniority` seconds (the node passes provider_seniority; 0: no ramp-up) may hold newcomer_per_publisher, a tenth of it. own_id's records (the node recording itself) aren't limited by - either, and are never refused for room: they are as many as the node provides. + either, nor counted toward max_per_key, and are never refused for room or evicted: they are as many as + the node provides (beyond max_records, in the rare case that it provides more than the store holds). """ self.max_records = max_records self.max_per_publisher = (max_per_publisher if max_per_publisher is not None @@ -241,6 +244,9 @@ def __init__(self, max_records: int, state_dir: str | None, ttl: float = PROVIDE self._dirty_publishers: set[bytes] = set() self._dirty_gone: set[bytes] = set() self._last_expire = float("-inf") + # The latest clock reading any time held here came from: none is later, so expire() needs to clamp times + # only when the clock reads earlier than this (it was corrected backwards), not on every pass + self._latest = float("-inf") self._lock = threading.RLock() self._save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last self._db = None if state_dir is None else statedb.connect(state_dir) @@ -263,12 +269,14 @@ def store(self, key: bytes, publisher: bytes) -> bool: newest one there evicts a record of that newest publisher; any other goes into the pool. A full pool evicts the record pooled first by one of the publishers holding the most pooled records, and refuses a publisher already holding that many (so it can't churn its own records). So new identities can't - displace older publishers, and a new publisher always gets in. + displace older publishers, and a new publisher always gets in. own_id's records are never evicted by + either limit, nor counted toward max_per_key: Sybil stores to a key could otherwise push the node's own + record out of its own store. """ check_id(key) check_id(publisher) with self._lock: - now = self.clock() + now = self._now() record = self._live_record(key, publisher, now) if record is not None: record.last_stored = now @@ -294,16 +302,25 @@ def store(self, key: bytes, publisher: bytes) -> bool: entry = self._publishers[publisher] = _Publisher(now if first_seen is None else first_seen, next(self._order)) self._touch_publisher(publisher) - entry.keys.add(key) - record = self._records.setdefault(key, {})[publisher] = _Record(now, now, pooled) - self._count += 1 - if pooled: - self._pool_add(key, publisher) - else: - self._add_main(publisher, entry) - self._touch(key, publisher) + self._insert(key, publisher, _Record(now, now, pooled)) return True + def _insert(self, key: bytes, publisher: bytes, record: _Record, loading: bool = False): + """Add a new record of publisher (which has an entry) for key, in the pool if record.pooled, keeping every + index in step: store() and _load() both add records through here, so the indexes can't drift apart after a + restart. loading: _load's, which rebuilds the newest-publisher heap once at the end and works out the rows + to save from what it read, so neither is kept up record by record. Caller holds the lock.""" + entry = self._publishers[publisher] + entry.keys.add(key) + self._records.setdefault(key, {})[publisher] = record + self._count += 1 + if record.pooled: + self._pool_add(key, publisher) + else: + self._add_main(publisher, entry, loading) + if not loading: + self._touch(key, publisher) + def withdraw(self, key: bytes, publisher: bytes) -> bool: """Remove publisher's record for key. True if there was one.""" with self._lock: @@ -375,15 +392,18 @@ def _within_limits(self, key: bytes, publisher: bytes, now: float) -> bool: self._sweep_publisher(publisher, entry, now) if publisher in self._publishers and len(self._publishers[publisher].keys) >= allowance: return False + if publisher == self.own_id: + return True # the node's own record doesn't count toward max_per_key, so it evicts no one records = self._records.get(key) - if records is not None and len(records) >= self.max_per_key: + # The node's own record is neither counted nor a candidate: otherwise about a thousand Sybil stores to a + # key it provides would evict it from its own store, and its own /find replies would leave it out + if records is not None and len(records) - (self.own_id in records) >= self.max_per_key: self._drop_expired([(key, other) for other in records], now) - records = self._records.get(key, {}) - if len(records) >= self.max_per_key: + others = [other for other in self._records.get(key, {}) if other != self.own_id] + if len(others) >= self.max_per_key: # Records are in first-stored order: the K/2 oldest (the half /find replies favour) stay senior = min(K // 2, self.max_per_key - 1) - victim = list(records)[self.rng.randrange(senior, len(records))] - self._remove(key, victim) + self._remove(key, others[self.rng.randrange(senior, len(others))]) return True def _sweep_publisher(self, publisher: bytes, entry: _Publisher, now: float): @@ -431,18 +451,33 @@ def _make_room(self, publisher: bytes, now: float) -> bool | None: if self._pool_count < self.pool_size: return True # Fair shares: a publisher already holding as many pooled records as anyone can't push others out (nor - # churn its own); any other takes a record from one of those holding the most - if not own and len(self._pooled.get(publisher, ())) >= self._most_pooled: + # churn its own); any other takes a record from one of those holding the most. The node's own records are + # never taken, nor counted among the most held (it holds what it provides, not a share) + most = self._most_pooled_by_others() + if not own and len(self._pooled.get(publisher, ())) >= most: return None - holders = self._holders[self._most_pooled] + if most == 0: + return True # only own records are pooled: the node's own goes over the pool rather than evict its own + holders = self._holders[most] index = self.rng.randrange(len(holders)) victim_id = holders[index] - if victim_id == publisher and len(holders) > 1: # own records: another's first, if another holds as many + if victim_id == self.own_id: # at most one entry, and not the only one at `most` victim_id = holders[(index + 1) % len(holders)] key = next(iter(self._pooled[victim_id])) # the one it had pooled first self._remove(key, victim_id, evicted=True) return True + def _most_pooled_by_others(self) -> int: + """The most pooled records any publisher but own_id holds (0: none). Caller holds the lock. + + _most_pooled unless own_id alone holds that many: then the next count held, found by stepping down (only + then, so O(1) otherwise). + """ + most = self._most_pooled + while most and self._holders.get(most, [self.own_id]) == [self.own_id]: + most -= 1 + return most + @staticmethod def _rank(entry: _Publisher) -> tuple[float, int]: """Sort key for first-seen order: lower was seen earlier.""" @@ -468,11 +503,12 @@ def _newest_main(self, skip: set) -> tuple[bytes, _Publisher] | None: heapq.heappush(self._newest, item) return found - def _add_main(self, publisher: bytes, entry: _Publisher): - """Count a record of publisher outside the pool. Caller holds the lock.""" + def _add_main(self, publisher: bytes, entry: _Publisher, loading: bool = False): + """Count a record of publisher outside the pool (loading: _load's, which rebuilds the heap of newest + publishers at the end rather than push to it here). Caller holds the lock.""" self._main_count += 1 entry.main += 1 - if entry.main == 1: + if entry.main == 1 and not loading: heapq.heappush(self._newest, (-entry.first_seen, -entry.order, publisher)) if len(self._newest) > 2 * len(self._publishers) + 64: self._rebuild_newest() @@ -548,7 +584,7 @@ def _remove(self, key: bytes, publisher: bytes, evicted: bool = False) -> bool: if not entry.keys: del self._publishers[publisher] self._touch_publisher(publisher) - self._depart(publisher, entry.first_seen, self.clock(), evicted) + self._depart(publisher, entry.first_seen, self._now(), evicted) return True def _depart(self, publisher: bytes, first_seen: float, now: float, evicted: bool = False): @@ -619,12 +655,22 @@ def _live_record(self, key: bytes, publisher: bytes, now: float) -> _Record | No def expire(self) -> int: """Remove every expired record. Returns how many were removed.""" with self._lock: - return self._expire(self.clock()) + return self._expire(self._now()) + + def _now(self) -> float: + """The clock's time, for a time about to be held: noted in _latest. Caller holds the lock.""" + now = self.clock() + self._latest = max(self._latest, now) + return now def _expire(self, now: float) -> int: - """Remove records last stored ttl or more before now, clamping times in the future. Caller holds the lock.""" + """Remove records last stored ttl or more before now, clamping times in the future if the clock went + backwards. Caller holds the lock.""" self._last_expire = now - self._clamp(now) + if now < self._latest: + # A time may be in the future. Otherwise none is, and the clamping pass (half of an expiry pass, + # under the lock) would change nothing + self._clamp(now) expired = [(key, publisher) for key, records in self._records.items() for publisher, record in records.items() if now - record.last_stored >= self.ttl] for key, publisher in expired: @@ -639,7 +685,8 @@ def _expire(self, now: float) -> int: return len(expired) def _clamp(self, now: float): - """Bring every time after now back to now (the clock was corrected backwards). Caller holds the lock.""" + """Bring every time after now back to now (the clock was corrected backwards), so none is later than now + any more. Caller holds the lock.""" stored_moved = False for key, records in self._records.items(): for publisher, record in records.items(): @@ -663,6 +710,7 @@ def _clamp(self, now: float): self._touch_gone(publisher) if moved: self._rebuild_newest() + self._latest = now # Queries @@ -695,7 +743,7 @@ def providers(self, key: bytes, exclude: Callable[[bytes], bool] = _never) -> li return self._live(key, exclude) def _live_unsorted(self, key: bytes) -> list[bytes]: - """Unexpired providers of key in the order their records were stored (at most max_per_key).""" + """Unexpired providers of key in the order their records were stored (at most max_per_key, and own_id).""" with self._lock: now = self.clock() return [publisher for publisher, record in self._records.get(key, {}).items() @@ -727,6 +775,75 @@ def first_seen(self, publisher: bytes) -> float | None: entry = self._publishers.get(publisher) return None if entry is None else entry.first_seen + def _check_consistency(self): + """Recompute every index from _records and raise AssertionError where one differs (for tests: test_providers + runs it after each test). An index added to the store belongs here too.""" + def check(condition: bool, what: str): + """Raise AssertionError(what) unless condition holds.""" + if not condition: + raise AssertionError(f"ProviderStore inconsistent: {what}") + + with self._lock: + keys: dict[bytes, set[bytes]] = {} + main: dict[bytes, int] = {} + pooled: dict[bytes, set[bytes]] = {} + for key, records in self._records.items(): + check(bool(records), f"key {key.hex()} has an empty record dict") + for publisher, record in records.items(): + keys.setdefault(publisher, set()).add(key) + if record.pooled: + pooled.setdefault(publisher, set()).add(key) + else: + main[publisher] = main.get(publisher, 0) + 1 + count = sum(len(held) for held in keys.values()) + pool_count = sum(len(held) for held in pooled.values()) + check(self._count == count, f"_count {self._count}, records {count}") + check(self._pool_count == pool_count, f"_pool_count {self._pool_count}, pooled records {pool_count}") + check(self._main_count == count - pool_count, + f"_main_count {self._main_count}, records outside the pool {count - pool_count}") + check(set(self._publishers) == set(keys), "_publishers isn't the publishers with records") + for publisher, entry in self._publishers.items(): + check(entry.keys == keys[publisher], f"{publisher.hex()}'s keys") + check(entry.main == main.get(publisher, 0), f"{publisher.hex()}'s main count") + check({publisher: set(held) for publisher, held in self._pooled.items()} == pooled, + "_pooled isn't the pooled records") + # Holders by pooled count: each publisher listed once, under its count, at its index + listed = {} + for held, holders in self._holders.items(): + check(bool(holders), f"empty holder list for {held}") + for index, publisher in enumerate(holders): + check(publisher not in listed, f"{publisher.hex()} listed twice among holders") + listed[publisher] = held + check(self._holder_index.get(publisher) == index, f"{publisher.hex()}'s holder index") + check(listed == {publisher: len(held) for publisher, held in pooled.items()}, + "_holders isn't the pooled counts") + check(set(self._holder_index) == set(listed), "_holder_index lists publishers holding nothing pooled") + check(self._most_pooled == max(listed.values(), default=0), + f"_most_pooled {self._most_pooled}, most held {max(listed.values(), default=0)}") + # The newest-publisher heap: a heap, holding a current entry for each publisher with records outside + # the pool (stale ones are allowed: they are dropped as they reach the top) + heap = self._newest + check(all(heap[(i - 1) // 2] <= heap[i] for i in range(1, len(heap))), "_newest isn't a heap") + current = {(-entry.first_seen, -entry.order, publisher) for publisher, entry in self._publishers.items() + if entry.main > 0} + check(current <= set(heap), "_newest lacks a publisher with records outside the pool") + # First-seen memories: only of publishers without records, each in at most one, within their sizes + check(not set(self._departed) & set(self._publishers), "a departed publisher has records") + check(not set(self._evicted) & set(self._publishers), "an evicted publisher has records") + check(not set(self._departed) & set(self._evicted), "a publisher is both departed and evicted") + check(len(self._departed) <= self._departed_size, "_departed over its size") + check(len(self._evicted) <= self._evicted_size, "_evicted over its size") + # No time held is later than _latest, or expire() could skip clamping a time in the future + latest = max([time for records in self._records.values() for record in records.values() + for time in (record.first_stored, record.last_stored)] + + [entry.first_seen for entry in self._publishers.values()] + + [time for memory in (self._departed, self._evicted) for times in memory.values() + for time in times], default=float("-inf")) + check(latest <= self._latest, f"a time held ({latest}) is later than _latest ({self._latest})") + if self._db is None: + check(not (self._dirty_records or self._dirty_publishers or self._dirty_gone), + "changes tracked with no state database") + # Persistence def save(self): @@ -796,7 +913,7 @@ def _load(self): record_rows = self._db.query("SELECT key, publisher, first_stored, last_stored, pooled FROM provider_records") departed_rows = self._db.query("SELECT publisher, first_seen, at FROM provider_departed") evicted_rows = self._db.query("SELECT publisher, first_seen, at FROM provider_evicted") - now = self.clock() + now = self._now() # every time loaded is clamped to it malformed = 0 # Times in the future (a clock that ran ahead, since corrected) count as now; NaN is dropped first_seen = {} @@ -818,26 +935,22 @@ def _load(self): self._publishers[publisher] = _Publisher(seen, next(self._order)) # Oldest records first, so each key's records are in first-stored order and the limits keep the oldest for key, publisher, first, last, pooled in sorted(records, key=lambda record: record[2]): + own = publisher == self.own_id + held = self._records.get(key, {}) + # As store() keeps them: the node's own records aren't limited, nor counted toward max_per_key if (publisher not in self._publishers or now - last >= self.ttl - or (publisher != self.own_id - and len(self._publishers[publisher].keys) >= self.max_per_publisher) - or len(self._records.get(key, ())) >= self.max_per_key): + or (not own and len(self._publishers[publisher].keys) >= self.max_per_publisher) + or (not own and len(held) - (self.own_id in held) >= self.max_per_key)): continue - # Where it was if there is room there, else wherever there is + # Where it was if there is room there, else wherever there is (the node's own: where it was, as + # store() lets them exceed the store rather than evict them) main_room = self._main_count < self.main_size pool_room = self._pool_count < self.pool_size - if not (main_room or pool_room): + if not (main_room or pool_room or own): continue - pooled = pool_room if pooled or not main_room else False - entry = self._publishers[publisher] - self._records.setdefault(key, {})[publisher] = _Record(first, last, pooled) - entry.keys.add(key) - self._count += 1 - if pooled: - self._pool_add(key, publisher) - else: - self._main_count += 1 - entry.main += 1 + if main_room or pool_room: + pooled = pool_room if pooled or not main_room else False + self._insert(key, publisher, _Record(first, last, pooled), loading=True) for publisher in [p for p, entry in self._publishers.items() if not entry.keys]: del self._publishers[publisher] self._rebuild_newest() diff --git a/rnfs/dht/routing.py b/rnfs/dht/routing.py index 1a34247..35309b5 100644 --- a/rnfs/dht/routing.py +++ b/rnfs/dht/routing.py @@ -16,7 +16,7 @@ from collections.abc import Callable from rnfs.config import Config -from rnfs.constants import BUCKET_COUNT, ID_LENGTH, K +from rnfs.constants import BUCKET_COUNT, ID_LENGTH, K, MISSED_ANNOUNCES AVOID_AFTER = 300.0 # seconds an unreliable node isn't retried after max_failures failed queries in a row ... AVOID_MAX = 3600.0 # ... doubling with each further failure up to this @@ -69,6 +69,8 @@ def __init__(self, own_id: bytes, config: Config, clock: Callable[[], float] = t self.clock = clock self.admit = admit self._buckets: list[dict[bytes, _Entry]] = [{} for _ in range(BUCKET_COUNT)] + # Both in the order their entries expire (when dropped, ban end), each popped and re-inserted when set, so + # _prune only looks at the front of each self._dropped: dict[bytes, tuple[float, float]] = {} # node ID -> (first seen, when dropped) self._bans: dict[bytes, float] = {} # node ID -> ban end # Node ID -> [failed queries in a row, when it may be retried], for any node; only an answer clears it @@ -77,8 +79,8 @@ def __init__(self, own_id: bytes, config: Config, clock: Callable[[], float] = t @property def _silence_limit(self) -> float: - """Silence after which a node is presumed gone: it missed two announces.""" - return 2 * self.config.announce_interval + """Silence after which a node is presumed gone: it missed MISSED_ANNOUNCES (two) announces.""" + return MISSED_ANNOUNCES * self.config.announce_interval def bucket_index(self, node_id: bytes) -> int: """Index of the bucket node_id belongs in: i for distance in [2^i, 2^(i+1)). ValueError for own_id.""" @@ -92,12 +94,31 @@ def _stale(self, entry: _Entry, now: float) -> bool: return entry.failures > 0 or now - entry.last_heard > self._silence_limit def _prune(self, now: float): - """Forget expired bans, and dropped nodes whose seniority can no longer be regained.""" - for node_id in [n for n, until in self._bans.items() if until <= now]: + """Forget expired bans, and dropped nodes whose seniority can no longer be regained. Caller holds the lock. + + Runs on every announce heard, so it stops at the first entry still current: both dicts are in expiry order. + Their readers (_hear, _add, is_banned) check expiry themselves, so an entry left out of order by a clock step + or a changed setting only costs memory until those before it expire.""" + expired = [] + for node_id, until in self._bans.items(): + if until > now: + break + expired.append(node_id) + for node_id in expired: del self._bans[node_id] - for node_id in [n for n, (_, dropped) in self._dropped.items() if now - dropped > self._silence_limit]: + expired = [] + for node_id, (_, dropped) in self._dropped.items(): + if now - dropped <= self._silence_limit: + break + expired.append(node_id) + for node_id in expired: del self._dropped[node_id] + def _set_dropped(self, node_id: bytes, first_seen: float, now: float): + """Remember node_id's seniority as dropped now, at the end of _dropped's expiry order. Caller holds the lock.""" + self._dropped.pop(node_id, None) + self._dropped[node_id] = (first_seen, now) + def heard(self, node_id: bytes) -> bool: """Record that node_id proved to be a reachable server (server-mode announce, or answered /find; for an answer, call answered(), which also makes it reliable again). @@ -127,7 +148,7 @@ def _hear(self, node_id: bytes, answered: bool) -> bool: self._unanswered.pop(node_id, None) now = self.clock() self._prune(now) - if node_id in self._bans: + if self._bans.get(node_id, now) > now: # banned; an expired ban may be left (see _prune) return False bucket = self._buckets[self.bucket_index(node_id)] entry = bucket.get(node_id) @@ -152,10 +173,12 @@ def _add(self, bucket: dict[bytes, _Entry], node_id: bytes, now: float) -> bool: return False del bucket[victim.node_id] # Displaced, not dropped for failures: it may come back with its seniority like a dropped node - self._dropped[victim.node_id] = (victim.first_seen, now) + self._set_dropped(victim.node_id, victim.first_seen, now) # A node dropped recently (e.g. by a lossy link) regains its seniority - first_seen, _ = self._dropped.pop(node_id, (now, now)) + first_seen, dropped = self._dropped.pop(node_id, (now, now)) + if now - dropped > self._silence_limit: # expired, but not yet pruned (see _prune) + first_seen = now bucket[node_id] = _Entry(node_id, first_seen, now) return True @@ -178,7 +201,7 @@ def failed(self, node_id: bytes) -> bool: if entry.failures < self.config.max_failures: return False del bucket[node_id] - self._dropped[node_id] = (entry.first_seen, now) + self._set_dropped(node_id, entry.first_seen, now) return True def _count_unanswered(self, node_id: bytes, now: float): @@ -233,6 +256,7 @@ def ban(self, node_id: bytes): return with self._lock: self.remove(node_id) + self._bans.pop(node_id, None) # re-inserted last, keeping _bans in expiry order self._bans[node_id] = self.clock() + self.config.ban_time def is_banned(self, node_id: bytes) -> bool: diff --git a/rnfs/embedded.py b/rnfs/embedded.py index ef71ed4..ba6efe0 100644 --- a/rnfs/embedded.py +++ b/rnfs/embedded.py @@ -12,7 +12,8 @@ - If starting fails, nothing of it is left in the process (RnfsNode releases its rnfs.dht destination), so the program may try again. - It locks the rnfs dir as the daemon does (node.lock_rnfs_dir), so a daemon or another node can't share it, - and only one Node runs in a process at a time. + and only one Node runs in a process at a time. As in the daemon, the rnfs dir is made private before its + rnfs.toml is read, and what was changed to trust that file is logged as a warning. Name records are heard through NameManager's record listener, which only queues them for the event thread; jobs are the jobs feature's, polled for progress. @@ -28,8 +29,8 @@ from rnfs import fetcher, fsutil, interface from rnfs.config import Config from rnfs.errors import RnfsError -from rnfs.log import LOG_ERROR, log -from rnfs.node import RnfsNode, lock_rnfs_dir +from rnfs.log import LOG_ERROR, LOG_WARNING, log +from rnfs.node import RnfsNode, lock_rnfs_dir, make_private_root from rnfs.objects import Record # An embedded node's settings where neither rnfs.toml nor the program's settings give them: an app's node is @@ -66,7 +67,13 @@ def __init__(self, rnfs_dir, *, reticulum: "RNS.Reticulum | None" = None, settin unknown = settings.keys() - ({f.name for f in fields(Config)} - {"rnfs_dir"}) if unknown: raise ValueError(f"unknown settings: {', '.join(sorted(unknown))}") - config = Config.load(os.fspath(rnfs_dir), defaults=EMBEDDED_DEFAULTS, **settings) + rnfs_dir = os.fspath(rnfs_dir) + # As the daemon starts: the rnfs dir made private, then its settings read, then its subdirectories made + # private and the dir locked (lock_rnfs_dir) + make_private_root(os.path.expanduser(rnfs_dir)) + config = Config.load(rnfs_dir, defaults=EMBEDDED_DEFAULTS, **settings) + for warning in config.warnings: + log(warning, LOG_WARNING) with _process_lock: if _running is not None: raise RnfsError("an rnfs.Node is already running in this process; close it first") @@ -185,9 +192,7 @@ def dir_set(self, target: str, name: str, child: str, executable: bool) -> dict: return self.node.dir_set(target, name, child, executable=executable) def dir_rm(self, target: str) -> dict: - """RnfsNode.dir_rm, which needs a path to remove.""" - if "/" not in target.rstrip("/"): - raise ValueError("dir_rm needs /: the entry to remove") + """RnfsNode.dir_rm, which needs a path to remove (and checks it has one).""" return self.node.dir_rm(target) def pin_add(self, target: str) -> dict: diff --git a/rnfs/errors.py b/rnfs/errors.py index ffcedf6..db9373a 100644 --- a/rnfs/errors.py +++ b/rnfs/errors.py @@ -1,6 +1,9 @@ """Errors shared across rnfs, and raised by its public API (rnfs.Node, rnfs.Client), which exports each as -rnfs.. The API maps them to HTTP statuses (see design.md); rnfs.apiclient's errors for those statuses are -subclasses of these, so one except clause catches either class's.""" +rnfs.. The API maps them to HTTP statuses by STATUSES (see design.md); rnfs.apiclient's errors for those +statuses are subclasses of these, so one except clause catches either class's. + +Standard library only: rnfs.apiclient, which never loads RNS, derives its table from STATUSES. +""" class RnfsError(Exception): @@ -41,3 +44,28 @@ class Cancelled(RnfsError): class Closed(RnfsError): """The rnfs.Node or rnfs.Client was closed.""" + + +class BlockMismatch(RnfsError, ValueError): + """A block handed to the block store doesn't match its CID (or is over MAX_BLOCK_SIZE). + + The add and import paths only store blocks they built or checked, so there it is a bug, not a bad request: + the API logs it at WARNING. Still a ValueError (400), as before, so every `except ValueError` still holds. + """ + + +class SealError(RnfsError, ValueError): + """A sealed file is malformed or has been tampered with (rnfs.content.seal). A ValueError (400), as before.""" + + +class NotRecipient(ValueError): + """This node's identity isn't among a sealed file's recipients (rnfs.content.seal), or it has none. A + ValueError (400).""" + + +# Error class -> the local API's HTTP status (design.md, "Node facade"), checked in order: an error gets the status +# of the first class it is an instance of, and anything else 500. One class per status: rnfs.api maps errors by +# it (BlockMismatch, SealError and NotRecipient are ValueErrors: 400), rnfs.apiclient each status back to its +# error, and rnfs.gateway has a fixed message for each +STATUSES = ((ValueError, 400), (NotFound, 404), (TooLarge, 413), (Denied, 451), (InvalidDag, 422), (Busy, 503), + (StorageFull, 507)) diff --git a/rnfs/features/__init__.py b/rnfs/features/__init__.py index 4ab6a4a..811c82f 100644 --- a/rnfs/features/__init__.py +++ b/rnfs/features/__init__.py @@ -9,12 +9,27 @@ its parser, and client commands must not pay for loading RNS. Import it inside functions. - The constructor must be cheap and must not need Reticulum, so tests can build an offline node. - API handlers take the ApiHandler and use its helpers (param, json_body, send_json); they reach the - feature through `_common.feature(handler, NAME)`. Every feature route requires the API token. + feature through `_common.feature(handler, NAME)`. Every feature route requires the API token. Routes and + request paths are checked when the node loads the features (api_routes, request_handlers): one that is + malformed, or that rnfs or another feature defines too, stops the node with a FeatureError. - CLI commands are added with the `command(subparsers, name, run, help)` helper the CLI passes in, and run functions talk to the daemon with `rnfs.cli.client(args)` (import rnfs.cli inside the function). +- Background threads are started with `self._spawn(...)`; `stop(deadline)` asks them to end, then waits for them + with `self._join(deadline)`, so the node's stop budget bounds every feature's. +- A feature that uses another declares it: in `requires` if it can't work without it (that feature must come + before it in FEATURE_MODULES, or the node refuses to load), in `optional` if it uses it when loaded. It + reaches it with `self.dependency(name)`, which refuses a name it didn't declare. What the contacts feature + offers the others is the ContactsApi Protocol (tests/test_contacts.py checks the feature against it). """ import importlib -from collections.abc import Callable +import inspect +import re +import threading +import time +from collections.abc import Callable, Iterable +from typing import Literal, Protocol, overload, runtime_checkable + +from rnfs.constants import BLOCKS_PATH, FIND_PATH, PIN_PATH, PROVIDE_PATH # Module paths, in start order. A feature may use the features before it FEATURE_MODULES: list[str] = ["rnfs.features.peers", "rnfs.features.jobs", "rnfs.features.keepers", @@ -22,22 +37,111 @@ "rnfs.features.deny"] ApiRoute = Callable[[object], None] # handler(api_handler) +API_METHODS = ("GET", "POST") # what a feature's API route may answer +API_PATH = re.compile(r"/api/[a-z0-9][a-z0-9_/-]*") # a feature API route's path +REQUEST_PATH = re.compile(r"/[a-z0-9][a-z0-9_/-]*") # a feature request path (rnfs.dht) +CORE_REQUEST_PATHS = (FIND_PATH, PROVIDE_PATH, BLOCKS_PATH, PIN_PATH) # served by rnfs itself (network/server.py) + +STOP_WAIT = 5.0 # seconds stop() waits for a feature's threads when given no deadline (the node's own budget) + + +class FeatureError(RuntimeError): + """A feature that can't be loaded as declared (raised while the node loads its features, before any runs), or + one reaching a feature it didn't declare.""" + + +@runtime_checkable +class ContactsApi(Protocol): + """What other features may call on the contacts feature (rnfs.features.contacts.ContactsFeature, which + tests/test_contacts.py checks against this): a renamed method fails that test instead of silently turning + off what used it.""" + + def lookup(self, nick: str, kind: str | None = None) -> bytes | None: + """The hash of contact nick (of that kind, if given), or None.""" + + def has(self, ident: bytes, kind: str) -> bool: + """Whether some contact of that kind has hash ident.""" + + def nick_of(self, ident: bytes, kind: str) -> str | None: + """The nick of the contact of that kind with hash ident, or None.""" + + def resolve_id(self, text: str, kind: str) -> bytes: + """32 hex characters as bytes, or the hash of a contact of kind with that nick. ValueError otherwise.""" + + def expand_target(self, target): + """target with a leading name contact's nick replaced by the CID its name points at now.""" + + +@overload +def find(node, name: Literal["contacts"]) -> ContactsApi | None: ... +@overload +def find(node, name: str) -> "Feature | None": ... + + +def find(node, name): + """node's feature called name, or None if it isn't loaded (or node is a stand-in without features).""" + return (getattr(node, "features", None) or {}).get(name) class Feature: """Base class: override what the feature needs.""" name: str = "" # unique, used for node.feature(name) and in status + requires: tuple[str, ...] = () # features this one can't work without, each before it in FEATURE_MODULES + optional: tuple[str, ...] = () # features this one uses when they are loaded def __init__(self, node): """Set up state from node.config without touching the network.""" self.node = node + self._spawned: list[threading.Thread] = [] # started by _spawn and not yet seen ended; _join waits for them + self._spawned_lock = threading.Lock() + + @overload + def dependency(self, name: Literal["contacts"]) -> ContactsApi | None: ... + @overload + def dependency(self, name: str) -> "Feature | None": ... + + def dependency(self, name): + """The node's feature called name, which this one declares in requires or optional; None if it isn't + loaded. FeatureError for a name not declared, so a use the load-time check doesn't know of can't slip in.""" + if name not in self.requires and name not in self.optional: + raise _refused(self.name, f"it uses feature {name} without declaring it") + return find(self.node, name) def start(self): """Start background work, once the node's own services run. Not called on offline test nodes.""" - def stop(self): - """Stop background work and save state. Called before the node stops its own services.""" + def stop(self, deadline: float | None = None): + """Stop background work and save state. Called before the node stops its own services. + + deadline is when to stop waiting for the feature's threads, a time.monotonic() value (the node passes + its stop budget's); None waits STOP_WAIT from now. An override asks its threads to end, then calls + self._join(deadline); the base only waits for the threads _spawn started. + """ + self._join(deadline) + + def _spawn(self, target: Callable[..., object], name: str, *args) -> threading.Thread: + """Start a daemon thread named name running target(*args), which _join (and so stop) waits for. Raises + RuntimeError if it can't be started.""" + thread = threading.Thread(target=target, args=args, name=name, daemon=True) + thread.start() + with self._spawned_lock: + # Pruned here, so a feature starting threads now and then doesn't keep every ended one + self._spawned = [t for t in self._spawned if t.is_alive()] + self._spawned.append(thread) + return thread + + def _join(self, deadline: float | None = None, threads: list[threading.Thread] | None = None) -> bool: + """Wait until threads (by default those _spawn started) end or deadline (a time.monotonic() value; None: + STOP_WAIT from now) passes; True if they all ended. Ask them to end first.""" + if deadline is None: + deadline = time.monotonic() + STOP_WAIT + if threads is None: + with self._spawned_lock: + threads = list(self._spawned) + for thread in threads: + thread.join(max(0.0, deadline - time.monotonic())) + return not any(thread.is_alive() for thread in threads) def save(self): """Save state that is written in batches (called periodically and at stop).""" @@ -60,13 +164,19 @@ def requests(self) -> dict[str, Callable]: check it before decoding): True, False or None, a /find answer (two lists of at most K IDs), or a list of at most MAX_BLOCKS_CIDS entries, each bytes of at most MAX_BLOCK_SIZE or None (a /blocks answer). Any other shape (a dict, a string, a longer list) is refused as malformed, and widening that check wouldn't - reach the nodes already running. + reach the nodes already running. The request server checks each response the same way + (`protocol.body_refusal`) and answers False instead of one requesters would refuse, logging an error. + + Called once, when the node loads its features: a path that isn't "/" and a name (REQUEST_PATH), one rnfs + serves itself, or one another feature adds too stops the node (FeatureError, request_handlers). """ return {} @classmethod def api_routes(cls) -> dict[tuple[str, str], ApiRoute]: - """{(method, path): handler} for this feature's API endpoints (paths under /api/).""" + """{(method, path): handler} for this feature's API endpoints: GET or POST of a path under /api/ + (API_PATH). A route the core API or another feature defines too stops the node from loading (FeatureError, + api_routes).""" return {} @classmethod @@ -74,17 +184,88 @@ def cli_commands(cls, commands, command): """Add this feature's subcommands: command(commands, name, run, help) returns an argparse parser.""" +def stop_feature(feature: Feature, deadline: float): + """feature.stop(deadline=deadline), or feature.stop() for a feature whose stop takes no deadline (written for + rnfs before 0.1.6: it waits as it always did).""" + try: + inspect.signature(feature.stop).bind(deadline=deadline) + except (TypeError, ValueError): # no deadline parameter; ValueError: no signature to read + feature.stop() + return + feature.stop(deadline=deadline) + + def feature_classes() -> list[type[Feature]]: """The Feature subclass of every module in FEATURE_MODULES, in order.""" return [importlib.import_module(path).FEATURE for path in FEATURE_MODULES] +def checked_classes() -> list[type[Feature]]: + """feature_classes(), for the node to load in order, once checked: FeatureError if two share a name or one + has none, if one requires a feature that isn't loaded before it, or for an API route api_routes() refuses.""" + classes = feature_classes() + loaded: list[str] = [] + for cls in classes: + if not isinstance(cls.name, str) or not cls.name: + raise _refused(f"{cls.__module__}.{cls.__qualname__}", "it has no name") + if cls.name in loaded: + raise _refused(cls.name, "another feature has that name") + for kind in ("requires", "optional"): + names = getattr(cls, kind) + if not isinstance(names, tuple) or not all(isinstance(name, str) for name in names): + raise _refused(cls.name, f"{kind} must be a tuple of feature names") + for name in cls.requires: + if name not in loaded: + raise _refused(cls.name, f"it requires feature {name}, which isn't loaded before it") + loaded.append(cls.name) + _api_routes(classes) + return classes + + def api_routes() -> dict[tuple[str, str], ApiRoute]: - """Every feature's API routes.""" + """Every feature's API routes, checked: FeatureError for a route that isn't GET or POST of a path under /api/ + with a handler, or that the core API (rnfs.api.ApiHandler.ROUTES) or another feature defines too.""" + return _api_routes(feature_classes()) + + +def _api_routes(classes: list[type[Feature]]) -> dict[tuple[str, str], ApiRoute]: + """api_routes() of classes.""" + from rnfs.api import ApiHandler # here: only what serves or loads features needs the API's table + owners = dict.fromkeys(ApiHandler.ROUTES, "rnfs") # route -> what defines it, for the error routes: dict[tuple[str, str], ApiRoute] = {} - for cls in feature_classes(): + for cls in classes: for route, handler in cls.api_routes().items(): - if route in routes: - raise RuntimeError(f"API route {route} is defined by two features") + if not (isinstance(route, tuple) and len(route) == 2 and route[0] in API_METHODS + and isinstance(route[1], str) and API_PATH.fullmatch(route[1]) and callable(handler)): + raise _refused(cls.name, f"API route {route!r} must be (GET or POST, a path under /api/) with a " + "handler") + _claim(owners, route, cls.name, f"API route {route[0]} {route[1]}") routes[route] = handler return routes + + +def request_handlers(loaded: Iterable[Feature]) -> dict[str, Callable]: + """Every loaded feature's request paths (Feature.requests), checked: FeatureError for a path that isn't "/" + and a name with a handler, or that rnfs serves itself (CORE_REQUEST_PATHS) or another feature adds too.""" + owners = dict.fromkeys(CORE_REQUEST_PATHS, "rnfs") + handlers: dict[str, Callable] = {} + for feature in loaded: + for path, handler in feature.requests().items(): + if not (isinstance(path, str) and REQUEST_PATH.fullmatch(path) and callable(handler)): + raise _refused(feature.name, f"request path {path!r} must be \"/\" and a name, with a handler") + _claim(owners, path, feature.name, f"request path {path}") + handlers[path] = handler + return handlers + + +def _claim(owners: dict, key, name: str, what: str): + """Record that feature name defines key (what: how to say it); FeatureError if rnfs or a feature already does.""" + if key in owners: + other = owners[key] + raise _refused(name, f"{what} is already defined by {'rnfs' if other == 'rnfs' else 'feature ' + other}") + owners[key] = name + + +def _refused(name: str, problem: str) -> FeatureError: + """The one error for a feature that can't be loaded as it is: "feature : ".""" + return FeatureError(f"feature {name}: {problem}") diff --git a/rnfs/features/contacts.py b/rnfs/features/contacts.py index 0d6caf4..6aa2e6d 100644 --- a/rnfs/features/contacts.py +++ b/rnfs/features/contacts.py @@ -9,8 +9,9 @@ - A target may start with a name contact's nick: `rnfs get alice/docs/report.pdf` resolves alice's name to its CID first (RnfsNode._parse_target calls expand_target; never for the gateway, so web pages can't use or probe the address book). Wherever the CLI takes a node ID (`get --from`, `ping`, `pin remote`), a node - contact's nick works too (node_arg). The LXMF feature looks nicks up with lookup(nick, "lxmf"), and trusts - senders that has(address, "lxmf") finds. + contact's nick works too (node_arg). The LXMF feature resolves a recipient's nick with resolve_id(nick, "lxmf"), + trusts senders that has(address, "lxmf") finds, and names them in the inbox listing with nick_of(address, + "lxmf"). What other features may call is rnfs.features.ContactsApi (tests/test_contacts.py checks it). - Stored in the state database's contacts table (rnfs.statedb), each change written at once, durably; at most MAX_CONTACTS entries. @@ -20,6 +21,7 @@ import threading import time +from rnfs import ids from rnfs.content.cid import is_valid as is_valid_cid from rnfs.errors import NotFound from rnfs.features import Feature, _common @@ -31,14 +33,20 @@ MAX_CONTACTS = 10_000 # an address book is typed in by hand; this only bounds the saved state EXPAND_AGE = 60.0 # seconds a nick's resolved CID is reused when a target starts with a contact's nick _NICK = re.compile(r"[a-z0-9][a-z0-9._-]{0,31}") -_HEX_ID = re.compile(r"[0-9a-f]{32}") # RNS destination aspects a hash is checked against to tell its kind _KIND_ASPECTS = {"name": ("rnfs", "name"), "node": ("rnfs", "dht"), "lxmf": ("lxmf", "delivery")} +# Each kind in resolve_id's errors: (article, noun, the word before "contact") +_KIND_WORDS = {"name": ("a", "name", "name"), "node": ("a", "node", "node"), "lxmf": ("an", "LXMF address", "LXMF")} + + +def _log(message: str, level: int = LOG_INFO): + """Log with this feature's prefix.""" + _common.log(f"contacts: {message}", level) def check_nick(nick) -> str: """nick if it is a valid contact nick; ValueError otherwise.""" - if not isinstance(nick, str) or not _NICK.fullmatch(nick) or _HEX_ID.fullmatch(nick) or is_valid_cid(nick): + if not isinstance(nick, str) or not _NICK.fullmatch(nick) or ids.HEX16_LOWER.fullmatch(nick) or is_valid_cid(nick): raise ValueError(f"invalid nick {nick!r}: use 1-32 lowercase letters, digits, '-', '_' or '.', starting " "with a letter or digit (and not 32 hex digits)") return nick @@ -55,11 +63,7 @@ def is_nick(text) -> bool: def parse_hex(text) -> bytes: """A 16-byte destination hash from 32 hex characters (RNS's allowed). ValueError.""" - if isinstance(text, str): - text = text.strip().strip("<>").lower() - if _HEX_ID.fullmatch(text): - return bytes.fromhex(text) - raise ValueError(f"invalid hash {text!r}: must be 32 hex characters") + return ids.parse_hex16(text, "hash", brackets=True) def check_kind(kind) -> str: @@ -85,7 +89,7 @@ def import_contacts(value, tx) -> int: (nick, check_kind(entry["kind"]), parse_hex(entry["id"]))) count += 1 except (KeyError, TypeError, ValueError): - ContactsFeature._log(f"ignoring malformed contact {nick!r} in contacts.json", LOG_WARNING) + _log(f"ignoring malformed contact {nick!r} in contacts.json", LOG_WARNING) if count >= MAX_CONTACTS: break return count @@ -95,6 +99,7 @@ class ContactsFeature(Feature): """The address book. Thread-safe.""" name = NAME + optional = ("keepers",) # a name it keeps is a name (detect_kind) def __init__(self, node): """Load the address book from the state database.""" @@ -102,23 +107,21 @@ def __init__(self, node): super().__init__(node) self._lock = threading.Lock() self._contacts: dict[str, tuple[str, bytes]] = {} # nick -> (kind, hash) - self._nicks: dict[tuple[str, bytes], int] = {} # (kind, hash) -> how many nicks name it, for has() - self._dirty: set[str] = set() # nicks changed (or deleted) whose save failed, written with the next change + self._nicks: dict[tuple[str, bytes], set[str]] = {} # (kind, hash) -> the nicks naming it, for has() # name -> (CID, monotonic time of the resolve that found it), reused for EXPAND_AGE self._expanded: dict[bytes, tuple[str, float]] = {} - self._db = statedb.connect(node.config.state_dir) - try: - self._load() - except BaseException: - self.close() - raise + self._db = statedb.open_store(node.config.state_dir, self._load) # State - def _load(self): + def _load(self, db): """Read the contacts table, by nick; malformed rows are logged and deleted, as are any past MAX_CONTACTS.""" + from rnfs import statedb + # Written through, durably: a change is on disk when it returns; one whose write failed is written with the + # next change, or by save() + self._store = statedb.RowStore(db, "contacts", ("nick",), durable=True, lock=self._lock, row=self._row) bad = [] - for nick, kind, ident in self._db.query("SELECT nick, kind, id FROM contacts ORDER BY nick"): + for nick, kind, ident in db.query("SELECT nick, kind, id FROM contacts ORDER BY nick"): if len(self._contacts) >= MAX_CONTACTS: bad.append((nick,)) # never added (the address book is full): dropped, as from contacts.json continue @@ -128,43 +131,39 @@ def _load(self): raise ValueError("not a 16-byte hash") self._set(nick, (check_kind(kind), ident)) except (TypeError, ValueError): - self._log(f"ignoring malformed contact {nick!r}", LOG_WARNING) + _log(f"ignoring malformed contact {nick!r}", LOG_WARNING) bad.append((nick,)) - if bad: - with self._db.transaction() as tx: - tx.executemany("DELETE FROM contacts WHERE nick = ?", bad) + statedb.delete_rows(db, "contacts", ("nick",), bad) + + def _row(self, nick: str) -> tuple | None: + """nick's contacts row, or None if there is no such contact (lock held).""" + entry = self._contacts.get(nick) + return None if entry is None else (nick, *entry) def _set(self, nick: str, value: tuple[str, bytes] | None): """Set or (None) delete a contact, keeping _nicks in step (lock held, or loading).""" old = self._contacts.pop(nick, None) if old is not None: - self._nicks[old] -= 1 + self._nicks[old].discard(nick) if not self._nicks[old]: del self._nicks[old] if value is not None: self._contacts[nick] = value - self._nicks[value] = self._nicks.get(value, 0) + 1 + self._nicks.setdefault(value, set()).add(nick) def _save(self, nick: str): - """Write nick's row, and those of earlier changes whose save failed, durably (lock held). If it fails they - are written with the next change.""" - self._dirty.add(nick) - rows = [(other, *self._contacts[other]) for other in self._dirty if other in self._contacts] - gone = [(other,) for other in self._dirty if other not in self._contacts] - with self._db.transaction(durable=True) as tx: - tx.executemany("DELETE FROM contacts WHERE nick = ?", gone) - tx.executemany("INSERT OR REPLACE INTO contacts (nick, kind, id) VALUES (?, ?, ?)", rows) - self._dirty.clear() + """Write nick's row, and those of earlier changes whose save failed, durably (lock held). Raises OSError if + it fails (the change stays in memory); they are then written with the next change, or by save().""" + self._store.changed(nick) + + def save(self): + """Write the contacts whose write failed, if any (each change is written as it is made).""" + self._store.save() def close(self): """Release the state database (the node calls this once it has saved everything at shutdown).""" self._db.close() - @staticmethod - def _log(message: str, level: int = LOG_INFO): - """Log with this feature's prefix.""" - _common.log(f"contacts: {message}", level) - # The address book def add(self, nick: str, ident_hex: str, kind: str | None = None) -> dict: @@ -182,7 +181,7 @@ def add(self, nick: str, ident_hex: str, kind: str | None = None) -> dict: raise ValueError(f"the address book is full ({MAX_CONTACTS} contacts)") self._set(nick, (kind, ident)) self._save(nick) - self._log(f"added {nick}: {kind} {ident.hex()}") + _log(f"added {nick}: {kind} {ident.hex()}") return {"nick": nick, "kind": kind, "id": ident.hex()} def remove(self, nick: str): @@ -215,6 +214,13 @@ def has(self, ident: bytes, kind: str) -> bool: with self._lock: return (kind, bytes(ident)) in self._nicks + def nick_of(self, ident: bytes, kind: str) -> str | None: + """The nick of the contact of that kind with hash ident (the first by nick if several name it), or None + (for other features: the LXMF inbox listing names its senders' contacts).""" + with self._lock: + nicks = self._nicks.get((kind, bytes(ident))) + return min(nicks) if nicks else None + def get(self, nick: str) -> dict: """One contact. NotFound if there is none; ValueError for an invalid nick.""" check_nick(nick) @@ -225,20 +231,24 @@ def get(self, nick: str) -> dict: return {"nick": nick, "kind": entry[0], "id": entry[1].hex()} def resolve_id(self, text: str, kind: str) -> bytes: - """32 hex characters as bytes, or the hash of a contact of kind with that nick. ValueError otherwise.""" + """32 hex characters as bytes, or the hash of a contact of kind with that nick. ValueError otherwise. + + Other features call it too (keepers' and follow's names, LXMF's recipients), so they all word it alike. + """ try: return parse_hex(text) except ValueError: pass + article, noun, word = _KIND_WORDS.get(kind, ("a", kind, kind)) if is_nick(text): ident = self.lookup(text, kind) if ident is not None: return ident known = self.lookup(text) if known is not None: - raise ValueError(f"contact {text} is not a {kind}") - raise ValueError(f"no {kind} contact {text}") - raise ValueError(f"invalid {kind} {text!r}: give 32 hex characters or a contact's nick") + raise ValueError(f"contact {text} is not {article} {noun}") + raise ValueError(f"no {word} contact {text}") + raise ValueError(f"invalid {noun} {text!r}: give 32 hex characters or a contact's nick") # Kinds @@ -248,7 +258,7 @@ def detect_kind(self, ident: bytes) -> str | None: names = node.names if any(entry["name"] == ident.hex() for entry in names.list()) or names.record(ident) is not None: return "name" - keepers = node.features.get("keepers") + keepers = self.dependency("keepers") if keepers is not None and keepers.is_kept(ident): return "name" if ident == node.node_id or node.routing.contains(ident): @@ -398,7 +408,7 @@ def node_arg(args, text: str | None) -> str | None: For the CLI's --from, ping and pin remote. Fails the command for an unknown nick. """ - if text is None or _HEX_ID.fullmatch(text.strip().strip("<>").lower()) or not is_nick(text): + if text is None or ids.is_hex16(text, brackets=True) or not is_nick(text): return text # hex, or something the API will reject with its own message import rnfs.cli contact = rnfs.cli.client(args).get("/api/contacts/get", nick=text) # exits with "no contact " diff --git a/rnfs/features/deny.py b/rnfs/features/deny.py index 877e9c1..d5e150f 100644 --- a/rnfs/features/deny.py +++ b/rnfs/features/deny.py @@ -8,7 +8,7 @@ can't be fetched, read or parsed is logged, and the source keeps the version it had. - A name is resolved as `name resolve` does (persisted, so its announces keep the record held up to date), checked every CHECK_INTERVAL from the record held, and through the network only if nothing was heard for - STALE_ANNOUNCES x announce_interval. A failed check is retried after RETRY_INTERVAL, doubling up to + MISSED_ANNOUNCES x announce_interval. A failed check is retried after RETRY_INTERVAL, doubling up to announce_interval; a CID source once accepted is never checked again. - At start, sources no longer in deny_lists are forgotten and every "deny" pin no source uses is released. - One background thread does every check and fetch, one at a time. @@ -19,7 +19,7 @@ import threading import time -from rnfs.constants import ID_LENGTH +from rnfs.constants import ID_LENGTH, MISSED_ANNOUNCES from rnfs.content.cid import Cid from rnfs.errors import RnfsError from rnfs.features import Feature, _common @@ -29,7 +29,6 @@ NAME = "deny" NAME_PREFIX = "name/" CHECK_INTERVAL = 600.0 # seconds between checks of a name source's record held (no network) -STALE_ANNOUNCES = 2 # a record not heard for this many announce_intervals is refreshed from the network FAR_FUTURE = float("inf") @@ -86,7 +85,6 @@ def __init__(self, node): self._failures: dict[str, int] = {} # source -> checks failed in a row self._wake = threading.Event() self._stop = threading.Event() - self._thread: threading.Thread | None = None for source in self.deny.accepted(): if source not in self._sources: self.deny.forget(source) @@ -112,13 +110,14 @@ def start(self): with self._lock: for source in self._sources: self._due[source] = first - self._thread = threading.Thread(target=self._loop, name="rnfs-deny", daemon=True) - self._thread.start() + self._spawn(self._loop, "rnfs-deny") - def stop(self): - """Stop the background thread (a fetch in progress stops at its next network wait).""" + def stop(self, deadline: float | None = None): + """Stop the background thread and wait for it until deadline (a fetch in progress stops at its next network + wait), so the node doesn't close the state database under it.""" self._stop.set() self._wake.set() + self._join(deadline) def status(self) -> dict: """Entries denied (the local list's and every source's) and sources configured.""" @@ -142,7 +141,7 @@ def check(self, source: str) -> bool: root, name, names = parse_source(source) previous = self.deny.accepted().get(source) if name is not None: - max_age = STALE_ANNOUNCES * self.node.config.announce_interval + max_age = MISSED_ANNOUNCES * self.node.config.announce_interval try: result = self.node.names.resolve(name, persist=True, max_age=max_age) except Exception as e: # noqa: BLE001 - resolving is retried later @@ -161,7 +160,7 @@ def check(self, source: str) -> bool: if previous == expect.cid and self.deny.in_use(source): self._set_state(source, "in use") return True - if walker.kind_of(expect, fetch.get(expect)) != "file": + if walker.kind_of(expect, fetch.get(expect)) != walker.FILE_TYPE: raise ValueError("it is a directory, not a file") if fetch.size_of(expect) > denylist.MAX_LIST_BYTES: raise ValueError(f"it is over {denylist.MAX_LIST_BYTES // 1024} KiB") diff --git a/rnfs/features/follow.py b/rnfs/features/follow.py index 47725f6..bb5594e 100644 --- a/rnfs/features/follow.py +++ b/rnfs/features/follow.py @@ -12,7 +12,11 @@ held, without a path request. - One background thread does every check and fetch, one at a time, so following many names never runs more than one transfer. Fetches obey the usual limits (max_download_size, the DAG byte bound of the new - target's declared size, disk reserve); a followed target is a pin, so it is provided. + target's declared size, disk reserve) and follow_quota: a new target whose size, or the bytes of its DAG's + blocks, would exceed it is refused (TooLarge, shown as the check's state; the old target stays pinned), and 0 + fetches nothing. A pinned fetch never stops at max_storage, so the quota is what keeps a name's owner from + filling the disk; while a target replaces another both are held, so up to twice the quota per name. A target + pinned before the quota was lowered stays. A followed target is a pin, so it is provided. - Followed names count against keep_max_names. State: the follow table of the state database (rnfs.statedb), each change committed durably as it is made (the rows of the names it changes), and a meta value saying it was saved whole ("clean"), which the reconcile at start needs. @@ -22,9 +26,9 @@ import threading import time -from rnfs.constants import ID_LENGTH +from rnfs.constants import ID_LENGTH, MISSED_ANNOUNCES from rnfs.content.cid import Cid -from rnfs.errors import NotFound, RnfsError +from rnfs.errors import NotFound, RnfsError, TooLarge from rnfs.features import Feature, _common from rnfs.features._common import LOG_ERROR, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING, RETRY_INTERVAL, retry_delay from rnfs.features.keepers import FOLLOW, name_subcommands, parse_name_arg @@ -33,7 +37,6 @@ NAME = "follow" SOURCE_PREFIX = "follow:" FAR_FUTURE = float("inf") -STALE_ANNOUNCES = 2 # a record not heard for this many announce_intervals is refreshed from the network def follow_source(name: bytes) -> str: @@ -75,6 +78,8 @@ class FollowFeature(Feature): """Followed names and their pins. Thread-safe.""" name = NAME + requires = ("keepers",) # a followed name is kept + optional = ("contacts",) # a name contact's nick names a name (parse_name_arg) def __init__(self, node): """Load followed names; no network.""" @@ -90,8 +95,7 @@ def __init__(self, node): self._resolved: set[bytes] = set() # names whose last check resolved them (a failed fetch, then) self._wake = threading.Event() self._stop = threading.Event() - self._thread: threading.Thread | None = None - self._keepers = node.feature("keepers") + self._keepers = self.dependency("keepers") self._keepers.add_listener(self._changed) try: self._reconcile(self._load()) @@ -187,13 +191,14 @@ def start(self): with self._lock: for name in self._pinned: self._due[name] = (first, True) - self._thread = threading.Thread(target=self._loop, name="rnfs-follow", daemon=True) - self._thread.start() + self._spawn(self._loop, "rnfs-follow") - def stop(self): - """Stop the background thread (a fetch in progress stops at its next network wait).""" + def stop(self, deadline: float | None = None): + """Stop the background thread and wait for it until deadline (a fetch in progress stops at its next network + wait), so the node doesn't close the state database under it.""" self._stop.set() self._wake.set() + self._join(deadline) def close(self): """Release the state database.""" @@ -271,7 +276,7 @@ def check(self, name: bytes, network: bool = True) -> bool: """Bring name's follow pin up to date with its latest target; True if it is (or nothing to do). network=True resolves through the network (a flooded path request, maybe a DHT lookup) only if the - record held wasn't heard within STALE_ANNOUNCES x announce_interval; network=False uses the record held + record held wasn't heard within MISSED_ANNOUNCES x announce_interval; network=False uses the record held whatever its age (the network only when none is held). """ with self._lock: @@ -280,7 +285,7 @@ def check(self, name: bytes, network: bool = True) -> bool: pinned = self._pinned[name] self._resolved.discard(name) try: - max_age = STALE_ANNOUNCES * self.node.config.announce_interval if network else FAR_FUTURE + max_age = MISSED_ANNOUNCES * self.node.config.announce_interval if network else FAR_FUTURE result = self.node.names.resolve(name, persist=True, max_age=max_age) except Exception as e: # noqa: BLE001 - resolving is retried later self._set_state(name, f"resolve failed: {e}") @@ -296,9 +301,25 @@ def check(self, name: bytes, network: bool = True) -> bool: return True self._set_state(name, f"fetching {target}") source = follow_source(name) + quota = self.node.config.follow_quota try: + if quota == 0: + raise TooLarge("follow fetches are off (follow_quota is 0)") with self.node.fetcher.open(target, pinned=True) as fetch: - fetch.fetch_dag(fetch.resolve()) + # A pinned fetch never stops at max_storage, so without this the name's owner could fill the disk + expect = fetch.resolve() + content = fetch.size_of(expect) + if content > quota: + raise TooLarge(f"{content} bytes would exceed follow_quota ({quota})") + # A DAG of empty files has size 0 yet can hold gigabytes of directory nodes: count its blocks too + # (fetch_dag also bounds them by what the size can hold, whichever is lower) + fetch.max_dag_bytes = quota + try: + fetch.fetch_dag(expect) + except TooLarge as e: + if fetch.dag_bytes <= quota: + raise # the bound of its size, not the quota + raise TooLarge(f"{e}, more than follow_quota") from e with self._lock: followed = name in self._pinned if followed: diff --git a/rnfs/features/jobs.py b/rnfs/features/jobs.py index fe344d8..6bd376e 100644 --- a/rnfs/features/jobs.py +++ b/rnfs/features/jobs.py @@ -26,7 +26,8 @@ - Finished jobs (done, failed, cancelled) are kept for `rnfs jobs`, at most max_finished_jobs, but each for at least FINISHED_GRACE: a CLI following a fast job must still find it once it ends. - Job threads are daemon threads, so they never hold up the process; stop() asks running fetches to stop - and waits at most STOP_WAIT for them. A job interrupted by the shutdown is saved as queued. + and waits for them until the deadline the node passes (Feature.stop: its stop budget). A job interrupted by + the shutdown is saved as queued. No RNS at module level (the CLI imports this module): node-side modules are imported inside functions. """ @@ -41,9 +42,11 @@ from collections import deque from contextlib import ExitStack +from rnfs import ids from rnfs.constants import PARTIAL_SUFFIX from rnfs.features import Feature, _common from rnfs.features._common import LOG_ERROR, LOG_INFO, LOG_NOTICE, LOG_WARNING +from rnfs.fsutil import printable NAME = "jobs" KINDS = ("get", "pin", "export") @@ -54,13 +57,12 @@ MAX_ACTIVE = 1000 # queued and running jobs at most, so a script can't queue without bound RATE_WINDOW = 300.0 # seconds the rate is averaged over: LoRa responses can arrive minutes apart SAMPLE_INTERVAL = 1.0 # seconds between rate samples -STOP_WAIT = 2.0 # seconds stop() waits for running jobs to notice they must stop MAX_WAIT = 30 # longest wait= a GET /api/jobs/get may ask for, in seconds POLL_TTY = 1 # seconds between progress updates the CLI shows POLL_QUIET = 10 # seconds a CLI without a progress line waits per request (it returns early when the job ends) FINISHED_GRACE = 60.0 # seconds a finished job is kept whatever max_finished_jobs says _JOB_ID = re.compile(r"[0-9a-f]{8}") -_NODE_ID = re.compile(r"[0-9a-f]{32}") +_NODE_ID = ids.HEX16_LOWER # a saved job's from_node, as bytes.hex() writes it (a row that fails is deleted) def _log(message: str, level: int = LOG_INFO): @@ -98,6 +100,7 @@ def __init__(self, job_id: str, kind: str, target: str, output: str | None = Non self.ended = threading.Event() # set once finished, for waiters self.stop_event: threading.Event | None = None # the feature's, set when the node stops self.protection: ExitStack | None = None # the session keeping its blocks while queued or running + self.position: int | None = None # its row's place in the jobs table (creation order), once queued or loaded self._lock = threading.Lock() # guards _samples and _seen self._samples: deque[tuple[float, int]] = deque() # (monotonic time, downloaded_bytes) self._seen: set = set() # CIDs met this run, so each counts once @@ -116,6 +119,9 @@ def block(self, cid, length: int, downloaded: bool): self.downloaded_bytes += length if not self._samples or now - self._samples[-1][0] >= SAMPLE_INTERVAL: self._samples.append((now, self.downloaded_bytes)) + # Prune here too, not only in rate(): a job nobody polls would otherwise keep a sample a + # second for as long as it runs (about 10 MB a day) + self._prune(now) if cid in self._seen: return self._seen.add(cid) @@ -139,15 +145,19 @@ def begin_run(self): self._seen = set() self._samples = deque([(time.monotonic(), 0)]) + def _prune(self, now: float): + """Drop samples older than RATE_WINDOW, keeping one at or before the window's start as the base to + measure from. Called with _lock held.""" + while len(self._samples) > 1 and self._samples[1][0] <= now - RATE_WINDOW: + self._samples.popleft() + def rate(self, now: float | None = None) -> float | None: """Bytes/s downloaded over the last RATE_WINDOW of a running job (it decays while nothing arrives).""" now = time.monotonic() if now is None else now with self._lock: if self.state != RUNNING or not self._samples: return None - # Keep one sample at or before the window's start, as the base to measure from - while len(self._samples) > 1 and self._samples[1][0] <= now - RATE_WINDOW: - self._samples.popleft() + self._prune(now) since, base = self._samples[0] downloaded = self.downloaded_bytes elapsed = now - since @@ -284,42 +294,37 @@ class JobsFeature(Feature): """The job manager: queue, threads, persistence, protection of resumable blocks. Thread-safe.""" name = NAME + optional = ("contacts",) # a target may start with a name contact's nick def __init__(self, node): """Load saved jobs, queueing again those that hadn't finished and protecting their blocks.""" from rnfs import statedb super().__init__(node) self._lock = threading.Lock() - self._save_lock = threading.Lock() # keeps saves in order without holding _lock while writing self._jobs: dict[str, Job] = {} # in creation order self._threads: dict[str, threading.Thread] = {} self._started = False - self._dirty = False # jobs changed since the last save self._stop_event = threading.Event() - # What the jobs table holds: id -> (position, dump()ed record); and the next new job's position (after - # every one saved). Both used under _save_lock only - self._saved: dict[str, tuple[int, str]] = {} - self._position = 0 - self._db = statedb.connect(node.config.state_dir) - try: - self._load() - except BaseException: - self.close() - raise + self._position = 0 # the next new job's position (after every one saved) + self._db = statedb.open_store(node.config.state_dir, self._load) @property def max_jobs(self) -> int: """Jobs running at once: max_jobs, by default 2, or 1 on slow nodes.""" - configured = getattr(self.node.config, "max_jobs", None) + configured = self.node.config.max_jobs return configured if configured is not None else (1 if self.node.slow else 2) # Persistence - def _load(self): + def _load(self, db): """Read the jobs table, oldest first; a malformed row is logged and deleted.""" from rnfs import statedb + # Saved durably at each change and periodically (a running job's progress), outside the lock; only the + # jobs whose record changed are written + self._store = statedb.RowStore(db, "jobs", ("id",), durable=True, mode=statedb.SNAPSHOT, lock=self._lock, + row=self._row, skip_unchanged=True) bad = [] - for job_id, position, text in self._db.query("SELECT id, position, record FROM jobs ORDER BY position"): + for job_id, position, text in db.query("SELECT id, position, record FROM jobs ORDER BY position"): try: if not isinstance(position, int): raise TypeError("bad position") @@ -331,8 +336,10 @@ def _load(self): _log(f"ignoring a saved job ({e})", LOG_WARNING) bad.append((job_id,)) continue - self._saved[job.id] = (position, text) + job.position = position if job.state in ACTIVE: + if job.state == RUNNING: + self._store.changed(job.id) # saved as queued by the next save job.state = QUEUED # interrupted: it resumes once the node runs job.stop_event = self._stop_event self._protect(job) @@ -343,45 +350,25 @@ def _load(self): resumed = sum(1 for job in self._jobs.values() if job.state == QUEUED) if resumed: _log(f"{resumed} unfinished job(s) will resume") - if bad: - with self._db.transaction() as tx: - tx.executemany("DELETE FROM jobs WHERE id = ?", bad) + statedb.delete_rows(db, "jobs", ("id",), bad) - def save(self): - """Save the jobs changed since the last save, and forget those gone, durably, if jobs changed or one runs - (the node calls this periodically, keeping saved progress recent).""" + def _row(self, job_id: str) -> tuple | None: + """job_id's jobs row, or None once it is forgotten (lock held).""" from rnfs import statedb - with self._save_lock: - with self._lock: - if not self._dirty and not any(job.state == RUNNING for job in self._jobs.values()): - return - records = [(job_id, statedb.dump(job.to_record())) for job_id, job in self._jobs.items()] - self._dirty = False - saved, position, rows = {}, self._position, [] - for job_id, text in records: - old = self._saved.get(job_id) - if old is None: - old = (position, None) - position += 1 - if old[1] != text: - rows.append((job_id, old[0], text)) - saved[job_id] = (old[0], text) - gone = [(job_id,) for job_id in self._saved if job_id not in saved] - try: - if rows or gone: - with self._db.transaction(durable=True) as tx: # a queued job must survive a crash - tx.executemany("DELETE FROM jobs WHERE id = ?", gone) - tx.executemany("INSERT OR REPLACE INTO jobs (id, position, record) VALUES (?, ?, ?)", rows) - except BaseException: - with self._lock: - self._dirty = True - raise - self._saved, self._position = saved, position + job = self._jobs.get(job_id) + return None if job is None else (job_id, job.position, statedb.dump(job.to_record())) - def _save_quietly(self): - """save() after a change, logging rather than raising: a job's outcome mustn't be lost to a full disk.""" + def save(self): + """Save the jobs changed since the last save, and forget those gone, durably (a queued job must survive a + crash); running jobs whose progress changed too (the node calls this periodically, keeping saved progress + recent). Each change is noted with _store.changed(job ID), under the lock.""" with self._lock: - self._dirty = True + self._store.changed(*(job_id for job_id, job in self._jobs.items() if job.state == RUNNING)) + self._store.save() + + def _save_quietly(self): + """save() after a change, logging rather than raising: a job's outcome mustn't be lost to a full disk (it + is saved again by the next save).""" try: self.save() except OSError as e: @@ -404,6 +391,7 @@ def _cap_finished(self): for job in finished[:max(0, len(finished) - limit)]: if (job.finished or job.created) < recent: del self._jobs[job.id] + self._store.changed(job.id) # Protection @@ -431,15 +419,14 @@ def start(self): self._started = True self._schedule() - def stop(self): - """Ask running jobs to stop (they are saved as queued, to resume), wait briefly, save.""" + def stop(self, deadline: float | None = None): + """Ask running jobs to stop (they are saved as queued, to resume), wait for them until deadline (as + Feature.stop), save.""" with self._lock: self._started = False threads = list(self._threads.values()) self._stop_event.set() - deadline = time.monotonic() + STOP_WAIT - for thread in threads: - thread.join(max(0.0, deadline - time.monotonic())) + self._join(deadline, threads) try: self.save() # only if something changed: offline test nodes with no jobs write nothing except OSError as e: @@ -464,6 +451,7 @@ def _schedule(self) -> bool: if job.state != QUEUED: continue job.begin_run() + self._store.changed(job.id) thread = threading.Thread(target=self._run, args=(job,), name=f"rnfs-job-{job.id}", daemon=True) self._threads[job.id] = thread starting.append(thread) @@ -571,6 +559,7 @@ def _finish(self, job: Job, state: str, error: str | None, result: dict | None): if state in FINISHED: job.finished = time.time() self._threads.pop(job.id, None) + self._store.changed(job.id) self._cap_finished() if state in FINISHED: job.ended.set() @@ -588,7 +577,7 @@ def submit(self, kind: str, target: str, output: str | None = None, from_node: s by DHT discovery if from_node fails. """ from rnfs.errors import Busy - contacts = getattr(self.node, "features", {}).get("contacts") + contacts = self.dependency("contacts") if contacts is not None: # `alice/docs`: a name contact's nick becomes the CID its name points at now (rnfs.features.contacts) target = contacts.expand_target(target) @@ -607,8 +596,11 @@ def submit(self, kind: str, target: str, output: str | None = None, from_node: s job_id = secrets.token_hex(4) job = Job(job_id, kind, target, output, from_node, sealed_name=sealed_name, fallback_dht=fallback_dht) job.stop_event = self._stop_event + job.position = self._position + self._position += 1 self._protect(job) self._jobs[job_id] = job + self._store.changed(job_id) _log(f"job {job_id} queued: {kind} {target}" + (f" to {output}" if output else "")) if not self._schedule(): # else saved as it started self._save_quietly() @@ -644,6 +636,7 @@ def cancel(self, job_id: str) -> dict: if job.state == QUEUED: job.state = CANCELLED job.finished = time.time() + self._store.changed(job.id) ended = True elif job.state == RUNNING: job.cancel_requested = True @@ -664,6 +657,7 @@ def clear(self, job_id: str | None = None) -> int: if job.state in FINISHED and (job_id is None or jid == job_id)] for jid in removed: del self._jobs[jid] + self._store.changed(*removed) if removed: self._save_quietly() return len(removed) @@ -820,7 +814,6 @@ def format_progress(job: dict, width: int = 20) -> str: def format_job(job: dict) -> str: """One `rnfs jobs` line: ID, state, kind, progress, target (and output).""" - from rnfs.cli import printable where = f" -> {job['output']}" if job.get("output") else "" return printable(f"{job['id']} {job['state']:<9} {job['kind']:<6} {format_progress(job)} {job['target']}{where}") @@ -927,14 +920,14 @@ def follow(api, job_id: str, cancel_on_interrupt: bool) -> dict: if job["state"] in FINISHED: return job if line: - line.show(format_progress(job)) + line.show(printable(format_progress(job))) except KeyboardInterrupt: if line: line.clear() if cancel_on_interrupt: api.post("/api/jobs/cancel", id=job_id) - print(f"rnfs: cancelled job {job_id} (with --background, a transfer keeps running after the command " - "ends)", file=sys.stderr) + print(f"rnfs: cancelled job {printable(job_id)} (with --background, a transfer keeps running after " + "the command ends)", file=sys.stderr) sys.exit(130) finally: if line: @@ -959,9 +952,11 @@ def run_transfer(args, fields: dict, fallback) -> dict | None: return fallback(api) job = api.post("/api/jobs", **fields) if getattr(args, "background", False): - print(job["id"]) + # Everything the daemon sends is escaped before it reaches the terminal, its job IDs too + job_id = printable(job["id"]) + print(job_id) if sys.stderr.isatty(): - print(f"rnfs: started job {job['id']}; follow it with: rnfs jobs wait {job['id']}", file=sys.stderr) + print(f"rnfs: started job {job_id}; follow it with: rnfs jobs wait {job_id}", file=sys.stderr) return None job = follow(api, job["id"], cancel_on_interrupt=True) if job["state"] != DONE: @@ -1014,18 +1009,18 @@ def run_cancel(args): from rnfs import cli job = cli.client(args).post("/api/jobs/cancel", id=_cli_id(args.id)) if job["state"] == RUNNING: - print(f"cancelling {job['id']}") + print(f"cancelling {printable(job['id'])}") elif job["state"] == CANCELLED: - print(f"cancelled {job['id']}") + print(f"cancelled {printable(job['id'])}") else: - print(f"{job['id']} already {job['state']}") + print(printable(f"{job['id']} already {job['state']}")) def run_clear(args): """Forget finished jobs.""" from rnfs import cli removed = cli.client(args).post("/api/jobs/clear", id=_cli_id(args.id) if args.id else None)["removed"] - print(f"removed {removed} finished job{'' if removed == 1 else 's'}") + print(printable(f"removed {removed} finished job{'' if removed == 1 else 's'}")) def _cli_id(text: str) -> str: diff --git a/rnfs/features/keepers.py b/rnfs/features/keepers.py index db44971..0d0649f 100644 --- a/rnfs/features/keepers.py +++ b/rnfs/features/keepers.py @@ -16,11 +16,13 @@ 36- or 39-byte binary CID). A name is advertised only once a record of it is held. - **Resolving** (NameManager.resolve): when the path request heard nothing (or always, with keeper_query = "always"), find providers of the name's key (past the first alpha, with local records only candidates, until - two established servers list some), rank them by routing-table standing as it was before the search, then at - random, and ask them with /name a batch of alpha at a time, each batch drawing first on sources no earlier - pick came from, asking another batch (up to a bound) while no answer reaches the sequence already known or - only one keeper (one established keeper, when no sequence is known) answered; only verifying records are - offered to the usual sequence rules (persisted highest sequence, no rollback). + two established servers list some, or some are known and lookup_timeout / 5 has passed), rank them by + routing-table standing as it was before the search, then at random, and ask them with /name a batch of alpha + at a time, each batch drawing first on sources no earlier pick came from and waited for at least name_wait + (87.5 s), asking another batch (up to a bound) while no answer reaches the sequence already known or only one + keeper (one established keeper, when no sequence is known) answered, resuming a search stopped early when the + candidates run out; only verifying records are offered to the usual sequence rules (persisted highest + sequence, no rollback). - **What is kept.** Names kept by the user (`rnfs name keep`), followed names (rnfs.features.follow), and this node's own published names, which it signs itself. At most keep_max_names, own names not counted. - **Refreshing.** On hearing an announce with a higher sequence for a kept name, the signed announce is read @@ -45,7 +47,7 @@ from rnfs.constants import ID_LENGTH from rnfs.content.cid import Cid from rnfs.errors import NotFound -from rnfs.features import Feature, _common +from rnfs.features import Feature, _common, find from rnfs.features._common import LOG_ERROR, LOG_NOTICE, LOG_VERBOSE NAME = "keepers" @@ -68,6 +70,8 @@ LOOP_INTERVAL = 60.0 # the background thread syncs own names and saves at least this often KEEPER_CANDIDATES = 3 # the search for a name's keepers runs until 3 × alpha are known (from 2+ sources) KEEPER_ROUNDS = 3 # batches of alpha keepers one query asks at most (2 on slow nodes) +SEARCH_SHARE = 5 # with candidates in hand, the search for keepers stops once lookup_timeout / 5 has passed +NAME_HOPS = 5 # hops of RNS link establishment timeout a /name batch allows for REQUEST_ID_LENGTH = 16 _UNVOUCHED = b"" # ask_keepers: the source of candidates no established node (nor this one) listed @@ -77,6 +81,16 @@ def name_key(name: bytes) -> bytes: return hashlib.sha256(KEY_DOMAIN + name).digest()[:ID_LENGTH] +def name_wait(config) -> float: + """How long a batch of /name queries is waited for at least, apart from the lookup's budget: a path request + (path_timeout), a link set up over NAME_HOPS hops with LinkPool's grace, and the request (LinkPool's deadline + for a request other than /blocks, before an RTT is measured). 87.5 s at the defaults.""" + import RNS + from rnfs.network import links, protocol + return (config.path_timeout + links.ESTABLISH_GRACE + NAME_HOPS * RNS.Link.ESTABLISHMENT_TIMEOUT_PER_HOP + + links.DEADLINE_FACTOR * protocol.REQUEST_TIMEOUT_WITHOUT_RTT) + + def _log(message: str, level: int = LOG_VERBOSE): """Log with this feature's prefix.""" _common.log(f"keepers: {message}", level) @@ -417,6 +431,178 @@ def count(self) -> int: return len(self._reasons) +# Keeper queries + + +class _KeeperQuery: + """One keeper query of KeepersFeature.ask_keepers for a name: the search for its keepers, the candidates it + found, and the verifying records they answered with so far. + + The search and the choice of batches run in the calling thread; ask runs on a thread per keeper asked, so the + answers are guarded by a lock. A search stopped early may be resumed for more candidates (resume). + """ + + def __init__(self, feature: "KeepersFeature", name: bytes): + """The query's starting state; nothing is sent until search() runs.""" + node = self.node = feature.node + self.feature = feature + self.name = name + self.key = name_key(name) + self.alpha = node.config.effective_alpha(node.slow) + self.banned = node.routing.is_banned + # Taken before the lookup, whose answers add nodes to the routing table + self.before = node.routing.standings(node.config.provider_seniority) + self.local = [p for p in node.records.providers(self.key, exclude=self.banned) if p != node.node_id] + self.found = dict.fromkeys(self.local) # every provider of the key found, in the order found + self.considered: set[bytes] = set() # providers found by an earlier search: made candidates then + self.lookup = node._lookup(self.key) + self.wanted = KEEPER_CANDIDATES * self.alpha + # Records stored here count as a source only if an established publisher stored one, so identities + # nobody knows can't end the search by storing records for the key here first + self.local_source = any(self.established(p) for p in self.local) + self.candidates: list[bytes] = [] # not taken by a batch yet, in the order to ask them + self.vouched: dict[bytes, set[bytes]] = {} # candidate -> the sources that listed it + self.floor: int | None = None # the highest sequence known for the name when the search ended + self.asked = 0 + self.cut_short = False # the last search was stopped by should_stop, not ended by the lookup + self._search_started = 0.0 + self._asked_since_search = False # a batch was taken from the last search's candidates + self._lock = threading.Lock() + self._results: list[SignedRecord] = [] + self._answered: set[bytes] = set() # keepers that answered with a verifying record + # Unknown keepers given a path request: at most alpha, apart from the lookup's + self._unknown: set[bytes] = set() + self._used: set[bytes] = set() # sources that listed a keeper already picked + + def established(self, node_id: bytes) -> bool: + """Senior and healthy in the table before the search, and not failed or gone silent since.""" + return self.feature._standing(node_id, self.before) == 0 + + def add(self, provider: bytes): + """Collect a provider the lookup reports.""" + if provider != self.node.node_id and provider not in self.found and not self.banned(provider): + self.found[provider] = None + + def _fresh(self) -> list[bytes]: + """Providers found that no earlier search made candidates, in the order found.""" + return [provider for provider in self.found if provider not in self.considered] + + def enough(self) -> bool: + """Enough new candidates, listed by more than one established source.""" + if len(self._fresh()) < self.wanted: + return False + sources = sum(1 for lister in self.lookup.listing_nodes() if self.established(lister)) + return sources + (1 if self.local_source else 0) >= 2 + + def should_stop(self) -> bool: + """True if the search should stop: enough new candidates, or some once lookup_timeout / SEARCH_SHARE has + passed since it started. Candidates are asked only once it stops, so reply order doesn't choose them.""" + if not self._fresh(): + return False + budget = self.node.config.lookup_timeout / SEARCH_SHARE + if self.enough() or time.monotonic() - self._search_started >= budget: + self.cut_short = True + return self.cut_short + + def search(self) -> list[bytes]: + """Find providers of the name's key and rank those no earlier search found; those candidates, in the order + to ask them. + + Who vouches for each candidate: established nodes that listed it, and this node if it holds a record of + it; candidates nobody established vouches for count as one source between them. + """ + self._search_started = time.monotonic() + self.cut_short = self._asked_since_search = False + self.lookup.find_providers(self.add, should_stop=self.should_stop) + fresh = self._fresh() + self.considered.update(fresh) + ranked = self.feature.rank_keepers(fresh, self.before) + if not ranked: + return [] + here = set(self.local) + self.vouched.update({provider: ({lister for lister in self.lookup.listers(provider) + if self.established(lister)} + | ({self.node.node_id} if provider in here else set())) or {_UNVOUCHED} + for provider in ranked}) + self.candidates.extend(ranked) + self.floor = self.feature._known_sequence(self.name) + return ranked + + def resume(self) -> bool: + """Search again, once the candidates have run out without settling the name; True if it found new ones. + + Only a search that was stopped early, not ended by the lookup (complete, bounded or out of time), is + resumed, and only after some of its candidates were asked: each resume is followed by a batch, so the + batch bound also bounds the resumes, all within the lookup's deadline. + """ + if not (self.cut_short and self._asked_since_search) or self.lookup.remaining() <= 0: + return False + return bool(self.search()) + + def may_ask(self, provider: bytes) -> bool: + """True if provider's identity is known, or the unknown-keeper budget has room.""" + if self.node._is_known(provider) or provider in self._unknown: + return True + if len(self._unknown) >= self.alpha: + return False + self._unknown.add(provider) + return True + + def next_batch(self) -> list[bytes]: + """Up to alpha candidates: first those listed by a source not used yet, then in rank order. Each candidate + considered leaves candidates, asked or not (an unknown one over the budget).""" + batch = [] + for fresh in (True, False): + for provider in list(self.candidates): + if len(batch) >= self.alpha: + break + if fresh and not self.vouched[provider] - self._used: + continue + self.candidates.remove(provider) + if self.may_ask(provider): + batch.append(provider) + self._used.update(self.vouched[provider]) + self.asked += len(batch) + if batch: + self._asked_since_search = True + return batch + + def ask(self, provider: bytes): + """Ask one keeper; keep its record if it verifies. Runs on a thread of its own.""" + try: + record = parse_response(self.name, self.feature.client.name_record(provider, self.name)) + except Exception as e: # noqa: BLE001 - a failed request is just no answer + _log(f"/name to {provider.hex()} failed: {e!r}") + return + if record is None: + return + with self._lock: + self._results.append(record) + self._answered.add(provider) + + def settled(self) -> bool: + """True if the answers so far settle the name: one reaches the highest sequence known (any, if none is + known), and enough keepers answered to confirm it.""" + with self._lock: + best = max((record.sequence for record in self._results), default=None) + responders = list(self._answered) + current = best is not None and (self.floor is None or best >= self.floor) + if self.floor is None: + # Nothing to check answers against: replayed old records from identities that answered the + # lookup must not settle it, so only established keepers confirm each other + confirmed = sum(1 for provider in responders if self.established(provider)) >= 2 + else: + # A second keeper's answer is only worth its airtime on a slow node if there's nothing to check + # the first against + confirmed = len(responders) >= 2 or self.node.slow + return current and confirmed + + def records(self) -> list[SignedRecord]: + """The verifying records answered so far.""" + with self._lock: + return list(self._results) + + # The feature @@ -424,6 +610,7 @@ class KeepersFeature(Feature): """Keeps and serves signed name records, and asks keepers when resolving. Thread-safe.""" name = NAME + optional = ("contacts",) # a name contact's nick names a name (parse_name_arg) def __init__(self, node): """Load kept names, hook into names and the publisher; no network.""" @@ -435,7 +622,6 @@ def __init__(self, node): self._pending: dict[bytes, None] = {} # names to refresh in the background, in order self._wake = threading.Event() self._stop = threading.Event() - self._thread: threading.Thread | None = None self._listeners: list = [] # The last name the periodic retry queued: the next retry starts after it, so past MAX_PENDING # names without a record every one is asked in turn @@ -460,13 +646,14 @@ def __init__(self, node): def start(self): """Start the background refresher.""" - self._thread = threading.Thread(target=self._loop, name="rnfs-keepers", daemon=True) - self._thread.start() + self._spawn(self._loop, "rnfs-keepers") - def stop(self): - """Stop the refresher and save.""" + def stop(self, deadline: float | None = None): + """Stop the refresher, wait for it until deadline, then save: so the last save comes after its last + change, and the node doesn't close the state database under it.""" self._stop.set() self._wake.set() + self._join(deadline) self.save() def save(self): @@ -708,135 +895,48 @@ def ask_keepers(self, name: bytes) -> list[SignedRecord]: (a node the lookup meets doesn't gain rank from answering it; one that fails meanwhile loses rank), and an *established* node is one senior and healthy then. Keepers recorded here are candidates, never a reason to skip the search, which runs until KEEPER_CANDIDATES × alpha are known and two established - sources listed some (this node counts as one if an established publisher stored a record here), or - the lookup is complete. Candidates are ranked by standing then at random (rank_keepers), and each batch - takes first, in that order, those listed by a source no earlier pick was listed by (this node, an - established listing node, or, as one source, nobody established), then the rest in rank order. Unknown - keepers (a path request each) get a budget of alpha of their own, apart from the lookup's. Another batch - is asked, up to keeper_rounds() in all, while no answer reaches the highest sequence already known for - the name (from an announce, a resolve or a kept record), or fewer than two keepers answered (when no - sequence is known, two established keepers; on a slow node a second batch is asked for this only when - nothing is known to compare against). Each verifying record is returned; the caller keeps the highest - sequence. + sources listed some (this node counts as one if an established publisher stored a record here), until + some are known and lookup_timeout / SEARCH_SHARE has passed, or until the lookup ends. Candidates are + asked only once the search stops, never as they arrive, and are ranked by standing then at random + (rank_keepers), and each batch takes first, in that order, those listed by a source no earlier pick was + listed by (this node, an established listing node, or, as one source, nobody established), then the rest + in rank order. Unknown keepers (a path request each) get a budget of alpha of their own, apart from the + lookup's. Another batch is asked, up to keeper_rounds() in all, while no answer reaches the highest + sequence already known for the name (from an announce, a resolve or a kept record), or fewer than two + keepers answered (when no sequence is known, two established keepers; on a slow node a second batch is + asked for this only when nothing is known to compare against). Each batch is waited for the lookup's + time left or name_wait, whichever is longer, so a search that used all of lookup_timeout still leaves + keepers time to answer. If the candidates run out first, a search stopped early is resumed for more + (_KeeperQuery.resume). Each verifying record is returned; the caller keeps the highest sequence. """ - node = self.node - key = name_key(name) - alpha = node.config.effective_alpha(node.slow) - banned = node.routing.is_banned - # Taken before the lookup, whose answers add nodes to the routing table - before = node.routing.standings(node.config.provider_seniority) - local = [p for p in node.records.providers(key, exclude=banned) if p != node.node_id] - found = dict.fromkeys(local) - lookup = node._lookup(key) - wanted = KEEPER_CANDIDATES * alpha - - def established(node_id: bytes) -> bool: - """Senior and healthy in the table before the search, and not failed or gone silent since.""" - return self._standing(node_id, before) == 0 - - # Records stored here count as a source only if an established publisher stored one, so identities - # nobody knows can't end the search by storing records for the key here first - local_source = any(established(p) for p in local) - - def add(provider: bytes): - """Collect a provider the lookup reports.""" - if provider != node.node_id and provider not in found and not banned(provider): - found[provider] = None - - def enough() -> bool: - """Enough candidates, listed by more than one established source.""" - if len(found) < wanted: - return False - sources = sum(1 for lister in lookup.listing_nodes() if established(lister)) - return sources + (1 if local_source else 0) >= 2 - - lookup.find_providers(add, should_stop=enough) - candidates = self.rank_keepers(list(found), before) - if not candidates: + query = _KeeperQuery(self, name) + if not query.search(): _log(f"no keepers found for name {name.hex()}") return [] - # Who vouches for each candidate: established nodes that listed it, and this node if it holds a record of - # it; candidates nobody established vouches for count as one source between them - here = set(local) - vouched = {provider: ({lister for lister in lookup.listers(provider) if established(lister)} - | ({node.node_id} if provider in here else set())) or {_UNVOUCHED} - for provider in candidates} - floor = self._known_sequence(name) - results: list[SignedRecord] = [] - answered: set[bytes] = set() - lock = threading.Lock() - unknown: set[bytes] = set() # unknown keepers given a path request: at most alpha, apart from the lookup's - - def may_ask(provider: bytes) -> bool: - """True if provider's identity is known, or the unknown-keeper budget has room.""" - if node._is_known(provider) or provider in unknown: - return True - if len(unknown) >= alpha: - return False - unknown.add(provider) - return True - - used: set[bytes] = set() # sources that listed a keeper already picked - - def next_batch() -> list[bytes]: - """Up to alpha candidates: first those listed by a source not used yet, then in rank order.""" - batch = [] - for fresh in (True, False): - for provider in list(candidates): - if len(batch) >= alpha: - break - if fresh and not vouched[provider] - used: - continue - candidates.remove(provider) - if may_ask(provider): - batch.append(provider) - used.update(vouched[provider]) - return batch - - def ask(provider: bytes): - """Ask one keeper; keep its record if it verifies.""" - try: - record = parse_response(name, self.client.name_record(provider, name)) - except Exception as e: # noqa: BLE001 - a failed request is just no answer - _log(f"/name to {provider.hex()} failed: {e!r}") - return - if record is None: - return - with lock: - results.append(record) - answered.add(provider) - - deadline = time.monotonic() + max(1.0, lookup.remaining()) - asked = 0 - for _ in range(self.keeper_rounds()): - batch = next_batch() + wait = name_wait(self.node.config) + batches = 0 + while batches < self.keeper_rounds(): + batch = query.next_batch() if not batch: + if query.resume(): + continue break - asked += len(batch) - threads = [threading.Thread(target=ask, args=(provider,), name="rnfs-keeper-query", daemon=True) + batches += 1 + # /name has a budget of its own: a link to a keeper may take many seconds to set up, however long + # the search took + deadline = time.monotonic() + max(query.lookup.remaining(), wait) + threads = [threading.Thread(target=query.ask, args=(provider,), name="rnfs-keeper-query", daemon=True) for provider in batch] for thread in threads: thread.start() for thread in threads: thread.join(max(0.0, deadline - time.monotonic())) - with lock: - best = max((record.sequence for record in results), default=None) - responders = list(answered) - current = best is not None and (floor is None or best >= floor) - if floor is None: - # Nothing to check answers against: replayed old records from identities that answered the - # lookup must not settle it, so only established keepers confirm each other - confirmed = sum(1 for provider in responders if established(provider)) >= 2 - else: - # A second keeper's answer is only worth its airtime on a slow node if there's nothing to check - # the first against - confirmed = len(responders) >= 2 or node.slow - if (current and confirmed) or time.monotonic() >= deadline: + if query.settled(): break - with lock: - _log(f"asked {asked} of {asked + len(candidates)} keepers of name {name.hex()}: " - f"{len(results)} verifying records") - return list(results) + records = query.records() + _log(f"asked {query.asked} of {query.asked + len(query.candidates)} keepers of name {name.hex()}: " + f"{len(records)} verifying records") + return records def keeper_rounds(self) -> int: """How many batches of alpha keepers one query asks at most: KEEPER_ROUNDS, or 2 on a slow node, where @@ -917,7 +1017,7 @@ def name_subcommands(commands): def parse_name_arg(node, text) -> bytes: """A name from 32 hex characters, or a name contact's nick (rnfs.features.contacts). ValueError.""" from rnfs.names import parse_name - contacts = node.features.get("contacts") if hasattr(node, "features") else None + contacts = find(node, "contacts") if contacts is not None: return contacts.resolve_id(text, "name") return parse_name(text) diff --git a/rnfs/features/lxmf.py b/rnfs/features/lxmf.py index fa67f03..65b5207 100644 --- a/rnfs/features/lxmf.py +++ b/rnfs/features/lxmf.py @@ -15,7 +15,8 @@ - Receiving: messages with rnfs links go to a bounded inbox. Links from senders in lxmf_auto_fetch_from are fetched and pinned with source `lxmf:`, within lxmf_auto_fetch_quota per sender; removing the message from the inbox releases those pins, and a message holding pins is never dropped to make room for - others. lxmf_accept_from restricts senders. + others' (each such sender keeps at most 1/n of the inbox, so none fills it). lxmf_accept_from restricts + senders. - The inbox and outbox are kept in the state database (rnfs.statedb, the lxmf_messages table) and saved in batches, each save writing the records changed since the last; outbox messages still unfinished at a start are marked failed (LXMF doesn't persist its outbound queue). @@ -25,7 +26,8 @@ printed. A message is only accepted once LXMF has validated its signature against the sender's identity; one from a sender whose identity isn't known yet is held while a path request fetches it. -LXMF is optional: without the package (or with lxmf_enabled = false) the feature stays off and says so once. +The LXMF package is a dependency (pyproject.toml), but the node runs without it: if it can't be imported (or with +lxmf_enabled = false), the feature stays off and says so once. Nothing here imports RNS or LXMF at module level (CLI commands mustn't load them); LxmfNetwork, made in start(), holds every RNS and LXMF object, so the rest works on offline test nodes with a stand-in network. """ @@ -37,12 +39,12 @@ import sys import threading import time -from collections import OrderedDict, deque +from collections import Counter, OrderedDict, deque from contextlib import contextmanager from dataclasses import dataclass from urllib.parse import quote, unquote_to_bytes -from rnfs import features, fsutil +from rnfs import features, fsutil, ids from rnfs.features import _common from rnfs.features._common import LOG_ERROR, LOG_INFO, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING from rnfs.constants import ID_LENGTH, MAX_DEPTH @@ -100,10 +102,9 @@ PROPAGATION_NODES_LIMIT = 64 # propagation nodes remembered from announces PROPAGATION_NODE_AGE = 7 * 24 * 3600.0 # forget a propagation node not heard this long IDENTITY_WAIT = 15.0 # seconds `send` waits for an unknown recipient's identity (capped by path_timeout) -TICK = 1.0 # seconds between polls of outbound message states +TICK = 1.0 # seconds between polls of outbound message states (while any is followed, held or downloading) SYNC_TIMEOUT = 900.0 # a propagation node download still unfinished after this is abandoned -_HEX32 = re.compile(r"[0-9a-fA-F]{32}") _SEGMENT = re.compile(r"(?:[A-Za-z0-9\-._~!$&'()*+,;=:@]|%[0-9A-Fa-f]{2})+") _QUERY_VALUE = re.compile(r"(?:[A-Za-z0-9\-._~!$'()*+,;=:@/]|%[0-9A-Fa-f]{2})*") # "&" separates parameters # An rnfs: link in free text: not glued to a preceding word, ending at whitespace or a character URIs never hold @@ -133,9 +134,7 @@ def _check_name(name: str): def parse_address(text) -> bytes: """An LXMF address (destination hash) from 32 hex characters, with or without .""" - if not isinstance(text, str) or not _HEX32.fullmatch(text.strip().strip("<>")): - raise ValueError(f"invalid LXMF address {text!r}: must be 32 hex characters") - return bytes.fromhex(text.strip().strip("<>")) + return ids.parse_hex16(text, "LXMF address", brackets=True) def validate_settings(config): @@ -252,7 +251,7 @@ def parse_uri(text) -> Link: segments.pop() # "/dir/" names the same directory as "/dir" cid = name = None if segments[0] == NAME_SEGMENT: - if len(segments) < 2 or not _HEX32.fullmatch(segments[1]): + if len(segments) < 2 or not ids.HEX16.fullmatch(segments[1]): raise ValueError("rnfs:name/ must be followed by a name: 32 hex characters") name = bytes.fromhex(segments[1]) segments = segments[2:] @@ -282,7 +281,7 @@ def parse_uri(text) -> Link: if key == "from": if from_node is not None: raise ValueError("from is given twice") - if not _HEX32.fullmatch(value): + if not ids.HEX16.fullmatch(value): raise ValueError("from must be a node ID: 32 hex characters") from_node = bytes.fromhex(value) return Link(cid, name, tuple(names), None, from_node) @@ -485,13 +484,16 @@ class MessageLog: per record in its box: a save writes the records added, changed or removed since the last. Thread-safe. At most limit records and max_bytes of JSON are kept. To make room, the oldest record protect() is false - for is dropped; a protected one only when every other is protected too. add() may also name records that - are dropped only for each other's (reserved). valid() checks records loaded (malformed ones are dropped; - clean says whether the box was saved whole, or imported from a whole file, and every record was read). + for is dropped; a protected one only when every other is protected too, and then only one with the new + record's "from". add() may also name records that are dropped only for each other's (reserved), and limit + the new record's "from" to a share. holds(record) is what a protected record holds (the inbox: pinned CIDs + and queued links): one whose holdings another record of its sender also holds is dropped first among a + sender's protected records. valid() checks records loaded (malformed ones are dropped; clean says whether + the box was saved whole, or imported from a whole file, and every record was read). """ def __init__(self, state_dir: str, box: str, limit: int, max_bytes: int | None = None, protect=None, - valid=None): + valid=None, holds=None): """Load box's records from the state database in state_dir (malformed ones are dropped). Close it when done.""" from rnfs import statedb @@ -499,6 +501,7 @@ def __init__(self, state_dir: str, box: str, limit: int, max_bytes: int | None = self.limit = limit self.max_bytes = max_bytes self.protect = protect or (lambda entry: False) + self.holds = holds or (lambda entry: ()) self._lock = threading.Lock() self._save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last self._dirty: set[int] = set() # ids of records added, changed or removed since the last save @@ -540,12 +543,15 @@ def _load(self, valid): if bad: statedb.set_meta(tx, self.box, "clean", 0) - def add(self, entry: dict, per_from: int | None = None, reserved=None) -> tuple[int | None, list[dict]]: + def add(self, entry: dict, per_from: int | None = None, reserved=None, + share: tuple[int, int] | None = None) -> tuple[int | None, list[dict]]: """Append entry with a new id; returns the id (None if entry itself was dropped: every other record is - protected and it isn't) and the entries dropped to stay within the bounds. per_from: also keep at most - that many unprotected records with entry's "from". reserved(record): records it is true for are dropped - to make room only for an entry it is true for too, and only once no other unprotected record is left - (trusted senders' records, so new identities can't push them out).""" + protected and it isn't, or none of the others it may push out has its "from") and the entries dropped to + stay within the bounds. per_from: also keep at most that many unprotected records with entry's "from". + share (records, bytes): keep at most that many records and bytes with entry's "from", protected ones + included (entry is kept even if it alone exceeds them). reserved(record): records it is true for are + dropped to make room only for an entry it is true for too, and only once no other unprotected record is + left (trusted senders' records, so new identities can't push them out).""" reserved = reserved or (lambda record: False) with self._lock: if self._next_id > MAX_ID: # only a hand-edited next id gets here: refused, so saves keep working @@ -559,6 +565,8 @@ def add(self, entry: dict, per_from: int | None = None, reserved=None) -> tuple[ if per_from is not None: same = [e for e in self._entries if e.get("from") == entry.get("from") and not self.protect(e)] dropped += same[:max(0, len(same) - per_from)] + if share is not None: + dropped += self._over_share(entry, share, dropped) entry_reserved = reserved(entry) while self._over(dropped): gone = {id(e) for e in dropped} @@ -568,9 +576,11 @@ def add(self, entry: dict, per_from: int | None = None, reserved=None) -> tuple[ victim = others[0] if victim is None: # Every other record holds pins (or is reserved, and entry isn't): refuse entry rather than - # drop them, unless it holds pins too (then the oldest goes; only a trusted sender gets here) - victim = entry if not self.protect(entry) or len(self._entries) - len(dropped) == 1 else \ - next(e for e in self._entries if id(e) not in gone and e is not entry) + # drop them, unless it holds pins too. Then one of its own sender's records goes, never another + # sender's, so no sender pushes out another's pins (FEAT-1); with none, entry is refused + own = [e for e in self._entries + if e.get("from") == entry.get("from") and id(e) not in gone and e is not entry] + victim = self._own_victim(own, own + [entry]) if self.protect(entry) and own else entry dropped.append(victim) if victim is entry: break @@ -581,6 +591,31 @@ def add(self, entry: dict, per_from: int | None = None, reserved=None) -> tuple[ self._dirty.update(e["id"] for e in [entry, *dropped]) # a refused entry still took an id return (None if id(entry) in gone else entry["id"]), [e for e in dropped if e is not entry] + def _over_share(self, entry: dict, share: tuple[int, int], dropped: list[dict]) -> list[dict]: + """The records of entry's "from" to drop (besides dropped) so that its records, entry and protected ones + included, are within share (records, bytes), chosen by _own_victim; never entry (lock held).""" + max_records, max_bytes = share + gone = {id(e) for e in dropped} + own = [e for e in self._entries if e.get("from") == entry.get("from") and id(e) not in gone] + size = sum(self._sizes[e["id"]] for e in own) + victims = [] + while len(own) > 1 and (len(own) > max_records or size > max_bytes): + victim = self._own_victim([e for e in own if e is not entry], own) + victims.append(victim) + own = [e for e in own if e is not victim] + size -= self._sizes[victim["id"]] + return victims + + def _own_victim(self, candidates: list[dict], own: list[dict]) -> dict: + """Which of candidates (records of one sender, oldest first) to drop: the oldest unprotected; else the + oldest whose holdings another of own (that sender's records kept so far) also holds, which drops no pin + (a sender re-linking one CID over and over has many such records); else the oldest (lock held).""" + victim = next((e for e in candidates if not self.protect(e)), None) + if victim is not None: + return victim + held = Counter(item for e in own for item in set(self.holds(e))) + return next((e for e in candidates if all(held[item] > 1 for item in set(self.holds(e)))), candidates[0]) + def _over(self, dropped: list[dict]) -> bool: """Whether the records left after dropping these exceed the limit or max_bytes (lock held).""" if len(self._entries) - len(dropped) > self.limit: @@ -680,7 +715,7 @@ def _valid_inbox_entry(entry: dict) -> bool: """Whether a loaded inbox record has the shape received() gives it.""" auto = entry.get("auto") return (isinstance(entry.get("hash"), str) and isinstance(entry.get("from"), str) - and _HEX32.fullmatch(entry["from"]) is not None and _valid_links(entry.get("links")) + and ids.HEX16.fullmatch(entry["from"]) is not None and _valid_links(entry.get("links")) and (auto is None or (isinstance(auto, list) and len(auto) == len(entry["links"]) and all(isinstance(a, dict) and _str_or_none(a.get("state")) for a in auto))) and all(_str_or_none(entry.get(key)) for key in ("display_name", "title", "text", "method"))) @@ -692,11 +727,28 @@ def _valid_outbox_entry(entry: dict) -> bool: and _valid_links(entry.get("links")) and all(_str_or_none(entry.get(key)) for key in ("text", "error"))) +def _unlisted() -> dict: + """The auto-fetch result of a link whose sender was taken off lxmf_auto_fetch_from before it was fetched.""" + return {"state": "refused", "error": "the sender is no longer in lxmf_auto_fetch_from"} + + def _holds_pins(entry: dict) -> bool: """Whether an inbox record has auto-fetched pins, or is about to (queued): a flood mustn't release them.""" return any(isinstance(a, dict) and a.get("state") in ("queued", "pinned") for a in entry.get("auto") or []) +def _held(entry: dict) -> set: + """What an inbox record holds for its sender (MessageLog's holds): the CIDs pinned for it, and each link + still queued (its own: nobody else's record holds what it will pin).""" + held = set() + for index, result in enumerate(entry.get("auto") or []): + if isinstance(result, dict) and result.get("state") == "pinned": + held.add(str(result.get("cid"))) + elif isinstance(result, dict) and result.get("state") == "queued": + held.add((entry.get("id"), index)) + return held + + @dataclass class Incoming: """A received LXMF message whose signature LXMF has validated, as plain data.""" @@ -729,7 +781,7 @@ def _fit_entry(entry: dict): """Bound an inbox record to MAX_ENTRY_BYTES as JSON: drop links from the last (keeping one), then cut text. Room is left for the auto-fetch results it may get.""" - reserve = 128 * len(entry["links"]) # {"state": "pinned", "cid": ...} per link + reserve = 128 * len(entry["links"]) # {"state": "pinned", "cid": ..., "sent_cid": true} per link while len(entry["links"]) > 1 and _json_bytes(entry) + reserve > MAX_ENTRY_BYTES: entry["links"].pop() reserve -= 128 @@ -805,6 +857,8 @@ class LxmfFeature(features.Feature): """Sends and receives rnfs links over LXMF; see the module docstring.""" name = NAME + # contacts: send to a nick, trust LXMF contacts, name senders in the inbox; jobs: links fetched in the background + optional = ("contacts", "jobs") def __init__(self, node): """Load the inbox and outbox; LXMF itself is started in start().""" @@ -815,8 +869,23 @@ def __init__(self, node): self.off_reason = "LXMF has not been started" self.accept_from = {parse_address(a).hex() for a in config.lxmf_accept_from} self.auto_fetch_from = {parse_address(a).hex() for a in config.lxmf_auto_fetch_from} + # Every attribute is set before any method runs: the start-up steps below can reach the inbox, the + # auto-fetch queue and the lock (_release_unrecorded -> _record_kept -> _release) + self._lock = threading.RLock() # inbox removals and auto-fetch pins, so a pin is never left behind + self._resend: dict[str, tuple] = {} # direct message hash -> (address, content, fields, nearest) + self._told_no_pn = False # logged once how to set a propagation node + self._auto_queue = _RoundRobin() # (entry id, link index) by sender + self.clock = time.monotonic # for the rate limit on senders not trusted (tests set their own) + self._untrusted_tokens = float(UNTRUSTED_BURST) + self._untrusted_at: float | None = None + self._untrusted_dry = False # logged (notice) that the bucket ran dry; verbose until a token is taken + self._untrusted_dropped = 0 # untrusted senders' messages the rate limits dropped since the start + self._sender_tokens: OrderedDict[bytes, tuple[float, float]] = OrderedDict() # sender -> (tokens, when) + self._auto_wanted = threading.Condition() + self._stop = threading.Event() + self._wake = threading.Event() self.inbox = MessageLog(config.state_dir, INBOX, INBOX_LIMIT, MAX_INBOX_BYTES, protect=_holds_pins, - valid=_valid_inbox_entry) + valid=_valid_inbox_entry, holds=_held) try: self.outbox = MessageLog(config.state_dir, OUTBOX, OUTBOX_LIMIT, valid=_valid_outbox_entry) except BaseException: @@ -828,23 +897,11 @@ def __init__(self, node): # LXMF doesn't persist its outbound queue self.outbox.update(entry["id"], state="failed", error="the daemon stopped before it was delivered") - self._lock = threading.RLock() # inbox removals and auto-fetch pins, so a pin is never left behind self._release_unrecorded() + self._refuse_unlisted() except BaseException: self.close() raise - self._resend: dict[str, tuple] = {} # direct message hash -> (address, content, fields, nearest) - self._told_no_pn = False # logged once how to set a propagation node - self._auto_queue = _RoundRobin() # (entry id, link index) by sender - self.clock = time.monotonic # for the rate limit on senders not trusted (tests set their own) - self._untrusted_tokens = float(UNTRUSTED_BURST) - self._untrusted_at: float | None = None - self._untrusted_dry = False # logged (notice) that the bucket ran dry; verbose until a token is taken - self._sender_tokens: OrderedDict[bytes, tuple[float, float]] = OrderedDict() # sender -> (tokens, when) - self._auto_wanted = threading.Condition() - self._stop = threading.Event() - self._wake = threading.Event() - self._threads: list[threading.Thread] = [] def _release_unrecorded(self): """Release lxmf: pins no inbox message records as pinned. Pins are saved at once and the inbox every @@ -876,11 +933,22 @@ def _release_unrecorded(self): except OSError as e: # e.g. a read-only state directory: tried again at the next start _log(f"could not release {cid}: {e}", LOG_ERROR) + def _refuse_unlisted(self): + """Mark "refused" the auto-fetches still queued for senders no longer in lxmf_auto_fetch_from: the user took + them off the list since, and a queued link holds its message in the inbox like a pin. Runs whether or not + LXMF then starts.""" + for entry in self.inbox.entries(): + if entry.get("from") in self.auto_fetch_from: + continue + for index, result in enumerate(entry.get("auto") or []): + if isinstance(result, dict) and result.get("state") == "queued": + self._set_auto(entry["id"], index, _unlisted()) + def _record_kept(self, sender: str, cids: list[Cid]): """Keep sender's lxmf: pins of cids, which an inbox that didn't load whole doesn't record: add an inbox message from sender recording them as pinned, and save the inbox.""" listed = ", ".join(str(cid) for cid in cids) - if _HEX32.fullmatch(sender) is None: # not a source this feature makes: kept, never recorded + if ids.HEX16.fullmatch(sender) is None: # not a source this feature makes: kept, never recorded _log(f"keeping {listed}, pinned for {sender!r}, which isn't an LXMF address", LOG_WARNING) return _log(f"the saved inbox is missing, damaged or incomplete: keeping {listed}, pinned for a message from " @@ -893,8 +961,11 @@ def _record_kept(self, sender: str, cids: list[Cid]): "release them.", "auto": [{"state": "pinned", "cid": str(cid)} for cid in cids]} with self._lock: - _, dropped = self.inbox.add(entry) + entry_id, dropped = self.inbox.add(entry) self._release(dropped) + if entry_id is None: # never another sender's pins for these: kept unrecorded, released at a later start + _log(f"no room for that message: the inbox is full of other senders' messages holding pins; {listed} " + f"stay pinned until a start finds the inbox whole", LOG_WARNING) try: self.inbox.save() except OSError as e: # the pins are kept; recorded again at the next start @@ -927,23 +998,20 @@ def start(self): self.attach(network) self._requeue_auto_fetches() for name, target in (("rnfs-lxmf", self._run), ("rnfs-lxmf-fetch", self._run_auto_fetch)): - thread = threading.Thread(target=target, name=name, daemon=True) - thread.start() - self._threads.append(thread) + self._spawn(target, name) _log(f"LXMF address {network.address.hex()} ({network.display_name})", LOG_NOTICE) def attach(self, network): """Use network (LxmfNetwork, or a stand-in in tests) for sending and receiving.""" self.network = network - def stop(self): - """Stop the loops, LXMF's router, and save.""" + def stop(self, deadline: float | None = None): + """Stop the loops (waiting for them until deadline, as Feature.stop), LXMF's router, and save.""" self._stop.set() self._wake.set() with self._auto_wanted: self._auto_wanted.notify_all() - for thread in self._threads: - thread.join(2.0) + self._join(deadline) if self.network is not None: try: self.network.stop() @@ -962,9 +1030,19 @@ def close(self): self.outbox.close() def status(self) -> dict: - """The LXMF address (None when off) and the number of inbox messages.""" + """The LXMF address (None when off), the number of inbox messages, and how many messages from untrusted + senders the rate limits dropped since the start: many identities can keep the strangers' bucket empty, so + a rising count is a reason to add the senders the user expects as contacts.""" return {"lxmf_address": self.network.address.hex() if self.network is not None else None, - "lxmf_inbox": len(self.inbox)} + "lxmf_inbox": len(self.inbox), "untrusted_dropped": self._untrusted_dropped} + + def sync(self, nearest: bool = False) -> dict: + """Start downloading waiting messages from the propagation node (nearest: without lxmf_propagation_node + set, the nearest heard): {"propagation_node", "started"}. ValueError if LXMF is off.""" + network = self._require_network() + node = network.propagation_node(nearest) + return {"propagation_node": node.hex() if node is not None else None, + "started": bool(node is not None and network.sync(nearest))} def _require_network(self): """The network, or ValueError explaining why LXMF is off.""" @@ -974,13 +1052,17 @@ def _require_network(self): def _run(self): """Announce (at start, after startup_announce_delay, then every announce_interval), download from the - propagation node every lxmf_sync_interval, and poll outbound messages.""" + propagation node every lxmf_sync_interval, and poll outbound messages every TICK. While the network has + nothing to poll (network.idle()), sleep until the next announce or download is due instead: the network + wakes the loop (wake()) when it queues or holds a message or starts a download, so an idle node doesn't + wake each second.""" network = self.network now = time.monotonic() announce_at = [now, now + self.config.startup_announce_delay] sync_at = now + max(self.config.startup_announce_delay, 10.0) if self.config.lxmf_sync_interval else None while not self._stop.is_set(): now = time.monotonic() + timeout = TICK try: if announce_at and now >= announce_at[0]: announce_at.pop(0) @@ -995,11 +1077,19 @@ def _run(self): network.sync() for message_hash, state, progress in network.poll(): self.outbound_changed(message_hash, state, progress) + # A message queued or held after this check sets _wake, so the wait below returns at once + if network.idle(): + due = announce_at[0] if sync_at is None else min(announce_at[0], sync_at) + timeout = max(0.0, due - time.monotonic()) except Exception as e: # noqa: BLE001 - keep the loop alive _log(f"loop failed: {e!r}", LOG_ERROR) - self._wake.wait(TICK) + self._wake.wait(timeout) self._wake.clear() + def wake(self): + """Wake the loop to poll the network now: it queued or held a message, or started a download.""" + self._wake.set() + def _no_propagation_node(self, consequence: str): """Log, once, that no propagation node is set and how to set one.""" if self._told_no_pn: @@ -1024,26 +1114,20 @@ def address(self) -> dict: "nearest_propagation_node": nearest.hex() if nearest is not None else None} def _recipient(self, to: str) -> bytes: - """An LXMF address, or the LXMF address of a contact nick.""" + """An LXMF address, or the LXMF address of a contact nick (contacts.resolve_id).""" if not isinstance(to, str) or not to.strip(): raise ValueError("a recipient is required: an LXMF address or a contact") - if _HEX32.fullmatch(to.strip().strip("<>")): + if ids.is_hex16(to, brackets=True): return parse_address(to) - # A contact nick, through rnfs/features/contacts.py when that feature is loaded - contacts = getattr(self.node, "features", {}).get("contacts") - if contacts is not None and hasattr(contacts, "lookup"): - found = contacts.lookup(to, "lxmf") - if isinstance(found, (bytes, bytearray)) and len(found) == ID_LENGTH: - return bytes(found) - if contacts.lookup(to) is not None: - raise ValueError(f"contact {to} is not an LXMF address") - raise ValueError(f"no LXMF contact {to!r}, and it isn't an LXMF address (32 hex characters)") - raise ValueError(f"{to!r} is not an LXMF address (32 hex characters), and contacts aren't available") + contacts = self.dependency("contacts") + if contacts is None: + raise ValueError(f"{to!r} is not an LXMF address (32 hex characters), and contacts aren't available") + return contacts.resolve_id(to, "lxmf") def _contact_target(self, target): """target with a leading name contact's nick replaced by name/, so the link follows the name.""" - contacts = getattr(self.node, "features", {}).get("contacts") - if not isinstance(target, str) or contacts is None or not hasattr(contacts, "lookup"): + contacts = self.dependency("contacts") + if not isinstance(target, str) or contacts is None: return target first, slash, rest = target.partition("/") if first == NAME_SEGMENT or target[:len(URI_SCHEME)].lower() == URI_SCHEME: @@ -1172,13 +1256,18 @@ def received(self, message: Incoming) -> int | None: _log(f"message {message_hash} from {sender} not kept: too many new messages from this sender, or from " f"senders not in contacts, lxmf_accept_from or lxmf_auto_fetch_from", LOG_VERBOSE if self._untrusted_dry else LOG_NOTICE) - self._untrusted_dry = True + with self._lock: + self._untrusted_dry = True + self._untrusted_dropped += 1 return None with self._lock: # A sender not trusted with auto-fetch gets INBOX_PER_SENDER records, so one can't push out others', - # and records of trusted senders are dropped only for each other's: identities are free + # and records of trusted senders are dropped only for each other's: identities are free. An + # auto-fetch sender's records hold pins, which others' records never push out, so each such sender + # gets 1/n of the inbox, protected records included: one can't fill it and push out another's (FEAT-1) entry_id, dropped = self.inbox.add(entry, per_from=None if auto else INBOX_PER_SENDER, - reserved=lambda record: self._trusted_hex(record.get("from"))) + reserved=lambda record: self._trusted_hex(record.get("from")), + share=self._auto_share() if auto else None) self._release(dropped) if entry_id is None: _log(f"message {message_hash} from {sender} not kept: the inbox is full of messages from trusted " @@ -1191,6 +1280,12 @@ def received(self, message: Incoming) -> int | None: self._queue_auto_fetch(entry_id, index, sender) return entry_id + def _auto_share(self) -> tuple[int, int]: + """The inbox records and bytes one auto-fetch sender may hold: 1/n of the inbox's limits for n senders in + lxmf_auto_fetch_from, and always room for one record.""" + senders = max(1, len(self.auto_fetch_from)) + return max(1, self.inbox.limit // senders), max(MAX_ENTRY_BYTES, self.inbox.max_bytes // senders) + def accepts(self, source: bytes) -> bool: """Whether messages from LXMF address source are kept: lxmf_accept_from (empty: everyone), and lxmf_auto_fetch_from.""" @@ -1207,8 +1302,8 @@ def _trusted_hex(self, sender) -> bool: """trusted() for an address as 32 hex characters (an inbox record's "from").""" if sender in self.accept_from or sender in self.auto_fetch_from: return True - contacts = getattr(self.node, "features", {}).get("contacts") - if contacts is None or not hasattr(contacts, "has") or not isinstance(sender, str): + contacts = self.dependency("contacts") + if contacts is None or not isinstance(sender, str): return False try: return bool(contacts.has(bytes.fromhex(sender), "lxmf")) @@ -1240,8 +1335,20 @@ def _untrusted_token(self, source: bytes) -> bool: return True def inbox_list(self) -> list[dict]: - """Inbox messages, oldest first.""" - return self.inbox.entries() + """Inbox messages, oldest first, each with "trusted" (whether its sender is trusted now) and "contact" (the + nick of the LXMF contact with its address, or None) added: the display name is the sender's own word, so + `rnfs inbox` shows these instead.""" + contacts = self.dependency("contacts") + entries = self.inbox.entries() + for entry in entries: + entry["trusted"] = self._trusted_hex(entry.get("from")) + entry["contact"] = None + if contacts is not None and entry["trusted"]: + try: + entry["contact"] = contacts.nick_of(bytes.fromhex(entry["from"]), "lxmf") + except (TypeError, ValueError): + pass + return entries def outbox_list(self) -> list[dict]: """Sent messages, oldest first.""" @@ -1278,27 +1385,37 @@ def _release(self, removed: list[dict]): if released: self.node.publisher.refresh() - def _resolve(self, link: Link, persist: bool = True) -> tuple[Cid, list[str]]: - """The root CID and path a link names, resolving a name (falling back to the CID it was sent with). + def _resolve(self, link: Link, persist: bool = True, + allow_sent_cid: bool = False) -> tuple[Cid, list[str], bool]: + """The root CID and path a link names, resolving a name, and whether it fell back to the CID the name link + was sent with: only if allow_sent_cid and the name can't be resolved. NotFound if it can't be resolved + and that isn't allowed (naming --use-sent-cid if the link has a sent CID). + + The sender chose the sent CID: nothing ties it to the name, so a stranger could pose as a well-known name + with it. A name that resolves is always used. persist False (auto-fetch): the name's record goes only into the bounded table, so names arriving unasked don't grow the held records; its sequence still goes into the bounded, saved floor, so a later auto-fetch of the name never resolves to an older record.""" if link.name is None: - return link.cid, list(link.path) + return link.cid, list(link.path), False from rnfs.errors import NotFound try: if persist: - return Cid.parse(self.node.name_resolve(link.name.hex())["cid"]), list(link.path) + return Cid.parse(self.node.name_resolve(link.name.hex())["cid"]), list(link.path), False record = self.node.names.resolve(link.name, persist=False) if record is None: raise NotFound(f"name {link.name.hex()} could not be resolved") - return Cid.parse(record["cid"]), list(link.path) + return Cid.parse(record["cid"]), list(link.path), False except NotFound: if link.cid is None: raise + if not allow_sent_cid: + raise NotFound(f"name {link.name.hex()} could not be resolved; the sender says it pointed at " + f"{link.cid}, but nothing ties that CID to the name: to get it anyway, use " + f"--use-sent-cid") from None _log(f"name {link.name.hex()} could not be resolved; using the CID it was sent with", LOG_NOTICE) - return link.cid, list(link.path) + return link.cid, list(link.path), True def _hints(self, link: Link) -> list[bytes | None]: """Where to fetch from: the link's provider hint first (unless it is this node), then the DHT.""" @@ -1307,11 +1424,14 @@ def _hints(self, link: Link) -> list[bytes | None]: return [None] def inbox_get(self, entry_id: int, index: int | None = None, output: str | None = None, - directory: str | None = None) -> list[dict]: + directory: str | None = None, use_sent_cid: bool = False) -> list[dict]: """Fetch the links of inbox message entry_id (only link index, 1-based, if given) and write each out: to output, or into directory under its last path name (the root CID if that name is hidden, non-printable or not a plain file name here: the sender chose it). Tries the sender's node first, then - the DHT. Once one link has started, a later one's failure is a result with "error", not raised.""" + the DHT. Once one link has started, a later one's failure is a result with "error", not raised. + + A name link whose name can't be resolved is NotFound before anything is written or started, unless + use_sent_cid: then the CID it was sent with is fetched, and its result says so in "warnings".""" from rnfs.errors import NotFound entry = self.inbox.get(entry_id) if entry is None: @@ -1328,29 +1448,33 @@ def inbox_get(self, entry_id: int, index: int | None = None, output: str | None # Every output is checked before anything starts, so a clash can't stop the fetches halfway planned = [] for number, link in enumerate(links, 1): - root, names = self._resolve(link) + root, names, fell_back = self._resolve(link, allow_sent_cid=use_sent_cid) + warnings = [f"name {link.name.hex()} could not be resolved: fetching {root}, the CID the sender says it " + f"pointed at (nothing ties that CID to the name)"] if fell_back else [] target = "/".join([str(root), *names]) if output is not None: path = output else: - from rnfs import fsutil path = os.path.join(directory, fsutil.safe_default_name(names[-1] if names else None, str(root))) if os.path.lexists(path): raise ValueError(f"{fsutil.printable(path)} already exists; remove it or choose another with -o") - same = next((i for i, (_, other, _) in enumerate(planned) if other == path), None) + same = next((i for i, (_, other, _, _) in enumerate(planned) if other == path), None) if same is not None: raise ValueError(f"links {same + 1} and {number} would both be written to {fsutil.printable(path)}: " f"get them one at a time with --link and -o") - planned.append((link, path, target)) + planned.append((link, path, target, warnings)) results = [] - for link, path, target in planned: + for link, path, target, warnings in planned: try: - results.append(self._get_link(link, path, target)) - except Exception as e: # noqa: BLE001 - raised as is when nothing has started + result = self._get_link(link, path, target) + except Exception as e: # raised as is when nothing has started if not results: raise # Something has started already: report it, and this failure with it - results.append({"target": target, "output": path, "error": str(e) or type(e).__name__}) + result = {"target": target, "output": path, "error": str(e) or type(e).__name__} + if warnings: + result["warnings"] = [*result.get("warnings", []), *warnings] + results.append(result) return results def _get_link(self, link: Link, path: str, target: str) -> dict: @@ -1379,8 +1503,8 @@ def _start_job(self, target: str, output: str, link: Link): Like a fetch here: the link's provider hint first, then (fallback_dht) ordinary DHT discovery. """ - jobs = getattr(self.node, "features", {}).get("jobs") - if jobs is None or not hasattr(jobs, "submit"): + jobs = self.dependency("jobs") + if jobs is None: return None from_node = self._hints(link)[0] if from_node is None: @@ -1408,6 +1532,8 @@ def _requeue_auto_fetches(self): """Queue again, oldest first, the auto-fetches still "queued" when the daemon stopped (also one that was running then): the queue itself isn't saved, and a queued record is never dropped for others.""" for entry in self.inbox.entries(): + if entry.get("from") not in self.auto_fetch_from: + continue # refused at construction (_refuse_unlisted) for index, result in enumerate(entry.get("auto") or []): if isinstance(result, dict) and result.get("state") == "queued": self._queue_auto_fetch(entry["id"], index, entry["from"]) @@ -1446,25 +1572,32 @@ def _set_auto(self, entry_id: int, index: int, result: dict): def auto_fetch(self, entry_id: int, index: int): """Fetch one link of an inbox message and pin it with source lxmf:, within the sender's quota. - The fetch stops (within fetcher.POLL) when the node stops, leaving the link queued for the next start, - or when the message is removed.""" + The sender is trusted, so a name link whose name can't be resolved falls back to the CID it was sent with; + the result then records "sent_cid": true, which the inbox listing shows. The fetch stops (within + fetcher.POLL) when the node stops, leaving the link queued for the next start, or when the message is + removed. A sender taken off lxmf_auto_fetch_from while the link waited is refused.""" from rnfs import fetcher from rnfs.content.storage import LXMF_PREFIX from rnfs.errors import Denied, InvalidDag, RnfsError, StorageFull, TooLarge entry = self.inbox.get(entry_id) if entry is None: return + if entry.get("from") not in self.auto_fetch_from: # taken off the list while the link waited + _log(f"auto-fetch of inbox #{entry_id} link {index + 1}: refused: the sender is no longer in " + f"lxmf_auto_fetch_from", LOG_NOTICE) + self._set_auto(entry_id, index, _unlisted()) + return observer = fetcher.Observer() observer.cancelled = lambda: self._stop.is_set() or self.inbox.get(entry_id) is None try: link = Link.from_json(entry["links"][index]) source = LXMF_PREFIX + entry["from"] - root, names = self._resolve(link, persist=False) + root, names, sent_cid = self._resolve(link, persist=False, allow_sent_cid=True) hints = self._hints(link) for attempt, from_node in enumerate(hints): try: with fetcher.observing(observer): - cid = self._fetch_and_pin(entry_id, index, root, names, from_node, source) + cid = self._fetch_and_pin(entry_id, index, root, names, from_node, source, sent_cid) break except (TooLarge, InvalidDag, StorageFull, fetcher.Cancelled): raise @@ -1490,27 +1623,32 @@ def auto_fetch(self, entry_id: int, index: int): f"{fsutil.printable(result['error'])}", LOG_NOTICE) self._set_auto(entry_id, index, result) - def _pin_recorded(self, entry_id: int, index: int, cid: Cid, source: str) -> bool: - """Pin cid with source and record it in the inbox entry, in one locked step so a removal can't come - between them and leave a pin no entry releases; False (nothing pinned) if the entry is gone.""" + def _pin_recorded(self, entry_id: int, index: int, cid: Cid, source: str, sent_cid: bool = False) -> bool: + """Pin cid with source and record it in the inbox entry (with "sent_cid": true if it is a name link's sent + CID, the name unresolved), in one locked step so a removal can't come between them and leave a pin no + entry releases; False (nothing pinned) if the entry is gone.""" + result = {"state": "pinned", "cid": str(cid)} + if sent_cid: + result["sent_cid"] = True with self._lock: if self.inbox.get(entry_id) is None: return False self.node.storage.pin(cid, source) - self._set_auto(entry_id, index, {"state": "pinned", "cid": str(cid)}) + self._set_auto(entry_id, index, result) return True def _fetch_and_pin(self, entry_id: int, index: int, root: Cid, names: list[str], from_node: bytes | None, - source: str) -> Cid | None: + source: str, sent_cid: bool = False) -> Cid | None: """Fetch / completely and pin the target with source, unless it would exceed the - sender's quota (TooLarge). Returns the CID pinned, or None if the message went away meanwhile.""" + sender's quota (TooLarge). Returns the CID pinned, or None if the message went away meanwhile. sent_cid: + root is a name link's sent CID (recorded with the pin).""" from rnfs.errors import TooLarge quota = self.config.lxmf_auto_fetch_quota storage = self.node.storage with self.node.fetcher.open(root, names, from_node, pinned=True) as fetch: expect = fetch.resolve() - if source in storage.pins().get(expect.cid, set()): - return expect.cid if self._pin_recorded(entry_id, index, expect.cid, source) else None + if storage.has_pin(expect.cid, source): + return expect.cid if self._pin_recorded(entry_id, index, expect.cid, source, sent_cid) else None content = fetch.size_of(expect) used = storage.source_usage(source) if used + content > quota: @@ -1519,7 +1657,7 @@ def _fetch_and_pin(self, entry_id: int, index: int, root: Cid, names: list[str], # A DAG of empty files has size 0 yet can hold many directory nodes: count its blocks too fetch.max_dag_bytes = quota - used fetch.fetch_dag(expect) - if not self._pin_recorded(entry_id, index, expect.cid, source): + if not self._pin_recorded(entry_id, index, expect.cid, source, sent_cid): return None self.node.publisher.refresh() return expect.cid @@ -1553,6 +1691,9 @@ def cli_commands(cls, commands, command): get.add_argument("n", type=int, metavar="N", help="message number, from `rnfs inbox`") get.add_argument("--link", type=int, metavar="K", help="only the K-th link of the message") get.add_argument("-o", "--output", help="where to write (one link only; default: its name, here)") + get.add_argument("--use-sent-cid", action="store_true", + help="if a name link's name can't be resolved, get the CID the sender says it pointed at " + "(nothing ties that CID to the name: only for a sender you trust)") rm = command(inbox_commands, "rm", _run_inbox_rm, "remove inbox messages, releasing what they auto-fetched") rm.add_argument("n", type=int, nargs="?", metavar="N", help="message number") @@ -1608,13 +1749,14 @@ def _int(value, name: str, required: bool = True) -> int | None: def _api_inbox_get(handler): - """POST /api/lxmf/inbox/get {id, link?, output? | directory?}: {"results": [...]}.""" - from rnfs.api import parse_path + """POST /api/lxmf/inbox/get {id, link?, output? | directory?, use_sent_cid?}: {"results": [...]}. A name link + whose name can't be resolved is 404 unless use_sent_cid (then its result carries a warning).""" + from rnfs.api import parse_bool, parse_path body = handler.json_body() output = parse_path(body["output"]) if body.get("output") is not None else None directory = parse_path(body["directory"]) if body.get("directory") is not None else None results = _feature(handler).inbox_get(_int(body.get("id"), "id"), _int(body.get("link"), "link", False), - output, directory) + output, directory, parse_bool(body.get("use_sent_cid", False))) handler.send_json(200, {"results": results}) @@ -1640,10 +1782,7 @@ def _api_sync(handler): from rnfs.api import parse_bool body = handler.json_body() nearest = parse_bool(body.get("nearest", False)) - network = _feature(handler)._require_network() - node = network.propagation_node(nearest) - handler.send_json(200, {"propagation_node": node.hex() if node is not None else None, - "started": bool(node is not None and network.sync(nearest))}) + handler.send_json(200, _feature(handler).sync(nearest)) # CLI commands @@ -1657,7 +1796,8 @@ def _when(timestamp) -> str: def _print_links(entry: dict, printable): - """Print a message's links, one per line, with size and auto-fetch state.""" + """Print a message's links, one per line, with size, the CID a name link was sent as (the sender's word: only + the name's own record ties a CID to it) and auto-fetch state.""" auto = entry.get("auto") if isinstance(entry.get("auto"), list) else [] for i, link in enumerate(entry.get("links") or []): if not isinstance(link, dict): @@ -1665,8 +1805,12 @@ def _print_links(entry: dict, printable): notes = [] if isinstance(link.get("size"), int): notes.append(human_size(link["size"])) + if link.get("name") is not None and link.get("cid") is not None: + notes.append(f"sent as {link['cid']}") if i < len(auto) and isinstance(auto[i], dict): state = auto[i].get("state") + if auto[i].get("sent_cid") is True: + state = f"{state} (sent CID; name unresolved)" notes.append(f"auto-fetch {state}" + (f": {auto[i]['error']}" if auto[i].get("error") else "")) print(f" [{i + 1}] {printable(link.get('uri'))}" + (f" ({printable(', '.join(notes))})" if notes else "")) @@ -1685,18 +1829,35 @@ def _run_send(args): print(cli.printable(result["hash"])) +def _trust_label(entry: dict) -> str: + """How `rnfs inbox` marks a message's sender: "contact ", "trusted" or "untrusted"; "?" when the daemon + (older than 0.1.6) doesn't say.""" + if isinstance(entry.get("contact"), str): + return f"contact {fsutil.printable_label(entry['contact'])}" + if isinstance(entry.get("trusted"), bool): + return "trusted" if entry["trusted"] else "untrusted" + return "?" + + +def _quoted_label(name) -> str: + """A display name, the sender's own word, in double quotes and escaped as a label (fsutil.printable_label, + which escapes backslashes, so an escaped quote can't be faked either).""" + return '"' + fsutil.printable_label(name).replace('"', '\\"') + '"' + + def _run_inbox(args): - """List inbox messages: number, time, sender, links and text.""" + """List inbox messages: number, time, the sender's address and whether it is a contact or trusted, method, + then the display name it announced (last and quoted: it is the sender's own word, and could otherwise pose as + another address), links and text.""" from rnfs import cli messages = cli.client(args).get("/api/lxmf/inbox")["messages"] if not messages: print("no messages") for entry in messages: - sender = entry.get("from") or "?" name = entry.get("display_name") - who = f"{cli.printable(name)} <{cli.printable(sender)}>" if name else f"<{cli.printable(sender)}>" - print(f"#{cli.printable(entry.get('id'))} {_when(entry.get('received'))} {who} " - f"{cli.printable(entry.get('method') or '')}") + print(f"#{cli.printable(entry.get('id'))} {_when(entry.get('received'))} " + f"<{cli.printable(entry.get('from') or '?')}> {_trust_label(entry)} " + f"{cli.printable(entry.get('method') or '?')}" + (f" {_quoted_label(name)}" if name else "")) _print_links(entry, cli.printable) _print_text(_strip_links(entry), cli.printable) @@ -1715,7 +1876,7 @@ def _run_inbox_get(args): if output is not None and os.path.lexists(output): cli.fail(f"{output} already exists; remove it or choose another with -o") result = cli.client(args).post("/api/lxmf/inbox/get", id=args.n, link=args.link, output=output, - directory=os.getcwd()) + directory=os.getcwd(), use_sent_cid=args.use_sent_cid) failed = False for item in result["results"]: cli.print_warnings(item) @@ -1958,7 +2119,7 @@ def sync(self, nearest: bool = False) -> bool: if node is None: return False router = self.router - busy = router.PR_PATH_REQUESTED <= router.propagation_transfer_state < router.PR_COMPLETE + busy = self._downloading() if busy and time.monotonic() - self._sync_started < SYNC_TIMEOUT: return False if busy: @@ -1966,8 +2127,22 @@ def sync(self, nearest: bool = False) -> bool: router.acknowledge_sync_completion(reset_state=True) self._sync_started = time.monotonic() router.request_messages_from_propagation_node(self.identity, SYNC_MAX_MESSAGES) + self.feature.wake() # poll() bounds the download's link from now on, until it ends return True + def idle(self) -> bool: + """True while poll() has nothing to do: no outbound message followed, none held, and no propagation node + download running. The feature's loop then sleeps until its next announce or download.""" + with self._lock: + if self._outbound or self._held: + return False + return not self._downloading() + + def _downloading(self) -> bool: + """True while a propagation node download runs: from its path request until it completes or fails.""" + router = self.router + return router.PR_PATH_REQUESTED <= router.propagation_transfer_state < router.PR_COMPLETE + # Sending def wait_identity(self, address: bytes, timeout: float) -> bool: @@ -2001,6 +2176,7 @@ def send(self, address: bytes, content: str, fields: dict, propagated: bool, nea self.router.handle_outbound(message) with self._lock: self._outbound[bytes(message.hash)] = [message, None, None] + self.feature.wake() # follow its state from now on return bytes(message.hash) def poll(self) -> list[tuple[bytes, str, float | None]]: @@ -2127,6 +2303,7 @@ def _hold(self, source: bytes, packed: bytes, method) -> bool: _log(f"dropped a held message from unknown sender {victim[1][ID_LENGTH:2 * ID_LENGTH].hex()} for one " f"from trusted sender {source.hex()}", LOG_VERBOSE) self._held.append((time.monotonic() + HOLD_TIME, packed, method)) + self.feature.wake() # poll() retries it once its sender's identity is known return True def _request_path(self, destination: bytes) -> bool: diff --git a/rnfs/features/peers.py b/rnfs/features/peers.py index d629597..d07775c 100644 --- a/rnfs/features/peers.py +++ b/rnfs/features/peers.py @@ -2,8 +2,8 @@ Counters live in a mesh.Traffic shared by the node's PeerClient (requests and verified bytes received per provider) and RequestServer (/blocks bytes sent, per requester where it identified, and airtime refusals); -this feature reads them. It also registers an rnfs.dht announce handler of its own, so the DHT mode and the -last-heard time of client-mode nodes (which never enter the routing table) are known too. +this feature reads them. The node's rnfs.dht announce handler counts every announce in them too (Traffic.heard), +so the DHT mode and the last-heard time of client-mode nodes (which never enter the routing table) are known. Nothing is persisted: counts are since the daemon started. Memory is bounded by mesh.MAX_PEERS. """ @@ -28,23 +28,9 @@ def __init__(self, node): super().__init__(node) traffic = getattr(node.client, "traffic", None) self.traffic: mesh.Traffic = traffic if isinstance(traffic, mesh.Traffic) else mesh.Traffic() - self._handler = None - - def start(self): - """Listen to rnfs.dht announces for modes and last-heard times.""" - if getattr(self.node, "reticulum", None) is None: - return - from rnfs.network.announces import DhtAnnounceHandler - self._handler = DhtAnnounceHandler(lambda node_id: self.traffic.heard(node_id, "server"), - lambda node_id: self.traffic.heard(node_id, "client"), - self.node.routing.is_banned, self.node.node_id) - self._handler.register() - - def stop(self): - """Stop listening to announces.""" - if self._handler is not None: - self._handler.deregister() - self._handler = None + from rnfs.log import ThrottledLog # here, not at module level: it loads RNS, and the CLI imports features + # A listing that can't read something is a bug or an RNS change, met per peer: a line a minute + self._failure_log = ThrottledLog() def status(self) -> dict: """Traffic totals and airtime refusals for `rnfs status`.""" @@ -68,23 +54,27 @@ def peers(self) -> list[dict]: node = self.node routing = node.routing paths = getattr(node, "paths", None) - path_info = paths.info if paths is not None else mesh.no_path_info counters = self.traffic.peers() try: linked = set(node.client.open_links()) - except Exception: + except Exception as e: # noqa: BLE001 - a listing shows what it can; logged + self._failed(f"couldn't list the open links: {e!r}") linked = set() ids = [n for n in dict.fromkeys([*routing.nodes(), *linked, *reversed(list(counters))]) if n != node.node_id] + ids = ids[:mesh.MAX_PEERS] + # All at once: attached to a shared instance, RnsPaths reads the interfaces from one RPC, not one each + known = paths.infos(ids) if paths is not None else {} peers = [] - for node_id in ids[:mesh.MAX_PEERS]: + for node_id in ids: counts = counters.get(node_id) or {} seniority = routing.seniority(node_id) - path = path_info(node_id) + path = known.get(node_id, mesh.NO_PATH) rate = None if node_id in linked: try: rate = node.client.rate_estimate(node_id) - except Exception: + except Exception as e: # noqa: BLE001 - a listing shows what it can; logged + self._failed(f"couldn't read the rate estimate of {node_id.hex()}: {e!r}") rate = None peers.append({ "node_id": node_id.hex(), @@ -118,11 +108,12 @@ def _recalled_mode(self, node_id: bytes) -> str | None: import RNS from rnfs.constants import MODE_SERVER from rnfs.network.announces import parse_mode - app_data = RNS.Identity.recall_app_data(node_id) + app_data = RNS.Identity.recall_app_data(node_id, _no_use=True) # a listing doesn't mark it used if app_data is None: return None return "server" if parse_mode(app_data) == MODE_SERVER else "client" - except Exception: + except Exception as e: # noqa: BLE001 - RNS internals: unknown if they fail; logged + self._failed(f"couldn't read {node_id.hex()}'s last announce from RNS: {e!r}") return None def _recalled_heard(self, node_id: bytes) -> float | None: @@ -133,9 +124,15 @@ def _recalled_heard(self, node_id: bytes) -> float | None: import RNS entry = RNS.Identity.known_destinations.get(node_id) return float(entry[0]) if entry else None - except Exception: + except Exception as e: # noqa: BLE001 - RNS internals: unknown if they fail; logged + self._failed(f"couldn't read when RNS last heard {node_id.hex()}: {e!r}") return None + def _failed(self, message: str): + """Log, at most once a minute, that a listing couldn't read something. Called inside the except, so the + debug log has the traceback.""" + self._failure_log(f"peers: {message}", _common.LOG_WARNING) + # API @classmethod @@ -201,7 +198,7 @@ def human_age(seconds: float | None) -> str: def format_peers(peers: list[dict], now: float) -> list[str]: - """The `rnfs peers` table, one line per node.""" + """The `rnfs peers` table, one line per node, escaped (the daemon's fields mustn't drive the terminal).""" lines = [f"{'NODE':<32} {'MODE':<7} {'TABLE':>6} {'HOPS':>4} {'INTERFACE':<20} {'RATE':>10} {'HEARD':>6} " f"{'SENT':>9} {'RECV':>9} {'FAIL':>5}"] for peer in peers: @@ -210,15 +207,17 @@ def format_peers(peers: list[dict], now: float) -> list[str]: rate = human_rate(peer["rate"]) if peer["link"] else "-" interface = (peer.get("interface") or "-")[:20] flags = (" banned" if peer["banned"] else "") + (f" bad:{peer['bad']}" if peer.get("bad") else "") - lines.append(f"{printable(peer['node_id']):<32} {printable(peer['mode'] or '?'):<7} {table:>6} " - f"{peer['hops'] if peer['hops'] is not None else '?':>4} {printable(interface):<20} " - f"{rate:>10} {heard:>6} {human_size(peer['sent']):>9} {human_size(peer['received']):>9} " - f"{peer['failed']:>5}{flags}") + # Each text field escaped before it is padded, so the columns line up; the whole line again for the rest + lines.append(printable( + f"{printable(peer['node_id']):<32} {printable(peer['mode'] or '?'):<7} {table:>6} " + f"{peer['hops'] if peer['hops'] is not None else '?':>4} {printable(interface):<20} " + f"{rate:>10} {heard:>6} {human_size(peer['sent']):>9} {human_size(peer['received']):>9} " + f"{peer['failed']:>5}{flags}")) return lines def format_stats(stats: dict) -> list[str]: - """The `rnfs stats` report.""" + """The `rnfs stats` report, escaped (the daemon's fields mustn't drive the terminal).""" totals = stats["totals"] lines = [f"since: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(stats['since']))}", f"sent: {human_size(totals['sent'])} in {totals['responses']} /blocks responses", @@ -230,9 +229,9 @@ def format_stats(stats: dict) -> list[str]: lines.append(f"upload budget: {human_size(stats.get('upload_budget_left'))} left of " f"{human_size(stats['upload_budget'])} per hour") for name, counters in sorted(stats["interfaces"].items()): - lines.append(f"interface {printable(name)}: sent {human_size(counters['sent'])}, " + lines.append(f"interface {name}: sent {human_size(counters['sent'])}, " f"received {human_size(counters['received'])}") - return lines + return [printable(line) for line in lines] def run_peers(args): diff --git a/rnfs/fetcher.py b/rnfs/fetcher.py index 59688b2..6101f35 100644 --- a/rnfs/fetcher.py +++ b/rnfs/fetcher.py @@ -24,6 +24,13 @@ longer matches its CID is local damage: it is deleted and fetched again). A fetch that gets no verified block and no response progress for lookup_timeout + stall_timeout fails, so a hung one ends. +End-game duplication (NET-4; section 7): once nothing is left to ask for, an idle provider is also asked for a +batch another has had in flight for longer than deadline_slack, so a response dripping in (never stalling, its +deadline hours away at unmeasured_bitrate) doesn't hold the fetch up. Each request is duplicated at most once, +and only while fewer than max_parallel_providers requests are in flight (never on a slow node). The first +verified copy wins: the request still sending what has arrived has its link closed (unless another fetch has a +request on it: then it is left to finish), and its outcome isn't counted against its provider. + Upload pacing (section 3): a responder starts a slow link's next /blocks response no sooner than its size at the upload cap (by default half the rate estimate) after the previous one started, and refuses a request it would have to hold back for longer than a few seconds. So after a response on a slow link, a provider isn't @@ -43,7 +50,7 @@ so without these a file declaring 1 byte, or a directory of one entry, could hold any number of distinct nodes. What the builder writes never comes near them (design.md, "DAG byte bound"). While one applies, requests ask for no more than it leaves, counting a DAG node as MAX_BLOCK_SIZE until it arrives (a directory or index node -as nothing under the size-derived bound: see _reserve), so a fetch downloads at most about one block beyond it +as nothing under the size-derived bound: see _DagBudget.reserve), so a fetch downloads at most about one block beyond it before failing with TooLarge. max_storage never refuses a fetch: one that isn't being pinned evicts other cached fetches to make room for each response, and goes over max_storage if nothing is evictable (the node evicts again once it ends). What refuses a fetch, pinned or not, with StorageFull is the disk: its remaining @@ -53,8 +60,23 @@ blocks against a quota). Requests are spread over up to max_parallel_providers providers (1 on slow nodes), one /blocks request in -flight per provider. Request threads only call the client; all fetch state is changed by the thread that -runs the fetch, from the events they queue. +flight per provider. Request threads only call the client and note when a response last made progress +(Fetch._heard_at, which the no-progress deadline reads with the fetch thread's own _progress_at); all other +fetch state is changed by the thread that runs the fetch, from the events they queue (ResultEvent, and the +lookup thread's ProviderEvent and LookupDone). + +fetch_dag and fetch_listing walk the DAG inside one _download: a block's children still to fetch are queued as +soon as it is met, stored or downloaded, rather than once its whole tree level has arrived, so no provider waits +for a level's last response. A parent is always stored before its children are asked for, so whatever an +interrupted fetch stored hangs together from the target, and the next fetch resumes from it. + +read_range fetches WINDOW chunks at a time, and asks for the next window's before handing the reader this one's +bytes, so the providers don't sit idle between windows; a block that a request brings after the call that asked +for it has returned is stored all the same. A fetch that ends by an error, is cancelled, or ends with requests it +no longer needs (a reader that stopped early, which has had at most one window fetched that it didn't read) +closes the links of its requests still in flight, unless another fetch has a request on the same (shared) link: +then its request is left to finish, rather than fail the other's. It never cancels a response's Resource: a +responder would take that for a rejection and cool the requester down (section 3). Within `with observing(observer):`, every fetch the thread opens reports its total size and each block it meets to the Observer, and raises Cancelled once observer.cancelled() is true (background jobs use this). @@ -65,12 +87,11 @@ from collections import OrderedDict, deque from contextlib import contextmanager from itertools import islice -from typing import Callable, Iterator +from typing import Callable, Iterator, NamedTuple from rnfs import statedb from rnfs.config import Config -from rnfs.constants import (CHUNK_SIZE, KIB, MAX_BLOCK_SIZE, MAX_CHILDREN, MAX_DEPTH, MAX_REQUEST_SIZE, - MAX_RESPONSE_SIZE, RESPONSE_OVERHEAD) +from rnfs.constants import CHUNK_SIZE, KIB, MAX_BLOCK_SIZE, MAX_CHILDREN, MAX_DEPTH, MAX_REQUEST_SIZE from rnfs.content import walker from rnfs.content.cid import Cid from rnfs.content.storage import Storage @@ -197,27 +218,233 @@ def observing(observer: Observer): _observers.current = previous -class _RangeBudget: - """The file node bytes one read_range call may still meet (range_bound), and the nodes it has met.""" +class _DagBudget: + """The byte limits on the distinct blocks a fetch meets, and what it has counted against them (the module + docstring's "Limits"). Pure bookkeeping: no storage, network or clock, and only the fetch thread uses it. + + A fetch's own budget (Fetch.budget) bounds the distinct blocks of its DAG, each counted once: by max_dag_bytes + (a quota its caller sets) and, once fetch_dag or fetch_listing runs, by dag_bound of the size their targets + declare plus what the directory and index nodes counted bring (node_credit). A read_range call passes + Fetch.ensure a budget of its own (for_range), which also bounds the file nodes that call meets by range_bound, + each counted once per call, and counts every block against the fetch's own budget as well. - __slots__ = ("limit", "length", "used", "counted") + The room a budget leaves is a tuple (bytes under the size-derived bound, under max_dag_bytes, under the + read_range call's bound), each None if that limit doesn't apply: room() computes it, reserve() takes from it + what a block not yet received may add, and fits() checks it. + """ - def __init__(self, limit: int, length: int): - """A budget of limit bytes for reading length bytes.""" - self.limit = limit - self.length = length - self.used = 0 + __slots__ = ("root", "whole", "max_dag_bytes", "declared", "credit", "used", "counted", "range_limit", + "range_length", "range_used", "range_counted") + + def __init__(self, root: Cid): + """A fetch's own budget for root's DAG, with no limit until one is set (max_dag_bytes) or declared.""" + self.root = root + self.whole = self # the fetch's own budget, which a read_range call's counts against too + self.max_dag_bytes: int | None = None + self.declared: int | None = None # content bytes fetch_dag's targets declare (fetch_listing's count as 0) + self.credit = 0 # what the counted directory and index nodes bring to the bound (node_credit) + self.used = 0 # bytes of the distinct blocks counted self.counted: set[Cid] = set() + self.range_limit: int | None = None # a read_range call's: range_bound for its range_length bytes + self.range_length = 0 + self.range_used = 0 + self.range_counted: set[Cid] = set() + + def for_range(self, limit: int, length: int) -> "_DagBudget": + """The budget of a read_range call reading length bytes: limit bytes of the file nodes it meets, and this + fetch's own budget.""" + budget = _DagBudget(self.root) + budget.whole = self.whole + budget.range_limit = limit + budget.range_length = length + return budget + + def declare(self, size: int): + """Bound the DAG by what size more content bytes can hold (fetch_dag; fetch_listing declares 0).""" + whole = self.whole + whole.declared = (whole.declared or 0) + size + + def limit(self) -> int | None: + """Bytes the distinct blocks of the DAG may hold: max_dag_bytes, and once declared, dag_bound of the size + declared plus what the directory and index nodes counted bring (node_credit), whichever is lower. None + if neither applies.""" + whole = self.whole + limits = [] if whole.max_dag_bytes is None else [whole.max_dag_bytes] + if whole.declared is not None: + limits.append(dag_bound(whole.declared) + whole.credit) + return min(limits, default=None) - def count(self, root: Cid, cid: Cid, length: int): - """Count a DAG node of length bytes once. Raises TooLarge once over the limit.""" - if cid in self.counted: + def count(self, cid: Cid, length: int, children: list[Expect] = ()): + """Count a DAG node of length bytes against a read_range call's bound, once per call, then any block of the + DAG against the fetch's limits, once, with what it brings to them (node_credit). Raises TooLarge once over + either.""" + if self.range_limit is not None and not cid.is_raw and cid not in self.range_counted: + self.range_counted.add(cid) + self.range_used += length + if self.range_used > self.range_limit: + raise TooLarge(f"reading {plural(self.range_length, 'byte')} of {self.root} would take more than " + f"{self.range_limit} bytes of file nodes, more than such a range can need: refusing it") + whole = self.whole + if (whole.max_dag_bytes is None and whole.declared is None) or cid in whole.counted: return - self.counted.add(cid) - self.used += length - if self.used > self.limit: - raise TooLarge(f"reading {plural(self.length, 'byte')} of {root} would take more than {self.limit} " - "bytes of file nodes, more than such a range can need: refusing it") + whole.counted.add(cid) + whole.used += length + whole.credit += node_credit(length, children) + limit = self.limit() + if whole.used > limit: + if limit == whole.max_dag_bytes: + raise TooLarge(f"{self.root} holds more than {limit} bytes of blocks") + content = f"its {plural(whole.declared, 'byte')} of content and " if whole.declared else "" + raise TooLarge(f"{self.root} holds more than {limit} bytes of blocks, more than {content}its " + "directory entries can take: refusing it") + + def room(self, in_flight, wanted: dict[Cid, list[Expect]]) -> tuple | None: + """What the limits leave for blocks not yet asked for, after the most the CIDs in_flight (asked for, not + yet received) can bring; None if no limit applies.""" + whole = self.whole + if whole.declared is None and whole.max_dag_bytes is None and self.range_limit is None: + return None + bound = None if whole.declared is None else dag_bound(whole.declared) + whole.credit - whole.used + quota = None if whole.max_dag_bytes is None else whole.max_dag_bytes - whole.used + budget = None if self.range_limit is None else self.range_limit - self.range_used + room = (bound, quota, budget) + for cid in in_flight: + room = self.reserve(cid, wanted, room) + return room + + def reserve(self, cid: Cid, wanted: dict[Cid, list[Expect]], room: tuple) -> tuple: + """room after the most cid can add to the bytes counted: a raw chunk's declared size, otherwise + MAX_BLOCK_SIZE, and nothing for a raw chunk under a read_range call's bound. + + Under the size-derived bound a directory or index node takes nothing: a directory node brings more + than it takes, and an index node takes at most INDEX_NODE_COST, so a request of MAX_BATCH of them + overshoots by at most 2 KiB. Holding them to INDEX_NODE_COST each would ask for a directory's + subdirectories a few at a time. + """ + bound, quota, budget = room + if budget is not None and not cid.is_raw and cid not in self.range_counted: + budget -= MAX_BLOCK_SIZE + if cid in self.whole.counted: + return bound, quota, budget + expects = wanted.get(cid) + worst = MAX_BLOCK_SIZE + if expects and cid.is_raw and expects[0].size is not None: + worst = expects[0].size + if bound is not None and not (expects and not cid.is_raw + and not any(walker.FILE_TYPE in expect.types for expect in expects)): + bound -= worst + return bound, None if quota is None else quota - worst, budget + + @staticmethod + def fits(room: tuple) -> bool: + """True if room leaves nothing below zero.""" + return all(left is None or left >= 0 for left in room) + + +class ResultEvent(NamedTuple): + """A /blocks request's result, queued by its request thread.""" + + node_id: bytes + cids: list[Cid] # what it asked for + result: peer_client.BlocksResult + sent_at: float | None # when it went out (None: unknown) + rate: float | None # the link's rate estimate it was sent with (the responder's upload cap derives from it) + + +class ProviderEvent(NamedTuple): + """A provider the lookup found, queued by the lookup thread as its reply arrives.""" + + node_id: bytes + + +class LookupDone(NamedTuple): + """A lookup run ended, queued by the lookup thread.""" + + found: int # providers it reported + + +class _Pending: + """The CIDs a _download has still to ask for, in the walk's order, each at most once. + + A deque with a set alongside: requeueing a result asks whether each of its CIDs is already pending, and + scanning a deque of a large file's chunks for every CID held up the fetch thread (16k pending, 20 ms a result). + """ + + __slots__ = ("_order", "_members") + + def __init__(self, cids): + """Pending cids, in order (distinct, as a dict's keys are).""" + self._order: deque[Cid] = deque(cids) + self._members: set[Cid] = set(self._order) + + def __len__(self) -> int: + """How many CIDs are pending.""" + return len(self._order) + + def __contains__(self, cid: Cid) -> bool: + """True if cid is pending.""" + return cid in self._members + + def __iter__(self) -> Iterator[Cid]: + """The pending CIDs, front first.""" + return iter(self._order) + + def __getitem__(self, index: int) -> Cid: + """The CID at index (0 is the next one asked for).""" + return self._order[index] + + def popleft(self) -> Cid: + """Take the next CID to ask for.""" + cid = self._order.popleft() + self._members.discard(cid) + return cid + + def push_front(self, cids: list[Cid]): + """Put cids (none of them pending) back at the front, keeping their order.""" + self._order.extendleft(reversed(cids)) + self._members.update(cids) + + def extend(self, cids: list[Cid]): + """Add cids (none of them pending) at the back, keeping their order.""" + self._order.extend(cids) + self._members.update(cids) + + +class _Walk: + """The walk of fetch_dag or fetch_listing below a target, met block by block (A4): what each block met adds to + fetch, and the directory entries met so far. Pure bookkeeping, used by the fetch thread only.""" + + __slots__ = ("seen", "listing", "max_entries", "entries", "index_queued") + + def __init__(self, target: Expect, listing: bool, max_entries: int | None): + """A walk from target: of its whole DAG, or for a listing (listing=True) of its index nodes only.""" + self.seen = {target} + self.listing = listing + self.max_entries = max_entries + self.entries = 0 # entries of the distinct directory nodes met + self.index_queued: set[Expect] = set() # index children queued, not met yet + + def met(self, expect: Expect, children: list[Expect]) -> list[Expect]: + """The children of expect, a block just met (stored or downloaded), to fetch next: each not seen before, + and for a listing only an index's (a directory's entries and a file's chunks carry no first_name). + + Raises TooLarge once more than max_entries entries are met, counted per block before its children are + fetched: the entries of the distinct directory nodes met, plus one for each index child queued and not + met yet, which holds at least one entry below it (its first_name), so an index tree can't grow unchecked + either. + """ + self.index_queued.discard(expect) + self.entries += sum(1 for child in children if child.kind is not None) + follow = [] + for child in children: + if child not in self.seen and (not self.listing or walker.is_index_step(child)): + self.seen.add(child) + follow.append(child) + if walker.is_index_step(child): + self.index_queued.add(child) + if self.max_entries is not None and self.entries + len(self.index_queued) > self.max_entries: + raise TooLarge(f"more than max_output_entries ({self.max_entries}) entries") + return follow class _Provider: @@ -267,6 +494,9 @@ def __init__(self, storage: Storage, client, config: Config, own_id: bytes, rout # node ID -> mesh.PathInfo (hops, next-hop interface bitrate) for ranking providers; the node sets # RNS's (mesh.RnsPaths.info), and without it nothing is known and ranking falls back to reply order self.path_info: Callable[[bytes], mesh.PathInfo] = mesh.no_path_info + # path_info of several nodes at once (mesh.RnsPaths.infos: one RPC to a shared instance, not one each) + self.path_infos: Callable[[list[bytes]], dict[bytes, mesh.PathInfo]] = lambda node_ids: { + node_id: self.path_info(node_id) for node_id in node_ids} self._lock = threading.Lock() self._db = None if state_dir is None else statedb.connect(state_dir) self._invalid: OrderedDict[Cid, None] = OrderedDict() # oldest first @@ -372,16 +602,11 @@ def __init__(self, fetcher: Fetcher, root: Cid, names: list[str], from_node: byt self.downloaded_blocks = 0 self.downloaded_bytes = 0 self.node_bytes = 0 # DAG node bytes downloaded, which max_download_size also bounds - # Set by the caller to bound the bytes of the distinct blocks of the DAG met, stored or downloaded - # (a remote pin's quota); TooLarge once over. fetch_dag adds the bound its target's size implies - self.max_dag_bytes: int | None = None - self.dag_bytes = 0 - # Content bytes the targets of fetch_dag declare, once it runs (fetch_listing's count as 0) - self._declared: int | None = None - self._credit = 0 # what the counted directory and index nodes bring to the bound (node_credit) - self._counted: set[Cid] = set() - self._range: _RangeBudget | None = None # the budget of the read_range call ensuring blocks, if any - self._progress_at = fetcher.clock() # last verified block or response progress, for the no-progress deadline + self.budget = _DagBudget(root) # the DAG byte limits (max_dag_bytes, dag_limit()) + # For the no-progress deadline: the fetch thread's last verified block (or pacing wait), and the last + # response progress a request thread heard; the deadline counts from the later of the two + self._progress_at = fetcher.clock() + self._heard_at = self._progress_at self.root_delivered = False self.closed = False self.session = None @@ -395,9 +620,39 @@ def __init__(self, fetcher: Fetcher, root: Cid, names: list[str], from_node: byt self._events: queue.Queue = queue.Queue() # provider -> (CIDs asked for, is a probe); kept across _download calls, since a request may outlive one self._in_flight: dict[bytes, tuple[list[Cid], bool]] = {} + # What each CID of a request in flight (not a probe) was asked for as, so a block arriving after the + # _download call that asked for it has returned is checked and stored rather than dropped + self._expects: dict[Cid, list[Expect]] = {} + # End-game duplication (_duplicate): when each request in flight (not a probe) was sent, by the fetch + # thread's clock; a request duplicated -> its duplicate, and back, while each is in flight; and the + # requests whose links were closed because the other copy won, whose outcome isn't counted + self._sent_at: dict[bytes, float] = {} + self._duplicated: dict[bytes, bytes] = {} + self._duplicates: dict[bytes, bytes] = {} + self._abandoned: set[bytes] = set() + # The idents of this fetch's request threads while they run, and the providers whose links it closed: + # a link another fetch has a request on isn't closed (PeerClient.close_unless_used) + self._threads: set[int] = set() + self._links_closed: set[bytes] = set() if from_node is not None: self._new_provider(from_node) + @property + def max_dag_bytes(self) -> int | None: + """Set by the caller to bound the bytes of the distinct blocks of the DAG met, stored or downloaded (a + remote pin's quota); TooLarge once over. fetch_dag adds the bound its target's size implies.""" + return self.budget.max_dag_bytes + + @max_dag_bytes.setter + def max_dag_bytes(self, value: int | None): + """Set the quota.""" + self.budget.max_dag_bytes = value + + @property + def dag_bytes(self) -> int: + """Bytes of the distinct blocks of the DAG counted against its limits so far (none while none applies).""" + return self.budget.used + # Context def __enter__(self) -> "Fetch": @@ -410,25 +665,29 @@ def __enter__(self) -> "Fetch": def __exit__(self, exc_type, exc, tb): """Stop the lookup and end the session; an InvalidDag raised by this fetch marks the root invalid. - A fetch ended by an error (or cancelled) closes the links of its requests still in flight, so their - responses don't keep using the link and hold up the next /blocks to those nodes. + A fetch ended by an error (or cancelled, or by its reader closing a read_range early) closes the links of + its requests still in flight, so their responses don't keep using the link and hold up the next /blocks + to those nodes. One that ends normally does so for requests it no longer needs (a read_range's next window, + left unread), but leaves a probe for the first block, which another node sent, to finish. A link another + fetch has a request on (in progress or waiting for the link's /blocks slot) isn't closed: this fetch's + request is left to finish, and its blocks dropped, rather than fail the other fetch's and make it drop + the provider. Closing the link is how a requester stops a response: it never cancels the response's + Resource, which a responder would take for a rejection and cool the requester down for (section 3). """ self.closed = True if self._lookup is not None: self._lookup.stop() - if exc_type is not None: - while True: # requests already answered aren't in flight any more - try: - event = self._events.get_nowait() - except queue.Empty: - break - if event[0] == "result": - self._in_flight.pop(event[1], None) - for node_id in list(self._in_flight): - try: - self.client.close(node_id) - except Exception as e: - log(f"could not close the link to {node_name(node_id)}: {e}", LOG_WARNING) + while True: # requests already answered aren't in flight any more + try: + event = self._events.get_nowait() + except queue.Empty: + break + if isinstance(event, ResultEvent): + self._in_flight.pop(event.node_id, None) + for node_id, (_, probe) in list(self._in_flight.items()): + if (exc_type is None and probe) or node_id in self._links_closed: + continue + self._close_link(node_id) # Only this fetch's own error: one from another fetch nested in the same `with` is about another root if isinstance(exc, InvalidDag) and getattr(exc, "root", None) == self.root: self.fetcher.mark_invalid(self.root) @@ -497,7 +756,8 @@ def check_disk(self, target: Expect, size: int): self.storage.check_disk(size) def fetch_dag(self, target: Expect, max_entries: int | None = None): - """Fetch every block of target's DAG not already stored, level by level, checking each. + """Fetch every block of target's DAG not already stored, checking each; a block's children are asked for + as soon as it is met (_Walk). Raises TooLarge once its distinct blocks hold more than dag_limit() bytes (what target's declared size and the directory entries met allow, or max_dag_bytes if lower), and with max_entries once more @@ -508,7 +768,7 @@ def fetch_dag(self, target: Expect, max_entries: int | None = None): size = self.size_of(target) self.check_size(size) # From here on the DAG's blocks are counted (the target's again) against what its size can hold - self._declared = (self._declared or 0) + size + self.budget.declare(size) if self.observer is not None: self.observer.total(size) # The root's own DAG, already complete, was checked from the same expectation when it was memoised; @@ -524,19 +784,7 @@ def fetch_dag(self, target: Expect, max_entries: int | None = None): return # What is stored already needn't fit again (its DAG nodes make this an estimate) self.check_disk(target, size - self.storage.dag_bytes(target.cid)) - seen = {target} - frontier = [target] - entries = 0 - while frontier: - children = self.ensure(frontier) - next_frontier = [] - for expect in frontier: - for child in children[expect]: - if child not in seen: - seen.add(child) - next_frontier.append(child) - entries = self._count_entries(entries, frontier, children, next_frontier, max_entries) - frontier = next_frontier + self.ensure([target], walk=_Walk(target, False, max_entries)) def fetch_listing(self, target: Expect, max_entries: int | None = None): """Fetch the directory and index nodes of the directory target (not its entries' contents). @@ -548,39 +796,18 @@ def fetch_listing(self, target: Expect, max_entries: int | None = None): if max_entries is None: max_entries = self.config.max_output_entries with self._checking(): - if self._declared is None: - self._declared = 0 # from here on its blocks are counted, the target's again - frontier = [target] - seen = {target} - entries = 0 - while frontier: - children = self.ensure(frontier) - # Only an index's children carry first_name; a directory's entries and a file's chunks don't - next_frontier = [child for expect in frontier for child in children[expect] - if walker.is_index_step(child) and child not in seen] - entries = self._count_entries(entries, frontier, children, next_frontier, max_entries) - seen.update(next_frontier) - frontier = next_frontier - - @staticmethod - def _count_entries(entries: int, frontier: list[Expect], children: dict[Expect, list[Expect]], - next_frontier: list[Expect], max_entries: int | None) -> int: - """Entries met so far, after a level: raises TooLarge past max_entries, before the next level is fetched. - - Entries of distinct directory nodes are counted, plus the index children about to be fetched: each - holds at least one entry below it (its first_name), so an index tree can't grow unchecked either. - """ - entries += sum(1 for expect in frontier for child in children[expect] if child.kind is not None) - if max_entries is not None: - pending = sum(1 for child in next_frontier if walker.is_index_step(child)) - if entries + pending > max_entries: - raise TooLarge(f"more than max_output_entries ({max_entries}) entries") - return entries + self.budget.declare(0) # from here on its blocks are counted, the target's again + self.ensure([target], walk=_Walk(target, True, max_entries)) def read_range(self, target: Expect, offset: int = 0, length: int | None = None) -> Iterator[bytes]: """The bytes of the file target in [offset, offset + length), fetching only the chunks covering them and the nodes above, WINDOW chunks at a time. Raises ValueError for a directory. + The next window's chunks are asked for before this one's bytes are handed to the reader, so the + providers keep working while it reads; a reader that stops early has fetched at most one window more + than it read, and ending the fetch closes the links still bringing it (but not one another fetch has a + request on: that request is left to finish). + Lazy all the way down: the range is walked depth-first as it is read, so a DAG claiming billions of chunks costs nothing until they are read, and small chunks count against max_output_entries. The file nodes it meets, stored or downloaded, may hold range_bound(size, range length) bytes (TooLarge past @@ -590,7 +817,7 @@ def read_range(self, target: Expect, offset: int = 0, length: int | None = None) raise ValueError("offset and length must be non-negative") with self._checking(): self.ensure([target]) - if walker.kind_of(target, self.get(target)) != "file": + if walker.kind_of(target, self.get(target)) != walker.FILE_TYPE: raise ValueError(f"{target.cid} is a directory") size = self.size_of(target) end = size if length is None else min(size, offset + length) @@ -598,22 +825,32 @@ def read_range(self, target: Expect, offset: int = 0, length: int | None = None) return self.check_size(end - offset) self.check_disk(target, end - offset) - budget = _RangeBudget(range_bound(size, end - offset), end - offset) + budget = self.budget.for_range(range_bound(size, end - offset), end - offset) if not target.cid.is_raw: - budget.count(self.root, target.cid, len(self.get(target))) + budget.count(target.cid, len(self.get(target))) # the fetch's own budget has counted it already small = walker.SmallLeaves(self.config.max_output_entries) - window: list[tuple[Expect, int, int]] = [] - for leaf in self._range_leaves(target, offset, end - offset, budget): - small.meet(leaf[2]) - window.append(leaf) - if len(window) >= WINDOW: - yield from self._read_window(window) - window = [] - yield from self._read_window(window) - - def _read_window(self, window: list[tuple[Expect, int, int]]) -> Iterator[bytes]: - """Fetch a window of chunks in one go, then yield the range within each, in order.""" - self.ensure([expect for expect, _, _ in window]) + leaves = self._range_leaves(target, offset, end - offset, budget) + window = self._next_window(leaves, small) + while window: + ahead = self._next_window(leaves, small) + yield from self._read_window(window, ahead) + window = ahead + + @staticmethod + def _next_window(leaves: Iterator[tuple[Expect, int, int]], + small: walker.SmallLeaves) -> list[tuple[Expect, int, int]]: + """The next WINDOW chunks of a range's walk (fewer at its end, none past it), each met by small.""" + window = [] + for leaf in islice(leaves, WINDOW): + small.meet(leaf[2]) + window.append(leaf) + return window + + def _read_window(self, window: list[tuple[Expect, int, int]], + ahead: list[tuple[Expect, int, int]]) -> Iterator[bytes]: + """Fetch a window of chunks in one go, asking for ahead's (the next window's) once nothing of it is left to + ask for, then yield the range within each, in order.""" + self.ensure([expect for expect, _, _ in window], ahead=[expect for expect, _, _ in ahead]) for expect, child_offset, child_length in window: block = self.get(expect) try: @@ -629,10 +866,10 @@ def _read_window(self, window: list[tuple[Expect, int, int]]) -> Iterator[bytes] yield block[child_offset:child_offset + child_length] def _range_leaves(self, target: Expect, offset: int, length: int, - budget: _RangeBudget) -> Iterator[tuple[Expect, int, int]]: + budget: _DagBudget) -> Iterator[tuple[Expect, int, int]]: """The raw chunks covering a range, in order, each with the range within it, depth-first; the DAG - nodes below each node are fetched together when it is reached, counted against budget (the stack - holds at most MAX_DEPTH × MAX_CHILDREN items).""" + nodes below each node are fetched together when it is reached, counted against budget, the read_range + call's (the stack holds at most MAX_DEPTH × MAX_CHILDREN items).""" stack: list[tuple[Expect, int, int]] = [(target, offset, length)] while stack: expect, child_offset, child_length = stack.pop() @@ -640,46 +877,78 @@ def _range_leaves(self, target: Expect, offset: int, length: int, yield expect, child_offset, child_length continue parts = walker.range_children(expect, self.get(expect), child_offset, child_length) - # Only while ensuring: another read_range of this fetch may run between this generator's steps - self._range = budget - try: - self.ensure([child for child, _, _ in parts if not child.cid.is_raw]) - finally: - self._range = None + # Passed, not kept on the fetch: another read_range of this fetch may run between this generator's steps + self.ensure([child for child, _, _ in parts if not child.cid.is_raw], budget=budget) stack.extend(reversed(parts)) # Getting blocks - def ensure(self, expects: list[Expect]) -> dict[Expect, list[Expect]]: + def ensure(self, expects: list[Expect], budget: _DagBudget | None = None, ahead: list[Expect] = (), + walk: _Walk | None = None) -> dict[Expect, list[Expect]]: """Make sure every block of expects is stored and passes its reader checks; their children's expects. + Their blocks are counted against budget (a read_range call's), by default the fetch's own. The blocks of + ahead (a read_range's next window) are asked for once nothing of expects is left to ask for, and stored + as they arrive, during this call or a later one, but not waited for; one stored already, denied, or not + passing its CID check is left out, to be met when it is ensured. Under walk (fetch_dag's or + fetch_listing's; not with ahead), each block met adds the children walk.met returns, ensured in the same + call, and nothing is returned. Raises InvalidDag, NotFound when no provider can supply a block, StorageFull, TooLarge. """ with self._checking(): - return self._ensure(expects) + return self._ensure(expects, self.budget if budget is None else budget, ahead, walk) - def _ensure(self, expects: list[Expect]) -> dict[Expect, list[Expect]]: + def _ensure(self, expects: list[Expect], budget: _DagBudget, ahead: list[Expect] = (), + walk: _Walk | None = None) -> dict[Expect, list[Expect]]: """ensure(), untagged.""" self._check_cancelled() children: dict[Expect, list[Expect]] = {} wanted: dict[Cid, list[Expect]] = {} - for expect in dict.fromkeys(expects): + self._meet(expects, wanted, children, budget, walk) + prefetch: dict[Cid, list[Expect]] = {} + for expect in ahead: + cid = expect.cid + if cid in wanted or self.storage.has(cid) or self.storage.deny.denies(cid): + continue + try: + walker.check_cid(expect) + except InvalidDag: + continue + prefetch.setdefault(cid, []).append(expect) + if wanted or prefetch: + self._download(wanted, children, budget, prefetch, walk) + self.fetcher.after_store() + return children + + def _meet(self, expects, wanted: dict[Cid, list[Expect]], children: dict, budget: _DagBudget, + walk: _Walk | None) -> list[Cid]: + """Check each of expects before fetching or reading it (the deny list, its CID) and, if it is stored, + against its reader checks (counted against budget), its children going into children; the rest go into + wanted. Under walk, a stored block's children walk.met returns are met in turn, and children isn't kept. + Returns the CIDs newly wanted, in the order met.""" + queue = deque(dict.fromkeys(expects)) + new = [] + while queue: + expect = queue.popleft() self.fetcher.check_denied(expect.cid) # before fetching or reading it, stored or not walker.check_cid(expect) # before fetching a block that can't be right - checked = self._check_stored(expect) + checked = self._check_stored(expect, budget) if checked is None: + if expect.cid not in wanted: + new.append(expect.cid) wanted.setdefault(expect.cid, []).append(expect) - else: + continue + if self.observer is not None: + self.observer.block(expect.cid, self.storage.block_size(expect.cid) or 0, False) + if walk is None: children[expect] = checked - if self.observer is not None: - self.observer.block(expect.cid, self.storage.block_size(expect.cid) or 0, False) - if wanted: - self._download(wanted, children) - self.fetcher.after_store() - return children + else: + queue.extend(walk.met(expect, checked)) + return new - def _check_stored(self, expect: Expect) -> list[Expect] | None: - """Children of a stored block after checking it against expect, or None if it isn't stored. + def _check_stored(self, expect: Expect, budget: _DagBudget) -> list[Expect] | None: + """Children of a stored block after checking it against expect (counted against budget), or None if it + isn't stored. A stored block failing the checks is verified against its CID: if it no longer matches (damaged on disk), it is deleted and None returned, so it is fetched again rather than the root marked invalid. @@ -690,7 +959,7 @@ def _check_stored(self, expect: Expect) -> list[Expect] | None: if length is None: return None walker.check_raw(expect, length) # stored blocks were verified, so their length is enough - self._count_dag(expect.cid, length) + budget.count(expect.cid, length) return [] block = self.storage.get(expect.cid) if block is None: @@ -701,82 +970,16 @@ def _check_stored(self, expect: Expect) -> list[Expect] | None: log(f"stored block {expect.cid} of {self.root} is damaged; fetching it again", LOG_WARNING) return None raise - self._count_dag(expect.cid, len(block), result) + budget.count(expect.cid, len(block), result) return result def dag_limit(self) -> int | None: - """Bytes the distinct blocks of the DAG may hold: max_dag_bytes, and once fetch_dag (or fetch_listing) - runs, dag_bound of the size its target declares plus what the directory and index nodes counted bring - (node_credit), whichever is lower. None if neither applies.""" - limits = [] if self.max_dag_bytes is None else [self.max_dag_bytes] - if self._declared is not None: - limits.append(dag_bound(self._declared) + self._credit) - return min(limits, default=None) - - def _count_dag(self, cid: Cid, length: int, children: list[Expect] = ()): - """Count a block of the DAG once against dag_limit(), if there is one, with what it brings to it - (node_credit), and a DAG node against the budget of the read_range call ensuring it, if any. Raises - TooLarge once over either.""" - if self._range is not None and not cid.is_raw: - self._range.count(self.root, cid, length) - if (self.max_dag_bytes is None and self._declared is None) or cid in self._counted: - return - self._counted.add(cid) - self.dag_bytes += length - self._credit += node_credit(length, children) - limit = self.dag_limit() - if self.dag_bytes > limit: - if limit == self.max_dag_bytes: - raise TooLarge(f"{self.root} holds more than {limit} bytes of blocks") - content = f"its {plural(self._declared, 'byte')} of content and " if self._declared else "" - raise TooLarge(f"{self.root} holds more than {limit} bytes of blocks, more than {content}its " - "directory entries can take: refusing it") - - def _room(self, wanted: dict[Cid, list[Expect]]) -> tuple | None: - """What the limits leave for blocks not yet asked for, after the most the requests in flight can bring: - (bytes under the size-derived bound, bytes under max_dag_bytes, bytes under the budget of the - read_range call ensuring blocks), each None if that limit doesn't apply; None if none does.""" - if self._declared is None and self.max_dag_bytes is None and self._range is None: - return None - bound = None if self._declared is None else dag_bound(self._declared) + self._credit - self.dag_bytes - quota = None if self.max_dag_bytes is None else self.max_dag_bytes - self.dag_bytes - budget = None if self._range is None else self._range.limit - self._range.used - room = (bound, quota, budget) - for cids, _ in self._in_flight.values(): - for cid in cids: - room = self._reserve(cid, wanted, room) - return room + """Bytes the distinct blocks of the DAG may hold (_DagBudget.limit), None if no limit applies.""" + return self.budget.limit() - def _reserve(self, cid: Cid, wanted: dict[Cid, list[Expect]], room: tuple) -> tuple: - """room (as _room) after the most cid can add to the bytes counted: a raw chunk's declared size, - otherwise MAX_BLOCK_SIZE, and nothing for a raw chunk under a read_range budget. - - Under the size-derived bound a directory or index node takes nothing: a directory node brings more - than it takes, and an index node takes at most INDEX_NODE_COST, so a request of MAX_BATCH of them - overshoots by at most 2 KiB. Holding them to INDEX_NODE_COST each would ask for a directory's - subdirectories a few at a time. - """ - bound, quota, budget = room - if budget is not None and not cid.is_raw and cid not in self._range.counted: - budget -= MAX_BLOCK_SIZE - if cid in self._counted: - return bound, quota, budget - expects = wanted.get(cid) - worst = MAX_BLOCK_SIZE - if expects and cid.is_raw and expects[0].size is not None: - worst = expects[0].size - if bound is not None and not (expects and not cid.is_raw - and not any(walker.FILE_TYPE in expect.types for expect in expects)): - bound -= worst - return bound, None if quota is None else quota - worst, budget - - @staticmethod - def _fits(room: tuple) -> bool: - """True if room (as _room) leaves nothing below zero.""" - return all(left is None or left >= 0 for left in room) - - def _accept(self, cid: Cid, block: bytes, wanted: dict[Cid, list[Expect]], children: dict): - """Check a verified block against every expectation of it and the limits, then store it. + def _accept(self, cid: Cid, block: bytes, wanted: dict[Cid, list[Expect]], children: dict, budget: _DagBudget): + """Check a verified block against every expectation of it and the limits (budget's among them), then + store it. Raises InvalidDag, TooLarge. """ @@ -787,7 +990,7 @@ def _accept(self, cid: Cid, block: bytes, wanted: dict[Cid, list[Expect]], child if limit is not None and self.node_bytes + len(block) > limit: raise TooLarge(f"{self.root}'s DAG nodes would exceed max_download_size ({limit})") self.node_bytes += len(block) - self._count_dag(cid, len(block), children[wanted[cid][0]]) + budget.count(cid, len(block), children[wanted[cid][0]]) self.session.put(cid, block) self._progress_at = self.fetcher.clock() self.downloaded_blocks += 1 @@ -799,22 +1002,39 @@ def _accept(self, cid: Cid, block: bytes, wanted: dict[Cid, list[Expect]], child if self._lookup is not None: self._lookup.stop() # it may now have enough providers to stop searching - def _download(self, wanted: dict[Cid, list[Expect]], children: dict): - """Fetch the wanted blocks from providers until all are stored. Raises NotFound if none can.""" - outstanding = set(wanted) - pending = deque(wanted) + def _download(self, wanted: dict[Cid, list[Expect]], children: dict, budget: _DagBudget, + ahead: dict[Cid, list[Expect]] | None = None, walk: _Walk | None = None): + """Fetch the wanted blocks from providers until all are stored, counted against budget. Raises NotFound if + none can. + + ahead's blocks (ensure's) are asked for after wanted's and stored as they arrive, but not waited for: + whatever of them is still to ask for once wanted's are stored goes out to the providers idle by then, + before returning, so they keep working while the caller uses wanted's. Under walk, each block stored adds + its children still to fetch (_meet) to wanted's, at the back of the queue. + """ + outstanding = set(wanted) # still to store: wanted's, then ahead's + needed = outstanding # what the call waits for: wanted's + if ahead: + needed = set(outstanding) + for cid, expects in ahead.items(): + wanted[cid] = expects + outstanding.add(cid) in_flight = self._in_flight + # A CID that a request still in flight (an earlier call's ahead) asked for isn't asked for again: its + # result is waited for, and requeued if it doesn't bring the block + asked = {cid for cids, _ in in_flight.values() for cid in cids} + pending = _Pending(cid for cid in wanted if cid not in asked) self._progress_at = max(self._progress_at, self.fetcher.clock()) # it may be after a pacing wait give_up = self.config.lookup_timeout + self.config.stall_timeout - while outstanding: + while needed: self._check_cancelled() # each pass waits at most POLL, so a cancel takes effect promptly # Requests and lookups have deadlines of their own; this catches whatever hangs regardless - if self.fetcher.clock() - self._progress_at > give_up: + if self.fetcher.clock() - max(self._progress_at, self._heard_at) > give_up: raise NotFound(f"fetching {self.root} made no progress for {give_up:g}s") - self._assign(pending, in_flight, outstanding, wanted) + self._assign(pending, in_flight, outstanding, wanted, budget) if not in_flight and not self._lookup_running and not self._waiting_for_busy(): if not self._rerun_lookup(): - raise NotFound(self._not_found_message(outstanding)) + raise NotFound(self._not_found_message(needed)) continue try: events = [self._events.get(timeout=self._wait_time())] @@ -827,24 +1047,49 @@ def _download(self, wanted: dict[Cid, list[Expect]], children: dict): except queue.Empty: break for event in events: - self._handle_event(event, pending, in_flight, outstanding, wanted, children) - - def _handle_event(self, event: tuple, pending: deque, in_flight: dict, outstanding: set[Cid], - wanted: dict[Cid, list[Expect]], children: dict): - """Apply one event from a request thread or the lookup thread.""" - kind = event[0] - if kind == "provider": - self._add_provider(event[1]) - elif kind == "lookup_done": + self._handle_event(event, pending, in_flight, outstanding, wanted, children, budget, needed, walk) + if pending and self._usable(): + # ahead's requests go out before the caller is handed wanted's blocks (never a lookup just for them) + self._assign(pending, in_flight, outstanding, wanted, budget) + + def _handle_event(self, event: ResultEvent | ProviderEvent | LookupDone, pending: _Pending, in_flight: dict, + outstanding: set[Cid], wanted: dict[Cid, list[Expect]], children: dict, budget: _DagBudget, + needed: set[Cid] | None = None, walk: _Walk | None = None): + """Apply one event from a request thread or the lookup thread. needed is _download's, if it isn't + outstanding itself: the blocks stored are taken out of it too. Under walk, the children of the blocks + stored are met, and those still to fetch queued.""" + if isinstance(event, ProviderEvent): + self._add_provider(event.node_id) + elif isinstance(event, LookupDone): self._lookup_running = False - self._lookup_found = event[1] - elif kind == "result": - _, provider, cids, result, sent_at, rate = event - _, probe = in_flight.pop(provider, (cids, False)) - self._handle(provider, cids, result, probe, wanted, children, outstanding, sent_at, rate) - requeue = [cid for cid in cids if cid in outstanding and cid not in pending - and not any(cid in asked for asked, _ in in_flight.values())] - pending.extendleft(reversed(requeue)) # back at the front, keeping the walk's order + self._lookup_found = event.found + elif isinstance(event, ResultEvent): + cids = event.cids + _, probe = in_flight.pop(event.node_id, (cids, False)) + self._sent_at.pop(event.node_id, None) + abandoned = event.node_id in self._abandoned + self._abandoned.discard(event.node_id) + asked_now = [cid for cid in cids if cid in outstanding] + self._handle(event.node_id, cids, event.result, probe, wanted, children, outstanding, budget, + event.sent_at, event.rate, counted=not abandoned) + self._settle_duplicate(event.node_id, in_flight) + still_asked = {cid for asked, _ in in_flight.values() for cid in asked} + requeue = [cid for cid in cids if cid in outstanding and cid not in pending and cid not in still_asked] + pending.push_front(requeue) # back at the front, keeping the walk's order + for cid in cids: + if cid not in still_asked: + self._expects.pop(cid, None) + if needed is not None and needed is not outstanding: + needed.difference_update(cid for cid in cids if cid not in outstanding) + if walk is not None: + # In the order asked for, so the queue keeps the walk's order (breadth first, as met) + met = [child for cid in asked_now if cid not in outstanding + for expect in wanted[cid] for child in walk.met(expect, children.pop(expect))] + new = self._meet(met, wanted, children, budget, walk) + outstanding.update(new) + if needed is not None and needed is not outstanding: + needed.update(new) + pending.extend([cid for cid in new if cid not in still_asked]) def _make_room(self, nbytes: int): """Before storing nbytes: evict other cached fetches over max_storage if not pinned (going over it if @@ -880,9 +1125,17 @@ def _path(self, node_id: bytes) -> mesh.PathInfo: """What RNS knows about the path to node_id (nothing if the lookup fails).""" try: return self.fetcher.path_info(node_id) - except Exception: + except Exception: # noqa: BLE001 - RNS internals: no path known if they fail return mesh.NO_PATH + def _paths(self, node_ids: list[bytes]) -> dict[bytes, mesh.PathInfo]: + """_path of each of node_ids, looked up together (fetcher.path_infos).""" + try: + found = self.fetcher.path_infos(node_ids) + except Exception: # noqa: BLE001 - RNS internals: no path known if they fail + found = {} + return {node_id: found.get(node_id) or self._path(node_id) for node_id in node_ids} + def _waiting_for_busy(self) -> bool: """True if a usable provider is only waiting out a busy delay.""" now = self.fetcher.clock() @@ -894,7 +1147,8 @@ def _wait_time(self) -> float: waits = [p.busy_until - now for p in self._providers.values() if not p.failed and p.busy_until > now] return max(0.01, min([POLL, *waits])) - def _assign(self, pending: deque, in_flight: dict, outstanding: set[Cid], wanted: dict[Cid, list[Expect]]): + def _assign(self, pending: _Pending, in_flight: dict, outstanding: set[Cid], wanted: dict[Cid, list[Expect]], + budget: _DagBudget): """Send requests to idle providers, after finding providers if there are none.""" now = self.fetcher.clock() usable = self._usable() @@ -907,29 +1161,102 @@ def _assign(self, pending: deque, in_flight: dict, outstanding: set[Cid], wanted break if provider.node_id in in_flight or provider.busy_until > now: continue - batch = self._take_batch(pending, provider.node_id, wanted) + batch = self._take_batch(pending, provider.node_id, wanted, budget) if batch: in_flight[provider.node_id] = (batch, False) + for cid in batch: + self._expects[cid] = wanted[cid] + self._sent_at[provider.node_id] = now self._send(provider.node_id, batch) busy_providers += 1 + if not pending and busy_providers < self.fetcher.max_parallel: + self._duplicate(usable, in_flight, busy_providers, now) + + def _duplicate(self, usable: list[_Provider], in_flight: dict, busy_providers: int, now: float): + """End-game duplication (NET-4; section 7, item 7): with nothing left to ask for, ask idle providers too + for the batches others have had in flight for longer than deadline_slack, oldest first, each at most + once, and within max_parallel requests in flight. A duplicate is sized for its own provider's link and + what one response carries (_take_batch).""" + idle = [provider for provider in usable if provider.node_id not in in_flight and provider.busy_until <= now] + if not idle: + return + slow = sorted((sent_at, node_id) for node_id, sent_at in self._sent_at.items() + if node_id in in_flight and now - sent_at > self.config.deadline_slack + and node_id not in self._duplicated and node_id not in self._duplicates + and node_id not in self._abandoned) + for sent_at, node_id in slow: + if not idle or busy_providers >= self.fetcher.max_parallel: + return + # Its blocks not stored yet, as they were asked for (every CID in flight has its expectations here) + left = [cid for cid in in_flight[node_id][0] if cid in self._expects and not self.storage.has(cid)] + if not left: + continue + provider = idle[0] + # Not reserved against the DAG limits again: a block is counted once, whichever copy brings it + batch = self._take_batch(_Pending(left), provider.node_id, self._expects, None) + if not batch: + continue + idle.pop(0) + log(f"{node_name(node_id)} has been sending {plural(len(left), 'block')} of {self.root} for " + f"{now - sent_at:.0f}s; asking {node_name(provider.node_id)} too", LOG_VERBOSE) + in_flight[provider.node_id] = (batch, False) + self._sent_at[provider.node_id] = now + self._duplicated[node_id] = provider.node_id + self._duplicates[provider.node_id] = node_id + self._send(provider.node_id, batch) + busy_providers += 1 + + def _settle_duplicate(self, node_id: bytes, in_flight: dict): + """node_id's request has been answered: if it was one of a duplicated pair and the other copy, still in + flight, now carries nothing not stored, close that one's link (the first verified copy won), unless another + fetch has a request on it: then it is left to finish. Either way its outcome isn't counted against its + provider (_handle's counted). Closing the link stops the response without cancelling its Resource + (section 3).""" + other = self._duplicated.pop(node_id, None) + if other is not None: + self._duplicates.pop(other, None) + else: + other = self._duplicates.pop(node_id, None) # the original stays marked as duplicated until answered + if other is None or other not in in_flight or other in self._abandoned: + return + if any(not self.storage.has(cid) for cid in in_flight[other][0]): + return # it still carries blocks nobody else has sent + self._abandoned.add(other) + log(f"{node_name(node_id)} sent what {node_name(other)} was still sending; closing that link unless " + "another fetch uses it", LOG_VERBOSE) + self._close_link(other) + + def _close_link(self, node_id: bytes): + """Close the link of this fetch's request to node_id, unless another fetch has a request on it.""" + try: + if self.client.close_unless_used(node_id, set(self._threads)): + self._links_closed.add(node_id) + else: + log(f"leaving the request to {node_name(node_id)} to finish: another fetch uses its link", LOG_VERBOSE) + except Exception as e: # noqa: BLE001 - closing is best effort; logged + log(f"could not close the link to {node_name(node_id)}: {e}", LOG_WARNING) - def _take_batch(self, pending: deque, node_id: bytes, wanted: dict[Cid, list[Expect]]) -> list[Cid]: + def _take_batch(self, pending: _Pending, node_id: bytes, wanted: dict[Cid, list[Expect]], + budget: _DagBudget | None) -> list[Cid]: """The next CIDs to ask node_id for: what one response can carry and one request packet on its link can hold, at least one. - Under a DAG limit, only what the room it leaves can take for sure (_room), but at least one CID when no - request is in flight: so a fetch downloads at most about one block beyond the limit. + Under a DAG limit, only what the room budget leaves can take for sure (_DagBudget.room), but at least one + CID when no request is in flight: so a fetch downloads at most about one block beyond the limit. With no + budget (a duplicate's, whose blocks are reserved already), no limit is applied. """ - budget = rates.requester_max_bytes(self.client.rate_estimate(node_id), self.config) - # However large max_bytes is, the responder's budget is at most this (server.blocks_budget, section 7) - carried = min(budget, MAX_RESPONSE_SIZE - RESPONSE_OVERHEAD) - # The client sends what fits its link's packet and returns the rest: the open link's MDU, else the - # largest request any link allows (test fakes may not know links). Whether a prefix fits only gets - # worse as it grows, so one count covers every batch taken from the front of pending - link_mdu = getattr(self.client, "link_mdu", None) - mdu = (link_mdu(node_id) if link_mdu is not None else None) or MAX_REQUEST_SIZE - fit = max(1, protocol.blocks_batch_size(budget, list(islice(pending, MAX_BATCH)), mdu)) - room = self._room(wanted) + rate = self.client.rate_estimate(node_id) + max_bytes = rates.requester_max_bytes(rate, self.config) + # However large max_bytes is, the responder's budget is at most this (section 7); its upload cap isn't + # known here, and a lower one only cuts the response short (the rest comes back unsent) + carried = rates.response_capacity(max_bytes, rate, None, self.config.response_target) + # The client sends what fits its link's packet and returns the rest: the open link's MDU, else (no link + # open yet) the largest request any link allows. Whether a prefix fits only gets worse as it grows, so + # one count covers every batch taken from the front of pending + mdu = self.client.link_mdu(node_id) or MAX_REQUEST_SIZE + fit = max(1, protocol.blocks_batch_size(max_bytes, list(islice(pending, MAX_BATCH)), mdu)) + room = None if budget is None else budget.room((cid for cids, _ in self._in_flight.values() for cid in cids), + wanted) batch: list[Cid] = [] total = 0 while pending and len(batch) < fit: @@ -939,8 +1266,8 @@ def _take_batch(self, pending: deque, node_id: bytes, wanted: dict[Cid, list[Exp if batch and total + guess > carried: break if room is not None: - room = self._reserve(cid, wanted, room) - if not self._fits(room) and (batch or self._in_flight): + room = budget.reserve(cid, wanted, room) + if not budget.fits(room) and (batch or self._in_flight): break batch.append(pending.popleft()) total += guess @@ -949,36 +1276,45 @@ def _take_batch(self, pending: deque, node_id: bytes, wanted: dict[Cid, list[Exp def _send(self, node_id: bytes, cids: list[Cid]): """Ask node_id for cids on a request thread, which queues the result.""" def progress(fraction: float): - """The response is arriving: the fetch isn't hung.""" - self._progress_at = self.fetcher.clock() + """The response is arriving: the fetch isn't hung (on the request thread, so only _heard_at is written).""" + self._heard_at = self.fetcher.clock() def work(): """Run the blocking request and queue its result, with when it was sent and the rate estimate it was sent with (the responder's upload cap derives from the same response).""" sent_at = self.fetcher.clock() rate = None + ident = threading.get_ident() + self._threads.add(ident) # until its request is over: the link's users that are this fetch's try: rate = self.client.rate_estimate(node_id) result = self.client.blocks(node_id, cids, on_progress=progress) - except Exception as e: + except Exception as e: # noqa: BLE001 - any client failure is a failed request, retried elsewhere; logged + # The client reports network failures in its result, so this is a bug: logged here, inside the + # except, so the debug log has its traceback + log(f"/blocks to {node_name(node_id)} raised {e!r}", LOG_WARNING) result = peer_client.BlocksResult(peer_client.FAILED, unsent=list(cids), reason=str(e)) + finally: + self._threads.discard(ident) if result.elapsed is not None: # When it actually went out: opening the link or waiting for its /blocks slot isn't its time sent_at = max(sent_at, self.fetcher.clock() - result.elapsed) - self._events.put(("result", node_id, cids, result, sent_at, rate)) + self._events.put(ResultEvent(node_id, cids, result, sent_at, rate)) threading.Thread(target=work, name="rnfs-fetch", daemon=True).start() # Finding providers - def _discover(self, pending: deque, in_flight: dict): + def _discover(self, pending: _Pending, in_flight: dict): """No usable provider: ask nodes with open links, else start the lookup.""" if not self._probed: self._probed = True banned = self.fetcher.routing.is_banned linked = [n for n in self.client.open_links() if n != self.fetcher.own_id and not banned(n)] - # Ranked like providers; most recently used breaks ties - ranked = sorted(enumerate(linked), key=lambda item: self._rank(item[1], self._path(item[1]), item[0])) + # Ranked like providers; most recently used breaks ties. Their paths in one go: attached to a shared + # instance, that is one RPC rather than one per link before the first request (plan C1) + paths = self._paths(linked) + ranked = sorted(enumerate(linked), key=lambda item: self._rank(item[1], paths[item[1]], item[0])) candidates = [node_id for _, node_id in ranked][:self.fetcher.max_parallel] if candidates and pending: # Each is asked for just the first block missing, which is the root at the start @@ -1006,10 +1342,10 @@ def work(): """Search, reporting each provider as its reply arrives.""" found = [] try: - found = lookup.find_providers(lambda node_id: self._events.put(("provider", node_id)), should_stop) - except Exception as e: + found = lookup.find_providers(lambda node_id: self._events.put(ProviderEvent(node_id)), should_stop) + except Exception as e: # noqa: BLE001 - a failed lookup ends with the providers found so far log(f"provider lookup for {self.root} failed: {e}", LOG_WARNING) - self._events.put(("lookup_done", len(found))) + self._events.put(LookupDone(len(found))) threading.Thread(target=work, name="rnfs-find-providers", daemon=True).start() @@ -1052,10 +1388,14 @@ def _fail(self, provider: _Provider, why: str): log(f"provider {node_name(provider.node_id)} failed for {self.root}: {why}", LOG_VERBOSE) def _handle(self, node_id: bytes, cids: list[Cid], result, probe: bool, wanted: dict, children: dict, - outstanding: set[Cid], sent_at: float | None = None, rate: float | None = None): - """Store a response's blocks and apply the provider failure and pacing rules. Raises InvalidDag. - - sent_at is when the request was sent and rate the link's rate estimate then (None: unknown). + outstanding: set[Cid], budget: _DagBudget, sent_at: float | None = None, rate: float | None = None, + counted: bool = True): + """Store a response's blocks (counted against budget) and apply the provider failure and pacing rules. + Raises InvalidDag. + + sent_at is when the request was sent and rate the link's rate estimate then (None: unknown). A request + not counted (one whose link the fetch closed, as its duplicate won) has its blocks stored, but its outcome + applies none of the rules: closing the link isn't the provider's stall or failure. """ provider = self._providers.get(node_id) if result.bad: @@ -1067,10 +1407,16 @@ def _handle(self, node_id: bytes, cids: list[Cid], result, probe: bool, wanted: provider = self._new_provider(node_id) self._fail(provider, "sent a block not matching its CID") new = {cid: block for cid, block in result.blocks.items() if cid in outstanding} - self._make_room(sum(len(block) for cid, block in new.items() if not self.storage.has(cid))) + # Asked for by an earlier _download call (a read_range's next window, while its walk fetched a file node): + # checked against what it was asked for as and stored, not dropped and asked for again + late = {cid: block for cid, block in result.blocks.items() + if cid not in outstanding and cid in self._expects and not self.storage.has(cid)} + self._make_room(sum(len(block) for cid, block in (new | late).items() if not self.storage.has(cid))) for cid, block in new.items(): - self._accept(cid, block, wanted, children) + self._accept(cid, block, wanted, children, budget) outstanding.discard(cid) + for cid, block in late.items(): + self._accept(cid, block, self._expects, {}, self.budget) if probe: # A node with an open link that has the block becomes a provider; one that hasn't isn't penalised if result.blocks and provider is None: @@ -1078,7 +1424,7 @@ def _handle(self, node_id: bytes, cids: list[Cid], result, probe: bool, wanted: log(f"{node_name(node_id)} (open link) has {self.root}", LOG_VERBOSE) self._pace(provider, sent_at, rate) return - if provider is None or provider.failed: + if provider is None or provider.failed or not counted: return now = self.fetcher.clock() if result.status == peer_client.OK: @@ -1112,12 +1458,16 @@ def _pace(self, provider: _Provider, sent_at: float | None, rate: float | None): return now = self.fetcher.clock() took = max(0.0, now - sent_at) # about the response's transfer time at the link's rate, plus its RTT + # Refusals until 4 × took after sending count as pacing: at the default cap the next response may start + # 2 × took after the last one did (rates.UPLOAD_CAP_DIVISOR), and twice that leaves room for a responder + # whose rate estimate or cap is lower than ours and the default. Retrying them every eighth of took loses + # at most that much once the responder is ready, for one refusal (a request packet) each time provider.pacing_until = sent_at + 4 * took provider.pacing_retry = max(self.fetcher.busy_delay, took / 8) if rate is not None and rate < self.config.slow_bitrate: - # At the default cap (half the rate estimate) the next response may start once the last one's - # transfer time has passed twice since it started - provider.busy_until = max(provider.busy_until, now + took) + # At the default cap (rates.default_upload_cap) the next response may start once the last one's + # transfer time has passed UPLOAD_CAP_DIVISOR times since it started: took has passed once already + provider.busy_until = max(provider.busy_until, now + (rates.UPLOAD_CAP_DIVISOR - 1) * took) self._progress_at = max(self._progress_at, provider.busy_until) # waiting isn't hanging diff --git a/rnfs/fsutil.py b/rnfs/fsutil.py index f403632..ef3ad0a 100644 --- a/rnfs/fsutil.py +++ b/rnfs/fsutil.py @@ -10,6 +10,7 @@ import stat import sys import time +import unicodedata # The one platform switch for rnfs's Windows behaviour; tests patch it to exercise that behaviour anywhere IS_WINDOWS = os.name == "nt" @@ -255,6 +256,38 @@ def is_link(st: os.stat_result) -> bool: return stat.S_ISLNK(st.st_mode) or tag == _REPARSE_JUNCTION or bool(tag & _REPARSE_NAME_SURROGATE) +def check_owned_fd(path: str, fd: int, info: os.stat_result, what: str) -> str | None: + """Check that what (e.g. "the settings file") at path, open as fd and whose fstat is info, belongs to this + user and is private to them, making it so if others may access it: a warning saying what was changed, or + None if nothing was. Raises PermissionError if another user owns it (they could change it again), OSError if + it can't be made private. + + Through the descriptor, so a file swapped in at path after the open can't pass for it. POSIX: owned by the + effective user, and a file with group or other bits set is made 0600 (a directory 0700). Windows, whose + st_mode always shows others allowed and whose st_uid is always 0: the owner and access list instead + (winacl.tighten). + """ + directory = stat.S_ISDIR(info.st_mode) + if IS_WINDOWS: + from rnfs import winacl + reason = winacl.tighten(path, directory=directory, fd=fd) + if reason is None: + return None + return f"{what} {path} was accessible to other users ({reason}); restricted it to you and SYSTEM" + if info.st_uid != os.geteuid(): + raise PermissionError(f"{what} {path} belongs to another user (uid {info.st_uid})") + mode = stat.S_IMODE(info.st_mode) + if not mode & 0o077: + return None + private = 0o700 if directory else 0o600 + try: + os.fchmod(fd, private) + except OSError as e: + raise PermissionError(f"{what} {path} is accessible to other users (mode {mode:o}) and can't be " + f"restricted: {e.strerror or e}") from None + return f"{what} {path} was accessible to other users (mode {mode:o}); restricted it to {private:o}" + + def is_below(path: str, root: str) -> bool: """True if path is root or inside it; both should be real (resolved) paths. Case-insensitive on Windows.""" path, root = os.path.normcase(path), os.path.normcase(root) @@ -347,6 +380,33 @@ def printable(text) -> str: return "".join(c if c.isprintable() else c.encode("unicode_escape").decode("ascii") for c in text) +# Printable characters that show nothing: the Hangul fillers and the blank braille pattern +_BLANK_FILLERS = frozenset("ᅟᅠㅤﾠ⠀") +MAX_COMBINING_RUN = 2 # combining marks kept in a row (some scripts stack two); more spill over the text around + + +def printable_label(text) -> str: + """text, a label someone else chose (an LXMF display name), escaped as printable() does, and also where it + could pass for another label or for none: backslashes (so it can't spell out an escape), the blank fillers, + a combining mark that follows no printed character (it would join what was printed before) or makes more + than MAX_COMBINING_RUN in a row, and a label of nothing but spaces. Not for paths: printable() keeps a + Windows path's backslashes.""" + out, run, after_base = [], 0, False # run: combining marks kept in a row; after_base: one may follow + for c in str(text): + mark = unicodedata.category(c).startswith("M") + if c == "\\" or c in _BLANK_FILLERS or not c.isprintable() \ + or (mark and (not after_base or run >= MAX_COMBINING_RUN)): + out.append(c.encode("unicode_escape").decode("ascii")) # "\\" for a backslash, "ㅤ" for a filler + after_base = False + continue + out.append(c) + run = run + 1 if mark else 0 + after_base = True + if out and all(c == " " for c in out): + return "\\x20" * len(out) + return "".join(out) + + def unsafe_name_reason(name: str) -> str | None: """Why the local filesystem can't take name as a plain file name (it would be read as a path), or None.""" if name in (".", "..") or "/" in name or "\0" in name: diff --git a/rnfs/gateway.py b/rnfs/gateway.py index 94eda68..42b6404 100644 --- a/rnfs/gateway.py +++ b/rnfs/gateway.py @@ -8,7 +8,8 @@ segment is exactly one name. - Files: Content-Type from ?filename=, else the path's last name, else sniffed from the first bytes. The user's own HEAD, or range not starting at byte 0, uses the first bytes only if they are stored (else the - name, or application/octet-stream), so it never fetches what it doesn't send. A web page's request (see + name, or application/octet-stream), so it never fetches what it doesn't send; the deny list applies to + them as to any read (451). A web page's request (see limited() below) always reads them, fetching them if need be, so what a page can observe of the type (e.g. through nosniff or opaque-response blocking) never depends on what is stored. Single byte ranges (206, 416, Accept-Ranges; If-Range honoured); several ranges get the whole file. @@ -31,8 +32,10 @@ - Requests that aren't the user's own navigation (Sec-Fetch-Site present and not "none") may hold at most gateway_fetches network fetches or name resolutions at a time; over that, 503 with Retry-After. Content already stored (node.is_local) is served without limit. A request with no Sec-Fetch-Site is - treated as the user's own: every browser sends the header to 127.0.0.1/localhost (potentially - trustworthy origins), so its absence means a local tool such as curl, run by the user. + treated as the user's own only if its User-Agent doesn't start with "Mozilla/": browsers with Fetch + Metadata send the header to 127.0.0.1/localhost (potentially trustworthy origins), so its absence means + a local tool such as curl, run by the user, or a browser without Fetch Metadata (Safari < 16.4, Firefox + < 90), whose every request is then treated as a web page's (no ?format=, held to gateway_fetches). - Everything is requested with for_gateway=True: fetched content is cache, never provided, and names go only into the capped table. Name resolutions are reused for NAME_CACHE_TIME, so a page's many subresources don't each start a resolution against gateway_fetches. @@ -71,8 +74,9 @@ NAME_CACHE_TIME = 60.0 NAME_CACHE_SIZE = 1000 INDEX = "index.html" -# What an error from the node says on the gateway, by status: never its own text (see the module docstring) -ERROR_MESSAGES = {400: "invalid request", 413: "too large for this node's download limits", +# What an error from the node says on the gateway, by status: never its own text (see the module docstring). Every +# status of rnfs.errors.STATUSES has one, and 500; a NotFound's own text is shown, this only if it has none +ERROR_MESSAGES = {400: "invalid request", 404: "not found", 413: "too large for this node's download limits", 451: "unavailable: this node's deny list blocks this content", 422: "invalid content: it breaks the rnfs format's rules", 503: "the node is busy; try again shortly", 507: "not enough free disk space on this node", 500: "internal error; see the rnfs daemon's log"} @@ -423,7 +427,7 @@ def handle_request(self): self.respond_error(e.status, str(e), e.headers) except NotFound as e: log(f"gateway: {loggable(self.path)} not found: {e}", LOG_DEBUG) - self.respond_error(404, str(e) or "not found", {}) + self.respond_error(404, str(e) or ERROR_MESSAGES[404], {}) except Exception as e: # noqa: BLE001 - every error becomes a response, never a dropped connection status = error_status(e) # str(e) may name local paths (a StorageFull's or an OSError's) or the node's limits: logged only @@ -449,7 +453,11 @@ def respond_error(self, status: int, message: str, headers: dict): def limited(self) -> bool: """True if this request isn't the user's own navigation, so its network use is limited.""" site = self.headers.get("Sec-Fetch-Site") - return site is not None and site.strip().lower() != "none" + if site is None: + # Browsers without Fetch Metadata (Safari < 16.4, Firefox < 90) send no Sec-Fetch-Site for a web page's + # requests either, and every browser's User-Agent starts with "Mozilla/": only a tool's is the user's own + return (self.headers.get("User-Agent") or "").strip().lower().startswith("mozilla/") + return site.strip().lower() != "none" def need_network(self): """Take a gateway_fetches slot for a limited request about to use the network (once per request).""" diff --git a/rnfs/gateway_formats.py b/rnfs/gateway_formats.py index be04f28..cdabe80 100644 --- a/rnfs/gateway_formats.py +++ b/rnfs/gateway_formats.py @@ -11,15 +11,16 @@ plain response's "". Archives keep max_download_size and max_output_entries. /name/ paths don't take ?format= (the result must be named by the CID it is checked against). -Every format is answered only to the user's own GET (handler.limited() false: no Sec-Fetch-Site, or "none"), -which fetches what isn't stored (an archive's fetch is cancelled once the client disconnects). A web page's -request or a HEAD always gets 409, stored or not, before the node is asked anything, so no format ever takes -a gateway_fetches slot. Neither type is one a browser shows, so a format's 200 is a download, and an iframe -of a download fires no load event while an error page does: answering a page 200 for stored content but a -fetch (an error once nobody has it, or 503 once the page holds every gateway_fetches slot) for the rest -would tell web pages what the node stores (L6). A bare-CID If-None-Match revalidation, which needs nothing -stored, is answered 304 first. An archive also needs the target's whole DAG before its first byte (its -length is in the headers), and a request whose body nobody reads shouldn't make the node fetch a whole DAG. +Every format is answered only to the user's own GET (handler.limited() false: Sec-Fetch-Site "none", or none sent +by a client whose User-Agent doesn't start with "Mozilla/"), which fetches what isn't stored (an archive's fetch is +cancelled once the client disconnects). A web page's request or a HEAD always gets 409, stored or not, before the +node is asked anything, so no format ever takes a gateway_fetches slot. Neither type is one a browser shows, so a +format's 200 is a download, and an iframe of a download fires no load event while an error page does: answering a +page 200 for stored content but a fetch (an error once nobody has it, or 503 once the page holds every +gateway_fetches slot) for the rest would tell web pages what the node stores (L6). A bare-CID If-None-Match +revalidation, which needs nothing stored, is answered 304 first. An archive also needs the target's whole DAG +before its first byte (its length is in the headers), and a request whose body nobody reads shouldn't make the node +fetch a whole DAG. """ import select import socket diff --git a/rnfs/ids.py b/rnfs/ids.py new file mode 100644 index 0000000..d1aa0eb --- /dev/null +++ b/rnfs/ids.py @@ -0,0 +1,41 @@ +"""16-byte IDs written as 32 hex digits: node IDs, names, RNS identity hashes and LXMF addresses. + +The one parser for these, so a rule about case, brackets or whitespace holds everywhere. Text may have whitespace +around it and, where brackets=True, RNS's (as RNS prints hashes); nothing between the digits: +bytes.fromhex alone would skip whitespace there, giving fewer than 16 bytes from 32 characters. + +Stdlib only: rnfs.config imports it, and the CLI and rnfs.Client load config without RNS. +""" +import re + +HEX16 = re.compile(r"[0-9a-fA-F]{32}") # exactly 32 hex digits, either case +# As rnfs writes them (bytes.hex()). Persisted rows are checked against this, unstripped: a row that fails is +# deleted when loaded, so what they accept mustn't change +HEX16_LOWER = re.compile(r"[0-9a-f]{32}") + + +def _digits(text: str, brackets: bool) -> str: + """text without the whitespace around it and, if brackets, then the < and > around that.""" + stripped = text.strip() + return stripped.strip("<>") if brackets else stripped + + +def is_hex16(text, *, brackets: bool = False, upper: bool = True) -> bool: + """True if parse_hex16 would accept text.""" + if not isinstance(text, str): + return False + return (HEX16 if upper else HEX16_LOWER).fullmatch(_digits(text, brackets)) is not None + + +def parse_hex16(text, what: str, *, brackets: bool = False, upper: bool = True) -> bytes: + """16 bytes from 32 hex digits, with whitespace around allowed and, if brackets, inside that; + upper=False refuses uppercase digits. Raises ValueError naming what (e.g. "node ID") for anything else.""" + if not is_hex16(text, brackets=brackets, upper=upper): + raise ValueError(f"invalid {what} {text!r}: must be 32 hex characters") + return bytes.fromhex(_digits(text, brackets)) + + +def normal_hex16(text: str) -> str: + """text as rnfs writes an ID (bytes.hex()): whitespace and around it removed, lowercase. For + settings Config.validate has checked; it doesn't check text is hex.""" + return _digits(text, True).lower() diff --git a/rnfs/interface.py b/rnfs/interface.py index 741c740..66e9fcf 100644 --- a/rnfs/interface.py +++ b/rnfs/interface.py @@ -20,10 +20,11 @@ import os import threading import time +import warnings from collections import OrderedDict from dataclasses import dataclass, field -from rnfs.config import _HEX_ID +from rnfs import ids from rnfs.content.cid import Cid from rnfs.errors import Closed, NotFound, RnfsError # A job's final states, and the longest wait one request for a job asks for (the API's cap) @@ -32,6 +33,9 @@ Verified, Written) FILE_BUFFER = 64 * 1024 # what a File reads at least per refill: one read() of the node, one GET /api/cat +# The most one refill of a File read sequentially fetches: each sequential refill doubles from FILE_BUFFER up +# to it, so the per-request cost is spread over more bytes; a seek starts again at FILE_BUFFER +READ_AHEAD = 4 * 1024 * 1024 STOP_WAIT = 5.0 # seconds close() waits for the event and job threads to end @@ -41,7 +45,7 @@ def node_hex(node_id) -> str | None: return None if isinstance(node_id, (bytes, bytearray)) and len(node_id) == 16: return bytes(node_id).hex() - if isinstance(node_id, str) and _HEX_ID.fullmatch(node_id): + if isinstance(node_id, str) and ids.HEX16.fullmatch(node_id): return node_id.lower() raise ValueError(f"invalid node ID or name {node_id!r}: 16 bytes or 32 hex characters") @@ -422,8 +426,9 @@ def get(self, target, output, *, from_node=None, raw: bool = False) -> Written: return Written.from_dict({**result, "output": output}) def open(self, target, *, from_node=None, raw: bool = False) -> "File": - """A file to read and seek in, each read fetching only the chunks it covers (sealed files read as - plaintext unless raw). Raises the fetch errors, ValueError (a directory); reads raise the fetch errors.""" + """A file to read and seek in, each read after a seek fetching only the chunks it covers, sequential reads + reading ahead (sealed files read as plaintext unless raw). Raises the fetch errors, ValueError (a + directory); reads raise the fetch errors.""" self._check() text, hint = self._target(target) from_node = self._from(from_node, hint) @@ -600,7 +605,8 @@ def _baseline(self, name: bytes) -> int | None: class _FileReader(io.RawIOBase): - """File's raw stream: each readinto is one read of [position, position + its length) of the target opened. + """File's raw stream: each refill is one read of the target opened from the position, of what readinto asks + for, or (read sequentially) of up to READ_AHEAD bytes ahead, kept for the next readintos. By the target (root and path), not the file's own CID: providers announce roots (provide_strategy), so the file's CID may have none.""" @@ -614,6 +620,9 @@ def __init__(self, owner: Rnfs, target: str, info: Info, from_node: str | None, self.from_node = from_node self.raw = raw self.position = 0 + self._ahead = memoryview(b"") # bytes read from the position on but not handed out yet + self._fetched_to: int | None = None # where the last read ended, while no seek has moved the position + self._window = FILE_BUFFER # the last refill's read-ahead size, doubled while reads are sequential def readable(self) -> bool: """True.""" @@ -624,12 +633,43 @@ def seekable(self) -> bool: return True def readinto(self, buffer) -> int: - """Fill buffer from the position, fetching only what it covers; 0 at the end.""" + """Fill buffer from the position, from what was read ahead or else one read; 0 at the end.""" wanted = min(len(buffer), self.info.size - self.position) if wanted <= 0: return 0 + if not self._ahead: + self._ahead = memoryview(self._read(self._refill_size(wanted))) + count = min(wanted, len(self._ahead)) + buffer[:count] = self._ahead[:count] + self._ahead = self._ahead[count:] + self.position += count + return count + + def readall(self) -> bytes: + """The rest of the file in one read (RawIOBase's would make one per 8 KiB), after what was read ahead.""" + chunks = [bytes(self._ahead)] + self.position += len(self._ahead) + self._ahead = memoryview(b"") + while self.position < self.info.size: + data = self._read(self.info.size - self.position) + chunks.append(data) + self.position += len(data) + return b"".join(chunks) + + def _refill_size(self, wanted: int) -> int: + """How much a refill of wanted bytes reads: just those, unless it continues where the last read ended. + Then the read-ahead doubles (up to READ_AHEAD), so a sequential reader makes few requests, and no more + than about what it has already read is fetched beyond what it reads.""" + if self._fetched_to != self.position: + return wanted + self._window = min(2 * self._window, READ_AHEAD) + return min(max(wanted, self._window), self.info.size - self.position) + + def _read(self, length: int) -> bytes: + """One read of [position, position + length), fetching only what it covers. Raises RnfsError if the + file ends before it.""" self.owner._check() - chunks = self.owner._backend.read(self.name, self.position, wanted, self.raw, self.from_node) + chunks = self.owner._backend.read(self.name, self.position, length, self.raw, self.from_node) try: data = b"".join(chunks) finally: @@ -638,25 +678,21 @@ def readinto(self, buffer) -> int: close() if not data: raise RnfsError(f"{self.name} ended at {self.position} of its {self.info.size} bytes") - buffer[:len(data)] = data - self.position += len(data) - return len(data) - - def readall(self) -> bytes: - """The rest of the file in one read (RawIOBase's would make one per 8 KiB).""" - chunks = [] - while self.position < self.info.size: - buffer = bytearray(self.info.size - self.position) - chunks.append(bytes(buffer[:self.readinto(buffer)])) - return b"".join(chunks) + self._fetched_to = self.position + len(data) + return data def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: - """Move the position (past the end is allowed: reads there return nothing).""" + """Move the position (past the end is allowed: reads there return nothing). A move drops what was read + ahead, and the next refill reads only what it covers.""" base = {io.SEEK_SET: 0, io.SEEK_CUR: self.position, io.SEEK_END: self.info.size}.get(whence) if base is None: raise ValueError(f"invalid whence {whence!r}") if base + offset < 0: raise ValueError("negative seek position") + if base + offset != self.position: + self._ahead = memoryview(b"") + self._fetched_to = None + self._window = FILE_BUFFER self.position = base + offset return self.position @@ -664,10 +700,16 @@ def tell(self) -> int: """The position.""" return self.position + def close(self): + """Close the stream, dropping what was read ahead.""" + self._ahead = memoryview(b"") + super().close() + class File(io.BufferedReader): """A file in rnfs, from open(): read, readinto, readline, iteration, seek and tell, buffered by FILE_BUFFER - bytes, each refill fetching only the chunks it covers. Reads go by the target opened (an rnfs: name URI + bytes. A refill after open or a seek fetches only the chunks it covers; each refill continuing the last reads + ahead twice as far, up to READ_AHEAD. Reads go by the target opened (an rnfs: name URI resolved once, at open; a contact nick is expanded by the node, which reuses its CID for 60 s). cid is the file's own CID, size its size, info its Info, as open() found them. Not thread-safe, like any file object.""" @@ -725,9 +767,13 @@ def to_dict(self) -> dict: def publish(self, target) -> Record: """Point the name at target (a path in it resolved locally) and announce it; the new record. Raises - NotFound (a path not stored here), ValueError, RnfsError (the highest sequence reached).""" + NotFound (a path not stored here), ValueError, RnfsError (the highest sequence reached). In private mode + the node's warning that the announce is visible outside the private network is raised as a + RuntimeWarning (warnings.warn), as the CLI prints it.""" self._owner._check() result = self._owner._backend.name_publish(self.alias, self._owner._target(target)[0]) + if result.get("warning"): + warnings.warn(result["warning"], RuntimeWarning, stacklevel=2) return Record.from_dict({**result, "age": 0.0}) def export_key(self) -> bytes: @@ -783,7 +829,7 @@ def resolve(self, name) -> Record: """The newest record of name (16 bytes, 32 hex characters, a Name or a name contact's nick), after asking the network; persisted, so it never goes back. Raises NotFound (unresolved), ValueError.""" self._owner._check() - text = name if isinstance(name, str) and not _HEX_ID.fullmatch(name) else node_hex(name_bytes(name)) + text = name if isinstance(name, str) and not ids.HEX16.fullmatch(name) else node_hex(name_bytes(name)) return Record.from_dict(self._owner._backend.name_resolve(text)) def watch(self, name, callback) -> Watch: diff --git a/rnfs/log.py b/rnfs/log.py index 8aa9fe3..6c4fcb6 100644 --- a/rnfs/log.py +++ b/rnfs/log.py @@ -3,17 +3,19 @@ With `debug_log` on, every message also goes to the debug log file (rnfs/debuglog.py), whatever RNS's loglevel. """ import threading +import time +from typing import Callable import RNS +# Re-exported so callers only need this module for logging +from RNS import LOG_DEBUG, LOG_ERROR, LOG_INFO, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING # noqa: F401 + from rnfs.fsutil import printable # Warnings a result keeps (a write-out skipping 50,000 entries mustn't make a 16 MB response or saved job) MAX_WARNINGS = 100 -# Re-exported so callers only need this module for logging -from RNS import LOG_DEBUG, LOG_ERROR, LOG_INFO, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING # noqa: F401 - # sink(level, text) given every message as well, at any level: the debug log's (set_debug_sink), or None _debug_sink = None # Set while an rnfs message is inside RNS.log, so the debug log's copy of RNS's own output can skip it @@ -48,6 +50,40 @@ def passing_through() -> bool: return getattr(_passing, "active", False) +class ThrottledLog: + """Log lines about one kind of event at most one every interval seconds, counting the events left out, so + whatever causes them (often another node) can't flood the log. Thread-safe. + + Called inside an except block, a line logged at LOG_WARNING or more severe carries the exception's traceback + to the debug log, like any other. + """ + + def __init__(self, interval: float = 60.0, clock: Callable[[], float] = time.monotonic): + """Nothing logged yet.""" + self.interval = interval + self.clock = clock + self.lock = threading.Lock() + self.logged_at: float | None = None + self.skipped = 0 + + def due(self) -> int | None: + """For an event now: how many were left out since the last line, if a line is due (the next is then due + interval seconds from now); None, counting this one as left out, if not.""" + with self.lock: + now = self.clock() + if self.logged_at is not None and now - self.logged_at < self.interval: + self.skipped += 1 + return None + skipped, self.skipped, self.logged_at = self.skipped, 0, now + return skipped + + def __call__(self, message: str, level: int): + """Log message at level, with how many were left out since the last line, if a line is due.""" + skipped = self.due() + if skipped is not None: + log(message + (f" ({skipped} more since the last)" if skipped else ""), level) + + def node_name(node_id: bytes | None) -> str: """Printable form of a node ID, e.g. "<9e0a158a...>", or a placeholder for an unknown node.""" return RNS.prettyhexrep(node_id) if node_id is not None else "an unidentified node" diff --git a/rnfs/mesh.py b/rnfs/mesh.py index da259bf..66e0106 100644 --- a/rnfs/mesh.py +++ b/rnfs/mesh.py @@ -24,6 +24,11 @@ MAX_PEERS = 1000 # peers tracked by Traffic; the least recently active is forgotten beyond this MAX_INTERFACES = 64 # interfaces tracked by Traffic, likewise STATS_CACHE = 30.0 # seconds a shared instance's interface stats are reused for bitrate lookups +# Seconds a shared instance's answer for a node's next-hop interface is reused, and how many nodes' answers are +# kept: each answer is an RPC to the instance, about 41 ms over TCP (macOS, Windows), asked per open link and +# per provider at the start of every fetch, and per peer by `rnfs peers` +PATH_CACHE = 15.0 +MAX_CACHED_PATHS = 1024 PATHFINDER_M = 128 # RNS.Transport.PATHFINDER_M: what hops_to returns for an unknown path # Why a /blocks request was refused by an airtime control (Traffic.refused keys) @@ -69,8 +74,9 @@ class RnsPaths: """path_info backed by RNS.Transport (or, attached to a shared instance, by its RPC). Thread-safe. In a shared-instance client, next_hop_interface is the local interface to the shared instance, so the - interface name comes from the instance (Reticulum.get_next_hop_if_name) and its bitrate from the - instance's interface stats, cached for STATS_CACHE. + interface name comes from the instance (Reticulum.get_next_hop_if_name, cached per node for PATH_CACHE) and + its bitrate from the instance's interface stats, cached for STATS_CACHE. Whether there is a path, and its + hops, are always read fresh (they are local); only the interface may be up to PATH_CACHE old. """ def __init__(self, reticulum=None, clock: Callable[[], float] = time.monotonic): @@ -80,6 +86,13 @@ def __init__(self, reticulum=None, clock: Callable[[], float] = time.monotonic): self._stats: dict[str, tuple[str, float | None]] = {} # full name -> (short name, bitrate) self._stats_at = -math.inf self._lock = threading.Lock() + # node ID -> (when asked, the instance's next-hop interface name), oldest first: an entry is moved to + # the end when stored, so expired entries and those evicted beyond MAX_CACHED_PATHS are at the front + self._names: OrderedDict[bytes, tuple[float, str | None]] = OrderedDict() + self._names_lock = threading.Lock() + from rnfs.log import ThrottledLog # here, not at module level: it loads RNS + # RNS failing to answer is a bug or an RNS change, asked about per provider and per peer: a line a minute + self._failure_log = ThrottledLog() def info(self, node_id: bytes) -> PathInfo: """Hops and next-hop interface of the path to node_id; NO_PATH if RNS has none or fails.""" @@ -89,16 +102,78 @@ def info(self, node_id: bytes) -> PathInfo: return NO_PATH hops = RNS.Transport.hops_to(node_id) hops = hops if isinstance(hops, int) and 0 <= hops < PATHFINDER_M else None - reticulum = self.reticulum or RNS.Reticulum.get_instance() - if reticulum is not None and getattr(reticulum, "is_connected_to_shared_instance", False): - name, bitrate = self._shared_interface(reticulum, reticulum.get_next_hop_if_name(node_id)) + reticulum = self._shared_instance() + if reticulum is not None: + name, bitrate = self._shared_interface(reticulum, self._next_hop_name(reticulum, node_id)) else: interface = RNS.Transport.next_hop_interface(node_id) name, bitrate = interface_label(interface), _bitrate(getattr(interface, "bitrate", None)) return PathInfo(hops, name, bitrate) - except Exception: + except Exception as e: # noqa: BLE001 - RNS internals: no path known if they fail; logged + # Inside the except, so the debug log has the traceback + self._failure_log(f"couldn't read the path to {RNS.prettyhexrep(node_id)} from RNS: {e!r}", RNS.LOG_WARNING) return NO_PATH + def infos(self, node_ids: list[bytes]) -> dict[bytes, PathInfo]: + """info() of each of node_ids. Attached to a shared instance, the next-hop interfaces not cached are + read from one path table RPC (Reticulum.get_path_table) instead of one RPC per node.""" + import RNS + try: + reticulum = self._shared_instance() + if reticulum is not None: + now = self.clock() + with self._names_lock: + missing = {node_id for node_id in node_ids if not self._cached(node_id, now)} + missing = {node_id for node_id in missing if RNS.Transport.has_path(node_id)} + # One RPC either way for a single node: its own answer is the smaller one + if len(missing) > 1: + table = reticulum.get_path_table() or [] + now = self.clock() + for entry in table: + node_id = entry.get("hash") if isinstance(entry, dict) else None + if node_id in missing: + # The "interface" here is str() of the same path table entry's interface that + # get_next_hop_if_name answers with; a node not in the table is asked on its own + self._remember(node_id, entry.get("interface"), now) + except Exception: # noqa: BLE001 - RNS internals: info() asks node by node if the table fails + pass + return {node_id: self.info(node_id) for node_id in node_ids} + + def _shared_instance(self): + """The Reticulum instance if it is attached to a shared instance (RPC answers paths), else None.""" + import RNS + reticulum = self.reticulum or RNS.Reticulum.get_instance() + if reticulum is not None and getattr(reticulum, "is_connected_to_shared_instance", False): + return reticulum + return None + + def _next_hop_name(self, reticulum, node_id: bytes): + """The shared instance's name for node_id's next-hop interface, asked at most once per PATH_CACHE.""" + with self._names_lock: + cached = self._cached(node_id, self.clock()) + if cached is not None: + return cached[1] + # Asked outside the lock: an RPC can take 41 ms, and other nodes' cached answers needn't wait for it + name = reticulum.get_next_hop_if_name(node_id) + self._remember(node_id, name, self.clock()) + return name + + def _cached(self, node_id: bytes, now: float) -> tuple[float, str | None] | None: + """node_id's cached (when, name) if younger than PATH_CACHE, else None. Caller holds _names_lock.""" + cached = self._names.get(node_id) + return cached if cached is not None and now - cached[0] <= PATH_CACHE else None + + def _remember(self, node_id: bytes, name, now: float): + """Cache the instance's next-hop interface name for node_id, dropping expired and excess entries.""" + with self._names_lock: + self._names[node_id] = (now, name) + self._names.move_to_end(node_id) + while self._names: + oldest = next(iter(self._names.values())) + if len(self._names) <= MAX_CACHED_PATHS and now - oldest[0] <= PATH_CACHE: + break + self._names.popitem(last=False) + def _shared_interface(self, reticulum, full_name) -> tuple[str | None, float | None]: """(short name, bitrate) of the shared instance's interface called full_name (str(interface)).""" if not isinstance(full_name, str) or not full_name or full_name == "None": @@ -238,15 +313,6 @@ def available(self) -> float: self._refill() return self.tokens - def take(self, nbytes: int) -> bool: - """Spend nbytes if allowed (see the class). False, spending nothing, if not.""" - with self.lock: - self._refill() - if self.tokens >= nbytes or self.tokens >= self.capacity: - self.tokens -= nbytes - return True - return False - def give_back(self, nbytes: int): """Return bytes spent on a response that wasn't sent after all.""" with self.lock: @@ -308,10 +374,6 @@ def limit(self, limit: int, committed: int = 0) -> int: return limit return max(0, min(limit, int(available - committed) - RESPONSE_OVERHEAD)) - def spend(self, nbytes: int) -> bool: - """Charge a response of nbytes to the upload budget. False if it must be refused.""" - return self.bucket is None or self.bucket.take(nbytes) - def reserve(self, nbytes: int, reservation: int, committed: int = 0) -> bool: """Admit a response of nbytes, charging reservation of it now (the rest is charged as it is sent). False if it must be refused; committed is what responses in flight are still committed to send.""" diff --git a/rnfs/names.py b/rnfs/names.py index 863b6c1..f33cea3 100644 --- a/rnfs/names.py +++ b/rnfs/names.py @@ -5,7 +5,9 @@ any validly signed announce, including a replayed old one, so the highest sequence seen is persisted for every name this node publishes or resolves for the user, and lower sequences are never accepted from any source. RNS also remembers a replayed announce as just heard, so an announce shows its record current only up to -ANNOUNCE_VALIDITY announce intervals after it was signed (_announce_heard): past that, resolves ask keepers. +ANNOUNCE_VALIDITY announce intervals after it was signed (_announce_heard): past that, resolves ask keepers. One +older than the announce RNS took the path from, or carrying another record, is a suspected replay, as old as it is +(RnsNetwork.recalled_emission), and resolves ask keepers at once. Names resolved for the gateway or LXMF auto-fetch keep their records in a bounded table, where announces of names this node never resolved can't push them out, and their highest sequence in a bounded persisted floor, saved within seconds of a resolve or an announce raising it, where names resolved in the last FLOOR_KEEP_AGE are forgotten @@ -31,14 +33,15 @@ import threading import time from collections import OrderedDict +from collections.abc import Mapping from dataclasses import dataclass import RNS from RNS.vendor import umsgpack -from rnfs import fsutil, statedb -from rnfs.config import _HEX_ID, Config # _HEX_ID: exactly 32 hex digits, as identity lists are checked -from rnfs.constants import APP_NAME, ID_LENGTH, NAME_ASPECT +from rnfs import fsutil, ids, statedb +from rnfs.config import Config +from rnfs.constants import APP_NAME, ID_LENGTH, MISSED_ANNOUNCES, NAME_ASPECT from rnfs.content.cid import Cid from rnfs.errors import NotFound, RnfsError from rnfs.log import LOG_ERROR, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING, log @@ -58,6 +61,10 @@ FLOOR_SAVE_INTERVAL = 5.0 # seconds: a resolve raising the floor saves it at once, or this long after the last save FLOOR_TOUCH_SAVE = 3600.0 # a resolve that only renews a floor entry's time marks it for the next save after this PIN_SOURCE_PREFIX = "name:" +# Private mode restricts rnfs's links, not RNS announces: a name's record rides in its announce, which RNS +# propagates to every node that hears it (specification sections 3 and 10) +PRIVATE_NAME_WARNING = ("private mode doesn't hide names: this name's announce, with its sequence and target CID, " + "is visible outside the private network, though its content is served to members only") _ALIAS = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}") _PARTIAL = re.compile(re.escape(fsutil.PARTIAL_PREFIX) + r"[0-9a-f]{16}") # a key write's temp file (write_atomic) _DECIMAL = re.compile(r"-?[0-9]+") # a sequence column (decimal text: sequences don't fit SQLite's integers) @@ -65,9 +72,11 @@ _POLL_INTERVAL = 0.25 # how often a refresh checks whether RNS has heard a path response # An announce shows its record current for this many announce_intervals after it was emitted, however recently # it was heard: RNS remembers a replayed announce as just heard (_announce_heard) -ANNOUNCE_VALIDITY = 2 +ANNOUNCE_VALIDITY = MISSED_ANNOUNCES _PATH_RANDOM_BLOBS = 4 # RNS's IDX_PT_RANDBLOBS: where a path table entry holds its announces' random blobs +_PATH_PACKET = 6 # RNS's IDX_PT_PACKET: where a path table entry holds the packet hash of the path's announce _MAX_LOOP_WAIT = 60 # the announce loop wakes at least this often, so new schedules are picked up +REFRESHED_SIZE = 4096 # how many names' last network refresh a resolve remembers (E1), least recent forgotten first # Records and names @@ -85,7 +94,7 @@ def parse_record(app_data) -> tuple[int, Cid] | None: app_data = bytes(app_data) try: value = umsgpack.unpackb(app_data) - except Exception: + except Exception: # noqa: BLE001 - umsgpack raises several types for hostile bytes return None if not isinstance(value, list) or len(value) != 2: return None @@ -108,13 +117,7 @@ def parse_record(app_data) -> tuple[int, Cid] | None: def parse_name(text: str) -> bytes: """A name from its 32 hex character form, with or without RNS's . Raises ValueError.""" - if not isinstance(text, str): - raise ValueError("name must be a string") - stripped = text.strip().strip("<>") - # Hex digits only: bytes.fromhex alone would skip whitespace between them, giving a shorter name - if not _HEX_ID.fullmatch(stripped): - raise ValueError(f"name must be {NAME_LENGTH * 2} hex characters: {text!r}") - return bytes.fromhex(stripped) + return ids.parse_hex16(text, "name", brackets=True) def check_alias(alias) -> str: @@ -269,7 +272,7 @@ def import_floor(value: dict, tx) -> int: first, replace the name_floor rows, each checked as rnfs 0.1.4 loaded it: one that doesn't parse is skipped (and counted in a warning), one without a valid resolve time counts as resolved long ago, and a name listed twice keeps the higher sequence, at its later place. Held names, future times and the floor's bound are left - to the load (_load_floor). Returns the number of entries imported.""" + to the load (_NameFloor.load). Returns the number of entries imported.""" floor: OrderedDict[bytes, tuple[int, float]] = OrderedDict() skipped = 0 for entry in value["floor"]: @@ -340,8 +343,58 @@ def emitted_time(self, name: bytes) -> float | None: return None emitted = RNS.Transport.timebase_from_random_blobs(entry[_PATH_RANDOM_BLOBS]) return float(emitted) if type(emitted) is int and emitted > 0 else None - except Exception: + except Exception: # noqa: BLE001 - RNS internals: unknown if they fail + return None + + def recalled_emission(self, name: bytes, app_data: bytes) -> tuple[float | None, bool]: + """(emitted, takes_path) for the announce RNS last stored for name, whose app_data (recall_app_data) is + app_data: when that announce was emitted (its owner's clock, signed), or None if RNS no longer has it; and + False if RNS shows it isn't the record of the announce RNS took name's path from (a replay, usually of an + older announce, which RNS stores all the same). + + RNS internals (Identity.known_destinations[hash][1], the packet hash Identity.remember stores; + Transport.path_table[hash][IDX_PT_PACKET], the packet hash of the path's announce; Transport's announce + cache, written only for an announce that takes the path, read with Transport.get_cached_packet). The path's + announce is read first: if it carries app_data, so does the path (an announce heard again, or repeated in + a path response, which RNS hashes with its context, gets another packet hash and is never cached). Else + the stored one is read for its own emission time. Without a path, or with neither announce cached (a + shared instance's client whose cache isn't rnsd's), nothing shows a replay: (None, True), or the path's + emission time if the stored announce is the path's. If an internal changes in another RNS version this + returns (None, True) and callers go by emitted_time alone. + """ + try: + known = RNS.Identity.known_destinations.get(name) + path = RNS.Transport.path_table.get(name) + # known[3] is another record if RNS stored a newer announce since app_data was read: judge that one later + if not known or not path or known[3] != app_data: + return None, True + stored, path_packet = known[1], path[_PATH_PACKET] + in_path = self._cached_announce(name, path_packet) + if in_path is not None and in_path[1] == app_data: + return in_path[0], True + own = in_path if stored == path_packet else self._cached_announce(name, stored) + emitted = own[0] if own is not None and own[1] == app_data else None + if in_path is not None: + return emitted, False # the path's announce carries another record + if stored == path_packet: + return self.emitted_time(name), True + return emitted, True + except Exception: # noqa: BLE001 - RNS internals: unknown if they fail + return None, True + + @staticmethod + def _cached_announce(name: bytes, packet_hash) -> tuple[float, bytes] | None: + """(emission time, app_data) of name's announce with packet_hash in RNS's announce cache, or None if it + isn't there (or isn't an announce of name). Not verified again: RNS caches only announces it validated.""" + packet = RNS.Transport.get_cached_packet(packet_hash, packet_type="announce") + if packet is None or packet.unpack() is False or packet.packet_type != RNS.Packet.ANNOUNCE \ + or packet.destination_hash != name: return None + data = bytes(packet.data) + blob = RNS.Identity.KEYSIZE // 8 + RNS.Identity.NAME_HASH_LENGTH // 8 # the random blob: emission time + ratchet = RNS.Identity.RATCHETSIZE // 8 if packet.context_flag == RNS.Packet.FLAG_SET else 0 + app_data = data[blob + 10 + ratchet + RNS.Identity.SIGLENGTH // 8:] + return float(RNS.Transport.timebase_from_random_blob(data[blob:blob + 10])), app_data def heard_time(self, name: bytes) -> float | None: """When RNS last stored a validly signed announce for name (wall clock), or None. @@ -353,7 +406,7 @@ def heard_time(self, name: bytes) -> float | None: try: entry = RNS.Identity.known_destinations.get(name) return float(entry[0]) if entry else None - except Exception: + except Exception: # noqa: BLE001 - RNS internals: unknown if they fail return None @@ -378,9 +431,232 @@ def received_announce(self, destination_hash, announced_identity, app_data, anno self._on_announce(destination_hash, app_data, announce_packet_hash) +# The floor + + +class _NameFloor(Mapping): + """The highest sequence of each name resolved without persist (gateway, LXMF auto-fetch), and when it was last + resolved (wall clock), least recently resolved first, persisted in the name_floor table. One value per name, + (sequence, resolved, used), used being its row's place in the table; read as a mapping of name -> (sequence, + resolved). + + At most FLOOR_FACTOR × name_table_size names (size()), more (up to FLOOR_HARD_FACTOR times that) while those + resolved within FLOOR_KEEP_AGE need the room (trim). Saved, so such a resolve never goes back after table churn + or a restart: at once, or FLOOR_SAVE_INTERVAL after the last save (a timer), writing only the rows changed. + + Changed only under the manager's lock, which every caller but save() holds; save() snapshots the rows under it + and writes without it. + """ + + def __init__(self, db, lock, clock, monotonic, size): + """An empty floor over the state database db (load() reads it); lock is the manager's, clock and monotonic + its clocks, and size() how many names the floor holds before trimming.""" + self._db = db + self._lock = lock + self._clock = clock # resolve times: persisted, so wall clock + self._monotonic = monotonic # save pacing + self._size = size + self._entries: OrderedDict[bytes, tuple[int, float, int]] = OrderedDict() + self._next_used = 0 # above every used value given out + self._changed: set[bytes] = set() # names whose row the next save writes (or deletes) + self._dirty = False # a save is due + self._saved_at = float("-inf") # monotonic time of the last save + self._timer: threading.Timer | None = None # a save waiting for FLOOR_SAVE_INTERVAL to pass + self._closed = False # close() has run: no timer is started + # Saves write without the lock, one at a time, so an older snapshot never lands last. Taken before the + # manager's lock, never while holding it + self._save_lock = threading.Lock() + + # Reading: name -> (sequence, resolved) + + def __getitem__(self, name: bytes) -> tuple[int, float]: + """name's (sequence, resolve time); KeyError if it has no entry.""" + sequence, resolved, _ = self._entries[name] + return sequence, resolved + + def __iter__(self): + """The names, least recently resolved first.""" + return iter(self._entries) + + def __len__(self) -> int: + """How many names the floor holds.""" + return len(self._entries) + + def __contains__(self, name) -> bool: + """True if name has an entry.""" + return name in self._entries + + # Changes; callers hold the manager's lock + + def raise_to(self, name: bytes, sequence: int): + """A higher record was accepted for name: raise its entry (if it has one) to sequence, keeping its place and + resolve time, and save it within FLOOR_SAVE_INTERVAL (by the timer, since callers hold the lock, some on + RNS's thread).""" + entry = self._entries.get(name) + if entry is not None and sequence > entry[0]: + self._entries[name] = (sequence, entry[1], entry[2]) + self._changed.add(name) + self._dirty = True + self.save_soon(inline=False) + + def drop(self, name: bytes): + """Forget name (held for the user now: its held record is its floor); the next save deletes its row.""" + if self._entries.pop(name, None) is not None: + self._changed.add(name) + self._dirty = True + + def resolved(self, name: bytes, sequence: int) -> bool: + """A resolve without persist returned name's record at sequence: it becomes the most recently resolved, and + its sequence the floor for later resolves (trim says which are forgotten). A floor raised (or a new one) is + saved at once, or FLOOR_SAVE_INTERVAL after the last save, so a crash soon after loses little. True if the + caller is to save it now (save()), once it has released the lock.""" + now = self._clock() + previous = self._entries.pop(name, None) + # Last, with a new used value: the most recently resolved + self._entries[name] = (sequence if previous is None else max(previous[0], sequence), now, self._next_used) + self._next_used += 1 + self._changed.add(name) # written by the next save, even if this change doesn't call for one + trimmed = self.trim() + if previous is None or self._entries[name][0] != previous[0] or trimmed: + self._dirty = True + return self.save_soon() + if not 0 <= now - previous[1] < FLOOR_TOUCH_SAVE: + self._dirty = True # only its time renewed: saved with the next change or by announce_due + return False + + def trim(self) -> bool: + """Forget the least recently resolved names over size(), but not those resolved within FLOOR_KEEP_AGE unless + the floor holds FLOOR_HARD_FACTOR times that. True if any went.""" + size = self._size() + now = self._clock() + trimmed = False + while len(self._entries) > size: + _, at, _ = next(iter(self._entries.values())) + if len(self._entries) <= FLOOR_HARD_FACTOR * size and now - at < FLOOR_KEEP_AGE: + break + name, _ = self._entries.popitem(last=False) + self._changed.add(name) + trimmed = True + return trimmed + + def save_soon(self, inline: bool = True) -> bool: + """True if the floor is to be saved now (the last save was FLOOR_SAVE_INTERVAL ago), by the caller once it + has released the lock; else it is saved once it is (a timer). Without inline, a save due now is the + timer's too, started at once.""" + wait = self._saved_at + FLOOR_SAVE_INTERVAL - self._monotonic() + if wait <= 0 and inline: + return True + if self._timer is None and not self._closed: + timer = threading.Timer(max(0.0, wait), self._save_later) + timer.daemon = True + self._timer = timer + timer.start() + return False + + def cancel_timer(self): + """Cancel a save waiting for its time (stop() saves at once instead).""" + if self._timer is not None: + self._timer.cancel() + self._timer = None + + def close(self): + """Start no timer from now on, and cancel a waiting one.""" + self._closed = True + self.cancel_timer() + + # Loading and saving + + def load(self, held): + """Read the name_floor rows, least recently resolved first. A row that doesn't parse is logged and deleted + (the floor only bounds rollback of gateway and auto-fetch resolves), as are those of names in held (resolved + for the user since: their held record is their floor) and those trim drops. A resolve time in the future + (the clock was corrected since) counts as now, and is written so by the next save that has a change to + write.""" + now = self._clock() + dropped = [] + for name, sequence, resolved, used in self._db.query( + "SELECT name, sequence, resolved, used FROM name_floor ORDER BY used"): + try: + _check_name(name) + sequence = _sequence(sequence) + if not 0 <= sequence <= MAX_SEQUENCE: + raise ValueError(f"sequence {sequence} out of range") + if type(resolved) not in (int, float) or not math.isfinite(resolved): + raise ValueError(f"invalid resolve time {resolved!r}") + if type(used) is not int or not 0 <= used < _MAX_USED: + raise ValueError(f"invalid used value {used!r}") + except (TypeError, ValueError) as e: + log(f"dropping an invalid name floor entry {name!r}: {e}", LOG_WARNING) + dropped.append(name) + continue + if name in held: + dropped.append(name) + continue + self._entries[name] = (sequence, min(float(resolved), now), used) + self._next_used = max(self._next_used, used + 1) + if resolved > now: + self._changed.add(name) + self.trim() + trimmed = [name for name in self._changed if name not in self._entries] + self._changed.difference_update(trimmed) + if dropped or trimmed: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM name_floor WHERE name = ?", [(name,) for name in dropped + trimmed]) + + def _snapshot(self) -> tuple[set[bytes], list[tuple], list[tuple]]: + """(the names changed, the rows to write for them, the rows to delete), taken as saved. The lock held.""" + changed, self._changed = self._changed, set() + rows = [(name, str(self._entries[name][0]), self._entries[name][1], self._entries[name][2]) + for name in changed if name in self._entries] + gone = [(name,) for name in changed if name not in self._entries] + self._dirty = False + return changed, rows, gone + + def save(self): + """Write the entries changed (or forgotten) since the last save, if a save is due, with synchronous=NORMAL. + Saved by a resolve that raised it (at most every FLOOR_SAVE_INTERVAL, since it holds thousands of names), + the timer, announce_due (at least every minute while running) and stop(). + + Called without the lock: only the rows are made under it, so resolves and announces needn't wait for the + write. The save counts from when they were made; a failed one is retried by the next.""" + with self._save_lock: + with self._lock: + if not self._dirty: + return + changed, rows, gone = self._snapshot() + saved_at, self._saved_at = self._saved_at, self._monotonic() + saved = False + try: + try: + with self._db.transaction() as tx: + tx.executemany("DELETE FROM name_floor WHERE name = ?", gone) + tx.executemany("INSERT OR REPLACE INTO name_floor (name, sequence, resolved, used) " + "VALUES (?, ?, ?, ?)", rows) + saved = True + except OSError as e: + log(f"could not save the name floor: {e}", LOG_WARNING) + finally: + if not saved: + with self._lock: + self._changed |= changed + self._dirty = True + self._saved_at = saved_at + + def _save_later(self): + """The timer's save.""" + with self._lock: + self._timer = None + self.save() + + # Names +def _wait_event(event: threading.Event, timeout: float) -> bool: + """Wait up to timeout seconds for event to be set: True if it was.""" + return event.wait(timeout) + + class NameManager: """This node's name keys and records, and the records it has heard. Thread-safe. @@ -389,13 +665,17 @@ class NameManager: shutdown. """ - def __init__(self, config: Config, pins, network=None, clock=time.time, monotonic=time.monotonic): - """Names under config.names_dir and config.state_dir; pins has pin(cid, source) and unpin(cid, source).""" + def __init__(self, config: Config, pins, network=None, clock=time.time, monotonic=time.monotonic, + wait=_wait_event): + """Names under config.names_dir and config.state_dir; pins has pin(cid, source) and unpin(cid, source). + wait(event, timeout) is how a refresh waits for an answer (_wait_event; tests pass one that doesn't + block).""" self.config = config self._pins = pins self._network = network if network is not None else RnsNetwork() self._clock = clock # wall clock: heard times are persisted across restarts self._monotonic = monotonic # announce schedule + self._wait = wait self._lock = threading.RLock() self._keys: dict[str, RNS.Identity] = {} @@ -415,26 +695,16 @@ def __init__(self, config: Config, pins, network=None, clock=time.time, monotoni # name this node isn't resolving and doesn't know may only take the place of one of these, so a flood of # new names can't push out the names it resolved self._heard_only: OrderedDict[bytes, None] = OrderedDict() - # Highest sequence of each name resolved without persist (gateway, LXMF auto-fetch), and when it was last - # resolved (wall clock), least recently resolved first; at most FLOOR_FACTOR × name_table_size, more (up to - # FLOOR_HARD_FACTOR times that) while those resolved within FLOOR_KEEP_AGE need the room. Saved, so such a - # resolve never goes back after table churn or a restart - self._floor: OrderedDict[bytes, tuple[int, float]] = OrderedDict() - self._floor_used: dict[bytes, int] = {} # each floor name's used value, its place in the name_floor table - self._floor_next_used = 0 # above every used value given out - self._floor_changed: set[bytes] = set() # floor names whose row the next save writes (or deletes) - self._floor_dirty = False # a floor save is due - self._floor_saved_at = float("-inf") # monotonic time of the last floor save - self._floor_timer: threading.Timer | None = None # a save waiting for FLOOR_SAVE_INTERVAL to pass - self._closed = False # close() has run: no floor timer is started - # Floor saves write without self._lock, one at a time, so an older snapshot never lands last. Taken before - # self._lock, never while holding it - self._floor_save_lock = threading.Lock() self._held_dirty = False # held records changed without a save (heard again, or a save that failed) self._destinations: dict[str, object] = {} self._next_announce: dict[str, float] = {} # alias -> monotonic time self._waiters: dict[bytes, list[threading.Event]] = {} + # A resolve's network refresh of each name (E1): when it last finished (monotonic) and whether a record was + # held after it, least recent first, at most REFRESHED_SIZE; and the refreshes running, which other + # resolves of the name wait for rather than refresh again + self._refreshed: OrderedDict[bytes, tuple[float, bool]] = OrderedDict() + self._refreshing: dict[bytes, threading.Event] = {} self._handler = NameAnnounceHandler(self._heard) self._started = False # The keepers feature (rnfs.features.keepers), if loaded: told of heard and published records, and asked @@ -460,10 +730,12 @@ def __init__(self, config: Config, pins, network=None, clock=time.time, monotoni self._remove_old_temps() self._db = statedb.connect(config.state_dir) + # The highest sequence of each name resolved without persist (gateway, LXMF auto-fetch), bounded and saved + self._floor = _NameFloor(self._db, self._lock, clock, monotonic, self._floor_size) try: self._load_keys() self._load_state() - self._load_floor() + self._floor.load(self._held) except BaseException: self._db.close() raise @@ -625,7 +897,8 @@ def list(self) -> list[dict]: # Publishing def publish(self, alias: str, cid: Cid) -> dict: - """Point alias's name at cid, pin it, announce it. Returns {"name", "cid", "sequence"}.""" + """Point alias's name at cid, pin it, announce it. Returns {"name", "cid", "sequence"}, and in private mode + a "warning" (PRIVATE_NAME_WARNING): the announce reaches outsiders all the same.""" if not isinstance(cid, Cid): raise ValueError("publish needs a CID") with self._lock: @@ -635,10 +908,7 @@ def publish(self, alias: str, cid: Cid) -> dict: # A freshly imported key: its previous home may have published sequences we've never heard, which # keepers may hold after its announces have expired (section 9: above the highest it can resolve) self._offer_kept(name) - self._refresh(name) - if self.keepers is not None: - for sequence, kept_cid, heard in self.keepers.query(name): - self.offer(name, sequence, kept_cid, heard) + self._refresh_name(name, ask="always") with self._lock: if self.name(alias) != name: # NotFound if removed while refreshing @@ -679,7 +949,10 @@ def publish(self, alias: str, cid: Cid) -> dict: if self.keepers is not None: # This node keeps its own names, so they resolve through it while the owner's announce is gone self.keepers.published(name, identity, encode_record(sequence, cid)) - return {"name": name.hex(), "cid": str(cid), "sequence": sequence} + result = {"name": name.hex(), "cid": str(cid), "sequence": sequence} + if self.config.private: + result["warning"] = PRIVATE_NAME_WARNING + return result def start(self): """Register the announce handler, re-create every local name's destination and announce published ones.""" @@ -709,19 +982,14 @@ def stop(self): for events in self._waiters.values(): for event in events: event.set() - if self._floor_timer is not None: - self._floor_timer.cancel() - self._floor_timer = None + self._floor.cancel_timer() self._save() - self._save_floor() + self._floor.save() def close(self): """Release the state database, once stop() has saved everything (idempotent).""" with self._lock: - self._closed = True - if self._floor_timer is not None: - self._floor_timer.cancel() - self._floor_timer = None + self._floor.close() self._db.close() def announce_due(self) -> float: @@ -737,7 +1005,7 @@ def announce_due(self) -> float: wait = self.config.announce_interval else: wait = max(0.0, min(self._next_announce.values()) - now) - self._save_floor() + self._floor.save() return wait def run(self, stop_event: threading.Event): @@ -745,7 +1013,7 @@ def run(self, stop_event: threading.Event): while not stop_event.is_set(): try: delay = self.announce_due() - except Exception as e: + except Exception as e: # noqa: BLE001 - the announce thread keeps running; logged log(f"name re-announce failed: {e}", LOG_ERROR) delay = _MAX_LOOP_WAIT stop_event.wait(min(delay, _MAX_LOOP_WAIT)) @@ -799,8 +1067,9 @@ def resolve(self, name: bytes, persist: bool, max_age: float = 0.0) -> dict | No persist is True for the CLI and API (the highest sequence is then kept forever) and False for the gateway and LXMF auto-fetch (the record goes only into the bounded table, and its sequence into the bounded floor). - The network refresh, which can take path_timeout, is skipped for this node's own names and when the held - record was heard less than max_age seconds ago. + The network refresh, which can take path_timeout and then a keeper query, is skipped for this node's own + names, when the held record was heard less than max_age seconds ago, and when a refresh of the name ended + less than max_age ago (_shared_refresh). A resolve finding a refresh of the name running waits for it. """ name = _check_name(name) self._offer_kept(name) @@ -812,16 +1081,7 @@ def resolve(self, name: bytes, persist: bool, max_age: float = 0.0) -> dict | No # This node's own names need no refresh, unless imported and not yet published here local = alias is not None and not self._published[alias].unsettled if not local and not fresh: - started = self._clock() - self._refresh(name) - if self.keepers is not None: - with self._lock: - self._absorb_recalled(name) - record = self._record(name, touch=False) - # Nothing heard during the refresh: the owner and the paths may be gone, so ask keepers - if record is None or record.heard < started or self.keepers.always_ask(): - for sequence, cid, heard in self.keepers.query(name): - self.offer(name, sequence, cid, heard) + self._shared_refresh(name, max_age) # Again: the bounded table may have evicted it meanwhile, letting a replayed announce in self._offer_kept(name) @@ -840,9 +1100,85 @@ def resolve(self, name: bytes, persist: bool, max_age: float = 0.0) -> dict | No result = {"name": name.hex(), "cid": str(record.cid), "sequence": record.sequence, "age": max(0.0, self._clock() - record.heard)} if save_floor: - self._save_floor() + self._floor.save() return result + def _shared_refresh(self, name: bytes, max_age: float): + """A resolve's network refresh of name (_refresh_name, asking keepers if nothing was heard), shared (E1). + + Skipped if one ended less than max_age ago (never for max_age 0, the CLI's), unless it left a record that + has gone since (the bounded table evicted it): with the owner offline, a keeper's record shows its old + emission time, so going by heard times alone every gateway request would refresh again. Nothing found + counts too, a short negative cache. A refresh leaving a suspected replay (DHT-2) isn't remembered. + Resolves of a name being refreshed wait for that refresh, whatever their max_age, then each takes the + record held. Called without the lock.""" + with self._lock: + if max_age > 0 and self._refreshed_within(name, max_age): + return + running = self._refreshing.get(name) + if running is None: + running = self._refreshing[name] = threading.Event() + leader = True + else: + leader = False + if not leader: + running.wait() + return + suspect = True # a refresh that raised isn't remembered + try: + suspect = self._refresh_name(name, ask="if_unheard") + finally: + with self._lock: + del self._refreshing[name] + if not suspect: + self._refreshed.pop(name, None) + self._refreshed[name] = (self._monotonic(), self._record(name, touch=False) is not None) + while len(self._refreshed) > REFRESHED_SIZE: + self._refreshed.popitem(last=False) + running.set() + + def _refreshed_within(self, name: bytes, max_age: float) -> bool: + """True if a refresh of name ended less than max_age ago, and the record it left (if any) is still held. + Lock held.""" + refreshed = self._refreshed.get(name) + if refreshed is None: + return False + at, found = refreshed + return 0 <= self._monotonic() - at < max_age and (not found or self._record(name, touch=False) is not None) + + def _refresh_name(self, name: bytes, *, ask: str) -> bool: + """Ask the network for name's newest record (_refresh), then keepers, offering what they answer. ask is + "always" (the first publish of an imported key: its previous home's sequences may be held by keepers alone) + or "if_unheard" (a resolve: only if nothing was heard during the refresh, the record RNS stored is a + suspected replay, or keeper_query is "always"). True if the record held after it is a suspected replay + (DHT-2; asked only with "if_unheard"). Called without the lock.""" + if ask not in ("always", "if_unheard"): + raise ValueError(f"ask is 'always' or 'if_unheard', not {ask!r}") + started = self._clock() + self._refresh(name) + suspect = None + if ask == "if_unheard": + with self._lock: + suspect = self._absorb_recalled(name) + record = self._record(name, touch=False) + # Heard during the refresh, and no replay suspected: the owner (or a path to it) answered. If not, the + # owner and the paths may be gone, or a neighbour repeats an old announce + heard = suspect is None and record is not None and record.heard >= started + if self.keepers is None or (heard and not self.keepers.always_ask()): + return self._held_is(name, suspect) + if self.keepers is not None: + for sequence, cid, kept_heard in self.keepers.query(name): + self.offer(name, sequence, cid, kept_heard) + return self._held_is(name, suspect) + + def _held_is(self, name: bytes, record: tuple[int, Cid] | None) -> bool: + """True if the record held for name is record, a (sequence, CID).""" + if record is None: + return False + with self._lock: + held = self._record(name, touch=False) + return held is not None and (held.sequence, held.cid) == record + def _offer_kept(self, name: bytes): """Offer the signed record keepers hold for name, so it is a floor no replayed announce gets under. @@ -863,18 +1199,19 @@ def _refresh(self, name: bytes): started = self._clock() try: self._network.request_path(name) - # Real time, not the injected clock: this is an actual wait - deadline = time.monotonic() + self.config.path_timeout - while True: - remaining = deadline - time.monotonic() - if remaining <= 0 or event.wait(min(_POLL_INTERVAL, remaining)): + # Counted in the steps waited rather than read from a clock, so a test's wait runs it in virtual time + remaining = self.config.path_timeout + while remaining > 0: + step = min(_POLL_INTERVAL, remaining) + if self._wait(event, step): break + remaining -= step # A path response repeating an announce RNS already has doesn't change the path table, so it # never reaches the handler; RNS still records when it heard it heard = self._network.heard_time(name) if heard is not None and heard >= started: break - except Exception as e: + except Exception as e: # noqa: BLE001 - the resolve goes on with what is held; logged log(f"path request for name {name.hex()} failed: {e}", LOG_WARNING) finally: with self._lock: @@ -884,16 +1221,17 @@ def _refresh(self, name: bytes): if not events: self._waiters.pop(name, None) - def _absorb_recalled(self, name: bytes): - """Offer the record RNS last stored for name (which may be a replay) to the sequence rules.""" + def _absorb_recalled(self, name: bytes) -> tuple[int, Cid] | None: + """Offer the record RNS last stored for name (which may be a replay) to the sequence rules. Its (sequence, + CID) if it is a suspected replay (_announce_heard), which a resolve takes as a reason to ask keepers.""" try: app_data = self._network.recall_app_data(name) - except Exception as e: + except Exception as e: # noqa: BLE001 - the resolve goes on with what is held; logged log(f"could not recall name {name.hex()}: {e}", LOG_WARNING) - return + return None parsed = parse_record(app_data) if parsed is None: - return + return None sequence, cid = parsed heard = self._network.heard_time(name) if heard is None: @@ -901,19 +1239,30 @@ def _absorb_recalled(self, name: bytes): held = self._record(name, touch=False) same = held is not None and held.sequence == sequence and held.cid == cid heard = held.heard if same else self._clock() - self._offer(name, sequence, cid, self._announce_heard(name, heard)) - - def _announce_heard(self, name: bytes, heard: float) -> float: - """The heard time of name's record from an announce RNS heard at heard: no later than ANNOUNCE_VALIDITY - announce_intervals after RNS's newest announce of name was emitted. Anyone may repeat an old announce, and - RNS remembers it as just heard, so by its time alone a record its owner has since replaced would look - current, and resolves would never ask keepers for the newer one. The owner announces every - announce_interval, so an announce older than ANNOUNCE_VALIDITY of them no longer shows its record current. + heard, suspect = self._announce_heard(name, bytes(app_data), heard) + self._offer(name, sequence, cid, heard) + return parsed if suspect else None + + def _announce_heard(self, name: bytes, app_data: bytes, heard: float) -> tuple[float, bool]: + """(heard time, suspected replay) of name's record from the announce RNS last stored, carrying app_data and + heard at heard. + + Anyone may repeat an old announce, and RNS stores it as just heard, so by its time alone a record its + owner has since replaced would look current, and resolves would never ask keepers for the newer one: + - The record is heard no later than ANNOUNCE_VALIDITY announce_intervals after RNS's newest announce of + name was emitted (the path's: emitted_time), since the owner announces every announce_interval. + - An announce emitted before the path's newest, or carrying another record than the path's announce + (RnsNetwork.recalled_emission), is a suspected replay (DHT-2): it is heard no later than its own + emission time, or the path's if its own is unknown. Without a path nothing is compared. """ - emitted = self._network.emitted_time(name) - if emitted is None: - return heard - return min(heard, emitted + ANNOUNCE_VALIDITY * self.config.announce_interval) + path_emitted = self._network.emitted_time(name) + if path_emitted is None: + return heard, False + emitted, takes_path = self._network.recalled_emission(name, app_data) + suspect = not takes_path or (emitted is not None and emitted < path_emitted) + if suspect: + heard = min(heard, emitted if emitted is not None else path_emitted) + return min(heard, path_emitted + ANNOUNCE_VALIDITY * self.config.announce_interval), suspect def _heard(self, name: bytes, app_data, packet_hash: bytes | None = None): """Announce handler: apply a heard record and wake resolves waiting for it.""" @@ -921,11 +1270,13 @@ def _heard(self, name: bytes, app_data, packet_hash: bytes | None = None): parsed = parse_record(app_data) # A name kept here may take a table place like a resolved one (asked outside our lock, as always) kept = parsed is not None and self.keepers is not None and self.keepers.local_record(name) is not None + # Read before taking the lock: it may read RNS's announce cache on disk + heard = self._announce_heard(name, bytes(app_data), self._clock())[0] if parsed is not None else None with self._lock: if parsed is not None: # Unasked: not being resolved (a path response would have a waiter), kept nor watched unasked = not kept and name not in self._waiters and name not in self._watched - self._offer(name, parsed[0], parsed[1], self._announce_heard(name, self._clock()), unasked=unasked) + self._offer(name, parsed[0], parsed[1], heard, unasked=unasked) else: log(f"ignored malformed record for name {RNS.prettyhexrep(name)}", LOG_VERBOSE) for event in self._waiters.get(name, ()): @@ -1023,12 +1374,7 @@ def _store(self, name: bytes, record: Record, held_here: bool, unasked: bool = F self._heard_only[name] = None self._table[name] = record self._touch(name) - floor = self._floor.get(name) - if floor is not None and record.sequence > floor[0]: - self._floor[name] = (record.sequence, floor[1]) - self._floor_changed.add(name) - self._floor_dirty = True - self._save_floor_soon(inline=False) + self._floor.raise_to(name, record.sequence) if len(self._table) > limit: # name_table_size 0: nothing is kept self._drop_from_table(name) return True @@ -1048,66 +1394,14 @@ def _hold(self, name: bytes): """name's record is held now: out of the table, and out of the floor (its held record is its floor), whose row the next floor save deletes.""" self._drop_from_table(name) - if self._floor.pop(name, None) is not None: - del self._floor_used[name] - self._floor_changed.add(name) - self._floor_dirty = True + self._floor.drop(name) def _resolved(self, name: bytes, sequence: int) -> bool: """A resolve without persist returned name's record: it is no longer only heard, and its sequence is the - floor for later resolves (_trim_floor says which are forgotten). A floor raised (or a new one) is saved at - once, or FLOOR_SAVE_INTERVAL after the last save, so a crash soon after loses little. True if the caller - is to save it now, once it has released the lock.""" + floor for later resolves (_NameFloor.resolved). True if the caller is to save the floor now, once it has + released the lock.""" self._heard_only.pop(name, None) - now = self._clock() - previous = self._floor.pop(name, None) - self._floor[name] = (sequence if previous is None else max(previous[0], sequence), now) - self._floor_used[name] = self._floor_next_used # last: the most recently resolved - self._floor_next_used += 1 - self._floor_changed.add(name) # written by the next save, even if this change doesn't call for one - trimmed = self._trim_floor() - if previous is None or self._floor[name][0] != previous[0] or trimmed: - self._floor_dirty = True - return self._save_floor_soon() - if not 0 <= now - previous[1] < FLOOR_TOUCH_SAVE: - self._floor_dirty = True # only its time renewed: saved with the next change or by announce_due - return False - - def _trim_floor(self) -> bool: - """Forget the least recently resolved names over FLOOR_FACTOR × name_table_size, but not those resolved - within FLOOR_KEEP_AGE unless the floor holds FLOOR_HARD_FACTOR times that. True if any went.""" - size = self._floor_size() - now = self._clock() - trimmed = False - while len(self._floor) > size: - _, at = next(iter(self._floor.values())) - if len(self._floor) <= FLOOR_HARD_FACTOR * size and now - at < FLOOR_KEEP_AGE: - break - name, _ = self._floor.popitem(last=False) - del self._floor_used[name] - self._floor_changed.add(name) - trimmed = True - return trimmed - - def _save_floor_soon(self, inline: bool = True) -> bool: - """True if the floor is to be saved now (the last save was FLOOR_SAVE_INTERVAL ago), by the caller once it - has released the lock; else it is saved once it is (a timer). Without inline, a save due now is the - timer's too, started at once.""" - wait = self._floor_saved_at + FLOOR_SAVE_INTERVAL - self._monotonic() - if wait <= 0 and inline: - return True - if self._floor_timer is None and not self._closed: - timer = threading.Timer(max(0.0, wait), self._save_floor_later) - timer.daemon = True - self._floor_timer = timer - timer.start() - return False - - def _save_floor_later(self): - """The timer's save.""" - with self._lock: - self._floor_timer = None - self._save_floor() + return self._floor.resolved(name, sequence) def _floor_size(self) -> int: """How many names the floor holds, past which only names resolved within FLOOR_KEEP_AGE are kept.""" @@ -1119,7 +1413,7 @@ def _open_destination(self, alias: str): """Register alias's destination with its record as default app_data (so path responses carry it).""" try: destination = self._network.name_destination(self._keys[alias]) - except Exception as e: + except Exception as e: # noqa: BLE001 - one name's failure doesn't stop the others; logged log(f"could not open name {alias}: {e}", LOG_ERROR) return self._destinations[alias] = destination @@ -1138,7 +1432,7 @@ def _announce(self, alias: str) -> bool: try: destination.set_default_app_data(encode_record(published.sequence, published.cid)) destination.announce() - except Exception as e: + except Exception as e: # noqa: BLE001 - one name's failure doesn't stop the others; logged log(f"could not announce name {alias}: {e}", LOG_ERROR) return False log(f"announced name {self._names[alias].hex()} ({alias}), sequence {published.sequence}", LOG_VERBOSE) @@ -1248,77 +1542,6 @@ def _load_state(self): # published elsewhere: treated as imported, so its first publish looks for its records first self._published.setdefault(alias, _Published(0, None, imported=True)) - def _load_floor(self): - """Read the floor of names resolved without persist, least recently resolved first. A row that doesn't - parse is logged and deleted (the floor only bounds rollback of gateway and auto-fetch resolves), as are - those of names held for the user and those _trim_floor drops. A resolve time in the future (the clock was - corrected since) counts as now, and is written so by the next floor save that has a change to write.""" - now = self._clock() - dropped = [] - for name, sequence, resolved, used in self._db.query( - "SELECT name, sequence, resolved, used FROM name_floor ORDER BY used"): - try: - _check_name(name) - sequence = _sequence(sequence) - if not 0 <= sequence <= MAX_SEQUENCE: - raise ValueError(f"sequence {sequence} out of range") - if type(resolved) not in (int, float) or not math.isfinite(resolved): - raise ValueError(f"invalid resolve time {resolved!r}") - if type(used) is not int or not 0 <= used < _MAX_USED: - raise ValueError(f"invalid used value {used!r}") - except (TypeError, ValueError) as e: - log(f"dropping an invalid name floor entry {name!r}: {e}", LOG_WARNING) - dropped.append(name) - continue - if name in self._held: - dropped.append(name) # resolved for the user since: its held record is its floor - continue - self._floor[name] = (sequence, min(float(resolved), now)) - self._floor_used[name] = used - self._floor_next_used = max(self._floor_next_used, used + 1) - if resolved > now: - self._floor_changed.add(name) - self._trim_floor() - trimmed = [name for name in self._floor_changed if name not in self._floor] - self._floor_changed.difference_update(trimmed) - if dropped or trimmed: - with self._db.transaction() as tx: - tx.executemany("DELETE FROM name_floor WHERE name = ?", [(name,) for name in dropped + trimmed]) - - def _save_floor(self): - """Write the floor entries changed (or forgotten) since the last save, if a save is due, with - synchronous=NORMAL. Saved by a resolve that raised it (at most every FLOOR_SAVE_INTERVAL, since it holds - thousands of names), announce_due (at least every minute while running) and stop(). - - Called without the lock: only the rows are made under it, so resolves and announces needn't wait for - the write. The save counts from when they were made; a failed one is retried by the next.""" - with self._floor_save_lock: - with self._lock: - if not self._floor_dirty: - return - changed, self._floor_changed = self._floor_changed, set() - rows = [(name, str(self._floor[name][0]), self._floor[name][1], self._floor_used[name]) - for name in changed if name in self._floor] - gone = [(name,) for name in changed if name not in self._floor] - saved_at, self._floor_saved_at = self._floor_saved_at, self._monotonic() - self._floor_dirty = False - saved = False - try: - try: - with self._db.transaction() as tx: - tx.executemany("DELETE FROM name_floor WHERE name = ?", gone) - tx.executemany("INSERT OR REPLACE INTO name_floor (name, sequence, resolved, used) " - "VALUES (?, ?, ?, ?)", rows) - saved = True - except OSError as e: - log(f"could not save the name floor: {e}", LOG_WARNING) - finally: - if not saved: - with self._lock: - self._floor_changed |= changed - self._floor_dirty = True - self._floor_saved_at = saved_at - def _save(self): """Write the published and held records changed since the last save, durably (on disk when it returns: this is a publish's commit point); the rows of _unreadable aliases are left as they are. Raises OSError if diff --git a/rnfs/network/announces.py b/rnfs/network/announces.py index 990304a..b3bd33c 100644 --- a/rnfs/network/announces.py +++ b/rnfs/network/announces.py @@ -29,7 +29,7 @@ def parse_mode(app_data) -> int: stream = io.BytesIO(bytes(app_data)) try: value = umsgpack.unpack(stream) - except Exception: + except Exception: # noqa: BLE001 - umsgpack raises several types for hostile bytes return MODE_CLIENT # Booleans are a separate msgpack type, and trailing bytes make it something other than msgpack 1 if type(value) is int and value == MODE_SERVER and not stream.read(1): @@ -54,7 +54,7 @@ def announce(self): try: self.destination.announce() log(f"announced as a DHT {'server' if self.server else 'client'}", LOG_VERBOSE) - except Exception as e: + except Exception as e: # noqa: BLE001 - announcing is retried at the next interval; logged log(f"could not announce: {e}", LOG_WARNING) def start(self): @@ -101,7 +101,7 @@ def received_announce(self, destination_hash, announced_identity, app_data): self.on_server(node_id) else: self.on_client(node_id) - except Exception as e: + except Exception as e: # noqa: BLE001 - runs on RNS's announce thread; logged log(f"error handling the announce of {node_name(node_id)}: {e}", LOG_WARNING) def register(self): diff --git a/rnfs/network/client.py b/rnfs/network/client.py index 449b936..1480edc 100644 --- a/rnfs/network/client.py +++ b/rnfs/network/client.py @@ -69,6 +69,13 @@ def close(self, node_id: bytes): """Close the shared link to node_id, cancelling any transfer on it.""" self.links.close(node_id) + def close_unless_used(self, node_id: bytes, own: set[int]) -> bool: + """Close the shared link to node_id, as close() does, unless a request made from a thread not in own + (thread idents: the caller's request threads) is in progress on it or waiting for a slot, so one fetch + giving up on its request doesn't fail another's. True if closed (or there was no link); False if left + open, its request left to finish. Never cancels a Resource: closing the link stops it (section 3).""" + return self.links.close_unless_used(node_id, own) + # /find def find(self, node_id: bytes, key: bytes) -> tuple[list[bytes], list[bytes]] | None: diff --git a/rnfs/network/guards.py b/rnfs/network/guards.py index 5b18899..c7d2aeb 100644 --- a/rnfs/network/guards.py +++ b/rnfs/network/guards.py @@ -39,6 +39,10 @@ link that negotiated a larger one, up to hundreds of KiB) is decrypted and checked first, so such a link decrypts an honest one twice. An rnfs.dht server sends no requests, so a response packet arriving on an incoming link is dropped unread: RNS would otherwise decode it before finding no request it answers. +Link.receive likewise drops, unread, a request packet arriving on an rnfs.dht link this node opened: the remote +destination of such a link has no request handlers and no max_request_size, so RNS would decode any request up +to the link's MDU and start a thread for it before finding nothing to run it. The node's server takes requests +on incoming links only. A refused response answers its request with None, which every rnfs.dht parser refuses as malformed, so the requester fails it at once, as it would a malformed response. @@ -60,7 +64,7 @@ from rnfs import __version__ from rnfs.constants import APP_NAME, DHT_ASPECT, MAX_RESPONSE_SIZE -from rnfs.log import LOG_ERROR, LOG_VERBOSE, LOG_WARNING, log, plural +from rnfs.log import LOG_ERROR, LOG_VERBOSE, LOG_WARNING, ThrottledLog, log, plural from rnfs.network import protocol # Resource data is a 4-byte random hash plus the payload, encrypted as an RNS Token: 16-byte IV, 32-byte HMAC @@ -148,7 +152,7 @@ def _fail_pending_request(packet, adv): if getattr(pending, "request_id", None) == request_id: try: pending.response_rejected() - except Exception: + except Exception: # noqa: BLE001 - RNS internals: best effort pass @@ -158,7 +162,7 @@ def guarded_accept(advertisement_packet, callback=None, progress_callback=None, link = getattr(advertisement_packet, "link", None) try: adv = RNS.ResourceAdvertisement.unpack(advertisement_packet.plaintext) - except Exception: + except Exception: # noqa: BLE001 - unreadable advertisements are left to RNS adv = None # let RNS handle an advertisement it can't read, as it always has reason = refusal(adv, link) if adv is not None else None if reason is not None: @@ -178,31 +182,10 @@ def guarded_accept(advertisement_packet, callback=None, progress_callback=None, PACKET_CHECK_SIZE = RNS.Reticulum.MTU # response packets larger than RNS's default MTU are checked first REFUSAL_LOG_INTERVAL = 60.0 # seconds between log lines about refused responses, after the first - -class _ThrottledLog: - """A log line at most every REFUSAL_LOG_INTERVAL seconds, counting the ones left out, so whoever sends - refused responses can't flood the log.""" - - def __init__(self, clock=time.monotonic): - """Nothing logged yet.""" - self.clock = clock - self.lock = threading.Lock() - self.logged_at = None - self.skipped = 0 - - def __call__(self, message: str, level: int): - """Log message at level now, or count it if a line was logged less than REFUSAL_LOG_INTERVAL ago.""" - with self.lock: - now = self.clock() - if self.logged_at is not None and now - self.logged_at < REFUSAL_LOG_INTERVAL: - self.skipped += 1 - return - skipped, self.skipped, self.logged_at = self.skipped, 0, now - log(message + (f" ({plural(skipped, 'more')} since the last)" if skipped else ""), level) - - -_log_refused_response = _ThrottledLog() -_log_unexpected_response = _ThrottledLog() +# Whoever sends refused or unexpected responses or requests can't flood the log +_log_refused_response = ThrottledLog(REFUSAL_LOG_INTERVAL) +_log_unexpected_response = ThrottledLog(REFUSAL_LOG_INTERVAL) +_log_unexpected_request = ThrottledLog(REFUSAL_LOG_INTERVAL) def _answer_as_malformed(link, request_id, size: int): @@ -211,7 +194,7 @@ def _answer_as_malformed(link, request_id, size: int): return try: link.handle_response(request_id, None, size, size) - except Exception as e: + except Exception as e: # noqa: BLE001 - runs inside RNS's packet handling; logged log(f"error failing a request whose response was refused: {e}", LOG_WARNING) @@ -221,7 +204,7 @@ def _read_resource(resource) -> bytes | None: try: packed = data.read() data.seek(0) - except Exception: + except Exception: # noqa: BLE001 - RNS's stream: unreadable means not checked here return None return packed if isinstance(packed, (bytes, bytearray)) else None @@ -243,7 +226,14 @@ def guarded_response_resource_concluded(link, resource): def guarded_receive(link, packet): - """RNS.Link.receive, but on rnfs.dht links checking a response packet before RNS decodes it.""" + """RNS.Link.receive, but on rnfs.dht links checking a response packet before RNS decodes it, and dropping a + request packet on a link this node opened.""" + if getattr(packet, "context", None) == RNS.Packet.REQUEST and getattr(link, "initiator", False) is True \ + and getattr(packet, "packet_type", None) == RNS.Packet.DATA and is_dht_link(link): + # The remote destination of a link this node opened has no max_request_size and no handlers, so RNS + # would decode the request (up to the link's MDU) and start a thread for it before finding nothing to run + _log_unexpected_request("dropped a request arriving on an outgoing link, which serves none", LOG_VERBOSE) + return None if getattr(packet, "context", None) == RNS.Packet.RESPONSE and is_dht_link(link) \ and getattr(packet, "packet_type", None) == RNS.Packet.DATA: if not getattr(link, "initiator", False): @@ -254,7 +244,7 @@ def guarded_receive(link, packet): if len(data) > PACKET_CHECK_SIZE: try: packed = link.decrypt(data) - except Exception: + except Exception: # noqa: BLE001 - undecryptable is left to RNS, as before packed = None reason = protocol.response_refusal(packed) if packed is not None else None if reason is not None: @@ -382,7 +372,7 @@ def _close(self, link): handshake was cut short learns at once and can try again.""" try: link.teardown() - except Exception as e: + except Exception as e: # noqa: BLE001 - runs on the guard's timer thread; logged log(f"error closing a half-open link: {e}", LOG_WARNING) return self.closed += 1 diff --git a/rnfs/network/identity.py b/rnfs/network/identity.py index fe9e7cd..2798386 100644 --- a/rnfs/network/identity.py +++ b/rnfs/network/identity.py @@ -4,8 +4,8 @@ import RNS -from rnfs import fsutil, winacl -from rnfs.constants import APP_NAME, DHT_ASPECT, ID_LENGTH +from rnfs import fsutil, ids +from rnfs.constants import APP_NAME, DHT_ASPECT from rnfs.errors import RnfsError from rnfs.log import LOG_WARNING, log @@ -116,28 +116,16 @@ def _load_identity(path: str) -> RNS.Identity: def _check_private(path: str, fd: int, info: os.stat_result): """Check that the identity file open as fd (whose fstat is info) is the user's own and private to them, - tightening its permissions if others may access it. Raises IdentityFileError, OSError.""" - if fsutil.IS_WINDOWS: - try: - reason = winacl.tighten(path, fd=fd) - except OSError as e: + tightening its permissions (to IDENTITY_MODE on POSIX) if others may access it, with a warning. Raises + IdentityFileError.""" + try: + warning = fsutil.check_owned_fd(path, fd, info, "the identity file") + except OSError as e: + if fsutil.IS_WINDOWS: raise IdentityFileError(f"the identity file {path} can't be made private to you: {e}") from None - if reason is not None: - log(f"identity file {path} was accessible to other users ({reason}); " - f"restricted it to you and SYSTEM", LOG_WARNING) - return - if info.st_uid != os.geteuid(): - raise IdentityFileError(f"the identity file {path} belongs to another user (uid {info.st_uid}); " - f"rnfs only uses a key file of your own") - mode = stat.S_IMODE(info.st_mode) - if mode & 0o077: - try: - os.fchmod(fd, IDENTITY_MODE) - except OSError as e: - raise IdentityFileError(f"the identity file {path} is accessible to other users (mode {mode:o}) " - f"and can't be restricted: {e}") from None - log(f"identity file {path} was accessible to other users (mode {mode:o}); " - f"restricted it to {IDENTITY_MODE:o}", LOG_WARNING) + raise IdentityFileError(f"{e}; rnfs only uses a key file of your own, private to you") from None + if warning is not None: + log(warning, LOG_WARNING) def node_id_for_identity(identity: RNS.Identity) -> bytes: @@ -146,9 +134,6 @@ def node_id_for_identity(identity: RNS.Identity) -> bytes: def parse_node_id(hex_id: str) -> bytes: - """Node ID from its hex form, with or without RNS's . Raises ValueError if malformed.""" - node_id = bytes.fromhex(hex_id.strip().strip("<>")) - if len(node_id) != ID_LENGTH: - raise ValueError(f"node ID must be {ID_LENGTH * 2} hex characters: {hex_id}") - return node_id + """Node ID from its 32 hex digits, with or without RNS's . Raises ValueError if malformed.""" + return ids.parse_hex16(hex_id, "node ID", brackets=True) diff --git a/rnfs/network/links.py b/rnfs/network/links.py index 35bd6d3..57ab6f5 100644 --- a/rnfs/network/links.py +++ b/rnfs/network/links.py @@ -27,6 +27,7 @@ from rnfs.config import Config from rnfs.constants import APP_NAME, DHT_ASPECT, MAX_REQUESTS_PER_LINK +from rnfs import log as rnfs_log from rnfs.log import LOG_DEBUG, LOG_VERBOSE, LOG_WARNING, log, node_name, size from rnfs.network import guards, protocol, rates @@ -56,12 +57,18 @@ def close_reason(link) -> str: return "closed" +def _debug_logged() -> bool: + """True if a LOG_DEBUG message is kept anywhere: RNS logs at debug level, or the debug log (which keeps + every message, whatever RNS's loglevel) is on.""" + return RNS.loglevel >= LOG_DEBUG or rnfs_log._debug_sink is not None + + def path_description(node_id: bytes) -> str: """How far RNS's path to node_id goes and through which interface, for the log ("" if it can't tell).""" try: hops = RNS.Transport.hops_to(node_id) interface = RNS.Transport.next_hop_interface(node_id) - except Exception: + except Exception: # noqa: BLE001 - RNS internals: no description if they fail return "" if hops >= RNS.Transport.PATHFINDER_M: return "" @@ -95,7 +102,7 @@ def on_response(self, receipt): """Record the response and the link's measured rate, which the next Resource would overwrite.""" try: self.expected_rate = receipt.link.get_expected_rate() - except Exception: + except Exception: # noqa: BLE001 - RNS internals: no rate if they fail self.expected_rate = None self.response_size = receipt.response_size self.response = receipt.get_response() @@ -132,6 +139,9 @@ def __init__(self, node_id: bytes, identified: bool, shared: bool): self.rate: float | None = None # rate estimate in bits/s (section 2) self.requests = 0 self.blocks = 0 + # Threads with a request in progress on the link or waiting for a slot -> how many each: who would lose a + # request if the link were closed now (PeerClient.close_unless_used) + self.users: dict[int, int] = {} self.closed = False self.handed_out: float | None = None # when LinkPool.get() last returned it (the pool's clock) self.cond = threading.Condition() @@ -144,7 +154,7 @@ def mdu(self) -> int | None: """The link's packet MDU, which bounds request sizes.""" try: return self.link.get_mdu() - except Exception: + except Exception: # noqa: BLE001 - RNS internals: no MDU if they fail return None def acquire(self, blocks: bool, timeout: float | None = None) -> bool: @@ -179,6 +189,26 @@ def busy(self) -> bool: with self.cond: return self.requests > 0 + def enter(self): + """The calling thread is about to wait for a slot and send a request (LinkPool.request).""" + ident = threading.get_ident() + with self.cond: + self.users[ident] = self.users.get(ident, 0) + 1 + + def leave(self): + """The calling thread's request is over, sent or not.""" + ident = threading.get_ident() + with self.cond: + if self.users.get(ident, 0) <= 1: + self.users.pop(ident, None) + else: + self.users[ident] -= 1 + + def used_by_others(self, own: set[int]) -> bool: + """True if a thread not in own (thread idents) has a request in progress on the link or waiting for a slot.""" + with self.cond: + return any(ident not in own for ident in self.users) + def mark_closed(self): """The link closed: wake everyone waiting for a slot.""" with self.cond: @@ -203,7 +233,7 @@ def __init__(self, identity: RNS.Identity, config: Config, clock: Callable[[], f self.config = config self.clock = clock # Private mode links only to members: every link identifies, and an outsider mustn't learn who we are - self.members = {h.strip().strip("<>").lower() for h in config.members} + self.members = config.member_set self.shared: dict[bytes, PeerLink] = {} self.oneshot: set[PeerLink] = set() self.connecting: dict[bytes, threading.Event] = {} # one thread opens a node's shared link at a time @@ -282,7 +312,8 @@ def _open(self, peer: PeerLink) -> bool: if not RNS.Transport.await_path(node_id, timeout=self.config.path_timeout): log(f"no path to {name} after {self.config.path_timeout:g}s") return False - log(f"path to {name}: {path_description(node_id) or 'known'}", LOG_DEBUG) + if _debug_logged(): # RNS path lookups, only for a message that will be kept + log(f"path to {name}: {path_description(node_id) or 'known'}", LOG_DEBUG) identity = self._recall(node_id) if identity is None: @@ -330,16 +361,23 @@ def on_closed(closed_link): return True def _recall(self, node_id: bytes): - """The node's identity from RNS, asking for a path again (whose response carries it) if needed.""" + """The node's identity from RNS, asking for a path again (whose response carries it) if needed. + + A recall marks the destination used, which keeps it in RNS's known destinations for longer (and is + an RPC of its own on a shared instance): right for a node being contacted, but not for each of the + polls while its identity is awaited, so those don't mark it, and one recall marks it once it is there. + """ identity = RNS.Identity.recall(node_id) if identity is not None: return identity RNS.Transport.request_path(node_id) deadline = time.monotonic() + self.config.path_timeout - while identity is None and time.monotonic() < deadline: + while time.monotonic() < deadline: time.sleep(0.1) - identity = RNS.Identity.recall(node_id) - return identity + found = RNS.Identity.recall(node_id, _no_use=True) + if found is not None: + return RNS.Identity.recall(node_id) or found + return None def identify(self, peer: PeerLink): """Identify again on peer's link (unacknowledged, so it may need repeating).""" @@ -374,6 +412,15 @@ def close(self, node_id: bytes): """Tear down the shared link to node_id, if any (e.g. after a stalled transfer).""" self.close_link(self.shared_peer(node_id)) + def close_unless_used(self, node_id: bytes, own: set[int]) -> bool: + """Tear down the shared link to node_id unless a thread not in own (thread idents) has a request in + progress on it or waiting for a slot. True if there was nothing else on it (the link, if any, is closed).""" + peer = self.shared_peer(node_id) + if peer is not None and peer.used_by_others(own): + return False + self.close_link(peer) + return True + def close_all(self): """Stop the idle sweeper and tear down every link.""" self.stop_event.set() @@ -392,7 +439,7 @@ def sweep(self) -> int: for peer in peers: try: quiet = peer.link.no_data_for() - except Exception: + except Exception: # noqa: BLE001 - a link closing meanwhile is skipped continue if not peer.busy() and quiet >= timeout: idle.append(peer) @@ -425,7 +472,7 @@ def _sweep_loop(self): while not self.stop_event.wait(interval): try: self.sweep() - except Exception as e: + except Exception as e: # noqa: BLE001 - the sweeper thread keeps running; logged log(f"error closing idle links: {e}", LOG_WARNING) # Queries @@ -445,7 +492,7 @@ def quiet_for(peer: PeerLink) -> float: """Seconds since data crossed the link.""" try: return peer.link.no_data_for() - except Exception: + except Exception: # noqa: BLE001 - a link closing meanwhile sorts last return float("inf") return [peer.node_id for peer in sorted(peers, key=quiet_for)] @@ -462,6 +509,16 @@ def request(self, peer: PeerLink, path: str, data, blocks: bool = False, max_res request_deadline has passed since it was sent, or if it waited that long for a slot. A request larger than the link's packet MDU isn't sent (the exchange is returned unanswered). """ + peer.enter() + try: + return self._request(peer, path, data, blocks, max_response_size, watchdog, on_progress) + finally: + peer.leave() + + def _request(self, peer: PeerLink, path: str, data, blocks: bool, max_response_size: int | None, + watchdog: Callable[[Exchange, float], str | None] | None, + on_progress: Callable[[float], None] | None) -> Exchange: + """request(), with the calling thread counted among the link's users.""" exchange = Exchange(self.clock) deadline = None if blocks else self.request_deadline(peer) if not peer.acquire(blocks, timeout=deadline): @@ -484,8 +541,9 @@ def request(self, peer: PeerLink, path: str, data, blocks: bool = False, max_res failed_callback=exchange.on_failed, progress_callback=exchange.on_progress, timeout=None, max_response_size=max_response_size) - except Exception as e: - log(f"could not send {path} to {node_name(peer.node_id)}: {e}", LOG_VERBOSE) + except Exception as e: # noqa: BLE001 - a failed send is a failed request; logged + # Link.request answers False when it can't send (a closed link too), so an exception is a bug + log(f"could not send {path} to {node_name(peer.node_id)}: {e!r}", LOG_WARNING) return exchange if not receipt: log(f"could not send {path} to {node_name(peer.node_id)}", LOG_VERBOSE) diff --git a/rnfs/network/protocol.py b/rnfs/network/protocol.py index 6d2a566..c974071 100644 --- a/rnfs/network/protocol.py +++ b/rnfs/network/protocol.py @@ -5,10 +5,15 @@ | /provide | [[key, ...], [withdrawn key, ...]]| True if recorded | | /blocks | [max_bytes, [cid, ...]] | [block or None, ...] | | /pin | [cid, pin] | True if accepted | +| /name | name | the record's fields, or [] | -Keys and node IDs are 16-byte bin, CIDs binary CIDs as bin, blocks bin, max_bytes a non-negative integer +Keys, node IDs and names are 16-byte bin, CIDs binary CIDs as bin, blocks bin, max_bytes a non-negative integer (never a boolean), pin a boolean. Any other shape, or a refusal, is answered False. +/name is the keepers feature's path (rnfs.features.keepers encodes and parses it). Its response is a kept +record's [public_key, name_hash, random_hash, ratchet, signature, app_data], every field bin, or [] if the node +keeps none: the shape of a /blocks answer, which is what lets response_refusal accept it. + Everything parsed here comes from other nodes, so parsers are strict and return None for anything malformed. Every request must fit in one packet: at most min(link MDU, MAX_REQUEST_SIZE) bytes including RNS's request framing, which is computed exactly here the way Link.request packs it. @@ -357,14 +362,25 @@ def response_refusal(packed: bytes) -> str | None: """Why packed (RNS's response framing, [request_id, response], as received) can't hold any rnfs.dht response, judged from msgpack headers alone without decoding it; None if it may. - The response may be True, False or None (a /provide or /pin answer, or a refusal), a /find answer, or a - /blocks answer (also the shape of a /name one). Nothing may follow it. + The response may be any that body_refusal accepts. Nothing may follow it. """ outer = _array_at(packed, 0) request_id = _bin_at(packed, outer[1]) if outer is not None and outer[0] == 2 else None if request_id is None or request_id[0] != REQUEST_ID_LENGTH or request_id[1] >= len(packed): return "not a response" - position = request_id[1] + return body_refusal(packed, request_id[1]) + + +def body_refusal(packed: bytes, position: int = 0) -> str | None: + """Why the packed response at position (to the end of packed) can't be any rnfs.dht response, from msgpack + headers alone; None if it may. + + The response may be True, False or None (a /provide or /pin answer, or a refusal), a /find answer, or a + /blocks answer (also the shape of a /name one). Nothing may follow it. Requesters check every response + with this (through response_refusal), so a node checks its features' responses with it before sending. + """ + if position >= len(packed): + return "no response" if packed[position] in (_NIL, _FALSE, _TRUE): end = position + 1 else: diff --git a/rnfs/network/rates.py b/rnfs/network/rates.py index 29db971..2a43bc1 100644 --- a/rnfs/network/rates.py +++ b/rnfs/network/rates.py @@ -1,4 +1,5 @@ -"""Rate estimates, slow links and slow nodes (specification section 2). Used by both server and client. +"""Rate estimates, slow links and slow nodes (specification section 2), and the /blocks size rules that follow +from them (sections 3 and 7). Used by both server and client. A link's rate estimate is what RNS measured (Link.get_expected_rate()) for the last response of at least min_rate_sample on it, recorded when that response completes, since RNS overwrites the value with every @@ -7,7 +8,7 @@ import threading from rnfs.config import Config -from rnfs.constants import CHUNK_SIZE +from rnfs.constants import CHUNK_SIZE, MAX_RESPONSE_SIZE, RESPONSE_OVERHEAD from rnfs.log import LOG_WARNING, log SLOW_MTU = 500 # a link whose MTU RNS never raised above this crosses an interface that doesn't support more @@ -68,7 +69,7 @@ def link_is_slow(link, rate_estimate: float | None, config: Config) -> bool: """is_slow_link for an RNS link, reading its MTU with Link.get_mtu().""" try: mtu = link.get_mtu() - except Exception: + except Exception: # noqa: BLE001 - RNS internals: MTU unknown if they fail mtu = None return is_slow_link(rate_estimate, mtu, config.slow_bitrate) @@ -97,6 +98,37 @@ def requester_max_bytes(rate_estimate: float | None, config: Config) -> int: return transfer_bytes(config.response_target, rate_estimate) +# The /blocks size rules both ends share (section 7): the responder enforces them (server.blocks_budget), the +# requester sizes its batches by them (fetcher), so changing one end's rule can't leave the other's estimate wrong + +def response_capacity(max_bytes: int, rate_estimate: float | None, upload_cap: float | None, + response_target: float) -> int: + """Total block bytes a /blocks response may hold after its first served block: max_bytes, at most what one + response carries, and response_target of transfer at the link's rate (capped by upload_cap, if any); CHUNK_SIZE + at most while the link has no rate estimate.""" + limit = min(max_bytes, MAX_RESPONSE_SIZE - RESPONSE_OVERHEAD) + if rate_estimate is None: + # What a requester asks for until the link has a rate estimate (section 7), enforced: otherwise one + # small request could ask for a ~1 MiB response with a deadline of many hours + return min(limit, CHUNK_SIZE) + rate = min(rate_estimate, upload_cap) if upload_cap else rate_estimate + return min(limit, transfer_bytes(response_target, rate)) + + +# Without max_upload_rate, a slow link's upload cap is its rate estimate divided by this (section 3). The responder +# then starts the next /blocks response on the link no sooner than this many times its transfer time at the rate +# estimate after the last one started, which the requester waits out (fetcher's _pace) +UPLOAD_CAP_DIVISOR = 2 + + +def default_upload_cap(slow_link: bool, rate_estimate: float | None) -> float | None: + """The upload cap of a link in bits/s when max_upload_rate isn't set: a slow link's rate estimate divided by + UPLOAD_CAP_DIVISOR. None (no cap) on a fast link, or one without a rate estimate.""" + if slow_link and rate_estimate is not None: + return rate_estimate / UPLOAD_CAP_DIVISOR + return None + + # Slow nodes def counted_bitrates(stats: dict) -> list[float]: @@ -140,7 +172,7 @@ def is_slow_node(config: Config, reticulum=None) -> bool: import RNS reticulum = RNS.Reticulum.get_instance() stats = reticulum.get_interface_stats() - except Exception as e: + except Exception as e: # noqa: BLE001 - not slow if the stats can't be read; logged log(f"could not read interface stats to decide whether this node is slow: {e}", LOG_WARNING) return False return slow_from_stats(stats, config.slow_bitrate) diff --git a/rnfs/network/server.py b/rnfs/network/server.py index 1f02aed..5afeaf2 100644 --- a/rnfs/network/server.py +++ b/rnfs/network/server.py @@ -23,12 +23,13 @@ from typing import Callable, Protocol import RNS +from RNS.vendor import umsgpack from rnfs.config import Config -from rnfs.constants import (BLOCKS_PATH, CHUNK_SIZE, FIND_PATH, MAX_REQUEST_SIZE, MAX_REQUESTS_PER_LINK, - MAX_RESPONSE_SIZE, PIN_PATH, PROVIDE_PATH, RESPONSE_OVERHEAD) +from rnfs.constants import (BLOCKS_PATH, FIND_PATH, MAX_REQUEST_SIZE, MAX_REQUESTS_PER_LINK, PIN_PATH, + PROVIDE_PATH, RESPONSE_OVERHEAD) from rnfs.content.cid import Cid -from rnfs.log import LOG_DEBUG, LOG_ERROR, LOG_VERBOSE, LOG_WARNING, log, node_name, plural, size +from rnfs.log import LOG_DEBUG, LOG_ERROR, LOG_VERBOSE, LOG_WARNING, ThrottledLog, log, node_name, plural, size from rnfs.network import guards, protocol, rates from rnfs.network.identity import node_id_for_identity from rnfs.network.links import close_reason @@ -64,6 +65,13 @@ # A response displaced for a newcomer while making progress at less than this fraction of the rate it needs to # be done when it is due (Response.due) counts as making none: its requester cools down FAR_BEHIND = 1 / 4 +# A response whose Resource its requester rejects or cancels before this fraction of it was sent made no real +# progress: its requester cools down (section 3). Otherwise one could take the first window of parts of each +# response, cancel it and ask again, having the node read, hash, pack and encrypt a whole response per window +# (on a slow link, a window of RNS.Resource.WINDOW_MAX_SLOW parts is about 7% of a 64 KiB response). An honest +# 0.1.5 fetcher never cancels a Resource (a reader giving up closes the link), so only RNS running out of +# retries on a lossy path does +NO_PROGRESS_FRACTION = 1 / 4 COOLDOWN_MEMORY = 4096 # links and identities on a /blocks cooldown remembered at most # Requests per incoming link (section 3): a token bucket of REQUEST_BURST, refilled at REQUEST_RATE a second. # An honest requester waits for its answers, with at most MAX_REQUESTS_PER_LINK in progress, so it sends at most @@ -99,8 +107,8 @@ def served_block(self, cid: Cid) -> bytes | None: """cid's block if it's in the DAG of a CID this node provides, re-verified; else None.""" def served_block_size(self, cid: Cid) -> int | None: - """Optional (looked up with getattr): the stored length of the block served_block(cid) would return, or - None if it isn't served (or isn't known). Lets /blocks stop at the block over its budget unread.""" + """The stored length of the block served_block(cid) would return, or None if it isn't served (or isn't + known). Lets /blocks stop at the block over its budget unread.""" def remote_pin(self, identity_hash: bytes, cid: Cid, pin: bool) -> bool: """Accept (True) or refuse a /pin from the identity with this 16-byte hash.""" @@ -109,22 +117,17 @@ def is_member(self, identity_hash: bytes) -> bool: """In private mode, True if the identity with this 16-byte hash is a member.""" def feature_requests(self) -> dict: - """Optional (looked up with getattr): request paths added by features, {path: handler(data, - remote_identity)}, served within the same per-link limits; a path rnfs serves itself is ignored.""" + """Request paths added by features, {path: handler(data, remote_identity)} ({} for none), served within the + same per-link limits; a path rnfs serves itself is ignored.""" # Pure policy def blocks_budget(max_bytes: int, rate_estimate: float | None, upload_cap: float | None, response_target: float) -> int: - """Total block bytes a /blocks response may hold after its first served block (section 7).""" - limit = min(max_bytes, MAX_RESPONSE_SIZE - RESPONSE_OVERHEAD) - if rate_estimate is None: - # What a requester asks for until the link has a rate estimate (section 7), enforced: otherwise one - # small request could ask for a ~1 MiB response with a deadline of many hours - return min(limit, CHUNK_SIZE) - rate = min(rate_estimate, upload_cap) if upload_cap else rate_estimate - return min(limit, rates.transfer_bytes(response_target, rate)) + """Total block bytes a /blocks response may hold after its first served block (section 7): the rule the + requester sizes its batches by too (rates.response_capacity).""" + return rates.response_capacity(max_bytes, rate_estimate, upload_cap, response_target) def select_blocks(cids: list[Cid], served_block: Callable[[Cid], bytes | None], limit: int, @@ -194,7 +197,7 @@ def fast_path(link) -> bool: KISS radio on it. A requester can lower its link's MTU, never raise it past what the path allows.""" try: mtu = link.get_mtu() - except Exception: + except Exception: # noqa: BLE001 - RNS internals: not a fast link if they fail return False return type(mtu) is int and mtu > rates.SLOW_MTU @@ -231,7 +234,44 @@ def progress_floor(bitrate: float | None, hops: int | None, config: Config) -> f class Response: - """A /blocks response in progress, holding a slot in one of the pools.""" + """A /blocks response in progress, holding a slot in one of the pools. + + Threads and locks. A response is touched by its request's handler thread (RNS starts one per request), the + next request's handler on the same link (_previous_done), the monitor thread (check_responses), RNS's threads + (the outgoing-Resource registration hook on the packet thread; the Resource's callback on the packet thread, + on a Resource's watchdog, on the thread RNS starts for a rejection's callback, or on whichever thread + cancelled it) and link-closed callbacks (_link_closed, on the thread that tore the link down). Python + assigns attributes atomically; what matters is who may write a field, under which lock, and in what order: + - link_id, request_id, pool and started: set at construction, never changed. + - grace, floor, interface and identity: set by ResponsePools.reserve under ResponsePools.lock, before the + response is visible to any other thread. grace is set again, with no lock, by the handler thread at + handover, dropping the pacing wait it covered until then: a reader sees one value or the other. + - size, deadline, due, find_until and untracked: written once each, with no lock, by the handler thread at + handover, size first and the others after it, before it returns the body (so before RNS can build the + Resource) and before it wakes the monitor. Readers take size None to mean not yet handed over and skip + the response (check_responses), so a reader that sees size set but an older deadline (inf) or + find_until (inf) only waits one more pass. requester is likewise set once by the handler, before _admit. + - sent and first_progress: written by the monitor thread only (note_progress), with no lock. Other threads + read them as a snapshot: a stale value only makes a response look slower than it is. + - grace_start and progress_at: moved forward only (max), with no lock, by the handler at handover, by + _attach (on the packet thread or the monitor) and, for progress_at, by the monitor. A lost update loses + at most the gap between two near-simultaneous "now"s. + - resource: set by _attach, with no lock, from the registration hook (packet thread) or the monitor's + poll, whichever sees the Resource first; both check it is still None and would attach the same + Resource, so a race only runs _hook twice, which installs the callback once (and _concluded runs once). + - idle: set under ResponsePools.lock when a newcomer displaces it (_displaceable), or with no lock by the + monitor (_check_progress) just before it cancels it, and read by the _cancel that follows. A displaced + response has left the pool, so only a monitor pass whose snapshot predates that can still reach it; both + then cancel it, which _finish makes harmless, and either one's idle decides the cooldown. + - finished and concluded: check-and-set under RequestServer.lock (_finish, _concluded), so each runs once + whichever thread gets there first. Read without the lock elsewhere as a hint: neither ever goes back to + False, so a stale False only means that thread's later _finish (or _concluded) call returns early. + - cost, charged and settled: written under RequestServer.lock (_admit, _charge_progress, _settle, and the + handler's settled=True for single-packet and untracked responses). Read without it by commitment() and + _committed() outside _admit, where a stale value only shifts an admission estimate. + - done: a threading.Event, set by _finish. + NW2's later steps (in a later release) give a response its own lock; until then, keep to these owners. + """ def __init__(self, link_id: bytes, request_id: bytes, pool: str, started: float): """A response that has just taken a slot; its size and deadline follow once it's built.""" @@ -681,10 +721,9 @@ def __init__(self, destination: RNS.Destination, config: Config, backend: Backen self.lock = threading.Lock() self.stop_event = threading.Event() self.wake = threading.Event() - self.pin_identities = {h.strip().strip("<>").lower() for h in config.remote_pin_identities} + self.pin_identities = config.remote_pin_set self.over_rate = 0 # requests over their link's rate limit, in all (refused or dropped) - self.over_rate_logged_at: float | None = None - self.over_rate_logged = 0 + self.over_rate_log = ThrottledLog(OVER_RATE_LOG_INTERVAL, clock) # Reject request Resources, compressed ones and advertisements whose transfer outgrows their size: on # every link in the daemon, which also turns RNS's compression off process-wide, or on rnfs.dht links @@ -701,7 +740,7 @@ def __init__(self, destination: RNS.Destination, config: Config, backend: Backen PIN_PATH: self._handle_pin, } # Paths added by features (rnfs.features: Feature.requests), within the same per-link limit - for path, handler in getattr(backend, "feature_requests", dict)().items(): + for path, handler in backend.feature_requests().items(): if path not in handlers: handlers[path] = self._feature_handler(handler) for path, handler in handlers.items(): @@ -751,7 +790,7 @@ def _member_links(self) -> set[bytes]: for link in links: try: identity = link.get_remote_identity() - except Exception: + except Exception: # noqa: BLE001 - RNS internals: no identity if they fail identity = None if self._is_member(identity): members.add(link.link_id) @@ -812,7 +851,7 @@ def response_generator(path, data, request_id, link_id, remote_identity, request log(f"refused {path} from a non-member", LOG_VERBOSE) return False return handler(path, data, request_id, link_id, remote_identity) - except Exception as e: + except Exception as e: # noqa: BLE001 - runs on RNS's request thread: refuse and log log(f"error answering {path}: {e}", LOG_ERROR) return False return response_generator @@ -821,32 +860,43 @@ def _over_rate(self): """Count a request over its link's rate limit, logging at most once a minute.""" with self.lock: self.over_rate += 1 - now = self.clock() - if self.over_rate_logged_at is not None and now - self.over_rate_logged_at < OVER_RATE_LOG_INTERVAL: - return - dropped, self.over_rate_logged, self.over_rate_logged_at = (self.over_rate - self.over_rate_logged, - self.over_rate, now) - log(f"refused or dropped {plural(dropped, 'request')} over the per-link rate limit ({REQUEST_BURST} at " + skipped = self.over_rate_log.due() + if skipped is None: + return + log(f"refused or dropped {plural(skipped + 1, 'request')} over the per-link rate limit ({REQUEST_BURST} at " f"once, {REQUEST_RATE:g} a second)", LOG_WARNING) def _is_member(self, remote_identity) -> bool: """True if the requester identified as a member (private mode).""" return remote_identity is not None and bool(self.backend.is_member(remote_identity.hash)) + def _in_slot(self, link_id: bytes, answer: Callable[[], object]): + """answer() run in one of the link's MAX_REQUESTS_PER_LINK request slots, which is freed when it returns + or raises; False at once if the link has none free. + + Every handler but /blocks (whose slot outlives the handler, until its Resource concludes) runs its + work here, after its own checks: a refusal from those comes first and takes no slot. + """ + if not self.incoming.start_request(link_id, blocks=False): + return False + try: + return answer() + finally: + self.incoming.finish_request(link_id, blocks=False) + def _handle_find(self, path, data, request_id, link_id, remote_identity): """Answer /find with providers of the key and the servers closest to it.""" key = protocol.parse_find_request(data) if key is None or not self.backend.is_dht_server(): return False - if not self.incoming.start_request(link_id, blocks=False): - return False - try: + + def answer(): + """Look the key up.""" providers, nodes = self.backend.find(key) log(f"/find {key.hex()}: {plural(len(providers), 'provider')}, {plural(len(nodes), 'node')}", LOG_VERBOSE) return protocol.encode_find_response(list(providers), list(nodes)) - finally: - self.incoming.finish_request(link_id, blocks=False) + return self._in_slot(link_id, answer) def _handle_provide(self, path, data, request_id, link_id, remote_identity): """Record the identified requester as a provider of the keys, and withdraw the withdrawn ones.""" @@ -854,29 +904,37 @@ def _handle_provide(self, path, data, request_id, link_id, remote_identity): if remote_identity is None or not self.backend.is_dht_server(): return False parsed = protocol.parse_provide_request(data) - if parsed is None or not self.incoming.start_request(link_id, blocks=False): + if parsed is None: return False - try: + + def answer(): + """Record the keys and withdraw the withdrawn ones.""" keys, withdrawn = parsed publisher = node_id_for_identity(remote_identity) recorded = self.backend.provide(publisher, keys, withdrawn) is True log(f"/provide from {node_name(publisher)}: {plural(len(keys), 'key')}, {len(withdrawn)} withdrawn, " f"{'recorded' if recorded else 'refused'}", LOG_VERBOSE) return recorded - finally: - self.incoming.finish_request(link_id, blocks=False) + return self._in_slot(link_id, answer) def _feature_handler(self, handler): - """A handler for a feature's path: counted against MAX_REQUESTS_PER_LINK, and never answering None.""" + """A handler for a feature's path: counted against MAX_REQUESTS_PER_LINK, never answering None, and + answering False rather than a response requesters would refuse.""" def handle(path, data, request_id, link_id, remote_identity): """Run the feature's handler(data, remote_identity) inside the per-link request limit.""" - if not self.incoming.start_request(link_id, blocks=False): - return False - try: + def answer(): + """The feature's response, False for None or a shape no requester accepts.""" response = handler(data, remote_identity) - return False if response is None else response - finally: - self.incoming.finish_request(link_id, blocks=False) + if response is None: + return False + # Requesters since 0.1.4 check every response's shape before decoding it (protocol.response_refusal): + # one they refuse would fail every request to this path, so it's a bug in the feature, said here + refusal = protocol.body_refusal(umsgpack.packb(response)) + if refusal is not None: + log(f"refused to answer {path} with a response no requester accepts: {refusal}", LOG_ERROR) + return False + return response + return self._in_slot(link_id, answer) return handle def _handle_pin(self, path, data, request_id, link_id, remote_identity): @@ -884,13 +942,10 @@ def _handle_pin(self, path, data, request_id, link_id, remote_identity): if remote_identity is None or remote_identity.hash.hex() not in self.pin_identities: return False parsed = protocol.parse_pin_request(data) - if parsed is None or not self.incoming.start_request(link_id, blocks=False): + if parsed is None: return False - try: - cid, pin = parsed - return self.backend.remote_pin(remote_identity.hash, cid, pin) is True - finally: - self.incoming.finish_request(link_id, blocks=False) + cid, pin = parsed + return self._in_slot(link_id, lambda: self.backend.remote_pin(remote_identity.hash, cid, pin) is True) def _handle_blocks(self, path, data, request_id, link_id, remote_identity): """Answer /blocks within the per-link limit, the response pools, upload pacing and the budget.""" @@ -942,8 +997,9 @@ def _handle_blocks(self, path, data, request_id, link_id, remote_identity): if response.finished: return False # displaced, or its link closed, while it waited - # The upload cap applies once the link has a rate estimate (section 3) - cap = self.config.effective_max_upload_rate(slow, rate) if rate is not None else None + # An explicit max_upload_rate applies to every response, also one sent as a single packet, which + # never gives a rate estimate; the default cap (half the estimate on slow links) needs one (section 3) + cap = self.config.effective_max_upload_rate(slow, rate) limit = self.airtime.limit(blocks_budget(max_bytes, rate, cap, self.config.response_target), self._committed(response)) response.requester = identity if identity is not None else link_id @@ -954,8 +1010,7 @@ def _handle_blocks(self, path, data, request_id, link_id, remote_identity): log("refused /blocks: that requester's responses in flight hold half of upload_budget", LOG_VERBOSE) return False limit = min(limit, max(0, share - RESPONSE_OVERHEAD)) - entries = select_blocks(cids, self.backend.served_block, limit, - getattr(self.backend, "served_block_size", None)) + entries = select_blocks(cids, self.backend.served_block, limit, self.backend.served_block_size) body = protocol.encode_blocks_response(entries) packed = protocol.packed_response_size(request_id, body) mdu = link.get_mdu() or protocol.DEFAULT_MDU @@ -1031,6 +1086,17 @@ def _after_previous(self, link, link_id: bytes) -> bool: finally: self.incoming.stop_waiting(link_id) + @staticmethod + def _proof_wait(link) -> float: + """Seconds a new /blocks waits for the previous response's proof: 2 × the link's RTT + 1, at most + PROOF_GRACE. The only server-side use of link.rtt, and it must stay capped: on an incoming link it is + whatever the requester claims (RNS keeps the larger of what it measured and what the requester's RTT + packet says), so anything but a finite, non-negative int or float (a bool included) counts as none.""" + rtt = getattr(link, "rtt", None) + if type(rtt) not in (int, float) or not 0 <= rtt < math.inf: # NaN fails both comparisons + rtt = 0 + return float(min(PROOF_GRACE, 2 * rtt + 1)) + def _previous_done(self, link, link_id: bytes) -> bool: """Whether the link's previous /blocks response, if any, has finished, so a new one may start. @@ -1041,7 +1107,7 @@ def _previous_done(self, link, link_id: bytes) -> bool: previous = [r for r in self.pools.responses() if r.link_id == link_id and not r.finished] if not previous: return True - deadline = time.monotonic() + min(PROOF_GRACE, 2 * (getattr(link, "rtt", None) or 0) + 1) + deadline = self.clock() + self._proof_wait(link) for response in previous: if response.untracked: # Without RNS internals, the requester asking again is the only sign the response arrived @@ -1051,7 +1117,7 @@ def _previous_done(self, link, link_id: bytes) -> bool: resource = response.resource if resource is None or getattr(resource, "status", 0) < RNS.Resource.AWAITING_PROOF: return False # not yet sent, or still transferring - while not response.finished and time.monotonic() < deadline: + while not response.finished and self.clock() < deadline: if getattr(resource, "status", 0) >= RNS.Resource.COMPLETE: self._concluded(response, link, resource) break @@ -1078,14 +1144,14 @@ def register_outgoing_resource(resource, *args, **kwargs): result = original(resource, *args, **kwargs) try: self._registered(link, resource) - except Exception as e: + except Exception as e: # noqa: BLE001 - runs inside RNS's Resource code; logged log(f"error tracking a /blocks response: {e}", LOG_WARNING) return result register_outgoing_resource.rnfs_watch = True try: link.register_outgoing_resource = register_outgoing_resource - except Exception: + except Exception: # noqa: BLE001 - a link that can't be watched isn't return self.watched.add(link.link_id) @@ -1178,7 +1244,7 @@ def _settle(self, response: Response): try: sent = resource.get_progress() * (response.size or 0) if resource is not None else response.sent final = min(response.cost, int(max(0.0, sent))) - except Exception: + except Exception: # noqa: BLE001 - progress unknown: charged in full pass # progress unknown: it is charged in full with self.lock: response.charged = final @@ -1206,7 +1272,7 @@ def _budget_share_left(self, response: Response) -> int | None: def _concluded(self, response: Response, link, resource): """The response's Resource concluded: record the rate if it completed, start a cooldown if its requester - rejected it with nothing sent, and free the slots.""" + rejected or cancelled it before it made real progress (_made_progress), and free the slots.""" with self.lock: # The Resource's callback, the monitor and the next request may all notice; handle it once first, response.concluded = not response.concluded, True @@ -1214,28 +1280,34 @@ def _concluded(self, response: Response, link, resource): status = getattr(resource, "status", None) if first and status == RNS.Resource.COMPLETE: self.rates.record(response.link_id, response.size, self._expected_rate(link)) - elif first and status == RNS.Resource.REJECTED and not self._sent_any(response, resource): - # Turned down at its advertisement (no honest requester does that): no progress, so its requester - # cools down as for a response cancelled for making none, rather than having the node build another - # at once (section 3) - log("a /blocks response was rejected with nothing sent: its requester cools down", LOG_VERBOSE) + elif first and status == RNS.Resource.REJECTED and not self._made_progress(response, resource): + # Turned down at its advertisement or cancelled by its requester (on RNS 1.5.6 both arrive as + # REJECTED) before a quarter of it was sent: no honest 0.1.5 fetcher does that, it closes the link. + # So no real progress, and its requester cools down as for a response cancelled for making none, + # rather than having the node build another at once (section 3) + log(f"a /blocks response ({size(response.size or 0)}) was rejected before a quarter of it was sent: " + "its requester cools down", LOG_VERBOSE) self.cooldowns.add(response.link_id, response.identity) elif first and not cancelled and status in (RNS.Resource.FAILED, RNS.Resource.CORRUPT): - # Not cancelled here: RNS gave up on the transfer, so the requester sees it stop arriving + # Not cancelled here: RNS gave up on the transfer, or the requester closed the link, so the requester + # sees it stop arriving. No cooldown, however little was sent: an honest reader that gives up early + # closes its links, and in private mode a cooldown would hold back its own identity log(f"a /blocks response ({size(response.size or 0)}) failed after {size(int(response.sent))} sent: " f"{'its link closed' if getattr(link, 'status', None) == RNS.Link.CLOSED else 'lost in transit'}", LOG_DEBUG) self._finish(response) # charges what was sent: all of a completed Resource @staticmethod - def _sent_any(response: Response, resource) -> bool: - """Whether any of response's Resource was sent (or that can't be read).""" - if response.sent > 0: + def _made_progress(response: Response, resource) -> bool: + """Whether at least NO_PROGRESS_FRACTION of response was sent: the more of what the monitor last saw and + what its Resource reads now. True if its size or the Resource's progress can't be read.""" + if response.size is None: return True try: - return resource.get_progress() > 0 - except Exception: + sent = max(response.sent, resource.get_progress() * response.size) + except Exception: # noqa: BLE001 - progress unknown counts as progress return True + return sent >= NO_PROGRESS_FRACTION * response.size def _cancel(self, response: Response, reason: str): """Stop sending a response: cancel its Resource, or close its link if we can't reach the Resource.""" @@ -1267,7 +1339,7 @@ def _expected_rate(link) -> float | None: """Link.get_expected_rate(), or None if unavailable.""" try: return link.get_expected_rate() - except Exception: + except Exception: # noqa: BLE001 - RNS internals: no rate if they fail return None def _hook(self, response: Response, link, resource): @@ -1279,7 +1351,7 @@ def concluded(resource): try: resource.callback = concluded - except Exception: + except Exception: # noqa: BLE001 - RNS internals: the deadline still applies pass if getattr(resource, "status", 0) >= RNS.Resource.COMPLETE: # It concluded before the hook was in place @@ -1322,7 +1394,7 @@ def check_responses(self) -> bool: continue try: response.note_progress(resource.get_progress() * response.size, now) - except Exception: + except Exception: # noqa: BLE001 - progress unknown: only the deadline applies resource = None # progress unknown: only the deadline applies else: self._charge_progress(response) @@ -1358,7 +1430,7 @@ def _monitor_loop(self): searching = self.check_responses() # check_responses only looks at responses handed to RNS, and each handover sets wake idle = not any(r.size is not None and not r.finished for r in self.pools.responses()) - except Exception as e: + except Exception as e: # noqa: BLE001 - the monitor thread keeps running; logged log(f"error tracking /blocks responses: {e}", LOG_WARNING) searching = idle = False self.wake.wait(None if idle else MONITOR_FAST_TICK if searching else MONITOR_TICK) diff --git a/rnfs/node.py b/rnfs/node.py index ef158ca..daa9e06 100644 --- a/rnfs/node.py +++ b/rnfs/node.py @@ -10,7 +10,9 @@ repeated while too few servers are known), the publish loop, the name re-announce loop, the features' own, a worker per source with remote pins waiting, and maintenance (shared-folder scans, provider record expiry and saves, max_storage enforcement, periodic state saves, publisher refreshes asked for, deny-list changes retried, -the remote pins waiting saved). +the remote pins waiting saved). Stopping ends the features, then the node's services and threads (waiting for +them, the features' too, at most STOP_TIMEOUT in all), saves every store and closes the state, each step guarded: +one that fails skips none of the others. Threading: RNS calls the announce and request handlers on its own threads; facade calls come from API and gateway threads. Every component locks internally; the node itself only guards the small sets it owns. @@ -19,6 +21,7 @@ (network/guards.py, PROCESS); an embedded node scopes both to its rnfs.dht links (DHT) and leaves the host program's Reticulum running when it stops. Neither ever stops Reticulum: the daemon calls RNS.exit() itself. """ +import errno import hashlib import os import random @@ -27,25 +30,27 @@ import stat import threading import time +from collections import OrderedDict from contextlib import contextmanager -from typing import BinaryIO, Callable, Iterator +from functools import partial +from typing import BinaryIO, Callable, Iterator, Literal, Protocol, TypeVar, overload import RNS from rnfs import __version__, denylist, features, fsutil, statedb, winacl from rnfs.config import LOCK_FILE, Config -from rnfs.constants import APP_NAME, DEFAULT_BOOTSTRAP_NODES, DHT_ASPECT, K, MODE_SERVER, PARTIAL_SUFFIX +from rnfs.constants import (APP_NAME, DEFAULT_BOOTSTRAP_NODES, DHT_ASPECT, K, MISSED_ANNOUNCES, MODE_SERVER, + PARTIAL_SUFFIX) from rnfs.content import archive, builder, dag, edit, seal, walker from rnfs.content.cid import Cid from rnfs import node_content # --- content: add --no-pin/--to/--chunker, sealed files, gateway formats --- -from rnfs.content import chunker as content_chunker from rnfs.content.shared_folder import SharedFolder from rnfs.content.storage import FOLDER, REMOTE_PREFIX, USER, Storage, disk_reserve, remote_source from rnfs.dht.lookup import Lookup from rnfs.dht.providers import ProviderStore from rnfs.dht.routing import RoutingTable -from rnfs.errors import Denied, InvalidDag, NotFound, RnfsError, TooLarge -from rnfs.fetcher import Fetch, Fetcher, describe +from rnfs.errors import Cancelled, Denied, InvalidDag, NotFound, RnfsError, TooLarge +from rnfs.fetcher import Fetch, Fetcher, Observer, describe, observing from rnfs.mesh import RnsPaths from rnfs.log import LOG_DEBUG, LOG_ERROR, LOG_NOTICE, LOG_VERBOSE, LOG_WARNING, cap_warnings, log, node_name, plural, size from rnfs.names import NameManager, parse_name @@ -63,13 +68,20 @@ ARCHIVE_BLOCK_OVERHEAD = 64 # bytes an archive spends on each block beyond the block (msgpack framing, CID) EVICT_INTERVAL = 60.0 # seconds between max_storage checks when nothing else triggers one GATEWAY_NAME_AGE = 60.0 # a gateway resolution reuses a record heard this recently -STOP_TIMEOUT = 5.0 +STOP_TIMEOUT = 5.0 # seconds stop() waits in all for the threads it ends (the features' and the node's) REMOTE_PIN_QUEUE = 64 # remote pins one source may have waiting or being fetched; more are refused BOOTSTRAP_ANSWERS = 2 # a join stops contacting bootstrap nodes once this many have answered MIN_SERVERS = 3 # while fewer live servers are known, the join runs again REJOIN_DELAY = 300.0 # seconds before joining again; doubles after each join, up to announce_interval DENY_RETRY = 60.0 # seconds before maintenance applies a deny-list change again after applying it failed -KIND_NAMES = {dag.KIND_FILE: "file", dag.KIND_EXECUTABLE: "executable", dag.KIND_DIRECTORY: "directory"} +HASH_NOTICES = 1024 # files whose denied-hash NOTICE was logged, kept so it isn't repeated (then forgotten) +# Resolved targets found not to be sealed files, so reading one again needn't read its first bytes to tell (a CID's +# content never changes); least recently used forgotten first, about 2 MB when full +NOT_SEALED_MEMO = 4096 +# Complete pins whose pin_ls type, name from inside and sealed status are remembered (about 3 MB when full); pins +# past it are read each time +PIN_MEMO = 16384 +_UNOPENED = object() # _read's opened when nothing has been opened yet def import_fetched(value: list, tx) -> int: @@ -109,20 +121,26 @@ def import_remote_pins(value: list, tx) -> int: return count -def _make_private_dirs(config: Config): - """Create the rnfs dir and the node's subdirectories readable by the owner only (0700; on Windows, a - protected access list for the user and SYSTEM from the moment each is created, see winacl). +# The node's subdirectories of the rnfs dir (Config.state_dir, blocks_dir, identity_path's, names_dir; LXMF's +# ratchets), made private at each start. Not files/, the shared folder, whose contents the user chooses +PRIVATE_SUBDIRS = ("state", "blocks", "storage", "names", "lxmf") +_DIRECTORY_FLAGS = (os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | fsutil.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NONBLOCK", 0)) - They hold the node's identity, name keys and the local API token, so other local users must not read - them; a root left group- or world-accessible by earlier versions is tightened. Raises RnfsError if the - root belongs to another user or can't be made private: its owner could swap the identity or plant a - config there. On Windows also if the root is a junction or symlink, or new but not empty. + +def make_private_root(root: str): + """Create the rnfs dir root readable by the owner only (0700; on Windows, a protected access list for the + user and SYSTEM from the moment it is created, see winacl), or check and tighten an existing one. + + The first step of a start, before the settings in it are read (then _make_private_dirs, then the lock). It + holds the node's identity, name keys, settings and the local API token, so other local users must not read + or change it; a root left group- or world-accessible by earlier versions is tightened. Raises RnfsError if it + belongs to another user or can't be made private: its owner could swap the identity or plant a config there. + On Windows also if it is a junction or symlink, or new but not empty. """ - root = config.root - subdirs = (config.state_dir, config.blocks_dir, os.path.dirname(config.identity_path), config.names_dir) if fsutil.IS_WINDOWS: # No Unix permission bits there (st_mode always shows others allowed): an explicit access list - winacl.secure_rnfs_dir(root, subdirs) + winacl.secure_rnfs_dir(root) return os.makedirs(root, mode=0o700, exist_ok=True) info = os.stat(root) @@ -135,8 +153,94 @@ def _make_private_dirs(config: Config): except OSError as e: raise RnfsError(f"couldn't make {root} private to its owner (mode {mode:o}): {e}") from None log(f"made {root} private to its owner (it had mode {mode:o})", LOG_NOTICE) - for directory in subdirs: - os.makedirs(directory, mode=0o700, exist_ok=True) + + +def _make_private_dirs(root: str): + """Make the rnfs dir root private (make_private_root), then each of its PRIVATE_SUBDIRS: created 0700 if + missing (on Windows with the private access list), else checked and tightened (audit API-1). + + Tightening the root later doesn't take back a subdirectory from a user who could once write in a loose rnfs + dir: they may own one they created, or hold a descriptor on it, and plant blocks (trusted by their length) + or name keys. So on POSIX each is opened without following a link and checked through that descriptor: a + symlink, anything but a directory or one of another user's stops the start; a loose one is made 0700 with a + notice, and if others could write to it, entries in it owned by another user are reported (a warning: they + may be planted). Raises RnfsError for any of those, or if the root can't be made private. + """ + if fsutil.IS_WINDOWS: + winacl.secure_rnfs_dir(root, tuple(os.path.join(root, name) for name in PRIVATE_SUBDIRS)) + return + make_private_root(root) + for name in PRIVATE_SUBDIRS: + _make_private_subdir(os.path.join(root, name)) + + +def _make_private_subdir(path: str): + """_make_private_dirs on POSIX for one subdirectory at path. Raises RnfsError.""" + try: + fd = _open_subdir(path) + except OSError as e: + # O_NOFOLLOW's answer for a symlink is ELOOP (EMLINK on FreeBSD), or with O_DIRECTORY on Linux ENOTDIR + if e.errno not in (errno.ELOOP, errno.EMLINK, errno.ENOTDIR): + raise RnfsError(f"can't use {path}: {e}") from None + try: + link = fsutil.is_link(os.lstat(path)) + except OSError: + link = False + what = "a symlink" if link else "not a directory" + raise RnfsError(f"{path} is {what}; rnfs uses only a real directory of your own there. Move it away (rnfs " + f"creates a new one)") from None + try: + info = os.fstat(fd) + mode = stat.S_IMODE(info.st_mode) + fsutil.check_owned_fd(path, fd, info, "the rnfs dir's subdirectory") + except OSError as e: + raise RnfsError(f"{e}. rnfs uses only directories of your own there: move it away, or as root give it to " + f"yourself (chown -R {os.geteuid()} {path})") from None + finally: + os.close(fd) + if mode & 0o077: + log(f"made {path} private to its owner (it had mode {mode:o})", LOG_NOTICE) + if mode & 0o022: + _warn_foreign_entries(path) + + +def _open_subdir(path: str) -> int: + """A descriptor of the directory at path, not following a link (created 0700 if missing). Raises OSError: + ELOOP, EMLINK or ENOTDIR for a symlink, ENOTDIR for anything else that isn't a directory.""" + try: + fd = os.open(path, _DIRECTORY_FLAGS) + except FileNotFoundError: + try: + os.mkdir(path, 0o700) + except FileExistsError: + pass # made meanwhile, by another start: checked like any other + fd = os.open(path, _DIRECTORY_FLAGS) + if not stat.S_ISDIR(os.fstat(fd).st_mode): # where O_DIRECTORY is missing + os.close(fd) + raise NotADirectoryError(errno.ENOTDIR, "not a directory", path) + return fd + + +def _warn_foreign_entries(directory: str): + """Warn about entries below directory, which others could write to until now, that another user owns: they + may have been planted (in blocks/, a raw block is trusted by its length). One lstat pass, links not followed; + nothing is removed.""" + uid = os.geteuid() + count, first = 0, None + for parent, directories, files in os.walk(directory): + for name in directories + files: + entry = os.path.join(parent, name) + try: + if os.lstat(entry).st_uid == uid: + continue + except OSError: + continue # gone meanwhile + count += 1 + first = first or entry + if count: + log(f"{directory} was writable by other users, and {count} entr{'y' if count == 1 else 'ies'} in it " + f"belong to another user (e.g. {first}): they may have been put there to mislead rnfs. Check them, or " + f"delete them (rnfs fetches blocks again)", LOG_WARNING) class DirInUse(RnfsError): @@ -149,12 +253,13 @@ def __init__(self, message: str, holder: int | None): def lock_rnfs_dir(config: Config) -> int: - """Make the rnfs dir private (as the node would) and lock /daemon.lock without waiting, before - anything else uses the dir: two nodes on one dir would load and save the same state and announce the same + """Make the rnfs dir and its subdirectories private (as the node would) and lock /daemon.lock + without waiting, before anything else uses the dir (the last steps of a start: make_private_root, then the + settings, come first): two nodes on one dir would load and save the same state and announce the same node ID. The open descriptor, whose lock lasts until it is closed or the process ends. Raises DirInUse if another process, or another node in this one, holds it; RnfsError or OSError if the dir can't be made private or the file locked. The daemon and rnfs.Node both lock through here.""" - _make_private_dirs(config) + _make_private_dirs(config.root) path = os.path.join(config.root, LOCK_FILE) try: return fsutil.lock_exclusive(path) @@ -187,8 +292,16 @@ def _check_free_space(path: str, needed: int, blocks_dir: str | None = None, to_ f"while leaving {size(reserve)} free") -def _remove_partial(path: str): - """Delete a partial get output (file or directory tree) if there is one; a symlink is removed, not followed.""" +def _remove_partial(path: str, identity: walker.Identity | None = None): + """Delete a partial get output (file or directory tree) if there is one; a symlink is removed, not followed. + + With identity (a directory output's, from write_out), only if path is still that directory: whatever someone + put in its place isn't this get's to delete.""" + if identity is not None: + try: + walker.check_unchanged(path, identity) + except (ValueError, OSError): + return try: if os.path.isdir(path) and not os.path.islink(path): shutil.rmtree(path) @@ -198,6 +311,17 @@ def _remove_partial(path: str): log(f"removing {path} failed: {e!r}", LOG_WARNING) +def _inherit_from_parent(partial: str, output: str): + """On Windows, give a complete directory output the access its folder passes on, where it was private to the + user while written (walker._PathOutput). If that fails it stays private, which is safe, so a warning is + logged rather than the get failed.""" + try: + winacl.inherit_from_parent(partial) + except OSError as e: + log(f"get: {output} stays private to you, as its folder's access list couldn't be applied to it: {e}", + LOG_WARNING) + + def _remove_created(path: str, created: os.stat_result): """Remove path if it is still the file this call created (same device and inode), and nothing else.""" try: @@ -210,7 +334,8 @@ def _remove_created(path: str, created: os.stat_result): def _rns_is_known(node_id: bytes) -> bool: """True if RNS knows node_id's identity, so contacting it needs no path request for the key.""" - return RNS.Identity.recall(node_id) is not None + # A check, not a contact: don't mark the destination used (an RPC each on a shared instance) + return RNS.Identity.recall(node_id, _no_use=True) is not None def _warn_unused_reticulum_config(config: Config): @@ -222,6 +347,34 @@ def _warn_unused_reticulum_config(config: Config): f"Set reticulum_dir (or --rnsconfig) to {config.root} to keep using it, or delete it", LOG_WARNING) +class Store(Protocol): + """A part of the node that keeps state in the state database; the node lists every one (RnfsNode._add_store). + + save() writes what changed since the last save, if anything; a store written through has nothing to save. It + raises OSError (StateDbError) if the database fails, keeping the change for the next save. close() releases + the store's reference to the state database, once everything is saved; the last reference closes it. + """ + + def save(self) -> None: + """Write the changes not yet saved, if any.""" + + def close(self) -> None: + """Release the state database.""" + + +_StoreT = TypeVar("_StoreT", bound=Store) + + +class _StoreParts: + """A Store made of two functions: for a store that has no save (written through) or no close of its own, or for + the node's own tables.""" + + def __init__(self, save: Callable[[], None] = lambda: None, close: Callable[[], None] = lambda: None): + """A store whose save and close call save and close.""" + self.save = save + self.close = close + + class _NamePins: """The pins NameManager uses: storage pins that also update what the node provides.""" @@ -295,9 +448,9 @@ def is_member(self, identity_hash: bytes) -> bool: return identity_hash.hex() in self.node.members def feature_requests(self) -> dict: - """Request paths added by features (Feature.requests): {path: handler(data, remote_identity)}.""" - return {path: handler for feature in self.node.features.values() - for path, handler in feature.requests().items()} + """Request paths added by features (Feature.requests): {path: handler(data, remote_identity)}, as checked + when the node loaded them.""" + return dict(self.node._feature_requests) class RnfsNode: @@ -311,7 +464,7 @@ def __init__(self, config: Config, reticulum: "RNS.Reticulum | None" = None, emb program's running instance, or starts one if there is none; its guards cover only rnfs.dht links. """ config.validate() - _make_private_dirs(config) # before Reticulum and the identity, which write under it + _make_private_dirs(config.root) # before Reticulum and the identity, which write under it self.embedded = embedded # Reject oversized, compressed and request Resource advertisements before RNS allocates for them; in the # daemon on every link, with RNS's compression off process-wide (section 3) @@ -351,6 +504,7 @@ def _start_network(self, config: Config, slow: bool): # mesh: hops and next-hop interfaces rank providers; one set of traffic counters for client and server self.paths = RnsPaths(self.reticulum) self.fetcher.path_info = self.paths.info + self.fetcher.path_infos = self.paths.infos self.server = RequestServer(self.destination, config, self.backend, traffic=self.client.traffic) self.announcer = DhtAnnouncer(self.destination, config) @@ -394,7 +548,7 @@ def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_netwo self.client = client self.slow = slow self._is_known = is_known - self.members = {h.strip().strip("<>").lower() for h in config.members} + self.members = config.member_set # Private mode: node IDs that sent /provide as a member this run (their identity was checked then) self._member_publishers: set[bytes] = set() self._bootstrap_nodes: list[bytes] = [] # bootstrap_nodes that parse, in the order given @@ -403,31 +557,58 @@ def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_netwo self._bootstrap_nodes.append(parse_node_id(text)) except ValueError as e: log(f"ignoring bootstrap node: {e}", LOG_WARNING) - _make_private_dirs(config) - # First, so the JSON state files of earlier versions are imported before any store loads its state + _make_private_dirs(config.root) + # Every store, in the order made, and those _save_state saves (the others save on a schedule of their own). + # A store is added as soon as it is made, so a _setup that fails releases exactly the ones it made. The + # features are stores too, listed by self.features (_every_store) + self._stores: list[Store] = [] + self._batched: list[Store] = [] + # First, so the JSON state files of earlier versions are imported before any store loads its state; closed + # last self._db = statedb.connect(config.state_dir) + self._add_store(_StoreParts(close=self._db.close), batched=False) self.routing = RoutingTable(node_id, config, admit=self._is_member_node if config.private else None) - self.records = ProviderStore(config.max_records, config.state_dir, own_id=node_id, - seniority=config.provider_seniority) + # Saved with their expiry, every SAVE_INTERVAL (maintain): they change with every /provide + self.records = self._add_store(ProviderStore(config.max_records, config.state_dir, own_id=node_id, + seniority=config.provider_seniority), batched=False) if config.private: # Records stored while the node was public, or from a member removed since, are never handed out removed = self.records.retain(self._is_member_publisher) if removed: log(f"private mode: dropped {plural(removed, 'provider record')} of non-members", LOG_NOTICE) - self.storage = Storage(config.blocks_dir, config.state_dir, config.max_storage) + self.storage = self._add_store(Storage(config.blocks_dir, config.state_dir, config.max_storage)) # Before anything is fetched or provided: the fetcher and the publisher check it through storage.deny - self.deny = denylist.DenyList(config.state_dir) + self.deny = self._add_store(denylist.DenyList(config.state_dir)) self.storage.deny = self.deny self.deny.reload(self._read_deny_list) - self.shared_folder = SharedFolder(config.files_dir, self.storage, config.state_dir) - self.pin_names = PinNames(config.state_dir) # where pins came from, for pin ls (local only) - self.fetcher = Fetcher(self.storage, client, config, node_id, self.routing, self._lookup, slow, - config.state_dir, after_store=self._enforce_storage, make_room=self._make_room) - self.publisher = Publisher(node_id, config, self.storage, self.records, client, self._lookup, slow, - config.state_dir, cached_eligible=self._eligible_targets, - generation=self._provide_generation) + self._hash_noticed: set[Cid] = set() # files _check_file refused by hash and logged the NOTICE for + # _open_sealed's memo of targets that aren't sealed files (NOT_SEALED_MEMO), and the deny list's CIDs it + # was filled under: reading the first bytes also checks the deny list, so a change to it empties the memo + self._not_sealed: OrderedDict[walker.Expect, None] = OrderedDict() + self._not_sealed_deny: frozenset[bytes] | None = None + self._not_sealed_lock = threading.Lock() + # pin_ls's memo: pinned CID -> (type, name from inside, sealed), for complete pins only (_pin_details) + self._pin_memo: dict[Cid, tuple[str | None, str | None, bool | None]] = {} + self._pin_memo_lock = threading.Lock() + # The shared folder is written through: nothing to save. Its per-file cache (what the files inside its entries + # were built as) is saved with the batched stores; the shared folder's close releases it too + self.shared_folder = SharedFolder(config.files_dir, self.storage, config.state_dir, rnfs_dir=config.root) + self._add_store(_StoreParts(close=self.shared_folder.close), batched=False) + self._add_store(self.shared_folder.files) + # Where pins came from, for pin ls (local only). Written through, not durably: a power loss can lose a + # name, never the pin it names, which is committed durably first; its save writes a name whose write failed + self.pin_names = self._add_store(PinNames(config.state_dir), batched=False) + self.fetcher = self._add_store(Fetcher(self.storage, client, config, node_id, self.routing, self._lookup, + slow, config.state_dir, after_store=self._enforce_storage, + make_room=self._make_room)) + # Saved by its own refreshes and publish loop + self.publisher = self._add_store(Publisher(node_id, config, self.storage, self.records, client, self._lookup, + slow, config.state_dir, cached_eligible=self._eligible_targets, + generation=self._provide_generation), batched=False) + # Saves its state itself, and at stop (names.stop) self.names = NameManager(config, _NamePins(self), network=names_network) + self._add_store(_StoreParts(close=self.names.close), batched=False) self.backend = NodeBackend(self) self._lock = threading.Lock() @@ -443,12 +624,16 @@ def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_netwo # Changed since last saved (by _save_state: maintenance, gc and stop); a row that didn't parse is deleted self._fetched_dirty = len(self._fetched) != len(self._fetched_saved) self._fetched_save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last + self._add_store(_StoreParts(save=self._save_fetched)) # closed with self._db # Remote source -> CIDs waiting or being fetched for it, in the order accepted (fetched one at a time) self._remote_reserved: dict[str, dict[Cid, None]] = {} self._remote_released: set[tuple[str, Cid]] = set() # remote pins released while still being fetched - self._remote_workers: set[str] = set() # remote sources with a thread fetching their pins + # Remote source -> the thread fetching its pins (one at most); stop waits for them + self._remote_workers: dict[str, threading.Thread] = {} self._remote_pins_dirty = False # changed since last saved (by maintenance, and at stop) self._remote_pins_save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last + # Saved about every second (maintain); closed with self._db + self._add_store(_StoreParts(save=self._save_remote_pins), batched=False) self._refresh_wanted = threading.Event() # Taken first (before the publisher's and storage's locks): one deny-list change applied at a time self._deny_lock = threading.Lock() @@ -459,8 +644,12 @@ def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_netwo self.threads: list[threading.Thread] = [] self._stopped = False self.features: dict[str, features.Feature] = {} - for cls in features.feature_classes(): + # In order, each made before the next, which may use it (requires); a feature that can't be loaded as it + # declares stops the node here + for cls in features.checked_classes(): self.features[cls.name] = cls(self) + # Their request paths, checked now: one rnfs serves itself, or two features adding the same, stops the node + self._feature_requests = features.request_handlers(self.features.values()) # Releasing a deleted name key's pin recomputes what is provided, so it waits for the features too; and # name pins are made to match the names state, which a crash inside a publish can leave them disagreeing with self.names.reconcile_pins(self.storage.pins()) @@ -470,7 +659,29 @@ def _setup(self, config: Config, node_id: bytes, client, slow: bool, names_netwo # Changes apply at once; what the list denied at start, maintenance applies first self.deny.add_listener(self._apply_deny) - def feature(self, name: str) -> "features.Feature": + def _add_store(self, store: _StoreT, batched: bool = True) -> _StoreT: + """Add store to the node's stores, which are saved at stop and closed by _close_state; returns it. + + batched: _save_state saves it (every STATE_SAVE_INTERVAL, and at gc); False for a store saved on a + schedule of its own, or written through. + """ + self._stores.append(store) + if batched: + self._batched.append(store) + return store + + def _every_store(self, batched: bool = False) -> list[Store]: + """The node's stores (only the batched ones if batched), in the order made, then the features (each saved + with the batched stores). After a _setup that failed, only those it made.""" + stores = getattr(self, "_batched" if batched else "_stores", []) + return [*stores, *getattr(self, "features", {}).values()] + + @overload + def feature(self, name: Literal["contacts"]) -> "features.ContactsApi": ... + @overload + def feature(self, name: str) -> "features.Feature": ... + + def feature(self, name): """The feature called name (rnfs.features). Raises KeyError if it isn't loaded.""" return self.features[name] @@ -483,6 +694,11 @@ def _start_threads(self): thread.start() self.threads.append(thread) self._resume_remote_pins() + self._start_features() + + def _start_features(self): + """Start every feature, in order. One whose start() raises is logged and the rest still start, where one + whose constructor raises stopped the node from being made (_setup).""" for feature in self.features.values(): try: feature.start() @@ -490,42 +706,53 @@ def _start_threads(self): log(f"feature {feature.name} failed to start: {e!r}", LOG_ERROR) def stop(self, timeout: float = STOP_TIMEOUT): - """Stop background work, save state and close links. Work still waiting on the network is abandoned.""" + """Stop background work, save state and close links. Work still waiting on the network is abandoned. + + The features' threads and the node's are waited for at most timeout seconds in all. Each step is guarded: + one that raises is logged and the rest still run, so the state is saved and closed, and an embedded + node's destination released, whatever failed before. + """ if self._stopped: return self._stopped = True log("stopping", LOG_NOTICE) - for feature in reversed(list(self.features.values())): - try: - feature.stop() - except Exception as e: # noqa: BLE001 - keep stopping the rest - log(f"feature {feature.name} failed to stop: {e!r}", LOG_ERROR) - self.stop_event.set() - self.publisher.wake() + deadline = time.monotonic() + timeout + steps: list[tuple[str, Callable[[], object]]] = [ + (f"stopping feature {feature.name}", partial(features.stop_feature, feature, deadline)) + for feature in reversed(list(self.features.values()))] + steps += [("signalling the threads to stop", self.stop_event.set), + ("waking the publisher", self.publisher.wake)] if self.announcer is not None: - self.announcer.stop() + steps.append(("stopping the announcer", self.announcer.stop)) if self.announce_handler is not None: - self.announce_handler.deregister() + steps.append(("deregistering the announce handler", self.announce_handler.deregister)) if self.server is not None: - self.server.stop() - try: - self.names.stop() # saves the names state - except OSError as e: # stopping goes on: the rest is saved, the state closed and the links too - log(f"saving state failed: {e!r}", LOG_ERROR) - deadline = time.monotonic() + timeout - for thread in self.threads: - thread.join(max(0.0, deadline - time.monotonic())) - for save in (self.records.save, self._save_remote_pins, self.publisher.save): - try: - save() - except OSError as e: - log(f"saving state failed: {e!r}", LOG_ERROR) - self._save_state() - self._close_state() + steps.append(("stopping the request server", self.server.stop)) + steps.append(("deregistering names, saving state", self.names.stop)) # names.stop saves the names state + if self.links is not None: + # Before the wait: a thread still connecting to a node that has gone (the publisher, say) is freed + # now, and ends while the state database is open, rather than once it is closed + steps.append(("closing links", self.links.close_all)) + steps.append(("waiting for the background threads", partial(self._join_threads, deadline))) + steps += [("saving state", store.save) for store in self._every_store()] + steps.append(("closing state", self._close_state)) if self.links is not None: - self.links.close_all() + steps.append(("closing links opened since", self.links.close_all)) if self.embedded: - self._release_destination() + steps.append(("releasing the rnfs.dht destination", self._release_destination)) + for label, step in steps: + try: + step() + except Exception as e: # noqa: BLE001 - the later steps run: the state is closed, the destination released + log(f"{label} failed: {e!r}", LOG_ERROR) + + def _join_threads(self, deadline: float): + """Wait until deadline (a time.monotonic() value) for the background loops and the remote-pin workers to + end, once stop_event is set.""" + with self._lock: + workers = list(self._remote_workers.values()) + for thread in (*self.threads, *workers): + thread.join(max(0.0, deadline - time.monotonic())) def _release_destination(self): """An embedded node's last step: RNS stops answering for its rnfs.dht destination and the links to it @@ -539,25 +766,22 @@ def _release_destination(self): log(f"releasing the rnfs.dht destination failed: {e!r}", LOG_ERROR) def _close_state(self): - """Release every store's reference to the state database, once everything is saved; the last closes it. - After a _setup that failed, only the stores it made.""" - names = ("records", "storage", "deny", "shared_folder", "pin_names", "fetcher", "publisher", "names") - stores = [getattr(self, name) for name in names if hasattr(self, name)] - for store in (*stores, *getattr(self, "features", {}).values(), getattr(self, "_db", None)): - if store is None: - continue + """Release every store's reference to the state database, once everything is saved, in the reverse of the + order made (the node's own reference last, which closes it). After a _setup that failed, only the stores + it made.""" + for store in reversed(self._every_store()): try: store.close() except Exception as e: # noqa: BLE001 - release the rest log(f"closing state failed: {e!r}", LOG_ERROR) def _save_state(self): - """Save cache records, fetched targets, invalid roots and remembered file hashes if they changed (each - change isn't written at once).""" - for save in (self.storage.save, self._save_fetched, self.fetcher.save, self.deny.save, - *(f.save for f in self.features.values())): + """Save what changed in the batched stores (cache records, fetched targets, invalid roots, remembered file + hashes and the features' state: each change isn't written at once). A failing save is logged, and the rest + are still saved.""" + for store in self._every_store(batched=True): try: - save() + store.save() except OSError as e: log(f"saving state failed: {e!r}", LOG_ERROR) @@ -593,17 +817,28 @@ def _answered(self, node_id: bytes): def _heard_server(self, node_id: bytes): """A server-mode announce (or path response) from node_id.""" + self._count_heard(node_id, "server") if self.routing.heard(node_id): log(f"heard DHT server {node_name(node_id)} ({self.routing.node_count()} known)", LOG_VERBOSE) self.publisher.wake() def _heard_client(self, node_id: bytes): """node_id announces client mode: it doesn't belong in the routing table.""" + self._count_heard(node_id, "client") self.routing.remove(node_id) + def _count_heard(self, node_id: bytes, mode: str): + """Count node_id's announce in the traffic counters, which `rnfs peers` reads its DHT mode and last-heard + time from (client-mode nodes never enter the routing table). Done here rather than by a second rnfs.dht + announce handler in the peers feature: RNS starts a thread per handler for every announce.""" + traffic = getattr(self.client, "traffic", None) + if traffic is not None: + traffic.heard(node_id, mode) + def _is_member_node(self, node_id: bytes) -> bool: """Private mode's routing table admission: node_id's identity is a member.""" - identity = RNS.Identity.recall(node_id) + # Runs on every admission: a check, not a contact, so it doesn't mark the destination used + identity = RNS.Identity.recall(node_id, _no_use=True) return identity is not None and identity.hash.hex() in self.members def _is_member_publisher(self, node_id: bytes) -> bool: @@ -619,16 +854,17 @@ def excluded_provider(self, node_id: bytes) -> bool: return self.config.private and not self._is_member_publisher(node_id) def _seed_routing(self): - """Add servers whose announces RNS heard (and cached) within 2 × announce_interval, before a restart. + """Add servers whose announces RNS heard (and cached) within MISSED_ANNOUNCES (2) × announce_interval + (the routing table's silence limit), before a restart. RNS internal (Identity.known_destinations: hash -> [time heard, packet hash, public key, app_data, ...]); skipped if it isn't there. Each is checked to be an rnfs.dht destination of the announced key. """ try: entries = list(RNS.Identity.known_destinations.items()) - except Exception: + except Exception: # noqa: BLE001 - RNS internals: no seeding if they fail return - horizon = time.time() - 2 * self.config.announce_interval + horizon = time.time() - MISSED_ANNOUNCES * self.config.announce_interval added = 0 for destination_hash, entry in entries: try: @@ -638,7 +874,7 @@ def _seed_routing(self): identity.load_public_key(entry[2]) if RNS.Destination.hash(identity, APP_NAME, DHT_ASPECT) == destination_hash: added += self.routing.heard(bytes(destination_hash)) - except Exception: + except Exception: # noqa: BLE001 - one bad entry is skipped continue if added: log(f"{plural(added, 'DHT server')} known from announces heard before", LOG_VERBOSE) @@ -653,7 +889,7 @@ def _join(self): if self.reticulum is not None: self._seed_routing() self.join() - except Exception as e: + except Exception as e: # noqa: BLE001 - joining is retried; logged log(f"joining the DHT failed: {e!r}", LOG_ERROR) finally: self.joined.set() @@ -664,7 +900,7 @@ def _join(self): continue try: self.join() - except Exception as e: + except Exception as e: # noqa: BLE001 - joining is retried; logged log(f"joining the DHT failed: {e!r}", LOG_ERROR) delay = min(2 * delay, self.config.announce_interval) @@ -707,7 +943,7 @@ def _contact_bootstrap(self) -> tuple[set[bytes], list[bytes]]: contacted.add(node_id) try: reply = self.client.find(node_id, self.node_id) - except Exception: + except Exception: # noqa: BLE001 - an unanswering bootstrap node is skipped reply = None if reply is None: log(f"bootstrap node {node_name(node_id)} didn't answer", LOG_VERBOSE) @@ -747,10 +983,9 @@ def maintain(self, now: float, last: dict) -> None: if now - last.get("scan", float("-inf")) >= self.config.scan_interval: last["scan"] = now try: - with builder.checking(self._hash_check()): - self.shared_folder.scan() + self.shared_folder.scan(builder.BuildOptions(check=self._hash_check())) self.publisher.refresh() - except Exception as e: + except Exception as e: # noqa: BLE001 - the next scan tries again; logged log(f"shared folder scan failed: {e!r}", LOG_ERROR) if now - last.get("save", now) >= SAVE_INTERVAL: last["save"] = now @@ -785,8 +1020,8 @@ def _read_deny_list(self, cid: Cid) -> bytes | None: return b"".join(walker.read_range(expect, get)) def _hash_check(self): - """What builder.checking takes for an add or import: raises Denied for a file whose content hash is - denied; None while the deny list has no file hashes (so nothing is hashed).""" + """The check of an add's or the shared folder's builder.BuildOptions: raises Denied for a file whose content + hash is denied; None while the deny list has no file hashes (so nothing is hashed).""" if not self.deny.hashes: return None @@ -796,22 +1031,52 @@ def check(digest: bytes): raise Denied(f"a file with SHA-256 {digest.hex()} is on this node's deny list") return check - def _check_file(self, expect: walker.Expect, read: Callable[[walker.Expect], Iterator[bytes]]): + def _check_file(self, expect: walker.Expect, get: walker.Get): """Raise Denied if file expect's content hashes to a deny-list entry, before any of it is written out or - sent. read(expect) gives its bytes, read only if the hash isn't known yet (a one-block file's CID is - it; others are remembered by the deny list).""" + sent. get serves its whole DAG, stored and checked (fetch_dag's, or a complete root's); its bytes are read + only if the hash isn't known yet (a one-block file's CID is it; others are remembered by the deny list). + A denied hash is logged once per file (a NOTICE suggesting `rnfs deny add `, which also stops its + blocks being served). + + A file declaring more than DENY_HASH_RATIO × the bytes of its distinct chunks + DENY_HASH_SLACK is refused + unread: hashing reads every occurrence of repeated content, so six small blocks can declare 2^56 bytes. + """ if not self.deny.hashes: return digest = expect.cid.digest if expect.cid.is_raw else self.deny.known_hash(expect.cid) if digest is None: + declared = walker.size_of(expect, get(expect)) + if declared > denylist.DENY_HASH_SLACK: + # The distinct chunk bytes it needs, at least: the walk stops once they are met + needed = -(-(declared - denylist.DENY_HASH_SLACK) // denylist.DENY_HASH_RATIO) + if walker.leaf_bytes(expect, get, needed) < needed: + raise Denied(f"{expect.cid} can't be checked against this node's deny-list file hashes: it " + f"repeats its content more than {denylist.DENY_HASH_RATIO}×") hasher = hashlib.sha256() - for data in read(expect): + for data in walker.read_range(expect, get): hasher.update(data) digest = hasher.digest() self.deny.remember_hash(expect.cid, digest) if self.deny.denies_hash(digest): + # A hash entry stops the file being assembled, not its blocks being served one by one (spec section 8) + if expect.cid not in self._hash_noticed: + if len(self._hash_noticed) >= HASH_NOTICES: + self._hash_noticed.clear() + self._hash_noticed.add(expect.cid) + log(f"{expect.cid} is a file whose SHA-256 is denied: it isn't read or sent whole, but its blocks can " + f"still be fetched, stored and served one by one. To stop that too, deny its CID: " + f"rnfs deny add {expect.cid}", LOG_NOTICE) raise Denied(f"{expect.cid} is a file whose SHA-256 ({digest.hex()}) is on this node's deny list") + def _check_fetched(self, fetch: Fetch, expect: walker.Expect): + """_check_file for a file a fetch reads (cat, the gateway, a sealed get): unless its hash is known, its + whole DAG is fetched first (fetch_dag: max_download_size, the disk reserve, the DAG byte bound).""" + if not self.deny.hashes: + return + if not expect.cid.is_raw and self.deny.known_hash(expect.cid) is None: + fetch.fetch_dag(expect, self.config.max_output_entries) + self._check_file(expect, fetch.get) + def _apply_deny(self): """Bring what is provided, cached and pinned in line with the deny list's CIDs, after a change and at start: stop providing (and withdraw) what reaches a denied block; drop cached fetches that do, deleting @@ -863,7 +1128,7 @@ def _maintenance_loop(self): while not self.stop_event.is_set(): try: self.maintain(time.monotonic(), last) - except Exception as e: + except Exception as e: # noqa: BLE001 - the maintenance thread keeps running; logged log(f"maintenance failed: {e!r}", LOG_ERROR) self.stop_event.wait(1.0) @@ -944,7 +1209,7 @@ def _fetching(self, root: Cid, names: list[str], from_node: bytes | None = None, finally: try: self._enforce_storage() - except Exception as e: # never hide the fetch's own outcome + except Exception as e: # noqa: BLE001 - never hide the fetch's own outcome log(f"enforcing max_storage after a fetch failed: {e!r}", LOG_ERROR) def _record_fetch(self, fetch: Fetch, for_gateway: bool): @@ -976,21 +1241,22 @@ def add_path(self, path: str, wrap: bool = False, only_hash: bool = False, pin: if not os.path.lexists(path): raise NotFound(f"no such file or directory: {path}") recipients = node_content.seal_recipients(self, to, to_self, only_hash) - with content_chunker.using(chunker): - if only_hash: - built, warnings = builder.add_path(path, builder.NO_PUT, wrap) - else: - with self._refusing(), self.storage.protect() as session: - with session.writer() as put: - if recipients: - built = node_content.add_sealed_path(path, put, wrap, recipients, seal_name) - warnings = [] - else: - built, warnings = builder.add_path(path, put, wrap) - node_content.keep_added(self, built.cid, pin) - if pin: - self.pin_names.record(built.cid, os.path.abspath(path)) - self._added(built) + options = builder.BuildOptions(chunker) # ValueError for an unknown chunker, before anything is read + if only_hash: + built, warnings = builder.add_path(path, builder.NO_PUT, wrap, options) + else: + options = builder.BuildOptions(options.chunker, self._hash_check()) + with self._refusing(), self.storage.protect() as session: + with session.writer() as put: + if recipients: + built = node_content.add_sealed_path(path, put, wrap, recipients, seal_name, options) + warnings = [] + else: + built, warnings = builder.add_path(path, put, wrap, options) + node_content.keep_added(self, built.cid, pin) + if pin: + self.pin_names.record(built.cid, os.path.abspath(path)) + self._added(built) return {"cid": str(built.cid), "size": built.size, "warnings": warnings, **({"sealed": True} if recipients else {})} def add_stream(self, stream: BinaryIO, size: int | None = None, name: str | None = None, @@ -1000,17 +1266,18 @@ def add_stream(self, stream: BinaryIO, size: int | None = None, name: str | None only_hash; sealed for to/to_self (with seal_name inside) if given; chunker as for add_path.""" recipients = node_content.seal_recipients(self, to, to_self, only_hash) stream, size = node_content.sealing_stream(stream, size, recipients, seal_name) - with content_chunker.using(chunker): - if only_hash: - built = builder.add_stream(stream, builder.NO_PUT, size, name) - else: - with self._refusing(), self.storage.protect() as session: - with session.writer() as put: - built = builder.add_stream(stream, put, size, name) - node_content.keep_added(self, built.cid, pin) - if pin: - self.pin_names.record(built.cid, name or seal_name or "(stdin)") - self._added(built) + options = builder.BuildOptions(chunker) # ValueError for an unknown chunker, before anything is read + if only_hash: + built = builder.add_stream(stream, builder.NO_PUT, size, name, options=options) + else: + options = builder.BuildOptions(options.chunker, self._hash_check()) + with self._refusing(), self.storage.protect() as session: + with session.writer() as put: + built = builder.add_stream(stream, put, size, name, options=options) + node_content.keep_added(self, built.cid, pin) + if pin: + self.pin_names.record(built.cid, name or seal_name or "(stdin)") + self._added(built) return {"cid": str(built.cid), "size": built.size, "warnings": [], **({"sealed": True} if recipients else {})} def block(self, cid: str, for_gateway: bool = False) -> bytes: @@ -1024,11 +1291,11 @@ def export_stream(self, target: str, start, for_gateway: bool = False) -> dict: @contextmanager def _refusing(self) -> Iterator[None]: - """Around an add or import: files whose content hash is denied stop it (builder.checking), and when the - deny list stops it, the blocks it stored are deleted once nothing reaches them.""" + """Around an add or import, which the deny list may stop (a denied block, or a file whose content hash is + denied: an add's BuildOptions check, _hash_check): when it does, the blocks it stored are deleted once + nothing reaches them.""" try: - with builder.checking(self._hash_check()): - yield + yield except Denied: self.storage.collect() raise @@ -1059,6 +1326,7 @@ def get(self, target: str, output: str, from_node: bytes | None = None, raw: boo # write-out never leaves a partial output under the name the user asked for partial = os.path.join(os.path.dirname(os.path.abspath(output)), f".rnfs-get-{secrets.token_hex(8)}{PARTIAL_SUFFIX}") + created: list[walker.Identity] = [] # a directory output's identity, once write_out has made it try: with self._fetching(root, names, from_node) as fetch: expect = fetch.resolve() @@ -1070,21 +1338,25 @@ def get(self, target: str, output: str, from_node: bytes | None = None, raw: boo _check_free_space(output, content, self.config.blocks_dir, to_fetch) sealed = None if raw else node_content.open_sealed(self, fetch, expect) if sealed is not None: # decrypted as it is fetched (rnfs/content/seal.py) - self._check_file(expect, fetch.read_range) # its stored (sealed) bytes, as others hold them + self._check_fetched(fetch, expect) # its stored (sealed) bytes, as others hold them written = node_content.write_sealed(sealed, partial, self.config.blocks_dir, to_fetch) result = {"cid": str(expect.cid), "warnings": [], **written} else: fetch.fetch_dag(expect, self.config.max_output_entries) # Each file is checked before it is written while the deny list has file hashes - check = {"check": lambda file: self._check_file(file, lambda e: walker.read_range(e, fetch.get)) - } if self.deny.hashes else {} - warnings = walker.write_out(expect, fetch.get, partial, self.config.max_output_entries, **check) + check = {"check": lambda file: self._check_file(file, fetch.get)} if self.deny.hashes else {} + warnings = walker.write_out(expect, fetch.get, partial, self.config.max_output_entries, + created=created, **check) # Every skipped entry is a warning: keep a response (and a job's saved result) small result = {"cid": str(expect.cid), **cap_warnings(warnings)} + if created: + # Still the directory written, not one put in its place by someone who can write beside it + walker.check_unchanged(partial, created[0]) + _inherit_from_parent(partial, output) fsutil.rename_new(partial, output) return result finally: - _remove_partial(partial) + _remove_partial(partial, created[0] if created else None) def info(self, target: str, for_gateway: bool = False, from_node: bytes | None = None, raw: bool = False) -> dict: """The target's own CID, type, size and executable bit (fetching the root, path and target blocks). @@ -1093,14 +1365,40 @@ def info(self, target: str, for_gateway: bool = False, from_node: bytes | None = """ root, names = self._parse_target(target, for_gateway) with self._fetching(root, names, from_node, for_gateway) as fetch: - return self._info(fetch, fetch.resolve(), for_gateway or raw) + return self._info(fetch, fetch.resolve(), for_gateway or raw)[0] - def _info(self, fetch: Fetch, expect: walker.Expect, stored: bool) -> dict: - """info() of a resolved target; with stored, a sealed file's as stored (its ciphertext size).""" + def _info(self, fetch: Fetch, expect: walker.Expect, stored: bool) -> tuple[dict, seal.SealedFile | None]: + """(info() of a resolved target, the sealed file it opened); with stored, a sealed file's info as stored + (its ciphertext size) and None. The sealed file is unlocked when this node is a recipient.""" block = fetch.get(expect) info = {"cid": str(expect.cid), "type": walker.kind_of(expect, block), "size": walker.size_of(expect, block), "executable": expect.kind == dag.KIND_EXECUTABLE} - return info if stored else node_content.sealed_info(self, fetch, expect, info) + if stored or info["type"] != walker.FILE_TYPE: + return info, None + sealed = self._open_sealed(fetch, expect, unlock=False) + return node_content.sealed_info(self, info, sealed), sealed + + def _open_sealed(self, fetch: Fetch, expect: walker.Expect, unlock: bool = True) -> seal.SealedFile | None: + """node_content.open_sealed, without reading the first bytes again of a target already found not to be + sealed under the same deny list (_not_sealed).""" + denied = self.deny.cids # swapped whole when the list changes + with self._not_sealed_lock: + if self._not_sealed_deny is not denied: + # Reading the first bytes meets the deny list (Denied for a denied first chunk), so what was + # found under another list is found again + self._not_sealed.clear() + self._not_sealed_deny = denied + elif expect in self._not_sealed: + self._not_sealed.move_to_end(expect) + return None + sealed = node_content.open_sealed(self, fetch, expect, unlock) + if sealed is None: + with self._not_sealed_lock: + if self._not_sealed_deny is denied: + self._not_sealed[expect] = None + if len(self._not_sealed) > NOT_SEALED_MEMO: + self._not_sealed.popitem(last=False) + return sealed def read(self, target: str, offset: int = 0, length: int | None = None, for_gateway: bool = False, from_node: bytes | None = None, raw: bool = False) -> Iterator[bytes]: @@ -1133,7 +1431,7 @@ def generate() -> Iterator: """The info, then the bytes; closed before reading them, the fetch ends as info()'s does.""" with self._fetching(root, names, from_node) as fetch: expect = fetch.resolve() - info = self._info(fetch, expect, raw) + info, sealed = self._info(fetch, expect, raw) try: yield info except GeneratorExit: @@ -1142,18 +1440,26 @@ def generate() -> Iterator: raise ValueError(f"{target} is a directory") available = max(info["size"] - offset, 0) yield from self._read(fetch, expect, offset, available if length is None else min(length, available), - False, raw) + False, raw, sealed) chunks = generate() return next(chunks), chunks def _read(self, fetch: Fetch, expect: walker.Expect, offset: int, length: int | None, for_gateway: bool, - raw: bool) -> Iterator[bytes]: - """read() of a resolved target, in fetch.""" - if self.deny.hashes and walker.kind_of(expect, fetch.get(expect)) == "file": + raw: bool, opened: "seal.SealedFile | None | object" = _UNOPENED) -> Iterator[bytes]: + """read() of a resolved target, in fetch. opened: what _info opened in the same fetch (read_file), so the + file's first bytes aren't read again to tell whether it is sealed.""" + if self.deny.hashes and walker.kind_of(expect, fetch.get(expect)) == walker.FILE_TYPE: # The whole file is fetched and hashed (once: then remembered) before any of it is sent - self._check_file(expect, fetch.read_range) - sealed = None if raw or for_gateway else node_content.open_sealed(self, fetch, expect) + self._check_fetched(fetch, expect) + if raw or for_gateway: + sealed = None + elif opened is _UNOPENED: + sealed = self._open_sealed(fetch, expect) + else: + sealed = opened + if sealed is not None and not node_content.is_recipient(self, sealed): + node_content.unlock_sealed(self, sealed) # raises NotRecipient after the deny check, as opening did if sealed is not None: yield from sealed.read(offset, length) return @@ -1164,10 +1470,10 @@ def ls(self, target: str, for_gateway: bool = False) -> list[dict]: root, names = self._parse_target(target, for_gateway) with self._fetching(root, names, None, for_gateway) as fetch: expect = fetch.resolve() - if walker.kind_of(expect, fetch.get(expect)) != "directory": + if walker.kind_of(expect, fetch.get(expect)) != walker.DIRECTORY_TYPE: raise ValueError(f"{target} is not a directory") fetch.fetch_listing(expect) - return [{"name": entry.name, "kind": KIND_NAMES[entry.kind], "size": entry.size, "cid": str(entry.cid)} + return [{"name": entry.name, "kind": dag.KIND_NAMES[entry.kind], "size": entry.size, "cid": str(entry.cid)} for entry in walker.list_directory(expect, fetch.get, self.config.max_output_entries)] def stat(self, target: str) -> dict: @@ -1187,7 +1493,8 @@ def is_local(self, target: str) -> bool: return False def stored_head(self, target: str, length: int) -> bytes | None: - """A file's first length bytes if every block they need is stored, else None. Never fetches.""" + """A file's first length bytes if every block they need is stored, else None. Never fetches. Raises + Denied as a read does (node_content.stored_head).""" return node_content.stored_head(self, target, length) def dir_set(self, target: str, name: str, child_cid: str, executable: bool = False) -> dict: @@ -1207,7 +1514,9 @@ def dir_set(self, target: str, name: str, child_cid: str, executable: bool = Fal return {"cid": str(new_root)} def dir_rm(self, target: str) -> dict: - """A new root without the entry target names; pinned "user" (the complete directory is fetched first).""" + """A new root without the entry target names; pinned "user" (the complete directory is fetched first). + Raises ValueError, before fetching anything, unless target is /.""" + edit.check_entry_target(target) root, names = self._parse_target(target) with self.fetcher.open(root, pinned=True) as fetch: fetch.fetch_dag(fetch.resolve()) @@ -1251,18 +1560,44 @@ def pin_ls(self, cid: str | None = None) -> list[dict]: """ pins = self.storage.pins() self.pin_names.prune(set(pins)) + with self._pin_memo_lock: + for gone in [c for c in self._pin_memo if c not in pins]: + del self._pin_memo[gone] if cid is not None: wanted = Cid.parse(cid) pins = {wanted: pins[wanted]} if wanted in pins else {} folder_names = self.shared_folder.names() result = [] for c, sources in sorted(pins.items(), key=lambda item: str(item[0])): - kind, inside = self._pin_shape(c) + complete = self.storage.is_complete(c) + kind, inside, sealed = self._pin_details(c, complete) name = self.pin_names.get(c) or (folder_names.get(c) if FOLDER in sources else None) or inside - result.append({"cid": str(c), "sources": sorted(sources), "complete": self.storage.is_complete(c), - "type": kind, "sealed": self._pin_sealed(c, kind), "name": name}) + result.append({"cid": str(c), "sources": sorted(sources), "complete": complete, + "type": kind, "sealed": sealed, "name": name}) return result + def _pin_details(self, cid: Cid, complete: bool) -> tuple[str | None, str | None, bool | None]: + """(type, name from inside, sealed) of a pin (_pin_shape, _pin_sealed), remembered while it is complete. + + They are read from stored blocks, which a complete pin has all of and whose content never changes, so + they hold for as long as it stays complete; an incomplete pin's are read each time (a block may arrive, + or have gone). pin_ls forgets the pins released; at most PIN_MEMO are remembered, the first met kept: + pin_ls goes through them in the same order each time, so replacing older ones would remember none. + """ + if complete: + with self._pin_memo_lock: + known = self._pin_memo.get(cid) + if known is not None: + return known + kind, inside = self._pin_shape(cid) + details = (kind, inside, self._pin_sealed(cid, kind)) + # A complete DAG's blocks always tell both; None means a block went while they were read + if complete and kind is not None and details[2] is not None: + with self._pin_memo_lock: + if len(self._pin_memo) < PIN_MEMO: + self._pin_memo[cid] = details + return details + def _pin_sealed(self, cid: Cid, kind: str | None) -> bool | None: """True if a pinned file is sealed (its first bytes carry the sealed-file magic), False if it isn't or is a directory, None if that isn't known from stored blocks. Never fetches.""" @@ -1270,7 +1605,8 @@ def _pin_sealed(self, cid: Cid, kind: str | None) -> bool | None: return False if kind is None: return None - head = node_content.stored_head(self, str(cid), len(seal.MAGIC)) + # The user's own list of what is stored: whether a denied pin is sealed is no secret, and isn't worth a hash + head = node_content.stored_head(self, str(cid), len(seal.MAGIC), deny=False) return None if head is None else seal.is_sealed(head) def _pin_shape(self, cid: Cid) -> tuple[str | None, str | None]: @@ -1358,7 +1694,7 @@ def _ensure_remote_worker(self, source: str, cid: Cid) -> bool: log(f"refused remote pin of {cid} for {source}: couldn't start its fetch: {e}", LOG_ERROR) return False # Registered after start() succeeds, still under the lock, so the worker can't look before it is - self._remote_workers.add(source) + self._remote_workers[source] = thread return True def _remote_pin_worker(self, source: str): @@ -1377,13 +1713,13 @@ def _remote_pin_worker(self, source: str): if cid is None: # Deregistered in the same critical section that saw nothing waiting: a /pin arriving after # it starts a new worker instead of relying on this one, which is about to return - self._remote_workers.discard(source) + self._remote_workers.pop(source, None) return try: self._remote_pin_fetch(cid, source) # removes cid from _remote_reserved when done except BaseException: with self._lock: - self._remote_workers.discard(source) + self._remote_workers.pop(source, None) raise def _save_remote_pins(self): @@ -1410,7 +1746,7 @@ def _save_remote_pins(self): def _load_remote_pins(self) -> list[tuple[str, Cid]]: """Remote pins saved as waiting, oldest first, from identities still on remote_pin_identities (at most REMOTE_PIN_QUEUE each); others are dropped. Caller holds no lock (it reads the database).""" - allowed = {remote_source(h.strip().strip("<>")) for h in self.config.remote_pin_identities} + allowed = {remote_source(h) for h in self.config.remote_pin_set} try: saved = self._db.query("SELECT source, cid FROM remote_pins ORDER BY position") except OSError as e: @@ -1456,13 +1792,15 @@ def _resume_remote_pins(self): def _remote_pin_fetch(self, cid: Cid, source: str): """Fetch cid for a remote pin, dropping it as soon as its verified size, or the bytes of its DAG's - blocks met so far, would exceed the quota.""" + blocks met so far, would exceed the quota. The fetch ends at its next step once the node stops.""" quota = self.config.remote_pin_quota + observer = Observer() + observer.cancelled = self.stop_event.is_set try: with self._lock: if (source, cid) in self._remote_released: return # released while it waited - with self.fetcher.open(cid, pinned=True) as fetch: + with observing(observer), self.fetcher.open(cid, pinned=True) as fetch: expect = fetch.resolve() content = fetch.size_of(expect) # Nothing else is being fetched for source (_remote_pin_worker), so its pins are all it uses @@ -1488,9 +1826,12 @@ def _remote_pin_fetch(self, cid: Cid, source: str): self.storage.pin(cid, source) log(f"pinned {cid} for {source}", LOG_NOTICE) self.publisher.refresh() + except Cancelled: + log(f"remote pin of {cid} for {source} stopped with the node: it is fetched after the next start", + LOG_VERBOSE) except RnfsError as e: log(f"remote pin of {cid} for {source} failed: {e}", LOG_WARNING) - except Exception as e: + except Exception as e: # noqa: BLE001 - the pin waits for the next try; logged log(f"remote pin of {cid} for {source} failed: {e!r}", LOG_ERROR) finally: with self._lock: @@ -1613,7 +1954,7 @@ def check(roots: list[Cid]): get = self.storage.getter() for root in roots: for file in walker.iter_files(walker.root_expect(root), get): - self._check_file(file, lambda e: walker.read_range(e, get)) + self._check_file(file, get) # a complete root: every block is stored and checked with stream, self._refusing(): stats = archive.import_archive(stream, self.storage, check=check) diff --git a/rnfs/node_content.py b/rnfs/node_content.py index 4c8fac5..56ccb0b 100644 --- a/rnfs/node_content.py +++ b/rnfs/node_content.py @@ -16,7 +16,7 @@ from rnfs.content import archive, builder, dag, seal, walker from rnfs.content.cid import Cid from rnfs.content.storage import USER -from rnfs.errors import InvalidDag, NotFound +from rnfs.errors import Denied, InvalidDag, NotFound def keep_added(node, cid: Cid, pin: bool): @@ -81,9 +81,10 @@ def sealing_stream(stream: BinaryIO, size: int | None, recipients: list[bytes], return seal.SealingReader(stream, recipients, seal_name, size), None -def add_sealed_path(path: str, put: builder.Put, wrap: bool, recipients: list[bytes], - seal_name: bool) -> builder.Built: - """Seal the regular file at path and build it (wrapped under its visible name if wrap).""" +def add_sealed_path(path: str, put: builder.Put, wrap: bool, recipients: list[bytes], seal_name: bool, + options: builder.BuildOptions | None = None) -> builder.Built: + """Seal the regular file at path and build it (wrapped under its visible name if wrap), with options (the + chunker, and the deny check of the sealed bytes; builder.build_file).""" handle, st = builder.open_path(path) try: if not stat.S_ISREG(st.st_mode): @@ -92,7 +93,7 @@ def add_sealed_path(path: str, put: builder.Put, wrap: bool, recipients: list[by sealed_name = name if seal_name and dag.is_valid_name(name) else None with os.fdopen(handle, "rb", closefd=False) as f: return builder.add_stream(seal.SealingReader(f, recipients, sealed_name), put, None, - name if wrap else None) + name if wrap else None, options=options) finally: builder.close(handle) @@ -103,7 +104,7 @@ def open_sealed(node, fetch, expect: walker.Expect, unlock: bool = True) -> seal Raises NotRecipient (unlocking) if the node's identity can't open it, SealError if it is damaged. """ block = fetch.get(expect) - if walker.kind_of(expect, block) != "file": + if walker.kind_of(expect, block) != walker.FILE_TYPE: return None stored_size = walker.size_of(expect, block) sealed = seal.open_sealed(lambda offset, length: fetch.read_range(expect, offset, length), stored_size) @@ -112,26 +113,35 @@ def open_sealed(node, fetch, expect: walker.Expect, unlock: bool = True) -> seal # would show "of ?": its stored size is what the chunks met add up to fetch.observer.total(stored_size) if sealed is not None and unlock: - identity = getattr(node, "identity", None) - if identity is None: - raise seal.NotRecipient("this node has no identity to open sealed files with (use --raw)") - sealed.unlock(identity) + unlock_sealed(node, sealed) return sealed -def sealed_info(node, fetch, expect: walker.Expect, info: dict) -> dict: - """info, plus for a sealed file "sealed", "recipient" and "stored_size", and when this node is a - recipient its plaintext "size" and sealed "name".""" - if info.get("type") != "file": - return info - sealed = open_sealed(node, fetch, expect, unlock=False) +def unlock_sealed(node, sealed: seal.SealedFile): + """Unlock sealed with the node's identity. Raises NotRecipient if the node has none or isn't a recipient, + SealError if the content key sealed for it doesn't decrypt.""" + identity = getattr(node, "identity", None) + if identity is None: + raise seal.NotRecipient("this node has no identity to open sealed files with (use --raw)") + sealed.unlock(identity) + + +def is_recipient(node, sealed: seal.SealedFile) -> bool: + """True if the node's identity is one of sealed's recipients.""" + identity = getattr(node, "identity", None) + return identity is not None and bytes(identity.hash) in sealed.recipients + + +def sealed_info(node, info: dict, sealed: seal.SealedFile | None) -> dict: + """A file's info, plus when it is sealed (sealed, as open_sealed opened it without unlocking) "sealed", + "recipient" and "stored_size", and when this node is a recipient its plaintext "size" and sealed "name" + (sealed is then unlocked).""" if sealed is None: return info - identity = getattr(node, "identity", None) - recipient = identity is not None and bytes(identity.hash) in sealed.recipients + recipient = is_recipient(node, sealed) info = {**info, "sealed": True, "recipient": recipient, "stored_size": info["size"]} if recipient: - sealed.unlock(identity) + unlock_sealed(node, sealed) info.update(size=sealed.size, name=sealed_name(sealed)) return info @@ -171,18 +181,43 @@ def write_sealed(sealed: seal.SealedFile, output: str, blocks_dir: str | None = return {"sealed": True, "name": name} -def stored_head(node, target: str, length: int) -> bytes | None: +def stored_head(node, target: str, length: int, deny: bool = True) -> bytes | None: """A file's first length bytes from stored blocks only (the gateway's sealed check and sniffing), or - None if one they need isn't stored (or the target is a directory or invalid). Never fetches.""" + None if one they need isn't stored (or the target is a directory or invalid). Never fetches. + + With deny, the deny list applies as to a read (a HEAD of a denied file answers 451, like GET): Denied for a + denied CID met, and while there are file hashes the file is checked first (node._check_file, bounded), which + takes it wholly stored (None otherwise: it can't be checked without fetching). `pin ls` passes deny=False. + """ try: root, names = walker.parse_target(target) get = node.storage.getter() + if deny: + get = denying(node, get) expect = walker.resolve(root, names, get) + if deny and node.deny.hashes and walker.kind_of(expect, get(expect)) == walker.FILE_TYPE: + if not expect.cid.is_raw and node.deny.known_hash(expect.cid) is None \ + and not node.storage.is_complete(expect.cid): + # leaf_bytes counts the chunks its file nodes list, so they must all be there to be counted + return None + node._check_file(expect, get) return b"".join(walker.read_range(expect, get, 0, length)) except (ValueError, NotFound, InvalidDag): return None +def denying(node, get: walker.Get) -> walker.Get: + """get, raising Denied for a CID on the node's deny list instead of reading it (as a fetch does).""" + + def checked(expect: walker.Expect) -> bytes: + """The block for expect, unless its CID is denied.""" + if node.deny.denies(expect.cid): + raise Denied(f"{expect.cid} is on this node's deny list") + return get(expect) + + return checked + + def block(node, cid: str, for_gateway: bool = False) -> bytes: """A CID's own block, fetched if needed and re-verified. A raw block is a whole file, so the deny list's file hashes apply to it (its digest is its content's SHA-256).""" @@ -190,7 +225,7 @@ def block(node, cid: str, for_gateway: bool = False) -> bytes: with node._fetching(root, [], None, for_gateway) as fetch: expect = fetch.resolve() if expect.cid.is_raw: - node._check_file(expect, fetch.read_range) # never reads: a raw CID carries the hash + node._check_file(expect, fetch.get) # never reads: a raw CID carries the hash data = node.storage.get(expect.cid, verify=True) if data is None: raise NotFound(f"block {cid} was damaged locally and has been deleted; try again") @@ -209,7 +244,7 @@ def export_stream(node, target: str, start: Callable[[int], BinaryIO | None], fo walker.count_entries(expect, fetch.get, node.config.max_output_entries) if node.deny.hashes: for file in walker.iter_files(expect, fetch.get): - node._check_file(file, lambda e: walker.read_range(e, fetch.get)) + node._check_file(file, fetch.get) blocks = node.storage.dag_blocks(expect.cid) length = archive.archive_size([expect.cid], ((cid, node.storage.block_size(cid) or 0) for cid in blocks)) out = start(length) diff --git a/rnfs/pin_names.py b/rnfs/pin_names.py index 86397f3..8d94008 100644 --- a/rnfs/pin_names.py +++ b/rnfs/pin_names.py @@ -3,7 +3,9 @@ A CID names only content, so `add photo.jpg` keeps no file name. This records, for this node's own user, what each pin was added from (an absolute path, "(stdin)", the target given to `pin add`, an archive) so listings can show it. It is local only: never sent anywhere, and it doesn't change any CID. Kept in the state -database's pin_names table (rnfs.statedb): each change is one row written at once, durably. +database's pin_names table (rnfs.statedb): each change is one row written at once, but not durably +(synchronous=NORMAL): the pin it names was committed durably before it, so a power loss can lose at most the +latest names, never a pin, and an add costs one durable commit rather than two. """ import threading @@ -38,20 +40,18 @@ class PinNames: def __init__(self, state_dir: str): """Load the names saved in state_dir; a row that doesn't parse is logged and deleted.""" - self._db = statedb.connect(state_dir) self._lock = threading.Lock() self._names: dict[Cid, str] = {} - self._dirty: set[Cid] = set() # CIDs whose name changed (or went) and whose save failed - try: - self._load() - except BaseException: - self._db.close() - raise + self._db = statedb.open_store(state_dir, self._load) - def _load(self): + def _load(self, db: statedb.StateDb): """Read the pin_names table.""" + # Written through: a change is on disk (not durably) when it returns; one whose write failed is written + # with the next change, or by save() + self._store = statedb.RowStore(db, "pin_names", ("cid",), durable=False, lock=self._lock, row=self._row, + key_values=lambda cid: (cid.binary,)) bad = [] - for binary, text in self._db.query("SELECT cid, name FROM pin_names"): + for binary, text in db.query("SELECT cid, name FROM pin_names"): try: name = statedb.load(text) if not isinstance(name, str): @@ -60,18 +60,22 @@ def _load(self): except (ValueError, TypeError): log(f"pin names: ignoring invalid record {binary!r} {text!r}", LOG_WARNING) bad.append((binary,)) - if bad: - with self._db.transaction() as tx: - tx.executemany("DELETE FROM pin_names WHERE cid = ?", bad) + statedb.delete_rows(db, "pin_names", ("cid",), bad) + + def _row(self, cid: Cid) -> tuple | None: + """cid's pin_names row, or None if it has no name (lock held).""" + name = self._names.get(cid) + return None if name is None else (cid.binary, statedb.dump(name)) def record(self, cid: Cid, name: str): - """Record that cid was added from name; a later add of the same content replaces it.""" + """Record that cid was added from name; a later add of the same content replaces it. Raises OSError if it + can't be saved (kept in memory, and saved with the next change).""" name = name[:MAX_NAME_LENGTH] with self._lock: if self._names.get(cid) == name: return self._names[cid] = name - self._save([cid]) + self._store.changed(cid) def record_if_unknown(self, cid: Cid, name: str): """Record name for cid unless it already has one (a pin add of something added here keeps its path).""" @@ -79,7 +83,7 @@ def record_if_unknown(self, cid: Cid, name: str): if cid in self._names: return self._names[cid] = name[:MAX_NAME_LENGTH] - self._save([cid]) + self._store.changed(cid) def get(self, cid: Cid) -> str | None: """Where cid was added from, or None if unknown.""" @@ -94,18 +98,11 @@ def prune(self, pinned: set[Cid]): return for cid in gone: del self._names[cid] - self._save(gone) - - def _save(self, changed: list[Cid]): - """Write the rows of changed, and of earlier changes whose save failed, durably (lock held). If it fails - they are written with the next change.""" - self._dirty.update(changed) - names = [(cid.binary, statedb.dump(self._names[cid])) for cid in self._dirty if cid in self._names] - gone = [(cid.binary,) for cid in self._dirty if cid not in self._names] - with self._db.transaction(durable=True) as tx: - tx.executemany("DELETE FROM pin_names WHERE cid = ?", gone) - tx.executemany("INSERT OR REPLACE INTO pin_names (cid, name) VALUES (?, ?)", names) - self._dirty.clear() + self._store.changed(*gone) + + def save(self): + """Write the names whose write failed, if any (each change is written as it is made).""" + self._store.save() def close(self): """Release the state database (the node calls this once it has saved everything at shutdown).""" diff --git a/rnfs/publisher.py b/rnfs/publisher.py index 833c369..63c3fde 100644 --- a/rnfs/publisher.py +++ b/rnfs/publisher.py @@ -100,6 +100,11 @@ def __hash__(self) -> int: return hash((DhtKey, self.key)) +# What the publisher provides: a CID, or a feature's DhtKey. Both have .key, hashing, str() and parse(), but only a +# Cid has .binary or is served, so code that needs those must check (provided() does) +ProvidedItem = Cid | DhtKey + + def key_interval(key: bytes, base: float) -> float: """The republish interval of DhtKey key: max(base, KEY_MIN_INTERVAL) with key's fixed jitter, below PROVIDER_TTL / 2 (base is already below it: Config.validate).""" @@ -110,12 +115,12 @@ def key_interval(key: bytes, base: float) -> float: return min(jittered, max(interval, PROVIDER_TTL / 2 - HOUR)) -def _parse_item(text: str) -> "Cid | DhtKey": +def _parse_item(text: str) -> ProvidedItem: """A provided item as str() writes it (the published tables' item). Raises ValueError.""" return DhtKey.parse(text) if isinstance(text, str) and text.startswith(KEY_PREFIX) else Cid.parse(text) -def _item(text) -> "Cid | DhtKey": +def _item(text) -> ProvidedItem: """A saved provided item: _parse_item, but only an ID_LENGTH-byte DhtKey. Raises ValueError.""" item = _parse_item(text) if isinstance(item, DhtKey) and len(item.key) != ID_LENGTH: @@ -187,6 +192,42 @@ def import_published(value: dict, tx) -> int: return sum(len(table) for table in rows) +_ITEM, _NODE = "item", "node" # the two kinds of key _PublishState writes + + +class _PublishState(statedb.RowStore): + """The publisher's three tables, saved together (not durably: NORMAL) in one transaction. A key is (_ITEM, + item), whose published row and published_earlier rows are written again, or (_NODE, node ID), whose + withdrawals rows are.""" + + def __init__(self, db: statedb.StateDb, publisher: "Publisher"): + """The saves of publisher's state, whose lock guards it.""" + super().__init__(db, "published", ("item",), durable=False, mode=statedb.SNAPSHOT, lock=publisher._lock) + self.publisher = publisher + + def snapshot(self, keys) -> tuple: + """The rows of keys (lock held): (item texts, published rows, earlier rows, node IDs, withdrawal rows).""" + publisher = self.publisher + items = [key for kind, key in keys if kind == _ITEM] + nodes = [key for kind, key in keys if kind == _NODE] + texts = {item: str(item) for item in items} + published = [(texts[item], publisher._published[item][0], b"".join(publisher._published[item][1])) + for item in items if item in publisher._published] + earlier = [(texts[item], node, at) for item in items for node, at in publisher._earlier.get(item, {}).items()] + withdrawals = [(node, key) for node in nodes for key in publisher._withdrawals.get(node, ())] + return list(texts.values()), published, earlier, nodes, withdrawals + + def write_rows(self, tx, snapshot: tuple): + """Replace the rows of the snapshot's items and nodes inside transaction tx.""" + texts, published, earlier, nodes, withdrawals = snapshot + tx.executemany("DELETE FROM published WHERE item = ?", [(text,) for text in texts]) + tx.executemany("DELETE FROM published_earlier WHERE item = ?", [(text,) for text in texts]) + tx.executemany("INSERT OR REPLACE INTO published (item, at, nodes) VALUES (?, ?, ?)", published) + tx.executemany("INSERT OR REPLACE INTO published_earlier (item, node, at) VALUES (?, ?, ?)", earlier) + tx.executemany("DELETE FROM withdrawals WHERE node = ?", [(node,) for node in nodes]) + tx.executemany("INSERT OR REPLACE INTO withdrawals (node, key) VALUES (?, ?)", withdrawals) + + class Publisher: """The provide set, the served set, and the publish/withdraw schedule. Thread-safe.""" @@ -213,9 +254,8 @@ def __init__(self, own_id: bytes, config: Config, storage: Storage, providers, c self.retry_interval = retry_interval self.generation = generation self._lock = threading.RLock() - self._save_lock = threading.Lock() # one save at a time, so an older snapshot never lands last self._wake = threading.Event() - self._provided: frozenset[Cid] = frozenset() + self._provided: frozenset[ProvidedItem] = frozenset() self._served: frozenset[Cid] = frozenset() # What _compute last built, and from what, so a refresh that changed nothing needn't build it again: the # served set from each provided CID's dag_blocks (memoised by storage, so the same object while its DAG @@ -227,26 +267,18 @@ def __init__(self, own_id: bytes, config: Config, storage: Storage, providers, c self._denied_dags: dict[Cid, tuple[frozenset[bytes], frozenset[Cid], bool]] = {} # (generation(), deny list's CID set, keys, strategy) of the last full refresh; None: the next computes self._inputs: tuple | None = None - self._published: dict[Cid, tuple[float, list[bytes]]] = {} # CID -> (when, nodes that recorded it) - # Other nodes that may still hold our record of a CID (earlier rounds' closest nodes, or a round cut + self._published: dict[ProvidedItem, tuple[float, list[bytes]]] = {} # item -> (when, nodes that recorded it) + # Other nodes that may still hold our record of an item (earlier rounds' closest nodes, or a round cut # short), with when it was stored there; withdrawn from too, and forgotten once PROVIDER_TTL has passed - self._earlier: dict[Cid, dict[bytes, float]] = {} + self._earlier: dict[ProvidedItem, dict[bytes, float]] = {} self._withdrawals: dict[bytes, set[bytes]] = {} # node -> keys to withdraw from it - self._retry_at: dict[Cid, float] = {} # CIDs whose last lookup found no nodes + self._retry_at: dict[ProvidedItem, float] = {} # items whose last lookup found no nodes self._next_lookup = 0.0 # slow nodes: clock time before which no lookup round starts # Functions returning the non-CID keys features provide (add_key_source); polled by refresh() self._key_sources: list[Callable[[], set[bytes]]] = [] - # Changed since the last save (only tracked when persisted): items whose published or earlier rows, and - # nodes whose withdrawals, save() writes again - self._dirty_items: set = set() - self._dirty_nodes: set = set() - self._db = None if state_dir is None else statedb.connect(state_dir) - if self._db is not None: - try: - self._load() - except BaseException: - self._db.close() - raise + # Writes what changed since the last save (_touch_item, _touch_node), when persisted + self._store: _PublishState | None = None + self._db = None if state_dir is None else statedb.open_store(state_dir, self._load) def add_key_source(self, source: Callable[[], set[bytes]]): """Also provide the keys source() returns (DhtKeys), recomputed at each refresh(). Must not block.""" @@ -259,7 +291,7 @@ def interval(self) -> float: """Seconds between republishes: republish_interval, 3 days on slow nodes by default.""" return self.config.effective_republish_interval(self.slow_node) - def interval_for(self, item: "Cid | DhtKey") -> float: + def interval_for(self, item: ProvidedItem) -> float: """Seconds between republishes of item: interval for a CID, key_interval for a DhtKey.""" if isinstance(item, DhtKey): return key_interval(item.key, self.interval) @@ -267,10 +299,11 @@ def interval_for(self, item: "Cid | DhtKey") -> float: # Persistence - def _load(self): + def _load(self, db: statedb.StateDb): """Load last-published times and nodes, earlier rounds' nodes and pending withdrawals. Rows of the wrong shape are skipped (logged) and deleted; those the load changes (times clamped, earlier nodes past PROVIDER_TTL dropped) are marked changed, so the next save writes them as memory has them.""" + self._store = _PublishState(db, self) now = self.clock() def when(number) -> float: @@ -280,7 +313,7 @@ def when(number) -> float: doomed: set = set() # item texts whose rows are deleted here: they don't parse, or aren't as str() writes malformed = 0 - def item_of(text) -> "Cid | DhtKey | None": + def item_of(text) -> ProvidedItem | None: """The item a row's text names, or None if it doesn't parse (counted and doomed).""" nonlocal malformed try: @@ -294,7 +327,7 @@ def item_of(text) -> "Cid | DhtKey | None": self._touch_item(item) return item - for text, at, nodes in self._db.query("SELECT item, at, nodes FROM published"): + for text, at, nodes in db.query("SELECT item, at, nodes FROM published"): item = item_of(text) if item is None: continue @@ -306,7 +339,7 @@ def item_of(text) -> "Cid | DhtKey | None": continue if self._published[item][0] != at: self._touch_item(item) - for text, node, stored in self._db.query("SELECT item, node, at FROM published_earlier"): + for text, node, stored in db.query("SELECT item, node, at FROM published_earlier"): item = item_of(text) if item is None: continue @@ -322,7 +355,7 @@ def item_of(text) -> "Cid | DhtKey | None": self._earlier.setdefault(item, {})[node] = stored_at if stored_at != stored or now - stored_at >= PROVIDER_TTL: self._touch_item(item) - for node, key in self._db.query("SELECT node, key FROM withdrawals"): + for node, key in db.query("SELECT node, key FROM withdrawals"): if not all(isinstance(value, bytes) and len(value) == ID_LENGTH for value in (node, key)): malformed += 1 self._touch_node(node) @@ -331,59 +364,31 @@ def item_of(text) -> "Cid | DhtKey | None": if malformed: log(f"publisher: ignoring {plural(malformed, 'malformed row')} in the state database", LOG_WARNING) if doomed: - with self._db.transaction() as tx: + with db.transaction() as tx: tx.executemany("DELETE FROM published WHERE item = ?", [(text,) for text in doomed]) tx.executemany("DELETE FROM published_earlier WHERE item = ?", [(text,) for text in doomed]) def save(self): """Persist the publish state changed since the last save, if any. Called without the lock: it is written - outside it. Raises OSError (StateDbError) if the database fails; what it held is then written next time.""" - if self._db is None: - return - # Saves come from refresh(), the publish loop and stop(): one at a time, or a snapshot taken before the - # other's could be written last - with self._save_lock: - with self._lock: - items, nodes = self._dirty_items, self._dirty_nodes - if not items and not nodes: - return - self._dirty_items, self._dirty_nodes = set(), set() - texts = {item: str(item) for item in items} - published = [(texts[item], self._published[item][0], b"".join(self._published[item][1])) - for item in items if item in self._published] - earlier = [(texts[item], node, at) - for item in items for node, at in self._earlier.get(item, {}).items()] - withdrawals = [(node, key) for node in nodes for key in self._withdrawals.get(node, ())] - try: - with self._db.transaction() as tx: - tx.executemany("DELETE FROM published WHERE item = ?", [(text,) for text in texts.values()]) - tx.executemany("DELETE FROM published_earlier WHERE item = ?", - [(text,) for text in texts.values()]) - tx.executemany("INSERT OR REPLACE INTO published (item, at, nodes) VALUES (?, ?, ?)", published) - tx.executemany("INSERT OR REPLACE INTO published_earlier (item, node, at) VALUES (?, ?, ?)", - earlier) - tx.executemany("DELETE FROM withdrawals WHERE node = ?", [(node,) for node in nodes]) - tx.executemany("INSERT OR REPLACE INTO withdrawals (node, key) VALUES (?, ?)", withdrawals) - except BaseException: - with self._lock: # written by the next save - self._dirty_items |= items - self._dirty_nodes |= nodes - raise + outside it, one save at a time (saves come from refresh(), the publish loop and stop()). Raises OSError + (StateDbError) if the database fails; what it held is then written next time.""" + if self._store is not None: + self._store.save() def close(self): """Release the state database (the node calls this once it has saved everything at shutdown).""" if self._db is not None: self._db.close() - def _touch_item(self, item: "Cid | DhtKey"): + def _touch_item(self, item: ProvidedItem): """Note that item's published or earlier entry changed, for the next save. Caller holds the lock.""" - if self._db is not None: - self._dirty_items.add(item) + if self._store is not None: + self._store.changed((_ITEM, item)) def _touch_node(self, node_id: bytes): """Note that the withdrawals queued for node_id changed, for the next save. Caller holds the lock.""" - if self._db is not None: - self._dirty_nodes.add(node_id) + if self._store is not None: + self._store.changed((_NODE, node_id)) # The provide set @@ -411,7 +416,7 @@ def _keys(self) -> set[bytes]: log(f"a provided-key source failed: {e!r}", LOG_ERROR) return keys - def _compute(self, keys: set[bytes], denied: frozenset[bytes]) -> tuple[frozenset[Cid], frozenset[Cid]]: + def _compute(self, keys: set[bytes], denied: frozenset[bytes]) -> tuple[frozenset[ProvidedItem], frozenset[Cid]]: """(provided CIDs and DhtKeys of keys, blocks they reach) from storage, the strategy and denied (the deny list's binary CIDs). Lock held. @@ -419,7 +424,7 @@ def _compute(self, keys: set[bytes], denied: frozenset[bytes]) -> tuple[frozense (most refreshes): the same complete CIDs with the same dag_blocks objects, and the same keys. CIDs only added since add their blocks to it; only one gone or changed builds it again. """ - candidates = set(self.storage.pins()) + candidates = self.storage.pinned_cids() # the same frozenset until pins change: not copied if self.config.provide_strategy == "roots": candidates |= self.storage.cached_targets() & set(self.cached_eligible()) dags: dict[Cid, frozenset[Cid]] = {} @@ -487,7 +492,9 @@ def refresh(self): for node_id in set(record[1] if record else []) | set(earlier): self._withdrawals.setdefault(node_id, set()).add(cid.key) self._touch_node(node_id) - cancelled = [cid for cid in added if self._provide_again(cid)] + # The keys withdrawals are queued for, gathered once: most added items have none to cancel + pending = {key for keys in self._withdrawals.values() for key in keys} if added else set() + cancelled = [cid for cid in added if self._provide_again(cid, pending)] self._inputs = inputs # only once everything above is done: an exception computes again next time if removed: log(f"no longer providing {plural(len(removed), 'CID')}", LOG_VERBOSE) @@ -496,9 +503,10 @@ def refresh(self): if removed or any(self._is_due(cid, self.clock()) for cid in added): self.wake() - def _provide_again(self, cid: "Cid | DhtKey") -> bool: + def _provide_again(self, cid: ProvidedItem, pending: set[bytes]) -> bool: """cid is provided (again): record ourselves locally and cancel withdrawals of its key still queued; - True if any were. Lock held. + True if any were. pending holds every key a withdrawal is queued for (it may hold more: those already + cancelled), so the queues are searched only for those. Lock held. A withdrawal queued when it stopped being provided would otherwise reach a node after a round stored it there again, deleting the fresh record (section 6). Those nodes may still hold a record of it, so they @@ -506,6 +514,8 @@ def _provide_again(self, cid: "Cid | DhtKey") -> bool: """ key = cid.key self.providers.store(key, self.own_id) # refresh() withdrew it when it stopped being provided + if key not in pending: + return False now = self.clock() cancelled = False for node_id in [node_id for node_id, keys in self._withdrawals.items() if key in keys]: @@ -526,7 +536,7 @@ def wake(self): """Make the publish loop look for due work now.""" self._wake.set() - def _is_due(self, cid: Cid, now: float) -> bool: + def _is_due(self, cid: ProvidedItem, now: float) -> bool: """True if cid should be (re)published now (lock held or not; reads are atomic).""" retry = self._retry_at.get(cid) if retry is not None: @@ -592,7 +602,7 @@ def publish_due(self, should_stop: Callable[[], bool] = lambda: False) -> dict: del closest[cid] self.providers.withdraw(cid.key, self.own_id) - by_node: dict[bytes, list[Cid]] = {} + by_node: dict[bytes, list[ProvidedItem]] = {} for cid, nodes in closest.items(): for node_id in nodes: by_node.setdefault(node_id, []).append(cid) @@ -607,7 +617,7 @@ def publish_due(self, should_stop: Callable[[], bool] = lambda: False) -> dict: if not self._withdrawals[node_id]: del self._withdrawals[node_id] self._touch_node(node_id) - accepted: dict[Cid, list[bytes]] = {cid: [] for cid in closest} + accepted: dict[ProvidedItem, list[bytes]] = {cid: [] for cid in closest} sent_withdrawals: dict[bytes, set[bytes]] = {} unsaved, saved_at = 0, self.clock() for node_id in sorted(set(by_node) | set(withdrawals)): @@ -646,7 +656,7 @@ def publish_due(self, should_stop: Callable[[], bool] = lambda: False) -> dict: self.save() unsaved, saved_at = 0, answered - unfinished: list[Cid] = [] + unfinished: list[ProvidedItem] = [] with self._lock: for node_id, keys in sent_withdrawals.items(): remaining = self._withdrawals.get(node_id, set()) - keys @@ -695,12 +705,12 @@ def publish_due(self, should_stop: Callable[[], bool] = lambda: False) -> dict: return {"published": published, "nodes": sorted(set(by_node) | set(sent_withdrawals)), "withdrawn": sum(len(keys) for keys in sent_withdrawals.values())} - def _find_closest(self, cid: "Cid | DhtKey") -> list[bytes]: + def _find_closest(self, cid: ProvidedItem) -> list[bytes]: """Record this node as cid's provider locally, then look up the nodes closest to its key (but this).""" self.providers.store(cid.key, self.own_id) # providers also record themselves locally return [n for n in self.make_lookup(cid.key).find_closest() if n != self.own_id] - def _lookups(self, due: list, should_stop: Callable[[], bool]) -> dict: + def _lookups(self, due: list[ProvidedItem], should_stop: Callable[[], bool]) -> dict[ProvidedItem, list[bytes]]: """{item: its closest nodes} for due's items whose lookup ran, in due's order, so a round's batches are the same however the lookups were run. Called without the lock: refresh() may run meanwhile. @@ -764,7 +774,7 @@ def run(self, stop_event: threading.Event, ready: threading.Event | None = None) try: if self.next_due() <= 0: self.publish_due(stop_event.is_set) - except Exception as e: + except Exception as e: # noqa: BLE001 - the publisher thread keeps running; logged log(f"publishing failed: {e!r}", LOG_ERROR) self._wake.wait(min(MAX_WAIT, max(0.5, self.next_due()))) if stop_event.is_set(): diff --git a/rnfs/remote.py b/rnfs/remote.py index d5af42e..5abbd9b 100644 --- a/rnfs/remote.py +++ b/rnfs/remote.py @@ -20,7 +20,8 @@ import time from rnfs import interface -from rnfs.apiclient import ApiConnection, read_all +from rnfs.apiclient import ApiConnection +from rnfs.fsutil import printable from rnfs.objects import Record MAX_WATCH_WAIT = 30 # seconds one GET /api/name/watch or /api/jobs/get waits (the API's cap) @@ -63,8 +64,9 @@ def api(self) -> ApiConnection: return self._api def _report(self, message: str): - """Log through Python's logging (logger "rnfs").""" - _logger.error("rnfs: %s", message) + """Log through Python's logging (logger "rnfs"), escaped: the daemon's message can quote names from + other people's DAGs.""" + _logger.error("rnfs: %s", printable(message)) def _close(self): """Wake the name poller so it ends.""" @@ -186,11 +188,11 @@ def stat(self, target: str) -> dict: def dir_set(self, target: str, name: str, child: str, executable: bool) -> dict: """POST /api/dir/set.""" - return self.api.post("/api/dir/set", target=target, name=name, cid=child, executable=executable) + return self.api.dir_set(target, name, child, executable) def dir_rm(self, target: str) -> dict: """POST /api/dir/rm.""" - return self.api.post("/api/dir/rm", target=target) + return self.api.dir_rm(target) def pin_add(self, target: str) -> dict: """POST /api/pin/add.""" @@ -206,31 +208,31 @@ def pin_ls(self, cid: str | None) -> list[dict]: def pin_remote(self, node_id: str, cid: str, pin: bool) -> bool: """POST /api/pin/remote.""" - return self.api.post("/api/pin/remote", node=node_id, cid=cid, pin=pin)["accepted"] + return self.api.pin_remote(node_id, cid, pin) def providers(self, cid: str) -> list[str]: """GET /api/providers.""" - return self.api.get("/api/providers", cid=cid)["providers"] + return self.api.providers(cid) def ping(self, node_id: str) -> float | None: """GET /api/ping.""" - return self.api.get("/api/ping", node=node_id)["rtt"] + return self.api.ping(node_id) def export(self, target: str, output: str) -> dict: """POST /api/export.""" - return self.api.post("/api/export", target=target, output=output) + return self.api.export(target, output) def import_archive(self, path: str) -> dict: """POST /api/import.""" - return self.api.post("/api/import", path=path) + return self.api.import_archive(path) def gc(self) -> dict: """POST /api/gc.""" - return self.api.post("/api/gc") + return self.api.gc() def verify(self) -> dict: """POST /api/verify.""" - return self.api.post("/api/verify") + return self.api.verify() def name_create(self, alias: str) -> dict: """POST /api/name/create.""" @@ -238,16 +240,15 @@ def name_create(self, alias: str) -> dict: def name_import(self, alias: str, key: bytes) -> dict: """POST /api/name/import (the key as the body).""" - body = bytes(key) - return self.api.call("POST", "/api/name/import", params={"alias": alias}, body=body, length=len(body)) + return self.api.name_import(alias, key) def name_rm(self, alias: str): """POST /api/name/rm.""" - self.api.post("/api/name/rm", alias=alias) + self.api.name_rm(alias) def name_export(self, alias: str) -> bytes: """POST /api/name/export: the key bytes.""" - return read_all(self.api.request("POST", "/api/name/export", json_body={"alias": alias})) + return self.api.name_export(alias) def name_list(self) -> list[dict]: """GET /api/name/list.""" diff --git a/rnfs/statedb.py b/rnfs/statedb.py index 12f41c9..e2a6952 100644 --- a/rnfs/statedb.py +++ b/rnfs/statedb.py @@ -17,7 +17,10 @@ transaction recorded in the migrated table, then renames it to .migrated. A crash between the commit and the rename leaves the record, so the next start only renames the file: nothing is imported twice or by half. -Stores keep their state in memory and read the database only when they load. Text that may hold lone +Stores keep their state in memory and read the database only when they load (open_store), and write what changed +through a RowStore: it keeps the keys changed and not yet written, writes them at once (WRITE_THROUGH) or at the +store's saves (SNAPSHOT, outside the store's lock, one save at a time), and keeps them for the next write when one +fails. reconcile_pins makes a feature's pins match its saved state at start. Text that may hold lone surrogates (file names, remote strings) is stored through dump() and load(); sequences, which may not fit SQLite's signed 64-bit integers, as decimal text. """ @@ -42,7 +45,11 @@ # Each step brings PRAGMA user_version from its index to its index + 1, in one transaction. Steps are only ever # appended: a store whose layout changes gets a new step (ALTER TABLE, or a new table and an INSERT ... SELECT). -SCHEMA: tuple[tuple[str, tuple[str, ...]], ...] = ( +# The steps released in 0.1.5 (the first 16) are pinned by a digest in tests/test_statedb.py: a database 0.1.5 +# wrote has them already, so an edit there would never reach it. A step's item is an SQL statement, or a callable +# taking the transaction's connection, for a migration SQL alone can't express. +Step = str | Callable[[object], None] +SCHEMA: tuple[tuple[str, tuple[Step, ...]], ...] = ( ("statedb", ( # JSON state files imported, so each is imported once (migrate) "CREATE TABLE migrated (file TEXT PRIMARY KEY, records INTEGER NOT NULL, at REAL NOT NULL)", @@ -129,6 +136,13 @@ # Files' content SHA-256 (digest) by binary CID; used orders them (least recently used lowest) "CREATE TABLE deny_hashes (cid BLOB PRIMARY KEY, digest BLOB NOT NULL, used INTEGER NOT NULL) WITHOUT ROWID", )), + ("shared_folder_files", ( + # What each file in the shared folder was built as (rnfs 0.1.6). name: the dump()ed top-level entry holding + # it; state: the dump()ed [device, inode, size, mtime_ns, execute bit] (text: an inode may not fit SQLite's + # integers); digest: its content SHA-256, NULL if it was built without one + "CREATE TABLE shared_folder_files (name TEXT NOT NULL, state TEXT NOT NULL, cid BLOB NOT NULL," + " height INTEGER NOT NULL, digest BLOB, PRIMARY KEY (name, state)) WITHOUT ROWID", + )), ) @@ -232,8 +246,11 @@ def _upgrade(self): "one (rnfs never moves state back to an older format)") for number in range(version, len(SCHEMA)): with self.transaction(durable=True) as tx: - for statement in SCHEMA[number][1]: - tx.execute(statement) + for step in SCHEMA[number][1]: + if callable(step): + step(tx) + else: + tx.execute(step) tx.execute(f"PRAGMA user_version = {number + 1}") @contextmanager @@ -419,6 +436,186 @@ def open_databases() -> list[str]: return [database.path for database in _open.values()] +# Stores: what every store does the same way + + +WRITE_THROUGH = "write_through" +SNAPSHOT = "snapshot" +_UNKNOWN = object() # RowStore's last-written memo has no entry for a key + + +def open_store(state_dir: str, load: Callable[[StateDb], object]) -> StateDb: + """A reference to the state database in state_dir (connect) once load(db) has read a store's rows from it + (making the store's RowStore first, so the load can mark rows it changed). If load raises, the reference is + released and the exception raised.""" + db = connect(state_dir) + try: + load(db) + except BaseException: + db.close() + raise + return db + + +def delete_rows(db: StateDb, table: str, key_cols: tuple[str, ...], keys: list[tuple]): + """Delete the rows of table whose key_cols hold each of keys (tuples of their values), in one transaction: the + rows a store's load found malformed. Not durable: a row a power loss brings back is only dropped again.""" + if keys: + with db.transaction() as tx: + tx.executemany(f"DELETE FROM {table} WHERE {_where(key_cols)}", keys) + + +def _where(key_cols: tuple[str, ...]) -> str: + """The WHERE clause matching one row's key_cols.""" + return " AND ".join(f"{column} = ?" for column in key_cols) + + +class RowStore: + """Writes a store's changed rows: one table whose key_cols are its primary key, one row per key. + + The store keeps its state in memory under its own lock, and tells changed(key) of each key it changes (lock + held). row(key) gives the row key has now, a tuple of the table's columns in the table's order, or None once + it is gone (lock held); key_values(key) gives its key_cols' values (by default (key,)). A write deletes the + rows of the keys gone and inserts or replaces the others, in one transaction, durable as given. + + This owns what each store used to do by hand: the set of keys changed and not yet written, writing them in + order, and keeping them for the next write when one fails (the change itself stays in memory): + - WRITE_THROUGH: changed() writes at once, under the store's lock (each change is on disk, as durable says, + when it returns); it raises OSError (StateDbError) if the write fails, and the keys are written with the + next change or save(). + - SNAPSHOT: changed() only notes the keys; save() takes them and their rows under the lock and writes them + outside it, so a slow disk never holds the store up. One save at a time, so an older snapshot never lands + last; if the write fails the keys are noted again and save() raises. + + skip_unchanged: remember each row as last written, and leave out a key whose row is as it was (for a store that + marks keys that may not have changed, as jobs mark the running ones at each save). A store whose rows span + several tables (the publisher) subclasses this, overriding snapshot() and write_rows(). + """ + + def __init__(self, db: StateDb, table: str, key_cols: tuple[str, ...], *, durable: bool, + mode: str = WRITE_THROUGH, lock, row: Callable[[object], tuple | None] | None = None, + key_values: Callable[[object], tuple] = lambda key: (key,), skip_unchanged: bool = False): + """Rows of table for a store whose state lock guards; see the class. Raises ValueError for an unknown + mode.""" + if mode not in (WRITE_THROUGH, SNAPSHOT): + raise ValueError(f"unknown RowStore mode {mode!r}") + self.db = db + self.table = table + self.durable = durable + self.mode = mode + self.lock = lock + self._row = row + self._key_values = key_values + self._dirty: set = set() # keys changed and not yet written (under lock) + self._save_lock = threading.Lock() # SNAPSHOT: one save at a time, so an older snapshot never lands last + # skip_unchanged: key -> its row as last committed (under _save_lock in SNAPSHOT mode, lock otherwise) + self._written: dict | None = {} if skip_unchanged else None + self._delete = f"DELETE FROM {table} WHERE {_where(key_cols)}" + columns = [info[1] for info in db.query(f"PRAGMA table_info({table})")] + self._insert = (f"INSERT OR REPLACE INTO {table} ({', '.join(columns)}) " + f"VALUES ({', '.join('?' * len(columns))})") + + @property + def pending(self) -> frozenset: + """The keys changed and not yet written.""" + with self.lock: + return frozenset(self._dirty) + + def changed(self, *keys): + """Note that keys changed (lock held); in WRITE_THROUGH mode, write them, and any a failed write left, at + once. Raises OSError (StateDbError) if that write fails; the keys are then written by the next one.""" + self._dirty.update(keys) + if self.mode == WRITE_THROUGH: + self._write_dirty() + + def save(self): + """Write the keys changed and not yet written, if any: in SNAPSHOT mode the store's changes since the last + save, in WRITE_THROUGH mode those a failed write left. Raises OSError (StateDbError) if the write fails; + they are then written by the next save.""" + if self.mode == WRITE_THROUGH: + with self.lock: + if self._dirty: + self._write_dirty() + return + with self._save_lock: + with self.lock: + keys, self._dirty = self._dirty, set() + if not keys: + return + snapshot = self.snapshot(keys) + try: + self._commit(snapshot) + except BaseException: + with self.lock: # written by the next save + self._dirty |= keys + raise + + def _write_dirty(self): + """WRITE_THROUGH: write every key not yet written (lock held), then forget them; kept if it fails.""" + self._commit(self.snapshot(self._dirty)) + self._dirty.clear() + + def _commit(self, snapshot): + """Write snapshot in one transaction, if it holds anything, then remember what was written.""" + if snapshot: + with self.db.transaction(self.durable) as tx: + self.write_rows(tx, snapshot) + self.written(snapshot) + + def snapshot(self, keys) -> list: + """What a write of keys writes, taken under the lock: [(key, row or None)], leaving out (skip_unchanged) + keys whose row is as last written. Empty (false) when there is nothing to write.""" + rows = [(key, self._row(key)) for key in keys] + if self._written is not None: + rows = [(key, row) for key, row in rows if self._written.get(key, _UNKNOWN) != row] + return rows + + def write_rows(self, tx, snapshot): + """Write snapshot inside transaction tx: delete the keys gone, insert or replace the others' rows.""" + tx.executemany(self._delete, [self._key_values(key) for key, row in snapshot if row is None]) + tx.executemany(self._insert, [row for _, row in snapshot if row is not None]) + + def written(self, snapshot): + """snapshot was committed (skip_unchanged: remember its rows; a key gone is forgotten).""" + if self._written is None: + return + for key, row in snapshot: + if row is None: + self._written.pop(key, None) + else: + self._written[key] = row + + +def reconcile_pins(pins: dict, prefix: str, recorded: Callable[[str, object], bool], clean: bool, + adopt: Callable[[dict[str, list]], None], release: Callable[[str, object], None]): + """At start, make a feature's pins match its saved state. Its pins are those whose source is prefix and an + owner ("lxmf:", "follow:"); pins is storage.pins() (CID -> sources). A pin recorded(owner, cid) + says the state doesn't record was left by a crash between the pin and the save, or by state that was lost. + + If the state is clean (saved whole, and every row of it read), release(owner, cid) releases each such pin: + nothing else would (`pin rm` releases only the user's pins). An OSError it raises (a read-only state + directory) is logged; the pin is tried again at the next start. Otherwise such a pin may be all that is left + of the state that was lost: adopt({owner: [cid, ...]}) records them again (owners and CIDs in order), so they + are kept, and the feature can release them later.""" + unrecorded: dict[str, list] = {} + for cid, sources in pins.items(): + for source in sources: + if source.startswith(prefix) and not recorded(source[len(prefix):], cid): + unrecorded.setdefault(source[len(prefix):], []).append(cid) + if not unrecorded: + return + if not clean: + adopt({owner: sorted(unrecorded[owner], key=str) for owner in sorted(unrecorded)}) + return + for owner, cids in unrecorded.items(): + for cid in cids: + try: + release(owner, cid) + except OSError as e: + log(f"state: could not release {cid}, pinned for {prefix}{owner} ({e}); trying again at the next " + "start", LOG_ERROR) + + # Migration of the JSON state files of rnfs 0.1.4 and earlier diff --git a/rnfs/winacl.py b/rnfs/winacl.py index 298e65b..59f11de 100644 --- a/rnfs/winacl.py +++ b/rnfs/winacl.py @@ -10,13 +10,15 @@ parent) that grants full control to the current user and SYSTEM only, inherited by everything created below it. The DACL is part of the CreateDirectoryW call, so there is no moment when the parent's DACL applies and another account could put a file in it; a dir that is new yet not empty is refused all the same. An existing - rnfs dir that lets anyone else in gets the same DACL (which Windows passes on to what is below it), and one - that is a junction or symlink is refused; + rnfs dir or subdirectory that lets anyone else in gets the same DACL (which Windows passes on to what is below + it), and one that is a junction or symlink is refused; - the API token, identity and name key files get that DACL explicitly when written (fsutil.write_atomic's private); an existing identity or settings file (rnfs.toml) is checked through the handle it is read from - before use: one owned by another account is refused, one that lets others in is tightened, and a DACL that - can't be read or tightened stops rnfs; -- every CLI command checks the token file's DACL before trusting it, and the daemon the rnfs dir's at start. + before use (fsutil.check_owned_fd): one owned by another account is refused, one that lets others in is + tightened, and a DACL that can't be read or tightened stops rnfs; +- every CLI command checks the token file's DACL before trusting it, and the daemon the rnfs dir's at start; +- a directory `get` writes out is created with that DACL too, so nobody else can rename inside it while it is + written (audit CONT-2), and given back its folder's inherited access list (inherit_from_parent) once complete. A DACL counts as private if it allows nobody but the current user, SYSTEM and Administrators (who can read any file anyway); deny entries and audit entries don't matter. The owner must be one of those too, since an owner can @@ -27,6 +29,7 @@ tested on any platform. No RNS dependency, so the CLI can use it. """ import os +import stat import struct from typing import NamedTuple @@ -133,6 +136,11 @@ def private_acl(user_sid: str, directory: bool) -> bytes: return struct.pack(" bytes: + """A binary ACL with no entries: set unprotected, the object keeps only what it inherits from its parent.""" + return struct.pack(" bytes: """A self-relative security descriptor whose DACL is private_acl, marked protected (nothing inherited from the parent), with no owner or group (the creating process's defaults apply), for CreateDirectoryW.""" @@ -187,6 +195,7 @@ class _Windows: OWNER_SECURITY_INFORMATION = 0x1 DACL_SECURITY_INFORMATION = 0x4 PROTECTED_DACL_SECURITY_INFORMATION = 0x80000000 + UNPROTECTED_DACL_SECURITY_INFORMATION = 0x20000000 TOKEN_QUERY = 0x8 TOKEN_USER = 1 ERROR_ALREADY_EXISTS = 183 @@ -294,6 +303,15 @@ def set_private(self, path: str, directory: bool): if code != 0: raise self._error(code, path) + def inherit(self, path: str): + """Replace path's DACL with an empty, unprotected one, so it has only what it inherits from its parent; + SetNamedSecurityInfoW passes the parent's inheritable entries on to everything below it too.""" + acl = self.ctypes.create_string_buffer(empty_acl()) + info = self.DACL_SECURITY_INFORMATION | self.UNPROTECTED_DACL_SECURITY_INFORMATION + code = self.set_named(path, self.SE_FILE_OBJECT, info, None, None, acl, None) + if code != 0: + raise self._error(code, path) + def create_private_dir(self, path: str) -> bool: """Create the directory path with the protected DACL of private_acl from the start (CreateDirectoryW with it in its SECURITY_ATTRIBUTES); False if something already exists at path. Raises OSError.""" @@ -344,6 +362,15 @@ def make_private_dir(path: str) -> bool: return True +def inherit_from_parent(path: str): + """On Windows, give path (and everything below it) only the access its parent passes on, as if it had been + created there plainly: for a get output written below a make_private_dir root, once it is complete. + Raises OSError.""" + acl = _backend() if fsutil.IS_WINDOWS else None + if acl is not None: + acl.inherit(path) + + def check_private(path: str, fd: int | None = None) -> str | None: """On Windows, why the file open as fd (or at path) isn't private to the current user, or None if it is (and always None elsewhere). Raises OSError if its DACL can't be read.""" @@ -378,18 +405,6 @@ def tighten(path: str, directory: bool = False, fd: int | None = None) -> str | return reason -def secure_settings(path: str, fd: int): - """On Windows, check the settings file (rnfs.toml) open as fd before its settings are used: one that another - account owns is refused, one that lets others in is made private (with a warning, as they could have - changed it). Its settings choose the Reticulum config, and with it programs rnfs starts, and the API port - the CLI sends its token to. Raises PermissionError or OSError if it can't be trusted.""" - reason = tighten(path, fd=fd) - if reason is not None: - from rnfs.log import LOG_WARNING, log - log(f"the settings file {path} was accessible to other users ({reason}); made it private to you and " - f"SYSTEM. Check that its settings are yours", LOG_WARNING) - - def outside_profile(path: str) -> bool: """On Windows, True if path is outside the user's profile (%USERPROFILE%), whose ACL keeps other users out.""" if not fsutil.IS_WINDOWS: @@ -411,10 +426,11 @@ def secure_rnfs_dir(root: str, subdirs: tuple[str, ...] = ()): A new root is created with the protected owner-only DACL in the same call (make_private_dir), and must then be empty: anything in it was put there by someone else. An existing one is made private if others may access it. A root that is a junction, symlink or other link-like reparse point is refused, as the - access list checked and set would be its target's. Missing subdirectories get the same DACL when created. + access list checked and set would be its target's. Missing subdirectories get the same DACL when created; + existing ones are checked and made private like the root (_make_private_below). - Raises RnfsError if root is a link, belongs to another account, can't be made private, or was found - non-empty when new. + Raises RnfsError if root or a subdirectory is a link, belongs to another account or can't be made private, + or if root was found non-empty when new. """ if not fsutil.IS_WINDOWS: return @@ -452,12 +468,26 @@ def secure_rnfs_dir(root: str, subdirs: tuple[str, ...] = ()): def _make_private_below(root: str, directory: str): - """Create directory (below root) and any missing directories between root and it with make_private_dir. - Raises OSError (FileExistsError if something other than a directory is in the way).""" + """Create directory (below root) and any missing directories between root and it with make_private_dir, or + make an existing one private if others may access it (audit API-1: an rnfs dir that was once open to them + may hold subdirectories they can still change). Raises RnfsError if one is a junction or symlink (the access + list checked and set would be its target's), OSError (FileExistsError if something other than a directory + is in the way, PermissionError if one belongs to another account).""" + from rnfs.errors import RnfsError + from rnfs.log import LOG_NOTICE, log path = root for part in os.path.relpath(directory, root).split(os.sep): if part == ".": continue path = os.path.join(path, part) - if not make_private_dir(path) and not os.path.isdir(path): - raise FileExistsError(f"{path} is in the way of a directory") + if make_private_dir(path): + continue + info = os.lstat(path) + if fsutil.is_link(info): + raise RnfsError(f"{path} is a junction or symlink; rnfs uses only a real directory there. Move it " + f"away (rnfs creates a new one)") + if not stat.S_ISDIR(info.st_mode): + raise FileExistsError(f"{path} is not a directory, and is in the way of one") + reason = tighten(path, directory=True) + if reason is not None: + log(f"made {path} private to you and SYSTEM ({reason})", LOG_NOTICE) diff --git a/roadmap.md b/roadmap.md index 06715c3..b88b7e7 100644 --- a/roadmap.md +++ b/roadmap.md @@ -1,33 +1,57 @@ -# 0.1.5 +# 0.1.5 (released) -## CID denylist +## CID denylist (done) -- Ability to point to a CID or name as a CID deny list +- Ability to point to a CID or name as a CID deny list (`deny_lists`, by CID or `name/`) - Refuse to fetch, provide or store a CID. - Deny lists can contain file hashes and/or CIDs -## Sqlite3 migration +## Sqlite3 migration (done) -## Migrate to JSON stores to sqlite3 if they exist on startup +- State is kept in one SQLite database, `/state/rnfs.db` -- If JSON store exists, convert to the sqlite3 stor +- The JSON stores of earlier versions are imported at the first start, and kept as `.migrated` -## Alternate Pin Sources -* List if a file was pinned by an application via an API +## Client Library (done) + +* `rnfs.Client` (the user's running daemon, through the local API) and `rnfs.Node` (a node inside a program), + with the same methods and objects; provisional until 0.2 (`rnfs.API_STATUS`) + +## Change notifications for other applications to subscribe to (done) + +* Name changes: `names.watch` in the library, `GET /api/name/watch` in the local API +* Job progress and completion: job callbacks in the library + +# 0.1.6 -## Change notifications for other applications to subscribe too -* New file pinned by daemon +Security fixes, maintainability work and performance work, with no wire-format or CID changes (see +CHANGELOG.md). +# 0.1.7 -# 0.1.6 +## Faster block writes on SD cards (plan item B, then A2) +Sync blocks to disk at the durability point (before a pin or cache save) instead of once per block, with one +`syncfs()` on Linux; keep the per-block fsync on other platforms and on stacked filesystems (ecryptfs, overlayfs), +where `syncfs()` may write nothing back. A block left damaged by a power loss is found and deleted at the next +start (a `blocks/.dirty` marker). Measure on a Pi SD card first (`tests/bench_block_writes.py`); then move block +writes off the fetch thread (A2). Deferred from 0.1.6: see 0.1.6-plan.md, item B. -## Client Library +# Later + +## Alternate Pin Sources + +* List if a file was pinned by an application via an API. `pin ls` shows each pin's sources (`user`, the shared + folder, a name, a deny list, ...), but not which application pinned it ## API versioning and reference +* The public library stays provisional until 0.2 (`rnfs.API_STATUS`) +* The local API's routes are listed in `rnfs/api.py`'s docstring (checked against the code by + `tests/test_api.py`); a reference with request and response shapes is still to come + ## Per-app API tokens * Destination.hash(identity, "rnfs", "api") -* Whitelist and/or API key for auth \ No newline at end of file +* Whitelist and/or API key for auth diff --git a/specification.md b/specification.md index e5c8b6a..cb891f7 100644 --- a/specification.md +++ b/specification.md @@ -4,6 +4,11 @@ rnfs is content-addressed file sharing over [Reticulum](https://reticulum.networ features of IPFS: content IDs, chunked files, directories, a provider DHT, verified multi-source transfer, pinning, mutable names and a gateway — built on what RNS already provides. +**Version.** This document describes rnfs 0.1.6. The wire format (request paths and messages, CIDs and DAG +bytes, chunk boundaries, provider and name records, announce app_data and the LXMF field) is byte-identical +since rnfs 0.1.3, so nodes of 0.1.3 and later work together. `tests/test_wire.py` checks the current code +against vectors that 0.1.3's and 0.1.4's own code produced (`tests/fixtures/wire`). + **Principles** - No compatibility with IPFS or libp2p is required. Follow IPFS where its design helps; drop what exists only because of IP networks or legacy. @@ -178,14 +183,25 @@ pass a `max_response_size` just above their largest honest answer. - A response that makes **no progress** is cancelled and its budget charge refunded: one with nothing sent, once its grace plus `deadline_slack` has passed; any response with no progress for `stall_timeout`. - **Cooldown.** A link (and its identity, if it identified) whose response was displaced or cancelled for - making no progress, or whose response Resource it rejected before any of it was sent (no honest requester - does: otherwise one could have the node build a response, reject it and ask again in a loop), is refused - new `/blocks` for `blocks_cooldown`. A response displaced as the slowest counts as making no progress if - it was moving at less than a quarter of the rate it needs to be done when due: by its deadline, or, on a - link with no rate estimate whose MTU RNS raised, by the deadline it would have at `slow_bitrate`. So a - requester taking a little of each response can't keep its slots young by displacing its own responses - from a spare link; an honest one keeping to its schedule is displaced without a cooldown. The refusal is - answered at once, before any slot is reserved or block read. + making no progress, or whose response Resource it rejected or cancelled before a quarter of it was sent, is + refused new `/blocks` for `blocks_cooldown`. Otherwise a requester could have the node build a response, + take its first window of parts, reject it and ask again in a loop: each round costs the node reading, + hashing, packing and encrypting the whole response, and the requester one window. On RNS 1.5.6 a receiver + cancelling a Resource, at its advertisement or mid-transfer, reaches the sender as a rejection + (`REJECTED`); the part sent is the more of the progress last read and the Resource's progress then, and + counts as a quarter or more if it can't be read. A requester that gives up on a response (a stall, a + reader that stops early) MUST close the link, so the Resource fails (`FAILED`), and MUST NOT cancel or + reject the Resource itself (`Resource.cancel()`), or it would cool itself down at its providers. A link + closing starts no cooldown however little was sent: in private mode that would hold back the member's own + identity. So the only honest cancel is RNS's receiver running out of retries on a lossy path; that + requester's next `/blocks` to the node are refused (`False`) until the cooldown ends, and it moves to + another provider after `max_failures` of them (section 6). A response displaced as the slowest counts as + making no progress if it was moving at less than a quarter of the rate it needs to be done when due: by + its deadline, or, on a link with no rate estimate whose MTU RNS raised, by the deadline it would have at + `slow_bitrate`. So a requester taking a little of each response can't keep its slots young by displacing + its own responses from a spare link; an honest one keeping to its schedule is displaced without a cooldown. + Every cooldown lasts `blocks_cooldown`, flat: it doesn't grow with repeats. The refusal is answered at + once, before any slot is reserved or block read. - A handler gets no direct handle on the response Resource RNS sends for it. Responses are tracked through the link's outgoing Resources (`Link.outgoing_resources`, matched by `request_id`) and cancelled with `Resource.cancel()` (which acts only while the Resource is not yet complete). These are RNS internals that @@ -230,11 +246,15 @@ pass a `max_response_size` just above their largest honest answer. nothing after it. rnfs checks responses sent as a Resource, and those sent as one packet larger than RNS's default MTU (only links that negotiated a larger MTU carry them; smaller ones decode to little). A refused response fails its request as a malformed one does. A responder sends no requests, so a response packet on - an incoming `rnfs.dht` link is dropped unread. -- `max_upload_rate` caps bytes sent for `/blocks` on a link once it has a rate estimate. RNS paces a Resource - by its receiver, so the cap is applied between responses: the response budget uses the lower of the rate - estimate and `max_upload_rate`, and the next response on that link starts no sooner than - `bytes × 8 / max_upload_rate` seconds after the previous one started. + an incoming `rnfs.dht` link is dropped unread. Likewise a node serves requests only on incoming links, so it + SHOULD drop, unread, a request packet on an `rnfs.dht` link it opened: the remote destination of such a link + has no request size limit or handlers, and RNS would decode the request and start a thread for it first. +- `max_upload_rate` caps bytes sent for `/blocks` on a link. An explicit one applies to every `/blocks` + response, including one sent as a single packet (which never gives a rate estimate: on a link with a large + MTU every response may be one); the default, half the rate estimate on slow links, applies once the link has + one. RNS paces a Resource by its receiver, so the cap is applied between responses: the response budget uses + the lower of the rate estimate and `max_upload_rate`, and the next response on that link starts no sooner + than `bytes × 8 / max_upload_rate` seconds after the previous one started. - A responder holds an accepted request back for this at most 5.625 s (half of RNS's request timeout without RTT, 1.125 × `RNS.Resource.RESPONSE_MAX_GRACE_TIME` (10 s), so the requester doesn't give up meanwhile; not RTT-based, since the requester can inflate the RTT), and refuses it at once if it would @@ -298,6 +318,11 @@ across shared interfaces and transport nodes, a node MAY run in private mode: - outgoing links go only to nodes whose identity is a member, so the node never identifies itself to an outsider (one listed as a provider anywhere, say). +Private mode restricts rnfs's links, not RNS announces. A member's names are announced as anywhere else (section +9), and RNS carries those announces to every node that hears them, so outsiders learn each name's +`[sequence, CID]` record and can follow its changes; only the content stays member-only. `name publish` in +private mode warns of this (rnfs: the result carries a `warning`, section 9). + ## 4. Content identifiers A CID is 36 (raw) or 39 (rnfs-dag) bytes, written as `b` + lowercase unpadded base32 for people (CLI, API, @@ -527,7 +552,15 @@ and repeat. live servers are unchanged. - Every lookup has a time budget, `lookup_timeout`. - **Termination.** As in IPFS, a lookup is complete once the `beta` closest nodes it knows of have all - answered, or once every known node has been queried without finding new ones. + answered, or once every known node has been queried without finding new ones. Nodes near a key can keep + answering with ever closer IDs, generated or ground for it, so a lookup also ends, without being complete: + - once it has sent 8 × `K` (80) `/find` queries in one run (counting retries of unreliable nodes), and the + answers to those still out are in; + - after 3 × max(`alpha`, `beta`) answers in a row (9 at the defaults) that don't shorten the bit length of + the distance from the key to the `beta`-th closest node it knows. + + Neither bound ends an honest lookup: over a simulated network of 2000 nodes, each of 2500 lookups (`alpha` + and `beta` 3) completed within 12 queries and at most 8 such answers in a row. - **Finding the closest nodes** (for publishing) runs until the lookup is complete. The result is the `K` closest nodes known; the stores sent to them also show which are reachable. - **Finding providers.** Downloading starts with the first provider found. The search ends when @@ -546,7 +579,9 @@ and repeat. 4. **Reply order.** With no hops or bitrates known, this is seniority, then reply order. Path information is read once per - provider per fetch, when the provider is found. + provider per fetch, when the provider is found. Through a shared instance, a node MAY reuse the instance's + answer for a node's next-hop interface for up to 15 seconds (each answer is an RPC to it); hops, and whether + there is a path at all, are read fresh. A provider *fails* when its link fails, it answers `None` for a block of the root it advertised, it answers `False` (busy or refused) `max_failures` times in a row, or it stalls twice (section 7). A `False` is retried @@ -592,7 +627,10 @@ and repeat. as many); a publisher that already holds as many as any other is refused instead, so a new publisher can always store something, a flood churns only the pool, and the pool is shared out evenly between publishers rather than emptied of a newcomer's records. A pooled record leaves the pool when it is - republished while the rest has room. The node's own records are always accepted. + republished while the rest has room. The node's own records are always accepted and never evicted to make + room for others' (rnfs: nor counted toward its local limit of 1000 publishers per key, a newcomer to a full + key evicting a random one of the others but the `K/2` oldest), so Sybil stores can't push a node's own record + out of its own store. - First-seen times are persisted, and kept for `PROVIDER_TTL` after a publisher's last record is gone (rnfs: for publishers seen for at least `provider_seniority`, at most `max_records / 10` of them, the oldest departure forgotten first), so a publisher that was away for a while isn't new again. A newcomer @@ -690,13 +728,22 @@ and repeat. chunks of 4 KiB or more (one level up to 1 MiB, seven for any size): a range ends inside at most two nodes of a level. Otherwise it fails ("too large"); a requester counts a file node not yet received as `MAX_BLOCK_SIZE`, as in item 5, so a small range ending in two nodes of a level asks for them one at a - time. -7. Spread requests over up to `max_parallel_providers` providers (1 on slow nodes, where parallel transfers - only contend for the same channel). A response *stalls* if it makes no progress for `stall_timeout`, or - misses the same deadline the responder uses (section 3). A stalled provider's link is closed, which cancels - the transfer, and its blocks are requested again from another provider; providers that keep failing are - handled as in section 6. A fetch that fails or is cancelled closes the links of its requests still in - flight, so their responses don't keep using the link and hold up the next `/blocks` to those nodes. + time. The chunks are fetched a window (32 chunks) at a time, and the next window's are asked for before the + reader is handed the current one's bytes, so the providers don't wait for the reader; a reader that stops + early has had at most one window fetched that it didn't read. +7. Spread requests over up to `max_parallel_providers` providers (1 on slow nodes, where parallel transfers only + contend for the same channel). A response *stalls* if it makes no progress for `stall_timeout`, or misses the + same deadline the responder uses (section 3). A stalled provider's link is closed, which cancels the transfer, + and its blocks are requested again from another provider; providers that keep failing are handled as in + section 6. A fetch that fails or is cancelled, or whose reader stops early (item 6), closes the links of its + requests still in flight, so their responses don't keep using the link and hold up the next `/blocks` to those + nodes, unless another fetch has a request on the same link: closing it would fail that one, so the request is + left to finish. It never cancels their Resources (section 3, Cooldown). When nothing is left to ask for and a + provider is idle, a requester MAY ask it too for a batch that has been in flight for longer than + `deadline_slack` (end-game duplication), so a response that keeps making just enough progress not to stall + doesn't hold up the fetch. The first verified copy wins; the request still sending blocks that have arrived + then has its link closed (left to finish, under the same rule), which doesn't count as a stall of its + provider. rnfs duplicates each request at most once, and not on slow nodes, which use one provider at a time. 8. Blocks are stored as they arrive, so an interrupted fetch resumes where it stopped. An RNS Resource is only delivered when complete, so a response lost mid-transfer yields nothing and its blocks are requested again. 9. When the target is complete, the node provides it if its strategy says so (section 6). @@ -763,7 +810,17 @@ and repeat. `add` (copied into the block store) and pinned with source `folder`. A changed or removed file releases its old pin. Unlike `add`, a scan MUST skip (with a log line) a regular file that has other hard links, at the top level and inside shared directories: it is also a file elsewhere, which whoever may write to the folder -could have linked in. It is imported at the next scan after its other links are gone. +could have linked in. It is imported at the next scan after its other links are gone. A scan recognises +unchanged content by its metadata rather than by reading it: an entry whose files' names, sizes, modification +times and execute bits are as before isn't read, nor, inside an entry that changed, a file imported before whose +device, inode, size, modification time and execute bit are as they were (its CID is reused while every block of it +is still stored and none is denied, and its content hash still goes through the deny list). Content rewritten in +place keeping all of these goes unnoticed until one changes. `files/` may be a +symlink, but a scan MUST NOT import from a folder that resolves, as opened, to the rnfs dir, to a directory +holding it, or into `storage/`, `names/`, `state/`, `blocks/` or `lxmf/`, which hold the node's keys and state: +rnfs logs a warning (once, while it stays so) and scans it as an empty folder: nothing is imported and every pin +with source `folder` is released (including those an earlier version made from it). rnfs also warns, once, when +a directory above the folder can be changed by other users, who could replace the folder with another. **Archives** move content without a network (the equivalent of IPFS CAR files). - `export` fetches the target if needed, then writes a msgpack stream: `["rnfs-archive", 1, [root CIDs]]`, @@ -818,8 +875,17 @@ off their node. Nothing on the wire changes. (so a range fetches the whole file first; its hash is then remembered; the gateway's `?format=raw` and `?format=archive` too), and as `add`, the shared folder or `import` read it (the add or import fails). By then the file's blocks have been downloaded. A file-hash entry doesn't stop those blocks being fetched (by `pin add`, `export` to a file, a remote pin or a followed name), stored, - served or provided: deny the CID for that. A file of one block has its content's hash as its CID's digest, + served or provided (a file of several chunks is still served chunk by chunk, by `/blocks` and by the gateway's + `?format=raw` of each block's CID): deny the CID for that. The first time a node refuses a file by its hash it + logs a notice naming the command that does (`rnfs deny add `). A file of one block has its content's hash as its CID's digest, so either entry stops it; a sealed file matches by its sealed bytes, as stored, not by what it decrypts to. +- Hashing a file reads every occurrence of content its DAG repeats, so a few small blocks can declare a file of + 2^56 bytes. While there are file-hash entries, a file whose hash isn't known yet and whose declared size is + more than 16 times the bytes of its distinct chunks plus 64 MiB (`DENY_HASH_RATIO`, `DENY_HASH_SLACK`, + section 12) can't be checked, and is refused as if denied, before any chunk is read (`451`; an add or import + fails as above). The chunks are counted from the file nodes, so a range read (`cat`, the gateway) fetches the + whole DAG first. Honest files repeat less, except highly repetitive ones over 64 MiB such as a zero-filled disk + image. - Provider records other nodes store here for a denied CID are kept and handed out as usual (`/find` sees only keys). @@ -852,7 +918,9 @@ A **name** is a mutable pointer to a CID, owned by a key pair: the IPNS equivale resolve, and persist it. If that fails, the publish fails and a pin step 2 added is released. 4. Release that source from the previous target (only now, so a crash leaves the target the persisted record names pinned; any other pin of the source is released at the next start, section 8). -5. Announce, and re-announce every `announce_interval`. +5. Announce, and re-announce every `announce_interval`. In private mode the announce reaches outsiders all the + same (section 3), and the publish warns of it: rnfs adds a `warning` to the result (`POST /api/name/publish` + and `rnfs name publish`, which prints it to stderr). Before the first publish of a key imported from elsewhere, or found without stored state, the node resolves the name through the network and asks keepers (below), since the key's earlier records may be held only there. @@ -862,16 +930,19 @@ the name through the network and asks keepers (below), since the key's earlier r - To refresh, call `Transport.request_path(name)`, wait up to `path_timeout`, then read `Identity.recall_app_data(name)`. - If the path request heard nothing within `path_timeout`, or heard only an announce emitted more than - 2 × `announce_interval` ago (or always, with `keeper_query = always`), ask keepers (below). Storing a provider record is free, and so is answering `/find`, so neither a record's age, + 2 × `announce_interval` ago or a suspected replay (below), or always with `keeper_query = always`, ask keepers + (below). Storing a provider record is free, and so is answering `/find`, so neither a record's age, nor the order a reply lists it in, nor how many nodes list it may decide which keepers are asked. Standing is read from the routing table as it was when the query started: a node the lookup meets doesn't gain rank from answering it, and one that fails a query meanwhile loses rank. An *established* node is one that was in the table for over `provider_seniority` then, and hasn't failed a query since last heard nor been silent for 2 × `announce_interval`. - Find providers of the name's key (section 6). Providers recorded locally are candidates, never a reason - to skip the search, which runs until 3 × `alpha` are known and two established sources listed some, or - until the lookup is complete. A source is a node whose `/find` answer listed providers, or this node if - it holds a record stored by an established publisher. + to skip the search, which runs until 3 × `alpha` are known and two established sources listed some, until + some are known and `lookup_timeout` / 5 has passed, or until the lookup ends. A source is a node whose + `/find` answer listed providers, or this node if it holds a record stored by an established publisher. + Candidates are asked only once the search has stopped, never as they arrive, so reply order doesn't choose + them. - Rank the candidates by routing-table seniority, as providers are (section 6): established servers first, then other servers in the table, then servers that failed a query or went silent (an owner that has left), then IDs not in the table; at random within each group. @@ -885,6 +956,13 @@ the name through the network and asks keepers (below), since the key's earlier r keepers have answered; when no sequence is known, only established keepers' answers count toward those two. At most 3 batches. A slow node (`alpha` 1) asks at most 2 batches, and asks the second for want of a second answer only when it knows no sequence to compare the first against. + - Each batch is waited for up to the lookup's time left or, if longer, a `/name` budget of its own: + `path_timeout` + 5 s + 5 hops × RNS's link establishment timeout per hop (6 s) + 2 × RNS's request timeout + without a measured round trip (11.25 s), 87.5 s at the defaults. A search that used all of + `lookup_timeout` still leaves keepers time to set up a link and answer. + - If the candidates run out before the answers settle the name, with batches left, and the search was + stopped early (not ended by the lookup) and some of its candidates asked, the search resumes within the + lookup's time left, stopping by the same rules for the candidates it newly finds, which are then asked. A keeper query (a DHT lookup plus a few requests) costs roughly ten to twenty times the airtime of a resolve a path response answers, so `always` SHOULD NOT be used on slow links. @@ -894,6 +972,18 @@ the name through the network and asks keepers (below), since the key's earlier r newest emission time (signed) RNS holds for the name, which repeating an old announce doesn't change (rnfs: from RNS's path table). A neighbour repeating an old announce can then keep a replaced record current for at most that long, plus the age a caller accepts (a follow's or deny list's 2 × `announce_interval`). +- An announce RNS stored that was emitted before the newest one its path to the name holds, or that carries + another record than the announce the path was taken from, is a *suspected replay* (rnfs: read from RNS's + announce cache, which holds the announces paths were taken from). Its record is heard no later than its own + emission time, or the path's if its own is no longer known, and the resolve asks keepers. Without a path + nothing is compared. +- A resolve that accepts a record up to some age (the gateway's 60 s, a follow's or deny list's + 2 × `announce_interval`) skips the refresh while the record held was heard within that age, or while a + refresh of the name ended within it and the record it left is still held (rnfs: one that found nothing + counts too, a short negative cache; one that left a suspected replay doesn't). With the owner offline a + keeper's record keeps its old emission time, so it would otherwise go to the network every time. A resolve + of a name being refreshed waits for that refresh, whatever age it accepts, and takes the record it leaves. + A name republished moments ago may so show its older record on the gateway for up to 60 s. **Sequence.** RNS stores any validly signed announce, including a replayed old one, so persisted sequences are what prevent rollback. @@ -951,7 +1041,11 @@ resolves after its owner's announce has expired from the network. source `follow:`. When the name moves, the new target is fetched completely in the background and pinned, and only then is the previous target's follow pin released, so a complete copy is always held. The fetch has the usual limits: `max_download_size`, the DAG byte bound of its new target's size (section 7) and -the disk reserve (section 8). A +the disk reserve (section 8). It also has `follow_quota` (section 12), since a pinned fetch never stops at +`max_storage`: a new target whose verified size, or whose DAG's distinct blocks (stored or downloaded), would +exceed it is refused (a failed fetch, so the old target stays pinned), and `follow_quota` = 0 fetches nothing. +While one target replaces another both are held, so a followed name can use up to 2 × `follow_quota` for a +time. A target pinned before the quota was lowered stays pinned. A followed name is checked when a newer record is heard and every `announce_interval`. - The periodic check uses the record held, unless nothing was heard for the name for 2 × `announce_interval`; only then does it resolve through the network (a path request, then keepers). @@ -969,9 +1063,11 @@ followed name is checked when a newer record is heard and every `announce_interv 6), so a name can be updated about once an hour sustained. - RNS orders announces by a signed emission timestamp from the publisher's clock, so a publisher whose clock goes backwards can't update its name until old paths expire. -- A resolver's first resolution of a name, through an announce or a keeper, can't detect a replayed old - record. A node that missed a name's newer announces and is next to someone repeating an older one gets the - newer record from keepers only once the repeated announce is 2 × `announce_interval` old (above). +- A resolver can't detect a replayed old record when RNS holds no newer path to the name (a first resolution + with no path, or after the path expired), nor when RNS's announce cache can't be read (a shared instance's + client whose cache isn't the instance's). A node that missed a name's newer announces and is next to someone + repeating an older one then gets the newer record from keepers only once the repeated announce is + 2 × `announce_interval` old (above). - The same name key used on two machines can produce two records with one sequence; each resolver keeps the one it heard first. @@ -983,7 +1079,7 @@ as is cheap. Rules live in the sections referenced. | Threat | Protection | |---|---| | Forged or corrupted data | Every block verified against its CID (7) | -| Size lies, repeated or empty-subtree bombs, deep trees | Verified sizes, `max_download_size`, `max_output_entries`, the DAG byte bounds of whole-DAG fetches, listings and byte-range reads, `MAX_DEPTH`, each block fetched once (5, 7) | +| Size lies, repeated or empty-subtree bombs, deep trees | Verified sizes, `max_download_size`, `max_output_entries`, the DAG byte bounds of whole-DAG fetches, listings and byte-range reads, `MAX_DEPTH`, each block fetched once; while the deny list has file hashes, a file repeating its content more than `DENY_HASH_RATIO` times (beyond `DENY_HASH_SLACK`) is refused rather than hashed (5, 7, 8) | | Malformed DAG nodes | Strict msgpack types, re-encoding and per-node checks (5). The tree rule gives honest writers one CID per content; readers don't enforce it | | Advertising on someone else's behalf | Records name only the identified requester (6) | | Advertising content it doesn't have, or stale records | Failed providers removed; the search runs again; withdrawals (6, 7) | @@ -991,14 +1087,14 @@ as is cheap. Rules live in the sections referenced. | Hiding content near a key | Provider search doesn't stop at the first reply that lists providers (6) | | Routing table takeover with new identities | Only proven-reachable nodes stored; full buckets keep long-lived nodes (6) | | Fake node IDs in replies | Unknown IDs never stored; at most `alpha` path requests for them per lookup; lookup time budget (6) | -| Slow or stalled transfers (both directions) | Rates measured from transfers, never from round-trip time; response deadlines; displacement protection earned by progress (a per-response grace, then a throughput floor), so a response that transfers nothing can't hold a slot; no-progress and stall cancellation with refund; a cooldown on a link whose response made no progress, was rejected unsent, or fell far behind its schedule; many fast-link slots; requester stall detection (2, 3, 7) | +| Slow or stalled transfers (both directions) | Rates measured from transfers, never from round-trip time; response deadlines; displacement protection earned by progress (a per-response grace, then a throughput floor), so a response that transfers nothing can't hold a slot; no-progress and stall cancellation with refund; a cooldown on a link whose response made no progress, was rejected or cancelled before a quarter of it was sent, or fell far behind its schedule; many fast-link slots; requester stall detection, and end-game duplication of a response still arriving after `deadline_slack` (2, 3, 7) | | Request flooding, link exhaustion | Request size, per-link and global limits; a per-link request rate limit; eviction of idle incoming links; half-open links capped in their own bucket, oldest closed first (3) | | Decompression and oversized-response bombs | Compressed Resources refused; response sizes capped; multi-segment Resources refused on `rnfs.dht` links, LXMF delivery links and links to LXMF propagation nodes; `rnfs.dht` responses' msgpack headers checked before RNS decodes them (3, 7, 13) | | Reading another node's download history | Only provided content is served (7). Under the default `roots`, complete cached downloads are provided; `pinned` keeps them private | | Name rollback or forgery | Self-certifying names; RNS-verified signatures; persisted highest sequence (9) | | Forged or withheld kept names | Records verified against the name's key; highest persisted sequence; several keepers asked, ranked by routing-table standing from before the query and drawn from different sources; answers from established keepers only settle a name with no known sequence (9) | | Tampered or spliced sealed files | Per-segment HMAC bound to the header, index and final flag, checked before decrypting (5) | -| Unsolicited LXMF links, inbox flooding | Strict field parsing, size caps per link, per message and for the whole inbox; at most 50 kept messages per untrusted sender; new ones from untrusted senders rate limited per sender, then node-wide; auto-fetched pins never pushed out by others' messages; signature-validated senders only; rate-limited path requests for unknown senders; `lxmf_accept_from`, per-sender `lxmf_auto_fetch_quota` (13) | +| Unsolicited LXMF links, inbox flooding | Strict field parsing, size caps per link, per message and for the whole inbox; at most 50 kept messages per untrusted sender; new ones from untrusted senders rate limited per sender, then node-wide; auto-fetched pins never pushed out by others' messages, and each auto-fetch sender held to 1/n of the inbox; signature-validated senders only; rate-limited path requests for unknown senders; `lxmf_accept_from`, per-sender `lxmf_auto_fetch_quota` (13) | | Unwanted content on a public gateway or provider node | Deny lists: denied CIDs never fetched, stored, served or provided, anywhere in a DAG; file hashes checked where content is assembled; lists size-capped and checked before use (8) | | Uploads exhausting a shared channel | `upload_budget` (charged for bytes sent, at most half of it per requester in flight), interface rules and quiet hours, applied to `/blocks` only (3) | | Names and paths chosen by others reaching the local user | Default output names only if printable, not hidden, not starting with `-` and one valid file name, else the CID; non-printable characters escaped in output and logs (11) | @@ -1037,7 +1133,11 @@ RNS's own `blackhole_identity` is not used: it is instance-wide and persistent. **Limits.** - RNS identities are free to create. A determined attacker can surround one key with responsive nodes and make that key hard to find (a Sybil attack); it can't forge data. `get --from ` bypasses - the DHT when a provider is known. + the DHT when a provider is known. Grinding IDs near a key is cheap: one CPU core makes about 20,000-30,000 + identities a second, each with its node ID (measured for rnfs 0.1.6), and getting `K` of them closer to a + key than the `K` closest of n honest nodes takes about n tries, so a fraction of a second against 10,000 + nodes and under a minute against a million. Raising this cost means changing how node IDs are derived, a + protocol change left for a later version. - Patient attackers that keep identities alive for days defeat the preference for old records and long-known providers. Such identities, established in a resolver's routing table near a name's keeper key, can also answer its keeper query with a replayed older record of the name, unless the resolver knows a newer @@ -1045,15 +1145,36 @@ RNS's own `blackhole_identity` is not used: it is instance-wide and persistent. - A flood of new identities with more of them than the newcomer pool has records leaves each, an honest newcomer included, about one pooled record at a time; a newcomer's first-seen time outlasts that only while fewer than the bounded number of newcomers first seen before it were evicted too. +- Private mode doesn't hide names: a member's names are announced over RNS like any other, so outsiders who + hear them learn each `[sequence, CID]` record and when it changes, though members won't serve them the content + (section 3). `name publish` in private mode warns of this. - Bans are evaded by switching identity; they only make an attacker pay for a new one. -- A provider can lower its own measured rate, and so slow a fetch from it; on slow nodes, which use one - provider at a time, this can delay a fetch for a long time. +- A provider can lower its own measured rate, and so slow a fetch from it, or drip a response just fast enough + not to stall: without a rate estimate its deadline assumes `unmeasured_bitrate`, hours for one 64 KiB block. + Once nothing else is left to ask for, another provider is asked for its blocks after `deadline_slack` + (section 7); on slow nodes, which use one provider at a time, this can delay a fetch for a long time. - Mass announces from new identities compete for RNS's announce bandwidth, which RNS itself caps. +- Many LXMF identities can keep the strangers' bucket empty (one message a minute, each from a new identity, + is enough), so no other untrusted sender's message is kept; contacts and other trusted senders are unaffected + (section 13). The node counts the messages dropped this way (`status`'s `untrusted_dropped`), so apps can + suggest adding the senders the user expects as contacts. LXMF stamps (proof of work) would make each message + cost the sender, but announcing a stamp cost changes the announce and refuses senders that haven't heard it. - RNS decrypts and decodes every request packet and starts a thread for it before rnfs sees it, so a link sending requests as fast as its interface carries them still costs the node that work (about 0.14 ms of CPU each over loopback): the request rate limit (section 3) removes only rnfs's share. Likewise a node rejecting `/blocks` responses only after RNS has received them can be made to build one per cooldown per link, and RNS spends a key exchange on every new link. +- A `/blocks` requester that takes a response and cancels it still has the node build the whole response (read, + hash, pack and encrypt it) each round, and the cooldown binds it only below a quarter sent (section 3): + - cancelling at a quarter or more starts no cooldown, so it costs the requester at least a quarter of each + response's bytes. Where a single window of parts is that much (a link whose MTU RNS raised, or a small + response), cancelling after the first window is not told apart from it; + - closing the link instead starts no cooldown however little was sent, deliberately: an honest reader that + gives up closes its links, and in private mode a cooldown would hold back its own identity. Each round + then costs a new link (a key exchange at both ends, and a place among `max_incoming_links`); + - an honest requester whose Resource RNS cancels after running out of retries on a lossy path, before a + quarter of it arrived, is cooled down at that node for `blocks_cooldown` and fetches from other providers + meanwhile. - Any web page can make the local gateway fetch content (a plain request to `127.0.0.1` passes the Host check), and can time responses to guess what is cached. The gateway's own limits (section 11) bound the airtime this costs: `gateway_fetches`, and no `?format=` response (raw block or archive) at all for a web @@ -1119,9 +1240,9 @@ are local conveniences, not part of the protocol. | `jobs [list]`, `jobs show\|wait\|cancel `, `jobs clear []` | Background jobs (below) | | `contacts add [--kind name\|node\|lxmf]`, `contacts rm `, `contacts ls` | Contacts (below) | | `deny add\|rm …`, `deny ls` | The local deny list (CIDs, `sha256:` file hashes); `ls` also shows each `deny_lists` source, the version in use and its state (section 8) | -| `send
[-m text] [--propagate] [--nearest]`, `inbox`, `inbox get [--link K] [-o out]`, `inbox rm \|--all`, `outbox`, `lxmf address`, `lxmf sync [--nearest]` | LXMF (section 13). `--nearest` uses the nearest propagation node heard when `lxmf_propagation_node` isn't set | +| `send
[-m text] [--propagate] [--nearest]`, `inbox`, `inbox get [--link K] [-o out] [--use-sent-cid]`, `inbox rm \|--all`, `outbox`, `lxmf address`, `lxmf sync [--nearest]` | LXMF (section 13). `--nearest` uses the nearest propagation node heard when `lxmf_propagation_node` isn't set; `--use-sent-cid` gets a name link's sent CID when the name can't be resolved | | `peers [--json]`, `stats [--json]` | Known nodes with hops, interface and traffic; traffic since start, refusals by cause, per interface | -| `status` | Node ID, RNS identity hash (the value other nodes list in `members` and `remote_pin_identities`), known nodes, records held, storage use, traffic, jobs, deny-list entries and sources, LXMF address (only the version without a readable API token) | +| `status` | Node ID, RNS identity hash (the value other nodes list in `members` and `remote_pin_identities`), known nodes, records held, storage use, traffic, jobs, deny-list entries and sources, LXMF address, inbox size and untrusted senders' messages dropped by the rate limits (`untrusted_dropped`, section 13) (only the version without a readable API token) | Wherever a command takes a target, it may start with a name contact's nick (`get alice/docs/report.pdf`); where it takes a node ID (`get --from`, `ping`, `pin remote`), a node contact's nick works too. @@ -1131,7 +1252,16 @@ links and messages) MUST be printed and logged with non-printable characters esc terminal escapes or fake log lines. A name chosen by someone else is used as a default output name only under the `get` rule above. `get` writes out under a temporary hidden name in the output's directory and renames it to the output once complete (refusing if the output appeared meanwhile), so a failed or interrupted -write-out never leaves a partial output under the requested name. +write-out never leaves a partial output under the requested name. A directory's entries MUST end up inside +it even if someone who can write beside or inside it swaps one of its directories for a link or another +directory while it is written; such a swap fails the `get`. rnfs: on POSIX the temporary directory is made +mode 0700 (setgid kept) and must still be empty once it is, every entry is created relative to its parent's +open directory without following links, and each directory is reopened from the root, component by +component, and checked against the device and inode it had when made; the temporary directory gets back the +mode it was created with when complete. On Windows it is created with the protected access list of the rnfs +directory (the user and SYSTEM only), every file's final path must be inside it, and once complete it +inherits its folder's access list again. Either way, it must still be the directory made before it is +renamed. A single file is created directly in the output's directory. **API** (`127.0.0.1:8788`) and **gateway** (`127.0.0.1:8789`) are separate ports, so pages served by the gateway can never reach the API. @@ -1151,11 +1281,20 @@ gateway can never reach the API. to its owner only (on Windows: a protected access list allowing only the user and SYSTEM, which the files below it inherit; Administrators are tolerated where an older version left them). On Windows a new rnfs directory and its subdirectories MUST get that access list when they are created (not after), the daemon - MUST refuse a new one that already holds anything, and one that is a junction or symlink. The settings file - (`rnfs.toml`) and the identity MUST be checked before use, through the handle they are read from: one owned - by another user (account) is refused, one others may access is made private (with a warning), and one that - can't be made private stops rnfs. The identity MUST also be a regular file (not a symlink, FIFO or device); - anything else stops the daemon with an error. + MUST refuse a new one that already holds anything, and one that is a junction or symlink. Its existing + subdirectories (rnfs: `state`, `blocks`, `storage`, `names` and `lxmf`) MUST be real directories (not + symlinks or junctions) owned by the user, checked without following a link, and one others may access is + made owner-only (rnfs: 0700, with a notice; if others could write to it, entries in it owned by another + user are reported, as a warning). Anything else stops the daemon with an error before Reticulum starts. + The settings file (`rnfs.toml`), the identity, and Reticulum's `config` when `reticulum_dir` is the rnfs + directory MUST be checked before use, through the handle they are read from: one owned by another user + (account) is refused, one others may access is made private (rnfs: 0600 on POSIX; with a warning), and one + that can't be made private stops rnfs. Each MUST also be a regular file (not a symlink, FIFO or device), + opened without following a link or waiting on a FIFO; anything else is refused. A symlinked `rnfs.toml` is + refused too. rnfs reads at most 1 MiB of `rnfs.toml`. A start checks the rnfs directory, then reads the + settings, then checks the subdirectories, then takes the directory's lock, in the daemon and in an embedded + node alike; clients (the CLI, `rnfs.Client`) check `rnfs.toml` the same way when they read it, and show + what was changed (the CLI prints it, `rnfs.Client` raises a `RuntimeWarning`, an embedded node logs it). - An API request carrying `X-Rnfs-Challenge: <32 hex digits>` is answered with `X-Rnfs-Proof: hex(HMAC-SHA256(token, "rnfs api proof "))`, `` being the API's. Before it sends the token, a client SHOULD check this proof with a `GET /api/status` on the connection @@ -1163,7 +1302,11 @@ gateway can never reach the API. runs on the port while the daemon is down learns neither the token nor a name key being imported, nor can answer in the daemon's place. The API keeps a connection open after a status request carrying a valid challenge and `Connection: keep-alive` (and no body), answering it with `Connection: keep-alive`, for one - more request, which is its last; every other connection is closed after its request. + more request, which is its last; every other connection is closed after its request. Any caller can ask for + this (the proof needs no token), and the second request gets a new read deadline, so one connection can hold + a connection slot (below) about twice as long as a single request could (rnfs: about 2 × 15 s). This is + accepted: holding every slot was already possible by opening connections, and the keep-open is bounded to + exactly one more request. - The API never sends CORS headers. - Both servers SHOULD time-limit reading a request (rnfs: 15 s for the request line, headers and small bodies; 60 s between reads of an upload), but not the node's work, and SHOULD cap concurrent @@ -1184,6 +1327,9 @@ gateway can never reach the API. and never fetches them: its Content-Type then comes from the name, or is `application/octet-stream`. A request that isn't the user's own navigation (below) always reads them, fetching them if need be, so its Content-Type doesn't depend on what the node stores. + - The deny list (section 8) applies to those bytes as to any read: a `HEAD` of a stored file that is denied, + by a CID read or by its content hash, answers `451` like `GET`. While the deny list has file hashes, the + stored bytes are used only once the whole file is stored (its hash is checked first, never fetching). - **Trustless responses** (`/rnfs/` only; `/name/` answers `?format=` with 400): - `/rnfs/?format=raw` returns the CID's own block, re-verified, as `application/vnd.rnfs.raw`; a path is refused (400). @@ -1193,7 +1339,8 @@ gateway can never reach the API. its first byte, so the user's own `GET` fetches the whole DAG first, within `max_download_size` and `max_output_entries` (413 otherwise), and the fetch is cancelled if the client disconnects. - Every format is answered only to the user's own `GET`, which fetches what isn't stored. A request that - isn't the user's own navigation (`Sec-Fetch-Site` present and not `none`) and any `HEAD` get + isn't the user's own (`Sec-Fetch-Site` present and not `none`, or absent with a browser's `User-Agent`; + below) and any `HEAD` get `409 Conflict`, whether the target is stored or not, without the node fetching anything. A browser downloads both types, and a frame of a download fires no load event while an error page does, so an answer that depended on what is stored (a download for stored content, a fetch that fails or a `503` for @@ -1222,7 +1369,12 @@ gateway can never reach the API. - Requests that aren't the user's own navigation (`Sec-Fetch-Site` other than `none`) may start at most `gateway_fetches` network fetches or name resolutions at a time. Content already stored is served without limit. Browsers send `Sec-Fetch-Site` to `127.0.0.1` and `localhost`, and `none` only for a navigation the - user started (address bar, bookmark); a request without it comes from a local tool the user runs. + user started (address bar, bookmark). A request without it is the user's own only if its `User-Agent` + doesn't start with `Mozilla/` (rnfs ignores case and leading spaces): then it comes from a local tool the + user runs, such as curl. Every browser's `User-Agent` starts with `Mozilla/`, and browsers without Fetch + Metadata (Safari before 16.4, Firefox before 90) send no `Sec-Fetch-Site` for a web page's requests either, + so all their requests are treated as a web page's: in them even the user's own navigation gets `409` for + `?format=` and is held to `gateway_fetches`. - Apart from timing, the `gateway_fetches` limit and content no provider has, a web page's request gets the same answer whether the node stores the content or has to fetch it. The limit is a known, accepted leak: while a page keeps every slot busy with fetches, stored content is still served and content that needs a fetch gets 503, @@ -1247,8 +1399,8 @@ optionally from one node), `pin` (fetch a target completely and pin it with sour - Cancelling a queued job MUST end it at once; cancelling a running one MUST stop its fetch at the fetch's next step (within 0.5 s), not at the end of a request in flight, whose link is then closed (section 7). Writing out (`get`) or the archive (`export`) after the fetch is local and is not interrupted. -- Stopping the node MUST NOT wait for jobs beyond a short bound (rnfs: 2 s); a job interrupted by the - shutdown is saved as queued and resumes at the next start. +- Stopping the node MUST NOT wait for jobs beyond a short bound (rnfs: the node's stop budget, 5 s for every + thread it ends); a job interrupted by the shutdown is saved as queued and resumes at the next start. - At most `max_finished_jobs` (at least 1) finished jobs are kept, the oldest finished forgotten first, but a finished job is kept for at least 60 s whatever the cap, so a client following a fast job always finds its result. At most 1000 jobs may be queued or running (more are refused as busy). Two queued or running @@ -1336,7 +1488,7 @@ setting. | `max_incoming_links` | 128 | | | `max_blocks_responses` | 4 | `/blocks` responses in progress on slow links: half of this; the rest for other links, unless `fast_blocks_responses` is more. At least 2 | | `fast_blocks_responses` | 16 | `/blocks` responses in progress on links that aren't slow, if more than the rest of `max_blocks_responses` | -| `max_upload_rate` | none; half the rate estimate on slow links that have one | | +| `max_upload_rate` | none; half the rate estimate on slow links that have one | An explicit value applies to every `/blocks` response, with or without a rate estimate | | `max_records` | 100 000 | | | `ban_time` | 1 h | | | `max_storage` | 10 GiB | | @@ -1363,6 +1515,7 @@ setting. | `quiet_hours` | none | Local time ranges such as `22:00-06:00` or `01:00-05:00,12:00-13:00`, during which `/blocks` is refused | | `keep_max_names` | 100 | Names kept (keep and follow) for others; this node's own names don't count. A whole number | | `keeper_query` | `fallback` | `fallback`: ask keepers when a path request heard nothing; `always`: on every network resolve (about 10× the airtime) | +| `follow_quota` | 1 GiB per followed name | Bytes a followed name's new target may hold, its size and its DAG's blocks; 0: followed names aren't fetched. A swap holds the old and new targets, so up to twice this (section 9) | | `max_jobs` | 2; 1 on slow nodes | Jobs running at once (a whole number ≥ 1). Foreground `get`, `pin add` and `export` run as jobs too, so they queue beyond this | | `max_finished_jobs` | 100 | Finished jobs kept (a whole number ≥ 1); each is kept at least 60 s regardless (section 11) | | `lxmf_enabled` | on if the LXMF package is installed | A boolean. `false` turns LXMF off; `true` without the package logs a warning and stays off | @@ -1375,6 +1528,13 @@ setting. | `lxmf_auto_fetch_quota` | 1 GiB per sender | Storage the pins of one auto-fetch sender may use | | `deny_lists` | empty | Deny-list sources: `[/path]` or `name/[/path]`, each a file (section 8) | +**Local limits.** Fixed by this implementation rather than set; other implementations may choose their own. + +| Name | Value | Notes | +|---|---|---| +| `DENY_HASH_RATIO` | 16 | While the deny list has file hashes, a file declaring more than this times the bytes of its distinct chunks, plus `DENY_HASH_SLACK`, is refused unhashed (section 8) | +| `DENY_HASH_SLACK` | 64 MiB | Any file up to this size can be hashed, however much it repeats | + ## 13. rnfs URIs and LXMF rnfs links can be sent and received as [LXMF](https://github.com/markqvist/LXMF) messages, so content can be @@ -1473,22 +1633,34 @@ unreserved, pct-encoded: as in RFC 3986 on a line of its own is read whole; elsewhere, trailing punctuation (`.,;:!?)]}'"`) is taken as the sentence's. At most 8 links, 4000 characters of text, 256 of title and 64 of display name are kept. Text from messages MUST be shown with control characters escaped. +- A sender's display name is its own word, so it MUST NOT be shown where it could pass for the sender's address + or trust. `inbox` shows each message's sender address first, then `untrusted`, `trusted` or `contact ` + (the API's computed `trusted` and `contact` fields; `?` from a daemon that doesn't give them), and the display + name last, quoted, with backslashes, quotes, blank fillers (U+115F, U+1160, U+3164, U+FFA0, U+2800), a + combining mark that follows nothing printed or is the third in a row, and a name of only spaces escaped too. - At most 500 messages and 4 MiB (as JSON) are kept, and at most 50 from one sender not in `lxmf_auto_fetch_from`. To make room, the oldest message holding no auto-fetched pins (none pinned or queued) is dropped; a message holding pins is not dropped for others' messages: if every kept message holds - pins, a new message without any is not kept (and one with pins pushes out the oldest). A trusted sender's - message is dropped only to make room for another trusted sender's, and only once no untrusted sender's + pins, a new message without any is not kept, and one with pins pushes out its own sender's oldest message + (or, if that sender has none, is not kept). Each of the n senders in `lxmf_auto_fetch_from` keeps at most + 1/n of the 500 messages and of the 4 MiB (at least one message and 16 KiB), messages holding pins included: + past its share, a new message from it drops its own messages, first those without pins, then those whose + pinned CIDs another of its messages also holds (dropping them releases nothing), then the oldest. A trusted + sender's message is dropped only to make room for another trusted sender's, and only once no untrusted sender's message without pins is left; a new message from an untrusted sender that would need one dropped is not kept. New messages from untrusted senders are also rate limited: each sender has an allowance of 4 at once, then one per 15 minutes, and within it messages are taken from a node-wide bucket of 32 at once, then one - per minute; others are not kept. So one sender can't keep the node-wide bucket empty for everyone else. Each message's record is at most 16 KiB as JSON: links beyond that are dropped (the + per minute; others are not kept, and counted (`untrusted_dropped` in `status`). So one sender can't keep the + node-wide bucket empty for everyone else (many can: section 10, Limits). Each message's record is at most 16 + KiB as JSON: links beyond that are dropped (the first is always kept), then text is cut. - **Auto-fetch.** Links from a sender in `lxmf_auto_fetch_from` are fetched completely and pinned with source `lxmf:`, one fetch at a time, taking senders in turn. At most 64 links wait: when that many do, a new link is skipped, unless its sender has at least two fewer waiting than the sender with the most, whose newest waiting link is skipped instead. Auto-fetches still waiting when the node stops are queued again, oldest first, at the next start; stopping the node or - removing the message stops a fetch in progress. The fetch is refused if the target's verified size plus what + removing the message stops a fetch in progress. A waiting link whose sender is no longer in + `lxmf_auto_fetch_from` is refused, not fetched: at the next start (with LXMF off too) and when its turn comes. The fetch is refused if the target's verified size plus what that source already pins would exceed `lxmf_auto_fetch_quota`, or if the blocks of its DAG would (as for remote pins, section 8); `max_download_size` applies too. A `name/` link is resolved without persisting the name's sequence (only the capped table of section 9 holds it), so names arriving unasked don't grow @@ -1498,6 +1670,13 @@ unreserved, pct-encoded: as in RFC 3986 the DHT if that fails. `inbox get` without `-o` names its output by the `get` rule (section 11): the link's last path name only if it is printable, not hidden, doesn't start with `-` and is a valid file name, else the root CID. +- A `name/` link is fetched at what the name resolves to. Its `cid` is only the sender's word: nothing ties + it to the name, so any sender could pose as a well-known name with it. If the name can't be resolved, + `inbox get` fails (404 through the API) before writing anything or starting a job, unless the user passes + `--use-sent-cid` (API: `use_sent_cid`): then it fetches the sent CID and that link's result carries a + warning saying so. A name that resolves is always used. Auto-fetch, whose senders are trusted, falls back + to the sent CID and records `"sent_cid": true` with the pin. `inbox` shows a name link's sent CID ("sent + as ``") and such a pin ("auto-fetch pinned (sent CID; name unresolved)"). ## Appendix A. Comparison with IPFS diff --git a/tests/bench_block_writes.py b/tests/bench_block_writes.py new file mode 100644 index 0000000..950183b --- /dev/null +++ b/tests/bench_block_writes.py @@ -0,0 +1,509 @@ +#!/usr/bin/env python3 +"""Benchmark of block write durability: the decision gate of 0.1.6 plan item B (decision Q1). + +Not a unittest module (discovery runs test_*.py only) and not run by the suite: a script the maintainer runs by +hand on the disks that matter. It stores blocks through the real builder, Storage and Session, swapping only the +BlockStore for one of four variants defined here (subclasses; no rnfs module is changed): + +- today rnfs as it is: BlockStore.put fsyncs each block's temporary file before renaming it, and + BlockStore.sync() (run before every pin and cache save) fsyncs the directory. +- fdatasync the same, with fdatasync in place of fsync (fsync where the OS has no fdatasync). +- deferred-fsync put writes and renames without syncing, remembering the path; the durability point (sync(), + and a Session writer's close, outside the storage lock) fdatasyncs every remembered file, at + most WRITE_THREADS at a time, then the directory. Includes the plan's blocks/.dirty marker + (created and synced before the first unsynced write, removed after sync()), but not the + startup check that would read it. +- deferred-syncfs the same, with one Linux syncfs() of the blocks' filesystem in place of the per-file loop. + Linux only: skipped elsewhere. + +Workloads (sizes from --size and --tree-files; content is random each run, so nothing dedups): + +- add a file added as `rnfs add` does: builder.add_path through Session.writer(), then the "user" pin + (its durability point: the pin is saved durably after BlockStore.sync()). +- fetch the same file's blocks (64 KiB chunks and their index nodes, root first) stored one at a time + through Session.put, as the fetch thread does, then record_fetch and Storage.save() (the cache + save's BlockStore.sync()). Only the storing is timed, not the network. +- tree-add a tree of 8 KiB files (64 per directory) added as `rnfs add` does. +- tree-fetch that tree's blocks stored as a fetch stores them: the plan's A2 case. + +Each run uses a fresh directory under --dir; the source data is written (and is in the page cache) before the +clock starts, and `sync` runs before each timed run so earlier writes don't land in it. The page cache can't be +dropped without root. Variants are run round-robin, --runs times each (default 3), so drift in machine load +spreads over all of them; the table shows the median and the minimum. + +Usage (the Python that runs rnfs, with its dependencies installed; from a copy of the repository): + + python3 tests/bench_block_writes.py --dir /path/on/the/disk/to/measure + python3 tests/bench_block_writes.py --dir /mnt/sd/bench --json > sd.json # machine-readable + +On a Raspberry Pi 4 SD card: copy the repository to the Pi (or check it out there), then run it with --dir on the +SD card's filesystem (the home directory of a stock Raspberry Pi OS is; check with `df -T DIR`: ext4, not tmpfs). +Keep the Pi otherwise idle. Disk space needed: about 2.2x --size (source file plus blocks), freed after each run; +memory: about 2x --size free (the fetch workloads hold their blocks in memory). +Duration: with the defaults 48 timed runs (4 workloads x 4 variants x 3). Today's code syncs each block of a +fetch one at a time, so the fetch workloads dominate: on a desktop NVMe the whole benchmark took 4 to 7 minutes +idle (11 under load); on an SD card expect one to two hours. For a quick look first: +`--size 64 --tree-files 1024 --runs 1` (a few minutes); `--workloads add,fetch` runs only what the gate needs. +--dir defaults to a new temporary directory under the current directory (not /tmp, which may be tmpfs); it is +removed afterwards. + +The gate (plan B, "Decision gate before implementing"): go ahead with the deferred design only if it is at +least about 1.5x faster than today on the SD card (add and fetch); if it isn't, fall back to fdatasync and keep +WRITE_THREADS at 16. Adopt syncfs over the per-file loop only if it is a clear win. The script prints these +ratios; NVMe numbers alone don't decide it. + +Caveats: syncfs flushes every dirty page on the filesystem, other programs' included, so it is only meaningful +on an otherwise idle machine. On a stacked filesystem (ecryptfs, overlayfs) syncfs may not reach the +filesystem below, so deferred-syncfs can look fast without being durable (on ecryptfs over ext4, Linux 6.8, it +was measured to write nothing back at all); the script warns about these, so measure syncfs on a plain one. +""" +import argparse +import ctypes +import ctypes.util +import json +import os +import platform +import shutil +import statistics +import sys +import tempfile +import threading +import time +from concurrent.futures import ThreadPoolExecutor + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # run from a plain checkout + +from rnfs import fsutil +from rnfs.constants import KIB, MAX_BLOCK_SIZE, MIB +from rnfs.content import builder +from rnfs.content.blockstore import PARTIAL_PREFIX, BlockStore +from rnfs.content.cid import Cid +from rnfs.content.storage import USER, WRITE_THREADS, Storage + +GATE = 1.5 # the plan's threshold: the deferred design must be about this much faster on the SD card +TREE_FILE_SIZE = 8 * KIB +TREE_FANOUT = 64 # files per directory in the tree workloads +DIRTY_MARKER = ".dirty" # the plan's marker: blocks may be unsynced (BlockStore.scan ignores the name) +STACKED_FILESYSTEMS = {"ecryptfs", "overlay", "fuse", "fuse.encfs", "fuse.gocryptfs", "aufs"} +data_sync = getattr(os, "fdatasync", os.fsync) # fdatasync skips metadata a later read doesn't need (mtime) +sync_all = getattr(os, "sync", lambda: None) # `sync`: write back every dirty page (Unix only) + + +# Variants + +class _ReplicaStore(BlockStore): + """BlockStore.put as it is today, with the per-file sync moved into hooks the variants override.""" + + def put(self, cid: Cid, block: bytes): + """Store block under cid unless already stored, as BlockStore.put does, calling _written and _renamed.""" + if len(block) > MAX_BLOCK_SIZE: + raise ValueError(f"block for {cid} exceeds MAX_BLOCK_SIZE") + if not cid.verify(block): + raise ValueError(f"block does not match {cid}") + path = self.path(cid) + if os.path.isfile(path): + return + fd, partial_path = tempfile.mkstemp(prefix=PARTIAL_PREFIX, dir=self.directory) + try: + with os.fdopen(fd, "wb") as out: + out.write(block) + out.flush() + self._written(out.fileno()) + try: + fsutil.replace(partial_path, path) + except PermissionError: + if not os.path.isfile(path): + raise + fsutil.remove(partial_path) + return + self._unsynced = True + self._renamed(path) + except BaseException: + try: + fsutil.remove(partial_path) + except OSError: + pass + raise + + def _written(self, fd: int): + """Called with the temporary file's descriptor once the block is written, before the rename.""" + + def _renamed(self, path: str): + """Called once the block is renamed to path.""" + + +class FdatasyncStore(_ReplicaStore): + """Today's design with fdatasync for fsync.""" + + def _written(self, fd: int): + """Sync the block's data before the rename.""" + data_sync(fd) + + +class DeferredStore(_ReplicaStore): + """No sync on put: the paths renamed since the last sync() are synced there, WRITE_THREADS at a time.""" + + def __init__(self, directory: str): + """Open the store at directory, with nothing unsynced.""" + super().__init__(directory) + self._paths: set[str] = set() + self._paths_lock = threading.Lock() + self._marked = False # DIRTY_MARKER exists (and is durable) + + def _renamed(self, path: str): + """Remember path, marking the store dirty first if it isn't already.""" + with self._paths_lock: + if not self._marked: + self._mark_dirty() + self._paths.add(path) + + def _mark_dirty(self): + """Create DIRTY_MARKER durably, so a startup after a power loss knows to check recent blocks.""" + marker = os.path.join(self.directory, DIRTY_MARKER) + with open(marker, "wb") as out: + out.flush() + os.fsync(out.fileno()) + fsutil.sync_directory(self.directory) + self._marked = True + + def sync(self): + """Sync every block renamed since the last sync, then the directory, then remove the marker.""" + with self._paths_lock: + paths, self._paths = self._paths, set() + self._unsynced = False + if not paths: + return + self._flush(paths) + fsutil.sync_directory(self.directory) + with self._paths_lock: + if not self._paths and self._marked: # nothing renamed meanwhile, so nothing is left unsynced + os.remove(os.path.join(self.directory, DIRTY_MARKER)) + self._marked = False + + def _flush(self, paths: set[str]): + """Make the data of the files at paths durable.""" + with ThreadPoolExecutor(WRITE_THREADS, thread_name_prefix="bench-sync") as pool: + for _ in pool.map(_sync_file, paths): + pass + + +def _sync_file(path: str): + """fdatasync (or fsync) the file at path; one deleted meanwhile needs nothing.""" + try: + fd = os.open(path, os.O_RDONLY) + except FileNotFoundError: + return + try: + data_sync(fd) + finally: + os.close(fd) + + +_libc = None + + +def _syncfs(directory: str): + """syncfs() the filesystem holding directory, through ctypes (Linux only). Raises OSError.""" + global _libc + if _libc is None: + _libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True) + fd = os.open(directory, os.O_RDONLY) + try: + if _libc.syncfs(fd) != 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error), directory) + finally: + os.close(fd) + + +class SyncfsStore(DeferredStore): + """The deferred design with one syncfs() in place of the per-file loop.""" + + def _flush(self, paths: set[str]): + """Sync the whole filesystem holding the blocks.""" + _syncfs(self.directory) + + +class Variant: + """One way of making blocks durable: its BlockStore class, and whether a writer's close syncs.""" + + def __init__(self, name: str, store: type[BlockStore], sync_at_writer_close: bool): + """A variant called name, storing through store.""" + self.name = name + self.store = store + self.sync_at_writer_close = sync_at_writer_close + + def open(self, run_dir: str) -> Storage: + """A fresh Storage in run_dir whose blocks are stored through this variant's BlockStore.""" + blocks_dir = os.path.join(run_dir, "blocks") + storage = Storage(blocks_dir, os.path.join(run_dir, "state"), max_storage=1 << 50) + storage.blocks = self.store(blocks_dir) # the directory is empty, so Storage's index is unchanged + return storage + + +VARIANTS = [ + Variant("today", BlockStore, False), + Variant("fdatasync", FdatasyncStore, False), + Variant("deferred-fsync", DeferredStore, True), + Variant("deferred-syncfs", SyncfsStore, True), +] + + +# Workloads + +def _write_random_file(path: str, nbytes: int): + """Write nbytes of random data to path, durably, so a timed run doesn't flush it.""" + with open(path, "wb") as out: + left = nbytes + while left: + piece = os.urandom(min(MIB, left)) + out.write(piece) + left -= len(piece) + out.flush() + os.fsync(out.fileno()) + + +def _prepare_file(run_dir: str, args) -> str: + """A random file of --size MiB in run_dir; its path.""" + path = os.path.join(run_dir, "source.bin") + _write_random_file(path, args.size * MIB) + return path + + +def _prepare_tree(run_dir: str, args) -> str: + """A tree of --tree-files random 8 KiB files, TREE_FANOUT per directory, in run_dir; its path.""" + root = os.path.join(run_dir, "source") + for n in range(args.tree_files): + directory = os.path.join(root, f"d{n // TREE_FANOUT:04d}") + if n % TREE_FANOUT == 0: + os.makedirs(directory) + with open(os.path.join(directory, f"f{n % TREE_FANOUT:03d}"), "wb") as out: + out.write(os.urandom(TREE_FILE_SIZE)) + sync_all() + return root + + +def _built_blocks(path: str) -> tuple[Cid, list[tuple[Cid, bytes]]]: + """The root and blocks of path as a fetch receives them: built in memory, root first.""" + blocks: list[tuple[Cid, bytes]] = [] + built, _ = builder.add_path(path, lambda cid, block: blocks.append((cid, block))) + blocks.reverse() # the builder makes leaves first; a fetch gets nodes before what they list + return built.cid, blocks + + +def _add(storage: Storage, variant: Variant, path: str) -> int: + """Add path as RnfsNode.add_path does, through to the "user" pin; the blocks stored.""" + with storage.protect() as session: + with session.writer() as put: + built, _ = builder.add_path(path, put) + if variant.sync_at_writer_close: + storage.blocks.sync() # the plan's writer close: outside the lock, so _save_pins finds nothing left + storage.pin(built.cid, USER) + return storage.block_count() + + +def _fetch(storage: Storage, variant: Variant, prepared: tuple[Cid, list[tuple[Cid, bytes]]]) -> int: + """Store the blocks one at a time as the fetch thread does, record the fetch and save the cache record.""" + root, blocks = prepared + with storage.protect() as session: + for cid, block in blocks: + session.put(cid, block) + storage.record_fetch(root, [], root) + storage.save() + return storage.block_count() + + +class Workload: + """A timed operation, with untimed preparation of its source data.""" + + def __init__(self, name: str, prepare, run, nbytes): + """name; prepare(run_dir, args) -> data; run(storage, variant, data) -> blocks; nbytes(args) -> bytes.""" + self.name = name + self.prepare = prepare + self.run = run + self.nbytes = nbytes + + +WORKLOADS = [ + Workload("add", _prepare_file, _add, lambda args: args.size * MIB), + Workload("fetch", lambda run_dir, args: _built_blocks(_prepare_file(run_dir, args)), _fetch, + lambda args: args.size * MIB), + Workload("tree-add", _prepare_tree, _add, lambda args: args.tree_files * TREE_FILE_SIZE), + Workload("tree-fetch", lambda run_dir, args: _built_blocks(_prepare_tree(run_dir, args)), _fetch, + lambda args: args.tree_files * TREE_FILE_SIZE), +] + + +# Running + +def filesystem(path: str) -> dict: + """The mount point, filesystem type and device holding path (from /proc/mounts; empty elsewhere).""" + path = os.path.realpath(path) + best = {} + try: + with open("/proc/mounts") as mounts: + for line in mounts: + device, mount_point, kind = line.split()[:3] + mount_point = mount_point.replace("\\040", " ") + inside = path == mount_point or path.startswith(mount_point.rstrip("/") + "/") + if inside and len(mount_point) >= len(best.get("mount_point", "")): + best = {"mount_point": mount_point, "type": kind, "device": device} + except OSError: + pass + return best + + +def warnings_for(fs: dict) -> list[str]: + """What makes results on the filesystem fs misleading.""" + kind = fs.get("type", "") + if kind in ("tmpfs", "ramfs"): + return [f"{kind} is memory: syncs cost nothing there, so these numbers mean nothing for a disk"] + if kind in STACKED_FILESYSTEMS or kind.startswith("fuse"): + return [f"{kind} is stacked on another filesystem: syncfs() may not reach the one below, so " + f"deferred-syncfs can look fast without being durable"] + return [] + + +def run_once(workload: Workload, variant: Variant, base: str, args) -> tuple[float, int]: + """One timed run of workload with variant in a fresh directory under base, removed afterwards: the seconds + taken and the blocks stored.""" + run_dir = tempfile.mkdtemp(prefix=f"{workload.name}-{variant.name}-", dir=base) + try: + prepared = workload.prepare(run_dir, args) + storage = variant.open(run_dir) + try: + sync_all() + start = time.perf_counter() + blocks = workload.run(storage, variant, prepared) + elapsed = time.perf_counter() - start + leftover = getattr(storage.blocks, "_paths", None) + if leftover: + raise AssertionError(f"{variant.name} left {len(leftover)} blocks unsynced") + if os.path.exists(os.path.join(storage.blocks.directory, DIRTY_MARKER)): + raise AssertionError(f"{variant.name} left {DIRTY_MARKER} behind") + finally: + storage.close() + return elapsed, blocks + finally: + shutil.rmtree(run_dir, ignore_errors=True) + sync_all() + + +def benchmark(args, base: str) -> dict: + """Run every selected workload and variant args.runs times; the results as a JSON-ready dict.""" + variants = [v for v in VARIANTS if v.name in args.variants] + if not sys.platform.startswith("linux"): + variants = [v for v in variants if v.store is not SyncfsStore] # syncfs() is Linux only + workloads = [w for w in WORKLOADS if w.name in args.workloads] + times = {w.name: {v.name: [] for v in variants} for w in workloads} + blocks = {} + started = time.time() + for rep in range(args.runs): + for workload in workloads: + order = variants[rep % len(variants):] + variants[:rep % len(variants)] # rotate: spread the drift + for variant in order: + elapsed, count = run_once(workload, variant, base, args) + times[workload.name][variant.name].append(elapsed) + blocks[workload.name] = count + print(f"run {rep + 1}/{args.runs} {workload.name:10} {variant.name:15} {elapsed:8.3f} s", + file=sys.stderr, flush=True) + results = {} + for workload in workloads: + nbytes = workload.nbytes(args) + rows = {} + for variant in variants: + samples = times[workload.name][variant.name] + rows[variant.name] = {"median_s": statistics.median(samples), "min_s": min(samples), + "runs_s": samples, "mib_per_s": nbytes / MIB / statistics.median(samples)} + if "today" in rows: + for row in rows.values(): + row["speedup_vs_today"] = rows["today"]["median_s"] / row["median_s"] + results[workload.name] = {"bytes": nbytes, "blocks": blocks[workload.name], "variants": rows} + return {"elapsed_s": time.time() - started, "workloads": results} + + +def print_table(report: dict): + """The results as a table, then the gate's ratios.""" + env = report["environment"] + fs = env["filesystem"] + print(f"rnfs block write benchmark: {env['dir']} ({fs.get('type', '?')} on {fs.get('device', '?')}), " + f"{env['runs']} runs each, WRITE_THREADS={env['write_threads']}, Python {env['python']}, {env['system']}") + print(f"load average before {env['load_before']}, after {env['load_after']}; took {report['elapsed_s']:.0f} s") + for warning in env["warnings"]: + print(f"WARNING: {warning}") + print() + print(f"{'workload':12} {'variant':16} {'median s':>9} {'min s':>9} {'MiB/s':>8} {'vs today':>9}") + for name, workload in report["workloads"].items(): + label = f"{name} ({workload['bytes'] / MIB:g} MiB, {workload['blocks']} blocks)" + print(label) + for variant, row in workload["variants"].items(): + speedup = f"{row['speedup_vs_today']:.2f}x" if "speedup_vs_today" in row else "-" + print(f"{'':12} {variant:16} {row['median_s']:9.3f} {row['min_s']:9.3f} {row['mib_per_s']:8.1f} " + f"{speedup:>9}") + print() + print(f"Gate (plan B): the deferred design goes ahead only if about {GATE}x faster than today ON THE SD CARD;") + print("otherwise fall back to fdatasync and keep 16 threads. Medians:") + for name, workload in report["workloads"].items(): + rows = workload["variants"] + if "today" not in rows: + continue + ratios = ", ".join(f"{variant} {row['speedup_vs_today']:.2f}x" for variant, row in rows.items() + if variant != "today") + print(f" {name:10} {ratios}") + + +def main(argv=None) -> int: + """Parse the arguments, run the benchmark and print the table (or JSON).""" + parser = argparse.ArgumentParser(description="Benchmark rnfs block write durability (0.1.6 plan item B).", + epilog="See the module docstring for how to run it on a Pi SD card.") + parser.add_argument("--dir", help="directory on the disk to measure (default: a new one under the current " + "directory, removed afterwards)") + parser.add_argument("--size", type=int, default=256, help="MiB added and fetched (default 256)") + parser.add_argument("--tree-files", type=int, default=4096, help="8 KiB files in the tree (default 4096)") + parser.add_argument("--runs", type=int, default=3, help="runs of each variant (default 3)") + parser.add_argument("--variants", default=",".join(v.name for v in VARIANTS), type=lambda s: s.split(","), + help="comma-separated variants (default all)") + parser.add_argument("--workloads", default=",".join(w.name for w in WORKLOADS), type=lambda s: s.split(","), + help="comma-separated workloads (default all)") + parser.add_argument("--json", action="store_true", help="print the results as JSON instead of the table") + args = parser.parse_args(argv) + unknown = (set(args.variants) - {v.name for v in VARIANTS}) | (set(args.workloads) - {w.name for w in WORKLOADS}) + if unknown: + parser.error(f"unknown variant or workload: {', '.join(sorted(unknown))}") + + if args.dir: + os.makedirs(args.dir, exist_ok=True) + parent = args.dir or os.getcwd() + base = tempfile.mkdtemp(prefix="rnfs-bench-", dir=parent) + fs = filesystem(base) + environment = {"dir": os.path.realpath(parent), "filesystem": fs, "warnings": warnings_for(fs), + "runs": args.runs, "size_mib": args.size, "tree_files": args.tree_files, + "write_threads": WRITE_THREADS, "python": platform.python_version(), + "system": platform.platform(), "machine": platform.machine(), + "fdatasync": hasattr(os, "fdatasync"), "load_before": _load()} + for warning in environment["warnings"]: + print(f"WARNING: {warning}", file=sys.stderr) + try: + report = benchmark(args, base) + finally: + shutil.rmtree(base, ignore_errors=True) + environment["load_after"] = _load() + report = {"environment": environment, **report} + if args.json: + json.dump(report, sys.stdout, indent=2) + print() + else: + print_table(report) + return 0 + + +def _load() -> list[float] | None: + """The 1, 5 and 15 minute load averages, where the OS has them.""" + try: + return [round(n, 2) for n in os.getloadavg()] + except (AttributeError, OSError): + return None + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fakes/__init__.py b/tests/fakes/__init__.py new file mode 100644 index 0000000..0822b0b --- /dev/null +++ b/tests/fakes/__init__.py @@ -0,0 +1,3 @@ +"""Test doubles shared by several test modules (not test modules themselves): FakeClient for PeerClient +(tests/fakes/client.py) and the quiet-log fixture (tests/fakes/log.py). tests/test_fake_contract.py checks the fakes +against the real classes they stand in for, so a change there can't leave tests passing against a stale fake.""" diff --git a/tests/fakes/client.py b/tests/fakes/client.py new file mode 100644 index 0000000..f714158 --- /dev/null +++ b/tests/fakes/client.py @@ -0,0 +1,109 @@ +"""FakeClient, the PeerClient stand-in the fetcher, publisher, mesh and names tests use. Its methods keep +PeerClient's names and parameters (tests/test_fake_contract.py), so production code calls them as it calls the +real client, without looking up with getattr what a fake might lack.""" +import threading + +from rnfs.content.cid import Cid +from rnfs.network.client import OK, BlocksResult + +BATCH = 4 # CIDs the fake client sends per /blocks request, like a packet limit; the rest come back unsent + + +class FakeClient: + """Stands in for PeerClient. + + /blocks: each node holds a dict of blocks, or behaves as a test function says. /provide: each request is + recorded; nodes in refuse record nothing, a node in limit records only its first keys. + """ + + def __init__(self): + """No nodes, no open links, no requests yet.""" + # /blocks and links + self.holdings: dict[bytes, dict[Cid, bytes]] = {} + self.behaviour: dict[bytes, object] = {} # node -> f(cids) -> BlocksResult, or None for the default + self.links: list[bytes] = [] + self.rate: float | None = None + self.mdu: int | None = None # every link's MDU; None, as for a node with no open link: not known + self.requests: list[tuple[bytes, list[Cid]]] = [] + self.closed: list[bytes] = [] + self.kept_open: list[bytes] = [] # close_unless_used calls that left the link open + self.users: dict[bytes, dict[int, int]] = {} # node -> threads with a /blocks call to it in progress + self.lock = threading.Lock() + # /provide + self.calls: list[tuple[bytes, list[bytes], list[bytes]]] = [] + self.refuse: set[bytes] = set() + self.limit: dict[bytes, int] = {} + + def open_links(self) -> list[bytes]: + """Nodes with an open link.""" + return list(self.links) + + def rate_estimate(self, node_id: bytes): + """The same estimate for every link.""" + return self.rate + + def link_mdu(self, node_id: bytes): + """The same MDU for every link (None: not known).""" + return self.mdu + + def close(self, node_id: bytes): + """Close the link to node_id (noted).""" + with self.lock: + self.closed.append(node_id) + + def close_unless_used(self, node_id: bytes, own: set[int]) -> bool: + """close(node_id) unless a thread not in own has a /blocks call to node_id in progress (noted in + kept_open).""" + with self.lock: + others = any(ident not in own for ident in self.users.get(node_id, {})) + if others: + self.kept_open.append(node_id) + if others: + return False + self.close(node_id) + return True + + def blocks(self, node_id: bytes, cids: list[Cid], on_progress=None) -> BlocksResult: + """Serve what node_id holds, BATCH CIDs per request (the calling thread counted among node_id's users).""" + ident = threading.get_ident() + with self.lock: + users = self.users.setdefault(node_id, {}) + users[ident] = users.get(ident, 0) + 1 + try: + return self._blocks(node_id, cids, on_progress) + finally: + with self.lock: + users[ident] -= 1 + if not users[ident]: + del users[ident] + + def _blocks(self, node_id: bytes, cids: list[Cid], on_progress) -> BlocksResult: + """blocks(), served.""" + with self.lock: + self.requests.append((node_id, list(cids))) + self.on_progress = on_progress + behaviour = self.behaviour.get(node_id) + if behaviour is not None: + result = behaviour(cids) + if result is not None: + return result + held = self.holdings.get(node_id, {}) + result = BlocksResult(OK) + for cid in cids[:BATCH]: + if cid in held: + result.blocks[cid] = held[cid] + else: + result.missing.append(cid) + result.unsent = list(cids[BATCH:]) + return result + + def provide(self, node_id, keys, withdrawn): + """Record one batched /provide; the keys the node may have recorded, as PeerClient.provide returns.""" + self.calls.append((node_id, list(keys), list(withdrawn))) + if node_id in self.refuse: + return set() + return set(list(keys)[:self.limit.get(node_id, len(keys))]) + + def asked(self, node_id: bytes) -> list[Cid]: + """Every CID sent to node_id in /blocks requests.""" + return [cid for node, cids in self.requests if node == node_id for cid in cids[:BATCH]] diff --git a/tests/fakes/log.py b/tests/fakes/log.py new file mode 100644 index 0000000..4ba23ed --- /dev/null +++ b/tests/fakes/log.py @@ -0,0 +1,19 @@ +"""The quiet-log fixture: a test module whose code under test logs through RNS silences it while its tests run.""" +import RNS + + +def quiet_log() -> tuple: + """setUpModule and tearDownModule functions for a test module (`setUpModule, tearDownModule = quiet_log()`): + RNS logs only critical messages while the module's tests run, and its loglevel is restored after them.""" + saved = [] + + def setUpModule(): + """Silence RNS's log below critical.""" + saved.append(RNS.loglevel) + RNS.loglevel = RNS.LOG_CRITICAL + + def tearDownModule(): + """Restore RNS's loglevel.""" + RNS.loglevel = saved.pop() + + return setUpModule, tearDownModule diff --git a/tests/fixtures/rnfs-0.1.5/README.md b/tests/fixtures/rnfs-0.1.5/README.md new file mode 100644 index 0000000..5946dfc --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/README.md @@ -0,0 +1,54 @@ +# rnfs 0.1.5 directory fixture + +An rnfs directory exactly as rnfs **0.1.5** (commit `5bd3075`, tag `v0.1.5`) leaves it: its state database, +`state/rnfs.db`, with rows in every one of its 27 tables, and the 17 JSON files 0.1.5 imported from 0.1.4, kept as +`.migrated`. `tests/test_upgrade_015.py` copies it, starts the current code on the copy and checks that +every row is still there. + +It was written by `generate.py` (in this directory), running 0.1.5's own code: + +1. 0.1.5 starts, offline, on a copy of the [rnfs-0.1.4 fixture](../rnfs-0.1.4/README.md), with that fixture's + blocks, clock and settings, as 0.1.5's own `tests/test_upgrade_014.py` does. That imports the JSON files. +2. Through 0.1.5's own objects it then adds what only 0.1.5 has: two local deny-list entries (a CID and a file + hash), a deny list followed through the `deny_lists` setting (accepted, its blocks pinned `deny`), and a + file's content hash remembered (`deny_hashes`), from reading a file of several blocks while file hashes are + denied. +3. The node stops, which saves every store and checkpoints the WAL, and the database is `VACUUM`ed. +4. 0.1.5 starts again on a copy, at the same time, and must change no row. + +`expected.json` holds every table's rows as plain `sqlite3` reads them (a BLOB is `{"blob": ""}`), the +schema version (`user_version`, 16), the time the test runs at (`now`) and the settings it starts with. + +## What is in it + +- `state/rnfs.db`: everything the rnfs-0.1.4 fixture holds (see its README), plus the 0.1.5 rows above. The + deny-list entries match nothing else in the fixture, so the upgrade test sees every pin and record as 0.1.5 + kept it. +- `state/*.json.migrated`: the 0.1.4 fixture's state files, unchanged. +- `names/photos`, `names/imported`: name keys. **Throwaway test keys, generated for the 0.1.4 fixture: never use + them for anything else.** +- `files/notes.txt`: the shared folder's one entry. +- `fixture_content.py`: the files 0.1.5 added on top of the 0.1.4 fixture's (data only). There is no `blocks/`: + the test adds both fixtures' files with the current code to make the blocks, so every pin is complete. + +Left out, as in the 0.1.4 fixture: the node identity (`storage/`), `lxmf/` router state, `api_token` and logs. +No `rnfs.toml`: the settings are in `expected.json`. + +## Times + +The fake wall clock is the 0.1.4 fixture's T0 plus a day, `now` in `expected.json` (1792141200), so no provider +record or publish round has expired. Rows 0.1.5 timed with the real clock (the deny-list entries' `added`, the +deny list's `accepted`) hold the time the generator ran. + +## Regenerating + +From the repository root: + + mkdir /tmp/rnfs-0.1.5 && git archive v0.1.5 | tar -x -C /tmp/rnfs-0.1.5 + python3 tests/fixtures/rnfs-0.1.5/generate.py /tmp/rnfs-0.1.5 /tmp/rnfs-fixture-out + +then replace `state/`, `names/`, `files/` and `expected.json` here with those in +`/tmp/rnfs-fixture-out/rnfs-0.1.5`. Keys, signatures, job IDs and message hashes are random, so every run writes +different bytes; the test reads its expectations from `expected.json`, so it needs no change. + +The fixture is test data only: the sdist prunes `tests/` (MANIFEST.in) and the wheel packages only `rnfs*`. diff --git a/tests/fixtures/rnfs-0.1.5/expected.json b/tests/fixtures/rnfs-0.1.5/expected.json new file mode 100644 index 0000000..233aab7 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/expected.json @@ -0,0 +1,1382 @@ +{ + "now": 1792141200.0, + "settings": { + "deny_lists": [ + "bafkreifuszzqadwxrcdl3baszyh4qizgboeanf4vfm6ly6p4vnsgonvoma" + ], + "lookup_timeout": 2, + "lxmf_auto_fetch_from": [ + "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf" + ], + "max_records": 40, + "path_timeout": 2, + "remote_pin_identities": [ + "0123456789abcdef0123456789abcdef" + ], + "slow": "no" + }, + "tables": { + "cache": [ + [ + { + "blob": "015512202a5740f76cb8e47558a10cd85a03f563be8da273306a54594ec4535f210d949b" + }, + "[]", + { + "blob": "015512202a5740f76cb8e47558a10cd85a03f563be8da273306a54594ec4535f210d949b" + }, + 5 + ], + [ + { + "blob": "01551220bb062752be144924cc42d662cf9c6d5ab7cc5c5be4ca26eb8596c9a8a98f7c1b" + }, + "[]", + { + "blob": "01551220bb062752be144924cc42d662cf9c6d5ab7cc5c5be4ca26eb8596c9a8a98f7c1b" + }, + 8 + ], + [ + { + "blob": "01551220d8bd1ae8a49d73ab61d6b59c9220f6696551c58a58006f5fcfad30a800ab773f" + }, + "[]", + { + "blob": "01551220d8bd1ae8a49d73ab61d6b59c9220f6696551c58a58006f5fcfad30a800ab773f" + }, + 6 + ], + [ + { + "blob": "01551220e48a9a8c088f18f129cbe714d52cbe2f262020efa62392f9698203f0cf5f9eca" + }, + "[]", + { + "blob": "01551220e48a9a8c088f18f129cbe714d52cbe2f262020efa62392f9698203f0cf5f9eca" + }, + 7 + ], + [ + { + "blob": "01551220f5130ee371cf7fc477243e10005c9ff101a67322e18e45099b9a22377282ba6c" + }, + "[]", + { + "blob": "01551220f5130ee371cf7fc477243e10005c9ff101a67322e18e45099b9a22377282ba6c" + }, + 0 + ], + [ + { + "blob": "018080c001122050e10779218db4a02e57c80824ad42faccc002b56b806a6910ee8ca25b36cf1d" + }, + "[]", + { + "blob": "018080c001122050e10779218db4a02e57c80824ad42faccc002b56b806a6910ee8ca25b36cf1d" + }, + 4 + ], + [ + { + "blob": "018080c00112209794acef0b890c1785d26330a941725abb059635df25d5831a9c786f085e6fae" + }, + "[\"café.txt\"]", + { + "blob": "015512204b5c2fdf195a98c4819c1b0991f08ac2771feaeb884f555c32b0e8ef61975f01" + }, + 9 + ], + [ + { + "blob": "018080c00112209794acef0b890c1785d26330a941725abb059635df25d5831a9c786f085e6fae" + }, + "[\"Ελληνικά\", \"αρχείο.txt\"]", + { + "blob": "0155122024a8cef76b030c400a45a986ed3ca0025864c83e14c1fcd992402fd6c7549cac" + }, + 3 + ], + [ + { + "blob": "018080c00112209794acef0b890c1785d26330a941725abb059635df25d5831a9c786f085e6fae" + }, + "[\"日本語\", \"メモ.md\"]", + { + "blob": "01551220350b6a3eac2aff841d8a1bedbffb6bba0f14e881893b79af5b1bba71aab5fec4" + }, + 2 + ], + [ + { + "blob": "018080c00112209794acef0b890c1785d26330a941725abb059635df25d5831a9c786f085e6fae" + }, + "[]", + { + "blob": "018080c00112209794acef0b890c1785d26330a941725abb059635df25d5831a9c786f085e6fae" + }, + 1 + ] + ], + "contacts": [ + [ + "alice", + "name", + { + "blob": "bcc7900726de195e4cc4b09e67f02966" + } + ], + [ + "bob", + "lxmf", + { + "blob": "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf" + } + ], + [ + "relay.1", + "node", + { + "blob": "10101010101010101010101010101010" + } + ] + ], + "deny_entries": [ + [ + "bafkreig5r66w4f7qbtvtsqwwlfvfql6cmsnhp42vymrhjfiwhtestirgie", + 1792235740.439997 + ], + [ + "sha256:b4c04cf45324f820d1061ae8f3955b3c5f9d4079553d52e6624dcc9386879bcb", + 1792235740.439997 + ] + ], + "deny_hashes": [ + [ + { + "blob": "018080c00112204049843d091df303313be2a6b054a172c90ccf64856626f28508897f3763c63a" + }, + { + "blob": "1a82694689881e1a2d798d52132fe43d091a5236d063c603b151f1a2ea7d9c65" + }, + 0 + ] + ], + "deny_lists": [ + [ + "\"bafkreifuszzqadwxrcdl3baszyh4qizgboeanf4vfm6ly6p4vnsgonvoma\"", + { + "blob": "01551220b49673000ed78886bd8412ce0fc823260b880697952b3cbc79fcab646736ae60" + }, + 1792235740.4456954 + ] + ], + "fetched": [ + [ + { + "blob": "0155122024a8cef76b030c400a45a986ed3ca0025864c83e14c1fcd992402fd6c7549cac" + } + ], + [ + { + "blob": "015512202a5740f76cb8e47558a10cd85a03f563be8da273306a54594ec4535f210d949b" + } + ], + [ + { + "blob": "01551220bb062752be144924cc42d662cf9c6d5ab7cc5c5be4ca26eb8596c9a8a98f7c1b" + } + ], + [ + { + "blob": "01551220d8bd1ae8a49d73ab61d6b59c9220f6696551c58a58006f5fcfad30a800ab773f" + } + ], + [ + { + "blob": "01551220e48a9a8c088f18f129cbe714d52cbe2f262020efa62392f9698203f0cf5f9eca" + } + ], + [ + { + "blob": "01551220f5130ee371cf7fc477243e10005c9ff101a67322e18e45099b9a22377282ba6c" + } + ], + [ + { + "blob": "018080c001122050e10779218db4a02e57c80824ad42faccc002b56b806a6910ee8ca25b36cf1d" + } + ], + [ + { + "blob": "018080c00112209794acef0b890c1785d26330a941725abb059635df25d5831a9c786f085e6fae" + } + ] + ], + "follow": [ + [ + { + "blob": "bcc7900726de195e4cc4b09e67f02966" + }, + { + "blob": "01551220d8bd1ae8a49d73ab61d6b59c9220f6696551c58a58006f5fcfad30a800ab773f" + } + ] + ], + "invalid": [ + [ + { + "blob": "0155122054c919d0a40be32c9e2999056b9741479e27cfb33f9ee720c25680dc724d5cb7" + }, + 0 + ], + [ + { + "blob": "01551220907dfadbd64c4e67dad7ea57c7a153edd2b1bc6dd47bd602948d0c2fc8386065" + }, + 1 + ], + [ + { + "blob": "01551220c22a55cf19e692c717ed51a8b0da88c0d43a0f667e07d7f983393f822198224d" + }, + 2 + ] + ], + "jobs": [ + [ + "3bd70342", + 0, + "{\"id\": \"3bd70342\", \"kind\": \"get\", \"target\": \"bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/café.txt\", \"output\": \"/home/alice/Downloads/café 😀.txt\", \"from\": null, \"sealed_name\": false, \"fallback_dht\": false, \"state\": \"done\", \"created\": 1792056540.0, \"started\": 1792056540.0, \"finished\": 1792056545.0, \"error\": null, \"result\": {\"cid\": \"bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae\", \"size\": 7, \"warnings\": [\"skipped symlink café/link → elsewhere\", \"kept 😀 as is\"], \"output\": \"/home/alice/Downloads/café 😀.txt\"}, \"runs\": 1, \"total_bytes\": 7, \"done_bytes\": 7, \"blocks\": 1, \"blocks_total\": null, \"downloaded_bytes\": 7}" + ], + [ + "6a322a52", + 1, + "{\"id\": \"6a322a52\", \"kind\": \"pin\", \"target\": \"bafkreiemqpndzewx4bioklzfvhtiegsfny5fxuc3nkjk2vmcpjxiammsti\", \"output\": null, \"from\": null, \"sealed_name\": false, \"fallback_dht\": false, \"state\": \"failed\", \"created\": 1792056605.0, \"started\": 1792056605.0, \"finished\": 1792056605.0, \"error\": \"no providers found for this CID\", \"result\": null, \"runs\": 1, \"total_bytes\": null, \"done_bytes\": 0, \"blocks\": 0, \"blocks_total\": null, \"downloaded_bytes\": 0}" + ], + [ + "da44b5c5", + 2, + "{\"id\": \"da44b5c5\", \"kind\": \"pin\", \"target\": \"bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/Ελληνικά\", \"output\": null, \"from\": null, \"sealed_name\": false, \"fallback_dht\": false, \"state\": \"queued\", \"created\": 1792056665.0, \"started\": null, \"finished\": null, \"error\": null, \"result\": null, \"runs\": 0, \"total_bytes\": null, \"done_bytes\": 0, \"blocks\": 0, \"blocks_total\": null, \"downloaded_bytes\": 0}" + ] + ], + "keepers": [ + [ + { + "blob": "05c7214ca11e7646c79340c6b6d9866c" + }, + "[\"own\"]", + "[\"e5ee52baf6bf767a65fc00d5acf5231c3ab7509438494eec22e4cdb9eae33d3cb4d1e823db4955856e0f57c46987475752a2aa95721399815e3f955d98e23cb2\", \"783c4f3005a78c29de83\", \"2e62f40c0a006ad097b4\", \"\", \"eb5f4d8539b58b495781c8ba83765ed04318d1718032b8d2af0ad694b340480aa6a9863e4fa057aedb7681f7e41145be6a61a8091e4dd02d10c71920ad932701\", \"9201c427018080c001122050e10779218db4a02e57c80824ad42faccc002b56b806a6910ee8ca25b36cf1d\"]" + ], + [ + { + "blob": "07defa27cdd333a37d1c1f425770a541" + }, + "[\"user\"]", + null + ], + [ + { + "blob": "bcc7900726de195e4cc4b09e67f02966" + }, + "[\"follow\"]", + "[\"8e6e4ec6192178ce7ea841c1603ee6756fe9bed343f7780234889a6fb4da5414274d5caf3ca30bb9017e5ac96eb3a75622171754072025b11210c98a0e8e546a\", \"783c4f3005a78c29de83\", \"ff7788273d006ad09958\", \"\", \"9c523b5797fa9e96ebcf18a1fef4e33aa25c682c158b34592f3942a63ab98dff38b8e8f890649ad259d3c1ce6ea25490077b1c26ed7ec0a891b4f0ae0d31db00\", \"920cc42401551220d8bd1ae8a49d73ab61d6b59c9220f6696551c58a58006f5fcfad30a800ab773f\"]" + ], + [ + { + "blob": "f35e61151054781eb9f985f470691a77" + }, + "[\"user\"]", + "[\"8db0daa5ca4c0979d432ce2b7ffac8f5057a4a77245494128a636f961309e54f4b2b7e004d1888419fa0de6e61ee896a40996edf79ecd1da36a56ac982962f3b\", \"783c4f3005a78c29de83\", \"d2a612e452006ad098e0\", \"\", \"f1cf556174e0341ee785bf34f236f06aae1cbda68547a6c5716ace79d29711641ac8b7afde40eb3ef226fad4ce416e4e6a343546be4c1de920fda5004b3d250d\", \"9207c424015512202a5740f76cb8e47558a10cd85a03f563be8da273306a54594ec4535f210d949b\"]" + ] + ], + "lxmf_messages": [ + [ + "inbox", + 1, + "{\"hash\": \"7b4121eec4d068e3eeee0a668158e14ad322a1ef925c0be28aa6a26a9dde4ebf\", \"from\": \"a0a1a2a3a4a5a6a7a8a9aaabacadaeaf\", \"display_name\": \"Zoë\", \"sent\": 1792056030.0, \"received\": 1792056060.0, \"title\": \"Grüße 🌻\", \"text\": \"Fotos vom Wochenende 📷\\n\\nrnfs:bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm?from=11111111111111111111111111111111\\nrnfs:bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/caf%C3%A9.txt\", \"method\": \"direct\", \"links\": [{\"uri\": \"rnfs:bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm?from=11111111111111111111111111111111\", \"cid\": \"bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm\", \"name\": null, \"path\": [], \"size\": 36, \"from\": \"11111111111111111111111111111111\"}, {\"uri\": \"rnfs:bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/caf%C3%A9.txt\", \"cid\": \"bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq\", \"name\": null, \"path\": [\"café.txt\"], \"size\": null, \"from\": null}], \"auto\": [{\"state\": \"pinned\", \"cid\": \"bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm\"}, {\"state\": \"pinned\", \"cid\": \"bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae\"}], \"id\": 1}" + ], + [ + "inbox", + 2, + "{\"hash\": \"b20fef4fb2d0e2869b3570019d9c05a64870b737d86a2e8d13660e452ef87c13\", \"from\": \"b0b1b2b3b4b5b6b7b8b9babbbcbdbebf\", \"display_name\": \"Bøb 🦊\", \"sent\": 1792056115.0, \"received\": 1792056120.0, \"title\": \"Привет\", \"text\": \"Посмотри это: rnfs:name/05c7214ca11e7646c79340c6b6d9866c/2026/%C3%A9t%C3%A9.jpg\", \"method\": \"propagated\", \"links\": [{\"uri\": \"rnfs:name/05c7214ca11e7646c79340c6b6d9866c/2026/%C3%A9t%C3%A9.jpg\", \"cid\": null, \"name\": \"05c7214ca11e7646c79340c6b6d9866c\", \"path\": [\"2026\", \"été.jpg\"], \"size\": null, \"from\": null}], \"auto\": null, \"id\": 2}" + ], + [ + "inbox", + 3, + "{\"hash\": \"584b41e5ba4074842544bf3368769200656aaddbca1d2863f897e6f2817952b6\", \"from\": \"b0b1b2b3b4b5b6b7b8b9babbbcbdbebf\", \"display_name\": null, \"sent\": null, \"received\": 1792056180.0, \"title\": \"\", \"text\": \"𝄞 music 🎵 rnfs:bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da\", \"method\": \"opportunistic\", \"links\": [{\"uri\": \"rnfs:bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da\", \"cid\": \"bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da\", \"name\": null, \"path\": [], \"size\": null, \"from\": null}], \"auto\": null, \"id\": 3}" + ], + [ + "outbox", + 1, + "{\"hash\": \"93d3f14fa1ad9e4c7b89988e68d6449f0b47b1bce1299bdb6fd33f26743991bd\", \"to\": \"b0b1b2b3b4b5b6b7b8b9babbbcbdbebf\", \"time\": 1792056240.0, \"links\": [{\"uri\": \"rnfs:bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da?from=4e0de1d04e0de1d04e0de1d04e0de1d0\", \"cid\": \"bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da\", \"name\": null, \"path\": [], \"size\": 16, \"from\": \"4e0de1d04e0de1d04e0de1d04e0de1d0\"}], \"text\": \"Voilà le fichier ✅\", \"method\": \"direct\", \"state\": \"delivered\", \"progress\": 1.0, \"id\": 1}" + ], + [ + "outbox", + 2, + "{\"hash\": \"59ae57cd8477123689cd483035d3386230f1a708b1d45dbb71338b2ac5a2197c\", \"to\": \"b0b1b2b3b4b5b6b7b8b9babbbcbdbebf\", \"time\": 1792056300.0, \"links\": [{\"uri\": \"rnfs:name/05c7214ca11e7646c79340c6b6d9866c/2026?from=4e0de1d04e0de1d04e0de1d04e0de1d0\", \"cid\": \"bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi\", \"name\": \"05c7214ca11e7646c79340c6b6d9866c\", \"path\": [\"2026\"], \"size\": 19, \"from\": \"4e0de1d04e0de1d04e0de1d04e0de1d0\"}], \"text\": \"via the propagation node 📡\", \"method\": \"propagated\", \"state\": \"sent\", \"progress\": 0.0, \"id\": 2}" + ], + [ + "outbox", + 3, + "{\"hash\": \"64bfcf1cb5c0b71e992de816e697e5ce0cb0f744b8a707bf8eb8fd602399cab0\", \"to\": \"b0b1b2b3b4b5b6b7b8b9babbbcbdbebf\", \"time\": 1792056360.0, \"links\": [{\"uri\": \"rnfs:bagaibqabciqen3e7u4m7fn7ak6q4vwpeihbka3hewbvwwkt7njnzaex4ifrk3pi?from=4e0de1d04e0de1d04e0de1d04e0de1d0\", \"cid\": \"bagaibqabciqen3e7u4m7fn7ak6q4vwpeihbka3hewbvwwkt7njnzaex4ifrk3pi\", \"name\": null, \"path\": [], \"size\": 22, \"from\": \"4e0de1d04e0de1d04e0de1d04e0de1d0\"}], \"text\": \"\", \"method\": \"propagated\", \"state\": \"failed\", \"progress\": 0.0, \"id\": 3, \"direct_hash\": \"e57ea396179755a3aa63ca5fdafa8ac20a47ab4cd2134df99da0535ef6d548ca\", \"fallback\": true, \"error\": \"the daemon stopped before it was delivered\"}" + ], + [ + "outbox", + 4, + "{\"hash\": \"75146bd05a2bb4c374a1490dc343371d424723b23f3073177d8cf35e5ceb431a\", \"to\": \"a0a1a2a3a4a5a6a7a8a9aaabacadaeaf\", \"time\": 1792056420.0, \"links\": [{\"uri\": \"rnfs:bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/%E6%97%A5%E6%9C%AC%E8%AA%9E?from=4e0de1d04e0de1d04e0de1d04e0de1d0\", \"cid\": \"bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq\", \"name\": null, \"path\": [\"日本語\"], \"size\": 7, \"from\": \"4e0de1d04e0de1d04e0de1d04e0de1d0\"}], \"text\": \"not delivered yet ⏳\", \"method\": \"direct\", \"state\": \"failed\", \"progress\": 0.0, \"id\": 4, \"error\": \"the daemon stopped before it was delivered\"}" + ], + [ + "outbox", + 5, + "{\"hash\": \"ae6c0f21ba27ea8a47bfb3607738da601e19cc3ff3eb58744c4c26ff89e27358\", \"to\": \"a0a1a2a3a4a5a6a7a8a9aaabacadaeaf\", \"time\": 1792056480.0, \"links\": [{\"uri\": \"rnfs:bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm?from=4e0de1d04e0de1d04e0de1d04e0de1d0\", \"cid\": \"bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm\", \"name\": null, \"path\": [], \"size\": 23, \"from\": \"4e0de1d04e0de1d04e0de1d04e0de1d0\"}], \"text\": \"half way 🚚\", \"method\": \"direct\", \"state\": \"failed\", \"progress\": 0.457, \"id\": 5, \"error\": \"the daemon stopped before it was delivered\"}" + ] + ], + "meta": [ + [ + "follow", + "clean", + 1 + ], + [ + "inbox", + "clean", + 1 + ], + [ + "inbox", + "next_id", + 4 + ], + [ + "outbox", + "clean", + 1 + ], + [ + "outbox", + "next_id", + 6 + ] + ], + "migrated": [ + [ + "cache.json", + 10, + 1792141200.0 + ], + [ + "contacts.json", + 3, + 1792141200.0 + ], + [ + "fetched.json", + 8, + 1792141200.0 + ], + [ + "follow.json", + 1, + 1792141200.0 + ], + [ + "invalid.json", + 3, + 1792141200.0 + ], + [ + "jobs.json", + 3, + 1792141200.0 + ], + [ + "keepers.json", + 4, + 1792141200.0 + ], + [ + "lxmf_inbox.json", + 3, + 1792141200.0 + ], + [ + "lxmf_outbox.json", + 5, + 1792141200.0 + ], + [ + "name-floor.json", + 2, + 1792141200.0 + ], + [ + "names.json", + 5, + 1792141200.0 + ], + [ + "pin_names.json", + 5, + 1792141200.0 + ], + [ + "pins.json", + 10, + 1792141200.0 + ], + [ + "providers.json", + 39, + 1792141200.0 + ], + [ + "published.json", + 28, + 1792141200.0 + ], + [ + "remote_pins.json", + 2, + 1792141200.0 + ], + [ + "shared_folder.json", + 1, + 1792141200.0 + ] + ], + "name_floor": [ + [ + { + "blob": "2a7485ee7ccbd5e2196e208cfaef1b9b" + }, + "5", + 1792055880.0, + 0 + ], + [ + { + "blob": "49154ef17f86f5401d7d07b4bf8f96f4" + }, + "18446744073709551615", + 1792055940.0, + 1 + ] + ], + "names_held": [ + [ + { + "blob": "05c7214ca11e7646c79340c6b6d9866c" + }, + "1", + { + "blob": "018080c001122050e10779218db4a02e57c80824ad42faccc002b56b806a6910ee8ca25b36cf1d" + }, + 1792055220.0 + ], + [ + { + "blob": "a9f96ee8afd8b8bee788a442a84e555e" + }, + "3", + { + "blob": "01551220e48a9a8c088f18f129cbe714d52cbe2f262020efa62392f9698203f0cf5f9eca" + }, + 1792055820.0 + ], + [ + { + "blob": "bcc7900726de195e4cc4b09e67f02966" + }, + "12", + { + "blob": "01551220d8bd1ae8a49d73ab61d6b59c9220f6696551c58a58006f5fcfad30a800ab773f" + }, + 1792055640.0 + ] + ], + "names_published": [ + [ + "imported", + "0", + null, + 1 + ], + [ + "photos", + "1", + { + "blob": "018080c001122050e10779218db4a02e57c80824ad42faccc002b56b806a6910ee8ca25b36cf1d" + }, + 0 + ] + ], + "pin_names": [ + [ + { + "blob": "01551220558abc6a7776d1f5ba7566a4ee1354b2ff53ede67bf18fa66b809fd413b532b9" + }, + "\"(stdin)\"" + ], + [ + { + "blob": "015512209cbf7c8bbb9697dfec87c2bfe1f2cc01c55493cd4c3c6da7960939fce7322d66" + }, + "\"/home/alice/Downloads/b\\udcffd name.txt\"" + ], + [ + { + "blob": "01551220ba0c105c88af68e8023121d7fad7982adfd2c7327b650d27961a25af9fb8c913" + }, + "\"(stdin)\"" + ], + [ + { + "blob": "018080c001122046ec9fa719f2b7e057a1cad9e441c2a06ce4b06b6b2a7f6a5b9012fc4162adbd" + }, + "\"/home/alice/Pictures/Fête à Montréal/photo 🎉.jpg\"" + ], + [ + { + "blob": "018080c001122086c1708170e6b241e782e2788e9e447818f56afb80cb72f0a453990628ec003a" + }, + "\"large.bin\"" + ], + [ + { + "blob": "018080c0011220b45b2792066d55160f297def05ba8a29eadd21d21ba107a842f4921f72ab1f0c" + }, + "\"Résumé – 2026.txt\"" + ] + ], + "pins": [ + [ + { + "blob": "015512204b5c2fdf195a98c4819c1b0991f08ac2771feaeb884f555c32b0e8ef61975f01" + }, + "lxmf:a0a1a2a3a4a5a6a7a8a9aaabacadaeaf" + ], + [ + { + "blob": "0155122084503737f15d2adbf191a708b0bdfecd6db1a0ec9321022247e4085ddbf38e92" + }, + "folder" + ], + [ + { + "blob": "015512209cbf7c8bbb9697dfec87c2bfe1f2cc01c55493cd4c3c6da7960939fce7322d66" + }, + "user" + ], + [ + { + "blob": "01551220b49673000ed78886bd8412ce0fc823260b880697952b3cbc79fcab646736ae60" + }, + "deny" + ], + [ + { + "blob": "01551220ba0c105c88af68e8023121d7fad7982adfd2c7327b650d27961a25af9fb8c913" + }, + "user" + ], + [ + { + "blob": "01551220bb062752be144924cc42d662cf9c6d5ab7cc5c5be4ca26eb8596c9a8a98f7c1b" + }, + "lxmf:a0a1a2a3a4a5a6a7a8a9aaabacadaeaf" + ], + [ + { + "blob": "01551220d8bd1ae8a49d73ab61d6b59c9220f6696551c58a58006f5fcfad30a800ab773f" + }, + "follow:bcc7900726de195e4cc4b09e67f02966" + ], + [ + { + "blob": "01551220f5130ee371cf7fc477243e10005c9ff101a67322e18e45099b9a22377282ba6c" + }, + "remote:0123456789abcdef0123456789abcdef" + ], + [ + { + "blob": "018080c001122046ec9fa719f2b7e057a1cad9e441c2a06ce4b06b6b2a7f6a5b9012fc4162adbd" + }, + "user" + ], + [ + { + "blob": "018080c001122050e10779218db4a02e57c80824ad42faccc002b56b806a6910ee8ca25b36cf1d" + }, + "name:photos" + ], + [ + { + "blob": "018080c001122086c1708170e6b241e782e2788e9e447818f56afb80cb72f0a453990628ec003a" + }, + "user" + ], + [ + { + "blob": "018080c0011220b45b2792066d55160f297def05ba8a29eadd21d21ba107a842f4921f72ab1f0c" + }, + "user" + ] + ], + "provider_departed": [ + [ + { + "blob": "32000000000000000000000000000000" + }, + 1791795600.0, + 1791925200.0 + ], + [ + { + "blob": "34000000000000000000000000000000" + }, + 1791925920.0, + 1792055520.0 + ], + [ + { + "blob": "35000000000000000000000000000000" + }, + 1791926580.0, + 1792054920.0 + ] + ], + "provider_evicted": [ + [ + { + "blob": "36000000000000000000000000000000" + }, + 1792051260.0, + 1792051500.0 + ] + ], + "provider_publishers": [ + [ + { + "blob": "30000000000000000000000000000000" + }, + 1791795600.0 + ], + [ + { + "blob": "31000000000000000000000000000000" + }, + 1791795600.0 + ], + [ + { + "blob": "33000000000000000000000000000000" + }, + 1791925260.0 + ], + [ + { + "blob": "37000000000000000000000000000000" + }, + 1792051320.0 + ], + [ + { + "blob": "38000000000000000000000000000000" + }, + 1792051380.0 + ], + [ + { + "blob": "39000000000000000000000000000000" + }, + 1792051440.0 + ], + [ + { + "blob": "3a000000000000000000000000000000" + }, + 1792051500.0 + ], + [ + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792054860.0 + ] + ], + "provider_records": [ + [ + { + "blob": "005bed22d2fd8d322fb80b140598a0c3" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792055160.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "075d5889d921fccee0f78cb9a3257b24" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792055340.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "0918207803da9867093b2cffe7b7bb2f" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792054860.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "24b2f721abc9dd6f9f7b8be02b20e1f8" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792055580.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "368f11e51cc31ac9cb717a07d8d12139" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792055280.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "5e3615897e5aabcf904002aa1c8c239c" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792055100.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "64f4edd9e5e844b50f7a0ae8b85a898f" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792055040.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "71e49204eaa570f938d3ba5bc497bdba" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792141200.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "7973a606727dce5446b2f0242d1c401e" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792055460.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "80000000000000000000000000000000" + }, + { + "blob": "30000000000000000000000000000000" + }, + 1791795600.0, + 1792056845.0, + 0 + ], + [ + { + "blob": "80010000000000000000000000000000" + }, + { + "blob": "30000000000000000000000000000000" + }, + 1791795600.0, + 1791795600.0, + 0 + ], + [ + { + "blob": "80020000000000000000000000000000" + }, + { + "blob": "30000000000000000000000000000000" + }, + 1791795600.0, + 1791795600.0, + 0 + ], + [ + { + "blob": "80030000000000000000000000000000" + }, + { + "blob": "30000000000000000000000000000000" + }, + 1791795600.0, + 1791795600.0, + 0 + ], + [ + { + "blob": "80040000000000000000000000000000" + }, + { + "blob": "30000000000000000000000000000000" + }, + 1791795600.0, + 1791795600.0, + 0 + ], + [ + { + "blob": "80040000000000000000000000000000" + }, + { + "blob": "31000000000000000000000000000000" + }, + 1791795600.0, + 1791795600.0, + 0 + ], + [ + { + "blob": "80050000000000000000000000000000" + }, + { + "blob": "30000000000000000000000000000000" + }, + 1791795600.0, + 1791795600.0, + 0 + ], + [ + { + "blob": "80050000000000000000000000000000" + }, + { + "blob": "31000000000000000000000000000000" + }, + 1791795600.0, + 1791795600.0, + 0 + ], + [ + { + "blob": "80060000000000000000000000000000" + }, + { + "blob": "30000000000000000000000000000000" + }, + 1791795600.0, + 1791795600.0, + 0 + ], + [ + { + "blob": "80060000000000000000000000000000" + }, + { + "blob": "31000000000000000000000000000000" + }, + 1791795600.0, + 1791795600.0, + 0 + ], + [ + { + "blob": "80070000000000000000000000000000" + }, + { + "blob": "30000000000000000000000000000000" + }, + 1791795600.0, + 1791795600.0, + 0 + ], + [ + { + "blob": "80070000000000000000000000000000" + }, + { + "blob": "31000000000000000000000000000000" + }, + 1791795600.0, + 1791795600.0, + 0 + ], + [ + { + "blob": "80080000000000000000000000000000" + }, + { + "blob": "31000000000000000000000000000000" + }, + 1791795600.0, + 1791795600.0, + 0 + ], + [ + { + "blob": "80090000000000000000000000000000" + }, + { + "blob": "31000000000000000000000000000000" + }, + 1791795600.0, + 1791795600.0, + 0 + ], + [ + { + "blob": "80250000000000000000000000000000" + }, + { + "blob": "33000000000000000000000000000000" + }, + 1791925680.0, + 1791925680.0, + 0 + ], + [ + { + "blob": "80260000000000000000000000000000" + }, + { + "blob": "33000000000000000000000000000000" + }, + 1791925740.0, + 1791925740.0, + 0 + ], + [ + { + "blob": "80270000000000000000000000000000" + }, + { + "blob": "33000000000000000000000000000000" + }, + 1791925800.0, + 1791925800.0, + 0 + ], + [ + { + "blob": "80650000000000000000000000000000" + }, + { + "blob": "37000000000000000000000000000000" + }, + 1792051320.0, + 1792051320.0, + 1 + ], + [ + { + "blob": "80660000000000000000000000000000" + }, + { + "blob": "38000000000000000000000000000000" + }, + 1792051380.0, + 1792051380.0, + 1 + ], + [ + { + "blob": "80670000000000000000000000000000" + }, + { + "blob": "39000000000000000000000000000000" + }, + 1792051440.0, + 1792051440.0, + 1 + ], + [ + { + "blob": "80680000000000000000000000000000" + }, + { + "blob": "3a000000000000000000000000000000" + }, + 1792051500.0, + 1792051500.0, + 1 + ], + [ + { + "blob": "8d616737498993836c17fa8fbc37cf05" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792056060.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "9fb861bafe2a0229fa850594ffa8d4e9" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792055760.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "b14a6a65ed6d7fd082f130f7d0c00ba9" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792055640.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "b362bf1c891e9f8bf935854c9a4a89da" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792055520.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "b60b450b4d0bfdb631405273f3800390" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792055460.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "b6a2e77295bec80e226b3d3c734bdecb" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792056000.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "bbbb4c2f859ccfb2b75447e3138b9695" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792141200.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "bf641d47a118e7693be211176667d353" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792054920.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "cc92f5dd8433c61479d0463e6adf4b50" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792054980.0, + 1792141200.0, + 0 + ], + [ + { + "blob": "ff3e2155432da6e46f967c4c688053b2" + }, + { + "blob": "4e0de1d04e0de1d04e0de1d04e0de1d0" + }, + 1792055220.0, + 1792141200.0, + 0 + ] + ], + "published": [ + [ + "bafkreibevdhpo2ydbraaurnjq3wtziaclbsmqpquyh6ntesaf7lmove4vq", + 1792056785.0, + { + "blob": "1010101010101010101010101010101011111111111111111111111111111111" + } + ], + [ + "bafkreibkk5apo3fy4r2vriim3bnah5ldx2g2e4zqnjkfstweknpscdmutm", + 1792053185.0, + { + "blob": "1111111111111111111111111111111112121212121212121212121212121212" + } + ], + [ + "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae", + 1792049585.0, + { + "blob": "1212121212121212121212121212121213131313131313131313131313131313" + } + ], + [ + "bafkreie4x56ixo4ws7p6zb6cx7q7ftabyvkjhtkmhrw2pfqjhh6oomrnmy", + 1792042385.0, + { + "blob": "1010101010101010101010101010101011111111111111111111111111111111" + } + ], + [ + "bafkreieeka3tp4k5fln7denhbcyl37wnnwy2b3eteebcer7ebbo5x44osi", + 1792038785.0, + { + "blob": "1111111111111111111111111111111112121212121212121212121212121212" + } + ], + [ + "bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm", + 1792035185.0, + { + "blob": "1212121212121212121212121212121213131313131313131313131313131313" + } + ], + [ + "bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm", + 1792031585.0, + { + "blob": "1313131313131313131313131313131310101010101010101010101010101010" + } + ], + [ + "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4", + 1792027985.0, + { + "blob": "1010101010101010101010101010101011111111111111111111111111111111" + } + ], + [ + "bafkreiherkniycepddysts7hctkszprpeyqcb35geojps2mcapym6x46zi", + 1792024385.0, + { + "blob": "1111111111111111111111111111111112121212121212121212121212121212" + } + ], + [ + "bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq", + 1792020785.0, + { + "blob": "1212121212121212121212121212121213131313131313131313131313131313" + } + ], + [ + "bagaibqabciqen3e7u4m7fn7ak6q4vwpeihbka3hewbvwwkt7njnzaex4ifrk3pi", + 1792017185.0, + { + "blob": "1313131313131313131313131313131310101010101010101010101010101010" + } + ], + [ + "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi", + 1792013585.0, + { + "blob": "1010101010101010101010101010101011111111111111111111111111111111" + } + ], + [ + "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq", + 1792009985.0, + { + "blob": "1111111111111111111111111111111112121212121212121212121212121212" + } + ], + [ + "bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da", + 1792006385.0, + { + "blob": "1212121212121212121212121212121213131313131313131313131313131313" + } + ], + [ + "key:b14a6a65ed6d7fd082f130f7d0c00ba9", + 1792002785.0, + { + "blob": "1313131313131313131313131313131310101010101010101010101010101010" + } + ], + [ + "key:b362bf1c891e9f8bf935854c9a4a89da", + 1791999185.0, + { + "blob": "1010101010101010101010101010101011111111111111111111111111111111" + } + ], + [ + "key:ff3e2155432da6e46f967c4c688053b2", + 1791995585.0, + { + "blob": "1111111111111111111111111111111112121212121212121212121212121212" + } + ] + ], + "published_earlier": [ + [ + "bafkreibevdhpo2ydbraaurnjq3wtziaclbsmqpquyh6ntesaf7lmove4vq", + { + "blob": "12121212121212121212121212121212" + }, + 1791797585.0 + ], + [ + "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae", + { + "blob": "10101010101010101010101010101010" + }, + 1791797585.0 + ], + [ + "bafkreie4x56ixo4ws7p6zb6cx7q7ftabyvkjhtkmhrw2pfqjhh6oomrnmy", + { + "blob": "12121212121212121212121212121212" + }, + 1791797585.0 + ], + [ + "bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm", + { + "blob": "10101010101010101010101010101010" + }, + 1791797585.0 + ], + [ + "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4", + { + "blob": "12121212121212121212121212121212" + }, + 1791797585.0 + ], + [ + "bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq", + { + "blob": "10101010101010101010101010101010" + }, + 1791797585.0 + ], + [ + "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi", + { + "blob": "12121212121212121212121212121212" + }, + 1791797585.0 + ], + [ + "bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da", + { + "blob": "10101010101010101010101010101010" + }, + 1791797585.0 + ], + [ + "key:b362bf1c891e9f8bf935854c9a4a89da", + { + "blob": "12121212121212121212121212121212" + }, + 1791797585.0 + ] + ], + "remote_pins": [ + [ + 0, + "remote:0123456789abcdef0123456789abcdef", + { + "blob": "015512209cfccdac5b4788e89d7eea492ee1d93f1615baa4f00a2e7105209703ea98d5a5" + } + ], + [ + 1, + "remote:0123456789abcdef0123456789abcdef", + { + "blob": "01551220b680d83a4b43738cf36c6b8e965d8b32d3b0fd2e1b643ee59cf4e0b787f394ee" + } + ] + ], + "shared_folder": [ + [ + "\"notes.txt\"", + "6baeb78e6b0bb791f691f2860e2fc0be964fc2d89cb04c82f1378c1f257c398e", + { + "blob": "0155122084503737f15d2adbf191a708b0bdfecd6db1a0ec9321022247e4085ddbf38e92" + } + ] + ], + "withdrawals": [ + [ + { + "blob": "10101010101010101010101010101010" + }, + { + "blob": "ffe8f9db57e41110e0abce7a6ce6bba0" + } + ], + [ + { + "blob": "13131313131313131313131313131313" + }, + { + "blob": "ffe8f9db57e41110e0abce7a6ce6bba0" + } + ] + ] + }, + "user_version": 16 +} diff --git a/tests/fixtures/rnfs-0.1.5/files/notes.txt b/tests/fixtures/rnfs-0.1.5/files/notes.txt new file mode 100644 index 0000000..713febb --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/files/notes.txt @@ -0,0 +1 @@ +shared folder notes diff --git a/tests/fixtures/rnfs-0.1.5/fixture_content.py b/tests/fixtures/rnfs-0.1.5/fixture_content.py new file mode 100644 index 0000000..d81d02b --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/fixture_content.py @@ -0,0 +1,22 @@ +"""The files the rnfs-0.1.5 fixture adds on top of the rnfs-0.1.4 fixture's (tests/fixtures/rnfs-0.1.4/ +fixture_content.py), shared by generate.py (which adds them with rnfs 0.1.5) and tests/test_upgrade_015.py (which +adds them again with the current code, to make the blocks: the fixture keeps no blocks/ of its own). Data only +(and hashlib), so either version of rnfs can import it. +""" +import hashlib + +# Local deny-list entries (rnfs deny add): a CID, and a file hash. Neither matches anything the fixture holds, +# so the upgrade test sees every pin and record as 0.1.5 kept it. +DENIED_CID = "bafkreig5r66w4f7qbtvtsqwwlfvfql6cmsnhp42vymrhjfiwhtestirgie" # raw block b"denied on this node only" +DENIED_HASH = "sha256:b4c04cf45324f820d1061ae8f3955b3c5f9d4079553d52e6624dcc9386879bcb" # b"a file this node refuses" + +# A deny list (the deny_lists setting names its CID), added with no name and pinned "deny" once accepted +DENY_LIST = (b"# a list a node operator follows\n" + b"bafkreibdtcf2jfx2vilaswpa3bmfkbli5w3qw3jnxz23mnpqzflu4y3dzm\n" # raw block b"denied by a deny list" + b"sha256:ab79b95e5c7747cd86f28d0d91b8339922032445112f1b6a48713847858bdaee\n", None) + +# A file of several blocks, pinned and read once while file hashes are denied, so 0.1.5 remembers its content +# hash (the deny_hashes table): a one-block file's CID is its hash, so only a larger one is remembered +LARGE = (b"".join(hashlib.sha256(b"%d" % i).digest() for i in range(20000)), "large.bin") + +STREAMS = (DENY_LIST, LARGE) diff --git a/tests/fixtures/rnfs-0.1.5/generate.py b/tests/fixtures/rnfs-0.1.5/generate.py new file mode 100644 index 0000000..d81a3b6 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/generate.py @@ -0,0 +1,176 @@ +"""Write the tests/fixtures/rnfs-0.1.5 fixture: an rnfs directory exactly as rnfs 0.1.5 leaves it, its state +database holding a row in every table (see README.md there). + +Run it with the rnfs 0.1.5 source tree (git archive v0.1.5, which includes its tests/ package and the rnfs-0.1.4 +fixture), never the current one: it puts that tree first on sys.path and checks rnfs is imported from there. From +the repository root: + + mkdir /tmp/rnfs-0.1.5 && git archive v0.1.5 | tar -x -C /tmp/rnfs-0.1.5 + python3 tests/fixtures/rnfs-0.1.5/generate.py /tmp/rnfs-0.1.5 /tmp/rnfs-fixture-out + # then replace tests/fixtures/rnfs-0.1.5/{state,names,files,expected.json} with those in + # /tmp/rnfs-fixture-out/rnfs-0.1.5 + +How: 0.1.5 starts, offline, on a copy of the rnfs-0.1.4 fixture, with that fixture's blocks made and its clock +and settings, as 0.1.5's own tests/test_upgrade_014.py does. That imports the 17 JSON state files. Then 0.1.5's +own objects add what only 0.1.5 has: local deny-list entries (a CID and a file hash), a deny list followed through +the deny_lists setting (accepted, its blocks pinned "deny"), and a file's content hash remembered (deny_hashes). +The node stops (which saves and checkpoints), and the database is VACUUMed. 0.1.5 then starts again on a copy and +must change no row. expected.json holds every table's rows, read with plain sqlite3. +""" +import io +import json +import os +import shutil +import sqlite3 +import sys +from unittest import mock + +V015, OUT = (os.path.abspath(arg) for arg in sys.argv[1:3]) +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRATCH = os.path.join(OUT, "scratch") +sys.path.insert(0, V015) +sys.path.insert(1, HERE) # fixture_content, next to this file + +import RNS # noqa: E402 + +import rnfs # noqa: E402 + +assert rnfs.__file__.startswith(V015 + os.sep), rnfs.__file__ +assert rnfs.__version__ == "0.1.5", rnfs.__version__ + +from rnfs import statedb # noqa: E402 +from rnfs.config import Config # noqa: E402 +from rnfs.content import builder # noqa: E402 +from rnfs.content.blockstore import BlockStore # noqa: E402 +from rnfs.content.cid import Cid # noqa: E402 +from rnfs.node import RnfsNode # noqa: E402 +from tests import test_upgrade_014 as v014 # noqa: E402 (0.1.5's own upgrade test: its fixture, clock, settings) +from tests.test_names import FakeNetwork # noqa: E402 +from tests.test_node import NetClient # noqa: E402 + +import fixture_content as content # noqa: E402 + +NOW = v014.NOW # T0 + 1 day: the 0.1.4 fixture's records haven't expired + + +def tables(db_path: str) -> dict[str, list]: + """Every row of every table, sorted, as JSON: a BLOB is {"blob": hex}.""" + conn = sqlite3.connect(db_path) + try: + names = [row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name")] + return {name: sorted(([{"blob": v.hex()} if isinstance(v, bytes) else v for v in row] + for row in conn.execute(f"SELECT * FROM {name}")), key=repr) for name in names} + finally: + conn.close() + + +def patched(clock): + """The patches 0.1.5's upgrade test starts its node under: time.time, and the stores taking it as a default + argument.""" + return [ + mock.patch("time.time", clock), + *(mock.patch.object(v014.node_module, cls, v014.functools.partial(getattr(v014.node_module, cls), + clock=clock)) + for cls in ("ProviderStore", "Publisher", "NameManager"))] + + +def start(root: str, settings: dict) -> RnfsNode: + """An offline 0.1.5 node on root.""" + config = Config(rnfs_dir=root, **settings) + config.validate() + return RnfsNode.offline(config, v014.NODE_ID, NetClient(), FakeNetwork()) + + +def build(root: str) -> dict: + """Copy the 0.1.4 fixture to root, make its blocks and upgrade it with 0.1.5, then add 0.1.5's rows. The + settings used.""" + for part in ("state", "names", "files"): + shutil.copytree(os.path.join(v014.FIXTURE, part), os.path.join(root, part)) + v014.make_blocks(os.path.join(root, "blocks"), SCRATCH) + store = BlockStore(os.path.join(root, "blocks")) + deny_list = builder.add_stream(io.BytesIO(content.DENY_LIST[0]), store.put, len(content.DENY_LIST[0])).cid + settings = {**v014.SETTINGS, "deny_lists": [str(deny_list)]} + clock = v014.FakeClock(NOW) + patches = patched(clock) + for patch in patches: + patch.start() + try: + node = start(root, settings) + try: + for name in v014.LEGACY: + assert os.path.exists(os.path.join(root, "state", name + statedb.MIGRATED_SUFFIX)), name + # deny_entries + assert node.deny.add([content.DENIED_CID, content.DENIED_HASH]) + # deny_lists, and its blocks pinned "deny" + assert node.feature("deny").check(str(deny_list)), node.feature("deny").lists() + # deny_hashes: a file of several blocks read while file hashes are denied + data, name = content.LARGE + large = Cid.parse(node.add_stream(io.BytesIO(data), name=name)["cid"]) + info, chunks = node.read_file(f"{large}/{name}") + assert b"".join(chunks) == data + assert node.deny.known_hash(Cid.parse(info["cid"])) is not None + finally: + node.stop() + finally: + for patch in reversed(patches): + patch.stop() + db_path = os.path.join(root, "state", statedb.DB_FILE) + conn = sqlite3.connect(db_path) + try: + conn.execute("VACUUM") + finally: + conn.close() + for suffix in ("-wal", "-shm"): + assert not os.path.exists(db_path + suffix), suffix + return settings + + +def check(root: str, settings: dict): + """Start 0.1.5 again on a copy of root (with the blocks), at the same time: it must change no row, so the test + can expect every row as it is.""" + copy = root + "-check" + shutil.copytree(root, copy) + before = tables(os.path.join(copy, "state", statedb.DB_FILE)) + patches = patched(v014.FakeClock(NOW)) + for patch in patches: + patch.start() + try: + start(copy, settings).stop() + finally: + for patch in reversed(patches): + patch.stop() + after = tables(os.path.join(copy, "state", statedb.DB_FILE)) + assert after == before, {t: (before[t], after[t]) for t in before if before[t] != after.get(t)} + empty = [t for t, rows in before.items() if not rows] + assert not empty, empty # every table has rows + shutil.rmtree(copy) + + +def prune(root: str): + """Drop what the fixture doesn't need, as the 0.1.4 fixture does: LXMF router state, the node identity, the + API token, logs, and the blocks (the test makes them again from the two fixture_content modules).""" + for name in ("lxmf", "api_token", "debug.log", "debug.log.1", "storage", "blocks"): + path = os.path.join(root, name) + if os.path.isdir(path): + shutil.rmtree(path) + elif os.path.exists(path): + os.remove(path) + + +if __name__ == "__main__": + RNS.loglevel = RNS.LOG_CRITICAL + root = os.path.join(OUT, "rnfs-0.1.5") + for path in (root, SCRATCH): + if os.path.exists(path): + shutil.rmtree(path) + os.makedirs(SCRATCH) + settings = build(root) + check(root, settings) + prune(root) + shutil.rmtree(SCRATCH) + expected = {"now": NOW, "settings": settings, "user_version": len(statedb.SCHEMA), + "tables": tables(os.path.join(root, "state", statedb.DB_FILE))} + with open(os.path.join(root, "expected.json"), "w", encoding="utf-8") as f: + json.dump(expected, f, indent=1, ensure_ascii=False, sort_keys=True) + f.write("\n") + print(root) diff --git a/tests/fixtures/rnfs-0.1.5/names/imported b/tests/fixtures/rnfs-0.1.5/names/imported new file mode 100644 index 0000000..decc608 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/names/imported @@ -0,0 +1 @@ +b?ub!PhV+Ec#nTfk٥AQ} "Ͱk5 \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/names/photos b/tests/fixtures/rnfs-0.1.5/names/photos new file mode 100644 index 0000000..d25014a --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/names/photos @@ -0,0 +1,2 @@ +(񹐤Ex>q? +T6ײQ`2ݍ2@|g?W!DӂUimp[ƃnW \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/cache.json.migrated b/tests/fixtures/rnfs-0.1.5/state/cache.json.migrated new file mode 100644 index 0000000..46e58ed --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/cache.json.migrated @@ -0,0 +1 @@ +{"fetches": [["bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq", [], "bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq"], ["bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq", [], "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq"], ["bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq", ["\u65e5\u672c\u8a9e", "\u30e1\u30e2.md"], "bafkreibvbnvd5lbk76cb3cq35w77w252b4koramjhn426wy3xjy2vnp6yq"], ["bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq", ["\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac", "\u03b1\u03c1\u03c7\u03b5\u03af\u03bf.txt"], "bafkreibevdhpo2ydbraaurnjq3wtziaclbsmqpquyh6ntesaf7lmove4vq"], ["bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi", [], "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi"], ["bafkreibkk5apo3fy4r2vriim3bnah5ldx2g2e4zqnjkfstweknpscdmutm", [], "bafkreibkk5apo3fy4r2vriim3bnah5ldx2g2e4zqnjkfstweknpscdmutm"], ["bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4", [], "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4"], ["bafkreiherkniycepddysts7hctkszprpeyqcb35geojps2mcapym6x46zi", [], "bafkreiherkniycepddysts7hctkszprpeyqcb35geojps2mcapym6x46zi"], ["bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm", [], "bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm"], ["bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq", ["caf\u00e9.txt"], "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae"]]} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/contacts.json.migrated b/tests/fixtures/rnfs-0.1.5/state/contacts.json.migrated new file mode 100644 index 0000000..924b480 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/contacts.json.migrated @@ -0,0 +1 @@ +{"version": 1, "contacts": {"alice": {"kind": "name", "id": "bcc7900726de195e4cc4b09e67f02966"}, "bob": {"kind": "lxmf", "id": "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf"}, "relay.1": {"kind": "node", "id": "10101010101010101010101010101010"}}} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/fetched.json.migrated b/tests/fixtures/rnfs-0.1.5/state/fetched.json.migrated new file mode 100644 index 0000000..4b6b62a --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/fetched.json.migrated @@ -0,0 +1 @@ +["bafkreibevdhpo2ydbraaurnjq3wtziaclbsmqpquyh6ntesaf7lmove4vq", "bafkreibkk5apo3fy4r2vriim3bnah5ldx2g2e4zqnjkfstweknpscdmutm", "bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm", "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4", "bafkreiherkniycepddysts7hctkszprpeyqcb35geojps2mcapym6x46zi", "bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq", "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi", "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq"] \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/follow.json.migrated b/tests/fixtures/rnfs-0.1.5/state/follow.json.migrated new file mode 100644 index 0000000..9532fd3 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/follow.json.migrated @@ -0,0 +1 @@ +{"version": 1, "names": {"bcc7900726de195e4cc4b09e67f02966": {"pinned": "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4"}}} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/invalid.json.migrated b/tests/fixtures/rnfs-0.1.5/state/invalid.json.migrated new file mode 100644 index 0000000..9bafe2f --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/invalid.json.migrated @@ -0,0 +1 @@ +["bafkreicuzem5bjal4mwj4kmzavvzoqkhtyt47mz7t3tsbqswqdohetk4w4", "bafkreieqpx5nxvsmjzt5vv7kk7d2cu7n2ky3y3oupplaffenbqx4qodamu", "bafkreigcfjk46gpgsldrp3krvcynvcga2q5a6zt6a7l7tazzh6bcdgbcju"] \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/jobs.json.migrated b/tests/fixtures/rnfs-0.1.5/state/jobs.json.migrated new file mode 100644 index 0000000..ea3ccfb --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/jobs.json.migrated @@ -0,0 +1 @@ +{"jobs": [{"id": "3bd70342", "kind": "get", "target": "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/caf\u00e9.txt", "output": "/home/alice/Downloads/caf\u00e9 \ud83d\ude00.txt", "from": null, "sealed_name": false, "fallback_dht": false, "state": "done", "created": 1792056540.0, "started": 1792056540.0, "finished": 1792056545.0, "error": null, "result": {"cid": "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae", "size": 7, "warnings": ["skipped symlink caf\u00e9/link \u2192 elsewhere", "kept \ud83d\ude00 as is"], "output": "/home/alice/Downloads/caf\u00e9 \ud83d\ude00.txt"}, "runs": 1, "total_bytes": 7, "done_bytes": 7, "blocks": 1, "blocks_total": null, "downloaded_bytes": 7}, {"id": "6a322a52", "kind": "pin", "target": "bafkreiemqpndzewx4bioklzfvhtiegsfny5fxuc3nkjk2vmcpjxiammsti", "output": null, "from": null, "sealed_name": false, "fallback_dht": false, "state": "failed", "created": 1792056605.0, "started": 1792056605.0, "finished": 1792056605.0, "error": "no providers found for this CID", "result": null, "runs": 1, "total_bytes": null, "done_bytes": 0, "blocks": 0, "blocks_total": null, "downloaded_bytes": 0}, {"id": "da44b5c5", "kind": "pin", "target": "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac", "output": null, "from": null, "sealed_name": false, "fallback_dht": false, "state": "queued", "created": 1792056665.0, "started": null, "finished": null, "error": null, "result": null, "runs": 0, "total_bytes": null, "done_bytes": 0, "blocks": 0, "blocks_total": null, "downloaded_bytes": 0}]} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/keepers.json.migrated b/tests/fixtures/rnfs-0.1.5/state/keepers.json.migrated new file mode 100644 index 0000000..08ffc5d --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/keepers.json.migrated @@ -0,0 +1 @@ +{"version": 1, "names": {"05c7214ca11e7646c79340c6b6d9866c": {"reasons": ["own"], "record": ["e5ee52baf6bf767a65fc00d5acf5231c3ab7509438494eec22e4cdb9eae33d3cb4d1e823db4955856e0f57c46987475752a2aa95721399815e3f955d98e23cb2", "783c4f3005a78c29de83", "2e62f40c0a006ad097b4", "", "eb5f4d8539b58b495781c8ba83765ed04318d1718032b8d2af0ad694b340480aa6a9863e4fa057aedb7681f7e41145be6a61a8091e4dd02d10c71920ad932701", "9201c427018080c001122050e10779218db4a02e57c80824ad42faccc002b56b806a6910ee8ca25b36cf1d"]}, "07defa27cdd333a37d1c1f425770a541": {"reasons": ["user"], "record": null}, "bcc7900726de195e4cc4b09e67f02966": {"reasons": ["follow"], "record": ["8e6e4ec6192178ce7ea841c1603ee6756fe9bed343f7780234889a6fb4da5414274d5caf3ca30bb9017e5ac96eb3a75622171754072025b11210c98a0e8e546a", "783c4f3005a78c29de83", "ff7788273d006ad09958", "", "9c523b5797fa9e96ebcf18a1fef4e33aa25c682c158b34592f3942a63ab98dff38b8e8f890649ad259d3c1ce6ea25490077b1c26ed7ec0a891b4f0ae0d31db00", "920cc42401551220d8bd1ae8a49d73ab61d6b59c9220f6696551c58a58006f5fcfad30a800ab773f"]}, "f35e61151054781eb9f985f470691a77": {"reasons": ["user"], "record": ["8db0daa5ca4c0979d432ce2b7ffac8f5057a4a77245494128a636f961309e54f4b2b7e004d1888419fa0de6e61ee896a40996edf79ecd1da36a56ac982962f3b", "783c4f3005a78c29de83", "d2a612e452006ad098e0", "", "f1cf556174e0341ee785bf34f236f06aae1cbda68547a6c5716ace79d29711641ac8b7afde40eb3ef226fad4ce416e4e6a343546be4c1de920fda5004b3d250d", "9207c424015512202a5740f76cb8e47558a10cd85a03f563be8da273306a54594ec4535f210d949b"]}}} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/lxmf_inbox.json.migrated b/tests/fixtures/rnfs-0.1.5/state/lxmf_inbox.json.migrated new file mode 100644 index 0000000..7eb06bd --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/lxmf_inbox.json.migrated @@ -0,0 +1 @@ +{"next_id": 4, "messages": [{"hash": "7b4121eec4d068e3eeee0a668158e14ad322a1ef925c0be28aa6a26a9dde4ebf", "from": "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf", "display_name": "Zoë", "sent": 1792056030.0, "received": 1792056060.0, "title": "Grüße 🌻", "text": "Fotos vom Wochenende 📷\n\nrnfs:bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm?from=11111111111111111111111111111111\nrnfs:bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/caf%C3%A9.txt", "method": "direct", "links": [{"uri": "rnfs:bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm?from=11111111111111111111111111111111", "cid": "bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm", "name": null, "path": [], "size": 36, "from": "11111111111111111111111111111111"}, {"uri": "rnfs:bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/caf%C3%A9.txt", "cid": "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq", "name": null, "path": ["café.txt"], "size": null, "from": null}], "auto": [{"state": "pinned", "cid": "bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm"}, {"state": "pinned", "cid": "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae"}], "id": 1}, {"hash": "b20fef4fb2d0e2869b3570019d9c05a64870b737d86a2e8d13660e452ef87c13", "from": "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf", "display_name": "Bøb 🦊", "sent": 1792056115.0, "received": 1792056120.0, "title": "Привет", "text": "Посмотри это: rnfs:name/05c7214ca11e7646c79340c6b6d9866c/2026/%C3%A9t%C3%A9.jpg", "method": "propagated", "links": [{"uri": "rnfs:name/05c7214ca11e7646c79340c6b6d9866c/2026/%C3%A9t%C3%A9.jpg", "cid": null, "name": "05c7214ca11e7646c79340c6b6d9866c", "path": ["2026", "été.jpg"], "size": null, "from": null}], "auto": null, "id": 2}, {"hash": "584b41e5ba4074842544bf3368769200656aaddbca1d2863f897e6f2817952b6", "from": "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf", "display_name": null, "sent": null, "received": 1792056180.0, "title": "", "text": "𝄞 music 🎵 rnfs:bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da", "method": "opportunistic", "links": [{"uri": "rnfs:bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da", "cid": "bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da", "name": null, "path": [], "size": null, "from": null}], "auto": null, "id": 3}]} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/lxmf_outbox.json.migrated b/tests/fixtures/rnfs-0.1.5/state/lxmf_outbox.json.migrated new file mode 100644 index 0000000..1e5c733 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/lxmf_outbox.json.migrated @@ -0,0 +1 @@ +{"next_id": 6, "messages": [{"hash": "93d3f14fa1ad9e4c7b89988e68d6449f0b47b1bce1299bdb6fd33f26743991bd", "to": "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf", "time": 1792056240.0, "links": [{"uri": "rnfs:bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da?from=4e0de1d04e0de1d04e0de1d04e0de1d0", "cid": "bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da", "name": null, "path": [], "size": 16, "from": "4e0de1d04e0de1d04e0de1d04e0de1d0"}], "text": "Voilà le fichier ✅", "method": "direct", "state": "delivered", "progress": 1.0, "id": 1}, {"hash": "59ae57cd8477123689cd483035d3386230f1a708b1d45dbb71338b2ac5a2197c", "to": "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf", "time": 1792056300.0, "links": [{"uri": "rnfs:name/05c7214ca11e7646c79340c6b6d9866c/2026?from=4e0de1d04e0de1d04e0de1d04e0de1d0", "cid": "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi", "name": "05c7214ca11e7646c79340c6b6d9866c", "path": ["2026"], "size": 19, "from": "4e0de1d04e0de1d04e0de1d04e0de1d0"}], "text": "via the propagation node 📡", "method": "propagated", "state": "sent", "progress": 0.0, "id": 2}, {"hash": "64bfcf1cb5c0b71e992de816e697e5ce0cb0f744b8a707bf8eb8fd602399cab0", "to": "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf", "time": 1792056360.0, "links": [{"uri": "rnfs:bagaibqabciqen3e7u4m7fn7ak6q4vwpeihbka3hewbvwwkt7njnzaex4ifrk3pi?from=4e0de1d04e0de1d04e0de1d04e0de1d0", "cid": "bagaibqabciqen3e7u4m7fn7ak6q4vwpeihbka3hewbvwwkt7njnzaex4ifrk3pi", "name": null, "path": [], "size": 22, "from": "4e0de1d04e0de1d04e0de1d04e0de1d0"}], "text": "", "method": "propagated", "state": "outbound", "progress": 0.0, "id": 3, "direct_hash": "e57ea396179755a3aa63ca5fdafa8ac20a47ab4cd2134df99da0535ef6d548ca", "fallback": true}, {"hash": "75146bd05a2bb4c374a1490dc343371d424723b23f3073177d8cf35e5ceb431a", "to": "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf", "time": 1792056420.0, "links": [{"uri": "rnfs:bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq/%E6%97%A5%E6%9C%AC%E8%AA%9E?from=4e0de1d04e0de1d04e0de1d04e0de1d0", "cid": "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq", "name": null, "path": ["日本語"], "size": 7, "from": "4e0de1d04e0de1d04e0de1d04e0de1d0"}], "text": "not delivered yet ⏳", "method": "direct", "state": "outbound", "progress": 0.0, "id": 4}, {"hash": "ae6c0f21ba27ea8a47bfb3607738da601e19cc3ff3eb58744c4c26ff89e27358", "to": "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf", "time": 1792056480.0, "links": [{"uri": "rnfs:bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm?from=4e0de1d04e0de1d04e0de1d04e0de1d0", "cid": "bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm", "name": null, "path": [], "size": 23, "from": "4e0de1d04e0de1d04e0de1d04e0de1d0"}], "text": "half way 🚚", "method": "direct", "state": "sending", "progress": 0.457, "id": 5}]} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/name-floor.json.migrated b/tests/fixtures/rnfs-0.1.5/state/name-floor.json.migrated new file mode 100644 index 0000000..6631290 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/name-floor.json.migrated @@ -0,0 +1 @@ +{"version":1,"floor":[["2a7485ee7ccbd5e2196e208cfaef1b9b",5,1792055880],["49154ef17f86f5401d7d07b4bf8f96f4",18446744073709551615,1792055940]]} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/names.json.migrated b/tests/fixtures/rnfs-0.1.5/state/names.json.migrated new file mode 100644 index 0000000..89a9228 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/names.json.migrated @@ -0,0 +1,32 @@ +{ + "version": 1, + "published": { + "imported": { + "sequence": 0, + "cid": null, + "imported": true + }, + "photos": { + "sequence": 1, + "cid": "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi", + "imported": false + } + }, + "held": { + "05c7214ca11e7646c79340c6b6d9866c": { + "sequence": 1, + "cid": "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi", + "heard": 1792055220.0 + }, + "a9f96ee8afd8b8bee788a442a84e555e": { + "sequence": 3, + "cid": "bafkreiherkniycepddysts7hctkszprpeyqcb35geojps2mcapym6x46zi", + "heard": 1792055820.0 + }, + "bcc7900726de195e4cc4b09e67f02966": { + "sequence": 12, + "cid": "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4", + "heard": 1792055640.0 + } + } +} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/pin_names.json.migrated b/tests/fixtures/rnfs-0.1.5/state/pin_names.json.migrated new file mode 100644 index 0000000..d098adb --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/pin_names.json.migrated @@ -0,0 +1 @@ +{"bafkreicvrk6gu53w2h23u5lgutxbgvfs75j63zt36gh2m24at7kbhnjsxe": "(stdin)", "bafkreie4x56ixo4ws7p6zb6cx7q7ftabyvkjhtkmhrw2pfqjhh6oomrnmy": "/home/alice/Downloads/b\udcffd name.txt", "bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm": "(stdin)", "bagaibqabciqen3e7u4m7fn7ak6q4vwpeihbka3hewbvwwkt7njnzaex4ifrk3pi": "/home/alice/Pictures/F\u00eate \u00e0 Montr\u00e9al/photo \ud83c\udf89.jpg", "bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da": "R\u00e9sum\u00e9 \u2013 2026.txt"} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/pins.json.migrated b/tests/fixtures/rnfs-0.1.5/state/pins.json.migrated new file mode 100644 index 0000000..8d16147 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/pins.json.migrated @@ -0,0 +1 @@ +{"pins": {"bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da": ["user"], "bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm": ["user"], "bagaibqabciqen3e7u4m7fn7ak6q4vwpeihbka3hewbvwwkt7njnzaex4ifrk3pi": ["user"], "bafkreie4x56ixo4ws7p6zb6cx7q7ftabyvkjhtkmhrw2pfqjhh6oomrnmy": ["user"], "bafkreieeka3tp4k5fln7denhbcyl37wnnwy2b3eteebcer7ebbo5x44osi": ["folder"], "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi": ["name:photos"], "bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq": ["remote:0123456789abcdef0123456789abcdef"], "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4": ["follow:bcc7900726de195e4cc4b09e67f02966"], "bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm": ["lxmf:a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"], "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae": ["lxmf:a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"]}} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/providers.json.migrated b/tests/fixtures/rnfs-0.1.5/state/providers.json.migrated new file mode 100644 index 0000000..12eb9d1 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/providers.json.migrated @@ -0,0 +1 @@ +{"version":1,"publishers":{"30000000000000000000000000000000":1791795600.0,"31000000000000000000000000000000":1791795600.0,"33000000000000000000000000000000":1791925260.0,"37000000000000000000000000000000":1792051320.0,"38000000000000000000000000000000":1792051380.0,"39000000000000000000000000000000":1792051440.0,"3a000000000000000000000000000000":1792051500.0,"4e0de1d04e0de1d04e0de1d04e0de1d0":1792054860.0},"records":[["80000000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1792056845.0,0],["80010000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80020000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80030000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80040000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80040000000000000000000000000000","31000000000000000000000000000000",1791795600.0,1791795600.0,0],["80050000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80050000000000000000000000000000","31000000000000000000000000000000",1791795600.0,1791795600.0,0],["80060000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80060000000000000000000000000000","31000000000000000000000000000000",1791795600.0,1791795600.0,0],["80070000000000000000000000000000","30000000000000000000000000000000",1791795600.0,1791795600.0,0],["80070000000000000000000000000000","31000000000000000000000000000000",1791795600.0,1791795600.0,0],["80080000000000000000000000000000","31000000000000000000000000000000",1791795600.0,1791795600.0,0],["80090000000000000000000000000000","31000000000000000000000000000000",1791795600.0,1791795600.0,0],["80240000000000000000000000000000","33000000000000000000000000000000",1791925620.0,1791925620.0,0],["80250000000000000000000000000000","33000000000000000000000000000000",1791925680.0,1791925680.0,0],["80260000000000000000000000000000","33000000000000000000000000000000",1791925740.0,1791925740.0,0],["80270000000000000000000000000000","33000000000000000000000000000000",1791925800.0,1791925800.0,0],["80650000000000000000000000000000","37000000000000000000000000000000",1792051320.0,1792051320.0,1],["80660000000000000000000000000000","38000000000000000000000000000000",1792051380.0,1792051380.0,1],["80670000000000000000000000000000","39000000000000000000000000000000",1792051440.0,1792051440.0,1],["80680000000000000000000000000000","3a000000000000000000000000000000",1792051500.0,1792051500.0,1],["0918207803da9867093b2cffe7b7bb2f","4e0de1d04e0de1d04e0de1d04e0de1d0",1792054860.0,1792054860.0,0],["bf641d47a118e7693be211176667d353","4e0de1d04e0de1d04e0de1d04e0de1d0",1792054920.0,1792054920.0,0],["cc92f5dd8433c61479d0463e6adf4b50","4e0de1d04e0de1d04e0de1d04e0de1d0",1792054980.0,1792054980.0,0],["64f4edd9e5e844b50f7a0ae8b85a898f","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055040.0,1792055040.0,0],["5e3615897e5aabcf904002aa1c8c239c","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055100.0,1792055100.0,0],["005bed22d2fd8d322fb80b140598a0c3","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055160.0,1792055160.0,0],["ff3e2155432da6e46f967c4c688053b2","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055220.0,1792055220.0,0],["368f11e51cc31ac9cb717a07d8d12139","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055280.0,1792055280.0,0],["075d5889d921fccee0f78cb9a3257b24","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055340.0,1792055340.0,0],["7973a606727dce5446b2f0242d1c401e","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055460.0,1792055460.0,0],["b60b450b4d0bfdb631405273f3800390","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055460.0,1792055460.0,0],["b362bf1c891e9f8bf935854c9a4a89da","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055520.0,1792055520.0,0],["24b2f721abc9dd6f9f7b8be02b20e1f8","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055580.0,1792055580.0,0],["b14a6a65ed6d7fd082f130f7d0c00ba9","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055640.0,1792055640.0,0],["9fb861bafe2a0229fa850594ffa8d4e9","4e0de1d04e0de1d04e0de1d04e0de1d0",1792055760.0,1792055760.0,0],["b6a2e77295bec80e226b3d3c734bdecb","4e0de1d04e0de1d04e0de1d04e0de1d0",1792056000.0,1792056000.0,0],["8d616737498993836c17fa8fbc37cf05","4e0de1d04e0de1d04e0de1d04e0de1d0",1792056060.0,1792056060.0,0]],"departed":[["32000000000000000000000000000000",1791795600.0,1791925200.0],["35000000000000000000000000000000",1791926580.0,1792054920.0],["34000000000000000000000000000000",1791925920.0,1792055520.0]],"evicted":[["36000000000000000000000000000000",1792051260.0,1792051500.0]]} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/published.json.migrated b/tests/fixtures/rnfs-0.1.5/state/published.json.migrated new file mode 100644 index 0000000..5fbe127 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/published.json.migrated @@ -0,0 +1 @@ +{"published": {"bafkreibevdhpo2ydbraaurnjq3wtziaclbsmqpquyh6ntesaf7lmove4vq": [1792056785.0, ["10101010101010101010101010101010", "11111111111111111111111111111111"]], "bafkreibkk5apo3fy4r2vriim3bnah5ldx2g2e4zqnjkfstweknpscdmutm": [1792053185.0, ["11111111111111111111111111111111", "12121212121212121212121212121212"]], "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae": [1792049585.0, ["12121212121212121212121212121212", "13131313131313131313131313131313"]], "bafkreie4x56ixo4ws7p6zb6cx7q7ftabyvkjhtkmhrw2pfqjhh6oomrnmy": [1792042385.0, ["10101010101010101010101010101010", "11111111111111111111111111111111"]], "bafkreieeka3tp4k5fln7denhbcyl37wnnwy2b3eteebcer7ebbo5x44osi": [1792038785.0, ["11111111111111111111111111111111", "12121212121212121212121212121212"]], "bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm": [1792035185.0, ["12121212121212121212121212121212", "13131313131313131313131313131313"]], "bafkreif3aytvfpqujesmyqwwmlhzy3k2w7gfyw7ezitoxbmwzguktd34dm": [1792031585.0, ["13131313131313131313131313131313", "10101010101010101010101010101010"]], "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4": [1792027985.0, ["10101010101010101010101010101010", "11111111111111111111111111111111"]], "bafkreiherkniycepddysts7hctkszprpeyqcb35geojps2mcapym6x46zi": [1792024385.0, ["11111111111111111111111111111111", "12121212121212121212121212121212"]], "bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq": [1792020785.0, ["12121212121212121212121212121212", "13131313131313131313131313131313"]], "bagaibqabciqen3e7u4m7fn7ak6q4vwpeihbka3hewbvwwkt7njnzaex4ifrk3pi": [1792017185.0, ["13131313131313131313131313131313", "10101010101010101010101010101010"]], "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi": [1792013585.0, ["10101010101010101010101010101010", "11111111111111111111111111111111"]], "bagaibqabciqjpffm54fysdaxqxjggmfjifzfvoyfsy256jovqmnjy6dpbbpg7lq": [1792009985.0, ["11111111111111111111111111111111", "12121212121212121212121212121212"]], "bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da": [1792006385.0, ["12121212121212121212121212121212", "13131313131313131313131313131313"]], "key:b14a6a65ed6d7fd082f130f7d0c00ba9": [1792002785.0, ["13131313131313131313131313131313", "10101010101010101010101010101010"]], "key:b362bf1c891e9f8bf935854c9a4a89da": [1791999185.0, ["10101010101010101010101010101010", "11111111111111111111111111111111"]], "key:ff3e2155432da6e46f967c4c688053b2": [1791995585.0, ["11111111111111111111111111111111", "12121212121212121212121212121212"]]}, "earlier": {"bafkreibevdhpo2ydbraaurnjq3wtziaclbsmqpquyh6ntesaf7lmove4vq": {"12121212121212121212121212121212": 1791797585.0}, "bafkreicllqx56gk2tdcidha3bgi7bcwco4p6v24ij5kvymvq5dxwdf27ae": {"10101010101010101010101010101010": 1791797585.0}, "bafkreie4x56ixo4ws7p6zb6cx7q7ftabyvkjhtkmhrw2pfqjhh6oomrnmy": {"12121212121212121212121212121212": 1791797585.0}, "bafkreif2bqifzcfpnduaemjb275npgbk37jmomt3mugspfq2ewxz7ogjcm": {"10101010101010101010101010101010": 1791797585.0}, "bafkreigyxunorje5oovwdvvvtsjcb5tjmvi4lcsyabxv7t5ngcuabk3xh4": {"12121212121212121212121212121212": 1791797585.0}, "bafkreihvcmhog4opp7chojb6caafzh7ragthgixbrzcqtg42ei3xfav2nq": {"10101010101010101010101010101010": 1791797585.0}, "bagaibqabciqfbyihpeqy3nfafzl4qcbevvbpvtgaak2wxadkneio5dfclm3m6hi": {"12121212121212121212121212121212": 1791797585.0}, "bagaibqabciqliwzhsidg2viwb4ux33yfxkfct2w5ehjbxiihvbbpjeq7okvr6da": {"10101010101010101010101010101010": 1791797585.0}, "key:b362bf1c891e9f8bf935854c9a4a89da": {"12121212121212121212121212121212": 1791797585.0}}, "withdraw": {"13131313131313131313131313131313": ["ffe8f9db57e41110e0abce7a6ce6bba0"], "10101010101010101010101010101010": ["ffe8f9db57e41110e0abce7a6ce6bba0"]}} \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/remote_pins.json.migrated b/tests/fixtures/rnfs-0.1.5/state/remote_pins.json.migrated new file mode 100644 index 0000000..de15fb5 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/remote_pins.json.migrated @@ -0,0 +1 @@ +[["remote:0123456789abcdef0123456789abcdef", "bafkreie47tg2yw2hrduj27xkjexodwj7cyk3vjhqbixhcbjas4b6vggvuu"], ["remote:0123456789abcdef0123456789abcdef", "bafkreifwqdmdus2doogpg3dlr2lf3czs2oyp2lq3mq7olhhu4c3yp44u5y"]] \ No newline at end of file diff --git a/tests/fixtures/rnfs-0.1.5/state/rnfs.db b/tests/fixtures/rnfs-0.1.5/state/rnfs.db new file mode 100644 index 0000000..a3b7f92 Binary files /dev/null and b/tests/fixtures/rnfs-0.1.5/state/rnfs.db differ diff --git a/tests/fixtures/rnfs-0.1.5/state/shared_folder.json.migrated b/tests/fixtures/rnfs-0.1.5/state/shared_folder.json.migrated new file mode 100644 index 0000000..2e6ca82 --- /dev/null +++ b/tests/fixtures/rnfs-0.1.5/state/shared_folder.json.migrated @@ -0,0 +1 @@ +{"entries": {"notes.txt": ["6baeb78e6b0bb791f691f2860e2fc0be964fc2d89cb04c82f1378c1f257c398e", "bafkreieeka3tp4k5fln7denhbcyl37wnnwy2b3eteebcer7ebbo5x44osi"]}} \ No newline at end of file diff --git a/tests/fixtures/wire/README.md b/tests/fixtures/wire/README.md new file mode 100644 index 0000000..94edc5c --- /dev/null +++ b/tests/fixtures/wire/README.md @@ -0,0 +1,27 @@ +# Wire known-answer vectors + +`vectors.json` holds what rnfs **0.1.3** and **0.1.4** put on the wire for fixed inputs, made by those releases' +own code: `/find`, `/provide`, `/blocks` and `/pin` requests and responses (and the `True`/`False` reply of +`/provide` and `/pin`), keepers' `/name` request and response, the `rnfs.dht` announce's mode, `rnfs.name` +records (sequence 0, 1 and 2^64 - 1), and the LXMF field carrying links. `tests/test_wire.py` checks that the +current code encodes the same inputs to the same bytes, and parses those bytes back to the same inputs. + +- `wire_cases.py`: the cases, and how each kind is encoded and parsed. It imports the rnfs tree it is given, so + the same file runs on the old releases and on the current code. +- `generate.py`: runs `wire_cases.py` on each tree given, in a subprocess with that tree first on `sys.path`, + checks the runs agree byte for byte, and writes `vectors.json` (one vector per line). + +Each vector is `{name, kind, inputs, hex}`. `hex` is `umsgpack.packb` of the request or response data only (RNS +owns the envelope); for announce app_data (the mode and name records) it is the bytes themselves. In `inputs`, a +byte string is `{"hex": ...}` and a CID `{"cid": }`. A name record's signature is Ed25519, which is +deterministic, so a fixed key (`NAME_KEY`, a throwaway for these vectors only) signs the same bytes each run. + +## Regenerating + +Only when adding cases: the bytes must never change. From the repository root: + + for v in v0.1.3 v0.1.4; do mkdir /tmp/rnfs-$v && git archive $v | tar -x -C /tmp/rnfs-$v; done + python3 tests/fixtures/wire/generate.py /tmp/rnfs-v0.1.3 /tmp/rnfs-v0.1.4 + +A new kind that 0.1.3 doesn't have can't be generated this way; give it its own generator run on the release that +introduced it. diff --git a/tests/fixtures/wire/generate.py b/tests/fixtures/wire/generate.py new file mode 100644 index 0000000..f411315 --- /dev/null +++ b/tests/fixtures/wire/generate.py @@ -0,0 +1,60 @@ +"""Write tests/fixtures/wire/vectors.json: the wire known-answer vectors (see README.md), from old releases' own +encoders. + +Each tree is run in a subprocess with that tree first on sys.path (and checked to be where rnfs comes from); the +runs must agree byte for byte, which also shows the wire didn't change between those releases. From the +repository root: + + for v in v0.1.3 v0.1.4; do mkdir /tmp/rnfs-$v && git archive $v | tar -x -C /tmp/rnfs-$v; done + python3 tests/fixtures/wire/generate.py /tmp/rnfs-v0.1.3 /tmp/rnfs-v0.1.4 + +Run with "--one TREE", it prints one tree's vectors as JSON instead. +""" +import json +import os +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def vectors(tree: str) -> list[dict]: + """Every case's vector from tree's own code (which must be first on sys.path).""" + import wire_cases + m = wire_cases.modules(tree + os.sep) + out = [] + for name, kind, inputs in wire_cases.cases(m): + data = wire_cases.wire(m, kind, inputs) + assert wire_cases.parse(m, kind, wire_cases.unwire(m, kind, data), inputs) == inputs, name + out.append({"name": name, "kind": kind, "inputs": wire_cases.to_json(inputs), "hex": data.hex()}) + return out + + +def main(): + """Run every tree, check they agree, write vectors.json with the first tree's version noted.""" + if sys.argv[1] == "--one": + tree = os.path.abspath(sys.argv[2]) + sys.path.insert(0, tree) + sys.path.insert(1, HERE) + import RNS + RNS.loglevel = RNS.LOG_CRITICAL + import rnfs + print(json.dumps({"version": rnfs.__version__, "vectors": vectors(tree)})) + return + runs = [] + for tree in sys.argv[1:]: + result = subprocess.run([sys.executable, os.path.abspath(__file__), "--one", tree], check=True, + capture_output=True, text=True) + runs.append(json.loads(result.stdout)) + for run in runs[1:]: + assert run["vectors"] == runs[0]["vectors"], f"{run['version']} differs from {runs[0]['version']}" + out = {"generated_by": [run["version"] for run in runs], "vectors": runs[0]["vectors"]} + with open(os.path.join(HERE, "vectors.json"), "w", encoding="utf-8") as f: # one vector per line + f.write('{"generated_by": %s, "vectors": [\n' % json.dumps(out["generated_by"])) + f.write(",\n".join(json.dumps(vector, ensure_ascii=False) for vector in out["vectors"])) + f.write("\n]}\n") + print(f"{len(out['vectors'])} vectors, identical from {', '.join(out['generated_by'])}") + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/wire/vectors.json b/tests/fixtures/wire/vectors.json new file mode 100644 index 0000000..e9ddb45 --- /dev/null +++ b/tests/fixtures/wire/vectors.json @@ -0,0 +1,29 @@ +{"generated_by": ["0.1.3", "0.1.4"], "vectors": [ +{"name": "find request", "kind": "find_request", "inputs": {"key": {"hex": "00112233445566778899aabbccddeeff"}}, "hex": "c41000112233445566778899aabbccddeeff"}, +{"name": "find response, empty", "kind": "find_response", "inputs": {"providers": [], "nodes": []}, "hex": "929090"}, +{"name": "find response, K nodes", "kind": "find_response", "inputs": {"providers": [], "nodes": [{"hex": "10101010101010101010101010101010"}, {"hex": "11111111111111111111111111111111"}, {"hex": "12121212121212121212121212121212"}, {"hex": "13131313131313131313131313131313"}, {"hex": "14141414141414141414141414141414"}, {"hex": "15151515151515151515151515151515"}, {"hex": "16161616161616161616161616161616"}, {"hex": "17171717171717171717171717171717"}, {"hex": "18181818181818181818181818181818"}, {"hex": "19191919191919191919191919191919"}]}, "hex": "92909ac41010101010101010101010101010101010c41011111111111111111111111111111111c41012121212121212121212121212121212c41013131313131313131313131313131313c41014141414141414141414141414141414c41015151515151515151515151515151515c41016161616161616161616161616161616c41017171717171717171717171717171717c41018181818181818181818181818181818c41019191919191919191919191919191919"}, +{"name": "find response, K providers and K nodes", "kind": "find_response", "inputs": {"providers": [{"hex": "40404040404040404040404040404040"}, {"hex": "41414141414141414141414141414141"}, {"hex": "42424242424242424242424242424242"}, {"hex": "43434343434343434343434343434343"}, {"hex": "44444444444444444444444444444444"}, {"hex": "45454545454545454545454545454545"}, {"hex": "46464646464646464646464646464646"}, {"hex": "47474747474747474747474747474747"}, {"hex": "48484848484848484848484848484848"}, {"hex": "49494949494949494949494949494949"}], "nodes": [{"hex": "10101010101010101010101010101010"}, {"hex": "11111111111111111111111111111111"}, {"hex": "12121212121212121212121212121212"}, {"hex": "13131313131313131313131313131313"}, {"hex": "14141414141414141414141414141414"}, {"hex": "15151515151515151515151515151515"}, {"hex": "16161616161616161616161616161616"}, {"hex": "17171717171717171717171717171717"}, {"hex": "18181818181818181818181818181818"}, {"hex": "19191919191919191919191919191919"}]}, "hex": "929ac41040404040404040404040404040404040c41041414141414141414141414141414141c41042424242424242424242424242424242c41043434343434343434343434343434343c41044444444444444444444444444444444c41045454545454545454545454545454545c41046464646464646464646464646464646c41047474747474747474747474747474747c41048484848484848484848484848484848c410494949494949494949494949494949499ac41010101010101010101010101010101010c41011111111111111111111111111111111c41012121212121212121212121212121212c41013131313131313131313131313131313c41014141414141414141414141414141414c41015151515151515151515151515151515c41016161616161616161616161616161616c41017171717171717171717171717171717c41018181818181818181818181818181818c41019191919191919191919191919191919"}, +{"name": "provide request", "kind": "provide_request", "inputs": {"keys": [{"hex": "00112233445566778899aabbccddeeff"}, {"hex": "ffeeddccbbaa99887766554433221100"}], "withdrawn": []}, "hex": "9292c41000112233445566778899aabbccddeeffc410ffeeddccbbaa9988776655443322110090"}, +{"name": "provide request, with withdrawn", "kind": "provide_request", "inputs": {"keys": [{"hex": "00112233445566778899aabbccddeeff"}], "withdrawn": [{"hex": "ffeeddccbbaa99887766554433221100"}]}, "hex": "9291c41000112233445566778899aabbccddeeff91c410ffeeddccbbaa99887766554433221100"}, +{"name": "provide request, only withdrawn", "kind": "provide_request", "inputs": {"keys": [], "withdrawn": [{"hex": "00112233445566778899aabbccddeeff"}, {"hex": "ffeeddccbbaa99887766554433221100"}]}, "hex": "929092c41000112233445566778899aabbccddeeffc410ffeeddccbbaa99887766554433221100"}, +{"name": "provide or pin reply, accepted", "kind": "accept_response", "inputs": {"accepted": true}, "hex": "c3"}, +{"name": "provide or pin reply, refused", "kind": "accept_response", "inputs": {"accepted": false}, "hex": "c2"}, +{"name": "blocks request, raw and DAG", "kind": "blocks_request", "inputs": {"max_bytes": 262144, "cids": [{"cid": "bafkreidknglcliqes4unda7xltg56zip5jmzv4t6sifwhzqjqrzroujyxa"}, {"cid": "bagaibqabciqc75srnyjyu74v36khiqzdwlfjnmsk5gyjk5bbqyrgxe2zckbayiq"}]}, "hex": "92ce0004000092c424015512206a699625a2049728d183f75ccddf650fea599af27e920b63e6098473175138b8c427018080c00112202ff6516e138a7f95df94744323b2ca96b24ae9b095742186226b935912820c22"}, +{"name": "blocks request, first block only", "kind": "blocks_request", "inputs": {"max_bytes": 0, "cids": [{"cid": "bagaibqabciqc75srnyjyu74v36khiqzdwlfjnmsk5gyjk5bbqyrgxe2zckbayiq"}]}, "hex": "920091c427018080c00112202ff6516e138a7f95df94744323b2ca96b24ae9b095742186226b935912820c22"}, +{"name": "blocks request, 4 MiB budget", "kind": "blocks_request", "inputs": {"max_bytes": 4194304, "cids": [{"cid": "bafkreidknglcliqes4unda7xltg56zip5jmzv4t6sifwhzqjqrzroujyxa"}]}, "hex": "92ce0040000091c424015512206a699625a2049728d183f75ccddf650fea599af27e920b63e6098473175138b8"}, +{"name": "blocks response, every block", "kind": "blocks_response", "inputs": {"requested": 2, "entries": [{"hex": "612072617720626c6f636b20666f7220746865207769726520766563746f72730a"}, {"hex": "920290"}]}, "hex": "92c421612072617720626c6f636b20666f7220746865207769726520766563746f72730ac403920290"}, +{"name": "blocks response, not held entries", "kind": "blocks_response", "inputs": {"requested": 3, "entries": [null, {"hex": "612072617720626c6f636b20666f7220746865207769726520766563746f72730a"}, null]}, "hex": "93c0c421612072617720626c6f636b20666f7220746865207769726520766563746f72730ac0"}, +{"name": "blocks response, cut by the budget", "kind": "blocks_response", "inputs": {"requested": 3, "entries": [{"hex": "920290"}]}, "hex": "91c403920290"}, +{"name": "pin request, pin", "kind": "pin_request", "inputs": {"cid": {"cid": "bafkreidknglcliqes4unda7xltg56zip5jmzv4t6sifwhzqjqrzroujyxa"}, "pin": true}, "hex": "92c424015512206a699625a2049728d183f75ccddf650fea599af27e920b63e6098473175138b8c3"}, +{"name": "pin request, release", "kind": "pin_request", "inputs": {"cid": {"cid": "bagaibqabciqc75srnyjyu74v36khiqzdwlfjnmsk5gyjk5bbqyrgxe2zckbayiq"}, "pin": false}, "hex": "92c427018080c00112202ff6516e138a7f95df94744323b2ca96b24ae9b095742186226b935912820c22c2"}, +{"name": "name request", "kind": "name_request", "inputs": {"name": {"hex": "4f8d9a364facd1c18716e77ea22e9e62"}}, "hex": "c4104f8d9a364facd1c18716e77ea22e9e62"}, +{"name": "name response, a record", "kind": "name_response", "inputs": {"name": {"hex": "4f8d9a364facd1c18716e77ea22e9e62"}, "record": {"public_key": {"hex": "8f40c5adb68f25624ae5b214ea767a6ec94d829d3d7b5e1ad1ba6f3e2138285f29acbae141bccaf0b22e1a94d34d0bc7361e526d0bfe12c89794bc9322966dd7"}, "random_hash": {"hex": "a1a2a3a4a5006ad09610"}, "signature": {"hex": "906023fd2c63ddb9801cb321d192e9e2298e715d2b1530e676736225ba8ae823688b6845f7a0fcbedfb4777a9f9ef3f902bfd24ad37e7bd25505eb548732e302"}, "app_data": {"hex": "9207c427018080c00112202ff6516e138a7f95df94744323b2ca96b24ae9b095742186226b935912820c22"}, "sequence": 7, "cid": {"cid": "bagaibqabciqc75srnyjyu74v36khiqzdwlfjnmsk5gyjk5bbqyrgxe2zckbayiq"}}}, "hex": "96c4408f40c5adb68f25624ae5b214ea767a6ec94d829d3d7b5e1ad1ba6f3e2138285f29acbae141bccaf0b22e1a94d34d0bc7361e526d0bfe12c89794bc9322966dd7c40a783c4f3005a78c29de83c40aa1a2a3a4a5006ad09610c400c440906023fd2c63ddb9801cb321d192e9e2298e715d2b1530e676736225ba8ae823688b6845f7a0fcbedfb4777a9f9ef3f902bfd24ad37e7bd25505eb548732e302c42b9207c427018080c00112202ff6516e138a7f95df94744323b2ca96b24ae9b095742186226b935912820c22"}, +{"name": "name response, none kept", "kind": "name_response", "inputs": {"name": {"hex": "4f8d9a364facd1c18716e77ea22e9e62"}, "record": null}, "hex": "90"}, +{"name": "announce mode, server", "kind": "announce_mode", "inputs": {"server": true}, "hex": "01"}, +{"name": "announce mode, client", "kind": "announce_mode", "inputs": {"server": false}, "hex": "00"}, +{"name": "name record, sequence 0", "kind": "name_record", "inputs": {"sequence": 0, "cid": {"cid": "bafkreidknglcliqes4unda7xltg56zip5jmzv4t6sifwhzqjqrzroujyxa"}}, "hex": "9200c424015512206a699625a2049728d183f75ccddf650fea599af27e920b63e6098473175138b8"}, +{"name": "name record, sequence 1", "kind": "name_record", "inputs": {"sequence": 1, "cid": {"cid": "bagaibqabciqc75srnyjyu74v36khiqzdwlfjnmsk5gyjk5bbqyrgxe2zckbayiq"}}, "hex": "9201c427018080c00112202ff6516e138a7f95df94744323b2ca96b24ae9b095742186226b935912820c22"}, +{"name": "name record, sequence 2^64 - 1", "kind": "name_record", "inputs": {"sequence": 18446744073709551615, "cid": {"cid": "bagaibqabciqc75srnyjyu74v36khiqzdwlfjnmsk5gyjk5bbqyrgxe2zckbayiq"}}, "hex": "92cfffffffffffffffffc427018080c00112202ff6516e138a7f95df94744323b2ca96b24ae9b095742186226b935912820c22"}, +{"name": "lxmf field, one link", "kind": "lxmf_field", "inputs": {"links": [{"cid": {"cid": "bafkreidknglcliqes4unda7xltg56zip5jmzv4t6sifwhzqjqrzroujyxa"}, "name": null, "path": [], "size": null, "from": null}]}, "hex": "82ccfba4726e6673ccfc9181a3636964c424015512206a699625a2049728d183f75ccddf650fea599af27e920b63e6098473175138b8"}, +{"name": "lxmf field, several links", "kind": "lxmf_field", "inputs": {"links": [{"cid": {"cid": "bagaibqabciqc75srnyjyu74v36khiqzdwlfjnmsk5gyjk5bbqyrgxe2zckbayiq"}, "name": null, "path": ["photos", "été.jpg"], "size": 123456, "from": {"hex": "10101010101010101010101010101010"}}, {"cid": {"cid": "bagaibqabciqc75srnyjyu74v36khiqzdwlfjnmsk5gyjk5bbqyrgxe2zckbayiq"}, "name": {"hex": "4f8d9a364facd1c18716e77ea22e9e62"}, "path": ["2026"], "size": null, "from": null}, {"cid": null, "name": {"hex": "4f8d9a364facd1c18716e77ea22e9e62"}, "path": [], "size": null, "from": null}, {"cid": {"cid": "bafkreidknglcliqes4unda7xltg56zip5jmzv4t6sifwhzqjqrzroujyxa"}, "name": null, "path": [], "size": 0, "from": null}]}, "hex": "82ccfba4726e6673ccfc9484a3636964c427018080c00112202ff6516e138a7f95df94744323b2ca96b24ae9b095742186226b935912820c22a47061746892a670686f746f73a9c3a974c3a92e6a7067a473697a65ce0001e240a466726f6dc4101010101010101010101010101010101083a3636964c427018080c00112202ff6516e138a7f95df94744323b2ca96b24ae9b095742186226b935912820c22a46e616d65c4104f8d9a364facd1c18716e77ea22e9e62a47061746891a43230323681a46e616d65c4104f8d9a364facd1c18716e77ea22e9e6282a3636964c424015512206a699625a2049728d183f75ccddf650fea599af27e920b63e6098473175138b8a473697a6500"} +]} diff --git a/tests/fixtures/wire/wire_cases.py b/tests/fixtures/wire/wire_cases.py new file mode 100644 index 0000000..acf6dda --- /dev/null +++ b/tests/fixtures/wire/wire_cases.py @@ -0,0 +1,221 @@ +"""The wire known-answer cases (specification sections 3, 4, 7, 9 and 13), shared by generate.py, which runs them on +the v0.1.3 and v0.1.4 trees, and tests/test_wire.py, which runs them on the current code. + +Each case is (name, kind, inputs). encode() turns a kind's inputs into what goes on the wire, with that tree's own +encoders, and parse() turns it back with that tree's own parsers; parse(encode(inputs)) must give inputs again. +For a request or response, the vector is umsgpack.packb of the request or response data only (RNS owns the +envelope); an announce's app_data (the DHT mode, a name record) is the bytes themselves. + +Inputs are fixed: node IDs, keys, CIDs (raw and DAG) and a name key. A name record's signature is Ed25519, which +is deterministic, so every run signs the same bytes. Imports nothing from rnfs itself: the caller passes the +modules of the tree under test (modules()). +""" +import hashlib +import types + +# Kinds whose wire form is bytes (announce app_data), not msgpack-packed data +RAW_KINDS = ("announce_mode", "name_record") + +KEY = bytes.fromhex("00112233445566778899aabbccddeeff") +OTHER_KEY = bytes.fromhex("ffeeddccbbaa99887766554433221100") +NODES = [bytes([0x10 + i]) * 16 for i in range(10)] # K = 10 +PROVIDERS = [bytes([0x40 + i]) * 16 for i in range(10)] +RAW_BLOCK = b"a raw block for the wire vectors\n" +DAG_BLOCK = b"\x92\x02\x90" # msgpack [2, []]: the empty directory +# The name key: 32 bytes of X25519 private key, then a 32-byte Ed25519 seed. A throwaway key for these vectors only +NAME_KEY = bytes(range(64)) +EMITTED = 1792054800 # the emission time in the record's random hash +RANDOM_HASH = bytes.fromhex("a1a2a3a4a5") + EMITTED.to_bytes(5, "big") +NAME_HASH = hashlib.sha256(b"rnfs.name").digest()[:10] + + +def modules(rnfs_root: str | None = None) -> types.SimpleNamespace: + """The modules of the rnfs tree on sys.path that the cases use (rnfs_root: check they come from there).""" + import RNS + from RNS.vendor import umsgpack + + import rnfs + from rnfs import constants, names + from rnfs.content import cid + from rnfs.features import keepers, lxmf + from rnfs.network import announces, protocol + if rnfs_root is not None: + assert rnfs.__file__.startswith(rnfs_root), rnfs.__file__ + return types.SimpleNamespace(RNS=RNS, umsgpack=umsgpack, rnfs=rnfs, constants=constants, names=names, cid=cid, + keepers=keepers, lxmf=lxmf, announces=announces, protocol=protocol) + + +def cases(m) -> list[tuple[str, str, dict]]: + """Every case, with typed inputs (bytes, the tree's Cid).""" + Cid = m.cid.Cid + raw, dag = Cid.for_block(m.cid.RAW, RAW_BLOCK), Cid.for_block(m.cid.DAG, DAG_BLOCK) + identity = m.RNS.Identity.from_bytes(NAME_KEY) + name = m.RNS.Destination.hash(identity, "rnfs", "name") + public_key = identity.get_public_key() + + def signed(sequence: int, target) -> dict: + """A signed name record's fields, as a /name response carries them (no ratchet, as rnfs announces).""" + app_data = m.names.encode_record(sequence, target) + signature = identity.sign(name + public_key + NAME_HASH + RANDOM_HASH + app_data) + return {"public_key": public_key, "random_hash": RANDOM_HASH, "signature": signature, "app_data": app_data, + "sequence": sequence, "cid": target} + + def link(**fields) -> dict: + """An LXMF link's fields, every one given.""" + return {"cid": None, "name": None, "path": [], "size": None, "from": None, **fields} + + return [ + # /find + ("find request", "find_request", {"key": KEY}), + ("find response, empty", "find_response", {"providers": [], "nodes": []}), + ("find response, K nodes", "find_response", {"providers": [], "nodes": NODES}), + ("find response, K providers and K nodes", "find_response", {"providers": PROVIDERS, "nodes": NODES}), + # /provide + ("provide request", "provide_request", {"keys": [KEY, OTHER_KEY], "withdrawn": []}), + ("provide request, with withdrawn", "provide_request", {"keys": [KEY], "withdrawn": [OTHER_KEY]}), + ("provide request, only withdrawn", "provide_request", {"keys": [], "withdrawn": [KEY, OTHER_KEY]}), + ("provide or pin reply, accepted", "accept_response", {"accepted": True}), + ("provide or pin reply, refused", "accept_response", {"accepted": False}), + # /blocks + ("blocks request, raw and DAG", "blocks_request", {"max_bytes": 262144, "cids": [raw, dag]}), + ("blocks request, first block only", "blocks_request", {"max_bytes": 0, "cids": [dag]}), + ("blocks request, 4 MiB budget", "blocks_request", {"max_bytes": 4 * 1024 * 1024, "cids": [raw]}), + ("blocks response, every block", "blocks_response", {"requested": 2, "entries": [RAW_BLOCK, DAG_BLOCK]}), + ("blocks response, not held entries", "blocks_response", + {"requested": 3, "entries": [None, RAW_BLOCK, None]}), + ("blocks response, cut by the budget", "blocks_response", {"requested": 3, "entries": [DAG_BLOCK]}), + # /pin + ("pin request, pin", "pin_request", {"cid": raw, "pin": True}), + ("pin request, release", "pin_request", {"cid": dag, "pin": False}), + # keepers' /name + ("name request", "name_request", {"name": name}), + ("name response, a record", "name_response", {"name": name, "record": signed(7, dag)}), + ("name response, none kept", "name_response", {"name": name, "record": None}), + # rnfs.dht announce app_data + ("announce mode, server", "announce_mode", {"server": True}), + ("announce mode, client", "announce_mode", {"server": False}), + # rnfs.name announce app_data + ("name record, sequence 0", "name_record", {"sequence": 0, "cid": raw}), + ("name record, sequence 1", "name_record", {"sequence": 1, "cid": dag}), + ("name record, sequence 2^64 - 1", "name_record", {"sequence": 2 ** 64 - 1, "cid": dag}), + # LXMF field + ("lxmf field, one link", "lxmf_field", {"links": [link(cid=raw)]}), + ("lxmf field, several links", "lxmf_field", {"links": [ + link(cid=dag, path=["photos", "été.jpg"], size=123456, **{"from": NODES[0]}), + link(name=name, cid=dag, path=["2026"]), + link(name=name), + link(cid=raw, size=0), + ]}), + ] + + +def encode(m, kind: str, inputs: dict): + """What goes on the wire for a case: the request or response data (packed by the caller), or app_data bytes.""" + p = m.protocol + if kind == "find_request": + return p.encode_find(inputs["key"]) + if kind == "find_response": + return p.encode_find_response(inputs["providers"], inputs["nodes"]) + if kind == "provide_request": + return p.encode_provide(inputs["keys"], inputs["withdrawn"]) + if kind == "accept_response": + return bool(inputs["accepted"]) # /provide and /pin handlers answer True or False + if kind == "blocks_request": + return p.encode_blocks(inputs["max_bytes"], inputs["cids"]) + if kind == "blocks_response": + return p.encode_blocks_response(inputs["entries"]) + if kind == "pin_request": + return p.encode_pin(inputs["cid"], inputs["pin"]) + if kind == "name_request": + return m.keepers.encode_request(inputs["name"]) + if kind == "name_response": + r = inputs["record"] + record = None if r is None else m.keepers.SignedRecord( + inputs["name"], r["public_key"], r["random_hash"], b"", r["signature"], r["app_data"], r["sequence"], + r["cid"]) + return m.keepers.encode_response(record) + if kind == "announce_mode": + return m.announces.encode_mode(inputs["server"]) + if kind == "name_record": + return m.names.encode_record(inputs["sequence"], inputs["cid"]) + if kind == "lxmf_field": + return m.lxmf.encode_field([m.lxmf.Link(cid=f["cid"], name=f["name"], path=tuple(f["path"]), size=f["size"], + from_node=f["from"]) for f in inputs["links"]]) + raise ValueError(kind) + + +def parse(m, kind: str, data, inputs: dict) -> dict: + """A case's inputs, back from its wire data with the tree's parsers. inputs supplies only what a parser is + given besides the data (the CIDs requested, the name asked for).""" + p = m.protocol + if kind == "find_request": + return {"key": p.parse_find_request(data)} + if kind == "find_response": + providers, nodes = p.parse_find_response(data) + return {"providers": providers, "nodes": nodes} + if kind == "provide_request": + keys, withdrawn = p.parse_provide_request(data) + return {"keys": keys, "withdrawn": withdrawn} + if kind == "accept_response": + return {"accepted": p.is_accepted(data)} + if kind == "blocks_request": + max_bytes, cids = p.parse_blocks_request(data) + return {"max_bytes": max_bytes, "cids": cids} + if kind == "blocks_response": + return {"requested": inputs["requested"], "entries": p.parse_blocks_response(data, inputs["requested"])} + if kind == "pin_request": + cid, pin = p.parse_pin_request(data) + return {"cid": cid, "pin": pin} + if kind == "name_request": + return {"name": m.keepers.parse_request(data)} + if kind == "name_response": + record = m.keepers.parse_response(inputs["name"], data) + return {"name": inputs["name"], "record": None if record is None else { + "public_key": record.public_key, "random_hash": record.random_hash, "signature": record.signature, + "app_data": record.app_data, "sequence": record.sequence, "cid": record.cid}} + if kind == "announce_mode": + return {"server": m.announces.parse_mode(data) == m.constants.MODE_SERVER} + if kind == "name_record": + sequence, cid = m.names.parse_record(data) + return {"sequence": sequence, "cid": cid} + if kind == "lxmf_field": + return {"links": [{"cid": link.cid, "name": link.name, "path": list(link.path), "size": link.size, + "from": link.from_node} for link in m.lxmf.decode_field(data)]} + raise ValueError(kind) + + +def wire(m, kind: str, inputs: dict) -> bytes: + """The vector's bytes: app_data as is, anything else packed as RNS packs request and response data.""" + data = encode(m, kind, inputs) + return data if kind in RAW_KINDS else m.umsgpack.packb(data) + + +def unwire(m, kind: str, data: bytes): + """wire()'s inverse, before parse().""" + return data if kind in RAW_KINDS else m.umsgpack.unpackb(data) + + +def to_json(value): + """Inputs as JSON: bytes as {"hex": ...}, a CID as {"cid": its text}.""" + if isinstance(value, bytes): + return {"hex": value.hex()} + if isinstance(value, dict): + return {key: to_json(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [to_json(item) for item in value] + if value is None or isinstance(value, (bool, int, str)): + return value + return {"cid": str(value)} # a Cid (the only other type the cases hold) + + +def from_json(m, value): + """to_json()'s inverse, with the tree's Cid.""" + if isinstance(value, dict): + if set(value) == {"hex"}: + return bytes.fromhex(value["hex"]) + if set(value) == {"cid"}: + return m.cid.Cid.parse(value["cid"]) + return {key: from_json(m, item) for key, item in value.items()} + if isinstance(value, list): + return [from_json(m, item) for item in value] + return value diff --git a/tests/interop.py b/tests/interop.py new file mode 100644 index 0000000..7652427 --- /dev/null +++ b/tests/interop.py @@ -0,0 +1,578 @@ +#!/usr/bin/env python3 +"""Cross-version interop: daemons of this tree against daemons of older releases, over local TCP. + +Not a unittest module (discovery runs test_*.py only) and not run by the suite: a script run by hand before a +release (the release checklist's "Interop" item). For each old tree given with --old (a `git archive` extract of a +release tag), it starts a network like tests/test_e2e.py's, a hub with transport enabled and nodes connected to +it, where every daemon runs `python3 /rnfs.py daemon` with its tree first on PYTHONPATH and is driven by +that tree's own CLI: + +- hub this tree; the TCP server and the bootstrap node. +- old/new one node of each version, default settings. +- old-cap/new-cap one node of each version with max_upload_rate set (CAP_RATE), the serving side of the paced + transfer, and later the name publishers of the keeper scenario. + +Every scenario runs in both directions, "A -> B" meaning A serves (or publishes, or sends) and B fetches (or +resolves, or receives): + +- version each daemon's `rnfs --version`, /api/status and imported rnfs come from the tree it was given. +- providers B finds A among the providers of a CID A added. +- fetch a multi-chunk file and a directory tree added on A: `get`, `cat --offset --length` and a path's + `cat` on B give the same bytes, execute bits included. +- bulk a BULK_SIZE file with default settings: identical bytes, and no refusals, failures or bad blocks + in B's counters for A, no fetch failures in B's log and no "refused" lines in A's (NET-1). +- small the same checks for a tree of SMALL_FILES small files, whose /blocks responses fit one packet each + (a local TCP link's MTU is large). +- bulk-cap, bulk and small from the *-cap node of A's version, whose max_upload_rate paces every /blocks + small-cap response, single-packet ones included since 0.1.6 (NET-2): an honest fetcher must see no refusals. +- names a name published on A resolves on B, and B's gateway serves a file through it. +- keepers publisher -> keeper -> resolver across versions (old-cap -> new -> a fresh old node, new-cap -> old + -> a fresh new node): the keeper keeps the name, the publisher stops, then the resolver starts (so + it never heard the name's announce) and must get the signed record by asking keepers, its log + showing the keeper query that returned it. +- lxmf A sends B a link over LXMF; it arrives in B's inbox, A's outbox says delivered, and `inbox get` + fetches it. + +Usage, from a checkout (the Python that runs rnfs, with RNS and LXMF installed): + + git archive v0.1.5 | (mkdir -p /tmp/rnfs-0.1.5 && tar -x -C /tmp/rnfs-0.1.5) + git archive v0.1.4 | (mkdir -p /tmp/rnfs-0.1.4 && tar -x -C /tmp/rnfs-0.1.4) + python3 tests/interop.py --old /tmp/rnfs-0.1.5 --old /tmp/rnfs-0.1.4 + +It prints a PASS/FAIL table per old tree and exits non-zero on any FAIL. Each old tree takes about five minutes. +Daemon directories and logs go to a temporary directory, kept (and named) when something fails or with --keep. +""" +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import traceback +import urllib.error +import urllib.request + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # run from a plain checkout + +from tests import support +from tests.test_e2e import HUB_CONFIG, INTERRUPT, NODE_CONFIG, POPEN_OPTIONS, SETTINGS, WINDOWS, same_tree + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +KIB, MIB = 1024, 1024 * 1024 +BULK_SIZE = 3 * MIB // 2 +SMALL_FILES, SMALL_SIZE = 64, 4 * KIB +CAP_RATE = 2_000_000 # bits/s: max_upload_rate on the *-cap nodes, so BULK_SIZE takes seconds (4 to 6) +# Lines in a fetching node's log that mean a provider let it down, and in a serving node's log that mean it +# refused (or cooled down) a requester. Both need RNS's loglevel at 5 (verbose), which every node here runs at +FETCHER_TROUBLE = re.compile(r"rnfs .*(failed for |banning |every provider of |refus|cooldown)", re.IGNORECASE) +SERVER_TROUBLE = re.compile(r"rnfs .*(refus|cooldown|no progress)", re.IGNORECASE) + + +def wait_for(condition, timeout: float, what: str): + """support.wait_for, polling every half second (each poll runs the CLI or calls an API).""" + return support.wait_for(condition, timeout, what, interval=0.5) + + +def tree_version(tree: str) -> str: + """The __version__ a source tree's rnfs/__init__.py declares.""" + with open(os.path.join(tree, "rnfs", "__init__.py"), encoding="utf-8") as f: + match = re.search(r'(?m)^__version__ = "([^"]+)"', f.read()) + if match is None: + raise SystemExit(f"{tree}: no __version__ in rnfs/__init__.py") + return match.group(1) + + +def current_label() -> str: + """This tree's name in the tables: its git commit, and its version.""" + try: + commit = subprocess.run(["git", "-C", ROOT, "rev-parse", "--short", "HEAD"], capture_output=True, + text=True, timeout=10).stdout.strip() + except OSError: + commit = "" + return f"current ({commit or 'checkout'}, {tree_version(ROOT)})" + + +class Daemon: + """One running daemon of one source tree: its directory, ports, node ID and log.""" + + def __init__(self, name: str, tree: str, label: str, base: str, config: str, tcp_port: int, + settings: str = SETTINGS, *extra: str): + """Start the daemon and wait for its API.""" + self.name, self.tree, self.label = name, tree, label + self.main = os.path.join(tree, "rnfs.py") + self.dir = os.path.join(base, name) + os.makedirs(self.dir, exist_ok=True) + # loglevel 5 (verbose): rnfs logs refusals, cooldowns and failed providers at that level + with open(os.path.join(self.dir, "config"), "w", encoding="utf-8") as f: + f.write(config.format(port=tcp_port).replace("loglevel = 4", "loglevel = 5")) + with open(os.path.join(self.dir, "rnfs.toml"), "w", encoding="utf-8") as f: + f.write(settings) + self.env = {**os.environ, "PYTHONUNBUFFERED": "1", "PYTHONPATH": tree} + self.api_port = support.free_port() + self.gateway_port = support.free_port() + self.log_path = os.path.join(base, f"{name}.log") + with open(self.log_path, "wb") as log: + self.proc = subprocess.Popen( + [sys.executable, self.main, "--dir", self.dir, "--api-port", str(self.api_port), "daemon", + "--rnsconfig", self.dir, "--gateway-port", str(self.gateway_port), *extra], + stdout=log, stderr=subprocess.STDOUT, env=self.env, cwd=base, **POPEN_OPTIONS) + self.node_id = wait_for(self.status, 60, f"{name} daemon to start")["node_id"] + + def status(self) -> dict | None: + """The daemon's status (with its API token), or None while its API is not up yet.""" + try: + with open(os.path.join(self.dir, "api_token"), encoding="ascii") as f: + token = f.read().strip() + request = urllib.request.Request(f"http://127.0.0.1:{self.api_port}/api/status", + headers={"X-Rnfs-Api": token}) + with urllib.request.urlopen(request, timeout=2) as response: + return json.load(response) + except OSError: + if self.proc.poll() is not None: + raise AssertionError(f"{self.name} daemon exited") from None + return None + + def run(self, *args: str, timeout: float = 300) -> subprocess.CompletedProcess: + """This tree's CLI with args, as is (no --dir).""" + return subprocess.run([sys.executable, self.main, *args], capture_output=True, timeout=timeout, + env=self.env, cwd=os.path.dirname(self.dir)) + + def cli(self, *args: str, check: bool = True) -> subprocess.CompletedProcess: + """A CLI command against this daemon; with check, fail unless it succeeds.""" + result = self.run("--dir", self.dir, "--api-port", str(self.api_port), *args) + if check and result.returncode != 0: + raise AssertionError(f"{self.name}: {' '.join(args)} failed: " + f"{result.stderr.decode('utf-8', 'replace').strip()}") + return result + + def out(self, *args: str) -> str: + """A successful command's stdout, stripped.""" + return self.cli(*args).stdout.decode("utf-8", "replace").strip() + + def providers(self, cid: str) -> list[str]: + """The provider IDs this node finds for cid (none if the lookup fails).""" + result = self.cli("providers", cid, check=False) + return result.stdout.decode("utf-8", "replace").split() if result.returncode == 0 else [] + + def peer(self, node_id: str) -> dict: + """This node's traffic counters for node_id (all zero if it has none).""" + for peer in json.loads(self.out("peers", "--json")): + if peer.get("node_id") == node_id: + return peer + return {} + + def log_size(self) -> int: + """The log's length now, to read only what is logged after it.""" + return os.path.getsize(self.log_path) + + def log_since(self, offset: int) -> list[str]: + """The log's lines from offset on.""" + with open(self.log_path, "rb") as f: + f.seek(offset) + return f.read().decode("utf-8", "replace").splitlines() + + def gateway(self, path: str) -> tuple[int, bytes]: + """(status, body) of a gateway GET.""" + try: + with urllib.request.urlopen(f"http://127.0.0.1:{self.gateway_port}{path}", timeout=120) as response: + return response.status, response.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + + def interrupt(self): + """Ask the daemon to shut down, as Ctrl+C would, if it is still running.""" + if self.proc.poll() is not None: + return + try: + self.proc.send_signal(INTERRUPT) + except OSError: + self.proc.terminate() + + def stop(self): + """Interrupt like Ctrl-C; kill if it doesn't exit in time.""" + self.interrupt() + try: + self.proc.wait(timeout=20) + except subprocess.TimeoutExpired: + self.proc.kill() + self.proc.wait() + + +class Network: + """A hub and the four nodes of one run (this tree against one old tree), and the scenarios run on it.""" + + def __init__(self, old_tree: str, base: str): + """Nothing started yet; old_tree is the old release's source tree, base the run's directory.""" + self.old_tree = os.path.abspath(old_tree) + self.old_label = f"v{tree_version(self.old_tree)}" + self.new_label = current_label() + self.base = base + self.daemons: list[Daemon] = [] + self.results: list[tuple[str, str, bool, str]] = [] + self.counter = 0 + + # Running + + def start(self): + """Start the hub, then the four nodes, all bootstrapping from the hub.""" + self.tcp_port = tcp_port = support.free_port() + capped = SETTINGS + f"max_upload_rate = {CAP_RATE}\n" + self.hub = self.daemon("hub", ROOT, self.new_label, HUB_CONFIG, tcp_port) + bootstrap = ("--bootstrap", self.hub.node_id) + self.old = self.daemon("old", self.old_tree, self.old_label, NODE_CONFIG, tcp_port, SETTINGS, *bootstrap) + self.new = self.daemon("new", ROOT, self.new_label, NODE_CONFIG, tcp_port, SETTINGS, *bootstrap) + self.old_cap = self.daemon("old-cap", self.old_tree, self.old_label, NODE_CONFIG, tcp_port, capped, + *bootstrap) + self.new_cap = self.daemon("new-cap", ROOT, self.new_label, NODE_CONFIG, tcp_port, capped, *bootstrap) + + def daemon(self, name: str, tree: str, label: str, config: str, tcp_port: int, settings: str = SETTINGS, + *extra: str) -> Daemon: + """Start one daemon, remembered for stopping.""" + daemon = Daemon(name, tree, label, self.base, config, tcp_port, settings, *extra) + self.daemons.append(daemon) + return daemon + + def stop_all(self): + """Interrupt every daemon at once, then wait for each (killing any that hangs).""" + for daemon in self.daemons: + daemon.interrupt() + for daemon in self.daemons: + daemon.stop() + + def check(self, scenario: str, direction: str, step, *args): + """Run one scenario step, recording PASS (with the detail it returns) or FAIL (with the error).""" + print(f"--- {scenario}: {direction}", flush=True) + started = time.monotonic() + try: + detail = step(*args) or "" + ok = True + except Exception as e: + detail = f"{type(e).__name__}: {e}" + if not isinstance(e, AssertionError): + detail += "\n" + traceback.format_exc() + ok = False + detail = f"{detail} [{time.monotonic() - started:.0f}s]".strip() + print(f" {'PASS' if ok else 'FAIL'} {detail}", flush=True) + self.results.append((scenario, direction, ok, detail)) + return ok + + def run(self): + """Every scenario, in both directions.""" + pairs = [(self.old, self.new), (self.new, self.old)] + self.check("version", "every daemon", self.versions) + added = {} + + def add(a: Daemon) -> str: + added[a.name] = self.add_content(a) + return f"file {added[a.name]['file']}, dir {added[a.name]['dir']}" + + for a, _ in pairs: + self.check("add", a.label.split(" (")[0], add, a) + for a, b in pairs: + self.check("providers", self.direction(a, b), self.providers, a, b, added[a.name]) + for a, b in pairs: + self.check("fetch", self.direction(a, b), self.fetch, a, b, added[a.name]) + for a, b in pairs: + self.check("bulk", self.direction(a, b), self.bulk, a, b) + self.check("small", self.direction(a, b), self.small, a, b) + # small-cap first: on a link with no rate estimate yet, only 0.1.6 paces single-packet responses (NET-2) + for a, b in [(self.old_cap, self.new), (self.new_cap, self.old)]: + self.check("small-cap", self.direction(a, b), self.small, a, b) + self.check("bulk-cap", self.direction(a, b), self.bulk, a, b) + for a, b in pairs: + self.check("names", self.direction(a, b), self.names, a, b, added[a.name]) + for a, b in pairs: + self.check("lxmf", self.direction(a, b), self.lxmf, a, b) + # Last: the publishers (the *-cap nodes) are stopped on the way + for publisher, keeper, resolver in [(self.old_cap, self.new, self.old), (self.new_cap, self.old, self.new)]: + self.check("keepers", self.direction(publisher, keeper, resolver), self.keepers, publisher, keeper, + resolver.tree, resolver.label) + + def direction(self, *daemons: Daemon) -> str: + """The direction's label: each daemon's version, in order.""" + return " -> ".join(daemon.label.split(" (")[0] for daemon in daemons) + + def path(self, *parts: str) -> str: + """A path in the run's directory.""" + return os.path.join(self.base, *parts) + + def unique(self, prefix: str) -> str: + """A path in the run's directory not used before.""" + self.counter += 1 + return self.path(f"{prefix}-{self.counter}") + + def random_file(self, size: int) -> tuple[str, bytes]: + """A new file of size random bytes: (path, content).""" + content = os.urandom(size) + path = self.unique("data") + with open(path, "wb") as f: + f.write(content) + return path, content + + # Scenarios + + def versions(self) -> str: + """Each daemon runs its own tree's code and reports that tree's version.""" + for daemon in self.daemons: + expected = tree_version(daemon.tree) + reported = daemon.status()["version"] + cli = daemon.run("--version").stdout.decode().strip() + probe = subprocess.run([sys.executable, "-c", "import rnfs; print(rnfs.__file__)"], capture_output=True, + text=True, env=daemon.env, cwd=self.base, timeout=60).stdout.strip() + if reported != expected or cli != f"rnfs {expected}": + raise AssertionError(f"{daemon.name} ({daemon.tree}): status says {reported!r}, --version says " + f"{cli!r}, expected {expected}") + if os.path.commonpath([os.path.realpath(probe), os.path.realpath(daemon.tree)]) != \ + os.path.realpath(daemon.tree): + raise AssertionError(f"{daemon.name}: rnfs imports from {probe}, not {daemon.tree}") + return ", ".join(f"{daemon.name}={daemon.status()['version']}" for daemon in self.daemons) + + def add_content(self, a: Daemon) -> dict: + """A multi-chunk file and a directory tree, added on a.""" + path, content = self.random_file(300 * KIB) + root = self.unique(f"tree-{a.name}") + os.makedirs(os.path.join(root, "sub", "deeper")) + with open(os.path.join(root, "big.bin"), "wb") as f: + f.write(os.urandom(200 * KIB)) + with open(os.path.join(root, "index.txt"), "wb") as f: + f.write(f"hello from {a.label}\n".encode()) + script = os.path.join(root, "sub", "run.sh") + with open(script, "wb") as f: + f.write(b"#!/bin/sh\necho hi\n") + os.chmod(script, 0o755) + with open(os.path.join(root, "sub", "deeper", "note.txt"), "wb") as f: + f.write(b"deep inside") + return {"file": a.out("add", path), "content": content, "dir": a.out("add", root), "tree": root} + + def providers(self, a: Daemon, b: Daemon, added: dict) -> str: + """b finds a providing the file and the directory.""" + wait_for(lambda: a.node_id in b.providers(added["file"]), 90, f"{b.name} to find {a.name} providing a file") + wait_for(lambda: a.node_id in b.providers(added["dir"]), 90, f"{b.name} to find {a.name} providing a dir") + return "file and directory" + + def fetch(self, a: Daemon, b: Daemon, added: dict) -> str: + """b gets and cats what a added, byte for byte.""" + out = self.unique("got") + b.cli("get", added["file"], "-o", out) + with open(out, "rb") as f: + if f.read() != added["content"]: + raise AssertionError("get: the file's bytes differ") + out = self.unique("got-tree") + b.cli("get", added["dir"], "-o", out) + if not same_tree(added["tree"], out): + raise AssertionError("get: the directory tree differs") + with open(os.path.join(added["tree"], "big.bin"), "rb") as f: + big = f.read() + part = b.cli("cat", f"{added['dir']}/big.bin", "--offset", "70000", "--length", "100000").stdout + if part != big[70000:170000]: + raise AssertionError("cat --offset --length: the bytes differ") + if b.cli("cat", f"{added['dir']}/sub/deeper/note.txt").stdout != b"deep inside": + raise AssertionError("cat /: the bytes differ") + if not WINDOWS: + out = self.unique("got-run") + b.cli("get", f"{added['dir']}/sub/run.sh", "-o", out) + if not os.stat(out).st_mode & 0o100: + raise AssertionError("get /: the execute bit is lost") + return "get file, get dir, cat range, cat path" + + def bulk(self, a: Daemon, b: Daemon) -> str: + """b gets a BULK_SIZE file from a, with no refusal, failure or bad block on either side.""" + path, content = self.random_file(BULK_SIZE) + cid = a.out("add", path) + + def check(out): + with open(out, "rb") as f: + if f.read() != content: + raise AssertionError("the bytes differ") + + return self.honest_transfer(a, b, cid, check, BULK_SIZE, f"{BULK_SIZE / MIB:.1f} MiB") + + def small(self, a: Daemon, b: Daemon) -> str: + """b gets a tree of SMALL_FILES small files from a, with no refusal, failure or bad block on either side. + + Over local TCP a link's MTU is large, so these responses go as single packets: the ones NET-2 paces when + max_upload_rate is set.""" + root = self.unique(f"small-{a.name}") + for i in range(SMALL_FILES): + os.makedirs(os.path.join(root, f"d{i % 4}"), exist_ok=True) + with open(os.path.join(root, f"d{i % 4}", f"f{i}.bin"), "wb") as f: + f.write(os.urandom(SMALL_SIZE)) + cid = a.out("add", root) + + def check(out): + if not same_tree(root, out): + raise AssertionError("the directory tree differs") + + return self.honest_transfer(a, b, cid, check, SMALL_FILES * SMALL_SIZE, + f"{SMALL_FILES} files of {SMALL_SIZE // KIB} KiB") + + def honest_transfer(self, a: Daemon, b: Daemon, cid: str, check, size: int, what: str) -> str: + """b gets cid from a, which check(path) then checks; no refusal, failure or bad block may show in b's + counters for a, b's log or a's log (NET-1, NET-2).""" + wait_for(lambda: a.node_id in b.providers(cid), 90, f"{b.name} to find {a.name} providing {cid}") + before = b.peer(a.node_id) + b_log, a_log = b.log_size(), a.log_size() + out = self.unique("transfer") + started = time.monotonic() + b.cli("get", cid, "-o", out) + took = time.monotonic() - started + after = b.peer(a.node_id) + check(out) + counts = {key: after.get(key, 0) - before.get(key, 0) for key in ("requests", "answered", "refused", + "failed", "bad")} + fetcher_lines = [line for line in b.log_since(b_log) if FETCHER_TROUBLE.search(line)] + server_lines = [line for line in a.log_since(a_log) if SERVER_TROUBLE.search(line)] + detail = (f"{what} in {took:.1f}s ({size * 8 / took / 1e6:.2f} Mbit/s); {b.name}'s counters for {a.name}: " + + ", ".join(f"{k} {v}" for k, v in counts.items())) + if counts["refused"] or counts["failed"] or counts["bad"] or fetcher_lines or server_lines: + lines = "\n".join([f" {b.name}.log: {line}" for line in fetcher_lines[:10]] + + [f" {a.name}.log: {line}" for line in server_lines[:10]]) + raise AssertionError(f"refusals or failures in an honest transfer: {detail}\n{lines}") + return detail + + def names(self, a: Daemon, b: Daemon, added: dict) -> str: + """A name a publishes resolves on b, and b's gateway serves a file through it.""" + alias = f"site-{self.counter}" + self.counter += 1 + name = a.out("name", "create", alias) + if added["dir"] not in a.out("name", "publish", alias, added["dir"]): + raise AssertionError("publish didn't print the target") + resolved = wait_for(lambda: b.cli("name", "resolve", name, check=False).stdout.decode().strip(), 90, + f"{b.name} to resolve {a.name}'s name") + if resolved != added["dir"]: + raise AssertionError(f"{b.name} resolved {resolved!r}, not {added['dir']}") + status, body = b.gateway(f"/name/{name}/index.txt") + if (status, body) != (200, f"hello from {a.label}\n".encode()): + raise AssertionError(f"gateway /name//index.txt: {status} {body[:200]!r}") + return f"resolve and gateway /name/ for {name[:12]}..." + + def keepers(self, publisher: Daemon, keeper: Daemon, resolver_tree: str, resolver_label: str) -> str: + """keeper keeps a name publisher published; with publisher stopped, a node of resolver_tree started + afterwards (so it never heard the name's announce, which it could take the signed record from) gets the + record by asking keepers, and resolves it.""" + path, _ = self.random_file(10 * KIB) + cid = publisher.out("add", path) + alias = f"kept-{self.counter}" + self.counter += 1 + name = publisher.out("name", "create", alias) + publisher.cli("name", "publish", alias, cid) + if name not in publisher.out("name", "kept"): + raise AssertionError(f"{publisher.name} doesn't keep its own name") + + def keeper_has(): + keeper.cli("name", "keep", name, check=False) + return cid in keeper.out("name", "kept") + + wait_for(keeper_has, 120, f"{keeper.name} to fetch the signed record from {publisher.name}") + publisher.stop() + resolver = self.daemon(f"resolver-{self.counter}", resolver_tree, resolver_label, NODE_CONFIG, + self.tcp_port, SETTINGS, "--bootstrap", self.hub.node_id) + self.counter += 1 + start = resolver.log_size() + + def resolver_has(): + """resolver asks keepers again (keep is idempotent) until it holds the signed record.""" + resolver.cli("name", "keep", name, check=False) + return cid in resolver.out("name", "kept") + + wait_for(resolver_has, 240, f"{resolver.name} to get the record from {keeper.name} with " + f"{publisher.name} stopped") + # Every version logs each keeper query's outcome; the record must have come from one (the keeper being + # the only one left), not from an announce + asked = re.compile(rf"rnfs keepers: (asked \d+ of \d+ keepers of name {name}: [1-9]\d* verifying records?)") + evidence = wait_for(lambda: next((m.group(1) for line in resolver.log_since(start) + if (m := asked.search(line))), None), 180, + f"{resolver.name}'s log to show a keeper query that got the record") + resolved = resolver.out("name", "resolve", name) + if resolved != cid: + raise AssertionError(f"{resolver.name} resolved {resolved!r}, not {cid}") + return f"{keeper.name} kept it; with {publisher.name} stopped, a fresh {resolver.name} {evidence}" + + def lxmf(self, a: Daemon, b: Daemon) -> str: + """a sends b a link over LXMF; b's inbox has it and `inbox get` fetches it.""" + path, content = self.random_file(100 * KIB) + cid = a.out("add", path) + address = b.out("lxmf", "address").splitlines()[0] + note = f"from {a.name} {self.counter}" + self.counter += 1 + wait_for(lambda: a.cli("send", address, cid, "-m", note, check=False).returncode == 0, 90, + f"{a.name} to send {b.name} an LXMF message") + + def arrived(): + """The inbox number of the message carrying cid (a "#N ..." line, its links indented below it).""" + number = None + for line in b.out("inbox").splitlines(): + if line.startswith("#"): + number = line.split()[0].lstrip("#") + elif cid in line and number is not None: + return number + return None + + number = wait_for(arrived, 120, f"the message in {b.name}'s inbox") + inbox = b.out("inbox") + if note not in inbox: + raise AssertionError(f"the message text is missing from {b.name}'s inbox:\n{inbox}") + wait_for(lambda: "delivered" in a.out("outbox"), 90, f"{a.name}'s outbox to show the message delivered") + out = self.unique("lxmf") + fetched = b.out("inbox", "get", number, "-o", out).split() + if fetched and fetched[0] == "job": + b.out("jobs", "wait", fetched[1].rstrip(":")) + with open(out, "rb") as f: + if f.read() != content: + raise AssertionError("inbox get: the bytes differ") + return "sent, delivered, fetched from the inbox" + + # Reporting + + def table(self) -> str: + """The results as a text table.""" + rows = [("scenario", "direction", "result", "detail")] + rows += [(scenario, direction, "PASS" if ok else "FAIL", detail.splitlines()[0]) + for scenario, direction, ok, detail in self.results] + widths = [max(len(row[i]) for row in rows) for i in range(3)] + lines = [f"{row[0]:<{widths[0]}} {row[1]:<{widths[1]}} {row[2]:<{widths[2]}} {row[3]}" for row in rows] + lines.insert(1, "-" * min(120, max(len(line) for line in lines))) + return "\n".join(lines) + + +def run_one(old_tree: str, keep: bool) -> bool: + """Run every scenario against one old tree; True if all passed.""" + base = tempfile.mkdtemp(prefix=f"rnfs-interop-{tree_version(old_tree)}-") + network = Network(old_tree, base) + print(f"=== {network.new_label} against {network.old_label} ({os.path.abspath(old_tree)}); files in {base}", + flush=True) + try: + if network.check("startup", "every daemon", network.start): + network.run() + finally: + network.stop_all() + passed = all(ok for _, _, ok, _ in network.results) + print(f"\n=== {network.new_label} against {network.old_label}\n{network.table()}\n", flush=True) + for scenario, direction, ok, detail in network.results: + if not ok and "\n" in detail: + print(f"{scenario} {direction}:\n{detail}\n") + if passed and not keep: + shutil.rmtree(base, ignore_errors=True) + else: + print(f"daemon directories and logs kept in {base}") + return passed + + +def main(): + parser = argparse.ArgumentParser(description="Run this tree's daemons against older releases' daemons.") + parser.add_argument("--old", action="append", required=True, metavar="DIR", + help="an old release's source tree (a `git archive vX.Y.Z` extract); repeat for several") + parser.add_argument("--keep", action="store_true", help="keep the daemon directories and logs even on success") + args = parser.parse_args() + for tree in args.old: + if not os.path.isfile(os.path.join(tree, "rnfs.py")): + parser.error(f"{tree}: no rnfs.py; give a release's source tree") + results = [run_one(tree, args.keep) for tree in args.old] + sys.exit(0 if all(results) else 1) + + +if __name__ == "__main__": + main() diff --git a/tests/security/README.md b/tests/security/README.md new file mode 100644 index 0000000..010c7b5 --- /dev/null +++ b/tests/security/README.md @@ -0,0 +1,169 @@ +# Security regression tests + +Each `test__.py` here is the regression test for one security finding, named by the finding's audit ID. +Its module docstring says what the attack was, how the test rebuilds it, and what the fix does. This file maps each +ID to its test modules, the code the test guards and the commits that fixed it, grouped by subsystem. To find every +test for a module, search this file for its name (for example `keepers.py` or `server.py`). + +Where the IDs come from: +- **Earlier audits** (H, M, L, N, NEW, I, R and T): the October 2026 audits, fixed before 0.1.4. A suffix `-R`, + `-R1` or `-R2` marks a residual found when the fix was re-tested, and "follow-up" a later fix to the same + finding. Their write-ups weren't kept; the test docstrings are the record. The fixes were made on `fix/*` and + `sec/*` branches, merged through `security/audit-2026-10` and released in 0.1.4. The commits named are those + branches' commits, all in the history of the `v0.1.4` tag. +- **0.1.5 audit** (CONT, DHT, API, FEAT, NET and SUP): `0.1.5-audit.md`, fixed in 0.1.6 (`0.1.6-plan.md`). +- **No ID:** tests from an earlier review or from the security tooling, listed under the subsystem they guard. + +Test modules are in this directory unless a path says otherwise. A commit marked "(test)" only added or updated +the test; the others changed the code. "Pages" are the NomadNet companion pages in `companion-nomadnet-sites/`. + +Run them with `python3 -m unittest discover -s tests/security -p "test_*.py"`. Those needing real RNS +(`test_h1_real_rns.py`, `test_m1_real_rns.py` and the real-daemon part of `test_h2_display_page_pins.py`) run only +with `RNFS_E2E=1`, and `test_fuzz_parsers.py` only where hypothesis is installed; see CONTRIBUTING.md. + +Shared fakes and helpers (not test modules): `_content_helpers.py` (a DAG declaring 1 byte but holding megabytes, +and two offline nodes counting what B downloads), `_server_helpers.py` (fake links and a `RequestServer` on a +FakeClock), `_fetchr2_helpers.py` (padded and chained DAGs, and a byte-range read's allowance), +`_dhtr2b_helpers.py` (SybilNet and the keeper scenario), `_pages_helpers.py` (the pages' own code against an offline node +and a fake CLI), and `_h1_real_helpers.py` and `_m1_real_helpers.py` (the real-RNS hub and attacker processes). + +## Content, storage and fetching + +| ID | Test modules | Code guarded | Fix commits | +|---|---|---|---| +| L3 | `test_l3_zero_size_children.py`, `test_dag_caps.py` | `rnfs/fetcher.py` (`dag_bound`, `node_credit`, `fetch_dag`) | `211c470`, `4bdd14a` (test) | +| N1 | `test_n1_user_pin_unbounded.py`, `test_n1_display_page_unbounded.py`, `test_dag_caps.py` | `rnfs/fetcher.py` (`fetch_dag`), `rnfs/node.py` (`pin_add`), display-mu-file page | `211c470`, `6b23f35` (page) | +| N1-R1 | `test_n1_read_range_cap.py` | `rnfs/fetcher.py` (`Fetch.read_range`) | `2290b66` | +| N1-R2 | `test_n1_listing_chains.py` | `rnfs/fetcher.py` (`fetch_listing`) | `2290b66` | +| N1-R3 | `test_n1_entry_credit.py` | `rnfs/fetcher.py` (`node_credit`) | `2290b66` | +| L8 | `test_l8_folder_swap.py`, `test_l8_path_walk_checks.py` | `rnfs/content/builder.py`, `rnfs/content/shared_folder.py`, `rnfs/fsutil.py` | `b77bf1a`, `b18fb02` (test) | +| L8-R2 | `test_l8r2_hard_links.py` | `rnfs/content/shared_folder.py`, `rnfs/content/builder.py` | `1d8e099` | +| L9 | `test_l9_get_free_space.py`, `test_l9_free_space_unknown.py` | `rnfs/node.py` (`_check_free_space`), `rnfs/node_content.py`, `rnfs/content/storage.py` | `7640d0b`; failing closed: `383f770` | +| NEW-5 | `test_new5_storage_lock_walks.py` | `rnfs/content/storage.py` (`evict`, `gc`) | `6f6b36e` | +| CONT-1 | `test_cont1_deny_hash_amplification.py` | `rnfs/denylist.py`, `rnfs/content/walker.py`, `rnfs/node_content.py`, `rnfs/node.py` (`_check_file`) | `5454d5d` | +| CONT-2 | `test_cont2_write_out_swap.py` | `rnfs/content/walker.py` (`write_out`), `rnfs/node.py` (`get`), `rnfs/winacl.py` | `04b3e2e` | +| CONT-3 | `test_cont3_shared_folder_target.py` | `rnfs/content/shared_folder.py` (`SharedFolder.scan`), `rnfs/node.py` | `3fb49ae`, `8d30c61` | +| CONT-4 / API-6 | `test_cont4_denied_hash_notice.py` | `rnfs/node.py` (`_check_file`); documented in specification section 8 | `c61a324` | + +## DHT, providers and names + +| ID | Test modules | Code guarded | Fix commits | +|---|---|---|---| +| M3 | `test_m3_keeper_discovery.py` | `rnfs/features/keepers.py`, `rnfs/dht/lookup.py`, `rnfs/dht/routing.py`, `rnfs/fetcher.py` | `424b20b` | +| M3-R1 | `test_m3r1_keeper_standing.py` | `rnfs/features/keepers.py` (`KeepersFeature.ask_keepers`), `rnfs/dht/lookup.py`, `rnfs/dht/routing.py` | `c83ceab`, `e2e78c2` (test) | +| M3-R2 | `test_m3r2_keeper_unknown_budget.py` | `rnfs/features/keepers.py`, `rnfs/dht/lookup.py` | `c83ceab` | +| M4 | `test_m4_provider_admission.py` | `rnfs/dht/providers.py` (`ProviderStore`), `rnfs/network/protocol.py`, `rnfs/node.py` | `641b9b2` | +| M4-R | `test_m4r_newcomer_pool.py` | `rnfs/dht/providers.py` | `2f72d80` | +| NEW-1 | `test_new1_publisher_withdraw_race.py` | `rnfs/publisher.py` (`publish_due`, `refresh`) | `c3b4d45` | +| NEW-4 | `test_new4_private_mode_records.py` | `rnfs/dht/providers.py`, `rnfs/node.py` (`NodeBackend.find`), `rnfs/network/links.py` (`LinkPool`) | `1ffc8eb` | +| NEW-6 | `test_new6_publisher_crash_cut_round.py`; remote-pin half: `test_new6_remote_pins_restart.py`; follow-up: `test_new6_remote_pins_bad_entry.py` | `rnfs/publisher.py`; `rnfs/node.py` (`_load_remote_pins`, `_resume_remote_pins`), `rnfs/statedb.py` | `c3b4d45`; `719c1bc`; `49df3c9` | +| NEW-8 | `test_new8_provide_at_limit.py` | `rnfs/dht/providers.py` (`ProviderStore.provide`, `_within_limits`) | `d74dcc5` | +| L4 | `test_l4_name_rollback.py` | `rnfs/names.py` (`NameManager`, the name table) | `97b2537` | +| L4-R | `test_l4r_name_floor.py` | `rnfs/names.py` (the gateway's name floor) | `a9c5640` | +| NEW-2 | `test_new2_name_publish_crash.py`, `test_new2_follow_lxmf_pins.py` | `rnfs/names.py` (`NameManager.publish`), `rnfs/node.py`, `rnfs/features/follow.py`, `rnfs/features/lxmf.py`, `rnfs/content/storage.py` | `8ecad5b`, `ef3cc9d` | +| R-2 | `test_r2_name_key_unreadable.py` | `rnfs/names.py` (`release_gone`, `reconcile_pins`) | `56ed5ac` | +| T1 | `test_t1_parse_name_whitespace.py`; residual: `test_t1_parse_hex_id_whitespace.py` | `rnfs/names.py` (`parse_name`); `rnfs/api.py` (`parse_hex_id`) | `63ebae9`; `bb23b64` | +| DHT-1 | `test_dht1_keeper_query_budget.py` | `rnfs/features/keepers.py`, `rnfs/dht/lookup.py` | `73c8295` | +| DHT-2 | `test_dht2_replayed_announce.py`; also `tests/test_names.py` | `rnfs/names.py` (`RnsNetwork.recalled_emission`) | `c31372b` | +| DHT-4 | `test_dht4_private_name_announces.py`; also `tests/test_public.py` | `rnfs/names.py` (`NameManager.publish`), `rnfs/cli.py` | `509c056` | +| DHT-5 | `test_dht5_own_provider_record.py`; also `tests/test_providers.py` | `rnfs/dht/providers.py` (`_within_limits`, `_make_room`) | `4b72ea7` | +| DHT-6 | none: documented only (the cost of grinding node IDs, specification section 10) | none | `4441878` (docs) | + +## Network and the `/blocks` server + +| ID | Test modules | Code guarded | Fix commits | +|---|---|---|---| +| H1 | `test_h1_response_slots.py`, `test_h1_real_rns.py` (`RNFS_E2E`), `test_rns_compat_security.py` | `rnfs/network/server.py` (`ResponsePools`, `RequestServer`), `rnfs/config.py` | `dbd1144`, `7b9f6b9` and `c8d09ad` (test) | +| H1 (follow-up) | `test_h1_regressions.py` | `rnfs/network/server.py` (`link_path`, the grace and fairness rules), `rnfs/config.py` | `126063d`, `c6f4a74`, `86c4475`, `90a2952` (test) | +| M1 | `test_m1_half_open_links.py`, `test_m1_real_rns.py` (`RNFS_E2E`), `test_rns_compat_security.py` | `rnfs/network/guards.py` (`limit_half_open`) | `09c6324` | +| M2 | `test_m2_upload_budget.py` | `rnfs/network/server.py` (upload_budget), `rnfs/mesh.py` | `0726417` | +| M2 (follow-up) | `test_m2_commitment_bound.py` | `rnfs/network/server.py` | `1af981f` | +| L1 | `test_l1_response_precheck.py`, `test_rns_compat_security.py` | `rnfs/network/guards.py`, `rnfs/network/protocol.py` (`response_refusal`) | `73041a7` | +| L2 | `test_l2_blocks_request_limits.py` | `rnfs/network/protocol.py` (a repeated CID), `rnfs/network/server.py` (the budget with no rate estimate) | `80b950d`, `a98b985` | +| L17 | `test_l17_rejected_responses.py` (L17a), `test_l17_request_rate.py` (L17b), `test_rns_compat_security.py` | `rnfs/network/server.py` (the cooldown; `RequestServer`'s per-link request rate) | `42aadca` (L17a), `6c79d4c` (L17b) | +| NEW-3 | `test_new3_remote_pin_wakeup.py` | `rnfs/node.py` (`_remote_pin_worker`) | `719c1bc` | +| NET-1 | `test_net1_cancelled_responses.py`, `test_l17_rejected_responses.py` | `rnfs/network/server.py` (`NO_PROGRESS_FRACTION`) | `0ec517b` | +| NET-2 | `test_net2_single_packet_pacing.py`; also `tests/test_pacing.py`, `tests/test_server.py` | `rnfs/network/server.py` (pacing under `max_upload_rate`) | `c0ec333` | +| NET-3 | `test_net3_outgoing_requests.py`, `test_l1_response_precheck.py` | `rnfs/network/guards.py` (`guarded_receive`) | `81afb98` | +| NET-4 | `test_net4_dripping_provider.py`; also `tests/test_fetcher.py` (`ReadAheadLinkTest`) | `rnfs/fetcher.py` (end-game duplication of a batch in flight past `deadline_slack`), `rnfs/network/client.py` and `rnfs/network/links.py` (`close_unless_used`) | `afa3820`, `ac9ed4f` | +| NET-5 | `test_net5_proof_wait.py` | `rnfs/network/server.py` (`RequestServer._proof_wait`) | `db27b8b` | +| none (RNS internals) | `test_rns_compat_security.py` | the RNS attributes `rnfs/network/server.py` and `rnfs/network/guards.py` rely on (H1, M1, L1, L17) | `dbd1144`, `09c6324`, `73041a7`, `42aadca`, `90a2952` (test) | + +## Local API, client and gateway + +| ID | Test modules | Code guarded | Fix commits | +|---|---|---|---| +| I1 | `test_i1_507_message.py` | `rnfs/gateway.py` (the 507 message), `rnfs/content/storage.py` (`StorageFull`) | `68dcb8f` | +| L6 | `test_l6_gateway_storage_oracle.py` | `rnfs/gateway.py`, `rnfs/gateway_formats.py` (`?format=`) | `a678443`, `3265621`, `4a765d4` (documented limit) | +| NEW-7 | `test_new7_api_status_open.py` | `rnfs/api.py` (`GET /api/status`), `rnfs/cli.py` | `863f0b7` | +| none (CORS) | `test_gateway_cors.py` | `rnfs/gateway.py`, `rnfs/gateway_formats.py` | `d77be66` | +| none (headers) | `test_response_headers.py` | `rnfs/gateway.py` (`?filename=`), `rnfs/api.py` (`no-store`, `nosniff`) | `bfe9bc7` (test; pins existing behaviour) | +| API-2 | `test_api2_proof_first.py` | `rnfs/apiclient.py` (`verify_daemon`, `PROOF_TIMEOUT`) | `dda4603` | +| API-3 | none: documented only (the proof keep-open, specification section 11) | none | `feba0f0` (docs) | +| API-4 | `test_api4_old_daemon_fallback.py`; also `tests/test_apiclient.py` | `rnfs/apiclient.py` (the pre-0.1.5 reconnect) | `b565ee8` | +| API-5 | `test_api5_head_deny.py` | `rnfs/node_content.py` (`stored_head`), `rnfs/gateway.py`, `rnfs/node.py` | `cb5c93f` | +| API-7 | `test_api7_missing_fetch_metadata.py` | `rnfs/gateway.py` (`limited`, the User-Agent rule) | `0d010b1` | + +## Features: LXMF and follow + +| ID | Test modules | Code guarded | Fix commits | +|---|---|---|---| +| L5 | `test_l5_follow_cap.py` | `rnfs/features/follow.py`, `rnfs/fetcher.py` (`dag_bound`) | `211c470` | +| L12 | `test_l12_lxmf_inbox.py` | `rnfs/features/lxmf.py` (the inbox), `rnfs/features/contacts.py` | `0cc8193` | +| L12-R | `test_l12r_untrusted_sender_allowance.py` | `rnfs/features/lxmf.py` (`SENDER_BURST`, `SENDER_INTERVAL`) | `dc1f71d` | +| L13 | `test_l13_lxmf_hold_queue.py` | `rnfs/features/lxmf.py` (`LxmfNetwork`'s hold queue) | `24898d2` | +| L13-R | `test_l13r_hold_replaces_oldest.py` | `rnfs/features/lxmf.py` | `dc1f71d` | +| R-1 | `test_r1_reconcile_damaged_state.py` | `rnfs/features/follow.py`, `rnfs/features/lxmf.py` (the start-up reconcile) | `0e350ae` | +| none (auto-fetch fairness) | `test_lxmf_auto_fetch_fairness.py` | `rnfs/features/lxmf.py` (the auto-fetch queue) | `84213fa` | +| DHT-3 | `test_dht3_follow_quota.py`, `test_l5_follow_cap.py` | `rnfs/features/follow.py` (`follow_quota`), `rnfs/config.py` | `63a0157` | +| FEAT-1 | `test_feat1_auto_sender_share.py` | `rnfs/features/lxmf.py` (each auto-fetch sender's share) | `81dc3a1` | +| FEAT-2 | `test_feat2_inbox_sender_line.py` | `rnfs/features/lxmf.py` (`inbox_list`), `rnfs/features/contacts.py` | `3707a08` | +| FEAT-3 | `test_feat3_untrusted_dropped_counter.py` | `rnfs/features/lxmf.py` (`untrusted_dropped`) | `6738659` | +| FEAT-4 | `test_feat4_removed_auto_sender.py` | `rnfs/features/lxmf.py` (`auto_fetch`, the re-queue at start) | `58fe37a` | +| FEAT-5 | `test_feat5_name_link_fallback.py` | `rnfs/features/lxmf.py` (`inbox get`, `use_sent_cid`) | `64ab89a` | +| FEAT-6 | `test_feat6_printable_label.py` | `rnfs/fsutil.py` (`printable_label`) | `0c97eb7` | +| FEAT-7 | `test_feat7_example_and_client_escaping.py` | `examples/rnfs_share.py`, `rnfs/remote.py` (`Client._report`) | `71d7cb2` | + +## Companion pages + +| ID | Test modules | Code guarded | Fix commits | +|---|---|---|---| +| H2 | `test_h2_display_page_pins.py` (part `RNFS_E2E`) | display-mu-file page | `6b23f35` | +| H2-R | `test_h2r_display_page_private_and_rate.py` | display-mu-file page (private mode, `VISITOR_LIMIT`) | `3707b53` | +| L10 | `test_l10_index_page_injection.py` | both pages' `escape()` | `9098381`, `6b23f35` | +| L11 | `test_l11_display_page_micron.py` | display-mu-file page (`plain_text`, the allowlist) | `6b23f35` | +| L14 | `test_l14_page_local_paths.py` | both pages (`log`, which writes page.log; display-mu-file's `fail`) | `6b23f35`, `9098381` | +| M6 | `test_m6_index_page_names.py`, `test_m6_pin_ls_sealed.py`; follow-up: `test_m6_index_page_full_run.py` | file-index-node page (`entry`, `shown`, `is_sealed`); `rnfs/node.py` (`pin_ls`), `rnfs/cli.py` | `9098381`, `9544fd6`; `5092f80` | +| FEAT-8 | `test_feat8_index_page_cache.py` | file-index-node page (`take_lock`, `cached`) | `be571fc` | + +N1's page half is in the content table. + +## Platform: files, permissions and Windows + +| ID | Test modules | Code guarded | Fix commits | +|---|---|---|---| +| L7 | `test_l7_windows_acl.py` | `rnfs/winacl.py`, `rnfs/fsutil.py`, `rnfs/network/identity.py`, `rnfs/names.py`, `rnfs/node.py`, `rnfs/cli.py` | `1d0b0e6`, `2be6acf` | +| L7-R1 | `test_l7r1_windows_dir_creation.py` | `rnfs/winacl.py` (`create_private_dir`), `rnfs/config.py`, `rnfs/fsutil.py`, `rnfs/node.py` | `4383bfa` | +| L7-R2 | `test_l7r2_junction_root.py` | `rnfs/fsutil.py`, `rnfs/node.py` (the root check) | `4383bfa` | +| I2 | `test_i2_identity_symlink.py`; packaging half: `test_i2_packaging_hardening.py` | `rnfs/network/identity.py` (`load_or_create_identity`); `.github/workflows/release.yml`, `contrib/rnfs.service`, `install.sh` | `ea88d13`; `cbf257e` | +| I2-R1 | `test_i2r1_identity_file.py` | `rnfs/network/identity.py` (`IdentityFileError`) | `2db7830` | +| API-1 | `test_api1_settings_file.py`, `test_api1_subdirs.py` | `rnfs/config.py`, `rnfs/fsutil.py`, `rnfs/node.py`, `rnfs/embedded.py`, `rnfs/cli.py`, `rnfs/apiclient.py` | `939ecae` | + +## Packaging and release + +| ID | Test modules | Code guarded | Fix commits | +|---|---|---|---| +| L15 | `test_l15_install_release_check.py` | `install.sh` (`download`, `verify_release`, `fetch_release`) | `e2b5686` | +| L16 | `test_l16_release_container.py` | `.github/workflows/release.yml` (the Debian 12 smoke test) | `1f3cb70` | +| SUP-1 | `tests/test_packaging.py` (`test_flatpak_builds_skip_the_actions_cache`) | `.github/workflows/release.yml` | `6fd0cc7` | +| SUP-2 | `test_l15_install_release_check.py`; also `tests/test_packaging.py` (`test_release_wheel_is_of_the_tag`, `test_provenance_names_the_tag`) | `install.sh` | `67bfafe` | +| SUP-3 | `test_l15_install_release_check.py` | `install.sh` | `c64ebf5` | +| SUP-4 | none: the PyPI name is the maintainer's to claim; the code side is a README note | `README.md` | `4ac4b89` (docs) | +| SUP-5 | `tests/test_packaging.py` (`test_flatpak_sandbox_is_narrow`) | `packaging/flatpak/io.github.afit21.rnfs.yml` | `021c7c2` | +| SUP-6 | `tests/test_packaging.py` (`test_unit_hardening_is_an_opt_in_drop_in`, `test_deb_postinst_python_is_isolated`, `test_plain_http_source_warns`) | `contrib/rnfs.service`, `contrib/rnfs.service.d/hardening.conf`, `packaging/build-deb.sh`, `install.sh` | `c00f8d7` | + +## Across subsystems + +| ID | Test modules | Code guarded | Fix commits | +|---|---|---|---| +| none (fuzzing) | `test_fuzz_parsers.py` (needs hypothesis) | the strict parsers of untrusted bytes: `rnfs/content/cid.py`, `rnfs/content/dag.py`, `rnfs/content/storage.py`, `rnfs/names.py`, `rnfs/network/`, `rnfs/features/lxmf.py`, `rnfs/gateway.py` | `077f33a` (test); found T1 | diff --git a/tests/security/_m1_real_helpers.py b/tests/security/_m1_real_helpers.py index 87bba35..de800fd 100644 --- a/tests/security/_m1_real_helpers.py +++ b/tests/security/_m1_real_helpers.py @@ -9,7 +9,7 @@ import time sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from tests.security import _h1_real_helpers as h # noqa: E402 +from tests.security import _h1_real_helpers as h def role_flood(configdir, node_id_hex, rate, seconds): diff --git a/tests/security/_pages_helpers.py b/tests/security/_pages_helpers.py index 073ca79..7d8a68b 100644 --- a/tests/security/_pages_helpers.py +++ b/tests/security/_pages_helpers.py @@ -71,9 +71,9 @@ def index_page_namespace(): """The file-index-node page's module-level defs (escape, display_name, entry, section).""" with open(INDEX_PAGE, encoding="utf-8") as f: source = f.read() - head = source.split('print("""#!c=0')[0] # everything before the page body - namespace = {} - exec(compile(head, INDEX_PAGE, "exec"), namespace) + # The page renders only under __main__, so running its source here just defines them + namespace = {"__name__": "index_page"} + exec(compile(source, INDEX_PAGE, "exec"), namespace) return namespace diff --git a/tests/security/_server_helpers.py b/tests/security/_server_helpers.py index a0a3b73..df69d53 100644 --- a/tests/security/_server_helpers.py +++ b/tests/security/_server_helpers.py @@ -1,5 +1,5 @@ -"""Helpers for the /blocks server security tests (H1, L2): fake links that know their RNS path, and a -RequestServer over tests/test_server.py's fakes driven by a FakeClock.""" +"""Helpers for the /blocks server security tests (H1, L2, L17, NET-1): fake links that know their RNS path, a +RequestServer over tests/test_server.py's fakes driven by a FakeClock, and a requester rejecting a response.""" import RNS from rnfs.config import Config @@ -12,6 +12,17 @@ LORA_BITRATE = 1_000 # a slow LoRa channel +def reject(resource, progress: float = 0.0): + """The requester rejects (or, mid-transfer, cancels) the response: RNS's Resource._rejected on an outgoing + Resource sets REJECTED, leaves the link's outgoing Resources and runs the callback. On RNS 1.5.6 every + receiver cancel arrives this way (RESOURCE_RCL), whether before or after parts were sent.""" + resource.progress = progress + resource.status = RNS.Resource.REJECTED + resource.conclude() + if resource.callback: + resource.callback(resource) + + class FakeInterface: """Stands in for an RNS Interface: a name and its configured bitrate.""" diff --git a/tests/security/test_api1_settings_file.py b/tests/security/test_api1_settings_file.py new file mode 100644 index 0000000..9d80f1e --- /dev/null +++ b/tests/security/test_api1_settings_file.py @@ -0,0 +1,303 @@ +"""API-1 (settings file): on POSIX, rnfs.toml was read by path with no check at all: a 0666 file loaded with no +warning, a symlink was followed, a FIFO blocked the start and one owned by another user was used. Its settings pick +the Reticulum config (and with it programs rnfs starts, such as a PipeInterface's command) and the API port the CLI +sends its token to, so whoever could write it chose them. Now it is opened with O_NOFOLLOW | O_NONBLOCK and checked +through that descriptor: a symlink, anything but a regular file and a file of another user's are refused, a file +others may access is made 0600 with a warning, and at most 1 MiB is read. RNS's own `config` gets the same check +when reticulum_dir is the rnfs dir. + +ND6: the warnings are returned to the caller (config.warnings) instead of being logged through rnfs.log, which +imports RNS. The daemon prints them, rnfs.Node logs them, the CLI client prints them and rnfs.Client raises them +as RuntimeWarnings; a client loading a loose settings file never loads RNS (checked in a `python -I` subprocess, +on the POSIX path and on the Windows one through a fake access-list backend). + +Reproduced with real files, symlinks and a FIFO in a temporary directory (POSIX); "another user" is an os.geteuid +that answers a different uid. +""" +import contextlib +import io +import os +import stat +import subprocess +import sys +import tempfile +import threading +import types +import unittest +from unittest import mock + +from rnfs import apiclient, cli +from rnfs.config import MAX_SETTINGS_SIZE, SETTINGS_FILE, Config +from tests.support import CAN_SYMLINK, HAS_FIFO, POSIX_MODES + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +SETTINGS = 'api_port = 9999\nreticulum_dir = "/tmp/planted-reticulum"\n' + +# Run with `python -I`: loads a client over a loose rnfs.toml with every import of RNS refused and recorded, on the +# POSIX path or (argv[3] == "windows") on the Windows one, through a fake access-list backend that reports the file +# open to Users until it is made private. Prints whether RNS was imported, then each warning raised +CLIENT_SCRIPT = r""" +import importlib.abc, sys, warnings +sys.path.insert(0, sys.argv[1]) +attempts = [] + +class NoRns(importlib.abc.MetaPathFinder): + def find_spec(self, name, path=None, target=None): + if name == "RNS" or name.startswith("RNS."): + attempts.append(name) + raise ImportError("RNS imported") + return None + +sys.meta_path.insert(0, NoRns()) +if sys.argv[3] == "windows": + from rnfs import fsutil, winacl + USER = "S-1-5-21-1-2-3-1001" + loose = winacl.Dacl(USER, [winacl.Ace(winacl.ACCESS_ALLOWED, 0, winacl.FILE_ALL_ACCESS, USER), + winacl.Ace(winacl.ACCESS_ALLOWED, 0, 0x120089, "S-1-5-32-545")]) + private = winacl.Dacl(USER, winacl.parse_acl(winacl.private_acl(USER, False))) + + class Acl: + made_private = False + def user_sid(self): + return USER + def read(self, path, fd=None): + return private if self.made_private else loose + def set_private(self, path, directory): + self.made_private = True + + fsutil.IS_WINDOWS = True + winacl.backend = Acl() +import rnfs +with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + client = rnfs.Client(sys.argv[2]) +print(client.api.port) +print(bool(attempts) or any(m == "RNS" or m.startswith("RNS.") for m in sys.modules)) +for warning in caught: + print(warning.category.__name__, warning.message) +""" + + +@unittest.skipUnless(POSIX_MODES, "Unix permission bits and owners") +class SettingsFileTest(unittest.TestCase): + """Config.load over a temporary rnfs dir, with rnfs.toml and a file of someone else's next to it.""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dir = tmp.name + self.path = os.path.join(self.dir, SETTINGS_FILE) + self.theirs = os.path.join(self.dir, "their-settings") + with open(self.theirs, "w", encoding="utf-8") as f: + f.write(SETTINGS) + os.chmod(self.theirs, 0o644) + + def write(self, text: str = SETTINGS, mode: int = 0o600): + """rnfs.toml holding text, with mode.""" + with open(self.path, "w", encoding="utf-8") as f: + f.write(text) + os.chmod(self.path, mode) + + def mode(self, path: str) -> int: + """path's permission bits.""" + return stat.S_IMODE(os.lstat(path).st_mode) + + def refused(self, error=PermissionError) -> str: + """Config.load, which must refuse the settings; its message.""" + with self.assertRaises(error) as caught: + Config.load(self.dir) + return str(caught.exception) + + def test_loose_settings_are_tightened_with_a_warning(self): + self.write(mode=0o666) + config = Config.load(self.dir) + self.assertEqual(self.mode(self.path), 0o600) + self.assertEqual(config.api_port, 9999) # still used, after the warning + [warning] = config.warnings + self.assertIn(self.path, warning) + self.assertIn("accessible to other users (mode 666)", warning) + self.assertIn("Check that its settings are yours", warning) + + def test_group_writable_settings_are_tightened(self): + """The 0664 a umask of 002 gives: made private, not refused.""" + self.write(mode=0o664) + self.assertEqual(len(Config.load(self.dir).warnings), 1) + self.assertEqual(self.mode(self.path), 0o600) + + def test_private_settings_load_quietly(self): + self.write() + self.assertEqual(list(Config.load(self.dir).warnings), []) + self.assertEqual(list(Config.load(os.path.join(self.dir, "missing")).warnings), []) # no file at all + + @unittest.skipUnless(CAN_SYMLINK, "symlinks") + def test_symlinked_settings_are_refused(self): + os.symlink(self.theirs, self.path) + self.assertIn("symlink", self.refused()) + self.assertEqual(self.mode(self.theirs), 0o644) # the target is left alone + + @unittest.skipUnless(CAN_SYMLINK, "symlinks") + def test_dangling_symlink_is_refused(self): + """A symlink to nothing is refused too, not taken for no settings.""" + os.symlink(os.path.join(self.dir, "nothing"), self.path) + self.assertIn("symlink", self.refused()) + + @unittest.skipUnless(HAS_FIFO, "FIFOs") + def test_fifo_is_refused_without_blocking(self): + os.mkfifo(self.path, 0o600) + outcome = [] + + def load(): + try: + Config.load(self.dir) + outcome.append("loaded") + except Exception as e: # whatever it is, the test reports it + outcome.append(e) + + thread = threading.Thread(target=load, daemon=True) + thread.start() + thread.join(5) + if thread.is_alive(): + # Unblock the old code's blocking open: a writer that closes at once gives it an empty read + os.close(os.open(self.path, os.O_WRONLY | os.O_NONBLOCK)) + thread.join(5) + self.fail("loading the settings hung opening the FIFO at rnfs.toml") + [result] = outcome + self.assertIsInstance(result, PermissionError) + self.assertIn("not a regular file", str(result)) + + def test_directory_is_refused(self): + os.mkdir(self.path) + self.assertIn("not a regular file", self.refused()) + + def test_settings_of_another_user_are_refused(self): + self.write(mode=0o644) + with mock.patch("os.geteuid", return_value=os.geteuid() + 1): + self.assertIn("belongs to another user", self.refused()) + self.assertEqual(self.mode(self.path), 0o644) # not someone else's file to change + + def test_settings_that_cant_be_tightened_are_refused(self): + self.write(mode=0o666) + with mock.patch("os.fchmod", side_effect=PermissionError(1, "Operation not permitted")): + self.assertIn("can't be restricted", self.refused()) + + def test_oversized_settings_are_refused(self): + self.write("# " + "x" * MAX_SETTINGS_SIZE + "\n") + self.assertIn("larger than", self.refused(ValueError)) + + def test_warnings_is_not_a_setting(self): + """config.warnings is a plain attribute: rnfs.toml can't set it.""" + self.write('warnings = ["all is well"]\n') + self.assertIn("unknown settings", self.refused(ValueError)) + self.assertEqual(list(Config().warnings), []) + + +@unittest.skipUnless(POSIX_MODES, "Unix permission bits and owners") +class ReticulumConfigTest(unittest.TestCase): + """RNS's `config` in the rnfs dir, checked when reticulum_dir is the rnfs dir (as earlier versions used it).""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dir = tmp.name + self.path = os.path.join(self.dir, "config") + with open(self.path, "w", encoding="utf-8") as f: + f.write("[reticulum]\n") + os.chmod(self.path, 0o644) + + def mode(self) -> int: + """The config file's permission bits.""" + return stat.S_IMODE(os.lstat(self.path).st_mode) + + def test_loose_config_is_tightened_with_a_warning(self): + config = Config.load(self.dir, reticulum_dir=self.dir + os.sep) + self.assertEqual(self.mode(), 0o600) + [warning] = config.warnings + self.assertIn(self.path, warning) + + def test_reticulum_dir_set_in_rnfs_toml_counts_too(self): + with open(os.path.join(self.dir, SETTINGS_FILE), "w", encoding="utf-8") as f: + f.write(f'reticulum_dir = "{self.dir}"\n') + os.chmod(os.path.join(self.dir, SETTINGS_FILE), 0o600) + Config.load(self.dir) + self.assertEqual(self.mode(), 0o600) + + def test_config_of_another_user_is_refused(self): + with mock.patch("os.geteuid", return_value=os.geteuid() + 1): + with self.assertRaisesRegex(PermissionError, "belongs to another user"): + Config.load(self.dir, reticulum_dir=self.dir) + + @unittest.skipUnless(CAN_SYMLINK, "symlinks") + def test_symlinked_config_is_refused(self): + target = os.path.join(self.dir, "elsewhere") + os.replace(self.path, target) + os.symlink(target, self.path) + with self.assertRaisesRegex(PermissionError, "symlink"): + Config.load(self.dir, reticulum_dir=self.dir) + + def test_config_elsewhere_is_left_alone(self): + """With reticulum_dir elsewhere (or RNS's default), a leftover config here isn't Reticulum's to check.""" + for reticulum_dir in (None, os.path.join(self.dir, "reticulum")): + self.assertEqual(list(Config.load(self.dir, reticulum_dir=reticulum_dir).warnings), []) + self.assertEqual(self.mode(), 0o644) + + def test_missing_config_is_fine(self): + os.remove(self.path) + self.assertEqual(list(Config.load(self.dir, reticulum_dir=self.dir).warnings), []) + + +@unittest.skipUnless(POSIX_MODES, "Unix permission bits") +class WarningsShownTest(unittest.TestCase): + """Where the warnings go: the CLI client prints them, rnfs.Client raises RuntimeWarnings, without RNS.""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dir = tmp.name + self.path = os.path.join(self.dir, SETTINGS_FILE) + with open(self.path, "w", encoding="utf-8") as f: + f.write("api_port = 9999\n") + + def loosen(self): + """Make rnfs.toml readable and writable by everyone.""" + os.chmod(self.path, 0o666) + + def test_cli_client_prints_the_warning(self): + self.loosen() + args = types.SimpleNamespace(api_port=None, dir=self.dir) + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + client = cli.client(args) + self.assertEqual(client.port, 9999) + self.assertIn("rnfs: warning: the settings file", stderr.getvalue()) + self.assertIn("accessible to other users", stderr.getvalue()) + + def test_library_raises_a_runtime_warning(self): + self.loosen() + with self.assertWarnsRegex(RuntimeWarning, "accessible to other users"): + self.assertEqual(apiclient.ApiConnection(self.dir).port, 9999) + + def run_client(self, platform: str) -> list[str]: + """CLIENT_SCRIPT in a `python -I` subprocess; its output lines, once it says RNS wasn't imported.""" + result = subprocess.run([sys.executable, "-I", "-c", CLIENT_SCRIPT, ROOT, self.dir, platform], + cwd=self.dir, capture_output=True, timeout=60, encoding="utf-8", errors="replace") + self.assertEqual(result.returncode, 0, result.stderr) + lines = result.stdout.splitlines() + self.assertEqual(lines[:2], ["9999", "False"], result.stdout + result.stderr) + return lines[2:] + + def test_client_with_loose_settings_does_not_import_rns(self): + self.loosen() + [warning] = self.run_client("posix") + self.assertTrue(warning.startswith("RuntimeWarning the settings file"), warning) + self.assertEqual(stat.S_IMODE(os.stat(self.path).st_mode), 0o600) + + def test_client_on_windows_with_loose_settings_does_not_import_rns(self): + """winacl reported a loose settings file through rnfs.log, which imports RNS (ND6).""" + os.chmod(self.path, 0o600) # the fake's access list, not these bits, says who may access it + [warning] = self.run_client("windows") + self.assertTrue(warning.startswith("RuntimeWarning the settings file"), warning) + self.assertIn("Users (S-1-5-32-545) may access it", warning) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_api1_subdirs.py b/tests/security/test_api1_subdirs.py new file mode 100644 index 0000000..49400ef --- /dev/null +++ b/tests/security/test_api1_subdirs.py @@ -0,0 +1,263 @@ +"""API-1 (subdirectories): on POSIX, the rnfs dir's subdirectories that already existed (state/, blocks/, storage/, +names/, lxmf/) were never checked: makedirs(exist_ok=True) took a symlink to someone else's directory, kept a 0777 +blocks/ as it was and used one another user owned. Tightening the root to 0700 doesn't take back access to them: a +user who could once write in a loose rnfs dir can keep a directory descriptor open, or own a subdirectory they +created, and plant raw blocks (trusted by their length) or swap name keys. + +Now each is opened with O_DIRECTORY | O_NOFOLLOW (created 0700 if missing) and checked through that descriptor: a +symlink, a non-directory or one of another user's stops the start with an RnfsError naming chown, before Reticulum +starts; a loose one is made 0700 with a notice, and if others could write to it, its entries owned by another user +are reported (a warning only). The start runs in the order root, settings, subdirectories, lock, in the daemon and +in rnfs.Node. On Windows an existing subdirectory gets the private access list too. + +Reproduced with real directories and symlinks in a temporary directory (POSIX); "another user" is an os.fstat or +os.lstat that answers a different uid for one directory or file. The Windows case runs on any platform through the +fake winacl backend of test_l7_windows_acl. +""" +import contextlib +import io +import os +import stat +import tempfile +import unittest +from unittest import mock + +from rnfs import cli, embedded, node as node_module +from rnfs.config import LOCK_FILE, SETTINGS_FILE, Config +from rnfs.errors import RnfsError +from rnfs.node import RnfsNode, lock_rnfs_dir +from tests.security.test_l7_windows_acl import SHARED, WindowsTestCase +from tests.support import CAN_SYMLINK, POSIX_MODES + +SUBDIRS = ("state", "blocks", "storage", "names", "lxmf") + + +def foreign(path: str, real): + """A stat function (real: os.fstat or os.lstat) that says another user owns the file at path.""" + target = os.lstat(path) + + def fake(*args, **kwargs): + info = real(*args, **kwargs) + if (info.st_dev, info.st_ino) != (target.st_dev, target.st_ino): + return info + fields = list(info[:10]) + fields[4] = os.geteuid() + 1 # st_uid + return os.stat_result(fields) + return fake + + +@unittest.skipUnless(POSIX_MODES, "Unix permission bits and owners") +class SubdirectoryTest(unittest.TestCase): + """node._make_private_dirs and the node's start over a temporary rnfs dir, and an attacker's directory.""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = os.path.join(tmp.name, "rnfs") + os.mkdir(self.root, 0o700) + self.attacker = os.path.join(tmp.name, "attacker") + os.mkdir(self.attacker, 0o777) + os.chmod(self.attacker, 0o777) + log = mock.patch.object(node_module, "log") + self.log = log.start() + self.addCleanup(log.stop) + + def mode(self, path: str) -> int: + """path's permission bits.""" + return stat.S_IMODE(os.lstat(path).st_mode) + + def messages(self) -> str: + """Everything logged.""" + return "\n".join(call.args[0] for call in self.log.call_args_list) + + def start_refused(self) -> str: + """Starting a node on the rnfs dir, which must stop with an RnfsError before Reticulum starts or + daemon.lock is made; its message.""" + with mock.patch.object(node_module.RNS, "Reticulum") as reticulum, \ + mock.patch.object(node_module.guards, "install") as guards: + with self.assertRaises(RnfsError) as caught: + RnfsNode(Config(rnfs_dir=self.root)) + with self.assertRaises(RnfsError): + lock_rnfs_dir(Config(rnfs_dir=self.root)) + reticulum.assert_not_called() + guards.assert_not_called() + self.assertFalse(os.path.exists(os.path.join(self.root, LOCK_FILE))) + return str(caught.exception) + + def test_missing_subdirectories_are_created_private(self): + node_module._make_private_dirs(self.root) + for name in SUBDIRS: + self.assertEqual(self.mode(os.path.join(self.root, name)), 0o700, name) + self.assertEqual(self.messages(), "") + + def test_loose_subdirectories_are_made_private(self): + for name in SUBDIRS: + os.mkdir(os.path.join(self.root, name)) + os.chmod(os.path.join(self.root, name), 0o777) + node_module._make_private_dirs(self.root) + for name in SUBDIRS: + self.assertEqual(self.mode(os.path.join(self.root, name)), 0o700, name) + self.assertIn(f"made {os.path.join(self.root, name)} private to its owner (it had mode 777)", + self.messages()) + + def test_group_readable_subdirectory_is_made_private_without_a_scan(self): + """The 0755 of a umask 022: tightened, and as nobody else could write to it, not scanned.""" + blocks = os.path.join(self.root, "blocks") + os.mkdir(blocks) + os.chmod(blocks, 0o755) + with mock.patch("os.walk", side_effect=AssertionError("scanned")): + node_module._make_private_dirs(self.root) + self.assertEqual(self.mode(blocks), 0o700) + + def test_private_subdirectories_are_left_alone(self): + node_module._make_private_dirs(self.root) + self.log.reset_mock() + node_module._make_private_dirs(self.root) + self.assertEqual(self.messages(), "") + + @unittest.skipUnless(CAN_SYMLINK, "symlinks") + def test_symlinked_subdirectory_is_refused(self): + for name in SUBDIRS: + with self.subTest(name): + link = os.path.join(self.root, name) + os.symlink(self.attacker, link) + try: + self.assertIn("symlink", self.start_refused()) + finally: + os.remove(link) + self.assertEqual(self.mode(self.attacker), 0o777) # not chmodded through the link + self.assertEqual(os.listdir(self.attacker), []) + + def test_file_in_place_of_a_subdirectory_is_refused(self): + with open(os.path.join(self.root, "blocks"), "wb"): + pass + message = self.start_refused() + self.assertIn("not a directory", message) + + def test_subdirectory_of_another_user_is_refused(self): + names = os.path.join(self.root, "names") + os.mkdir(names, 0o700) + with mock.patch("os.fstat", foreign(names, os.fstat)): + message = self.start_refused() + self.assertIn("belongs to another user", message) + self.assertIn("chown", message) + + def test_foreign_entries_of_a_once_writable_subdirectory_are_reported(self): + blocks = os.path.join(self.root, "blocks") + os.makedirs(os.path.join(blocks, "ab")) + planted = os.path.join(blocks, "ab", "planted") + with open(planted, "wb") as f: + f.write(b"x") + with open(os.path.join(blocks, "ab", "mine"), "wb") as f: + f.write(b"y") + os.chmod(blocks, 0o775) + with mock.patch("os.lstat", foreign(planted, os.lstat)): + node_module._make_private_dirs(self.root) + self.assertEqual(self.mode(blocks), 0o700) + warnings = [call.args[0] for call in self.log.call_args_list if call.args[1] == node_module.LOG_WARNING] + [warning] = warnings + self.assertIn("1 entry", warning) + self.assertIn(planted, warning) + self.assertTrue(os.path.exists(planted)) # reported, not removed + + def test_daemon_checks_in_order(self): + """The daemon: root, then settings, then subdirectories, then the lock, each before the next.""" + order = [] + real_root, real_load = node_module.make_private_root, Config.load + + def root(path): + order.append("root") + real_root(path) + + def load(*args, **kwargs): + order.append("settings") + return real_load(*args, **kwargs) + + def lock(config): + order.append("lock") + raise RnfsError("stop here") + + def subdirs(path): + order.append("subdirectories") + raise RnfsError("stop here") + + with open(os.path.join(self.root, SETTINGS_FILE), "w", encoding="utf-8") as f: + f.write("api_port = 9999\n") + os.chmod(os.path.join(self.root, SETTINGS_FILE), 0o666) + with mock.patch.object(node_module, "make_private_root", root), \ + mock.patch.object(Config, "load", load), \ + mock.patch.object(node_module, "lock_rnfs_dir", lock), \ + contextlib.redirect_stderr(io.StringIO()) as stderr, self.assertRaises(SystemExit): + cli.main(["--dir", self.root, "daemon"]) + self.assertEqual(order, ["root", "settings", "lock"]) + self.assertIn("rnfs: warning: the settings file", stderr.getvalue()) # printed before the lock + order.clear() + with mock.patch.object(node_module, "_make_private_dirs", subdirs), \ + mock.patch.object(node_module, "make_private_root", root), \ + contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + cli.main(["--dir", self.root, "daemon"]) + self.assertEqual(order, ["root", "subdirectories"]) + self.assertFalse(os.path.exists(os.path.join(self.root, LOCK_FILE))) + + def test_embedded_node_checks_in_order_and_logs_warnings(self): + """rnfs.Node: the same order, and the settings' warnings logged.""" + order = [] + real_root, real_load = node_module.make_private_root, Config.load + + def root(path): + order.append("root") + real_root(path) + + def load(*args, **kwargs): + order.append("settings") + return real_load(*args, **kwargs) + + def lock(config): + order.append("lock") + raise RnfsError("stop here") + + with open(os.path.join(self.root, SETTINGS_FILE), "w", encoding="utf-8") as f: + f.write("api_port = 9999\n") + os.chmod(os.path.join(self.root, SETTINGS_FILE), 0o666) + with mock.patch.object(embedded, "make_private_root", root), mock.patch.object(Config, "load", load), \ + mock.patch.object(embedded, "lock_rnfs_dir", lock), mock.patch.object(embedded, "log") as log: + with self.assertRaisesRegex(RnfsError, "stop here"): + embedded.Node(self.root) + self.assertEqual(order, ["root", "settings", "lock"]) + self.assertIn("the settings file", log.call_args.args[0]) + self.assertEqual(log.call_args.args[1], embedded.LOG_WARNING) + + +class WindowsSubdirectoryTest(WindowsTestCase): + """node._make_private_dirs on Windows: an existing subdirectory others may access is made private too.""" + + def setUp(self): + super().setUp() + self.root = os.path.join(self.tmp.name, "rnfs") + log = mock.patch("rnfs.log.log") + self.log = log.start() + self.addCleanup(log.stop) + profile = mock.patch.dict(os.environ, {"USERPROFILE": self.tmp.name}) + profile.start() + self.addCleanup(profile.stop) + + def test_existing_subdirectory_others_can_read_is_tightened(self): + node_module._make_private_dirs(self.root) + blocks = os.path.join(self.root, "blocks") + self.acl.set(blocks, SHARED) # as an older rnfs left it, inheriting a folder's access list + node_module._make_private_dirs(self.root) + self.assertEqual(self.acl.set_calls, [(blocks, True)]) + self.assertTrue(self.private(blocks)) + + @unittest.skipUnless(CAN_SYMLINK, "symlinks") + def test_linked_subdirectory_is_refused(self): + os.mkdir(self.root) + elsewhere = os.path.join(self.tmp.name, "elsewhere") + os.mkdir(elsewhere) + os.symlink(elsewhere, os.path.join(self.root, "state")) + with self.assertRaisesRegex(RnfsError, "junction or symlink"): + node_module._make_private_dirs(self.root) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_api2_proof_first.py b/tests/security/test_api2_proof_first.py new file mode 100644 index 0000000..ba0112d --- /dev/null +++ b/tests/security/test_api2_proof_first.py @@ -0,0 +1,229 @@ +"""API-2: before the proof, a program listening on the API port chose the client's error. verify_daemon mapped the +status of its proof request (error_for) before comparing X-Rnfs-Proof, so while the daemon was down a listener on +its port could answer 451, 404 or 507 with any text, and rnfs.Client raised Denied, NotFound or StorageFull carrying +that text instead of AuthFailed. The proof exchange also had no timeout, and its body was read whole: a listener +that never answered, or sent an endless body, hung the client. + +Now the proof is checked first: without a valid one, a 503 is Busy with the client's own message and every other +status AuthFailed, and the listener's text is never shown. The proof's response is read up to 64 KiB, and connecting +and proving take at most PROOF_TIMEOUT seconds, after which the request gets the client's own timeout. A proved +daemon's error statuses keep their meaning (error_for is still the one mapping). +""" +import json +import os +import socketserver +import tempfile +import threading +import time +import unittest +from unittest import mock + +from rnfs import apiclient, cli +from rnfs.api import ApiServer +from rnfs.apiclient import ApiConnection, AuthFailed, Busy, ConnectionFailed, Denied, NotFound +from tests.test_api import FakeNode + +TOKEN = "the daemon's token " + "7" * 32 +LISTENER_TEXT = "your files were deleted: run rnfs-repair" # what the listener would have the user read + + +def response(status: str, body: bytes = b"", headers: dict | None = None, length: int | None = None) -> bytes: + """A raw HTTP/1.1 response: status ("451 Unavailable"), headers, and body, with a Content-Length of length (default: + the body's).""" + lines = [f"HTTP/1.1 {status}", f"Content-Length: {len(body) if length is None else length}", + *(f"{name}: {value}" for name, value in (headers or {}).items())] + return ("\r\n".join(lines) + "\r\n\r\n").encode() + body + + +def error_body(message: str = LISTENER_TEXT) -> bytes: + """An error body as the API sends one.""" + return json.dumps({"error": message}).encode() + + +class FakeListener: + """A program listening on a local port, answering each request it reads with respond(listener, headers), which + returns (raw response, keep the connection open), or None to never answer. requests holds (request line, + headers) of everything received, in order.""" + + def __init__(self, test: unittest.TestCase, respond): + """Start listening on a free port; stopped when test ends.""" + self.respond = respond + self.requests = [] + self.release = threading.Event() # set at the end of the test, so connections never answered are closed + listener = self + + class Handler(socketserver.StreamRequestHandler): + def handle(self): + """Read requests (line and headers; the client sends no body to a proof) and answer them.""" + try: + self.serve() + except OSError: + pass # the client closed the connection + + def serve(self): + """Answer requests until the connection ends or one is answered for the last time.""" + while True: + line = self.rfile.readline().decode("latin-1").strip() + if not line: + return + headers = {} + for header in iter(self.rfile.readline, b"\r\n"): + if not header: + return + name, _, value = header.decode("latin-1").partition(":") + headers[name.strip().lower()] = value.strip() + listener.requests.append((line, headers)) + answer = listener.respond(listener, headers) + if answer is None: + listener.release.wait(30) + return + data, keep_open = answer + self.wfile.write(data) + self.wfile.flush() + if not keep_open: + return + + self.server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler) + self.server.daemon_threads = True + self.port = self.server.server_address[1] + threading.Thread(target=self.server.serve_forever, args=(0.05,), daemon=True).start() + test.addCleanup(self.server.server_close) + test.addCleanup(self.server.shutdown) + test.addCleanup(self.release.set) + + def proof(self, headers: dict, token: str = TOKEN) -> str: + """The proof for the challenge in headers, as a daemon holding token gives it.""" + return apiclient.api_proof(token, self.port, headers.get("x-rnfs-challenge", "")) + + def tokens(self) -> list: + """The X-Rnfs-Api header of every request received (None where there was none).""" + return [headers.get("x-rnfs-api") for _, headers in self.requests] + + +class ProofFirstTest(unittest.TestCase): + """A client whose token file holds TOKEN, against listeners on the API port.""" + + def setUp(self): + """The token file.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + cli.write_token(os.path.join(self.tmp, "api_token"), TOKEN) + + def client(self, listener: FakeListener, timeout: float | None = None) -> ApiConnection: + """A client of the API on listener's port.""" + return ApiConnection(self.tmp, port=listener.port, timeout=timeout) + + def finishes(self, call, within: float = 5.0) -> BaseException | None: + """What call() raised (None if it returned), failing the test if it is still running after within seconds.""" + outcome = [] + + def run(): + """Call, keeping what it raised.""" + try: + call() + outcome.append(None) + except BaseException as e: # whatever it raised is the result + outcome.append(e) + thread = threading.Thread(target=run, daemon=True) + thread.start() + thread.join(within) + self.assertFalse(thread.is_alive(), f"the client still waits after {within} s") + return outcome[0] + + def test_unproved_statuses_are_auth_failed(self): + # API-2: fails until a listener that can't prove it knows the token gets AuthFailed whatever it answers + for status in ("400 Bad Request", "403 Forbidden", "404 Not Found", "413 Too Large", "422 Invalid", + "451 Unavailable", "500 Internal Server Error", "502 Bad Gateway", "507 Insufficient Storage", + "200 OK"): + for proof in (None, "00" * 32, "not hex"): + headers = {"Content-Type": "application/json", **({apiclient.PROOF_HEADER: proof} if proof else {})} + listener = FakeListener(self, lambda listener, request, status=status, headers=headers: ( + response(status, error_body(), headers), False)) + api = self.client(listener) + with self.assertRaises(AuthFailed) as raised: + api.call("POST", "/api/name/import", params={"alias": "stolen"}, body=b"K" * 64) + context = (status, proof) + self.assertIs(type(raised.exception), AuthFailed, context) + self.assertNotIn(LISTENER_TEXT, str(raised.exception), context) + self.assertIn("couldn't prove it knows the API token", str(raised.exception), context) + self.assertEqual(listener.tokens(), [None], context) # only the proof request, without the token + self.assertTrue(listener.requests[0][0].startswith("GET /api/status "), context) + + def test_unproved_503_is_busy_with_the_clients_message(self): + # A full daemon answers 503 before reading the request, so without a proof: Busy, in the client's words + listener = FakeListener(self, lambda listener, request: ( + response("503 Service Unavailable", error_body(), {"Retry-After": "10"}), False)) + with self.assertRaises(Busy) as raised: + self.client(listener).status() + self.assertNotIn(LISTENER_TEXT, str(raised.exception)) + self.assertEqual(raised.exception.status, 503) + self.assertIn(f"API port {listener.port}", str(raised.exception)) + self.assertEqual(listener.tokens(), [None]) + + def test_proved_statuses_keep_their_meaning(self): + # The daemon itself (it knows the token) answering the proof request with an error: error_for, its message + for status, expected in (("451 Unavailable", Denied), ("404 Not Found", NotFound), + ("503 Service Unavailable", Busy)): + listener = FakeListener(self, lambda listener, request, status=status: ( + response(status, error_body("the daemon's own words"), + {apiclient.PROOF_HEADER: listener.proof(request)}), False)) + with self.assertRaises(expected) as raised: + self.client(listener).status() + self.assertEqual(str(raised.exception), "the daemon's own words") + self.assertEqual(raised.exception.status, int(status.split()[0])) + self.assertEqual(listener.tokens(), [None]) + + def test_a_listener_that_never_answers(self): + # API-2: fails until the proof exchange has a timeout of its own, even for a client without one + listener = FakeListener(self, lambda listener, request: None) + api = self.client(listener) + with mock.patch.object(apiclient, "PROOF_TIMEOUT", 0.5, create=True): + raised = self.finishes(api.status) + self.assertIsInstance(raised, ConnectionFailed) + self.assertEqual(listener.tokens(), [None]) + + def test_a_listener_that_never_ends_its_body(self): + # API-2: fails until the proof's response is read only up to 64 KiB, and its proof checked first + def endless(listener, request, proved): + """200 announcing 100 MiB, then 70 KiB of them, then nothing more.""" + headers = {apiclient.PROOF_HEADER: listener.proof(request)} if proved else {} + return response("200 OK", b"{" + b" " * (70 * 1024), headers, length=100 << 20), True + for proved in (False, True): + listener = FakeListener(self, lambda listener, request, proved=proved: endless(listener, request, proved)) + api = self.client(listener) + with mock.patch.object(apiclient, "PROOF_TIMEOUT", 30, create=True): + raised = self.finishes(api.status) + self.assertIsInstance(raised, AuthFailed if not proved else ConnectionFailed, proved) + self.assertEqual(listener.tokens(), [None], proved) + + def test_the_request_gets_the_clients_timeout(self): + """Once the daemon has proved itself, the request may take longer than PROOF_TIMEOUT (fetches can take + hours): the client's own timeout (none here) applies again.""" + node = FakeNode() + server = ApiServer(node, 0) + server.start() + self.addCleanup(server.stop) + cli.write_token(os.path.join(self.tmp, "api_token"), server.token) + cid = node.put_file(b"abc") + original = node.stat + + def slow(target): + """The stat, after a while.""" + time.sleep(1.0) + return original(target) + with mock.patch.object(apiclient, "PROOF_TIMEOUT", 0.3, create=True), \ + mock.patch.object(node, "stat", slow): + self.assertEqual(ApiConnection(self.tmp, port=server.port).stat(cid)["cid"], cid) + + +class ProofTimeoutTest(unittest.TestCase): + """The constants the plan fixes.""" + + def test_values(self): + """15 s to connect and prove; at most 64 KiB of the proof's response read.""" + self.assertEqual((apiclient.PROOF_TIMEOUT, apiclient.PROOF_READ_LIMIT), (15, 64 * 1024)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_api4_old_daemon_fallback.py b/tests/security/test_api4_old_daemon_fallback.py new file mode 100644 index 0000000..d37b54f --- /dev/null +++ b/tests/security/test_api4_old_daemon_fallback.py @@ -0,0 +1,62 @@ +"""API-4: a daemon before 0.1.5 closes the connection its proof was made on, so the client sent the request (and the +token) on a new connection. It did that for any daemon that closed it, which kept the old race for every daemon: the +daemon could stop between the proof and the new connection, and a program taking the port then got the token. + +Now the client reconnects only for a daemon whose proved version is below 0.1.5 (a deb-installed 0.1.4 still works); +any other daemon that closes the proof's connection gets ConnectionFailed, and nothing more is sent. The fallback is +to be dropped in 0.1.7. +""" +import json +import os +import tempfile +import unittest + +from rnfs import apiclient, cli +from rnfs.apiclient import ApiConnection, ConnectionFailed +from tests.security.test_api2_proof_first import TOKEN, FakeListener, response + + +def closing_daemon(version): + """A daemon that knows TOKEN, says version (None: says none) and closes every connection after its answer.""" + def respond(listener, request): + """The proof and version for a proof request, a status for the request after it.""" + if "x-rnfs-challenge" in request: + body = json.dumps({} if version is None else {"version": version}).encode() + return response("200 OK", body, {apiclient.PROOF_HEADER: listener.proof(request), "Connection": "close", + "Content-Type": "application/json"}), False + return response("200 OK", b'{"node_id": "ab"}', {"Connection": "close"}), False + return respond + + +class OldDaemonFallbackTest(unittest.TestCase): + """A client whose token file holds TOKEN, against daemons that close the proof's connection.""" + + def setUp(self): + """The token file.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = tmp.name + cli.write_token(os.path.join(self.tmp, "api_token"), TOKEN) + + def test_newer_daemon_closing_the_connection_gets_nothing_more(self): + # API-4: fails until only a daemon proved older than 0.1.5 gets the request on a new connection + for version in ("0.1.5", "0.1.6", "0.2.0", "1.0", "0.1.5.dev1", "garbage", "", None, 15): + listener = FakeListener(self, closing_daemon(version)) + api = ApiConnection(self.tmp, port=listener.port) + with self.assertRaises(ConnectionFailed, msg=version) as raised: + api.status() + self.assertIn("closed the connection", str(raised.exception), version) + self.assertEqual(listener.tokens(), [None], version) # only the proof request: no token sent + + def test_older_daemon_still_works(self): + # A pre-0.1.5 daemon (deb-installed 0.1.4) gets the request on a new connection, as its clients sent it + for version in ("0.1.4", "0.1.0", "0.1.4.post1", "0.0.9"): + listener = FakeListener(self, closing_daemon(version)) + api = ApiConnection(self.tmp, port=listener.port) + self.assertEqual(api.status(), {"node_id": "ab"}, version) + self.assertEqual(api.daemon_version, version) + self.assertEqual(listener.tokens(), [None, TOKEN], version) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_api5_head_deny.py b/tests/security/test_api5_head_deny.py new file mode 100644 index 0000000..f18b434 --- /dev/null +++ b/tests/security/test_api5_head_deny.py @@ -0,0 +1,130 @@ +"""API-5: the user's own HEAD, or a range not starting at byte 0, takes a file's Content-Type from its first bytes only +if they are stored (node_content.stored_head), and stored_head read them past the deny list: a HEAD sniffed a +denied stored chunk (its type leaked), and a HEAD of a stored file whose hash is denied answered 200 where GET +answered 451. + +Now stored_head raises Denied for a denied CID it meets, and while the deny list has file hashes it checks the file +(CONT-1's bounded check) against stored blocks only: a HEAD of a stored denied file answers 451, like GET. A file +not wholly stored is still never fetched for a HEAD, and `pin ls` is unchanged. +""" +import hashlib +import io +import os +import time +import unittest +from unittest import mock + +from rnfs.constants import CHUNK_SIZE +from rnfs.content import dag +from rnfs.content.cid import DAG, RAW, Cid +from rnfs.content.storage import USER +from rnfs.errors import Denied +from rnfs.gateway import GatewayServer +from tests.security._content_helpers import TwoNodeTest +from tests.security.test_cont1_deny_hash_amplification import repeated +from tests.test_api import raw_request + +PAGE = b"a page" + b"

text

" * 20_000 # several chunks, sniffed as HTML +DENIED_MESSAGE = b"unavailable: this node's deny list blocks this content\n" + + +class HeadDenyTest(TwoNodeTest): + """B's gateway answers the user's own HEAD of what B stores.""" + + def setUp(self): + super().setUp() + server = GatewayServer(self.b, self.b.config, 0) + server.start() + self.addCleanup(server.stop) + self.port = server.port + + def deny(self, *entries): + """Add entries to B's local list, its warnings silenced.""" + with mock.patch("rnfs.node.log"): + self.b.deny.add([str(entry) for entry in entries]) + + def stored(self, data: bytes) -> Cid: + """A file B added (pinned "user", so a deny-list change keeps it stored).""" + return Cid.parse(self.b.add_stream(io.BytesIO(data))["cid"]) + + def request(self, method: str, cid: Cid, headers: dict | None = None) -> tuple[int, str | None, bytes]: + """(status, Content-Type, body) of the user's own request (no Sec-Fetch-Site) for cid.""" + status, response, body = raw_request(self.port, method, f"/rnfs/{cid}", None, headers) + return status, response.get("Content-Type"), body + + def test_head_of_a_hash_denied_file(self): + """A stored file whose content hash is denied: HEAD and a later range answer 451, as GET does.""" + cid = self.stored(PAGE) + self.assertEqual(self.request("HEAD", cid)[:2], (200, "text/html")) + self.deny("sha256:" + hashlib.sha256(PAGE).hexdigest()) + status, _, body = self.request("GET", cid) + self.assertEqual((status, body), (451, DENIED_MESSAGE)) + self.b.deny._known.clear() # not remembered from the GET: the HEAD hashes it itself + self.assertEqual(self.request("HEAD", cid)[0], 451) + self.assertEqual(self.request("GET", cid, {"Range": f"bytes={CHUNK_SIZE}-{CHUNK_SIZE + 9}"})[0], 451) + self.assertEqual(self.request("HEAD", cid)[0], 451) # with the hash remembered + + def test_head_of_a_denied_first_chunk(self): + """A stored file whose first chunk's CID is denied isn't sniffed: HEAD answers 451.""" + cid = self.stored(PAGE) + first = Cid.for_block(RAW, PAGE[:CHUNK_SIZE]) + self.assertTrue(self.b.storage.has(first)) + self.deny(first) + self.assertTrue(self.b.storage.has(first), "a user pin's blocks stay stored") + status, content_type, _ = self.request("HEAD", cid) + self.assertEqual(status, 451) + self.assertNotEqual(content_type, "text/html") + with self.assertRaises(Denied): + self.b.stored_head(str(cid), 512) + + def test_head_of_the_bounded_poc(self): + """A stored file repeating one chunk 2^40 times answers 451 to a HEAD at once, its chunk unread.""" + blocks, root, _ = repeated(os.urandom(CHUNK_SIZE), 5) + with self.b.storage.protect() as session: + for cid, block in blocks.items(): + session.put(cid, block) + self.b.storage.pin(root, USER) + self.deny("sha256:" + hashlib.sha256(b"something else").hexdigest()) + get = self.b.storage.get + raw_reads = [] + with mock.patch.object(self.b.storage, "get", lambda cid, verify=False: ( + raw_reads.append(cid) if cid.is_raw else None) or get(cid, verify)): + started = time.monotonic() + self.assertEqual(self.request("HEAD", root)[0], 451) + self.assertLess(time.monotonic() - started, 10) + self.assertEqual(raw_reads, []) + + def test_allowed_and_not_wholly_stored(self): + """With hash entries, a stored file that isn't denied is still sniffed. One not wholly stored can't be + checked without fetching it, which a HEAD never does: it is typed by name only.""" + self.deny("sha256:" + hashlib.sha256(b"something else").hexdigest()) + cid = self.stored(PAGE) + self.assertEqual(self.request("HEAD", cid)[:2], (200, "text/html")) + # A file whose root and first chunk B stores, but not the rest: A has it all + rest = os.urandom(CHUNK_SIZE) + chunks = [PAGE[:CHUNK_SIZE], rest] + cids = [Cid.for_block(RAW, chunk) for chunk in chunks] + block = dag.encode(dag.FileNode([dag.Child(c, len(chunk)) for c, chunk in zip(cids, chunks)])) + partial = Cid.for_block(DAG, block) + self.serve({partial: block, **dict(zip(cids, chunks))}, partial) + with self.b.storage.protect() as session: + session.put(partial, block) + session.put(cids[0], chunks[0]) + self.b.storage.pin(partial, USER) + self.net.reset() + status, content_type, _ = self.request("HEAD", partial) + self.assertEqual((status, content_type), (200, "application/octet-stream")) + self.assertEqual(self.net.downloaded_blocks, 0) + self.assertFalse(self.b.storage.has(cids[1])) + + def test_pin_ls_is_unchanged(self): + """pin ls still reports whether a pinned file is sealed, denied or not, without hashing it.""" + cid = self.stored(PAGE) + self.deny("sha256:" + hashlib.sha256(PAGE).hexdigest(), Cid.for_block(RAW, PAGE[:CHUNK_SIZE])) + with mock.patch.object(self.b, "_check_file", side_effect=AssertionError("hashed")): + listed = {pin["cid"]: pin for pin in self.b.pin_ls()} + self.assertIs(listed[str(cid)]["sealed"], False) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_api7_missing_fetch_metadata.py b/tests/security/test_api7_missing_fetch_metadata.py new file mode 100644 index 0000000..b6f242a --- /dev/null +++ b/tests/security/test_api7_missing_fetch_metadata.py @@ -0,0 +1,111 @@ +"""API-7: the gateway counted every request without Sec-Fetch-Site as the user's own (a local tool such as curl). +Browsers without Fetch Metadata (Safari before 16.4, Firefox before 90) send none either, so in them any web page's +request was the user's own: it got ?format= downloads (reopening L6's storage oracle: a download for stored content, a fetch +for the rest) and wasn't held to gateway_fetches. + +Now (decision Q7) a request without Sec-Fetch-Site is the user's own only if its User-Agent doesn't start with +"Mozilla/", which every browser's does. Such a browser is treated as a web page's: ?format= gets 409 and network use +counts against gateway_fetches, whether or not the user typed the address. curl, scripts and other tools keep the +user's own access; so does a browser's own navigation that says so (Sec-Fetch-Site: none). +""" +import unittest + +from rnfs.config import Config +from rnfs.gateway import GatewayServer +from tests.security.test_l6_gateway_storage_oracle import PAGE, PNG, NetworkNode +from tests.test_api import raw_request + +# Browsers without Fetch Metadata: no Sec-Fetch-Site on any request, whoever started it +OLD_BROWSERS = { + "Safari 16.3": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) " + "Version/16.3 Safari/605.1.15", + "Firefox 89": "Mozilla/5.0 (X11; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0", + "Mozilla/ only": "Mozilla/", + "lower case": "mozilla/5.0 (compatible)", + "leading space": " Mozilla/5.0", +} +# The user's own tools: no Sec-Fetch-Site and a User-Agent that isn't a browser's (or none) +TOOLS = {"curl": "curl/8.5.0", "wget": "Wget/1.21.4", "python": "Python-urllib/3.12", "rnfs": "rnfs", + "no User-Agent": None, "empty": "", "Mozilla later": "MyTool/1.0 (like Mozilla/5.0)"} + + +class MissingFetchMetadataTest(unittest.TestCase): + """Two gateways, one whose node stores the content and one whose node has to fetch it.""" + + def setUp(self): + self.gateways = {} + for kind in ("stored", "unstored"): + node = NetworkNode() + remote = kind == "unstored" + page = node.put_file(PAGE, remote=remote) + data = node.put_file(PNG, remote=remote) + site = node.put_dir({"index.html": page, "data": data}, remote=remote) + server = GatewayServer(node, Config(gateway_fetches=2), 0) + server.start() + self.addCleanup(server.stop) + self.gateways[kind] = (node, server) + self.page, self.data, self.site = page, data, site + + def format_paths(self) -> list[str]: + """A block and an archive.""" + return [f"/rnfs/{self.data}?format=raw", f"/rnfs/{self.site}?format=archive"] + + def request(self, kind: str, method: str, path: str, user_agent: str | None, extra: dict | None = None) -> tuple: + """(status, body) of the request to the gateway of kind, with that User-Agent (None: none sent).""" + headers = {**({"User-Agent": user_agent} if user_agent is not None else {}), **(extra or {})} + status, _, body = raw_request(self.gateways[kind][1].port, method, path, None, headers) + return status, body + + def hold_fetch_slots(self): + """A page holds every gateway_fetches slot of both gateways with fetches that never finish.""" + for _, server in self.gateways.values(): + limiter = server.server.RequestHandlerClass.limiter + for _ in range(limiter.limit): + self.assertTrue(limiter.try_acquire()) + self.addCleanup(lambda limiter=limiter: [limiter.release() for _ in range(limiter.limit)]) + + def test_old_browsers_get_no_formats(self): + # API-7: fails until a request without Sec-Fetch-Site from a browser gets 409 for ?format=, stored or not + for name, user_agent in OLD_BROWSERS.items(): + for path in self.format_paths(): + for kind in self.gateways: + status, _ = self.request(kind, "GET", path, user_agent) + self.assertEqual(status, 409, (name, kind, path)) + for node, _ in self.gateways.values(): + self.assertEqual((node.called("block"), node.called("export_stream")), ([], [])) + self.assertEqual(self.gateways["unstored"][0].fetches, 0) + + def test_old_browsers_are_held_to_gateway_fetches(self): + # API-7: fails until a browser's request without Sec-Fetch-Site counts against gateway_fetches + self.hold_fetch_slots() + for name, user_agent in OLD_BROWSERS.items(): + self.assertEqual(self.request("unstored", "GET", f"/rnfs/{self.data}", user_agent)[0], 503, name) + self.assertEqual(self.request("stored", "GET", f"/rnfs/{self.data}", user_agent), + (200, PNG), name) # stored content is served without limit, as for any page + self.assertEqual(self.gateways["unstored"][0].fetches, 0) + + def test_tools_keep_the_users_own_access(self): + # curl and scripts still get ?format= (fetching what isn't stored) and aren't held to gateway_fetches + self.hold_fetch_slots() + for name, user_agent in TOOLS.items(): + for kind in self.gateways: + status, body = self.request(kind, "GET", f"/rnfs/{self.site}?format=archive", user_agent) + self.assertEqual((status, body), (200, f"archive of {self.site}".encode()), (name, kind)) + self.assertEqual(self.request(kind, "GET", f"/rnfs/{self.data}?format=raw", user_agent), + (200, PNG), (name, kind)) + self.assertEqual(self.request(kind, "GET", f"/rnfs/{self.data}", user_agent), (200, PNG), (name, kind)) + + def test_a_browsers_own_navigation_still_works(self): + # A browser with Fetch Metadata says Sec-Fetch-Site: none for the user's own navigation: the User-Agent + # doesn't matter then, nor for a page's request (cross-site stays limited whatever the User-Agent) + for user_agent in (*OLD_BROWSERS.values(), *TOOLS.values()): + status, _ = self.request("unstored", "GET", f"/rnfs/{self.site}?format=archive", user_agent, + {"Sec-Fetch-Site": "none"}) + self.assertEqual(status, 200, user_agent) + status, _ = self.request("unstored", "GET", f"/rnfs/{self.site}?format=archive", user_agent, + {"Sec-Fetch-Site": "cross-site"}) + self.assertEqual(status, 409, user_agent) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_cont1_deny_hash_amplification.py b/tests/security/test_cont1_deny_hash_amplification.py new file mode 100644 index 0000000..0f6d7a3 --- /dev/null +++ b/tests/security/test_cont1_deny_hash_amplification.py @@ -0,0 +1,234 @@ +"""CONT-1: while the deny list held a `sha256:` entry, a file was hashed whole before any of it was sent or +imported, and the hash read every occurrence of repeated content. Six blocks (about 128 KB: one 64 KiB chunk and +five file nodes, each listing the node below it 256 times) declare 2^56 bytes, so a 1-byte `cat`, a gateway +`Range: bytes=0-0` request, `?format=archive` or `import` of the 128 KB archive hashed for about a year, holding a +gateway slot all along. + +Now a file whose declared size is more than DENY_HASH_RATIO times the bytes of its distinct chunks, plus +DENY_HASH_SLACK, is refused as Denied (451) before any chunk is read: B holds the PoC whole (as an attacker can make +it, by having B fetch every block with ?format=raw), and its storage counts every raw block read, failing the read +past a limit so the old behaviour fails at once rather than hanging. A repetitive file inside the slack is still +served, an honest denied file still gets 451, and with no hash entries nothing is walked. +""" +import hashlib +import io +import os +import time +import unittest +from unittest import mock + +from rnfs import denylist +from rnfs.constants import CHUNK_SIZE, MAX_CHILDREN +from rnfs.content import archive, dag, walker +from rnfs.content.cid import DAG, RAW, Cid +from rnfs.content.storage import USER +from rnfs.errors import Denied, StorageFull +from rnfs.gateway import GatewayServer +from tests.security._content_helpers import TwoNodeTest +from tests.test_api import raw_request +from tests.test_node import A_ID + +DENIED_ELSEWHERE = "sha256:" + hashlib.sha256(b"some other file").hexdigest() # makes deny.hashes non-empty +QUICK = 10.0 # seconds a refusal may take, with room for a loaded machine + + +class RawRead(Exception): + """A raw block was read past the test's limit: the content is being hashed.""" + + +def repeated(chunk: bytes, levels: int, width: int = MAX_CHILDREN) -> tuple[dict[Cid, bytes], Cid, int]: + """A file of chunk repeated: levels file nodes, each listing the one below it width times. + + Returns (its blocks, its root, the size the root declares).""" + below = Cid.for_block(RAW, chunk) + blocks = {below: chunk} + size = len(chunk) + for _ in range(levels): + block = dag.encode(dag.FileNode([dag.Child(below, size)] * width)) + below = Cid.for_block(DAG, block) + blocks[below] = block + size *= width + return blocks, below, size + + +def joined(blocks: dict[Cid, bytes], parts: list[tuple[Cid, int]]) -> tuple[Cid, int]: + """A file node over parts (CID, size), added to blocks; (its CID, its size).""" + block = dag.encode(dag.FileNode([dag.Child(cid, size) for cid, size in parts])) + cid = Cid.for_block(DAG, block) + blocks[cid] = block + return cid, sum(size for _, size in parts) + + +class DenyHashAmplificationTest(TwoNodeTest): + """The 6-block PoC and its neighbours, read by B every way that checks file hashes.""" + + def setUp(self): + super().setUp() + self.chunk = os.urandom(CHUNK_SIZE) + self.blocks, self.root, self.size = repeated(self.chunk, 5) + self.assertEqual(self.size, 2 ** 56) + self.assertEqual(len(self.blocks), 6) + self.raw_reads = 0 + self.raw_limit = 0 # raw block reads allowed before RawRead + get = self.b.storage.get + + def counting_get(cid, verify=False): + """Storage.get, counting raw blocks read.""" + if cid.is_raw: + self.raw_reads += 1 + if self.raw_reads > self.raw_limit: + raise RawRead(f"raw block {cid} read {self.raw_reads} times") + return get(cid, verify) + + patcher = mock.patch.object(self.b.storage, "get", counting_get) + patcher.start() + self.addCleanup(patcher.stop) + + def deny(self, *entries): + """Add entries to B's local list, its warnings silenced.""" + with mock.patch("rnfs.node.log"): + self.b.deny.add(list(entries)) + + def store_on_b(self, blocks: dict[Cid, bytes], root: Cid): + """B holds root whole, pinned.""" + with self.b.storage.protect() as session: + for cid, block in blocks.items(): + session.put(cid, block) + self.b.storage.pin(root, USER) + self.assertTrue(self.b.storage.is_complete(root)) + + def gateway(self) -> int: + """B's gateway's port.""" + server = GatewayServer(self.b, self.b.config, 0) + server.start() + self.addCleanup(server.stop) + return server.port + + def assert_quick_and_unread(self, started: float): + """Refused within QUICK, without reading a raw block.""" + self.assertLess(time.monotonic() - started, QUICK) + self.assertEqual(self.raw_reads, 0) + + def test_constants(self): + self.assertEqual(denylist.DENY_HASH_RATIO, 16) + self.assertEqual(denylist.DENY_HASH_SLACK, 64 * 1024 * 1024) + + def test_cat(self): + """cat of one byte of the stored PoC is refused before any chunk is read.""" + self.store_on_b(self.blocks, self.root) + self.deny(DENIED_ELSEWHERE) + started = time.monotonic() + with self.assertRaisesRegex(Denied, "repeats its content more than 16"): + next(self.b.read(str(self.root), offset=0, length=1)) + self.assert_quick_and_unread(started) + # /api/cat: its info reads the first chunk (is it sealed?), and nothing more + self.raw_limit = 1 + info, chunks = self.b.read_file(str(self.root), length=1) + self.assertEqual(info["size"], 2 ** 56) + with self.assertRaises(Denied): + next(chunks) + self.assertLess(time.monotonic() - started, QUICK) + + def test_gateway_range(self): + """A gateway Range: bytes=0-0 request (and the type sniffing it implies) answers 451 at once.""" + self.store_on_b(self.blocks, self.root) + self.deny(DENIED_ELSEWHERE) + port = self.gateway() + started = time.monotonic() + status, _, body = raw_request(port, "GET", f"/rnfs/{self.root}", None, {"Range": "bytes=0-0"}) + self.assertEqual(status, 451) + self.assertEqual(body, b"unavailable: this node's deny list blocks this content\n") + self.assert_quick_and_unread(started) + + def test_gateway_archive(self): + """?format=archive answers 451 before anything is sent, and export_stream never starts.""" + self.store_on_b(self.blocks, self.root) + self.deny(DENIED_ELSEWHERE) + port = self.gateway() + started = time.monotonic() + status, _, _ = raw_request(port, "GET", f"/rnfs/{self.root}?format=archive") + self.assertEqual(status, 451) + begun = [] + with self.assertRaises(Denied): + self.b.export_stream(str(self.root), begun.append, for_gateway=True) + self.assertEqual(begun, []) + self.assert_quick_and_unread(started) + + def test_import(self): + """Importing the 128 KB archive is refused as Denied, pins nothing and leaves none of its blocks.""" + path = os.path.join(self.tmp.name, "poc.rnfs") + with open(path, "wb") as out: + archive.export_archive([self.root], lambda expect: self.blocks[expect.cid], out) + self.assertLess(os.path.getsize(path), 200 * 1024) + self.deny(DENIED_ELSEWHERE) + before = self.b.storage.block_count() + started = time.monotonic() + with self.assertRaisesRegex(Denied, "repeats its content"): + self.b.import_archive(path) + self.assert_quick_and_unread(started) + self.assertEqual(self.b.storage.pins(), {}) + self.assertEqual(self.b.storage.block_count(), before) + + def test_not_stored_is_refused_before_downloading_content(self): + """Fetched from A, the PoC is refused (its declared size can't fit on the disk, or by the ratio) without B + reading a chunk.""" + self.serve(self.blocks, self.root) + self.deny(DENIED_ELSEWHERE) + started = time.monotonic() + with self.assertRaises((Denied, StorageFull)): + next(self.b.read(str(self.root), offset=0, length=1)) + self.assert_quick_and_unread(started) + + def test_over_the_ratio_is_refused(self): + """A repeated-chunk file just past 16 × its chunk + 64 MiB (80 MiB of one chunk) is refused unread; one + inside it is served.""" + blocks, sixteen, sixteen_size = repeated(self.chunk, 1) # 16 MiB + over, over_size = joined(blocks, [(sixteen, sixteen_size)] * 5) + self.assertGreater(over_size, denylist.DENY_HASH_RATIO * CHUNK_SIZE + denylist.DENY_HASH_SLACK) + self.store_on_b(blocks, over) + self.deny(DENIED_ELSEWHERE) + with self.assertRaises(Denied): + next(self.b.read(str(over), offset=0, length=1)) + self.assertEqual(self.raw_reads, 0) + + def test_repetitive_file_inside_the_slack_is_served(self): + """A 32 MiB file of one repeated chunk is hashed (once: then remembered) and served, by cat and the + gateway.""" + blocks, sixteen, sixteen_size = repeated(self.chunk, 1) + root, size = joined(blocks, [(sixteen, sixteen_size)] * 2) + self.assertEqual(size, 32 * 1024 * 1024) + self.store_on_b(blocks, root) + self.deny(DENIED_ELSEWHERE) + self.raw_limit = 10 ** 6 + self.assertEqual(b"".join(self.b.read(str(root), offset=5, length=10)), self.chunk[5:15]) + self.assertEqual(self.b.deny.known_hash(root), hashlib.sha256(self.chunk * 512).digest()) + status, _, body = raw_request(self.gateway(), "GET", f"/rnfs/{root}", None, {"Range": "bytes=0-0"}) + self.assertEqual((status, body), (206, self.chunk[:1])) + + def test_honest_denied_file(self): + """A multi-chunk file whose hash is denied still gets Denied and 451, stored or fetched.""" + data = os.urandom(3 * CHUNK_SIZE + 7) + cid = Cid.parse(self.a.add_stream(io.BytesIO(data))["cid"]) + self.a.records.store(cid.key, A_ID) + self.deny("sha256:" + hashlib.sha256(data).hexdigest()) + self.raw_limit = 10 ** 6 + with self.assertRaises(Denied): + next(self.b.read(str(cid), offset=CHUNK_SIZE, length=1)) # fetched from A, then denied + status, _, _ = raw_request(self.gateway(), "GET", f"/rnfs/{cid}", None, {"Range": "bytes=0-0"}) + self.assertEqual(status, 451) + + def test_no_hash_entries_nothing_walked(self): + """With no sha256: entries, nothing is walked or hashed: a 1-byte read of the PoC reads one chunk.""" + self.store_on_b(self.blocks, self.root) + self.deny(str(Cid.for_block(RAW, b"denied by CID"))) + self.raw_limit = 8 # the read, and the gateway's sniff and range: a chunk each + with mock.patch.object(walker, "leaf_bytes", side_effect=AssertionError("walked")) as walk: + self.assertEqual(b"".join(self.b.read(str(self.root), offset=0, length=1)), self.chunk[:1]) + status, _, body = raw_request(self.gateway(), "GET", f"/rnfs/{self.root}", None, + {"Range": "bytes=0-0"}) + self.assertEqual((status, body), (206, self.chunk[:1])) + walk.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_cont2_write_out_swap.py b/tests/security/test_cont2_write_out_swap.py new file mode 100644 index 0000000..387a2d7 --- /dev/null +++ b/tests/security/test_cont2_write_out_swap.py @@ -0,0 +1,318 @@ +"""CONT-2: `get` created a directory's entries by path, after all its siblings, and never checked that the +directory was still the one it had made. Someone who could rename inside the output (a group member writing to a +umask 002 output in a shared setgid directory; on Windows, any user a folder's inherited ACL grants Modify) could +swap `partial/sub` for a symlink or junction between its mkdir and its children's writes, and get then created the +DAG's files, execute bit included, in a directory of their choosing (~/.config/autostart, the Startup folder). + +Now, on POSIX, the partial root is made private (0700, setgid kept) and must be empty, and every entry is created +relative to its parent's descriptor: directories are reopened from the root, one component at a time without +following links, and checked against the device and inode recorded when they were made. On Windows the root is +created with a private DACL and each file's final path must be inside it; the output then inherits its parent's +access list again before the rename. node.get checks the partial root is still the one write_out made before +renaming it. + +The swaps happen at a fixed point (while writing a file that comes after the directory in name order, so after the +directory's mkdir and before it is reopened for its children), so there is no timing race. The Windows logic runs +on any platform through the fake winacl backend of test_l7_windows_acl, with a symlink standing in for a junction; +the real advapi32 calls aren't exercised here. +""" +import os +import stat +import sys +import tempfile +import unittest +from unittest import mock + +from rnfs import fsutil, node as node_module, winacl +from rnfs.config import Config +from rnfs.content import builder, dag, walker +from rnfs.content.cid import EMPTY_DIRECTORY +from rnfs.content.dag import DirectoryNode, Entry +from rnfs.content.walker import root_expect +from rnfs.node import RnfsNode +from tests.security.test_l7_windows_acl import SHARED, USER, FakeAcl +from tests.support import CAN_SYMLINK, POSIX_MODES +from tests.test_walker import Blocks + +CHANGED = "output was changed while being written" +HAS_FINAL_PATH = os.name == "nt" or sys.platform.startswith("linux") or sys.platform == "darwin" +EVIL = b"[Desktop Entry]\nExec=sh -c 'echo pwned'\n" + + +class SwapTestCase(unittest.TestCase): + """A DAG {sub/evil.desktop, z.txt} in memory, a temporary directory and an attacker's directory in it.""" + + def setUp(self): + """The DAG and the directories.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dir = tmp.name + self.blocks = Blocks() + evil = builder.build_bytes(EVIL, self.blocks.put, kind=dag.KIND_EXECUTABLE) + self.z = builder.build_bytes(b"after sub in name order", self.blocks.put) + sub = builder.wrap("evil.desktop", evil, self.blocks.put) + self.root = builder.build_directory([Entry("sub", sub.cid, sub.size, dag.KIND_DIRECTORY), + Entry("z.txt", self.z.cid, self.z.size, dag.KIND_FILE)], self.blocks.put) + self.attacker = os.path.join(self.dir, "autostart") + os.mkdir(self.attacker) + self.dest = os.path.join(self.dir, "out") + + def swap_to_link(self, dest: str): + """What the attacker does: move dest/sub aside and put a symlink (a junction on Windows) to theirs there.""" + os.rename(os.path.join(dest, "sub"), os.path.join(self.dir, "moved")) + os.symlink(self.attacker, os.path.join(dest, "sub")) + + def swap_to_directory(self, dest: str): + """Or: move dest/sub aside and put a real directory of their own there.""" + os.rename(os.path.join(dest, "sub"), os.path.join(self.dir, "moved")) + os.mkdir(os.path.join(dest, "sub")) + + def write_out(self, swap=None, **kwargs) -> list[str]: + """walker.write_out of the DAG to self.dest, calling swap(dest) once, just before z.txt is created.""" + def check(file): + """The deny list's hook, called before each file is created: the attacker's moment.""" + if swap is not None and file.cid == self.z.cid: + swap(self.dest) + + return walker.write_out(root_expect(self.root.cid), self.blocks.get, self.dest, 100, check, **kwargs) + + +@unittest.skipUnless(CAN_SYMLINK and POSIX_MODES, "needs symlinks and POSIX modes") +class PosixSwapTest(SwapTestCase): + """The descriptor write-out (Linux, macOS).""" + + def test_swap_to_a_symlink_writes_nothing_outside(self): + with self.assertRaisesRegex(ValueError, CHANGED): + self.write_out(self.swap_to_link) + self.assertEqual(os.listdir(self.attacker), []) + self.assertEqual(os.listdir(os.path.join(self.dir, "moved")), []) # nor in the real one, moved away + + def test_swap_to_another_directory_fails_on_its_identity(self): + with self.assertRaisesRegex(ValueError, CHANGED): + self.write_out(self.swap_to_directory) + self.assertEqual(os.listdir(os.path.join(self.dest, "sub")), []) + + def test_unswapped_write_out_is_unchanged(self): + created = [] + self.assertEqual(self.write_out(created=created), []) + with open(os.path.join(self.dest, "sub", "evil.desktop"), "rb") as f: + self.assertEqual(f.read(), EVIL) + self.assertTrue(os.stat(os.path.join(self.dest, "sub", "evil.desktop")).st_mode & stat.S_IXUSR) + st = os.lstat(self.dest) + self.assertEqual(created, [(st.st_dev, st.st_ino)]) + + def test_root_is_private_while_written_and_normal_after(self): + # In a setgid directory, as shared ones are: setgid stays on the root, so what is created below it keeps + # the group, and the root gets back the mode a plain mkdir gives it + shared = os.path.join(self.dir, "shared") + os.mkdir(shared) + os.chmod(shared, 0o2775) + os.mkdir(os.path.join(shared, "reference")) + normal = stat.S_IMODE(os.lstat(os.path.join(shared, "reference")).st_mode) + self.dest = os.path.join(shared, "out") + modes = [] + self.write_out(lambda dest: modes.append(stat.S_IMODE(os.lstat(dest).st_mode))) + self.assertEqual(modes, [0o700 | (normal & stat.S_ISGID)]) + self.assertEqual(stat.S_IMODE(os.lstat(self.dest).st_mode), normal) + if normal & stat.S_ISGID: # a filesystem that honours setgid directories + self.assertTrue(os.lstat(os.path.join(self.dest, "sub")).st_mode & stat.S_ISGID) + + def test_entry_planted_before_the_chmod_aborts(self): + # Between the root's mkdir and its fchmod to 0700, anyone the parent's mode lets in can create an entry + # in it (say a symlink named like an entry of the DAG, or a directory they keep writing to) + real = os.mkdir + + def mkdir_then_plant(path, *args, **kwargs): + """os.mkdir, then the attacker's entry if it was the root.""" + real(path, *args, **kwargs) + if path == self.dest: + os.symlink(self.attacker, os.path.join(path, "sub")) + + with mock.patch("os.mkdir", mkdir_then_plant), self.assertRaisesRegex(ValueError, CHANGED): + self.write_out() + self.assertEqual(os.listdir(self.attacker), []) + + def test_root_swapped_for_a_symlink_before_it_is_opened(self): + real = os.mkdir + + def mkdir_then_swap(path, *args, **kwargs): + """os.mkdir, then the root moved aside and a symlink to the attacker's directory put in its place.""" + real(path, *args, **kwargs) + if path == self.dest: + os.rename(path, os.path.join(self.dir, "moved")) + os.symlink(self.attacker, path) + + with mock.patch("os.mkdir", mkdir_then_swap), self.assertRaisesRegex(ValueError, CHANGED): + self.write_out() + self.assertEqual(os.listdir(self.attacker), []) + + def test_ten_thousand_subdirectories_dont_exhaust_descriptors(self): + # The stack holds names and identities, not descriptors: a few are open at once, however many directories + self.blocks[EMPTY_DIRECTORY] = dag.encode(DirectoryNode([])) + count = 10_000 + empty = [Entry(f"d{i:05}", EMPTY_DIRECTORY, 0, dag.KIND_DIRECTORY) for i in range(count)] + middle = builder.build_directory(empty, self.blocks.put) + top = builder.build_directory([Entry("m", middle.cid, 0, dag.KIND_DIRECTORY)], self.blocks.put) + import resource # POSIX only, as this class is: imported here so the module loads on Windows + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + fd_dir = "/proc/self/fd" if os.path.isdir("/proc/self/fd") else "/dev/fd" + highest = max(int(name) for name in os.listdir(fd_dir)) + resource.setrlimit(resource.RLIMIT_NOFILE, (highest + 32, hard)) + try: + walker.write_out(root_expect(top.cid), self.blocks.get, self.dest, count + 1) + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + self.assertEqual(len(os.listdir(os.path.join(self.dest, "m"))), count) + + +@unittest.skipUnless(CAN_SYMLINK, "needs symlinks") +class NodeGetSwapTest(unittest.TestCase): + """node.get: a swap inside the partial output, or of the partial output itself, fails the get and leaves + no output.""" + + def setUp(self): + """An offline node holding {sub/evil.desktop, z.txt}, and an attacker's directory.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dir = tmp.name + self.node = RnfsNode.offline(Config(rnfs_dir=os.path.join(self.dir, "node")), b"\x01" * 16, + client=mock.Mock(), names_network=mock.Mock()) + self.addCleanup(self.node.stop) + src = os.path.join(self.dir, "src") + os.makedirs(os.path.join(src, "sub")) + for name, data in (("sub/evil.desktop", EVIL), ("z.txt", b"z")): + with open(os.path.join(src, name), "wb") as f: + f.write(data) + self.cid = self.node.add_path(src)["cid"] + self.attacker = os.path.join(self.dir, "autostart") + os.mkdir(self.attacker) + self.out = os.path.join(self.dir, "out") + + def partial(self) -> str: + """The get's partial output, beside out.""" + [name] = [n for n in os.listdir(self.dir) if n.startswith(".rnfs-get-")] + return os.path.join(self.dir, name) + + def test_swap_inside_the_partial_fails_the_get(self): + real = walker._write_file + + def write_and_swap(expect, *args, **kwargs): + """_write_file; before z.txt, the attacker's swap of sub.""" + if expect.size == 1: + sub = os.path.join(self.partial(), "sub") + os.rename(sub, os.path.join(self.dir, "moved")) + os.symlink(self.attacker, sub) + return real(expect, *args, **kwargs) + + with mock.patch.object(walker, "_write_file", write_and_swap), self.assertRaisesRegex(ValueError, CHANGED): + self.node.get(self.cid, self.out) + if fsutil.IS_WINDOWS: + # Writing by path, Windows creates the file where the link leads before its final path can be checked: + # it is refused unwritten, and left there empty, as removing it by path could be redirected too + # (walker._PathOutput). Other users are kept out of the partial output by its private access list + self.assertEqual({name: os.path.getsize(os.path.join(self.attacker, name)) + for name in os.listdir(self.attacker)}, {"evil.desktop": 0}) + else: + self.assertEqual(os.listdir(self.attacker), []) + self.assertFalse(os.path.lexists(self.out)) + self.assertEqual([n for n in os.listdir(self.dir) if n.startswith(".rnfs-get-")], []) + + def test_partial_swapped_after_write_out_is_not_renamed(self): + real = walker.write_out + + def write_then_swap(expect, get, dest, *args, **kwargs): + """write_out, then the whole partial output replaced by a directory of the attacker's.""" + warnings = real(expect, get, dest, *args, **kwargs) + os.rename(dest, os.path.join(self.dir, "moved")) + os.rename(self.attacker, dest) + return warnings + + with open(os.path.join(self.attacker, "theirs"), "wb"): + pass + with mock.patch.object(walker, "write_out", write_then_swap), self.assertRaisesRegex(ValueError, CHANGED): + self.node.get(self.cid, self.out) + self.assertFalse(os.path.lexists(self.out)) + # What was put in the partial's place isn't the get's to delete: it is left where it was put + self.assertEqual(os.listdir(self.partial()), ["theirs"]) + + def test_get_writes_the_output(self): + self.assertEqual(self.node.get(self.cid, self.out)["warnings"], []) + with open(os.path.join(self.out, "sub", "evil.desktop"), "rb") as f: + self.assertEqual(f.read(), EVIL) + if POSIX_MODES: + reference = os.path.join(self.dir, "reference") + os.mkdir(reference) + self.assertEqual(os.lstat(self.out).st_mode, os.lstat(reference).st_mode) + + +@unittest.skipUnless(CAN_SYMLINK, "needs symlinks") +class WindowsSwapTest(SwapTestCase): + """The path write-out of Windows, with a fake winacl backend and a symlink standing in for a junction.""" + + def setUp(self): + super().setUp() + self.acl = FakeAcl() + for patch in (mock.patch.object(fsutil, "IS_WINDOWS", True), mock.patch.object(winacl, "backend", self.acl)): + patch.start() + self.addCleanup(patch.stop) + + def test_root_is_created_private(self): + self.assertEqual(self.write_out(), []) + self.assertEqual(self.acl.created, [self.dest]) + self.assertIsNone(winacl.problem(self.acl.read(self.dest), USER)) + with open(os.path.join(self.dest, "sub", "evil.desktop"), "rb") as f: + self.assertEqual(f.read(), EVIL) + + @unittest.skipUnless(HAS_FINAL_PATH, "this platform can't give a file's final path") + def test_file_below_a_swapped_directory_is_refused(self): + # Windows can't open a directory as a descriptor, so the file is created (empty) through the junction, but + # its final path gives it away before a byte is written. It isn't removed: by path, that could be redirected + with self.assertRaisesRegex(ValueError, CHANGED): + self.write_out(self.swap_to_link) + self.assertEqual(os.listdir(self.attacker), ["evil.desktop"]) + self.assertEqual(os.path.getsize(os.path.join(self.attacker, "evil.desktop")), 0) + + def test_get_restores_inheritance_before_the_rename(self): + node = RnfsNode.offline(Config(rnfs_dir=os.path.join(self.dir, "node")), b"\x01" * 16, + client=mock.Mock(), names_network=mock.Mock()) + self.addCleanup(node.stop) + src = os.path.join(self.dir, "src") + os.makedirs(os.path.join(src, "sub")) + with open(os.path.join(src, "sub", "f"), "wb") as f: + f.write(b"f") + cid = node.add_path(src)["cid"] + out = os.path.join(self.dir, "got") + renamed = [] + real = fsutil.rename_new + + def rename_new(src, dst): + """fsutil.rename_new, noting what had been made to inherit by then.""" + renamed.append(list(self.acl.inherit_calls)) + real(src, dst) + + with mock.patch.object(fsutil, "rename_new", rename_new): + node.get(cid, out) + [[partial]] = renamed + self.assertTrue(os.path.basename(partial).startswith(".rnfs-get-")) + self.assertEqual(self.acl.read(out), SHARED) # its parent's access list, as for anything made there + # A single file is made in a folder the user chose, so it inherits from the start + self.acl.inherit_calls.clear() + node.get(cid + "/sub/f", os.path.join(self.dir, "f")) + self.assertEqual(self.acl.inherit_calls, []) + # If its access list can't be relaxed, the output stays private (safe) and the get still completes + again = os.path.join(self.dir, "again") + with mock.patch.object(self.acl, "inherit", side_effect=PermissionError(5, "Access is denied")), \ + mock.patch.object(node_module, "log") as log: + node.get(cid, again) + self.assertIsNone(winacl.problem(self.acl.read(again), USER)) + self.assertIn("stays private to you", log.call_args.args[0]) + + def test_empty_acl(self): + # What the real backend sets, unprotected, to inherit again: an ACL of no entries + acl = winacl.empty_acl() + self.assertEqual(int.from_bytes(acl[2:4], "little"), len(acl)) + self.assertEqual(winacl.parse_acl(acl), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_cont3_shared_folder_target.py b/tests/security/test_cont3_shared_folder_target.py new file mode 100644 index 0000000..708742f --- /dev/null +++ b/tests/security/test_cont3_shared_folder_target.py @@ -0,0 +1,253 @@ +"""CONT-3: the shared folder's scan follows a symlink at files/ (the owner's setting), and imported whatever it led +to. Pointed at (or, through a parent another user can write, retargeted to) the rnfs dir's storage/ or names/, it +pinned and provided the node's identity or the name keys (anyone who knew their CIDs could fetch them); pointed at +the rnfs dir or a directory above it, everything in it. + +Now the scan resolves the folder it opened (fsutil.final_path of its descriptor, or the real path the path walk +checks) and refuses, with a warning shown once, a folder that is the rnfs dir, a directory holding it, or one inside +storage/, names/, state/, blocks/ or lxmf/. It is scanned as an empty folder: nothing is imported, and every pin +with source "folder" is released, including those a 0.1.5 node made from such a folder before the upgrade. A folder +below a directory other users can change gets a warning, once, and is scanned. + +Reproduced with real symlinks in a temporary rnfs dir laid out as the node lays it out; the retarget is the +attacker's rename-and-link done between two scans. +""" +import os +import tempfile +import time +import unittest +from unittest import mock + +from rnfs import fsutil +from rnfs.config import Config +from rnfs.content import builder, shared_folder +from rnfs.content.cid import RAW, Cid +from rnfs.content.shared_folder import SharedFolder +from rnfs.content.storage import FOLDER, Storage +from rnfs.node import RnfsNode +from tests.support import CAN_SYMLINK, POSIX_MODES, state_rows +from tests.test_node import A_ID, FakeNetwork, NetClient + +IDENTITY = b"the node's private RNS identity\n" +NAME_KEY = b"a name's private signing key\n" +PRIVATE = ("storage", "names", "state", "blocks", "lxmf") + + +@unittest.skipUnless(CAN_SYMLINK, "creating symlinks isn't allowed here") +class SharedFolderTargetTest(unittest.TestCase): + """An rnfs dir holding an identity and a name key, its storage, and files/ as a symlink the test points.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = os.path.join(self.tmp.name, "rnfs") + for name in PRIVATE: + os.makedirs(os.path.join(self.root, name)) + with open(os.path.join(self.root, "storage", "identity"), "wb") as f: + f.write(IDENTITY) + with open(os.path.join(self.root, "names", "0" * 32 + ".key"), "wb") as f: + f.write(NAME_KEY) + self.files = os.path.join(self.root, "files") + self.state = os.path.join(self.root, "state") + self.storage = Storage(os.path.join(self.root, "blocks"), self.state, 10**9) + self.addCleanup(self.storage.close) + self.logged: list[str] = [] + patcher = mock.patch.object(shared_folder, "log", lambda message, level=None: self.logged.append(message)) + patcher.start() + self.addCleanup(patcher.stop) + + def open(self) -> SharedFolder: + """A SharedFolder over files/, closed when the test ends.""" + folder = SharedFolder(self.files, self.storage, self.state) + self.addCleanup(folder.close) + return folder + + def point_at(self, target: str): + """Make files/ a symlink to target (replacing the one there).""" + if os.path.lexists(self.files): + os.remove(self.files) + os.symlink(target, self.files, target_is_directory=True) + + def directory(self, *parts: str, content: bytes = b"shared") -> str: + """A directory below the temporary one holding a.txt with content.""" + path = os.path.join(self.tmp.name, *parts) + os.makedirs(path, exist_ok=True) + with open(os.path.join(path, "a.txt"), "wb") as f: + f.write(content) + return path + + def assert_refused(self, folder: SharedFolder, keys_stored: bool = False): + """A scan imports nothing and leaves no folder pin and no saved entry (as an empty folder's scan does), the + keys stay unstored (unless keys_stored), and one warning names the rnfs dir and the released pins.""" + before = len(self.logged) + self.assertEqual(folder.scan(), set()) + self.assertEqual(self.storage.pinned_by(FOLDER), set()) + self.assertEqual(folder.names(), {}) + self.assertEqual(state_rows(self.state, "SELECT name FROM shared_folder"), []) + if not keys_stored: + for secret in (IDENTITY, NAME_KEY): + self.assertFalse(self.storage.is_complete(Cid.for_block(RAW, secret))) + warnings = self.logged[before:] + self.assertEqual(len(warnings), 1, warnings) + self.assertIn("not scanning", warnings[0]) + self.assertIn("released", warnings[0]) + self.assertIn(os.path.realpath(self.root), warnings[0]) + + def test_private_subdirectories_are_refused(self): + """files/ leading to storage/, names/, state/, blocks/ or lxmf/, or a directory inside one, is refused.""" + os.makedirs(os.path.join(self.root, "names", "deeper")) + for target in [*PRIVATE, os.path.join("names", "deeper")]: + with self.subTest(target): + self.point_at(os.path.join(self.root, target)) + self.assert_refused(self.open()) + + def test_rnfs_dir_and_ancestors_are_refused(self): + """files/ leading to the rnfs dir itself or to a directory above it is refused.""" + for target in (self.root, self.tmp.name): + with self.subTest(target): + self.point_at(target) + self.assert_refused(self.open()) + + def test_warned_once(self): + """Scans of a refused folder warn once while it stays refused, and again if it is refused after a good + scan.""" + self.point_at(os.path.join(self.root, "storage")) + folder = self.open() + self.assert_refused(folder) + self.assertEqual(folder.scan(), set()) + self.assertEqual(len(self.logged), 1) + self.point_at(self.directory("elsewhere")) + self.assertEqual(len(folder.scan()), 1) + self.point_at(os.path.join(self.root, "names")) + self.assert_refused(folder) + + def test_retargeted_folder_releases_its_pins_and_imports_nothing(self): + """The audit's attack: files/ leads to shared/x, which someone who can write to shared/ swaps for a link to + names/. The next scan imports nothing and releases what the folder shared, as for an empty folder.""" + target = self.directory("shared", "x", content=b"the owner's shared file") + with open(os.path.join(target, "b.txt"), "wb") as f: + f.write(b"another") + self.point_at(target) + folder = self.open() + pins = folder.scan() + self.assertEqual(pins, {Cid.for_block(RAW, b"the owner's shared file"), Cid.for_block(RAW, b"another")}) + self.assertEqual(self.storage.pinned_by(FOLDER), pins) + os.rename(target, target + ".moved") + os.symlink(os.path.join(self.root, "names"), target, target_is_directory=True) + self.logged.clear() + self.assert_refused(folder) + + def test_file_cache_sits_behind_the_guard(self): + """The per-file cache (audit D1) is consulted only after the folder passed the check: a retargeted folder's + scan looks nothing up in it, and forgets what it held, so going back reads every file again.""" + target = self.directory("shared", "x", content=b"the owner's shared file") + os.makedirs(os.path.join(target, "sub")) + with open(os.path.join(target, "sub", "b.txt"), "wb") as f: + f.write(b"another") + self.point_at(target) + folder = self.open() + self.assertEqual(len(folder.scan()), 2) + moved = target + ".moved" + os.rename(target, moved) + os.symlink(os.path.join(self.root, "names"), target, target_is_directory=True) + self.logged.clear() + with mock.patch.object(shared_folder.FolderFiles, "get", side_effect=AssertionError("cache consulted")): + self.assert_refused(folder) + self.assertEqual(folder.files._index, {}) + os.remove(target) + os.rename(moved, target) + with mock.patch.object(builder, "build_file", wraps=builder.build_file) as build_file: + self.assertEqual(len(folder.scan()), 2) + self.assertEqual(build_file.call_count, 2) + + def test_upgrade_releases_keys_an_earlier_scan_imported(self): + """A 0.1.5 node whose files/ already led into names/ imported, pinned and provided the name key. The first + 0.1.6 scan (over the same state) releases that pin and empties the shared_folder table.""" + self.point_at(os.path.join(self.root, "names")) + earlier = self.open() + with mock.patch.object(shared_folder, "refused_reason", lambda real, rnfs_dir: None): # as 0.1.5 scanned + imported = earlier.scan() + key = Cid.for_block(RAW, NAME_KEY) + self.assertEqual(imported, {key}) + self.assertEqual(self.storage.pinned_by(FOLDER), {key}) + earlier.close() + upgraded = self.open() + self.assertEqual(set(upgraded.names()), {key}) # the saved entry, as 0.1.5 left it + self.assert_refused(upgraded, keys_stored=True) + self.assertNotIn(key, self.storage.pins()) + + def test_elsewhere_is_scanned(self): + """A folder outside the rnfs dir (or files/ as a real directory in it) is scanned as before.""" + self.point_at(self.directory("elsewhere")) + self.assertEqual(self.open().scan(), {Cid.for_block(RAW, b"shared")}) + os.remove(self.files) + os.mkdir(self.files) + with open(os.path.join(self.files, "b.txt"), "wb") as f: + f.write(b"in files") + self.assertEqual(self.open().scan(), {Cid.for_block(RAW, b"in files")}) + + def test_refused_without_the_descriptor_path(self): + """Where the system can't name an open directory's path (fsutil.final_path gives None), the folder's real + path is used, checked to be the directory opened.""" + with mock.patch.object(fsutil, "final_path", lambda fd: None): + self.point_at(os.path.join(self.root, "names")) + self.assert_refused(self.open()) + self.logged.clear() + self.point_at(self.directory("elsewhere")) + self.assertEqual(len(self.open().scan()), 1) + + def test_refused_by_the_path_walk(self): + """Where directories are walked by path (FD_WALK False, as on Windows), the same folders are refused.""" + with mock.patch.object(builder, "FD_WALK", False): + self.point_at(os.path.join(self.root, "storage")) + self.assert_refused(self.open()) + self.logged.clear() + self.point_at(self.directory("elsewhere")) + self.assertEqual(len(self.open().scan()), 1) + + @unittest.skipUnless(POSIX_MODES, "Unix permission bits") + def test_loose_ancestor_is_warned_about_once(self): + """A folder below a directory others can write to is scanned, with one warning naming that directory; a + sticky one (as /tmp) holding a directory of the user's own isn't.""" + loose = os.path.join(self.tmp.name, "loose") + target = self.directory("loose", "x") + os.chmod(loose, 0o777) + self.point_at(target) + folder = self.open() + self.assertEqual(len(folder.scan()), 1) + self.assertEqual(len(folder.scan()), 1) + warnings = [line for line in self.logged if "other users" in line] + self.assertEqual(len(warnings), 1, self.logged) + self.assertIn(os.path.realpath(loose), warnings[0]) + self.logged.clear() + sticky = os.path.join(self.tmp.name, "sticky") + target = self.directory("sticky", "x") + os.chmod(sticky, 0o1777) + self.point_at(target) + self.assertEqual(len(self.open().scan()), 1) + self.assertEqual([line for line in self.logged if "other users" in line], []) + + +@unittest.skipUnless(CAN_SYMLINK, "creating symlinks isn't allowed here") +class NodeSharedFolderTargetTest(unittest.TestCase): + """The node's maintenance scan refuses a files/ leading into its own rnfs dir.""" + + def test_names_dir_not_shared(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + config = Config(rnfs_dir=os.path.join(tmp.name, "node"), slow="no") + node = RnfsNode.offline(config, A_ID, NetClient(), FakeNetwork()) + self.addCleanup(node.stop) + with open(os.path.join(config.names_dir, "0" * 32 + ".key"), "wb") as f: + f.write(NAME_KEY) + if os.path.lexists(config.files_dir): + os.rmdir(config.files_dir) + os.symlink(config.names_dir, config.files_dir, target_is_directory=True) + with mock.patch.object(shared_folder, "log"): + node.maintain(time.monotonic(), {}) + self.assertEqual(node.storage.pinned_by(FOLDER), set()) + self.assertFalse(node.storage.is_complete(Cid.for_block(RAW, NAME_KEY))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_cont4_denied_hash_notice.py b/tests/security/test_cont4_denied_hash_notice.py new file mode 100644 index 0000000..82d067c --- /dev/null +++ b/tests/security/test_cont4_denied_hash_notice.py @@ -0,0 +1,57 @@ +"""CONT-4 / API-6: a `sha256:` deny entry is checked where a file's content is assembled (get, cat, the gateway, add, +import), so it doesn't stop a multi-chunk file's blocks being served one by one (/blocks, the gateway's +?format=raw of a chunk) or provided. Specification section 8 documents that, and says to deny the CID for it. + +Kept as documented (turning remembered hashes into CID denies is deferred), but the node now says so where it +matters: when _check_file finds a denied hash, it logs a NOTICE, once per file CID while it runs, suggesting +`rnfs deny add `. Doing that stops the blocks being served. +""" +import hashlib +import io +import os +from unittest import mock + +from rnfs.constants import CHUNK_SIZE +from rnfs.content import dag +from rnfs.content.cid import Cid +from rnfs.errors import Denied +from rnfs.log import LOG_NOTICE +from tests.test_node import NodeTest + + +class DeniedHashNoticeTest(NodeTest): + """B holds a three-chunk file of its own, pinned and provided, whose content hash it then denies.""" + + def setUp(self): + super().setUp() + self.data = os.urandom(3 * CHUNK_SIZE) + self.cid = Cid.parse(self.b.add_stream(io.BytesIO(self.data))["cid"]) + self.chunks = [child.cid for child in dag.decode(self.b.storage.get(self.cid)).children] + self.assertEqual(len(self.chunks), 3) + with mock.patch("rnfs.node.log"): + self.b.deny.add(["sha256:" + hashlib.sha256(self.data).hexdigest()]) + + def read(self) -> list: + """Read the file on B as cat does, expecting Denied; what B logged meanwhile, as (message, level).""" + logged = [] + with mock.patch("rnfs.node.log", lambda message, level=None: logged.append((message, level))): + with self.assertRaises(Denied): + next(self.b.read(str(self.cid))) + return logged + + def test_notice_suggests_denying_the_cid_once(self): + """The first refusal logs a NOTICE naming `rnfs deny add `; the next one doesn't repeat it.""" + notices = [message for message, level in self.read() if level == LOG_NOTICE] + self.assertEqual(len(notices), 1, notices) + self.assertIn(f"rnfs deny add {self.cid}", notices[0]) + self.assertEqual([message for message, level in self.read() if level == LOG_NOTICE], []) + + def test_blocks_still_served_until_the_cid_is_denied(self): + """What the notice is for: with only the hash denied, the file's blocks are still served to /blocks; + once its CID is denied, none of them are.""" + for cid in [self.cid, *self.chunks]: + self.assertEqual(self.b.backend.served_block(cid), self.b.storage.get(cid)) + with mock.patch("rnfs.node.log"): + self.b.deny.add([str(self.cid)]) + for cid in [self.cid, *self.chunks]: + self.assertIsNone(self.b.backend.served_block(cid)) diff --git a/tests/security/test_dht1_keeper_query_budget.py b/tests/security/test_dht1_keeper_query_budget.py new file mode 100644 index 0000000..11119dd --- /dev/null +++ b/tests/security/test_dht1_keeper_query_budget.py @@ -0,0 +1,152 @@ +"""DHT-1: a stalled lookup left keeper /name queries 1 s, censoring offline owners' names. + +Sybil server IDs ground close to a name's keeper key answer /find slowly, each time with fresh Sybil IDs closer to the +key, so the beta closest nodes never all answer. With fewer than 3 × alpha keepers the provider search never had +enough candidates to stop, so it used the whole lookup_timeout, and the /name queries then got +max(1 s, time left): a keeper whose link takes seconds to set up was asked, but its answer was dropped. The fix gives +/name its own budget, stops the search once candidates exist and lookup_timeout / 5 has passed (resuming it if their +answers don't settle the name), and bounds each lookup run (a query cap and a no-progress stop). + +Offline nodes over GrindingNet (a SybilNet, tests/security/_dhtr2b_helpers.py): the name is at sequence 2, kept only +by D. Resolver B (alpha 3) knows two grinding Sybils, and holds D's provider record unless a test says otherwise. +""" +import os +import threading +import time +import unittest +from unittest import mock + +from rnfs.config import Config +from rnfs.constants import K +from rnfs.dht import lookup +from rnfs.features import keepers +from rnfs.node import RnfsNode +from tests.security import _dhtr2b_helpers +from tests.security._dhtr2b_helpers import B_ID, D_ID, KeeperScenario, SybilNet +from tests.test_names import FakeNetwork + +S_ID = bytes([0x55] * 16) # an honest DHT server, slow to answer +X_ID = bytes([0x66] * 16) # a keeper that has left: its /name request fails +SYBIL_DELAY = 0.3 # seconds each grinding Sybil takes to answer /find +KEEPER_DELAY = 3.0 # seconds keeper D takes to answer /name (link setup over a slow path) + + +class GrindingNet(SybilNet): + """SybilNet with grinding Sybils: each answers /find after SYBIL_DELAY with K fresh Sybil IDs and no providers, + every second answer's a bit nearer the key (about 250 answers before they run out of bits). Nodes in find_delay + / name_delay answer late.""" + + def __init__(self): + """No grinding Sybils yet.""" + super().__init__() + self.grinders: set[bytes] = set() + self.closeness = 1 << 126 # the distance of the next IDs handed out, halved by every second answer + self.answers = 0 + self.find_delay: dict[bytes, float] = {} + self.name_delay: dict[bytes, float] = {} + self.name_times: dict[bytes, float] = {} # when each node was first asked /name (time.monotonic) + self.grind_lock = threading.Lock() + + def find(self, node_id, key): + """A grinding Sybil's /find answer, or the node's (late if find_delay says so).""" + if node_id in self.grinders: + with self.grind_lock: + self.find_requests.append(node_id) + time.sleep(SYBIL_DELAY) + with self.grind_lock: + self.answers += 1 + if self.answers % 2 == 0: + self.closeness = max(1 << 16, self.closeness // 2) + fresh = self.closeness + (self.answers % 2) * K # the two answers at one distance differ + closer = [(int.from_bytes(key, "big") ^ (fresh + i)).to_bytes(16, "big") for i in range(K)] + self.grinders.update(closer) + return [], closer + time.sleep(self.find_delay.get(node_id, 0)) + return super().find(node_id, key) + + def name_record(self, node_id, name): + """The node's /name answer, late if name_delay says so.""" + self.name_times.setdefault(node_id, time.monotonic()) + time.sleep(self.name_delay.get(node_id, 0)) + return super().name_record(node_id, name) + + +class KeeperQueryBudgetTest(KeeperScenario): + """The name at sequence 2 kept only by D; resolver B is made by each test.""" + + def setUp(self): + """The scenario over a GrindingNet.""" + with mock.patch.object(_dhtr2b_helpers, "SybilNet", GrindingNet): + super().setUp() + + def make_node(self, label: str, node_id: bytes, slow: bool = False, lookup_timeout: float = 5) -> RnfsNode: + """An offline node with its own directory and the given lookup_timeout.""" + config = Config(rnfs_dir=os.path.join(self.dir, label), slow="no", lookup_timeout=lookup_timeout, + path_timeout=0.2) + node = RnfsNode.offline(config, node_id, self.net, FakeNetwork(), slow=slow) + self.net.nodes[node_id] = node + self.addCleanup(node.stop) + return node + + def resolver(self, lookup_timeout: float, keeper_recorded: bool = True) -> RnfsNode: + """B, knowing two grinding Sybils near the key (and holding D's provider record if keeper_recorded).""" + b = self.make_node("b", B_ID, lookup_timeout=lookup_timeout) + seeds = [self.near_key(1 << 125), self.near_key((1 << 125) + 1)] + self.net.grinders.update(seeds) + for seed in seeds: + b.routing.heard(seed) + if keeper_recorded: + self.assertTrue(b.records.store(self.key, D_ID)) + self.net.name_delay[D_ID] = KEEPER_DELAY + return b + + def test_name_wait_at_the_defaults(self): + """path_timeout + link setup over 5 hops + twice RNS's request timeout: 87.5 s with the default settings.""" + self.assertEqual(keepers.name_wait(Config()), 87.5) + + def test_keeper_answer_survives_a_stalled_lookup(self): + b = self.resolver(lookup_timeout=10) + started = time.monotonic() + records = b.feature("keepers").ask_keepers(self.name) + took = time.monotonic() - started + asked = self.net.name_times.get(D_ID) + self.assertIsNotNone(asked, "keeper D was never asked") + # DHT-1: fails until /name has a budget of its own: D was asked with 1 s left, and its answer dropped + self.assertEqual(self.best(records), 2, f"D asked at {asked - started:.1f}s, the query took {took:.1f}s") + # The search stopped at lookup_timeout / 5 with D in hand; D was asked then, not when the lookup gave up + self.assertLess(asked - started, 10 / 5 + 1.5) + # Its answers didn't settle the name (D isn't established), so the search resumed for the rest of the budget + self.assertLess(took, 10 + KEEPER_DELAY) + + def test_stalled_lookup_is_bounded(self): + """With no keeper in sight the search can't stop early: the lookup's query cap ends it, long before + lookup_timeout, having sent at most MAX_QUERIES /find requests.""" + b = self.resolver(lookup_timeout=40, keeper_recorded=False) + started = time.monotonic() + records = b.feature("keepers").ask_keepers(self.name) + took = time.monotonic() - started + self.assertEqual(records, []) + # DHT-1: fails until a lookup run is bounded (it ran to lookup_timeout) + self.assertLess(took, 16, f"the lookup ran {took:.1f}s with {len(self.net.find_requests)} /find requests") + self.assertLessEqual(len(self.net.find_requests), lookup.MAX_QUERIES) + + def test_resumed_search_finds_a_later_keeper(self): + """B holds a record of X, which has left; slow server S lists D, but answers after the search's soft budget. + X is asked when the budget runs out; its answer doesn't settle the name, so the search resumes and finds D.""" + s = self.make_node("s", S_ID) + b = self.make_node("b", B_ID, lookup_timeout=10) + b.routing.heard(S_ID) + self.net.find_delay[S_ID] = 4.0 + self.assertTrue(s.records.store(self.key, D_ID)) + self.assertTrue(b.records.store(self.key, X_ID)) + started = time.monotonic() + records = b.feature("keepers").ask_keepers(self.name) + self.assertEqual(self.best(records), 2) + self.assertIn(X_ID, self.net.name_times) + # DHT-1: fails until the search stops at lookup_timeout / 5 once candidates exist (not once S answered) + self.assertLess(self.net.name_times[X_ID] - started, 10 / 5 + 1.5) + self.assertGreater(self.net.name_times[D_ID], self.net.name_times[X_ID]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_dht2_replayed_announce.py b/tests/security/test_dht2_replayed_announce.py new file mode 100644 index 0000000..ac152ed --- /dev/null +++ b/tests/security/test_dht2_replayed_announce.py @@ -0,0 +1,223 @@ +"""DHT-2: a replayed announce inherits the newest emission time. + +RNS's Identity.remember stores the app_data of every validly signed announce, a neighbour's replay of an old one +included, as just heard, while the path table keeps the random blobs of the announces the path was taken from, so +the emission time names read (emitted_time: the newest blob) is the newer announce's. An old record was so paired +with a new emission time and looked current: a resolve returned it with age 0 and didn't ask keepers, which hold the +newer record. + +The fix reads the remembered announce's own emission time from RNS's announce cache (RnsNetwork.recalled_emission) +and flags a remembered announce older than the path's, or carrying another record than the path's, as a suspected +replay: its heard time is its own emission time (or the path's, if its own is unknown), and the resolve asks keepers. + +RNS's state is built offline as Transport.inbound and Identity.validate_announce leave it (test_rns_compat checks +those internals): real announce packets, signed with a patched clock for their emission times, remembered in +Identity.known_destinations, and, for one that takes the path, cached and entered in Transport.path_table. +""" +import os +import tempfile +import time +import types +import unittest +from unittest import mock + +import RNS + +from rnfs import names +from rnfs.config import Config +from rnfs.constants import APP_NAME, NAME_ASPECT +from rnfs.content.cid import RAW, Cid +from rnfs.names import NameManager, RnsNetwork, encode_record +from tests.test_names import FakeKeepers, FakePins + +OLD = Cid.for_block(RAW, b"the page before") +NEW = Cid.for_block(RAW, b"the page now") +DAY = 24 * 3600 +BLOB_AT = RNS.Identity.KEYSIZE // 8 + RNS.Identity.NAME_HASH_LENGTH // 8 # where an announce's random blob starts + + +class OfflineNetwork(RnsNetwork): + """The real RnsNetwork, but a path request is answered by the test's answer() (RNS hearing an announce), if any, + and counted.""" + + def __init__(self): + """No path requests yet, and nobody answering.""" + self.path_requests = [] + self.answer = None + + def request_path(self, name: bytes): + """Record the request and run the answer.""" + self.path_requests.append(name) + if self.answer is not None: + self.answer() + + +class ReplayedAnnounceTest(unittest.TestCase): + """A name's owner announced sequence 3 two days ago and sequence 4 an hour ago; a NameManager over the real + RnsNetwork resolves it.""" + + def setUp(self): + """The owner's destination (not registered), an RNS announce cache in a temporary directory, and a + manager; RNS's tables are restored afterwards.""" + saved = RNS.loglevel + RNS.loglevel = RNS.LOG_CRITICAL + self.addCleanup(setattr, RNS, "loglevel", saved) + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + os.mkdir(os.path.join(tmp.name, "announces")) + for patch in (mock.patch.object(RNS.Reticulum, "cachepath", tmp.name), + mock.patch.object(RNS.Reticulum, "get_instance", return_value=types.SimpleNamespace( + _used_destination_data=lambda destination_hash: None))): + patch.start() + self.addCleanup(patch.stop) + with mock.patch.object(RNS.Transport, "register_destination"): + self.owner = RNS.Destination(RNS.Identity(), RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, + NAME_ASPECT) + self.name = self.owner.hash + self.addCleanup(RNS.Identity.known_destinations.pop, self.name, None) + self.addCleanup(RNS.Transport.path_table.pop, self.name, None) + now = int(time.time()) + self.emitted3, self.emitted4 = now - 2 * DAY, now - 3600 + self.announce3 = self.announce(3, OLD, self.emitted3) + self.announce4 = self.announce(4, NEW, self.emitted4) + self.config = Config(rnfs_dir=os.path.join(tmp.name, "rnfs"), path_timeout=0.05) + self.network = OfflineNetwork() + self.manager = NameManager(self.config, FakePins(), network=self.network) + self.addCleanup(self.manager.close) + + def announce(self, sequence: int, cid: Cid, emitted: int, path_response: bool = False) -> RNS.Packet: + """The owner's announce of [sequence, cid], emitted at emitted (as a path response if asked: RNS hashes the + context too, so the same announce then has another packet hash).""" + with mock.patch("time.time", return_value=float(emitted)): + packet = self.owner.announce(app_data=encode_record(sequence, cid), send=False) + if path_response: + packet = RNS.Packet(self.owner, packet.data, RNS.Packet.ANNOUNCE, context=RNS.Packet.PATH_RESPONSE, + context_flag=packet.context_flag) + packet.pack() + return packet + + def hear(self, packet: RNS.Packet, takes_path: bool, path_expired: bool = False): + """RNS hears packet: Identity.validate_announce remembers it, and Transport.inbound, if it takes the path, + caches it and makes it the path's announce, adding its random blob to those the path has.""" + app_data = bytes(packet.data[BLOB_AT + 10 + RNS.Identity.SIGLENGTH // 8:]) + RNS.Identity.remember(packet.get_hash(), self.name, self.owner.identity.get_public_key(), app_data) + if takes_path: + RNS.Transport.cache(packet, force_cache=True, packet_type="announce") + previous = RNS.Transport.path_table.get(self.name) + blobs = previous[4] if previous else [] + blob = bytes(packet.data[BLOB_AT:BLOB_AT + 10]) + if blob not in blobs: + blobs.append(blob) + expires = time.time() + (-60 if path_expired else 7 * DAY) + RNS.Transport.path_table[self.name] = [time.time(), self.name, 2, expires, blobs, None, packet.get_hash()] + + def forget_cached(self, packet: RNS.Packet): + """Transport.clean_announce_cache removes packet's file (it is no path's announce now).""" + os.remove(os.path.join(RNS.Reticulum.cachepath, "announces", packet.get_hash().hex())) + + def resolve(self, keepers: FakeKeepers | None) -> dict: + """A resolve for the user, with keepers (holding sequence 4) or none.""" + if keepers is not None: + keepers.answers[self.name] = [(4, NEW, float(self.emitted4))] + self.manager.keepers = keepers + return self.manager.resolve(self.name, persist=True) + + def test_replay_of_an_older_announce_asks_keepers(self): + """The path holds sequence 4; a neighbour repeats the announce of 3 (which took the path two days ago): + keepers are asked, and 4 is returned.""" + self.hear(self.announce3, takes_path=True) + self.hear(self.announce4, takes_path=True) + # A neighbour answers the path request with the announce of 3: RNS remembers it, the path stays + self.network.answer = lambda: self.hear(self.announce3, takes_path=False) + keepers = FakeKeepers() + result = self.resolve(keepers) + # DHT-2: fails until the replay's own emission time is read (it returned 3, and asked no keeper) + self.assertEqual(keepers.queried, [self.name]) + self.assertEqual(result["sequence"], 4) + + def test_replay_without_keepers_has_its_real_age(self): + """With no keepers the replayed record is all there is: 3 is returned, as old as its own announce.""" + self.hear(self.announce3, takes_path=True) + self.hear(self.announce4, takes_path=True) + self.network.answer = lambda: self.hear(self.announce3, takes_path=False) + result = self.resolve(None) + self.assertEqual(result["sequence"], 3) + # DHT-2: fails until the replay's own emission time is read (its age was 0) + self.assertAlmostEqual(result["age"], time.time() - self.emitted3, delta=60) + + def test_replay_no_longer_cached_is_as_old_as_the_path(self): + """A replayed announce RNS no longer caches, carrying another record than the path's, is heard no later + than the path's announce was emitted, and keepers are asked.""" + self.hear(self.announce3, takes_path=True) + self.hear(self.announce4, takes_path=True) + self.forget_cached(self.announce3) + self.network.answer = lambda: self.hear(self.announce3, takes_path=False) + result = self.resolve(None) + self.assertEqual(result["sequence"], 3) + self.assertAlmostEqual(result["age"], time.time() - self.emitted4, delta=60) + keepers = FakeKeepers() + self.assertEqual(self.resolve(keepers)["sequence"], 4) + self.assertEqual(keepers.queried, [self.name]) + + def test_replay_taking_an_expired_path(self): + """Once the path has expired, the replay (with more hops) takes it, but the path keeps the newer announce's + blob: the replay is older than the path, so keepers are asked and 4 is returned.""" + self.hear(self.announce3, takes_path=True) + self.hear(self.announce4, takes_path=True, path_expired=True) + self.network.answer = lambda: self.hear(self.announce3, takes_path=True) + keepers = FakeKeepers() + result = self.resolve(keepers) + # DHT-2: fails until the path's own announce is compared with the path's newest emission + self.assertEqual(keepers.queried, [self.name]) + self.assertEqual(result["sequence"], 4) + + def test_replay_heard_by_the_handler_is_as_old_as_its_announce(self): + """The replay taking the expired path reaches the announce handler: its record is stored as heard when its + announce was emitted, not now.""" + self.hear(self.announce3, takes_path=True) + self.hear(self.announce4, takes_path=True, path_expired=True) + self.hear(self.announce3, takes_path=True) + self.manager._handler.received_announce(self.name, None, encode_record(3, OLD), self.announce3.get_hash()) + record = self.manager.record(self.name) + self.assertEqual(record.sequence, 3) + # DHT-2: fails until the replay's own emission time is read (it was heard now) + self.assertEqual(record.heard, self.emitted3) + + def test_the_paths_own_announce_is_no_replay(self): + """The path's own announce, and that announce repeated in a path response (another packet hash, never + cached), show 4 as current and ask no keeper.""" + self.hear(self.announce3, takes_path=True) + self.hear(self.announce4, takes_path=True) + # Heard again (another path's rebroadcast: the same packet hash), then a transit node's path response + for repeat in (self.announce4, self.announce(4, NEW, self.emitted4, path_response=True)): + with self.subTest(context=repeat.context): + self.network.answer = lambda: self.hear(repeat, takes_path=False) + keepers = FakeKeepers() + result = self.resolve(keepers) + self.assertEqual((result["sequence"], keepers.queried), (4, [])) + self.assertLess(result["age"], 60) + + def test_an_older_announce_of_the_same_record_is_no_replay(self): + """A transit node answering a path request with an older announce of the path's record (its own path is + older) changes nothing: the record is the path's, current, and no keeper is asked.""" + self.hear(self.announce(4, NEW, self.emitted4 - 6 * 3600), takes_path=True) + self.hear(self.announce4, takes_path=True) + older = self.announce(4, NEW, self.emitted4 - 6 * 3600, path_response=True) + self.network.answer = lambda: self.hear(older, takes_path=False) + keepers = FakeKeepers() + result = self.resolve(keepers) + self.assertEqual((result["sequence"], keepers.queried), (4, [])) + self.assertLess(result["age"], 60) + + def test_without_a_path_nothing_is_compared(self): + """RNS remembers the announce of 3 but holds no path (it expired and was culled): nothing shows a newer + announce, so the record is as old as RNS last heard it, as before.""" + self.network.answer = lambda: self.hear(self.announce3, takes_path=False) + result = self.resolve(None) + self.assertEqual(result["sequence"], 3) + self.assertLess(result["age"], 60) + self.assertEqual(names.ANNOUNCE_VALIDITY, 2) # what bounds an announce's record when a path holds it + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_dht3_follow_quota.py b/tests/security/test_dht3_follow_quota.py new file mode 100644 index 0000000..12f7781 --- /dev/null +++ b/tests/security/test_dht3_follow_quota.py @@ -0,0 +1,144 @@ +"""DHT-3: a followed name's owner could fill the follower's disk. The follow fetch (follow.py, `fetcher.open(target, +pinned=True)`) had no size limit beyond max_download_size (no default) and the disk reserve, and a pinned fetch never +stops at, or evicts for, max_storage: pointing the name at a large target pinned all of it. + +B follows A's name with a small follow_quota. A first publishes a small file, which B pins, then points the name at +a target over the quota: a file larger than it, or a directory of empty files (size 0) whose directory nodes hold +more bytes than it. Each check now fails with the quota in its state, having downloaded little more than the root; +the old target stays pinned and the new one isn't. follow_quota = 0 fetches nothing at all, and a target pinned +before the quota was lowered stays pinned (no migration). +""" +import io +import os +import tempfile +import unittest + +import RNS + +from rnfs.config import Config +from rnfs.constants import GIB +from rnfs.content.cid import Cid +from rnfs.features.follow import follow_source +from rnfs.node import RnfsNode +from tests.security._content_helpers import SLACK +from tests.test_names import FakeNetwork +from tests.test_names_keep import KeeperNet + +A_ID, B_ID = bytes([0xA] * 16), bytes([0xB] * 16) +QUOTA = 50_000 + + +class FollowQuotaTest(unittest.TestCase): + """Owner A and follower B as offline nodes; B follows A's name, at a small file pinned already.""" + + def setUp(self): + """Two nodes that know each other, and A's name pointing at a small file B follows and pins.""" + saved = RNS.loglevel + RNS.loglevel = RNS.LOG_CRITICAL + self.addCleanup(setattr, RNS, "loglevel", saved) + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.net = KeeperNet() + self.net.sender = B_ID + self.a = self.make_node("a", A_ID) + self.b = self.make_node("b", B_ID, follow_quota=QUOTA) + self.a.routing.heard(B_ID) + self.b.routing.heard(A_ID) + self.name = bytes.fromhex(self.a.name_create("site")["name"]) + self.downloaded = 0 + real = self.net.blocks + + def counting(node_id, cids, on_progress=None): + """KeeperNet.blocks, adding up the bytes returned.""" + result = real(node_id, cids, on_progress) + self.downloaded += sum(len(block) for block in result.blocks.values()) + return result + + self.net.blocks = counting + self.follow = self.b.feature("follow") + self.source = follow_source(self.name) + self.honest = self.move(Cid.parse(self.a.add_stream(io.BytesIO(b"version one"))["cid"]), 1) + self.follow.follow(self.name) + self.assertTrue(self.follow.check(self.name, network=False)) + self.assertEqual(self.b.storage.pins().get(self.honest), {self.source}) + self.downloaded = 0 + + def make_node(self, label: str, node_id: bytes, **settings) -> RnfsNode: + """An offline node with its own directory.""" + config = Config(rnfs_dir=os.path.join(self.tmp.name, label), slow="no", lookup_timeout=5, path_timeout=0.2, + **settings) + node = RnfsNode.offline(config, node_id, self.net, FakeNetwork()) + self.net.nodes[node_id] = node + self.addCleanup(node.stop) + return node + + def move(self, target: Cid, sequence: int) -> Cid: + """Point A's name at target (which A provides), as if B heard the announce.""" + self.a.records.store(target.key, A_ID) + self.a.name_publish("site", str(target)) + self.b.names.offer(self.name, sequence, target, 0.0) + return target + + def empty_files(self, count: int) -> str: + """A directory on disk of count empty files with long names: size 0, but kilobytes of directory nodes.""" + root = os.path.join(self.tmp.name, "empties") + os.mkdir(root) + for i in range(count): + open(os.path.join(root, f"{i:04d}" + "n" * 120), "wb").close() + return root + + def assert_refused(self, target: Cid): + """The check of the moved name failed on the quota, downloading little; the old target is still pinned.""" + self.assertFalse(self.follow.check(self.name, network=False)) + entry = self.follow.following()[0] + self.assertIn("follow_quota", entry["state"]) + self.assertLessEqual(self.downloaded, SLACK) + pins = self.b.storage.pins() + self.assertEqual(pins.get(self.honest), {self.source}) + self.assertNotIn(target, pins) + self.assertEqual(entry["pinned"], str(self.honest)) + + def test_default_is_one_gib(self): + self.assertEqual(Config().follow_quota, 1 * GIB) + + def test_target_larger_than_the_quota_is_refused(self): + big = Cid.parse(self.a.add_stream(io.BytesIO(os.urandom(4 * QUOTA)))["cid"]) + self.assert_refused(self.move(big, 2)) + + def test_target_whose_blocks_exceed_the_quota_is_refused(self): + """Size 0, but more than QUOTA bytes of directory nodes: max_dag_bytes counts them.""" + empties = Cid.parse(self.a.add_path(self.empty_files(400))["cid"]) + self.assertGreater(self.a.storage.dag_bytes(empties), QUOTA) + self.move(empties, 2) + self.assertFalse(self.follow.check(self.name, network=False)) + entry = self.follow.following()[0] + self.assertIn("follow_quota", entry["state"]) + # The quota stops the fetch at about one request beyond it + self.assertLessEqual(self.downloaded, QUOTA + SLACK) + self.assertNotIn(empties, self.b.storage.pins()) + self.assertEqual(self.b.storage.pins().get(self.honest), {self.source}) + + def test_target_within_the_quota_is_pinned(self): + small = self.move(Cid.parse(self.a.add_stream(io.BytesIO(os.urandom(QUOTA // 2)))["cid"]), 2) + self.assertTrue(self.follow.check(self.name, network=False)) + self.assertEqual(self.b.storage.pins().get(small), {self.source}) + self.assertNotIn(self.honest, self.b.storage.pins()) + + def test_zero_quota_fetches_nothing(self): + self.b.config.follow_quota = 0 + small = self.move(Cid.parse(self.a.add_stream(io.BytesIO(b"version two"))["cid"]), 2) + self.assertFalse(self.follow.check(self.name, network=False)) + self.assertIn("follow_quota", self.follow.following()[0]["state"]) + self.assertEqual(self.downloaded, 0) + self.assertNotIn(small, self.b.storage.pins()) + self.assertEqual(self.b.storage.pins().get(self.honest), {self.source}) + + def test_target_pinned_before_the_quota_was_lowered_stays(self): + self.b.config.follow_quota = 1 + self.assertTrue(self.follow.check(self.name, network=False)) + self.assertEqual(self.follow.following()[0]["state"], "current") + self.assertEqual(self.b.storage.pins().get(self.honest), {self.source}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_dht4_private_name_announces.py b/tests/security/test_dht4_private_name_announces.py new file mode 100644 index 0000000..969190c --- /dev/null +++ b/tests/security/test_dht4_private_name_announces.py @@ -0,0 +1,94 @@ +"""DHT-4: in private mode, name records are still announced network-wide, so outsiders learn members' [sequence, +CID] records and can link a name's changes; only the content stays member-only. Nothing told the user. + +The announce itself is unchanged (the wire stays as it is): `name publish` in private mode now warns. The result of +NameManager.publish (and so of node.name_publish and POST /api/name/publish) gains an optional `warning` key, which +the CLI prints to stderr; outside private mode the result is as before, with no such key. +""" +import io +import os +import tempfile +import unittest +from unittest import mock + +import RNS + +from rnfs.config import Config +from rnfs.content.cid import RAW, Cid +from rnfs.names import NameManager +from rnfs.node import RnfsNode +from tests.support import FakeClock +from tests.test_cli import CliTestCase +from tests.test_names import FakeNetwork, FakePins + +CID = Cid.for_block(RAW, b"a member's page") +MEMBER = "ab" * 16 + + +class PrivateNamePublishTest(unittest.TestCase): + """A NameManager in a temporary rnfs dir, public or private.""" + + def setUp(self): + """The temporary directory; RNS's log silenced.""" + saved = RNS.loglevel + RNS.loglevel = RNS.LOG_CRITICAL + self.addCleanup(setattr, RNS, "loglevel", saved) + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + + def publish(self, **settings) -> dict: + """Create a name and publish CID under it with these settings; the result.""" + config = Config(rnfs_dir=self.tmp.name, path_timeout=0.2, **settings) + clock = FakeClock(1_000_000.0) + manager = NameManager(config, FakePins(), network=FakeNetwork(), clock=clock, monotonic=clock) + self.addCleanup(manager.close) + manager.create("site") + return manager.publish("site", CID) + + def test_private_mode_publish_warns(self): + result = self.publish(private=True, members=[MEMBER]) + self.assertIn("warning", result) + self.assertIn("private", result["warning"]) + self.assertIn("outside", result["warning"]) + self.assertEqual(result["cid"], str(CID)) # published all the same + + def test_public_publish_result_is_unchanged(self): + result = self.publish() + self.assertEqual(set(result), {"name", "cid", "sequence"}) + + def test_node_passes_the_warning_on(self): + config = Config(rnfs_dir=os.path.join(self.tmp.name, "node"), slow="no", private=True, members=[MEMBER]) + node = RnfsNode.offline(config, bytes([0xA] * 16), None, FakeNetwork()) + self.addCleanup(node.stop) + node.name_create("site") + cid = node.add_stream(io.BytesIO(b"a member's page"))["cid"] + self.assertIn("warning", node.name_publish("site", cid)) + + +class PrivateNamePublishCliTest(CliTestCase): + """`rnfs name publish` prints the result's warning to stderr.""" + + def test_cli_prints_the_warning(self): + real = self.node.name_publish + + def warning_publish(alias, target): + """FakeNode.name_publish, as a private node answers.""" + return {**real(alias, target), "warning": "name announces are visible outside the private network"} + + cid = self.node.put_file(b"a member's page") + self.ok("name", "create", "site") + with mock.patch.object(self.node, "name_publish", warning_publish): + code, out, err = self.run_cli("name", "publish", "site", cid) + self.assertEqual(code, 0, err) + self.assertIn("published", out.decode()) + self.assertIn("rnfs: warning: name announces are visible outside the private network", err) + + def test_cli_prints_no_warning_without_one(self): + cid = self.node.put_file(b"a member's page") + self.ok("name", "create", "site") + code, out, err = self.run_cli("name", "publish", "site", cid) + self.assertEqual((code, err), (0, "")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_dht5_own_provider_record.py b/tests/security/test_dht5_own_provider_record.py new file mode 100644 index 0000000..ea26f06 --- /dev/null +++ b/tests/security/test_dht5_own_provider_record.py @@ -0,0 +1,83 @@ +"""DHT-5: a node's own provider record could be evicted from its own store by Sybil stores to the key. + +Records are kept per key up to max_per_key publishers; a newcomer to a full key evicted a random publisher outside +the K/2 oldest records, and the node's own record (own_id, which the node records for every key it provides) was +one of the candidates. A handful of Sybil identities storing first take the senior places; after that about 1000 +stores to the key evicted the node's own record in 50 of 50 trials. The node's own records could likewise be the +victim of a full newcomer pool. Now own_id's record doesn't count toward max_per_key and is never the victim, of +either limit; everyone else's limits are unchanged. +""" +import random +import unittest + +from rnfs.constants import ID_LENGTH, K +from rnfs.dht.providers import ProviderStore +from tests.support import FakeClock + +OWN = bytes([0x0E] * ID_LENGTH) +KEY = bytes([0xAB] * ID_LENGTH) +SEEDS = 50 +SYBIL_STORES = 1000 +MAX_PER_KEY = 20 + + +def sybil(n: int) -> bytes: + """The n-th Sybil identity: a 16-byte ID unlike OWN.""" + return (0xF << 124 | n).to_bytes(ID_LENGTH, "big") + + +class OwnProviderRecordTest(unittest.TestCase): + """Stores with own_id set, flooded by fresh identities, over 50 eviction seeds.""" + + def setUp(self): + """The clock.""" + self.clock = FakeClock(1_700_000_000.0) + + def make_store(self, seed: int, max_records: int = 100_000) -> ProviderStore: + """A memory-only store recording OWN as this node, its eviction picks seeded.""" + return ProviderStore(max_records, None, clock=self.clock, rng=random.Random(seed), max_per_key=MAX_PER_KEY, + own_id=OWN) + + def test_sybil_stores_to_a_key_never_evict_the_own_record(self): + evicted = [] + for seed in range(SEEDS): + store = self.make_store(seed) + for n in range(K // 2): # the senior places, taken before the node records itself + self.assertTrue(store.store(KEY, sybil(n))) + self.assertTrue(store.store(KEY, OWN)) + for n in range(K // 2, K // 2 + SYBIL_STORES): + self.assertTrue(store.store(KEY, sybil(n))) # newcomers are still never locked out + providers = store.providers(KEY) + if OWN not in providers: + evicted.append(seed) + continue + # The own record doesn't count: the key still holds max_per_key others, the oldest K/2 of them kept + self.assertEqual(len(providers), MAX_PER_KEY + 1) + others = [p for p in providers if p != OWN] + self.assertEqual(others[:K // 2], [sybil(n) for n in range(K // 2)]) + self.assertEqual(evicted, [], f"own record evicted in {len(evicted)} of {SEEDS} trials") + + def test_own_store_to_a_full_key_evicts_no_one(self): + store = self.make_store(1) + for n in range(MAX_PER_KEY): + store.store(KEY, sybil(n)) + self.assertTrue(store.store(KEY, OWN)) + self.assertEqual(set(store.providers(KEY)), {sybil(n) for n in range(MAX_PER_KEY)} | {OWN}) + + def test_a_full_newcomer_pool_never_evicts_own_records(self): + """Own records fill the store outside the pool and spill into it; a flood of identities then fills the + pool and churns it, taking only each other's records.""" + for seed in range(SEEDS): + store = self.make_store(seed, max_records=100) + own_keys = [(1 << 100 | n).to_bytes(ID_LENGTH, "big") for n in range(store.main_size + 5)] + for key in own_keys: + self.assertTrue(store.store(key, OWN)) + for n in range(SYBIL_STORES): + store.store((2 << 100 | n).to_bytes(ID_LENGTH, "big"), sybil(n)) + missing = [key for key in own_keys if OWN not in store.providers(key)] + self.assertEqual(missing, [], f"seed {seed}: {len(missing)} own records evicted") + self.assertLessEqual(store.record_count(), store.max_records) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_feat1_auto_sender_share.py b/tests/security/test_feat1_auto_sender_share.py new file mode 100644 index 0000000..63615f4 --- /dev/null +++ b/tests/security/test_feat1_auto_sender_share.py @@ -0,0 +1,153 @@ +"""FEAT-1: one auto-fetch sender could push out another's pinned messages. + +A record holding auto-fetched pins (or queued to) is protected: it is dropped for other senders' messages only when +every record is protected, and then the oldest record of anyone went. Senders in lxmf_auto_fetch_from had no +per-sender cap, and re-linking a CID the sender already pins records "pinned" without using quota. So an auto-fetch +sender sending about 500 messages re-linking one tiny CID filled the inbox with protected records, and each further +message dropped the oldest record overall: another auto-fetch sender's message, whose `lxmf:` pin `_release` then +released. The spec's threat table promises "auto-fetched pins never pushed out by others' messages". + +Node B runs the lxmf feature on a stand-in network (tests/test_lxmf.py's FeatureTestCase) with VICTIM and ATTACKER +both in lxmf_auto_fetch_from. VICTIM sends one link, which B fetches and pins; ATTACKER sends 600 re-links of one +small file, each fetched (pinned at no quota) before the next. Fixed: each auto-fetch sender's records, protected +ones included, are limited to 1/n of the inbox's record and byte limits (n auto-fetch senders); over its share a +sender's own records go, unprotected first, then those whose pins another of its records also holds, then the +oldest; and when every other record is protected, only the incoming sender's own record may go (or the message is +refused), never another sender's. +""" +import hashlib +import io +import unittest + +from rnfs.content.cid import Cid +from rnfs.features import lxmf +from rnfs.features.lxmf import Incoming, Link, MessageLog +from tests import test_lxmf +from tests.test_lxmf import A_ID, OTHER, SENDER + +VICTIM, ATTACKER = OTHER, SENDER +RELINKS = 600 + + +def setUpModule(): + """Silence the node's log.""" + test_lxmf.setUpModule() + + +def tearDownModule(): + """Restore RNS's loglevel.""" + test_lxmf.tearDownModule() + + +class AutoSenderShareTest(test_lxmf.FeatureTestCase): + """VICTIM and ATTACKER in lxmf_auto_fetch_from; VICTIM's link fetched and pinned first.""" + + def setUp(self): + """B's feature, VICTIM's pinned message, and the small file ATTACKER re-links.""" + super().setUp() + self.feature = self.feature_node(lxmf_auto_fetch_from=[VICTIM.hex(), ATTACKER.hex()]) + self.victim = self.feature.received(self.link_message(source=VICTIM, number=1)) + self.feature.process_auto_fetches() + self.victim_cid = Cid.parse(self.a.info(f"{self.cid}/file.txt")["cid"]) + self.assertEqual(self.feature.inbox.get(self.victim)["auto"][0]["state"], "pinned") + small = Cid.parse(self.a.add_stream(io.BytesIO(b"x" * 47))["cid"]) + self.small = Link(small, from_node=A_ID) + self.sent = 0 + + def relink(self, text: bytes = b"") -> int | None: + """ATTACKER sends the small file's link again (with text), and B auto-fetches it; the inbox id.""" + self.sent += 1 + content = text + b"\n" + self.small.uri().encode() + message = Incoming(ATTACKER, hashlib.sha256(b"relink %d" % self.sent).digest(), 1.7e9, b"", content, + lxmf.encode_field([self.small]), "direct") + entry_id = self.feature.received(message) + self.feature.process_auto_fetches() + return entry_id + + def attacker_records(self) -> list[dict]: + """ATTACKER's inbox records.""" + return [e for e in self.feature.inbox_list() if e["from"] == ATTACKER.hex()] + + def test_relinks_keep_the_victims_record_and_pin(self): + for _ in range(RELINKS): + self.relink() + # FEAT-1: fails until one auto-fetch sender can't push out another's protected records + self.assertIsNotNone(self.feature.inbox.get(self.victim), + "another auto-fetch sender's re-links pushed out the victim's pinned message") + self.assertIn(f"lxmf:{VICTIM.hex()}", self.b.storage.pins().get(self.victim_cid, set()), + "the victim's auto-fetched pin was released") + # The attacker keeps its share of the records, and its pin + self.assertEqual(len(self.attacker_records()), lxmf.INBOX_LIMIT // 2) + self.assertIn(f"lxmf:{ATTACKER.hex()}", self.b.storage.pins()[self.small.cid]) + self.assertTrue(all(e["auto"][0]["state"] == "pinned" for e in self.attacker_records())) + + def test_share_holds_by_bytes(self): + self.feature.inbox.max_bytes = 200_000 + for _ in range(100): + self.relink(text=b"y" * 3000) + share = self.feature.inbox.max_bytes // 2 + used = sum(lxmf._json_bytes(e) for e in self.attacker_records()) + self.assertLessEqual(used, share) + self.assertGreater(used, share - 2 * lxmf.MAX_ENTRY_BYTES) # it does get its share + self.assertIsNotNone(self.feature.inbox.get(self.victim)) + + def test_share_counts_the_auto_fetch_senders(self): + # With a third auto-fetch sender, each gets a third + self.feature.auto_fetch_from.add(bytes([0x54] * 16).hex()) + self.feature.inbox.limit = 30 + for _ in range(40): + self.relink() + self.assertEqual(len(self.attacker_records()), 10) + self.assertIsNotNone(self.feature.inbox.get(self.victim)) + + +class SharedMessageLogTest(unittest.TestCase): + """MessageLog.add's share and the choice of victim when every other record is protected.""" + + def setUp(self): + """An inbox log whose records with "cids" hold those CIDs (protected).""" + import tempfile + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.log = MessageLog(tmp.name, lxmf.INBOX, 100, protect=lambda e: bool(e.get("cids")), + holds=lambda e: set(e.get("cids") or ())) + self.addCleanup(self.log.close) + + def add(self, sender: str, cids=(), share=None) -> int | None: + """Add a record from sender holding cids; its id, or None if refused.""" + return self.log.add({"from": sender, "cids": list(cids)}, share=share)[0] + + def senders(self) -> list[tuple[str, tuple]]: + """(sender, cids) of each record, oldest first.""" + return [(e["from"], tuple(e["cids"])) for e in self.log.entries()] + + def test_over_its_share_a_sender_drops_its_own_in_order(self): + self.add("v", ["v1"]) + self.add("a", ["c1"]) # protected, the only holder of c1 + self.add("a", ["c2"]) + self.add("a", ["c2"]) # c2 again: the older c2 record holds nothing the other doesn't + self.add("a") # unprotected + self.add("a", ["c3"], share=(4, 10 ** 6)) # five records of "a": the unprotected one goes first + self.assertEqual(self.senders(), [("v", ("v1",)), ("a", ("c1",)), ("a", ("c2",)), ("a", ("c2",)), + ("a", ("c3",))]) + self.add("a", ["c4"], share=(4, 10 ** 6)) # then the oldest whose CIDs another of its records holds + self.assertEqual(self.senders(), [("v", ("v1",)), ("a", ("c1",)), ("a", ("c2",)), ("a", ("c3",)), + ("a", ("c4",))]) + self.add("a", ["c5"], share=(4, 10 ** 6)) # then its oldest + self.assertEqual(self.senders(), [("v", ("v1",)), ("a", ("c2",)), ("a", ("c3",)), ("a", ("c4",)), + ("a", ("c5",))]) + + def test_when_all_are_protected_only_the_senders_own_may_go(self): + self.log.limit = 2 + self.add("v", ["v1"]) + self.add("v", ["v2"]) + # Another sender's protected record is never chosen: the new record is refused + self.assertIsNone(self.add("a", ["c1"])) + self.assertEqual(self.senders(), [("v", ("v1",)), ("v", ("v2",))]) + # The same sender's new record pushes out its own oldest + self.assertIsNotNone(self.add("v", ["v3"])) + self.assertEqual(self.senders(), [("v", ("v2",)), ("v", ("v3",))]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_feat2_inbox_sender_line.py b/tests/security/test_feat2_inbox_sender_line.py new file mode 100644 index 0000000..78deaa8 --- /dev/null +++ b/tests/security/test_feat2_inbox_sender_line.py @@ -0,0 +1,134 @@ +"""FEAT-2: display names could fake a trusted address in `rnfs inbox`. + +A sender's display name comes from its own announce. `rnfs inbox` printed it first and unmarked, followed by the +address: up to 64 printable characters fit `alice ` plus padding, pushing the real address off +to the right, and nothing said whether the sender was a contact or trusted at all: + + #7 2023-11-15 08:43 alice <51515151515151515151515151515151> direct + +Node B runs the lxmf feature on a stand-in network (tests/test_lxmf.py's FeatureTestCase). Fixed: `inbox_list()` +(GET /api/lxmf/inbox) adds computed `trusted` and `contact` (the LXMF contact's nick) fields, and each message's +line gives the sender's address first, then `untrusted`, `trusted` or `contact ` (`?` from an older daemon +that doesn't say), then the display name last, quoted and escaped with fsutil.printable_label (FEAT-6). +""" +import io +import re +import unittest +from types import SimpleNamespace +from unittest import mock + +from rnfs.features import lxmf +from tests import test_lxmf +from tests.test_lxmf import OTHER, SENDER, message + +ATTACKER = bytes([0xDE, 0xAD] * 8) +CONTACT = bytes([0xC0] * 16) +HEADER = re.compile(r"#\d+ \S+ \S+ <([0-9a-f]{32})> (untrusted|trusted|contact \S+|\?) (\S+)(?: (\".*\"))?$") + + +def setUpModule(): + """Silence the node's log.""" + test_lxmf.setUpModule() + + +def tearDownModule(): + """Restore RNS's loglevel.""" + test_lxmf.tearDownModule() + + +def listing(entries: list[dict]) -> list[str]: + """The message lines (not links or text) `rnfs inbox` prints for entries, as the API returned them.""" + client = mock.Mock() + client.get.return_value = {"messages": entries} + out = io.StringIO() + with mock.patch("rnfs.cli.client", return_value=client), mock.patch("sys.stdout", out): + lxmf._run_inbox(SimpleNamespace()) + return [line for line in out.getvalue().splitlines() if line.startswith("#")] + + +class InboxSenderLineTest(test_lxmf.FeatureTestCase): + """B with SENDER in lxmf_auto_fetch_from and CONTACT as LXMF contact carol.""" + + def setUp(self): + """B's feature and its contact.""" + super().setUp() + self.feature = self.feature_node(lxmf_auto_fetch_from=[SENDER.hex()]) + self.b.feature("contacts").add("carol", CONTACT.hex(), kind="lxmf") + self.uri = lxmf.Link(self.cid).uri().encode() + + def receive(self, source: bytes, display_name: str | None, number: int = 1) -> int: + """A message with an rnfs link from source, announcing display_name; its inbox id.""" + return self.feature.received(message(source, self.uri, number=number, display_name=display_name)) + + def test_display_name_cannot_pose_as_a_trusted_address(self): + fake = f"alice <{SENDER.hex()}>" + " " * 20 + self.receive(ATTACKER, fake[:lxmf.MAX_DISPLAY_NAME_CHARS]) + [line] = listing(self.feature.inbox_list()) + # FEAT-2: fails until the line gives the real address first and marks the sender + found = HEADER.fullmatch(line) + self.assertIsNotNone(found, f"the sender's address doesn't come first, marked: {line!r}") + self.assertEqual(found[1], ATTACKER.hex()) + self.assertEqual(found[2], "untrusted") + self.assertEqual(found[4], f'"alice <{SENDER.hex()}>' + " " * 20 + '"') + self.assertLess(line.index(ATTACKER.hex()), line.index(SENDER.hex())) + + def test_api_says_who_is_trusted_or_a_contact(self): + stranger = self.receive(ATTACKER, "bob", 1) + trusted = self.receive(SENDER, None, 2) + contact = self.receive(CONTACT, "Carol", 3) + by_id = {e["id"]: e for e in self.feature.inbox_list()} + self.assertEqual((by_id[stranger]["trusted"], by_id[stranger]["contact"]), (False, None)) + self.assertEqual((by_id[trusted]["trusted"], by_id[trusted]["contact"]), (True, None)) + self.assertEqual((by_id[contact]["trusted"], by_id[contact]["contact"]), (True, "carol")) + lines = listing(self.feature.inbox_list()) + self.assertEqual([HEADER.fullmatch(line)[2] for line in lines], ["untrusted", "trusted", "contact carol"]) + self.assertIsNone(HEADER.fullmatch(lines[1])[4]) # no display name: nothing quoted + self.assertEqual(HEADER.fullmatch(lines[2])[4], '"Carol"') + # Computed when listed: trust follows the contacts and settings now, and isn't saved in the record + self.b.feature("contacts").remove("carol") + self.assertEqual(self.feature.inbox_list()[2]["trusted"], False) + self.assertNotIn("trusted", self.feature.inbox.get(contact)) + + def test_older_daemon_shows_a_question_mark(self): + self.receive(ATTACKER, "bob") + entries = self.feature.inbox_list() + for entry in entries: + del entry["trusted"], entry["contact"] + self.assertEqual(HEADER.fullmatch(listing(entries)[0])[2], "?") + + def test_display_name_is_escaped_as_a_label(self): + # A blank filler, a combining mark after it, quotes and a spelled-out escape + self.receive(ATTACKER, 'a\u3164\u0301" trusted "\\x1b') + [line] = listing(self.feature.inbox_list()) + self.assertEqual(HEADER.fullmatch(line)[4], '"a\\u3164\\u0301\\" trusted \\"\\\\x1b"') + + def test_untrusted_sender_with_an_address_for_a_name(self): + # A name that is nothing but another address still comes after the real one, quoted + self.receive(OTHER, f"<{CONTACT.hex()}>") + [line] = listing(self.feature.inbox_list()) + found = HEADER.fullmatch(line) + self.assertEqual((found[1], found[2], found[4]), (OTHER.hex(), "untrusted", f'"<{CONTACT.hex()}>"')) + + +class NickOfTest(test_lxmf.FeatureTestCase): + """ContactsFeature.nick_of, which the inbox listing asks for each sender.""" + + def test_nick_of(self): + self.feature_node() + book = self.b.feature("contacts") + book.add("zed", CONTACT.hex(), kind="lxmf") + book.add("carol", CONTACT.hex(), kind="lxmf") + book.add("node", OTHER.hex(), kind="node") + self.assertEqual(book.nick_of(CONTACT, "lxmf"), "carol") # the first by nick when several name it + self.assertIsNone(book.nick_of(OTHER, "lxmf")) # a node contact isn't an LXMF one + self.assertEqual(book.nick_of(OTHER, "node"), "node") + book.remove("carol") + self.assertEqual(book.nick_of(CONTACT, "lxmf"), "zed") + book.add("zed", OTHER.hex(), kind="lxmf") # replaced + self.assertIsNone(book.nick_of(CONTACT, "lxmf")) + self.assertFalse(book.has(CONTACT, "lxmf")) + self.assertTrue(book.has(OTHER, "lxmf")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_feat3_untrusted_dropped_counter.py b/tests/security/test_feat3_untrusted_dropped_counter.py new file mode 100644 index 0000000..ebc85d0 --- /dev/null +++ b/tests/security/test_feat3_untrusted_dropped_counter.py @@ -0,0 +1,80 @@ +"""FEAT-3: many identities starve every stranger's messages. + +The L12-R allowance limits each sender not trusted, but strangers share one node-wide bucket (32 at once, then one a +minute), and an LXMF identity costs one announce. One message a minute, each from a fresh identity, keeps the +bucket empty, so every other stranger's message is dropped; trusted senders (contacts, lxmf_accept_from, +lxmf_auto_fetch_from) are unaffected. Stopping it would take LXMF stamps (proof of work), which change the announce +and refuse senders that haven't heard it: deferred, as an interop break. The limit is documented (spec section 10) +and counted: `status()["untrusted_dropped"]` is the number of untrusted senders' messages the rate limits dropped +since the start, so an app can suggest adding the people the user expects messages from as contacts. + +Node B runs the lxmf feature on a stand-in network (tests/test_lxmf.py's FeatureTestCase) with one LXMF contact +and a clock the test moves. +""" +import os +import unittest + +from rnfs.features import lxmf +from rnfs.features.lxmf import Link +from tests import test_lxmf +from tests.support import FakeClock +from tests.test_lxmf import message + +CONTACT = bytes([0xC0] * 16) +STRANGER = bytes([0x5E] * 16) # a sender not trusted, with its own allowance left + + +def setUpModule(): + """Silence the node's log.""" + test_lxmf.setUpModule() + + +def tearDownModule(): + """Restore RNS's loglevel.""" + test_lxmf.tearDownModule() + + +class StrangerBucketTest(test_lxmf.FeatureTestCase): + """B with contact CONTACT; fresh identities keep the strangers' bucket empty.""" + + def setUp(self): + """B's feature, its contact and its clock.""" + super().setUp() + self.feature = self.feature_node() + self.b.feature("contacts").add("carol", CONTACT.hex(), kind="lxmf") + self.clock = FakeClock(1000.0) + self.feature.clock = self.clock + self.junk = Link(self.cid).uri().encode() + self.number = 0 + + def send(self, source: bytes) -> int | None: + """A message with an rnfs link from source; its inbox id, or None if it wasn't kept.""" + self.number += 1 + return self.feature.received(message(source, self.junk, number=self.number % 256)) + + def test_drops_are_counted_and_contacts_are_unaffected(self): + # FEAT-3: fails until the status has the counter + self.assertEqual(self.b.status()["untrusted_dropped"], 0) + for _ in range(lxmf.UNTRUSTED_BURST): + self.assertIsNotNone(self.send(os.urandom(16))) + dropped = 0 + for _ in range(60): # an hour: a fresh identity takes each token as it comes back + self.clock.now += lxmf.UNTRUSTED_INTERVAL + self.assertIsNotNone(self.send(os.urandom(16))) + dropped += self.send(STRANGER) is None + self.assertEqual(dropped, 60, "a stranger got a message through the emptied bucket") + self.assertEqual(self.b.status()["untrusted_dropped"], 60) + self.assertEqual(self.feature.status()["untrusted_dropped"], 60) + # The per-sender allowance counts too + self.clock.now += 10 * lxmf.UNTRUSTED_INTERVAL + sender = os.urandom(16) + kept = [self.send(sender) is not None for _ in range(lxmf.SENDER_BURST + 1)] + self.assertEqual(kept, [True] * lxmf.SENDER_BURST + [False]) + self.assertEqual(self.b.status()["untrusted_dropped"], 61) + # Contacts are unaffected, and not counted + self.assertIsNotNone(self.send(CONTACT)) + self.assertEqual(self.b.status()["untrusted_dropped"], 61) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_feat4_removed_auto_sender.py b/tests/security/test_feat4_removed_auto_sender.py new file mode 100644 index 0000000..b71c53e --- /dev/null +++ b/tests/security/test_feat4_removed_auto_sender.py @@ -0,0 +1,101 @@ +"""FEAT-4: auto-fetches still queued for a sender ran after the sender was removed from lxmf_auto_fetch_from. + +The auto-fetch queue isn't saved: at start the node queues again every inbox link still recorded as "queued". +Neither that re-queue nor the fetch itself checked that the sender was still in lxmf_auto_fetch_from, so after the +user removed a sender and restarted, up to 64 of its links were fetched and pinned anyway (the remote-pin path +already filters against its current list). With LXMF off or failing to start nothing ran, but the records stayed +"queued", which protects them in the inbox like pins. + +Node B runs the lxmf feature on a stand-in network (tests/test_lxmf.py's FeatureTestCase); SENDER is an auto-fetch +sender whose message is received but not yet fetched. Fixed: building the feature marks a queued link of a sender +no longer listed "refused" (also when LXMF is off), the re-queue at start skips such senders, and auto_fetch() +checks the list again before fetching. +""" +import unittest +from unittest import mock + +from rnfs.content.cid import Cid +from rnfs.features import lxmf +from tests import test_lxmf +from tests.test_lxmf import OTHER, SENDER, FakeLxmf + + +def setUpModule(): + """Silence the node's log.""" + test_lxmf.setUpModule() + + +def tearDownModule(): + """Restore RNS's loglevel.""" + test_lxmf.tearDownModule() + + +class RemovedAutoSenderTest(test_lxmf.FeatureTestCase): + """SENDER (and OTHER) in lxmf_auto_fetch_from; SENDER's message is queued, not fetched.""" + + def setUp(self): + """B's feature with a queued link from SENDER and one from OTHER.""" + super().setUp() + self.feature = self.feature_node(lxmf_auto_fetch_from=[SENDER.hex(), OTHER.hex()]) + self.removed = self.feature.received(self.link_message(source=SENDER, number=1)) + self.kept = self.feature.received(self.link_message(source=OTHER, number=2)) + self.target = Cid.parse(self.a.info(f"{self.cid}/file.txt")["cid"]) + + def restart_without_sender(self, **settings) -> lxmf.LxmfFeature: + """Save, take SENDER out of lxmf_auto_fetch_from (and apply settings), build the feature again and start + it on a stand-in network (no threads run).""" + self.feature.save() + self.b.config.lxmf_auto_fetch_from = [OTHER.hex()] + for name, value in settings.items(): + setattr(self.b.config, name, value) + again = self.reloaded() + with mock.patch.object(lxmf, "LxmfNetwork", return_value=FakeLxmf()), \ + mock.patch.object(again, "_run"), mock.patch.object(again, "_run_auto_fetch"): + again.start() + self.addCleanup(again.stop) # before it is closed + return again + + def state(self, feature, entry_id) -> dict: + """The auto-fetch result of an inbox message's one link.""" + return feature.inbox.get(entry_id)["auto"][0] + + def test_queued_link_of_a_removed_sender_is_refused_at_start(self): + again = self.restart_without_sender() + again.process_auto_fetches() + # FEAT-4: fails until a removed sender's queued links are refused rather than fetched + self.assertNotIn(f"lxmf:{SENDER.hex()}", self.b.storage.pins().get(self.target, set()), + "a removed sender's queued link was fetched and pinned after the restart") + self.assertEqual(self.state(again, self.removed)["state"], "refused") + self.assertIn("lxmf_auto_fetch_from", self.state(again, self.removed)["error"]) + # A sender still listed is fetched as before + self.assertEqual(self.state(again, self.kept)["state"], "pinned") + self.assertIn(f"lxmf:{OTHER.hex()}", self.b.storage.pins()[self.target]) + + def test_refused_with_lxmf_off(self): + # Nothing is fetched with LXMF off, but the record mustn't stay "queued" (protected like a pin) + again = self.restart_without_sender(lxmf_enabled=False) + self.assertIsNone(again.network) + self.assertEqual(self.state(again, self.removed)["state"], "refused") + self.assertFalse(lxmf._holds_pins(again.inbox.get(self.removed))) + self.assertEqual(self.state(again, self.kept)["state"], "queued") # fetched once LXMF runs again + + def test_refused_when_lxmf_fails_to_start(self): + self.feature.save() + self.b.config.lxmf_auto_fetch_from = [OTHER.hex()] + again = self.reloaded() + with mock.patch.object(lxmf, "LxmfNetwork", side_effect=RuntimeError("boom")): + again.start() + self.assertIsNone(again.network) + self.assertEqual(self.state(again, self.removed)["state"], "refused") + + def test_auto_fetch_checks_the_list_again(self): + # The sender is removed while its link waits in the running node's queue + self.feature.auto_fetch_from.discard(SENDER.hex()) + self.feature.process_auto_fetches() + self.assertEqual(self.state(self.feature, self.removed)["state"], "refused") + self.assertNotIn(f"lxmf:{SENDER.hex()}", self.b.storage.pins().get(self.target, set())) + self.assertEqual(self.state(self.feature, self.kept)["state"], "pinned") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_feat5_name_link_fallback.py b/tests/security/test_feat5_name_link_fallback.py new file mode 100644 index 0000000..30c8961 --- /dev/null +++ b/tests/security/test_feat5_name_link_fallback.py @@ -0,0 +1,177 @@ +"""FEAT-5: a name link quietly fell back to the CID its sender put in the message. + +An rnfs link in an LXMF message's field carries both `name` and `cid` (the CID the name pointed at when sent), +and `rnfs inbox` showed only `rnfs:name//...`. When the name couldn't be resolved (owner offline, no +keeper reachable: common on a mesh), `inbox get` used the sender's CID without a word, so any sender, a stranger +included, could pose as a well-known name and have the user save the sender's file under the name's file name. + +Node B runs the lxmf feature on a stand-in network (tests/test_lxmf.py's FeatureTestCase): node A holds the +sender's file (`self.cid`, with `file.txt` in it), and the message links name NAME, sent as that CID. The name is +made unresolvable, or resolvable to other content, by replacing B's name table's `resolve`. + +Fixed: `inbox get` fails (NotFound, naming `--use-sent-cid`) before writing anything or starting a job, unless the +user asks for the sent CID, and then each result says so in `warnings`; the listing shows the sent CID next to a +name link; auto-fetch (trusted senders) keeps the fallback but records `"sent_cid": true`, which the listing +shows. +""" +import http.client +import io +import json +import os +import unittest +from types import SimpleNamespace +from unittest import mock + +from rnfs.api import ApiServer +from rnfs.content.cid import Cid +from rnfs.errors import NotFound +from rnfs.features import lxmf +from rnfs.features.lxmf import Link +from tests import test_lxmf +from tests.test_lxmf import A_ID, NAME, SENDER, FeatureTestCase + + +def setUpModule(): + """Silence the node's log.""" + test_lxmf.setUpModule() + + +def tearDownModule(): + """Restore RNS's loglevel.""" + test_lxmf.tearDownModule() + + +class NameLinkFallbackTest(FeatureTestCase): + """A message from SENDER linking name NAME, sent as A's directory holding the sender's file.txt.""" + + def setUp(self): + """The name link, an empty output directory, and the name unresolvable until a test says otherwise.""" + super().setUp() + self.name_link = Link(self.cid, NAME, ("file.txt",), from_node=A_ID) + self.out = os.path.join(self.tmp.name, "out") + os.makedirs(self.out) + self.resolved = None # what B's name table answers for NAME: None (unresolvable) or a record + + def resolve(self, name, persist, max_age=0.0): + """B's name table: the record the test set for NAME, else None.""" + return self.resolved if name == NAME else None + + def receive(self, **settings) -> int: + """Node B (with settings) receives the name link; its inbox id. B's names resolve as self.resolved.""" + feature = self.feature_node(**settings) + patch = mock.patch.object(self.b.names, "resolve", side_effect=self.resolve) + patch.start() + self.addCleanup(patch.stop) + return feature.received(self.link_message(self.name_link)) + + def name_points_elsewhere(self) -> bytes: + """Make NAME resolve to another directory with its own file.txt; that file's content.""" + data = b"the name owner's file" + other = Cid.parse(self.a.add_stream(io.BytesIO(data), name="file.txt")["cid"]) + self.resolved = {"name": NAME.hex(), "cid": str(other), "sequence": 1, "age": 0} + return data + + def test_unresolvable_name_fails_without_writing_or_starting_anything(self): + entry_id = self.receive() + with self.assertRaisesRegex(NotFound, "--use-sent-cid"): + self.feature.inbox_get(entry_id, directory=self.out) + self.assertEqual(os.listdir(self.out), [], "inbox get wrote the sender's file for an unresolved name") + # Through the jobs feature: no job is submitted either + jobs = mock.Mock() + jobs.submit.return_value = {"id": "0123abcd"} + self.b.features["jobs"] = jobs + with self.assertRaises(NotFound): + self.feature.inbox_get(entry_id, output=os.path.join(self.out, "x")) + jobs.submit.assert_not_called() + + def test_use_sent_cid_writes_the_file_with_a_warning(self): + entry_id = self.receive() + result = self.feature.inbox_get(entry_id, directory=self.out, use_sent_cid=True)[0] + with open(os.path.join(self.out, "file.txt"), "rb") as f: + self.assertEqual(f.read(), self.data) + self.assertTrue(any(NAME.hex() in w and str(self.cid) in w for w in result["warnings"]), result) + # A job's result carries the warning too + jobs = mock.Mock() + jobs.submit.return_value = {"id": "0123abcd"} + self.b.features["jobs"] = jobs + result = self.feature.inbox_get(entry_id, output=os.path.join(self.out, "y"), use_sent_cid=True)[0] + self.assertEqual(result["job"], {"id": "0123abcd"}) + self.assertTrue(any(str(self.cid) in w for w in result["warnings"]), result) + + def test_resolvable_name_wins(self): + entry_id = self.receive() + data = self.name_points_elsewhere() + for number, flag in enumerate((False, True)): + output = os.path.join(self.out, f"f{number}") + result = self.feature.inbox_get(entry_id, output=output, use_sent_cid=flag)[0] + with open(output, "rb") as f: + self.assertEqual(f.read(), data) + self.assertEqual(result["warnings"], []) + + def call(self, server, body): + """(status, decoded JSON) of POST /api/lxmf/inbox/get on server.""" + connection = http.client.HTTPConnection("127.0.0.1", server.port, timeout=30) + headers = {"Host": f"127.0.0.1:{server.port}", "X-Rnfs-Api": server.token, + "Content-Type": "application/json"} + connection.request("POST", "/api/lxmf/inbox/get", body=json.dumps(body).encode(), headers=headers) + response = connection.getresponse() + raw = response.read() + connection.close() + return response.status, json.loads(raw) + + def test_api_answers_404_without_use_sent_cid(self): + entry_id = self.receive() + server = ApiServer(self.b, 0) + server.start() + self.addCleanup(server.stop) + status, body = self.call(server, {"id": entry_id, "directory": self.out}) + self.assertEqual(status, 404, body) + self.assertEqual(os.listdir(self.out), []) + status, body = self.call(server, {"id": entry_id, "directory": self.out, "use_sent_cid": True}) + self.assertEqual(status, 200, body) + self.assertTrue(body["results"][0]["warnings"], body) + self.assertEqual(os.listdir(self.out), ["file.txt"]) + + def test_cli_passes_use_sent_cid(self): + client = mock.Mock() + client.post.return_value = {"results": [{"output": "/x/file.txt", "warnings": ["name unresolved"]}]} + err = io.StringIO() + with mock.patch("rnfs.cli.client", return_value=client), mock.patch("sys.stdout", io.StringIO()), \ + mock.patch("sys.stderr", err): + lxmf._run_inbox_get(SimpleNamespace(output=None, n=1, link=None, use_sent_cid=True)) + self.assertTrue(client.post.call_args.kwargs["use_sent_cid"]) + self.assertIn("warning: name unresolved", err.getvalue()) + + def listing(self, entry: dict) -> str: + """What `rnfs inbox` prints for entry.""" + client = mock.Mock() + client.get.return_value = {"messages": [entry]} + out = io.StringIO() + with mock.patch("rnfs.cli.client", return_value=client), mock.patch("sys.stdout", out): + lxmf._run_inbox(SimpleNamespace()) + return out.getvalue() + + def test_listing_shows_the_sent_cid(self): + entry_id = self.receive() + listed = self.listing(self.feature.inbox.get(entry_id)) + self.assertIn(f"rnfs:name/{NAME.hex()}/file.txt", listed) + self.assertIn(f"sent as {self.cid}", listed, "the inbox listing hides the CID a name link was sent as") + + def test_auto_fetch_marks_the_sent_cid(self): + entry_id = self.receive(lxmf_auto_fetch_from=[SENDER.hex()]) + self.feature.process_auto_fetches() + result = self.feature.inbox.get(entry_id)["auto"][0] + self.assertEqual((result["state"], result.get("sent_cid")), ("pinned", True), result) + self.assertIn("auto-fetch pinned (sent CID; name unresolved)", self.listing(self.feature.inbox.get(entry_id))) + # A resolvable name: no mark + self.name_points_elsewhere() + second = self.feature.received(self.link_message(self.name_link, number=2)) + self.feature.process_auto_fetches() + result = self.feature.inbox.get(second)["auto"][0] + self.assertEqual(result["state"], "pinned") + self.assertNotIn("sent_cid", result) + self.assertNotIn("sent CID", self.listing(self.feature.inbox.get(second))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_feat6_printable_label.py b/tests/security/test_feat6_printable_label.py new file mode 100644 index 0000000..430c0b4 --- /dev/null +++ b/tests/security/test_feat6_printable_label.py @@ -0,0 +1,68 @@ +"""FEAT-6: `fsutil.printable()` escapes terminal controls, BiDi overrides and zero-width characters, but lets through +characters that are printable yet show nothing or change their neighbours: the blank Hangul and braille fillers +(U+115F, U+1160, U+3164, U+FFA0, U+2800), combining marks (a stack of them spills over the text around it, and +one at the start joins whatever was printed before) and literal backslashes (so a label can spell out an escape +that printable() would print, and look like a different label). A display name made of fillers looks blank, and +one with a filler inside looks like another name. + +printable() stays as it is: it prints paths, and a Windows path is full of backslashes. Labels chosen by someone +else (an LXMF display name) go through the new `fsutil.printable_label()`, which escapes all of these too, and +escapes a label that would print as nothing but spaces. +""" +import unittest + +from rnfs import fsutil + +FILLERS = ("\u115f", "\u1160", "\u3164", "\uffa0", "\u2800") +ACUTE, GRAVE, TILDE = "\u0301", "\u0300", "\u0303" # combining marks + + +class PrintableLabelTest(unittest.TestCase): + """What printable_label() escapes beyond printable().""" + + def test_printable_lets_the_lookalikes_through(self): + # The finding: printable() returns these unchanged (it is kept that way for paths) + for filler in FILLERS: + self.assertEqual(fsutil.printable(f"al{filler}ice"), f"al{filler}ice") + self.assertEqual(fsutil.printable(f"{ACUTE}bob"), f"{ACUTE}bob") + self.assertEqual(fsutil.printable("C:\\Users\\bob"), "C:\\Users\\bob") + + def test_blank_fillers_are_escaped(self): + # FEAT-6: fails until printable_label() exists and escapes them + for filler in FILLERS: + label = fsutil.printable_label(f"al{filler}ice") + self.assertNotIn(filler, label) + self.assertEqual(label, f"al\\u{ord(filler):04x}ice") + self.assertEqual(fsutil.printable_label(FILLERS[2] * 3), "\\u3164" * 3) # a name that looked blank + + def test_backslashes_are_escaped(self): + # A label can't spell out an escape: "\x1b" typed out prints differently from an escaped ESC + self.assertEqual(fsutil.printable_label("a\\x1bb"), "a\\\\x1bb") + self.assertEqual(fsutil.printable_label("a\x1bb"), "a\\x1bb") + self.assertNotEqual(fsutil.printable_label("a\\x1bb"), fsutil.printable_label("a\x1bb")) + + def test_leading_and_stacked_combining_marks_are_escaped(self): + self.assertEqual(fsutil.printable_label(f"{ACUTE}bob"), "\\u0301bob") + # Up to two in a row stay (Vietnamese and others use two); the third and later are escaped + self.assertEqual(fsutil.printable_label(f"e{ACUTE}{GRAVE}"), f"e{ACUTE}{GRAVE}") + self.assertEqual(fsutil.printable_label(f"e{ACUTE}{GRAVE}{TILDE}{ACUTE}x{ACUTE}"), + f"e{ACUTE}{GRAVE}\\u0303\\u0301x{ACUTE}") + # A mark after an escape would join the escape's text: escaped too + self.assertEqual(fsutil.printable_label(f"a\x07{ACUTE}"), "a\\x07\\u0301") + # Spacing marks (Mc) and enclosing marks (Me) count as combining marks + self.assertEqual(fsutil.printable_label("\u20dd"), "\\u20dd") + + def test_blank_labels_are_escaped(self): + self.assertEqual(fsutil.printable_label(" "), "\\x20\\x20\\x20") + self.assertEqual(fsutil.printable_label(""), "") + self.assertEqual(fsutil.printable_label("a b"), "a b") # spaces between visible characters stay + + def test_everything_printable_escapes_stays_escaped(self): + for text in ("a\x1b[31mb\x07", "\u202eevil", "zero\u200bwidth", "nul\x00", "line\nbreak"): + self.assertEqual(fsutil.printable_label(text), fsutil.printable(text)) + self.assertEqual(fsutil.printable_label("plain été 名前"), "plain été 名前") + self.assertEqual(fsutil.printable_label(5), "5") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_feat7_example_and_client_escaping.py b/tests/security/test_feat7_example_and_client_escaping.py new file mode 100644 index 0000000..4bb4748 --- /dev/null +++ b/tests/security/test_feat7_example_and_client_escaping.py @@ -0,0 +1,115 @@ +"""FEAT-7: examples/rnfs_share.py printed names and errors unescaped, and rnfs.Client logged the daemon's messages +unescaped. + +The example prints entry names from a DAG (whoever made it chose them), warnings and errors that quote them, so a +name holding terminal controls (ESC sequences, BEL) drove the terminal it ran in. rnfs.Client's _report passed a +daemon message, which can quote such a name, to Python's logging as it was. + +The example's share() runs against a stand-in node whose add, ls, open, names and jobs answer with such names; +Client._report is called directly. Fixed: both go through rnfs.fsutil.printable. +""" +import contextlib +import importlib.util +import io +import os +import tempfile +import unittest +from types import SimpleNamespace +from unittest import mock + +from rnfs import remote + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +EVIL = "evil\x1b]0;owned\x07\x1b[2J" # sets the terminal's title and clears the screen + + +def load_example(): + """examples/rnfs_share.py as a module.""" + spec = importlib.util.spec_from_file_location("rnfs_share", os.path.join(ROOT, "examples", "rnfs_share.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class StandInNode: + """Just what share() calls, answering with EVIL in every name, warning and error it can.""" + + def __init__(self): + """Names and jobs (a CID is base32 text, so it stays plain).""" + self.cid = "bafyexample" + record = SimpleNamespace(name=b"\x01" * 16, cid=self.cid, sequence=1) + name = SimpleNamespace(alias="example", hex="01" * 16, publish=lambda cid: record) + + @contextlib.contextmanager + def watch(_name, on_record): + """Deliver the published record at once.""" + on_record(record) + yield + + self.names = SimpleNamespace(list=lambda: [name], create=lambda alias: name, watch=watch) + job = SimpleNamespace(state="done", done_bytes=1, total_bytes=1, error=None) + job.wait = lambda: job + + def get(_target, _output, on_progress): + """A job that is done at once.""" + on_progress(job) + return job + + self.jobs = SimpleNamespace(get=get) + + def add(self, _path): + """An added directory with a warning quoting a name.""" + return SimpleNamespace(cid=self.cid, size=1, warnings=[f"skipped {EVIL}"]) + + def ls(self, _cid): + """One file named EVIL.""" + return [SimpleNamespace(kind="file", size=1, name=EVIL, is_dir=False)] + + def open(self, _target): + """A one-byte file.""" + return contextlib.nullcontext(SimpleNamespace(size=1, seek=lambda offset: None, read=lambda n: b"x")) + + +class ExampleTest(unittest.TestCase): + """examples/rnfs_share.py escapes what it prints.""" + + def test_share_escapes_names(self): + """Entry names, warnings and targets are printed escaped.""" + example = load_example() + out = io.StringIO() + with tempfile.TemporaryDirectory() as folder, contextlib.redirect_stdout(out): + example.share(StandInNode(), folder) + printed = out.getvalue() + self.assertIn(EVIL.encode("unicode_escape").decode("ascii"), printed) + self.assertNotIn("\x1b", printed) + self.assertNotIn("\x07", printed) + + def test_errors_are_escaped(self): + """An rnfs error (whose text can quote a name) is printed escaped when the example exits on it.""" + import rnfs + example = load_example() + + def fail(*_args, **_kwargs): + """Fail as a daemon reporting a name would.""" + raise rnfs.RnfsError(f"not found: {EVIL}") + with mock.patch.object(rnfs, "Client", fail), mock.patch("sys.argv", ["rnfs_share", "--daemon", "x"]): + with self.assertRaises(SystemExit) as raised: + example.main() + self.assertIn("not found: evil\\x1b", str(raised.exception.code)) + self.assertNotIn("\x1b", str(raised.exception.code)) + + +class ClientReportTest(unittest.TestCase): + """rnfs.Client logs the daemon's messages escaped.""" + + def test_report_escapes(self): + """A message quoting a name reaches logging with its controls escaped.""" + with self.assertLogs("rnfs", "ERROR") as logged: + remote.Client._report(None, f"couldn't write {EVIL}") + self.assertEqual(len(logged.output), 1) + self.assertNotIn("\x1b", logged.output[0]) + self.assertIn("couldn't write evil\\x1b]0;owned\\x07", logged.output[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_feat8_index_page_cache.py b/tests/security/test_feat8_index_page_cache.py new file mode 100644 index 0000000..b646924 --- /dev/null +++ b/tests/security/test_feat8_index_page_cache.py @@ -0,0 +1,121 @@ +"""FEAT-8: the file-index-node page had no lock or rate limit, unlike the display page. + +Every visit ran `rnfs pin ls --json` (and a sealed check for each new file pin), however many visitors came at +once, so anyone reaching the NomadNet node could keep the daemon busy listing pins. + +These tests run the whole page script with the CLI stood in for by a fake subprocess.run that records each +call. Fixed: the page renders at most once per CACHE_SECONDS (60 s), one request at a time (the display page's +lock); it serves the cached page, written atomically, while it is fresh or while another request holds the lock. +""" +import json +import os +import tempfile +import time +import unittest + +from tests.security import _pages_helpers as helpers + +CID_A = "bafkreiailvfecuuvt6tctx3owax2jic7y5af2ndriwo6xymcq2uwyut34a" +CID_B = "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" + + +def pin(cid, name): + """A complete file pin the operator made, as `rnfs pin ls --json` lists it.""" + return {"cid": cid, "name": name, "sources": ["user"], "complete": True, "type": "file"} + + +class IndexPageCacheTest(unittest.TestCase): + """The whole file-index-node script, run repeatedly over one page data dir.""" + + def setUp(self): + """A page data dir, the pins the fake CLI lists, and the calls it records.""" + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.data_dir = tmp.name + self.pins = [pin(CID_A, "field-guide.pdf")] + self.calls = [] + self.daemon_down = False + + def run_cli(self, argv, *args, **kwargs): + """The fake CLI: pin ls lists self.pins, cat answers with a file's (unsealed) first bytes.""" + words = list(argv[1:]) + self.calls.append(words) + if self.daemon_down: + return helpers._Completed(1, stderr="rnfs: the daemon isn't running") + if words[:3] == ["pin", "ls", "--json"]: + return helpers._Completed(0, json.dumps(self.pins).encode("utf-8")) + if words[0] == "cat": + return helpers._Completed(0, b"%PDF-1.7\n") + return helpers._Completed(1, stderr="unknown command") + + def page(self) -> str: + """What the whole page prints now.""" + return helpers.exec_index_page(self.run_cli, self.data_dir) + + def cache_path(self) -> str: + """The cached page's path in the data dir.""" + return os.path.join(self.data_dir, "page.mu") + + def age_cache(self, seconds): + """Make the cached page seconds old.""" + then = time.time() - seconds + os.utime(self.cache_path(), (then, then)) + + def test_fresh_cache_is_served_without_running_rnfs(self): + """A second visit within the cache time gets the same page, and the CLI isn't run again.""" + first = self.page() + self.assertIn("`!field-guide.pdf`!", first) + self.assertIn(["pin", "ls", "--json"], self.calls) + self.calls.clear() + self.pins.append(pin(CID_B, "notes.txt")) + self.assertEqual(self.page(), first) + self.assertEqual(self.calls, []) + with open(self.cache_path(), encoding="utf-8") as f: + self.assertEqual(f.read(), first) + self.assertEqual(sorted(os.listdir(self.data_dir)), ["lock", "page.mu", "sealed.json"]) # no temp file + + def test_stale_cache_is_rendered_again(self): + """Once the cached page is 60 s old, the next visit lists the pins afresh.""" + self.page() + self.pins.append(pin(CID_B, "notes.txt")) + self.age_cache(59) + self.assertNotIn("notes.txt", self.page()) + self.age_cache(61) + self.calls.clear() + page = self.page() + self.assertIn("`!notes.txt`!", page) + self.assertIn(["pin", "ls", "--json"], self.calls) + # A cache dated in the future (the clock went back) isn't trusted either + self.pins.append(pin("bafkreie5737gdxlw5i64vzichcalba3z2v5mzd7mo7kzkwqvzrmxl5ofwu", "later.txt")) + then = time.time() + 3600 + os.utime(self.cache_path(), (then, then)) + self.assertIn("`!later.txt`!", self.page()) + + @unittest.skipIf(os.name == "nt", "holds the lock with fcntl") + def test_busy_lock_serves_the_cache(self): + """While another request holds the lock, a visit gets the cached page however old, without running the + CLI; with no cached page at all it is asked to try again.""" + import fcntl + first = self.page() + self.age_cache(3600) + self.calls.clear() + with open(os.path.join(self.data_dir, "lock"), "a+") as held: + fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) + self.assertEqual(self.page(), first) + os.remove(self.cache_path()) + busy = self.page() + self.assertEqual(self.calls, []) + self.assertIn("busy", busy) + self.assertNotIn("field-guide.pdf", busy) + + def test_failure_is_not_cached(self): + """With the daemon down, visitors get the generic message, and the next visit tries again.""" + self.daemon_down = True + self.assertIn("The file list isn't available right now", self.page()) + self.assertFalse(os.path.exists(self.cache_path())) + self.daemon_down = False + self.assertIn("`!field-guide.pdf`!", self.page()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_fuzz_parsers.py b/tests/security/test_fuzz_parsers.py index ca0c26b..3cc48e3 100644 --- a/tests/security/test_fuzz_parsers.py +++ b/tests/security/test_fuzz_parsers.py @@ -125,7 +125,7 @@ def unpacked(data: bytes): """(True, value) if data is msgpack umsgpack decodes, else (False, None): wire bytes as RNS hands them on.""" try: return True, umsgpack.unpackb(data) - except Exception: # noqa: BLE001 - anything umsgpack rejects never reaches a parser + except Exception: # anything umsgpack rejects never reaches a parser return False, None diff --git a/tests/security/test_h1_real_rns.py b/tests/security/test_h1_real_rns.py index ed1c189..c9cdf6e 100644 --- a/tests/security/test_h1_real_rns.py +++ b/tests/security/test_h1_real_rns.py @@ -16,7 +16,6 @@ """ import json import os -import signal import subprocess import sys import tempfile @@ -26,6 +25,7 @@ from tests import support from tests.security import _h1_real_helpers as helpers +from tests.test_e2e import INTERRUPT, POPEN_OPTIONS HELPER = helpers.__file__ ROOT = helpers.ROOT @@ -75,7 +75,7 @@ def setUp(self): def _spawn(self, argv, name, env=None): log = open(os.path.join(self.base, f"{name}.log"), "wb") self.logs.append(log) - proc = subprocess.Popen(argv, stdout=log, stderr=subprocess.STDOUT, env=env) + proc = subprocess.Popen(argv, stdout=log, stderr=subprocess.STDOUT, env=env, **POPEN_OPTIONS) self.procs.append(proc) return proc @@ -94,7 +94,7 @@ def _status(self): def _stop_all(self): for p in self.procs: if p.poll() is None: - p.send_signal(signal.SIGINT) + p.send_signal(INTERRUPT) # Ctrl+C; Windows can't send SIGINT, so CTRL_BREAK_EVENT (as test_e2e) time.sleep(2) for p in self.procs: if p.poll() is None: diff --git a/tests/security/test_h2_display_page_pins.py b/tests/security/test_h2_display_page_pins.py index e3f6020..59f4142 100644 --- a/tests/security/test_h2_display_page_pins.py +++ b/tests/security/test_h2_display_page_pins.py @@ -1,6 +1,6 @@ """H2: one anonymous view of the display-mu-file page made the node permanently pin (source "user") and provide content the visitor chose. The CID comes from var_cid; the page ran stat, ls, `pin add` and cat on it -with no allowlist and no per-visitor limit. `pin add` pins USER (node.py:821-833), which the node then +with no allowlist and no per-visitor limit. `pin add` pins USER (RnfsNode.pin_add), which the node then provides; and with the default provide_strategy="roots", stat/cat alone already make a cached complete fetch be provided, so dropping only the pin would not stop the seeding. That part is documented behaviour of "roots" (kept as the default by the user's decision; "pinned" avoids it), tested as such below. @@ -307,6 +307,8 @@ def test_an_invalid_cid_runs_nothing_and_is_not_counted(self): @E2E +@unittest.skipIf(os.name == "nt", "drives the page through a /bin/sh shim of the rnfs command: on Windows the " + "page's subprocess finds the installed rnfs.exe instead (the page's logic runs above on Windows)") class DisplayPageDaemonTest(unittest.TestCase): """Run the actual page script against a real victim daemon served by a real provider.""" diff --git a/tests/security/test_i2r1_identity_file.py b/tests/security/test_i2r1_identity_file.py index 1ee9084..459e938 100644 --- a/tests/security/test_i2r1_identity_file.py +++ b/tests/security/test_i2r1_identity_file.py @@ -73,7 +73,7 @@ def start(): try: load_or_create_identity(self.path) outcome.append("loaded") - except Exception as e: # noqa: BLE001 - whatever it is, the test reports it + except Exception as e: # whatever it is, the test reports it outcome.append(e) thread = threading.Thread(target=start, daemon=True) diff --git a/tests/security/test_l10_index_page_injection.py b/tests/security/test_l10_index_page_injection.py index f8f8099..5e6bf4d 100644 --- a/tests/security/test_l10_index_page_injection.py +++ b/tests/security/test_l10_index_page_injection.py @@ -1,5 +1,5 @@ """L10: Micron line injection through DAG entry names on the file-index-node page. The page's escape() only -escapes backslash and backtick (index.mu:28-30), but a directory entry name may contain newlines and other +escaped backslash and backtick, but a directory entry name may contain newlines and other control characters (dag.check_name allows them). A name with a newline therefore injects whole Micron lines into the page: a fake ">" heading, or a fake entry line. diff --git a/tests/security/test_l14_page_local_paths.py b/tests/security/test_l14_page_local_paths.py index 5376719..bac54d8 100644 --- a/tests/security/test_l14_page_local_paths.py +++ b/tests/security/test_l14_page_local_paths.py @@ -1,6 +1,7 @@ """L14: the pages show daemon errors and CLI stderr to remote visitors, which can carry local filesystem paths (for example "no API token at /...: is the daemon running"). The display-mu-file page echoed -the CLI's stderr verbatim on every error path (index.mu:54, 66, 86, 90, 94). +the CLI's stderr verbatim on every error path (size_of, is_directory, the `pin add` and `cat` calls and the +OSError handler, each through fail()). This runs the page's real script with a CLI that fails exactly as rnfs does when the daemon is down, and checks the visitor is not shown the local path the error carries. The fixed pages show a generic message and @@ -16,7 +17,7 @@ from tests.security import _pages_helpers as helpers _saved_loglevel = RNS.loglevel -# A stderr line the real CLI prints when it can't reach the daemon (cli.py:84-88), carrying a local path. +# A stderr line the real CLI prints when it can't reach the daemon (apiclient.read_token), carrying a local path. RNFS_DIR = os.path.join(os.path.expanduser("~"), ".config", "rnfs") DAEMON_DOWN = f"rnfs: no API token at {RNFS_DIR}/apitoken: is the daemon running?" CID = "bafkreiezrecqkty6ertxgtpi4mt6ns733iqnb2bfqwbiv6bk4b3lky4n4e" diff --git a/tests/security/test_l15_install_release_check.py b/tests/security/test_l15_install_release_check.py index b8a511a..90da9d8 100644 --- a/tests/security/test_l15_install_release_check.py +++ b/tests/security/test_l15_install_release_check.py @@ -1,6 +1,7 @@ """L15 (fix): install.sh installed GitHub's auto-generated archive of the release tag, which neither SHA256SUMS nor the signed provenance covers and which a moved tag changes. Now it installs the release's sdist, checked against the release's SHA256SUMS and, where the GitHub CLI can, its signed provenance (gh attestation verify), failing closed. +SUP-2 ties that provenance to the release's tag; SUP-3 stops the install when gh could check it but isn't logged in. install.sh's download, verify_release and fetch_release functions are run with sh against a fake release (file:// URLs) and a fake gh on a PATH holding only the tools they need, so the real network and any real gh are never @@ -24,14 +25,15 @@ REPO=afit21/rnfs """ # A gh that logs its arguments and answers as FAKE_GH says: "old" (no attestation command), "logged-out", "fail" -# (verification fails) or "ok" +# (verification fails) or "ok". Its verify --help lists FAKE_GH_HELP's options (by default only --signer-workflow, +# as in gh 2.49) FAKE_GH = r"""#!/bin/sh printf '%s\n' "$*" >> "$FAKE_GH_LOG" case "$1 $2" in "auth status") [ "$FAKE_GH" != logged-out ] ;; "attestation verify") [ "$FAKE_GH" != old ] || { echo 'unknown command "attestation" for "gh"' >&2; exit 1; } - case " $* " in *" --help "*) echo " --signer-workflow string"; exit 0 ;; esac + case " $* " in *" --help "*) printf '%s\n' "${FAKE_GH_HELP:- --signer-workflow string}"; exit 0 ;; esac [ "$FAKE_GH" = ok ] || { echo "Error: no attestations found" >&2; exit 1; } ;; *) exit 2 ;; esac @@ -82,7 +84,7 @@ def fetch(self, **env) -> subprocess.CompletedProcess: """Run fetch_release for v9.9.9 as do_install would; prints the SOURCE it leaves.""" repo_url = "file://" + os.path.join(self.tmp.name, "repo") script = "\n".join([PRELUDE, shell_function("download"), shell_function("verify_release"), - shell_function("gh_can_verify"), shell_function("fetch_release"), + shell_function("gh_has_attestation"), shell_function("fetch_release"), 'fetch_release "$1"', 'printf "SOURCE=%s\\n" "$SOURCE"']) environment = {"PATH": self.bin, "PYTHON": sys.executable, "REPO_URL": repo_url, "RELEASE_TAG": "v9.9.9", "SOURCE": repo_url + "/releases/download/v9.9.9/rnfs-9.9.9.tar.gz", @@ -142,20 +144,50 @@ def test_attestation_checked_with_gh(self): self.assertIn(f"attestation verify {local} --repo afit21/rnfs " f"--signer-workflow=afit21/rnfs/.github/workflows/release.yml", self.gh_calls()) + def test_attestation_names_the_tag(self): + """SUP-2: where gh has the options, the check asks for a build of this release's tag on a GitHub-hosted + runner, so an older release's attested sdist renamed to this one's file name is refused.""" + self.install_gh("ok") + result = self.fetch(FAKE_GH_HELP=" --signer-workflow string\n --source-ref string\n" + " --deny-self-hosted-runners") + self.assert_installs_checked_file(result) + local = os.path.join(self.work, "release", "rnfs-9.9.9.tar.gz") + self.assertIn(f"attestation verify {local} --repo afit21/rnfs " + f"--signer-workflow=afit21/rnfs/.github/workflows/release.yml --source-ref=refs/tags/v9.9.9 " + f"--deny-self-hosted-runners", self.gh_calls()) + # gh is asked for its options once + self.assertEqual(len([call for call in self.gh_calls() if "--help" in call]), 1) + def test_failed_attestation_is_refused(self): self.install_gh("fail") result = self.fetch() self.assert_refused(result, "no valid signed provenance") self.assertIn("no attestations found", result.stderr) # gh's own explanation is shown - def test_gh_that_cant_verify_counts_as_absent(self): - # Ubuntu 24.04's gh 2.45 has no attestation command; a gh not logged in can't query attestations - for behaviour in ("old", "logged-out"): - with self.subTest(behaviour): - self.install_gh(behaviour) - self.assert_installs_checked_file(self.fetch()) - self.assertFalse([call for call in self.gh_calls() if " --repo " in call]) - self.assert_refused(self.fetch(RNFS_REQUIRE_ATTESTATION="1"), "RNFS_REQUIRE_ATTESTATION=1") + def test_gh_without_attestation_counts_as_absent(self): + # Ubuntu 24.04's gh 2.45 has no attestation command + self.install_gh("old") + result = self.fetch() + self.assert_installs_checked_file(result) + self.assertIn("against the release's SHA256SUMS only", result.stderr) + self.assertFalse([call for call in self.gh_calls() if " --repo " in call]) + self.assert_refused(self.fetch(RNFS_REQUIRE_ATTESTATION="1"), "RNFS_REQUIRE_ATTESTATION=1") + + def test_logged_out_gh_stops_the_install(self): + """SUP-3: a gh that could check the provenance but isn't logged in (a root install's, usually) stops the + install, and the error names the opt-out, RNFS_REQUIRE_ATTESTATION=0. (Until 0.1.6 it counted as absent.)""" + self.install_gh("logged-out") + for env in ({}, {"RNFS_REQUIRE_ATTESTATION": "1"}): + with self.subTest(env): + result = self.fetch(**env) + self.assert_refused(result, "the GitHub CLI (gh) is installed but not logged in") + self.assertIn("RNFS_REQUIRE_ATTESTATION=0", result.stderr) + self.assertFalse([call for call in self.gh_calls() if " --repo " in call]) + result = self.fetch(RNFS_REQUIRE_ATTESTATION="0") + self.assert_installs_checked_file(result) + self.assertIn("RNFS_REQUIRE_ATTESTATION=0: checked rnfs-9.9.9.tar.gz against the release's SHA256SUMS only", + result.stderr) + self.assertFalse([call for call in self.gh_calls() if " --repo " in call]) class DefaultSourceTest(unittest.TestCase): diff --git a/tests/security/test_l17_rejected_responses.py b/tests/security/test_l17_rejected_responses.py index 3a52398..9649893 100644 --- a/tests/security/test_l17_rejected_responses.py +++ b/tests/security/test_l17_rejected_responses.py @@ -4,8 +4,9 @@ Fixed: a response whose Resource is rejected with nothing sent counts as making no progress, so its link (and identity) cools down for blocks_cooldown, as after one cancelled for making none (H1). These tests drive the server on a FakeClock with fake Resources concluding as RNS's do on a rejection (status REJECTED, then the -callback): the loop builds one response per cooldown, and honest outcomes (completed, or rejected mid-transfer) -start no cooldown. +callback): the loop builds one response per cooldown, and honest outcomes (completed, or rejected once a good +part of it was sent) start no cooldown. NET-1 extends the cooldown to a rejection before a quarter of the +response was sent (tests/security/test_net1_cancelled_responses.py). """ import unittest @@ -14,22 +15,12 @@ from rnfs.config import Config from rnfs.constants import CHUNK_SIZE from rnfs.content.cid import RAW, Cid -from tests.security._server_helpers import ServerHarness +from tests.security._server_helpers import ServerHarness, reject BLOCK = bytes(CHUNK_SIZE) CID = Cid.for_block(RAW, BLOCK) -def reject(resource, progress: float = 0.0): - """The requester rejects (or, mid-transfer, cancels) the response: RNS's Resource._rejected on an outgoing - Resource sets REJECTED, leaves the link's outgoing Resources and runs the callback.""" - resource.progress = progress - resource.status = RNS.Resource.REJECTED - resource.conclude() - if resource.callback: - resource.callback(resource) - - class RejectedResponseTest(unittest.TestCase): """Rejected /blocks responses and the cooldown.""" @@ -82,7 +73,7 @@ def test_honest_outcomes_start_no_cooldown(self): self.respond(link).complete() resource = self.respond(link) self.assertIsNotNone(resource) - # Rejected after some of it was sent (RNS's receiver gives up mid-transfer the same way): it made progress + # Rejected after half of it was sent (RNS's receiver gives up mid-transfer the same way): it made progress reject(resource, progress=0.5) self.assertIsNotNone(self.respond(link)) self.assertFalse(self.harness.server.cooldowns.active(link.link_id)) diff --git a/tests/security/test_l1_response_precheck.py b/tests/security/test_l1_response_precheck.py index edcaff0..6ca4068 100644 --- a/tests/security/test_l1_response_precheck.py +++ b/tests/security/test_l1_response_precheck.py @@ -235,7 +235,8 @@ def test_responses_on_incoming_links_dropped_unread(self): link.decrypt.assert_not_called() def test_other_packets_and_destinations_untouched(self): - link = ResponseLink() + # A request on an incoming link, where the node's server takes them (NET-3 drops one on a link it opened) + link = ResponseLink(initiator=False) request = types.SimpleNamespace(context=RNS.Packet.REQUEST, packet_type=RNS.Packet.DATA, data=b"x" * 600) RNS.Link.receive(link, request) lxmf_link = ResponseLink(LXMF, initiator=False) diff --git a/tests/security/test_l5_follow_cap.py b/tests/security/test_l5_follow_cap.py index 4369651..72942b7 100644 --- a/tests/security/test_l5_follow_cap.py +++ b/tests/security/test_l5_follow_cap.py @@ -4,8 +4,9 @@ B follows A's name. A first publishes an honest file, then points the name at a root that declares 1 byte but holds about 2 MB of empty file nodes (the L3 bomb). The follow fetch now runs under the same DAG byte bound as every fetch_dag (rnfs.fetcher.dag_bound of the new target's declared size), so the check fails with "refusing it" after -the root, the old target stays pinned and the bomb is not. Content size itself stays bounded by max_download_size -(no default) and the disk reserve, as before: a per-follow size limit was not added. +the root, the old target stays pinned and the bomb is not. Content size itself is bounded by follow_quota (1 GiB by +default; added in 0.1.6 for DHT-3, tested in test_dht3_follow_quota.py) as well as max_download_size and the disk +reserve; the bomb's size is far below the quota, so the DAG byte bound is what refuses it here. """ import io import os diff --git a/tests/security/test_l6_gateway_storage_oracle.py b/tests/security/test_l6_gateway_storage_oracle.py index 0445c82..48e748e 100644 --- a/tests/security/test_l6_gateway_storage_oracle.py +++ b/tests/security/test_l6_gateway_storage_oracle.py @@ -55,7 +55,7 @@ def stored_head(self, target, length): self.record("stored_head", target, length) try: path = self._path(target) - except Exception: # noqa: BLE001 - "invalid" is None too, as for RnfsNode + except Exception: # "invalid" is None too, as for RnfsNode return None obj = self.objects.get(path[-1]) if obj is None or obj[0] != "file" or set(path) & self.remote: diff --git a/tests/security/test_l7_windows_acl.py b/tests/security/test_l7_windows_acl.py index ebd9c84..ddcfe32 100644 --- a/tests/security/test_l7_windows_acl.py +++ b/tests/security/test_l7_windows_acl.py @@ -16,7 +16,6 @@ import RNS from rnfs import cli, fsutil, node as node_module, winacl -from rnfs.config import Config from rnfs.errors import RnfsError from rnfs.network import identity as identity_module from rnfs.winacl import ADMINISTRATORS, SYSTEM, Ace, Dacl @@ -43,8 +42,8 @@ def allow(sid: str, mask: int = FULL, flags: int = INHERITED) -> Ace: class FakeAcl: - """A winacl backend: a DACL per file, set_private's and create_private_dir's calls, and `inherited` for files - never set.""" + """A winacl backend: a DACL per file, set_private's, create_private_dir's and inherit's calls, and `inherited` + for files never set (or made to inherit again).""" def __init__(self, inherited: Dacl = SHARED): self.inherited = inherited @@ -52,6 +51,7 @@ def __init__(self, inherited: Dacl = SHARED): self.set_calls: list[tuple[str, bool]] = [] self.created: list[str] = [] self.read_fds: list[int | None] = [] + self.inherit_calls: list[str] = [] @staticmethod def key(path: str, fd: int | None = None) -> tuple[int, int]: @@ -84,6 +84,11 @@ def set(self, path: str, dacl: Dacl): """Give path dacl, as someone else (or an older rnfs) left it.""" self.dacls[self.key(path)] = dacl + def inherit(self, path: str): + """Drop path's own DACL, so it inherits its parent's (`inherited`) again.""" + self.inherit_calls.append(path) + self.dacls.pop(self.key(path), None) + class WindowsTestCase(unittest.TestCase): """A temporary directory, fsutil.IS_WINDOWS patched on and a FakeAcl as winacl's backend.""" @@ -187,17 +192,18 @@ def messages(self) -> str: return "\n".join(call.args[0] for call in self.log.call_args_list) def test_new_dir_gets_a_protected_dacl(self): - node_module._make_private_dirs(Config(rnfs_dir=self.root)) + node_module._make_private_dirs(self.root) # Created with it (CreateDirectoryW's security attributes), never set afterwards self.assertEqual(self.acl.created, [self.root, *(os.path.join(self.root, name) - for name in ("state", "blocks", "storage", "names"))]) + for name in ("state", "blocks", "storage", "names", + "lxmf"))]) self.assertEqual(self.acl.set_calls, []) self.assertTrue(self.private(self.root)) self.assertEqual(self.messages(), "") def test_existing_dir_others_can_read_is_tightened(self): os.mkdir(self.root) - node_module._make_private_dirs(Config(rnfs_dir=self.root)) + node_module._make_private_dirs(self.root) self.assertEqual(self.acl.set_calls, [(self.root, True)]) self.assertTrue(self.private(self.root)) self.assertIn("made " + self.root + " private to you and SYSTEM (Users (S-1-5-32-545) may access it)", @@ -207,7 +213,7 @@ def test_existing_private_dir_is_left_alone(self): # The default ~/.rnfs, inheriting the profile's DACL: nothing changed, nothing logged at every start os.mkdir(self.root) self.acl.set(self.root, PROFILE) - node_module._make_private_dirs(Config(rnfs_dir=self.root)) + node_module._make_private_dirs(self.root) self.assertEqual(self.acl.set_calls, []) self.assertEqual(self.messages(), "") @@ -215,7 +221,7 @@ def test_dir_of_another_account_is_refused(self): os.mkdir(self.root) self.acl.set(self.root, Dacl(OTHER, [allow(OTHER), allow(USER)])) with self.assertRaisesRegex(RnfsError, f"couldn't make the rnfs dir .* private to you: .*belongs to {OTHER}"): - node_module._make_private_dirs(Config(rnfs_dir=self.root)) + node_module._make_private_dirs(self.root) self.assertEqual(self.acl.set_calls, []) self.assertFalse(os.path.exists(os.path.join(self.root, "state"))) @@ -223,16 +229,16 @@ def test_dir_that_cant_be_made_private_is_refused(self): os.mkdir(self.root) with mock.patch.object(self.acl, "set_private", side_effect=PermissionError(5, "Access is denied")): with self.assertRaisesRegex(RnfsError, "couldn't make the rnfs dir"): - node_module._make_private_dirs(Config(rnfs_dir=self.root)) + node_module._make_private_dirs(self.root) # Nor one whose DACL doesn't take (e.g. a filesystem without ACLs) with mock.patch.object(self.acl, "set_private"): with self.assertRaisesRegex(RnfsError, "still not private"): - node_module._make_private_dirs(Config(rnfs_dir=self.root)) + node_module._make_private_dirs(self.root) def test_dir_outside_the_profile_is_warned_about(self): with mock.patch.dict(os.environ, {"USERPROFILE": os.path.join(self.tmp.name, "profile")}): - node_module._make_private_dirs(Config(rnfs_dir=self.root)) - node_module._make_private_dirs(Config(rnfs_dir=self.root)) # as a start does more than once + node_module._make_private_dirs(self.root) + node_module._make_private_dirs(self.root) # as a start does more than once self.assertTrue(self.private(self.root)) self.assertEqual(self.messages().count("is outside your user profile"), 1) @@ -284,7 +290,7 @@ class PosixUnchangedTest(unittest.TestCase): def test_no_backend_calls(self): with tempfile.TemporaryDirectory() as directory, mock.patch.object(fsutil, "IS_WINDOWS", False), \ mock.patch.object(winacl, "backend", mock.Mock(side_effect=AssertionError)) as backend: - node_module._make_private_dirs(Config(rnfs_dir=os.path.join(directory, "rnfs"))) + node_module._make_private_dirs(os.path.join(directory, "rnfs")) token = os.path.join(directory, "rnfs", cli.TOKEN_FILE) cli.write_token(token, "t") self.assertEqual(cli.read_token(token), "t") diff --git a/tests/security/test_l7r1_windows_dir_creation.py b/tests/security/test_l7r1_windows_dir_creation.py index ebc2d50..9049bef 100644 --- a/tests/security/test_l7r1_windows_dir_creation.py +++ b/tests/security/test_l7r1_windows_dir_creation.py @@ -42,8 +42,9 @@ def setUp(self): def test_root_and_subdirectories_are_created_private(self): """Each directory has the private DACL from the call that creates it: none is ever set afterwards (the old code made the root with os.makedirs, then set its DACL).""" - node_module._make_private_dirs(Config(rnfs_dir=self.root)) - expected = [self.root] + [os.path.join(self.root, name) for name in ("state", "blocks", "storage", "names")] + node_module._make_private_dirs(self.root) + expected = [self.root] + [os.path.join(self.root, name) for name in ("state", "blocks", "storage", "names", + "lxmf")] self.assertEqual(self.acl.set_calls, [], "a DACL was set after the directory was created with its " "parent's access list") self.assertEqual(self.acl.created, expected) @@ -63,12 +64,12 @@ def mkdir_then_plant(path, *args, **kwargs): with mock.patch("os.mkdir", mkdir_then_plant): with self.assertRaisesRegex(RnfsError, "was just created, yet it already holds 1 entries"): - node_module._make_private_dirs(Config(rnfs_dir=self.root)) + node_module._make_private_dirs(self.root) self.assertEqual(sorted(os.listdir(self.root)), [SETTINGS_FILE], "rnfs went on to use the planted dir") def test_trailing_separator(self): """The default dir is given as "~/.rnfs/": its parent is the profile, not the dir itself.""" - node_module._make_private_dirs(Config(rnfs_dir=self.root + os.sep)) + node_module._make_private_dirs(self.root + os.sep) self.assertEqual(self.acl.created[0], self.root) @@ -102,11 +103,14 @@ def test_settings_of_another_account_are_refused(self): Config.load(self.root) def test_settings_others_may_access_are_tightened_with_a_warning(self): + # The warning is returned for the caller to show (ND6: logging it loaded RNS in the CLI and rnfs.Client) with mock.patch("rnfs.log.log") as log: config = Config.load(self.root) self.assertEqual(config.reticulum_dir, "C:/Users/Public/evil-reticulum") # read, after the warning self.assertTrue(self.private(self.path), "the settings file was used without making it private") - self.assertIn("Check that its settings are yours", log.call_args.args[0]) + [warning] = config.warnings + self.assertIn("Check that its settings are yours", warning) + log.assert_not_called() self.assertIsNotNone(self.acl.read_fds[0], "the DACL checked must be the opened file's") def test_settings_that_cant_be_tightened_are_refused(self): @@ -117,7 +121,7 @@ def test_settings_that_cant_be_tightened_are_refused(self): def test_private_settings_load_quietly(self): self.acl.set(self.path, Dacl(USER, [allow(USER), allow(winacl.SYSTEM)])) with mock.patch("rnfs.log.log") as log: - Config.load(self.root) + self.assertEqual(Config.load(self.root).warnings, []) log.assert_not_called() self.assertEqual(self.acl.set_calls, []) diff --git a/tests/security/test_l7r2_junction_root.py b/tests/security/test_l7r2_junction_root.py index 0f0a477..f8b5063 100644 --- a/tests/security/test_l7r2_junction_root.py +++ b/tests/security/test_l7r2_junction_root.py @@ -12,7 +12,6 @@ from unittest import mock from rnfs import node as node_module -from rnfs.config import Config from rnfs.errors import RnfsError from tests.security.test_l7_windows_acl import WindowsTestCase from tests.support import CAN_SYMLINK @@ -35,7 +34,7 @@ def setUp(self): def assert_refused(self): """The start is refused, and nothing is set or created through the link.""" with self.assertRaisesRegex(RnfsError, "is a junction or symlink"): - node_module._make_private_dirs(Config(rnfs_dir=self.root)) + node_module._make_private_dirs(self.root) self.assertEqual(self.acl.set_calls, [], "the target's DACL was changed through the link") self.assertEqual(os.listdir(self.target), [], "rnfs created its directories in the link's target") @@ -57,7 +56,7 @@ def lstat(path, *args, **kwargs): with mock.patch("os.lstat", lstat): with self.assertRaisesRegex(RnfsError, "is a junction or symlink"): - node_module._make_private_dirs(Config(rnfs_dir=self.root)) + node_module._make_private_dirs(self.root) self.assertEqual(self.acl.set_calls, [], "the target's DACL was changed through the junction") self.assertEqual(os.listdir(self.root), [], "rnfs created its directories through the junction") @@ -73,7 +72,7 @@ def lstat(path, *args, **kwargs): return mock.Mock(st_mode=stat.S_IFDIR | 0o777, st_reparse_tag=0x9000001A) # IO_REPARSE_TAG_CLOUD with mock.patch("os.lstat", lstat): - node_module._make_private_dirs(Config(rnfs_dir=self.root)) + node_module._make_private_dirs(self.root) self.assertTrue(self.private(self.root)) diff --git a/tests/security/test_m6_index_page_names.py b/tests/security/test_m6_index_page_names.py index 0cd6333..cc3dd7d 100644 --- a/tests/security/test_m6_index_page_names.py +++ b/tests/security/test_m6_index_page_names.py @@ -1,6 +1,6 @@ """M6: the file-index-node page publishes sealed files' plaintext names and pins that didn't come from the operator. Spec section 5 promises a sealed file's name stays confidential, yet a sealed add records its -plaintext path/seal_name (node.py:650, 669) and the page prints the basename to every visitor; a directory +plaintext path/seal_name (RnfsNode.add_path and add_stream) and the page prints the basename to every visitor; a directory pinned from a contact (remote:/lxmf:/follow:) always gets a name from _pin_shape, so it is shown too. The page's own entry() is run over the real pin_ls() of an offline node (JSON round-tripped, as the page diff --git a/tests/security/test_n1_display_page_unbounded.py b/tests/security/test_n1_display_page_unbounded.py index 2dd6bd5..bef0c6b 100644 --- a/tests/security/test_n1_display_page_unbounded.py +++ b/tests/security/test_n1_display_page_unbounded.py @@ -2,7 +2,7 @@ permanently pin an unbounded DAG, bypassing the page's 200 KB MAX_SIZE check. A file root may truthfully declare 1 byte while carrying many zero-size children (L3: no reader rejects size-0 file-node children). The page's size_of() read that declared 1 byte, so the MAX_SIZE check passed; `pin add` then fetched the whole -DAG with no max_dag_bytes and pinned it USER (node.py:821-826), bounded only by the disk reserve. +DAG with no max_dag_bytes and pinned it USER (RnfsNode.pin_add), bounded only by the disk reserve. The tests run the page's real script against an offline victim that could fetch the crafted DAG from an offline provider, and assert the node did not store far more than MAX_SIZE: for a visitor's CID (which the @@ -19,7 +19,7 @@ from rnfs.content.storage import USER from tests.security import _pages_helpers as helpers -MAX_SIZE = 200 * 1024 # display-mu-file index.mu:18 +MAX_SIZE = 200 * 1024 # display-mu-file's MAX_SIZE OVERHEAD = MAX_BLOCK_SIZE # generous per-view slack: even one extra full block stays well under the bomb _saved_loglevel = RNS.loglevel diff --git a/tests/security/test_net1_cancelled_responses.py b/tests/security/test_net1_cancelled_responses.py new file mode 100644 index 0000000..4ca8d62 --- /dev/null +++ b/tests/security/test_net1_cancelled_responses.py @@ -0,0 +1,163 @@ +"""NET-1: a requester could have the node build /blocks responses in a loop by cancelling each one after its +first parts arrived and asking again at once. L17 cooled down only a response rejected with nothing sent, so a +cancel after one part (or one window) of parts freed the slot with no cooldown, and each round made the node +read, hash and pack every block, then build and encrypt a whole Resource, for the cost of one window. + +Fixed: a response whose Resource the requester rejected or cancelled (on RNS 1.5.6 both arrive as REJECTED) +before a quarter of it was sent (server.NO_PROGRESS_FRACTION) made no real progress, so its link and identity +cool down for blocks_cooldown, flat, as for one rejected unsent. A link closed early (FAILED) starts no +cooldown: an honest reader that gives up closes its links, and in private mode it would cool its own identity +down. If progress can't be read it counts as progress, as before. These tests drive the server on a FakeClock +with fake Resources concluding as RNS's do. +""" +import unittest + +import RNS + +from rnfs.config import Config +from rnfs.constants import CHUNK_SIZE +from rnfs.content.cid import RAW, Cid +from rnfs.network import protocol +from tests.security._server_helpers import ServerHarness, reject + +BLOCK = bytes(CHUNK_SIZE) +CID = Cid.for_block(RAW, BLOCK) + + +def close_link(link): + """The requester closes the link mid-transfer: RNS's Link.link_closed cancels the link's outgoing Resources + (FAILED, then their callbacks) before it runs the link-closed callback.""" + for resource in list(link.outgoing_resources): + resource.cancel() + link.teardown() + + +class CancelledResponseTest(unittest.TestCase): + """/blocks responses cancelled before real progress, and the cooldown.""" + + def setUp(self): + self.harness = ServerHarness({CID: BLOCK}, Config(blocks_cooldown=60)) + self.clock = self.harness.clock + self.server = self.harness.server + + def respond(self, link, identity=None): + """Request a block on link and advertise the response; its Resource, or None if refused.""" + answer = self.harness.blocks(link, [CID], max_bytes=CHUNK_SIZE, identity=identity) + if answer is False: + return None + self.assertEqual(answer, [BLOCK]) + return self.harness.advertise(link) + + def send(self, link, resource, progress: float): + """RNS sending progress (a fraction) of resource's parts, noticed by one monitor pass.""" + resource.progress = progress + self.harness.tick(0.1) + self.assertEqual(self.harness.response(link).sent, progress * self.harness.response(link).size) + + def assert_cools_down(self, progress: float, link=None): + """A rejection at progress on link (default: a new one) cools down the link and its identity, and not + another requester.""" + identity = RNS.Identity() + link = link or self.harness.connect() + resource = self.respond(link, identity) + self.send(link, resource, progress) + reject(resource, progress) + self.assertTrue(self.server.cooldowns.active(link.link_id)) + self.assertTrue(self.server.cooldowns.active(bytes(identity.hash))) + self.assertIsNone(self.respond(link, identity)) + self.assertIsNone(self.respond(self.harness.connect(), identity)) # the identity, on a new link + self.respond(self.harness.connect()).complete() # another requester is unaffected + + def assert_no_cooldown(self, link, identity): + """Neither link nor identity is cooling down, and the identity gets a new response at once.""" + self.assertFalse(self.server.cooldowns.active(link.link_id)) + self.assertFalse(self.server.cooldowns.active(bytes(identity.hash))) + self.respond(self.harness.connect(), identity).complete() # completed: the slow pool has two slots + + def test_reject_at_one_percent_cools_down(self): + self.assert_cools_down(0.01) + + def test_reject_after_one_window_cools_down(self): + # A full window of parts on a slow link (RNS's largest there; its first, RNS.Resource.WINDOW, is smaller) + # of a 64 KiB block's response: all an attacker has to receive before cancelling + link = self.harness.connect() + size = protocol.packed_response_size(link.link_id, protocol.encode_blocks_response([BLOCK])) + window = RNS.Resource.WINDOW_MAX_SLOW * link.get_mdu() / size + self.assertLess(window, 0.25) + self.assert_cools_down(window, link) + + def test_reject_loop_builds_one_response_per_cooldown(self): + link = self.harness.connect() + built = 0 + end = self.clock.now + 10 + while self.clock.now < end: # take 1% of each response, cancel it and ask again 10 ms later + resource = self.respond(link) + if resource is not None: + built += 1 + reject(resource, 0.01) + self.clock.now += 0.01 + self.server.check_responses() + self.assertEqual(built, 1) # 1,001 before the fix + self.clock.now += 60 + self.assertIsNotNone(self.respond(link)) # and again once the cooldown is over + + def test_reject_noticed_by_the_monitor_also_counts(self): + # Without the callback (the monitor finds the Resource concluded), the same applies + link = self.harness.connect() + resource = self.respond(link) + resource.callback = None + resource.progress = 0.01 + resource.status = RNS.Resource.REJECTED + resource.conclude() + self.server.check_responses() + self.assertIsNone(self.respond(link)) + + def test_reject_at_a_quarter_or_more_starts_no_cooldown(self): + for progress in (0.25, 0.5): + with self.subTest(progress=progress): + identity = RNS.Identity() + link = self.harness.connect() + resource = self.respond(link, identity) + self.send(link, resource, progress) + reject(resource, progress) + self.respond(link, identity).complete() # the same link may ask again at once + self.assert_no_cooldown(link, identity) + + def test_progress_seen_by_the_monitor_counts(self): + # A Resource that reads less at its rejection than the monitor last saw counts what was seen + identity = RNS.Identity() + link = self.harness.connect() + resource = self.respond(link, identity) + self.send(link, resource, 0.5) + reject(resource, 0.0) + self.assert_no_cooldown(link, identity) + + def test_link_closed_early_starts_no_cooldown(self): + # A reader that gives up early closes its links (an honest 0.1.5 fetcher never cancels a Resource): in + # private mode a cooldown would hit its own identity, so a FAILED response never starts one + identity = RNS.Identity() + link = self.harness.connect() + resource = self.respond(link, identity) + self.send(link, resource, 0.01) + close_link(link) + self.assertEqual(resource.status, RNS.Resource.FAILED) + self.assertIsNone(self.harness.response(link)) + self.assert_no_cooldown(link, identity) + + def test_unreadable_progress_starts_no_cooldown(self): + identity = RNS.Identity() + link = self.harness.connect() + resource = self.respond(link, identity) + + def broken(): + """RNS internals changed: progress can't be read.""" + raise AttributeError("no progress") + + resource.get_progress = broken + reject(resource) + self.respond(link, identity).complete() + self.assert_no_cooldown(link, identity) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_net2_single_packet_pacing.py b/tests/security/test_net2_single_packet_pacing.py new file mode 100644 index 0000000..82b3dc1 --- /dev/null +++ b/tests/security/test_net2_single_packet_pacing.py @@ -0,0 +1,87 @@ +"""NET-2: a /blocks response that fits one packet frees its slot at once and never produces a rate estimate, and +the upload cap applied only once a link had one. So on a large-MTU path (RNS's TCP interfaces negotiate 16 KiB +and more) max_upload_rate never applied to a requester that only asked for blocks fitting one packet: the PoC +served 50 such responses (1 MB) on a 32000-byte MDU under max_upload_rate = 1000 bit/s with no pacing at all. + +Fixed (decision Q5): an explicit max_upload_rate applies to every /blocks response, with or without a rate +estimate; the default cap (half the rate estimate on slow links) still needs an estimate. Waits up to +MAX_PACING_WAIT are held, longer ones refused at once, as for multi-packet responses. +""" +import unittest + +from rnfs.config import Config +from rnfs.constants import KIB +from rnfs.network import server + +from tests.security._server_helpers import TCP_BITRATE, FakeInterface, ServerHarness +from tests.test_server import block_of + +MTU, MDU = 32768, 32000 # a TCP path (RNS: 16 KiB at 10 Mbit/s, 32 KiB at 100 Mbit/s) + + +class SinglePacketPacingTest(unittest.TestCase): + """Responses that fit one packet on a large-MTU link, with no rate estimate.""" + + def setUp(self): + self.small = [block_of(20 * KIB, i) for i in range(60)] # 20 KiB: one packet each on this link + self.interface = FakeInterface("TCP", TCP_BITRATE) + + def harness(self, config: Config) -> tuple[ServerHarness, object, list]: + """A server under config, a large-MTU link to it, and the waits it holds responses back for.""" + harness = ServerHarness(dict(self.small), config) + link = harness.connect(self.interface, mtu=MTU, mdu=MDU) + waits = [] + + def pace(response, wait): + """Hold the response back in virtual time.""" + waits.append(wait) + harness.clock.now += wait + harness.server._pace = pace + return harness, link, waits + + def ask(self, harness, link, i: int): + """Request one small block as request i; the handler's answer.""" + return harness.blocks(link, [self.small[i][0]], request_id=i.to_bytes(16, "big")) + + def test_poc_responses_refused_under_a_low_explicit_cap(self): + harness, link, waits = self.harness(Config(max_upload_rate=1000)) + served = 0 + for i in range(50): + body = self.ask(harness, link, i) + if body is not False: + served += 1 + self.assertIsNone(harness.server.rates.get(link.link_id)) # still no rate estimate + # One 20 KiB response takes ~164 s at 1000 bit/s: every later request is refused at once + self.assertEqual(served, 1) + self.assertEqual(waits, []) + self.assertGreater(harness.server.pacer.wait_time(link.link_id), 160) + + def test_waits_within_the_bound_are_held(self): + cap = 1_000_000 # one 20 KiB response every ~0.16 s + harness, link, waits = self.harness(Config(max_upload_rate=cap)) + start = harness.clock.now + for i in range(20): + self.assertIsNot(self.ask(harness, link, i), False) + self.assertEqual(len(waits), 19) + self.assertTrue(all(0 < wait <= server.MAX_PACING_WAIT for wait in waits)) + # 19 responses before the last had to have started by now: their bytes at the cap + self.assertGreaterEqual(harness.clock.now - start, 19 * 20 * KIB * 8 / cap) + + def test_default_cap_still_needs_a_rate_estimate(self): + # No explicit cap: the default (half the rate estimate on slow links) has nothing to halve + harness, link, waits = self.harness(Config()) + for i in range(20): + self.assertIsNot(self.ask(harness, link, i), False) + self.assertEqual(waits, []) + self.assertEqual(harness.server.pacer.wait_time(link.link_id), 0) + + def test_multi_packet_responses_also_paced_without_an_estimate(self): + harness, link, waits = self.harness(Config(max_upload_rate=1000)) + big = block_of(100 * KIB, 99) + harness.backend.blocks[big[0]] = big[1] + self.assertIsNot(harness.blocks(link, [big[0]], request_id=b"b" * 16), False) + self.assertGreater(harness.server.pacer.wait_time(link.link_id), server.MAX_PACING_WAIT) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_net3_outgoing_requests.py b/tests/security/test_net3_outgoing_requests.py new file mode 100644 index 0000000..2b34af4 --- /dev/null +++ b/tests/security/test_net3_outgoing_requests.py @@ -0,0 +1,98 @@ +"""NET-3: on links rnfs opens, the remote destination has no max_request_size, so RNS msgpack-decodes any request +packet (up to the link MDU) and starts a thread for it before finding there is no handler. Any node rnfs links to +could make it do so. + +Fixed: guards.guarded_receive drops a request packet on an rnfs.dht link this node opened (link.initiator) +without calling RNS or decrypting it, mirroring the drop of responses on incoming links. These tests run the +guard with fake links: an initiator rnfs.dht link drops the request unread; an incoming rnfs.dht link (where the +node's server takes requests) and any LXMF link pass it to RNS; and the response check is unchanged. +""" +import types +import unittest +from unittest import mock + +import RNS +from RNS.vendor import umsgpack + +from rnfs.network import guards + +from tests.security.test_l1_response_precheck import DHT, LXMF, REQUEST_ID, ResponseLink, bomb, response_packet + + +def request_packet(size: int = 600): + """A request packet of size bytes (its content is never read by the guard).""" + return types.SimpleNamespace(context=RNS.Packet.REQUEST, packet_type=RNS.Packet.DATA, data=b"\x93" * size) + + +class OutgoingRequestTest(unittest.TestCase): + """Request packets by link direction and destination, through the guard to a recorded RNS receive.""" + + def setUp(self): + guards.install() + self.received = [] + patcher = mock.patch.object(guards, "_original_receive", lambda link, packet: self.received.append(packet)) + patcher.start() + self.addCleanup(patcher.stop) + self.logged = {} + for name in ("_log_refused_response", "_log_unexpected_response", "_log_unexpected_request"): + self.logged[name] = mock.Mock() + patcher = mock.patch.object(guards, name, self.logged[name], create=True) + patcher.start() + self.addCleanup(patcher.stop) + + def link(self, destination=DHT, initiator=True) -> ResponseLink: + """A fake link whose decrypt is recorded.""" + link = ResponseLink(destination, initiator=initiator) + link.decrypt = mock.Mock(side_effect=lambda data: data) + return link + + def test_request_on_an_initiator_dht_link_dropped_unread(self): + for size in (100, 16 * 1024): + with self.subTest(size=size): + link = self.link() + self.assertIsNone(RNS.Link.receive(link, request_packet(size))) + self.assertEqual(len(self.received), 0) # RNS's receive never called + link.decrypt.assert_not_called() + self.logged["_log_unexpected_request"].assert_called_with(mock.ANY, guards.LOG_VERBOSE) + + def test_request_on_an_incoming_dht_link_passed(self): + # Where this node's server takes requests: RNS's own size limit and handlers apply + link = self.link(initiator=False) + packet = request_packet() + RNS.Link.receive(link, packet) + self.assertEqual(self.received, [packet]) + + def test_request_on_an_lxmf_initiator_link_passed(self): + for destination in (LXMF, types.SimpleNamespace(name="host.app." + "ab" * 16)): + with self.subTest(destination.name): + self.received.clear() + link = self.link(destination) + packet = request_packet() + RNS.Link.receive(link, packet) + self.assertEqual(self.received, [packet]) + + def test_other_packet_types_on_an_initiator_dht_link_passed(self): + link = self.link() + proof = types.SimpleNamespace(context=RNS.Packet.REQUEST, packet_type=RNS.Packet.PROOF, data=b"x" * 64) + plain = types.SimpleNamespace(context=RNS.Packet.NONE, packet_type=RNS.Packet.DATA, data=b"x" * 64) + for packet in (proof, plain): + RNS.Link.receive(link, packet) + self.assertEqual(self.received, [proof, plain]) + + def test_response_check_still_applies(self): + # An honest response on an initiator link passes; a bomb is refused; a response on an incoming link drops + link = self.link() + honest = response_packet(umsgpack.packb([REQUEST_ID, True])) + RNS.Link.receive(link, honest) + self.assertEqual(self.received, [honest]) + RNS.Link.receive(link, response_packet(bomb(16 * 1024, 0x90))) + self.assertEqual(self.received, [honest]) + self.assertEqual(link.handled, [(REQUEST_ID, None)]) + incoming = self.link(initiator=False) + RNS.Link.receive(incoming, response_packet(umsgpack.packb([REQUEST_ID, True]))) + self.assertEqual(self.received, [honest]) + incoming.decrypt.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_net4_dripping_provider.py b/tests/security/test_net4_dripping_provider.py new file mode 100644 index 0000000..0c0eba9 --- /dev/null +++ b/tests/security/test_net4_dripping_provider.py @@ -0,0 +1,142 @@ +"""NET-4: a provider could hold one of our fetches for hours by dripping a /blocks response. A response that keeps +making progress never stalls, and its deadline, with no rate estimate, assumes unmeasured_bitrate (100 bit/s): 64 KiB +takes about 4.4 h. Once every other block had arrived the fetch waited for it, its other providers idle. + +Fixed with end-game duplication, requester-side (spec section 7, item 7, a MAY): once nothing is left to ask for and +a provider is idle, a batch in flight for longer than deadline_slack is asked of it too. The first verified copy +wins; the slow request's link is then closed, unless another fetch has a request on it (then the request is left +to finish), and its outcome doesn't count against its provider (no stall, no failure). The remote load stays bounded: nothing is asked twice before deadline_slack, while blocks are still to be asked for, or +on a slow node (one provider at a time), and each request is duplicated at most once. +""" +import os +import threading +import time +import unittest + +from rnfs.constants import CHUNK_SIZE +from rnfs.content import builder, walker +from rnfs.content.dag import Entry +from rnfs.network.client import FAILED, STALLED, BlocksResult +from tests.fakes.client import FakeClient +import tests.test_fetcher as fetcher_tests # a module, so its test cases don't run twice + +P1, P2 = fetcher_tests.P1, fetcher_tests.P2 +DRIP = 5.0 # seconds the dripping provider takes over its response, unless its link is closed first + + +class DrippingProviderTest(fetcher_tests.FetcherTest): + """P1 drips its response for one file's first chunk; P2 serves everything at once (each request carries one + chunk: no rate estimate, so CHUNK_SIZE).""" + + def setUp(self): + """A directory of two files of eight chunks each, held by P1 and P2; P1 drips the first file's first chunk + until DRIP passes or its link is closed (then it answers as a torn-down link does).""" + super().setUp() + self.config.deadline_slack = 0.3 + self.closing = threading.Event() + self.addCleanup(self.closing.set) # so no request thread outlives the test + first = self.file(os.urandom(8 * CHUNK_SIZE)) + second = self.file(os.urandom(8 * CHUNK_SIZE)) + self.root = builder.build_directory([Entry("a", first.cid, first.size, 0), + Entry("b", second.cid, second.size, 0)], self.put) + self.dripped = walker.expand(walker.root_expect(first.cid), self.network[first.cid])[0].cid + for node_id in (P1, P2): + self.hold(node_id) + self.runs.append([P1, P2]) + + def close(node_id): + """FakeClient.close, which also ends a dripping response: the link is gone.""" + FakeClient.close(self.client, node_id) + if node_id == P1: + self.closing.set() + self.client.close = close + + def drip(cids): + """Take DRIP seconds over the dripped chunk, as a response trickling in would.""" + if self.dripped in cids: + self.closing.wait(DRIP) + if P1 in self.client.closed: + return BlocksResult(FAILED, unsent=list(cids), reason="no answer") + return None + self.client.behaviour[P1] = drip + + def files(self, fetch) -> list[walker.Expect]: + """The root directory's two entries, the root fetched.""" + root = fetch.resolve() + return fetch.ensure([root])[root] + + def test_a_dripping_response_is_asked_of_an_idle_provider(self): + start = time.monotonic() + with self.fetcher.open(self.root.cid) as fetch: + first, second = self.files(fetch) + fetch.fetch_dag(first) + took = time.monotonic() - start + self.assertTrue(self.storage.is_complete(first.cid)) + self.assertLess(took, DRIP / 2, "the fetch waited for the dripping provider") + # The first copy won: P1's slow request was ended by closing its link... + self.assertEqual(self.client.closed, [P1]) + # ...which doesn't count against P1: its answer (the link gone) arrives, and P1 is used again + deadline = time.monotonic() + 5 + while fetch._events.empty() and time.monotonic() < deadline: + time.sleep(0.01) + fetch.fetch_dag(second) + provider = fetch._providers[P1] + self.assertEqual((provider.failed, provider.stalls, provider.busy), (False, 0, 0)) + self.assertTrue(self.storage.is_complete(second.cid)) + second_chunks = {child.cid for child in walker.expand(second, self.network[second.cid])} + self.assertTrue(second_chunks & set(self.client.asked(P1)), "P1 wasn't used again") + # Only the dripped chunk was asked twice, and of P2 only once nothing else was left to ask for + asked = self.client.asked(P1) + self.client.asked(P2) + self.assertEqual(sorted(cid for cid in set(asked) if asked.count(cid) > 1), [self.dripped]) + first_cids = [cid for cid in self.client.asked(P2) if cid not in second_chunks and cid != second.cid] + self.assertEqual(first_cids[-1], self.dripped) + + def test_a_link_another_fetch_uses_is_left_open(self): + self.client.users[P1] = {-1: 1} # another fetch's thread has a request on P1's link + + def drip(cids): + """Drip, then end as stalled: an outcome that would count, if it were counted.""" + if self.dripped in cids: + self.closing.wait(DRIP) + return BlocksResult(STALLED, unsent=list(cids), reason="no progress") + return None + self.client.behaviour[P1] = drip + start = time.monotonic() + with self.fetcher.open(self.root.cid) as fetch: + first, second = self.files(fetch) + fetch.fetch_dag(first) + self.assertLess(time.monotonic() - start, DRIP / 2, "the fetch waited for the dripping provider") + self.assertEqual((self.client.closed, self.client.kept_open), ([], [P1])) # left to finish + self.closing.set() + deadline = time.monotonic() + 5 + while fetch._events.empty() and time.monotonic() < deadline: + time.sleep(0.01) + fetch.fetch_dag(second) # P1's answer is handled first: ignored + provider = fetch._providers[P1] + self.assertEqual((provider.failed, provider.stalls, provider.busy), (False, 0, 0)) + self.assertEqual(self.client.closed, []) + + def test_nothing_is_asked_twice_within_deadline_slack(self): + self.config.deadline_slack = 30 + with self.fetcher.open(self.root.cid) as fetch: + first, _ = self.files(fetch) + threading.Timer(1.0, self.closing.set).start() # P1 finishes after a second + fetch.fetch_dag(first) + asked = self.client.asked(P1) + self.client.asked(P2) + self.assertEqual(len(asked), len(set(asked))) + self.assertEqual(self.client.closed, []) + + def test_a_slow_node_never_asks_twice(self): + self.fetcher = self.make_fetcher(slow=True) + self.config.max_parallel_providers = None # so one provider at a time + with self.fetcher.open(self.root.cid) as fetch: + first, _ = self.files(fetch) + threading.Timer(1.0, self.closing.set).start() + fetch.fetch_dag(first) + asked = self.client.asked(P1) + self.client.asked(P2) + self.assertEqual(len(asked), len(set(asked))) + self.assertEqual(self.client.closed, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_net5_proof_wait.py b/tests/security/test_net5_proof_wait.py new file mode 100644 index 0000000..301c023 --- /dev/null +++ b/tests/security/test_net5_proof_wait.py @@ -0,0 +1,132 @@ +"""NET-5: on the responder, link.rtt is whatever the requester claims (RNS takes the larger of what it measured +and the value in the requester's RTT packet, decoded with msgpack), so it can be huge, infinite, NaN, negative or +not a number at all. The server used it once, for how long a new /blocks waits for the previous response's +proof, as min(PROOF_GRACE, 2 × rtt + 1) on the real monotonic clock: a negative rtt gave no grace at all, a +string raised in the request handler, and the wait ignored the server's injected clock. + +Fixed: RequestServer._proof_wait(link) is the only server-side use of link.rtt. It takes a finite, non-bool +rtt ≥ 0 (anything else counts as none), caps the wait at PROOF_GRACE, and _previous_done measures it on +self.clock. +""" +import ast +import math +import threading +import time +import unittest + +import RNS + +from rnfs.config import Config +from rnfs.constants import KIB +from rnfs.network import server + +from tests.security._server_helpers import ServerHarness +from tests.support import wait_for +from tests.test_server import block_of + +NO_RTT_WAIT = 1.0 # 2 × 0 + 1: what a link without a usable rtt gets + + +class ProofWaitTest(unittest.TestCase): + """_proof_wait for every rtt a requester can claim.""" + + def setUp(self): + self.harness = ServerHarness({}) + + def wait(self, rtt) -> float: + """_proof_wait for a link whose rtt is rtt.""" + link = self.harness.connect() + link.rtt = rtt + return self.harness.server._proof_wait(link) + + def test_honest_rtts(self): + self.assertAlmostEqual(self.wait(0.05), 1.1) + self.assertAlmostEqual(self.wait(0), NO_RTT_WAIT) + self.assertAlmostEqual(self.wait(1), 3.0) + self.assertEqual(self.wait(2), server.PROOF_GRACE) + + def test_inflated_rtts_capped(self): + for rtt in (1e9, 1e308, 10**400): + with self.subTest(rtt=rtt): + self.assertEqual(self.wait(rtt), server.PROOF_GRACE) + + def test_unusable_rtts_count_as_none(self): + for rtt in (math.inf, -math.inf, math.nan, True, False, -1.0, -1e9, None, "5", b"\x05", [1.0], 1j): + with self.subTest(rtt=rtt): + wait = self.wait(rtt) + self.assertEqual(wait, NO_RTT_WAIT) + self.assertTrue(math.isfinite(wait)) + + def test_no_rtt_attribute(self): + link = self.harness.connect() + del link.rtt + self.assertEqual(self.harness.server._proof_wait(link), NO_RTT_WAIT) + + def test_link_rtt_used_only_through_the_helper(self): + # Any future server-side use of link.rtt must go through _proof_wait, which caps it + with open(server.__file__, encoding="utf-8") as f: + tree = ast.parse(f.read()) + uses = [] + for function in ast.walk(tree): + if not isinstance(function, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for node in ast.walk(function): + attribute = isinstance(node, ast.Attribute) and node.attr == "rtt" + looked_up = isinstance(node, ast.Constant) and node.value == "rtt" + if attribute or looked_up: + uses.append(function.name) + self.assertEqual(set(uses), {"_proof_wait"}) + + +class ProofGraceClockTest(unittest.TestCase): + """The grace a new /blocks waits for a withheld proof, on the server's clock, for claimed rtts.""" + + def setUp(self): + self.block = block_of(100 * KIB) + self.harness = ServerHarness(dict([self.block]), Config()) + + def withhold(self, rtt): + """A link whose first response has all its parts sent but its proof withheld.""" + link = self.harness.connect() + link.rtt = rtt + self.assertIsNot(self.harness.blocks(link, [self.block[0]], request_id=b"a" * 16), False) + self.harness.advertise(link, b"a" * 16).status = RNS.Resource.AWAITING_PROOF + return link + + def ask_in_background(self, link) -> list: + """A second /blocks on link, from another thread; its answer lands in the list returned.""" + results = [] + thread = threading.Thread(target=lambda: results.append( + self.harness.blocks(link, [self.block[0]], request_id=b"b" * 16)), daemon=True) + thread.start() + self.addCleanup(thread.join, 10) + return results + + def test_grace_ends_on_the_server_clock(self): + for rtt in (1e9, math.inf, math.nan, True, -5.0, "x"): + with self.subTest(rtt=rtt): + self.harness = ServerHarness(dict([self.block]), Config()) + link = self.withhold(rtt) + results = self.ask_in_background(link) + time.sleep(0.05) + self.assertEqual(results, []) # waiting out the grace + start = time.monotonic() + self.harness.clock.now += server.PROOF_GRACE # past any grace: the wait ends at once + wait_for(lambda: results, timeout=3, what="the waiting request to be refused") + self.assertLess(time.monotonic() - start, 2) + self.assertEqual(results, [False]) + self.assertEqual(len([r for r in self.harness.server.pools.responses() if r.link_id == link.link_id]), + 1) + + def test_proof_within_the_grace_accepted(self): + link = self.withhold(1e9) + resource = link.outgoing_resources[0] + results = self.ask_in_background(link) + time.sleep(0.05) + resource.complete() + wait_for(lambda: results, timeout=3, what="the waiting request to be answered") + self.assertIsNot(results[0], False) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/security/test_new4_private_mode_records.py b/tests/security/test_new4_private_mode_records.py index 12319f9..8f88198 100644 --- a/tests/security/test_new4_private_mode_records.py +++ b/tests/security/test_new4_private_mode_records.py @@ -48,7 +48,7 @@ def tearDownModule(): RNS.loglevel = _saved_loglevel -def recall(node_id): +def recall(node_id, _no_use=False): """RNS.Identity.recall as RNS would answer it for the test's node IDs (their announces were heard).""" identity_hash = IDENTITIES.get(node_id) return None if identity_hash is None else types.SimpleNamespace(hash=identity_hash) diff --git a/tests/security/test_r1_reconcile_damaged_state.py b/tests/security/test_r1_reconcile_damaged_state.py index e8c7469..7427ba8 100644 --- a/tests/security/test_r1_reconcile_damaged_state.py +++ b/tests/security/test_r1_reconcile_damaged_state.py @@ -166,8 +166,9 @@ def check_kept(self, how: str): feature.save() target = Cid.parse(feature.inbox.get(kept)["auto"][0]["cid"]) source = f"lxmf:{SENDER.hex()}" - # lxmf_inbox.json as rnfs 0.1.4 wrote it, with the message's record malformed - messages = feature.inbox_list() + # lxmf_inbox.json as rnfs 0.1.4 wrote it, with the message's record malformed: the stored records, not + # inbox_list(), which adds computed fields (trusted, contact) 0.1.4 never wrote + messages = feature.inbox.entries() legacy = {"next_id": 2, "messages": [{**messages[0], "links": "not a list"} if how == "malformed" else messages[0]]} self.b.stop() diff --git a/tests/test_api.py b/tests/test_api.py index 87943d4..56a5fc9 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -7,6 +7,7 @@ import inspect import json import os +import re import select import socket import tempfile @@ -15,9 +16,11 @@ import unittest from unittest import mock -from rnfs import cli, fsutil +from rnfs import api, cli, features, fsutil from rnfs.api import ApiHandler, ApiServer, BodyReader, LocalServer, loggable +from rnfs.content import edit from rnfs.content.cid import DAG, RAW, Cid +from rnfs import errors from rnfs.errors import Busy, Denied, InvalidDag, NotFound, RnfsError, StorageFull, TooLarge from tests.support import wait_for @@ -280,6 +283,7 @@ def dir_set(self, target, name, child_cid, executable=False): def dir_rm(self, target): self.record("dir_rm", target) + edit.check_entry_target(target) # as RnfsNode.dir_rm, before anything is fetched parent, name = target.rsplit("/", 1) obj = self._object(self._resolve(parent)) if name not in obj[1]: @@ -729,7 +733,9 @@ def test_error_mapping(self): file_cid = self.node.put_file(b"abc") for error, status in ((ValueError("v"), 400), (NotFound("n"), 404), (TooLarge("t"), 413), (Denied("d"), 451), (InvalidDag("i"), 422), (Busy("b"), 503), (StorageFull("s"), 507), - (RnfsError("r"), 500), (RuntimeError("boom"), 500)): + (RnfsError("r"), 500), (RuntimeError("boom"), 500), + (errors.BlockMismatch("m"), 400), (errors.SealError("s"), 400), + (errors.NotRecipient("n"), 400)): with mock.patch.object(self.node, "stat", side_effect=error), mock.patch("rnfs.api.log"): code, headers, data = self.call("GET", f"/api/stat?target={file_cid}") self.assertEqual(code, status, error) @@ -738,6 +744,39 @@ def test_error_mapping(self): if status == 503: self.assertIn("Retry-After", headers) + def test_every_error_class_keeps_its_status(self): + """The status of every error class rnfs raises, pinned (XC4): one table, rnfs.errors.STATUSES, decides them, + and the classes that became RnfsErrors (BlockMismatch, SealError) stay 400 as the ValueErrors they were.""" + from rnfs import apiclient + pinned = {ValueError: 400, errors.BlockMismatch: 400, errors.SealError: 400, errors.NotRecipient: 400, + apiclient.BadRequest: 400, NotFound: 404, TooLarge: 413, Denied: 451, InvalidDag: 422, Busy: 503, + StorageFull: 507, RnfsError: 500, errors.Cancelled: 500, errors.Closed: 500, OSError: 500, + RuntimeError: 500, KeyError: 500} + for error_class, status in pinned.items(): + with self.subTest(error_class.__name__): + self.assertEqual(api.error_status(error_class("x")), status) + self.assertIs(api.ERROR_STATUSES, errors.STATUSES) + self.assertEqual({status for _, status in errors.STATUSES}, {400, 404, 413, 422, 451, 503, 507}) + # The classes whose base changed are still caught as ValueErrors, and the content.seal names are the same + from rnfs.content import seal + self.assertIs(seal.SealError, errors.SealError) + self.assertIs(seal.NotRecipient, errors.NotRecipient) + for error_class in (errors.BlockMismatch, errors.SealError): + self.assertTrue(issubclass(error_class, RnfsError) and issubclass(error_class, ValueError)) + + def test_block_mismatch_is_logged(self): + """A BlockMismatch is a 400, as the ValueError it was, but a bug, so the API logs it at WARNING; another + ValueError (the caller's) isn't logged.""" + file_cid = self.node.put_file(b"abc") + for error, logged in ((errors.BlockMismatch(f"block does not match {file_cid}"), True), + (ValueError("bad"), False)): + with mock.patch.object(self.node, "stat", side_effect=error), mock.patch("rnfs.api.log") as log: + self.assertEqual(self.call("GET", f"/api/stat?target={file_cid}")[0], 400) + warnings = [c for c in log.call_args_list if c.args[1:] == (api.LOG_WARNING,)] + self.assertEqual(len(warnings), 1 if logged else 0, error) + if logged: + self.assertIn("BlockMismatch", warnings[0].args[0]) + def test_state_changing_endpoints(self): file_cid = self.node.put_file(b"abc") directory = self.node.put_dir({"a": file_cid}) @@ -919,6 +958,33 @@ def failing(*args, **kwargs): self.assertEqual(loggable("plain/path"), "plain/path") +class RouteTableTest(unittest.TestCase): + """ApiHandler.ROUTES, the features' routes and the module docstring's tables agree (ND4).""" + + def test_every_route_names_a_handler_method(self): + """ROUTES dispatches by name through getattr, so a typo would only show when that route is requested.""" + for route, name in ApiHandler.ROUTES.items(): + with self.subTest(route=route): + self.assertTrue(name.startswith("api_"), name) + self.assertTrue(callable(getattr(ApiHandler, name, None)), name) + self.assertLessEqual(ApiHandler.OPEN_ROUTES | {ApiHandler.PROOF_ROUTE}, set(ApiHandler.ROUTES)) + for route, handler in features.api_routes().items(): + with self.subTest(route=route): + self.assertTrue(callable(handler)) + self.assertNotIn(route, ApiHandler.ROUTES) + + def test_docstring_tables_list_every_route(self): + """Each (method, path) the docstring's tables list is a route, once, and every route is listed.""" + documented = [] + for line in api.__doc__.splitlines(): + row = re.match(r"\| (GET|POST) +\| (/api/[^|]*?) +\|", line) + if row is not None: + documented += [(row[1], path) for path in row[2].split(", ")] + self.assertEqual(len(documented), len(set(documented)), "a route is listed twice") + routes = set(ApiHandler.ROUTES) | set(features.api_routes()) + self.assertEqual(sorted(documented), sorted(routes)) + + class BindTest(unittest.TestCase): """The listening socket.""" diff --git a/tests/test_apiclient.py b/tests/test_apiclient.py index bfd9cf4..e9828f9 100644 --- a/tests/test_apiclient.py +++ b/tests/test_apiclient.py @@ -1,5 +1,6 @@ """Tests for the client library of the local API (rnfs/apiclient.py), against a live ApiServer with a FakeNode.""" import http.server +import inspect import io import os import subprocess @@ -10,7 +11,7 @@ from unittest import mock from rnfs import apiclient, cli -from rnfs.api import ApiServer +from rnfs.api import ApiHandler, ApiServer from rnfs.apiclient import (ApiConnection, ApiError, AuthFailed, BadRequest, Busy, ConnectionFailed, DaemonNotRunning, Denied, InvalidDag, NotFound, ServerError, SettingsError, StorageFull, TokenError, TooLarge) from rnfs.config import DEFAULT_API_PORT @@ -220,6 +221,56 @@ def test_names(self): with self.assertRaises(NotFound): self.api.name_resolve("ab" * 16) + def test_directories_archives_and_maintenance(self): + """dir_set and dir_rm, export and import_archive (paths made absolute), gc and verify (ND4).""" + file_cid = self.node.put_file(b"abc") + directory = self.node.put_dir({"a": file_cid}) + edited = self.api.dir_set(directory, "b", file_cid, executable=True)["cid"] + self.assertEqual(self.node.called("dir_set")[0], ((directory, "b", file_cid), {"executable": True})) + self.assertEqual(self.api.dir_rm(f"{edited}/b"), {"cid": directory}) + with self.assertRaises(BadRequest): + self.api.dir_rm(directory) + os.chdir(self.tmp) + self.addCleanup(os.chdir, ROOT) + self.assertEqual(self.api.export(directory, "archive")["roots"], [directory]) + sent = self.node.called("export")[0][0][1] + self.assertEqual(real(sent), real(os.path.join(self.tmp, "archive"))) + self.assertEqual(len(self.api.import_archive("archive")["roots"]), 1) + self.assertTrue(os.path.isabs(self.node.called("import_archive")[0][0][0])) + self.assertEqual(self.api.gc(), {"removed_blocks": 3, "freed_bytes": 300}) + self.assertEqual(self.api.verify()["bad"], []) + + def test_remote_pins_providers_and_ping(self): + """pin_remote, providers and ping, node IDs in hex (ND4).""" + cid = self.node.put_file(b"abc") + self.assertIs(self.api.pin_remote("ab" * 16, cid), True) + self.assertIs(self.api.pin_remote("ff" * 16, cid, pin=False), False) + self.assertEqual(self.node.called("pin_remote"), [((bytes.fromhex("ab" * 16), cid, True), {}), + ((bytes.fromhex("ff" * 16), cid, False), {})]) + self.node.provider_map[cid] = [bytes.fromhex("cd" * 16)] + self.assertEqual(self.api.providers(cid), ["cd" * 16]) + self.assertEqual(self.api.ping("cd" * 16), 0.25) + self.assertIsNone(self.api.ping("ff" * 16)) + + def test_name_keys(self): + """name_export, name_rm and name_import (the key as the body) (ND4).""" + name = self.api.name_create("site")["name"] + key = self.api.name_export("site") + self.assertEqual(key, self.node.keys["site"]) + self.api.name_rm("site") + with self.assertRaises(NotFound): + self.api.name_rm("site") + self.assertEqual(self.api.name_import("again", memoryview(key)), {"alias": "again", "name": name}) + + def test_every_core_route_has_a_method(self): + """Each of the API's own routes is sent by one of ApiConnection's methods, so neither rnfs.Client nor + another program writes a path by hand (ND4); the features' routes are reached through get() and post().""" + source = inspect.getsource(ApiConnection) + for method, path in ApiHandler.ROUTES: + with self.subTest(f"{method} {path}"): + self.assertRegex(source, rf'self\.(call|request)\("{method}", "{path}"|' + rf'self\._?{method.lower()}\("{path}"') + def test_jobs(self): """submit_job, job, jobs, cancel_job and clear_jobs reach the job manager.""" cid = self.node.put_file(b"abc") @@ -330,10 +381,28 @@ def spy(handler): def test_daemon_closing_every_connection(self): """A daemon before 0.1.5 closes the proof's connection: the request goes on a new one.""" seen = self.tokens_seen() - with mock.patch.object(self.server.server.RequestHandlerClass, "PROOF_ROUTE", None): + with mock.patch.object(self.server.server.RequestHandlerClass, "PROOF_ROUTE", None), \ + mock.patch("rnfs.api.__version__", "0.1.4"): self.assertEqual(self.api.status()["node_id"], self.node.status()["node_id"]) self.assertEqual(seen, [("/api/status", None), ("/api/status", self.server.token)]) + def test_newer_daemon_closing_the_connection(self): + """A daemon of 0.1.5 or later that closes the proof's connection gets nothing more (API-4).""" + seen = self.tokens_seen() + with mock.patch.object(self.server.server.RequestHandlerClass, "PROOF_ROUTE", None), \ + mock.patch("rnfs.api.__version__", "0.1.5"): + with self.assertRaises(ConnectionFailed) as raised: + self.api.status() + self.assertIn("closed the connection it proved itself on", str(raised.exception)) + self.assertEqual(seen, [("/api/status", None)]) + + def test_version_numbers(self): + """The numbers a version starts with, for the fallback's comparison.""" + for version, expected in (("0.1.4", (0, 1, 4)), ("0.1.5.dev1", (0, 1, 5)), ("0.1.4.post1", (0, 1, 4)), + ("1", (1,)), ("10.0rc1", (10, 0)), ("", None), ("v0.1.4", None), (None, None), + (15, None)): + self.assertEqual(apiclient.version_numbers(version), expected, version) + def test_daemon_restarted(self): """A client used before the daemon restarted reads its new token and goes on working.""" self.api.status() @@ -407,7 +476,8 @@ def test_api_statuses(self): (errors.TooLarge("t"), TooLarge, 413), (errors.InvalidDag("i"), InvalidDag, 422), (errors.Denied("d"), Denied, 451), (errors.Busy("b"), Busy, 503), (errors.StorageFull("s"), StorageFull, 507), - (errors.RnfsError("r"), ServerError, 500)) + (errors.RnfsError("r"), ServerError, 500), (errors.BlockMismatch("m"), BadRequest, 400), + (errors.SealError("s"), BadRequest, 400), (errors.NotRecipient("n"), BadRequest, 400)) for error, expected, status in cases: with mock.patch.object(self.node, "stat", side_effect=error), mock.patch("rnfs.api.log"): with self.assertRaises(expected) as raised: @@ -415,6 +485,17 @@ def test_api_statuses(self): self.assertEqual((str(raised.exception), raised.exception.status), (str(error), status)) self.assertIsInstance(raised.exception, ApiError) + def test_status_table_follows_the_node_errors(self): + """STATUS_ERRORS comes from rnfs.errors.STATUSES (XC4): each status's error is an ApiError and a subclass of + the node's class for it, so one except clause catches a Node's error and a Client's; 403 is AuthFailed.""" + self.assertEqual(set(apiclient.STATUS_ERRORS), {status for _, status in errors.STATUSES} | {403}) + for error_class, status in errors.STATUSES: + with self.subTest(status): + client_class = apiclient.STATUS_ERRORS[status] + self.assertTrue(issubclass(client_class, ApiError) and issubclass(client_class, error_class)) + self.assertIs(type(apiclient.error_for(status, "x")), client_class) + self.assertIs(apiclient.STATUS_ERRORS[403], AuthFailed) + def test_refused_without_the_token(self): """A protected endpoint asked without the token is AuthFailed (403); an unknown one NotFound.""" with self.assertRaises(AuthFailed) as raised: diff --git a/tests/test_archive.py b/tests/test_archive.py index d50ae90..a72b4e1 100644 --- a/tests/test_archive.py +++ b/tests/test_archive.py @@ -7,6 +7,7 @@ from RNS.vendor import umsgpack +from rnfs.constants import MAX_BLOCK_SIZE from rnfs.content import archive, builder, dag, walker from rnfs.content.cid import EMPTY_DIRECTORY, RAW, Cid from rnfs.content.dag import DirectoryNode, Entry @@ -203,6 +204,24 @@ def test_root_count_is_capped_before_reading_roots(self): self.assertEqual(len(archive.import_archive(io.BytesIO(data), self.destination)["complete"]), archive.MAX_ROOTS) + def test_archive_size_matches_export_at_msgpack_header_boundaries(self): + # archive_size computes the bin headers itself: 2 bytes up to 255 bytes, 3 up to 65535, 5 above (CT5) + roots = [] + for length in (0, 1, 255, 256, 65535, 65536, MAX_BLOCK_SIZE): + block = bytes([length % 251]) * length + cid = Cid.for_block(RAW, block) + self.source.put(cid, block) + roots.append(cid) + with self.subTest(length=length): + self.assertEqual(archive.archive_size([cid], [(cid, length)]), len(self.export([cid]))) + blocks = [(e.cid, len(block)) for e, block in walker.iter_dags([walker.root_expect(cid) for cid in roots], + self.source.getter())] + self.assertEqual(archive.archive_size(roots, blocks), len(self.export(roots))) + tree = [self.root.cid, self.other.cid] + blocks = [(e.cid, len(block)) for e, block in walker.iter_dags([walker.root_expect(cid) for cid in tree], + self.source.getter())] + self.assertEqual(archive.archive_size(tree, blocks), len(self.export(tree))) + def test_empty_file_root(self): empty = builder.build_bytes(b"", self.source.put) data = self.export([empty.cid]) diff --git a/tests/test_blockstore.py b/tests/test_blockstore.py index cc4fcfe..ff9d1dc 100644 --- a/tests/test_blockstore.py +++ b/tests/test_blockstore.py @@ -1,4 +1,5 @@ """Tests for BlockStore: one verified, atomically written file per CID (specification section 8).""" +import base64 import os import tempfile import unittest @@ -6,7 +7,7 @@ from rnfs import fsutil from rnfs.constants import MAX_BLOCK_SIZE -from rnfs.content.blockstore import PARTIAL_PREFIX, BlockStore +from rnfs.content.blockstore import PARTIAL_PREFIX, BlockStore, matches from rnfs.content.cid import DAG, RAW, Cid HELLO_WORLD = "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e" @@ -122,5 +123,56 @@ def test_ignores_foreign_files_and_deletes_partial_files(self): self.assertEqual(sorted(os.listdir(self.tmp.name)), sorted(["notes.txt", HELLO_WORLD.upper()])) + def test_scan_names_blocks_by_their_canonical_cids_only(self): + raw, node = Cid.for_block(RAW, b"leaf"), Cid.for_block(DAG, b"node") + self.store.put(raw, b"leaf") + self.store.put(node, b"node") + alphabet = "abcdefghijklmnopqrstuvwxyz234567" + spare_bits_set = str(raw)[:-1] + alphabet[alphabet.index(str(raw)[-1]) + 1] # parse() refuses it + dag_pb = "b" + base64.b32encode(bytes([1, 0x70, 0x12, 0x20]) + bytes(32)).decode().lower().rstrip("=") + names = [spare_bits_set, dag_pb, str(raw) + "="] + upper = "b" + str(node)[1:].upper() + # On a case-insensitive filesystem (macOS, Windows) the upper-case name is the node's own file: writing it + # would empty the block, and it can't be a second file there anyway + if not os.path.exists(os.path.join(self.tmp.name, upper)): + names.append(upper) + for name in names: + open(os.path.join(self.tmp.name, name), "wb").close() + scanned = sorted(BlockStore(self.tmp.name).scan()) + self.assertEqual(scanned, sorted([(raw, 4), (node, 4)])) + self.assertEqual(sorted(str(cid) for cid, _ in scanned), sorted([str(raw), str(node)])) + self.assertEqual({cid.codec for cid, _ in scanned}, {RAW, DAG}) + + +class MatchesTest(unittest.TestCase): + """matches(): the one block validity check (size limit and hash) that storing, reading and recovery share.""" + + def test_raw_and_dag_cids_match_their_blocks(self): + for codec in (RAW, DAG): + with self.subTest(codec=codec): + self.assertTrue(matches(Cid.for_block(codec, b"block"), b"block")) + + def test_wrong_block(self): + for codec in (RAW, DAG): + with self.subTest(codec=codec): + self.assertFalse(matches(Cid.for_block(codec, b"real"), b"fake")) + self.assertFalse(matches(Cid.for_block(codec, b"real"), b"real ")) + + def test_empty_block(self): + empty = Cid.for_block(RAW, b"") + self.assertTrue(matches(empty, b"")) + self.assertFalse(matches(empty, b"x")) + self.assertFalse(matches(Cid.for_block(RAW, b"x"), b"")) + + def test_size_limit(self): + largest = bytes(MAX_BLOCK_SIZE) + self.assertTrue(matches(Cid.for_block(RAW, largest), largest)) + too_large = bytes(MAX_BLOCK_SIZE + 1) + cid = Cid.for_block(RAW, too_large) + with mock.patch.object(Cid, "verify", wraps=cid.verify) as verify: + self.assertFalse(matches(cid, too_large)) + verify.assert_not_called() # refused on its size, without hashing it + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_builder.py b/tests/test_builder.py index 089787c..26abb56 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -1,18 +1,22 @@ -"""Tests for building trees (specification section 5): the tree rule, files, directories, wrap, only-hash.""" +"""Tests for building trees (specification section 5): the tree rule, files, directories, wrap, only-hash, and the +options a build is given.""" +import hashlib import io import os import tempfile import threading import unittest +from concurrent.futures import ThreadPoolExecutor from unittest import mock from rnfs import constants, fsutil -from rnfs.content import builder, dag +from rnfs.content import builder, chunker, dag from rnfs.content.builder import NO_PUT from rnfs.content.cid import DAG, EMPTY_DIRECTORY, EMPTY_FILE, RAW, Cid from rnfs.content.dag import Child, DirectoryNode, Entry, FileNode, IndexChild, IndexNode from rnfs.errors import TooLarge from tests.support import CAN_SYMLINK, HAS_FIFO, POSIX_MODES +from tests.test_chunker import FIXED_CID, VECTOR_CID, pseudo HELLO_WORLD = "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e" # What add records for a file with the execute bit: Windows has none, so there every file is kind 0 @@ -27,7 +31,7 @@ def run(): """Call function, keeping its result or exception.""" try: outcome.append(("result", function(*args))) - except BaseException as e: # noqa: B902 - handed back to the test + except BaseException as e: # handed back to the test outcome.append(("error", e)) thread = threading.Thread(target=run, daemon=True) # a hung call mustn't keep the test run alive @@ -508,5 +512,84 @@ def swap(): self.assertTrue(any("victim" in w and "changed" in w for w in warnings)) +class Refused(Exception): + """What the tests' deny checks raise.""" + + +class OptionsTest(unittest.TestCase): + """BuildOptions passed explicitly (maintainability audit CT2): a ContextVar isn't copied to a worker thread, so + the chunker and the deny check must travel with the build.""" + + def setUp(self): + """A temporary directory with a small tree.""" + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = os.path.join(self.tmp.name, "root") + os.makedirs(os.path.join(self.root, "sub")) + for relative, data in (("a.txt", b"a"), ("sub/denied.txt", b"denied content")): + with open(os.path.join(self.root, relative), "wb") as f: + f.write(data) + + @staticmethod + def deny(content: bytes): + """A check refusing files whose content is content.""" + denied = hashlib.sha256(content).digest() + + def check(digest: bytes): + """Raise Refused for the denied digest.""" + if digest == denied: + raise Refused(digest.hex()) + return check + + def on_worker(self, function, *args, **kwargs): + """function's result, run on a worker thread (which starts with a fresh context, as a pool's do).""" + with ThreadPoolExecutor(1) as pool: + return pool.submit(function, *args, **kwargs).result(10) + + def test_a_build_on_a_worker_thread_with_explicit_options_still_denies(self): + check = self.deny(b"denied content") + options = builder.BuildOptions(check=check) + with self.assertRaises(Refused): + self.on_worker(builder.add_path, self.root, NO_PUT, options=options) + with self.assertRaises(Refused): + self.on_worker(builder.add_stream, io.BytesIO(b"denied content"), NO_PUT, options=options) + # What CT2 fixes: the context's check doesn't reach a worker thread, so only explicit options are safe there + with builder.checking(check): + with self.assertRaises(Refused): + builder.add_path(self.root, NO_PUT) # the wrapper still works on its own thread + built, _ = self.on_worker(builder.add_path, self.root, NO_PUT) + self.assertEqual(built.cid, builder.add_path(self.root, NO_PUT)[0].cid) + + def test_explicit_options_override_the_context(self): + with builder.checking(self.deny(b"denied content")): + builder.add_path(self.root, NO_PUT, options=builder.BuildOptions()) # no check: not refused + with self.assertRaises(Refused): + builder.add_path(self.root, NO_PUT, options=builder.BuildOptions(check=self.deny(b"a"))) + + def test_check_sees_every_file_once(self): + seen = [] + builder.add_path(self.root, NO_PUT, options=builder.BuildOptions(check=seen.append)) + self.assertEqual(sorted(seen), sorted(hashlib.sha256(data).digest() for data in (b"a", b"denied content"))) + + def test_fastcdc_vector_with_explicit_options(self): + data = pseudo(1 << 20) + fastcdc = builder.BuildOptions(chunker.FASTCDC) + self.assertEqual(str(self.on_worker(builder.build_bytes, data, NO_PUT, options=fastcdc).cid), VECTOR_CID) + self.assertEqual(str(builder.build_bytes(data, NO_PUT, options=fastcdc).cid), VECTOR_CID) + self.assertEqual(str(builder.build_bytes(data, NO_PUT, options=builder.BuildOptions()).cid), FIXED_CID) + with chunker.using(chunker.FASTCDC): # the wrapper, kept + self.assertEqual(str(builder.build_bytes(data, NO_PUT).cid), VECTOR_CID) + self.assertEqual(str(self.on_worker(builder.build_bytes, data, NO_PUT).cid), FIXED_CID) + self.assertEqual(builder.current_options(), fastcdc) + self.assertEqual(str(self.on_worker(builder.build_bytes, data, NO_PUT, + options=builder.current_options()).cid), VECTOR_CID) + + def test_chunker_names(self): + self.assertEqual(builder.BuildOptions(None).chunker, chunker.FIXED) + self.assertEqual(builder.BuildOptions().chunker, chunker.FIXED) + with self.assertRaises(ValueError): + builder.BuildOptions("rabin") + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_chunker.py b/tests/test_chunker.py index 8025b5c..7a2db0c 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -42,8 +42,9 @@ def simple_cuts(data: bytes) -> list[int]: class ChunkerTest(unittest.TestCase): def test_parameters(self): - self.assertEqual((chunker.MIN_SIZE, chunker.AVG_SIZE, chunker.MAX_SIZE), (16384, 65536, 131072)) - self.assertEqual(chunker.MAX_SIZE, MAX_BLOCK_SIZE) + self.assertEqual((chunker.MIN_SIZE, chunker.AVG_SIZE, chunker.FASTCDC_MAX), (16384, 65536, 131072)) + self.assertEqual(chunker.FASTCDC_MAX, MAX_BLOCK_SIZE) + self.assertEqual(chunker.MAX_SIZE, chunker.FASTCDC_MAX) # the name before 0.1.6, kept self.assertEqual(chunker.GEAR[0], 0x61304b5b3811834c) self.assertEqual(chunker.GEAR[255], 0x7989ca9814accbdb) self.assertEqual(chunker.GEAR[7], int.from_bytes(hashlib.sha256(b"rnfs-fastcdc-gear\x07").digest()[:8], "big")) @@ -72,12 +73,12 @@ def test_bounds(self): self.assertEqual(sum(lengths), len(data)) for length in lengths[:-1]: self.assertGreaterEqual(length, chunker.MIN_SIZE) - self.assertLessEqual(length, chunker.MAX_SIZE) - self.assertLessEqual(lengths[-1], chunker.MAX_SIZE) + self.assertLessEqual(length, chunker.FASTCDC_MAX) + self.assertLessEqual(lengths[-1], chunker.FASTCDC_MAX) average = len(data) / len(lengths) self.assertTrue(48 * 1024 < average < 96 * 1024, average) # normalized chunking stays near 64 KiB - # No cut point at all: every chunk is MAX_SIZE - self.assertEqual(set(chunker.fastcdc_cuts(bytes(1 << 20))), {chunker.MAX_SIZE}) + # No cut point at all: every chunk is FASTCDC_MAX + self.assertEqual(set(chunker.fastcdc_cuts(bytes(1 << 20))), {chunker.FASTCDC_MAX}) self.assertEqual(chunker.fastcdc_cuts(b"x" * 100), [100]) def test_boundaries_survive_an_insertion(self): @@ -148,13 +149,13 @@ def get(expect: walker.Expect) -> bytes: def test_tree_rule_over_many_chunks(self): """More than MAX_CHILDREN chunks are grouped by the tree rule, as for fixed chunks.""" - data = bytes(300 * chunker.MAX_SIZE) # zeros: every chunk is MAX_SIZE, so 300 chunks + data = bytes(300 * chunker.FASTCDC_MAX) # zeros: every chunk is FASTCDC_MAX, so 300 chunks blocks: dict[Cid, bytes] = {} with chunker.using(chunker.FASTCDC): built = builder.build_bytes(data, lambda cid, block: blocks.setdefault(cid, block)) root = dag.decode(blocks[built.cid]) self.assertEqual(len(root.children), 2) - self.assertEqual([c.size for c in root.children], [256 * chunker.MAX_SIZE, 44 * chunker.MAX_SIZE]) + self.assertEqual([c.size for c in root.children], [256 * chunker.FASTCDC_MAX, 44 * chunker.FASTCDC_MAX]) def test_throughput(self): """Report MB/s (not a pass/fail threshold beyond a sanity floor).""" diff --git a/tests/test_cid.py b/tests/test_cid.py index 30568dc..537f380 100644 --- a/tests/test_cid.py +++ b/tests/test_cid.py @@ -3,6 +3,7 @@ import copy import hashlib import pickle +import random import unittest from RNS.vendor import umsgpack @@ -20,6 +21,30 @@ def written(binary: bytes) -> str: return "b" + base64.b32encode(binary).decode().lower().rstrip("=") +def malformed() -> dict[str, object]: + """Reason -> a value that is not a valid written CID.""" + digest = bytes(32) + return { + "not a string": b"bafk", + "wrong multibase": "z" + HELLO_WORLD[1:], + "empty": "", + "too short": "bae", + "truncated": HELLO_WORLD[:-4], + "not base32": "b!!!!", + "non-ascii": "bafkreié", + "padded": HELLO_WORLD + "====", + "uppercase": HELLO_WORLD.upper(), + "uppercase body": "b" + HELLO_WORLD[1:].upper(), + "unused bits set": HELLO_WORLD[:-1] + chr(ord(HELLO_WORLD[-1]) + 1), + "CIDv0-like version": written(bytes([0, 0x55, 0x12, 0x20]) + digest), + "dag-pb codec": written(bytes([1, 0x70, 0x12, 0x20]) + digest), + "wrong hash": written(bytes([1, 0x55, 0x13, 0x20]) + digest), + "short digest": written(bytes([1, 0x55, 0x12, 0x20]) + digest[:31]), + "long digest": written(bytes([1, 0x55, 0x12, 0x20]) + digest + b"\0"), + "dag codec, raw length": written(bytes.fromhex("018080c0011220") + digest[:29]), + } + + class CidTest(unittest.TestCase): """Building, parsing and rejecting CIDs.""" @@ -85,25 +110,7 @@ def test_empty_directory(self): def test_rejects_malformed(self): digest = bytes(32) - bad = { - "not a string": b"bafk", - "wrong multibase": "z" + HELLO_WORLD[1:], - "empty": "", - "too short": "bae", - "truncated": HELLO_WORLD[:-4], - "not base32": "b!!!!", - "non-ascii": "bafkreié", - "padded": HELLO_WORLD + "====", - "uppercase": HELLO_WORLD.upper(), - "uppercase body": "b" + HELLO_WORLD[1:].upper(), - "unused bits set": HELLO_WORLD[:-1] + chr(ord(HELLO_WORLD[-1]) + 1), - "CIDv0-like version": written(bytes([0, 0x55, 0x12, 0x20]) + digest), - "dag-pb codec": written(bytes([1, 0x70, 0x12, 0x20]) + digest), - "wrong hash": written(bytes([1, 0x55, 0x13, 0x20]) + digest), - "short digest": written(bytes([1, 0x55, 0x12, 0x20]) + digest[:31]), - "long digest": written(bytes([1, 0x55, 0x12, 0x20]) + digest + b"\0"), - "dag codec, raw length": written(bytes.fromhex("018080c0011220") + digest[:29]), - } + bad = malformed() for reason, value in bad.items(): with self.subTest(reason): with self.assertRaises(ValueError): @@ -115,6 +122,45 @@ def test_rejects_malformed(self): with self.assertRaises(ValueError): Cid.from_bytes(binary) + def test_from_block_name_accepts_exactly_what_parse_accepts(self): + # The block-directory scan's cheaper check (S2) must agree with parse() on every name, valid or not + rng = random.Random(4) + valid = [str(Cid.for_block(codec, bytes([i]))) for codec in (RAW, DAG) for i in range(4)] + names = list(valid) + list(malformed().values()) + [None, 1, HELLO_WORLD.encode()] + # One digit replaced: by base32 digits (most of which set spare bits in the final digit), by other ASCII, + # and by non-ASCII whose upper() is ASCII ("ı" -> "I", "ſ" -> "S") + pool = cid_module._BASE32 + cid_module._BASE32.upper() + "0189=_ -!\n" + "ıſéK" + for name in valid[::3]: + for i in range(len(name)): + names += [name[:i] + c + name[i + 1:] for c in pool] + for name in valid: + names += [name[:-1], name + "a", name + "=", name + "\n", " " + name, name[:-1] + "=", name[1:]] + # Random binaries of every length near a CID's, most with a wrong prefix, and the prefixes one bit off + for length in range(30, 45): + names += [written(rng.randbytes(length)) for _ in range(20)] + for prefix in cid_module._PREFIXES.values(): + for bit in range(len(prefix) * 8): + flipped = (int.from_bytes(prefix, "big") ^ (1 << bit)).to_bytes(len(prefix), "big") + names.append(written(flipped + rng.randbytes(32))) + names += [written(prefix + rng.randbytes(32)) for _ in range(50)] + accepted = 0 + for name in names: + with self.subTest(name=name): + try: + expected = Cid.parse(name) + except ValueError: + with self.assertRaises(ValueError): + Cid.from_block_name(name) + continue + cid = Cid.from_block_name(name) + accepted += 1 + self.assertEqual(cid, expected) + self.assertEqual((cid.binary, cid.codec, str(cid), cid.key), (expected.binary, expected.codec, + str(expected), expected.key)) + self.assertIs(str(cid), name) # kept as the written form, not encoded again + self.assertGreater(accepted, 100) # both outcomes were exercised + self.assertGreater(len(names) - accepted, 1000) + def test_ordering(self): a, b = sorted([Cid.for_block(RAW, b"1"), Cid.for_block(RAW, b"2")]) self.assertLess(a.binary, b.binary) diff --git a/tests/test_cli.py b/tests/test_cli.py index fce36c4..915d1e6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -507,14 +507,13 @@ class DaemonLockTest(unittest.TestCase): def test_second_daemon_stops_at_once(self): """With the lock held, `rnfs daemon` exits with an error naming the holder, before writing state or rotating the debug log.""" - from rnfs.config import Config from rnfs.node import _make_private_dirs from tests.test_fsutil import hold_lock tmp = tempfile.TemporaryDirectory() self.addCleanup(tmp.cleanup) rnfs_dir = os.path.join(tmp.name, "rnfs") - _make_private_dirs(Config(rnfs_dir=rnfs_dir)) # as the first daemon left it + _make_private_dirs(rnfs_dir) # as the first daemon left it debug_log = os.path.join(rnfs_dir, "debug.log") with open(debug_log, "w", encoding="utf-8") as f: f.write("the first daemon's log\n") diff --git a/tests/test_client.py b/tests/test_client.py index c46b168..13146fb 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -388,6 +388,14 @@ def is_member(self, identity_hash): """Whether the identity is a member.""" return False + def served_block_size(self, cid): + """Not known: blocks are read to size them.""" + return None + + def feature_requests(self): + """No feature paths.""" + return {} + def run_e2e_server(configdir: str): """Serve rnfs.dht until stdin closes, printing the node ID first.""" diff --git a/tests/test_config.py b/tests/test_config.py index 0b6c49b..c6e3596 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -34,6 +34,12 @@ def test_whole_numbers(self): self.assert_invalid(**{name: value}) Config(gateway_fetches=0, max_output_entries=0, max_storage=0, max_blocks_responses=2).validate() + def test_follow_quota(self): + """A whole number of bytes; 0 turns follow fetches off.""" + for value in (-1, "1", 1.5, True, None): + self.assert_invalid(follow_quota=value) + Config(follow_quota=0).validate() + def test_optional_whole_numbers(self): for name in ("alpha", "max_parallel_providers"): for value in (0, "3", True, 1.5): @@ -74,6 +80,16 @@ def test_identity_lists(self): self.assert_invalid(**{name: value}) Config(**{name: [ID, f"<{ID.upper()}>", f" {ID} "]}).validate() + def test_identity_sets(self): + """member_set and remote_pin_set hold each list once, as identity.hash.hex() writes it, so the node, the + link pool and the server all agree on who is listed; they follow later changes to the lists.""" + other = "cd" * 16 + config = Config(members=[f" <{ID.upper()}>", ID], remote_pin_identities=[f"{other.upper()}\n"]) + self.assertEqual(config.member_set, frozenset({ID})) + self.assertEqual(config.remote_pin_set, frozenset({other})) + config.remote_pin_identities = [] + self.assertEqual(config.remote_pin_set, frozenset()) + def test_other_lists_and_paths(self): for value in (ID, [1], None): self.assert_invalid(bootstrap_nodes=value) diff --git a/tests/test_contacts.py b/tests/test_contacts.py index 024ae9f..542c4cd 100644 --- a/tests/test_contacts.py +++ b/tests/test_contacts.py @@ -1,6 +1,7 @@ """Tests for the address book (rnfs/features/contacts.py): nicks, kinds, persistence, and nick targets and node IDs end to end through the node facade, the local API (including jobs) and the CLI helpers.""" import http.client +import inspect import json import os import tempfile @@ -15,6 +16,7 @@ from rnfs.config import Config from rnfs.content.cid import Cid from rnfs.content.storage import USER +from rnfs import features from rnfs.errors import NotFound from rnfs.features import contacts from rnfs.features.contacts import check_nick, is_nick, node_arg @@ -161,6 +163,19 @@ def test_failed_save_is_written_with_the_next_change(self): self.book.remove("bnode") self.assertEqual(self.rows(), []) + def test_changes_are_durable_and_a_failed_one_is_written_by_save(self): + """Each change is committed durably as it is made; one whose commit failed is written by the node's next + save of its features (periodic, and at stop).""" + real = self.book._db.transaction + with mock.patch.object(self.book._db, "transaction", side_effect=real) as opened: + self.book.add("anode", A_ID.hex(), "node") + self.assertEqual([call.args for call in opened.call_args_list], [(True,)]) + with failing_commits(), self.assertRaises(OSError): + self.book.remove("anode") + self.assertEqual(self.rows(), [("anode", "node", A_ID)]) + self.b._save_state() + self.assertEqual(self.rows(), []) + def test_kind_detection(self): self.assertEqual(self.book.add("alice", self.name.hex())["kind"], "name") # a record B holds self.assertEqual(self.book.add("anode", A_ID.hex())["kind"], "node") # in B's routing table @@ -209,6 +224,35 @@ def test_resolve_id(self): for text, kind in (("anode", "name"), ("nobody", "node"), ("Bad!", "node")): with self.assertRaises(ValueError): self.book.resolve_id(text, kind) + # Worded for each kind: LXMF's recipients come through here too + self.book.add("relay", B_ID.hex(), "lxmf") + self.assertEqual(self.book.resolve_id("relay", "lxmf"), B_ID) + for text, kind, message in (("anode", "lxmf", "contact anode is not an LXMF address"), + ("relay", "name", "contact relay is not a name"), + ("nobody", "lxmf", "no LXMF contact nobody"), + ("Bad!", "lxmf", "invalid LXMF address 'Bad!'"), + ("Bad!", "node", "invalid node 'Bad!'")): + with self.assertRaisesRegex(ValueError, message): + self.book.resolve_id(text, kind) + + def test_meets_the_contacts_api(self): + """The contract other features rely on (rnfs.features.ContactsApi): every method, with the same + parameters, so renaming one fails here instead of silently turning off what uses it (audit FT4).""" + self.assertIsInstance(self.book, features.ContactsApi) + self.assertIs(self.b.feature("contacts"), self.book) + methods = [name for name, value in vars(features.ContactsApi).items() + if callable(value) and not name.startswith("_")] + self.assertEqual(len(methods), 5) + for method in methods: + with self.subTest(method): + expected = inspect.signature(getattr(features.ContactsApi, method)) + actual = inspect.signature(getattr(contacts.ContactsFeature, method)) + self.assertEqual(list(actual.parameters), list(expected.parameters)) + self.assertEqual([p.default for p in actual.parameters.values()], + [p.default for p in expected.parameters.values()]) + # And the features using it declare it + for name in ("jobs", "keepers", "follow", "lxmf"): + self.assertIs(self.b.feature(name).dependency("contacts"), self.book) class ContactsApiTest(ContactsNodeTest.__base__): diff --git a/tests/test_daemon_wait.py b/tests/test_daemon_wait.py new file mode 100644 index 0000000..b8d3992 --- /dev/null +++ b/tests/test_daemon_wait.py @@ -0,0 +1,176 @@ +"""The daemon's main thread waits for a stop signal without waking each second on POSIX (S7, rnfs/cli.py).""" +import json +import os +import signal +import subprocess +import sys +import tempfile +import threading +import time +import unittest +import urllib.request +from unittest import mock + +from rnfs import cli, fsutil +from tests.support import free_port, wait_for + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +POSIX = os.name == "posix" + +# A Reticulum of its own with no interfaces: the daemon starts without touching any network +RNS_CONFIG = """ +[reticulum] + enable_transport = No + share_instance = No +[logging] + loglevel = 2 +[interfaces] +""" + + +class RecordingEvent(threading.Event): + """An Event recording each wait's timeout; it is set once stop_after waits returned unset (never blocking).""" + + def __init__(self, stop_after: int): + """Unset, nothing recorded.""" + super().__init__() + self.timeouts = [] + self.stop_after = stop_after + + def wait(self, timeout=None): + """Record timeout and return at once, set from the stop_after-th wait on.""" + self.timeouts.append(timeout) + if len(self.timeouts) >= self.stop_after: + self.set() + return super().wait(0) + + +class WaitUntilStoppingTest(unittest.TestCase): + """cli.wait_until_stopping, as run_daemon uses it.""" + + def test_waits_without_a_timeout_on_posix(self): + stopping = RecordingEvent(stop_after=1) + with mock.patch.object(fsutil, "IS_WINDOWS", False): + cli.wait_until_stopping(stopping) + self.assertEqual(stopping.timeouts, [None]) + + def test_wakes_each_second_on_windows(self): + # A wait isn't interrupted for signal handlers there + stopping = RecordingEvent(stop_after=3) + with mock.patch.object(fsutil, "IS_WINDOWS", True): + cli.wait_until_stopping(stopping) + self.assertEqual(stopping.timeouts, [1.0, 1.0, 1.0]) + stopping = RecordingEvent(stop_after=2) + with mock.patch.object(fsutil, "IS_WINDOWS", True): + cli.wait_until_stopping(stopping, 0.25, lambda: None) + self.assertTrue(all(0 <= timeout <= 0.25 for timeout in stopping.timeouts), stopping.timeouts) + + def test_task_runs_every_interval(self): + # The debug log's status line: the wait lasts until it is due + calls = [] + stopping = threading.Event() + + def task(): + """Count, and stop at the third.""" + calls.append(time.monotonic()) + if len(calls) == 3: + stopping.set() + + start = time.monotonic() + cli.wait_until_stopping(stopping, 0.1, task) + self.assertEqual(len(calls), 3) + self.assertGreaterEqual(calls[0] - start, 0.1) + self.assertTrue(all(b - a >= 0.1 for a, b in zip(calls, calls[1:])), calls) + + @unittest.skipUnless(POSIX, "a signal interrupts a wait on POSIX only") + def test_a_signal_ends_a_wait_without_timeout(self): + # Other threads are running, as in the daemon, and the signal is sent to the process (as kill does) + for signum in (signal.SIGTERM, signal.SIGINT): + with self.subTest(signal=signal.Signals(signum).name): + stopping = threading.Event() + handled = [] + + def handler(number, frame): + """As run_daemon's: set stopping.""" + handled.append(number) + stopping.set() + + previous = signal.signal(signum, handler) + idle = threading.Event() + busy = [threading.Thread(target=idle.wait, daemon=True) for _ in range(4)] + for thread in busy: + thread.start() + sender = threading.Timer(0.3, os.kill, (os.getpid(), signum)) + # If the signal never ended the wait, end it anyway so the suite goes on (and the test fails) + watchdog = threading.Timer(20, stopping.set) + start = time.monotonic() + sender.start() + watchdog.start() + try: + cli.wait_until_stopping(stopping) + elapsed = time.monotonic() - start + finally: + watchdog.cancel() + idle.set() + signal.signal(signum, previous) + self.assertEqual(handled, [signum]) + self.assertLess(elapsed, 10) + + +@unittest.skipUnless(POSIX, "Windows stops the daemon with CTRL_BREAK_EVENT, sent to a process group") +class DaemonSignalTest(unittest.TestCase): + """A real daemon (no interfaces) stops cleanly on SIGTERM and on SIGINT while its main thread waits unbounded.""" + + def test_daemon_stops_on_sigterm_and_sigint(self): + for signum in (signal.SIGTERM, signal.SIGINT): + with self.subTest(signal=signal.Signals(signum).name): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + rnfs_dir = os.path.join(tmp.name, "rnfs") + os.makedirs(rnfs_dir, mode=0o700) + with open(os.path.join(rnfs_dir, "config"), "w", encoding="utf-8") as f: + f.write(RNS_CONFIG) + with open(os.path.join(rnfs_dir, "rnfs.toml"), "w", encoding="utf-8") as f: + f.write("default_bootstrap = false\nlxmf_enabled = false\n") + api_port = free_port() + log_path = os.path.join(tmp.name, "daemon.log") + with open(log_path, "wb") as log: + proc = subprocess.Popen( + [sys.executable, "rnfs.py", "--dir", rnfs_dir, "--api-port", str(api_port), "daemon", + "--rnsconfig", rnfs_dir, "--gateway-port", str(free_port())], + cwd=ROOT, stdout=log, stderr=subprocess.STDOUT, env={**os.environ, "PYTHONUNBUFFERED": "1"}) + try: + wait_for(lambda: self.status(rnfs_dir, api_port, proc), 60, "the daemon to start") + time.sleep(1.5) # past the first waits of every loop + start = time.monotonic() + proc.send_signal(signum) + code = proc.wait(30) + elapsed = time.monotonic() - start + finally: + if proc.poll() is None: + proc.kill() + proc.wait(30) + with open(log_path, "rb") as f: + output = f.read().decode("utf-8", "replace") + self.assertEqual(code, 0, output) + self.assertLess(elapsed, 20, output) + self.assertFalse(os.path.exists(os.path.join(rnfs_dir, cli.TOKEN_FILE)), output) # a clean stop + + @staticmethod + def status(rnfs_dir: str, api_port: int, proc) -> dict | None: + """The daemon's status once its API answers, else None; fails if the daemon exited.""" + if proc.poll() is not None: + raise AssertionError("the daemon exited") + try: + with open(os.path.join(rnfs_dir, cli.TOKEN_FILE), encoding="ascii") as f: + token = f.read().strip() + request = urllib.request.Request(f"http://127.0.0.1:{api_port}/api/status", + headers={"X-Rnfs-Api": token}) + with urllib.request.urlopen(request, timeout=2) as response: + return json.load(response) + except OSError: + return None + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_dag.py b/tests/test_dag.py index bd693f9..8780218 100644 --- a/tests/test_dag.py +++ b/tests/test_dag.py @@ -55,6 +55,14 @@ def test_sizes(self): self.assertEqual(DirectoryNode([Entry("a", CHUNK, 3, 0), Entry("b", CHUNK, 4, 0)]).size, 7) self.assertEqual(IndexNode([IndexChild("a", NODE, 3)]).size, 3) + def test_child_cids_and_names(self): + file = FileNode([Child(CHUNK, 3), Child(NODE, 4)]) + directory = DirectoryNode([Entry("a", CHUNK, 3, 0), Entry("b", NODE, 4, 2)]) + index = IndexNode([IndexChild("a", NODE, 3), IndexChild("m", CHUNK, 4)]) + self.assertEqual((file.child_cids, file.names), ((CHUNK, NODE), ())) + self.assertEqual((directory.child_cids, directory.names), ((CHUNK, NODE), ("a", "b"))) + self.assertEqual((index.child_cids, index.names), ((NODE, CHUNK), ("a", "m"))) + class StrictDecodingTest(unittest.TestCase): """Every way a node can break the encoding rules is rejected.""" diff --git a/tests/test_edit.py b/tests/test_edit.py index 3330317..7414d5b 100644 --- a/tests/test_edit.py +++ b/tests/test_edit.py @@ -90,6 +90,16 @@ def test_errors(self): with self.assertRaises(ValueError): edit.dir_rm(self.root.cid, [], get, put) + def test_check_entry_target(self): + """dir rm's target must name an entry, /: a root alone (with or without slashes) or a non-string + is ValueError, with the one message every caller gives (ND3).""" + cid = str(self.root.cid) + for target in (f"{cid}/docs", f"{cid}/docs/a", f"{cid}/docs/", "nick/docs"): + edit.check_entry_target(target) + for target in (cid, f"{cid}/", f"{cid}//", "", None, 5): + with self.subTest(target), self.assertRaisesRegex(ValueError, "the entry to remove"): + edit.check_entry_target(target) + def test_through_index_nodes(self): """Edits inside a directory split into index nodes, and paths that step through them.""" with mock.patch.multiple(constants, CHUNK_SIZE=4, MAX_CHILDREN=3): diff --git a/tests/test_fake_contract.py b/tests/test_fake_contract.py index 9d2c670..a5fa12a 100644 --- a/tests/test_fake_contract.py +++ b/tests/test_fake_contract.py @@ -1,18 +1,37 @@ -"""The facade fake the API, gateway, CLI and jobs tests run against (tests.test_api.FakeNode) must keep matching -the real facade (rnfs.node.RnfsNode), so a change to RnfsNode can't leave those tests passing against a stale fake.""" +"""The fakes tests run against must keep matching the real classes they stand in for, so a change to the real class +can't leave those tests passing against a stale fake: +- the facade fake the API, gateway, CLI and jobs tests use (tests.test_api.FakeNode) and rnfs.node.RnfsNode; +- the PeerClient stand-ins of the fetcher, publisher and node tests (tests.fakes.client.FakeClient, + tests.test_node.NetClient) and rnfs.network.client.PeerClient; +- the request server's backend fake (tests.test_server.FakeBackend) and rnfs.network.server.Backend, every method + of which it must have: the server calls them all, none through getattr.""" import inspect import unittest +from rnfs.network.client import PeerClient +from rnfs.network.server import Backend from rnfs.node import RnfsNode +from tests.fakes.client import FakeClient from tests.test_api import FakeNode +from tests.test_node import NetClient +from tests.test_server import FakeBackend # FakeNode's own test helpers, not facade methods: building content, and inspecting the calls it recorded FAKE_ONLY = {"record", "called", "put_file", "put_dir"} +# FakeClient's: the CIDs a node was asked for +CLIENT_FAKE_ONLY = {"asked"} +# The PeerClient methods the fetcher and publisher call, which FakeClient (the client of their tests) must have +CLIENT_USED = {"open_links", "rate_estimate", "link_mdu", "close_unless_used", "blocks", "provide"} + + +def public_methods(cls) -> dict: + """The public methods cls defines itself, by name.""" + return {name: value for name, value in vars(cls).items() if not name.startswith("_") and callable(value)} def fake_methods() -> dict: """FakeNode's public methods, by name.""" - return {name: value for name, value in vars(FakeNode).items() if not name.startswith("_") and callable(value)} + return public_methods(FakeNode) def parameters(function) -> list[tuple]: @@ -20,6 +39,26 @@ def parameters(function) -> list[tuple]: return [(p.name, p.kind, p.default) for p in inspect.signature(function).parameters.values()] +class ContractTest(unittest.TestCase): + """Checks shared by every fake's contract.""" + + def check_fake(self, fake, real, helpers: set = frozenset()): + """Every public method of fake but helpers exists on real, with the same parameter names, kinds and + defaults, and no helper is named like one of real's methods (the checks would skip it).""" + methods = public_methods(fake) + for name, method in methods.items(): + if name in helpers: + continue + with self.subTest(fake=fake.__name__, method=name): + actual = getattr(real, name, None) + self.assertTrue(callable(actual), f"{real.__name__} has no method {name}") + self.assertEqual(parameters(method), parameters(actual), + f"{fake.__name__}.{name}{inspect.signature(method)} doesn't match " + f"{real.__name__}.{name}{inspect.signature(actual)}") + self.assertEqual(set(helpers) & set(dir(real)), set()) + self.assertLessEqual(set(helpers), methods.keys()) # no stale entries + + class FakeNodeContractTest(unittest.TestCase): """Every facade method FakeNode has exists on RnfsNode, with the same parameter names, kinds and defaults.""" @@ -44,5 +83,29 @@ def test_helpers_are_not_facade_methods(self): self.assertLessEqual(FAKE_ONLY, fake_methods().keys()) # no stale entries +class FakeClientContractTest(ContractTest): + """FakeClient and NetClient match PeerClient, and FakeClient has every method the fetcher and publisher call.""" + + def test_fake_client_matches_peer_client(self): + self.check_fake(FakeClient, PeerClient, CLIENT_FAKE_ONLY) + + def test_net_client_matches_peer_client(self): + self.check_fake(NetClient, PeerClient) + + def test_fake_client_has_what_the_fetcher_and_publisher_call(self): + self.assertLessEqual(CLIENT_USED, public_methods(FakeClient).keys()) + self.assertLessEqual(CLIENT_USED, public_methods(PeerClient).keys()) # no stale entries + + +class FakeBackendContractTest(ContractTest): + """FakeBackend has every method of the Backend protocol, with the same parameters, and no others.""" + + def test_matches_backend(self): + self.check_fake(FakeBackend, Backend) + + def test_has_every_backend_method(self): + self.assertEqual(public_methods(FakeBackend).keys(), public_methods(Backend).keys()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_feature_output.py b/tests/test_feature_output.py index d3d1b23..a9e468c 100644 --- a/tests/test_feature_output.py +++ b/tests/test_feature_output.py @@ -1,11 +1,17 @@ -"""Feature CLI commands escape what the daemon sends before printing it (follow, keepers, contacts, peers): -whatever answers on the API port mustn't drive the terminal.""" +"""Feature CLI commands escape what the daemon sends before printing it (follow, keepers, contacts, peers, stats, +jobs, deny): whatever answers on the API port mustn't drive the terminal. So do examples/rnfs_share.py and +rnfs.Client's log (FEAT-7; tests/security/test_feat7_example_and_client_escaping.py has the details).""" +import contextlib import io +import tempfile import unittest from types import SimpleNamespace from unittest import mock -from rnfs.features import contacts, follow, keepers, peers +from rnfs import remote +from rnfs.features import contacts, deny, follow, jobs, keepers, peers + +from tests.security.test_feat7_example_and_client_escaping import StandInNode, load_example EVIL = "ab\x1b]0;owned\x07\x1b[2J" @@ -13,16 +19,19 @@ class FeatureCliOutputTest(unittest.TestCase): """Every field of a response is printed escaped.""" - def output(self, run, response, args=None) -> str: - """What run prints when the daemon answers response to any request.""" + def output(self, run, response, args=None, tty=False) -> str: + """What run prints (stdout, then stderr) when the daemon answers response to any request; tty: whether + stderr is a terminal.""" client = mock.Mock() client.get.return_value = response client.post.return_value = response - out = io.StringIO() - with mock.patch("rnfs.cli.client", return_value=client), mock.patch("sys.stdout", out): + out, err = io.StringIO(), io.StringIO() + err.isatty = lambda: tty + with mock.patch("rnfs.cli.client", return_value=client), mock.patch("sys.stdout", out), \ + mock.patch("sys.stderr", err): run(args or SimpleNamespace(name="x", nick="x", id="x", kind=None)) self.assertTrue(out.getvalue()) - return out.getvalue() + return out.getvalue() + err.getvalue() def assertEscaped(self, text: str): """No control character reaches the terminal.""" @@ -47,6 +56,74 @@ def test_peers(self): "link": False, "interface": EVIL, "banned": False, "bad": 0, "hops": None, "sent": 0, "received": 0, "failed": 0} self.assertEscaped("\n".join(peers.format_peers([peer], 0.0))) + # Every field the table prints as it came, numbers too + peer.update(hops=EVIL, failed=EVIL, bad=EVIL) + self.assertEscaped("\n".join(peers.format_peers([peer], 0.0))) + + def test_stats(self): + totals = {"sent": 0, "responses": EVIL, "received": 0, "requests": EVIL, "answered": EVIL, "refused": EVIL, + "failed": EVIL} + stats = {"since": 0, "totals": totals, "refused": {EVIL: EVIL}, "upload_budget": 1, "upload_budget_left": 1, + "interfaces": {EVIL: {"sent": 0, "received": 0}}} + self.assertEscaped("\n".join(peers.format_stats(stats))) + self.assertEscaped(self.output(peers.run_stats, stats, SimpleNamespace(json=False))) + + def test_jobs(self): + job = {"id": EVIL, "kind": EVIL, "state": jobs.RUNNING, "target": EVIL, "output": EVIL, "from": EVIL, + "blocks": EVIL, "error": EVIL, "result": {"cid": EVIL}} + args = SimpleNamespace(id="0123abcd", background=True) + self.assertEscaped(self.output(jobs.run_list, {"jobs": [job, {**job, "state": EVIL}]})) + self.assertEscaped(self.output(jobs.run_show, {**job, "blocks_total": EVIL}, args)) + for state in (jobs.RUNNING, jobs.CANCELLED, EVIL): + self.assertEscaped(self.output(jobs.run_cancel, {**job, "state": state}, args)) + self.assertEscaped(self.output(jobs.run_clear, {"removed": EVIL}, args)) + with mock.patch.object(jobs, "has_jobs", return_value=True): + printed = self.output(lambda args: jobs.run_transfer(args, {}, None), job, args, tty=True) + self.assertIn("follow it with", printed) + self.assertEscaped(printed) + for kind in ("get", "pin", "export"): + self.assertEscaped(self.output(jobs.run_wait, {**job, "kind": kind, "state": jobs.DONE, + "result": {"output": EVIL, "cid": EVIL, "roots": [EVIL], + "warnings": [EVIL]}}, args)) + with self.assertRaises(SystemExit) as failed: + self.output(jobs.run_wait, {**job, "state": jobs.FAILED}, args) + self.assertEscaped(str(failed.exception.code)) + + def test_jobs_progress_line(self): + """The progress line a foreground transfer shows on a terminal, and the line Ctrl+C prints.""" + api = mock.Mock() + api.get.side_effect = [{"state": jobs.RUNNING, "blocks": EVIL}, KeyboardInterrupt] + err = io.StringIO() + err.isatty = lambda: True + with mock.patch("sys.stderr", err), self.assertRaises(SystemExit): + jobs.follow(api, EVIL, cancel_on_interrupt=True) + self.assertIn("finding the target", err.getvalue()) + self.assertEscaped(err.getvalue()) + + def test_deny(self): + args = SimpleNamespace(entries=["x"]) + self.assertEscaped(self.output(deny.run_add, {"added": [EVIL]}, args)) + self.assertEscaped(self.output(deny.run_rm, {"removed": [EVIL]}, args)) + self.assertEscaped(self.output(deny.run_ls, {"entries": [EVIL], "lists": [ + {"source": EVIL, "cid": EVIL, "entries": EVIL, "state": EVIL}]}, args)) + + +class ExampleAndClientOutputTest(FeatureCliOutputTest.__base__): + """FEAT-7: examples/rnfs_share.py prints names escaped, and rnfs.Client logs the daemon's messages escaped.""" + + assertEscaped = FeatureCliOutputTest.assertEscaped + + def test_example(self): + out = io.StringIO() + with tempfile.TemporaryDirectory() as folder, contextlib.redirect_stdout(out): + load_example().share(StandInNode(), folder) + self.assertIn("evil\\x1b", out.getvalue()) + self.assertEscaped(out.getvalue()) + + def test_client_log(self): + with self.assertLogs("rnfs", "ERROR") as logged: + remote.Client._report(None, EVIL) + self.assertEscaped("\n".join(logged.output)) if __name__ == "__main__": diff --git a/tests/test_features.py b/tests/test_features.py index fb82d29..8218293 100644 --- a/tests/test_features.py +++ b/tests/test_features.py @@ -1,5 +1,9 @@ """Tests for the feature framework (rnfs.features): loading, node wiring, API routes and CLI commands.""" +import os import sys +import tempfile +import threading +import time import types import unittest from unittest import mock @@ -7,6 +11,11 @@ from rnfs import features +def no_routes(cls): + """No API routes: a feature class's api_routes, so two test features don't define the same.""" + return {} + + class DummyFeature(features.Feature): """A feature recording what the framework calls.""" @@ -41,6 +50,29 @@ def cli_commands(cls, commands, command): sub.add_argument("--who") +class DeadlineFeature(DummyFeature): + """A feature whose stop takes the node's deadline, with one thread that runs until stopped.""" + + name = "deadline" + api_routes = classmethod(no_routes) # DummyFeature's are its own: the node refuses a route defined twice + + def __init__(self, node): + """Not stopped yet.""" + super().__init__(node) + self.deadlines = [] + self.ended = threading.Event() + + def start(self): + """Start the thread.""" + self._spawn(self.ended.wait, "test-deadline-feature") + + def stop(self, deadline=None): + """Record the deadline, end the thread and wait for it.""" + self.deadlines.append(deadline) + self.ended.set() + self._join(deadline) + + def install_dummy(): """Make rnfs.features load only DummyFeature, from a fake module.""" module = types.ModuleType("rnfs_test_dummy_feature") @@ -88,3 +120,372 @@ def test_api_route_is_dispatched_with_the_token(self): conn.close() self.assertEqual(response.status, expected, body) self.assertEqual(body, {"hello": "me"}) + + +def install(*classes): + """Make rnfs.features load classes, in that order, each from a fake module of its own.""" + modules = {} + for i, cls in enumerate(classes): + module = types.ModuleType(f"rnfs_test_feature_{i}") + module.FEATURE = cls + modules[module.__name__] = module + return (mock.patch.dict(sys.modules, modules), mock.patch.object(features, "FEATURE_MODULES", list(modules))) + + +def feature_class(name, base=DummyFeature, **attributes): + """A subclass of base called name, without API routes unless given, with the class attributes given.""" + return type(f"{name.title()}Feature", (base,), {"name": name, "api_routes": classmethod(no_routes), + **attributes}) + + +class FeatureDependencyTest(unittest.TestCase): + """requires and optional, checked when the node loads its features; dependency() (audit FT4).""" + + def classes(self, *classes): + """features.checked_classes() with only classes installed.""" + patchers = install(*classes) + for patcher in patchers: + patcher.start() + try: + return features.checked_classes() + finally: + for patcher in reversed(patchers): + patcher.stop() + + def test_the_shipped_features_load(self): + classes = features.checked_classes() + names = [cls.name for cls in classes] + self.assertEqual(len(set(names)), len(names)) + for cls in classes: + for name in (*cls.requires, *cls.optional): + self.assertIn(name, names, cls.name) + + def test_requires_a_feature_loaded_before(self): + base, user = feature_class("base"), feature_class("user", requires=("base",)) + self.assertEqual(self.classes(base, user), [base, user]) + for order in ((user,), (user, base)): + with self.assertRaisesRegex(features.FeatureError, "feature user: it requires feature base"): + self.classes(*order) + # An optional one may be missing, or come later + optional = feature_class("user", optional=("base", "absent")) + self.assertEqual(self.classes(optional, base), [optional, base]) + + def test_names_and_declarations_are_checked(self): + cases = ((("dummy", "dummy"), "feature dummy: another feature has that name"), (("",), "it has no name")) + for names, message in cases: + with self.subTest(names), self.assertRaisesRegex(features.FeatureError, message): + self.classes(*(feature_class(name) for name in names)) + for kind, value in (("requires", "base"), ("optional", ["base"]), ("optional", (1,))): + with self.subTest(kind=kind, value=value), \ + self.assertRaisesRegex(features.FeatureError, f"feature user: {kind} must be a tuple"): + self.classes(feature_class("user", **{kind: value})) + + def test_node_refuses_to_load_a_missing_requirement(self): + """Before any feature is made: none is half made, and the state is released (a retry can open it).""" + from rnfs.config import Config + from rnfs.node import RnfsNode + from tests.test_names import FakeNetwork + from tests.test_node import N1, NetClient + made = [] + + def init(self, node): + """Record that it was made.""" + DummyFeature.__init__(self, node) + made.append(self) + + user = feature_class("user", requires=("base",), __init__=init) + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + config = Config(rnfs_dir=os.path.join(tmp.name, "n"), slow="no") + patchers = install(user, feature_class("base")) + for patcher in patchers: + patcher.start() + self.addCleanup(patcher.stop) + with self.assertRaisesRegex(features.FeatureError, "feature user: it requires feature base"): + RnfsNode.offline(config, N1, NetClient(), FakeNetwork()) + self.assertEqual(made, []) + features.FEATURE_MODULES.reverse() # base before user: it loads + node = RnfsNode.offline(config, N1, NetClient(), FakeNetwork()) + self.addCleanup(node.stop) + self.assertEqual(made, [node.feature("user")]) + self.assertIs(node.feature("user").dependency("base"), node.feature("base")) + + def test_dependency_refuses_an_undeclared_feature(self): + node = types.SimpleNamespace(features={}) + user = feature_class("user", requires=("base",), optional=("extra",))(node) + base = feature_class("base")(node) + node.features.update(base=base, user=user) + self.assertIs(user.dependency("base"), base) + self.assertIsNone(user.dependency("extra")) # optional, not loaded + with self.assertRaisesRegex(features.FeatureError, "feature user: it uses feature dummy without declaring"): + user.dependency("dummy") + # A stand-in node without features (some tests use one) + self.assertIsNone(feature_class("user", optional=("base",))(types.SimpleNamespace()).dependency("base")) + + +class FeatureLifecycleTest(unittest.TestCase): + """Feature.stop(deadline): the base's _spawn and _join, and the node passing its stop budget (audit XC5).""" + + def test_join_waits_until_the_deadline_at_most(self): + feature = features.Feature(node=None) + release = threading.Event() + self.addCleanup(release.set) + thread = feature._spawn(release.wait, "test-feature-thread") + started = time.monotonic() + feature.stop(time.monotonic() + 0.2) # the base's stop only waits + self.assertLess(time.monotonic() - started, 2) + self.assertTrue(thread.is_alive()) + self.assertFalse(feature._join(time.monotonic())) + release.set() + self.assertTrue(feature._join()) + self.assertFalse(thread.is_alive()) + + def test_spawn_forgets_ended_threads(self): + feature = features.Feature(node=None) + first = feature._spawn(lambda: None, "test-feature-thread") + first.join(5) + release = threading.Event() + self.addCleanup(release.set) + second = feature._spawn(release.wait, "test-feature-thread") + self.assertEqual(feature._spawned, [second]) + + def test_stop_feature_is_call_compatible(self): + """A stop without a deadline parameter (written before 0.1.6) is called as before; a new one gets it.""" + legacy, current = DummyFeature(node=None), DeadlineFeature(node=None) + features.stop_feature(legacy, 123.0) + features.stop_feature(current, 123.0) + self.assertEqual(legacy.calls, ["init", "stop"]) + self.assertEqual(current.deadlines, [123.0]) + current.stop() # a host program calling stop() itself + self.assertEqual(current.deadlines, [123.0, None]) + + def test_node_passes_its_stop_budget(self): + from rnfs.config import Config + from rnfs.node import STOP_TIMEOUT, RnfsNode + from tests.test_names import FakeNetwork + from tests.test_node import N1, NetClient + legacy_module, current_module = types.ModuleType("rnfs_test_legacy"), types.ModuleType("rnfs_test_deadline") + legacy_module.FEATURE, current_module.FEATURE = DummyFeature, DeadlineFeature + modules = {legacy_module.__name__: legacy_module, current_module.__name__: current_module} + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + with mock.patch.dict(sys.modules, modules), \ + mock.patch.object(features, "FEATURE_MODULES", list(modules)): + node = RnfsNode.offline(Config(rnfs_dir=os.path.join(tmp.name, "n"), slow="no"), N1, NetClient(), + FakeNetwork()) + self.addCleanup(node.stop) + current = node.feature("deadline") + current.start() + before = time.monotonic() + with mock.patch("rnfs.node.log"): + node.stop() + after = time.monotonic() + self.assertEqual(node.feature("dummy").calls, ["init", "stop"]) + self.assertEqual(len(current.deadlines), 1) + self.assertTrue(before + STOP_TIMEOUT <= current.deadlines[0] <= after + STOP_TIMEOUT, current.deadlines) + self.assertEqual([t.is_alive() for t in current._spawned], [False]) + + def test_follow_deny_and_keepers_wait_for_their_threads(self): + """stop() returns only once the feature's own thread has ended (or the deadline passed), so the node + doesn't close the state database under it; keepers saves after its thread ends.""" + from rnfs.config import Config + from rnfs.node import RnfsNode + from tests.test_names import FakeNetwork + from tests.test_node import N1, NetClient + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + node = RnfsNode.offline(Config(rnfs_dir=os.path.join(tmp.name, "n"), slow="no"), N1, NetClient(), + FakeNetwork()) + self.addCleanup(node.stop) + for name in ("follow", "deny", "keepers"): + with self.subTest(name): + feature = node.feature(name) + order = [] + + def loop(feature=feature, order=order): + """The feature's loop: still busy a moment after stop is asked.""" + feature._stop.wait() + time.sleep(0.3) + order.append("loop ended") + + with mock.patch.object(feature, "_loop", loop), \ + mock.patch.object(feature, "save", lambda order=order: order.append("saved")): + feature.start() + feature.stop(time.monotonic() + 5) + self.assertEqual(order[0], "loop ended") + self.assertFalse(any(thread.is_alive() for thread in feature._spawned)) + + +def handler(*_args): + """A request or API handler that is never called.""" + return False + + +def routes(table): + """An api_routes classmethod answering table.""" + return classmethod(lambda cls: dict(table)) + + +def paths(table): + """A requests method answering table.""" + return lambda self: dict(table) + + +class OfflineNodes: + """Offline nodes loading only the features a test installs.""" + + def install(self, *classes): + """Load only classes, in that order, until the test ends.""" + for patcher in install(*classes): + patcher.start() + self.addCleanup(patcher.stop) + + def config(self): + """A config for a node in a new temporary directory.""" + from rnfs.config import Config + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + return Config(rnfs_dir=os.path.join(tmp.name, "n"), slow="no") + + def node(self, config=None): + """An offline node (stopped when the test ends).""" + from rnfs.node import RnfsNode + from tests.test_names import FakeNetwork + from tests.test_node import N1, NetClient + node = RnfsNode.offline(config or self.config(), N1, NetClient(), FakeNetwork()) + self.addCleanup(node.stop) + return node + + +class FeatureRouteTest(OfflineNodes, unittest.TestCase): + """API routes and request paths are checked when the node loads its features, with one error style: a route + or path that is malformed, that the core defines, or that two features define stops the node (audit FT5).""" + + def assertRefused(self, classes, message): + """Loading classes raises FeatureError with message, whether alone or by a node (which then releases + its state: a node with the same directory can be made after).""" + self.install(*classes) + with self.assertRaisesRegex(features.FeatureError, message): + features.api_routes() + features.request_handlers(cls(None) for cls in features.checked_classes()) + config = self.config() + with self.assertRaisesRegex(features.FeatureError, message): + self.node(config) + features.FEATURE_MODULES.clear() + self.node(config) + + def test_the_shipped_routes_and_paths_load(self): + from rnfs.api import ApiHandler + from rnfs.features import keepers + self.assertFalse(set(features.api_routes()) & set(ApiHandler.ROUTES)) + self.assertEqual(set(self.node().backend.feature_requests()), {keepers.NAME_PATH}) + + def test_a_core_api_route_is_refused(self): + self.assertRefused([feature_class("bad", api_routes=routes({("GET", "/api/status"): handler}))], + r"^feature bad: API route GET /api/status is already defined by rnfs$") + + def test_an_api_route_of_two_features_is_refused(self): + self.assertRefused([DummyFeature, feature_class("other", api_routes=routes({("GET", "/api/dummy"): handler}))], + r"^feature other: API route GET /api/dummy is already defined by feature dummy$") + + def test_a_malformed_api_route_is_refused(self): + for route, route_handler in ((("PUT", "/api/x"), handler), (("GET", "/x"), handler), + (("GET", "/api/x?y=1"), handler), (("GET", "/api/"), handler), + ("GET /api/x", handler), (("GET", "/api/x"), None)): + with self.subTest(route=route): + self.assertRefused([feature_class("bad", api_routes=routes({route: route_handler}))], + r"^feature bad: API route .* must be \(GET or POST, a path under /api/\)") + + def test_a_core_request_path_is_refused(self): + for path in features.CORE_REQUEST_PATHS: + with self.subTest(path): + self.assertRefused([feature_class("bad", requests=paths({path: handler}))], + rf"^feature bad: request path {path} is already defined by rnfs$") + + def test_a_request_path_of_two_features_is_refused(self): + self.assertRefused([feature_class("one", requests=paths({"/extra": handler})), + feature_class("two", requests=paths({"/extra": handler}))], + r"^feature two: request path /extra is already defined by feature one$") + + def test_a_malformed_request_path_is_refused(self): + for path, path_handler in (("extra", handler), ("/", handler), ("/a b", handler), (b"/extra", handler), + ("/extra", None)): + with self.subTest(path=path): + self.assertRefused([feature_class("bad", requests=paths({path: path_handler}))], + r"^feature bad: request path .* must be \"/\" and a name, with a handler$") + + +class Recorder(DummyFeature): + """A feature recording (name, call) in the shared list events, for the order across features.""" + + events: list = [] + + def start(self): + """Record start.""" + self.events.append((self.name, "start")) + + def stop(self, deadline=None): + """Record stop.""" + self.events.append((self.name, "stop")) + + def save(self): + """Record save.""" + self.events.append((self.name, "save")) + + def close(self): + """Record close.""" + self.events.append((self.name, "close")) + + +class FeatureNodeLifecycleTest(OfflineNodes, unittest.TestCase): + """What the node calls on its features, and in what order (audit FT5).""" + + def setUp(self): + """A fresh event list, and two features recording into it.""" + self.events = [] + self.first = feature_class("first", base=Recorder, events=self.events) + self.second = feature_class("second", base=Recorder, events=self.events) + + def test_requests_reach_the_request_server(self): + """The request server registers what the node's backend hands it: each feature's requests().""" + self.install(feature_class("dummy", requests=paths({"/dummy": handler}))) + self.assertEqual(self.node().backend.feature_requests(), {"/dummy": handler}) + + def test_start_stop_save_close_order(self): + """Started in order; stopped in reverse, before the node's own saves; saved in order, closed in reverse.""" + self.install(self.first, self.second) + node = self.node() + node._start_features() + self.assertEqual(self.events, [("first", "start"), ("second", "start")]) + del self.events[:] + node.stop() + self.assertEqual(self.events, [("second", "stop"), ("first", "stop"), ("first", "save"), + ("second", "save"), ("second", "close"), ("first", "close")]) + + def test_a_failing_start_is_isolated(self): + """One feature's start() raising is logged; the others still start, and the node runs on.""" + def broken(feature): + """Fail to start.""" + raise RuntimeError("no start") + + self.install(feature_class("broken", base=Recorder, events=self.events, start=broken), self.second) + node = self.node() + with mock.patch("rnfs.node.log") as log: + node._start_features() + self.assertEqual(self.events, [("second", "start")]) + self.assertIn("feature broken failed to start: RuntimeError('no start')", log.call_args.args[0]) + + def test_a_failing_constructor_stops_the_node(self): + """A feature's constructor raising stops the node from being made, and those made before it are closed, + so the state is released (a node with the same directory can be made after).""" + def broken(feature, node): + """Fail to be made.""" + raise RuntimeError("no init") + + self.install(self.first, feature_class("broken", __init__=broken), self.second) + config = self.config() + with self.assertRaisesRegex(RuntimeError, "no init"): + self.node(config) + self.assertEqual(self.events, [("first", "close")]) + features.FEATURE_MODULES.clear() + self.node(config) diff --git a/tests/test_fetcher.py b/tests/test_fetcher.py index c7fa933..44b4f85 100644 --- a/tests/test_fetcher.py +++ b/tests/test_fetcher.py @@ -9,8 +9,6 @@ from collections import deque from unittest import mock -import RNS - from rnfs.config import Config from rnfs.constants import CHUNK_SIZE, MAX_BLOCK_SIZE, MAX_REQUEST_SIZE, MAX_RESPONSE_SIZE, RESPONSE_OVERHEAD from rnfs.content import builder, dag, walker @@ -22,76 +20,16 @@ from rnfs.errors import InvalidDag, NotFound, StorageFull, TooLarge from rnfs import fetcher as fetcher_module from rnfs.fetcher import Fetcher -from rnfs.network import protocol, server +from rnfs.network import client as client_module, links, protocol, server from rnfs.network.client import BUSY, FAILED, OK, STALLED, BlocksResult +from tests.fakes.client import BATCH, FakeClient +from tests.fakes.log import quiet_log from tests.support import failing_commits, state_execute, state_rows +import tests.test_links as link_fakes # a module, so its test cases don't run twice OWN = bytes(16) P1, P2, P3, P4 = (bytes([i]) * 16 for i in (1, 2, 3, 4)) -BATCH = 4 # CIDs the fake client sends per request, like a packet limit - -_saved_loglevel = RNS.loglevel - - -def setUpModule(): - """Silence the fetcher's log.""" - RNS.loglevel = RNS.LOG_CRITICAL - - -def tearDownModule(): - """Restore RNS's loglevel.""" - RNS.loglevel = _saved_loglevel - - -class FakeClient: - """Stands in for PeerClient: each node holds a dict of blocks, or behaves as a test function says.""" - - def __init__(self): - """No nodes, no open links.""" - self.holdings: dict[bytes, dict[Cid, bytes]] = {} - self.behaviour: dict[bytes, object] = {} # node -> f(cids) -> BlocksResult, or None for the default - self.links: list[bytes] = [] - self.rate: float | None = None - self.requests: list[tuple[bytes, list[Cid]]] = [] - self.closed: list[bytes] = [] - self.lock = threading.Lock() - - def open_links(self) -> list[bytes]: - """Nodes with an open link.""" - return list(self.links) - - def rate_estimate(self, node_id: bytes): - """The same estimate for every link.""" - return self.rate - - def close(self, node_id: bytes): - """Close the link to node_id (noted).""" - with self.lock: - self.closed.append(node_id) - - def blocks(self, node_id: bytes, cids: list[Cid], on_progress=None) -> BlocksResult: - """Serve what node_id holds, BATCH CIDs per request.""" - with self.lock: - self.requests.append((node_id, list(cids))) - self.on_progress = on_progress - behaviour = self.behaviour.get(node_id) - if behaviour is not None: - result = behaviour(cids) - if result is not None: - return result - held = self.holdings.get(node_id, {}) - result = BlocksResult(OK) - for cid in cids[:BATCH]: - if cid in held: - result.blocks[cid] = held[cid] - else: - result.missing.append(cid) - result.unsent = list(cids[BATCH:]) - return result - - def asked(self, node_id: bytes) -> list[Cid]: - """Every CID sent to node_id.""" - return [cid for node, cids in self.requests if node == node_id for cid in cids[:BATCH]] +setUpModule, tearDownModule = quiet_log() class FakeLookup: @@ -342,7 +280,7 @@ def result(self, status: str, reason: str = "", sent_at: float | None = None, ra result = BlocksResult(status, reason=reason) if status == OK: result.blocks[cid] = self.network[cid] - self.fetch._handle(P1, [cid], result, False, {}, {}, set(), sent_at, rate) + self.fetch._handle(P1, [cid], result, False, {}, {}, set(), self.fetch.budget, sent_at, rate) def test_slow_link_waits_before_asking_again(self): self.result(OK, sent_at=self.now - 40, rate=50_000) # 40 s for that request @@ -350,6 +288,16 @@ def test_slow_link_waits_before_asking_again(self): self.assertEqual(self.provider.busy_until, self.now + 40) self.assertGreaterEqual(self.fetch._progress_at, self.now + 40) # waiting isn't counted as hanging + def test_response_progress_doesnt_cut_a_pacing_wait_short(self): + self.result(OK, sent_at=self.now - 40, rate=50_000) + paced = self.fetch._progress_at + self.now += 1 + self.client.behaviour[P1] = lambda cids: self.client.on_progress(0.5) # then served as usual + self.fetch._send(P1, [self.built.cid]) + self.fetch._events.get(timeout=5) + # The request thread only notes when it heard progress; the deadline counts from the later of the two + self.assertEqual((self.fetch._progress_at, self.fetch._heard_at), (paced, self.now)) + def test_fast_link_isnt_paced(self): self.result(OK, sent_at=self.now - 40, rate=200_000) self.assertEqual(self.provider.busy_until, 0) @@ -394,8 +342,9 @@ def answer(cids): return BlocksResult(OK, blocks={cid: self.network[cid]}, elapsed=elapsed) self.client.behaviour[P1] = answer self.fetch._send(P1, [cid]) - _, node_id, cids, result, sent_at, rate = self.fetch._events.get(timeout=5) - self.fetch._handle(node_id, cids, result, False, {}, {}, set(), sent_at, rate) + event = self.fetch._events.get(timeout=5) + self.fetch._handle(event.node_id, event.cids, event.result, False, {}, {}, set(), self.fetch.budget, + event.sent_at, event.rate) def test_pacing_counts_from_when_the_request_was_sent(self): self.sent(40) @@ -415,7 +364,7 @@ def old_take_batch(fetch, pending, node_id, wanted, mdu): reserved that it has now: the reference the single count over pending must match.""" budget = fetcher_module.rates.requester_max_bytes(fetch.client.rate_estimate(node_id), fetch.config) carried = min(budget, MAX_RESPONSE_SIZE - RESPONSE_OVERHEAD) - room = fetch._room(wanted) + room = fetch.budget.room((cid for cids, _ in fetch._in_flight.values() for cid in cids), wanted) batch, total = [], 0 while pending and len(batch) < fetcher_module.MAX_BATCH: cid = pending[0] @@ -426,8 +375,8 @@ def old_take_batch(fetch, pending, node_id, wanted, mdu): if batch and protocol.blocks_batch_size(budget, batch + [cid], mdu) <= len(batch): break if room is not None: - room = fetch._reserve(cid, wanted, room) - if not fetch._fits(room) and (batch or fetch._in_flight): + room = fetch.budget.reserve(cid, wanted, room) + if not fetch.budget.fits(room) and (batch or fetch._in_flight): break batch.append(pending.popleft()) total += guess @@ -448,7 +397,7 @@ def setUp(self): def take(self) -> list[Cid]: """The batch for P1 from every chunk.""" - return self.fetch._take_batch(deque(self.wanted), P1, self.wanted) + return self.fetch._take_batch(deque(self.wanted), P1, self.wanted, self.fetch.budget) def test_capped_at_what_a_response_carries(self): # At 1 Gbit/s max_bytes asks for ~7.5 GB, but a response carries at most MAX_RESPONSE_SIZE less overhead @@ -490,10 +439,93 @@ def test_one_count_matches_packing_each_prefix(self): with self.subTest(trial=trial): old, new = deque(wanted), deque(wanted) expected = old_take_batch(self.fetch, old, P1, wanted, mdu or MAX_REQUEST_SIZE) - self.assertEqual(self.fetch._take_batch(new, P1, wanted), expected) + self.assertEqual(self.fetch._take_batch(new, P1, wanted, self.fetch.budget), expected) self.assertEqual(list(new), list(old)) +class RequeueTest(FetcherTest): + """What a result puts back in the pending queue, and where, with a large file's chunks pending (A3).""" + + def test_requeue_keeps_the_walk_order_with_a_large_pending_queue(self): + fetch = self.fetcher.open(self.file(b"root").cid).__enter__() + self.addCleanup(fetch.__exit__, None, None, None) + chunks = [i.to_bytes(8, "big") for i in range(20_000)] + expects = [walker.Expect(Cid.for_block(RAW, data), len(data), walker.FILE_TYPES, 2) for data in chunks] + blocks = {expect.cid: data for expect, data in zip(expects, chunks)} + wanted = {expect.cid: [expect] for expect in expects} + walk = list(wanted) + outstanding = set(wanted) + pending = fetcher_module._Pending(wanted) + for node_id in (P1, P2): + fetch._new_provider(node_id) + first = [pending.popleft() for _ in range(32)] + second = [pending.popleft() for _ in range(32)] + in_flight = {P1: (first, False), P2: (second, False)} + + def result(node_id: bytes, cids: list[Cid], outcome: BlocksResult): + """Handle node_id's result for cids.""" + fetch._handle_event(fetcher_module.ResultEvent(node_id, cids, outcome, None, None), pending, in_flight, + outstanding, wanted, {}, fetch.budget) + + # P2's budget cut its response after 10 blocks: the rest go back to the front + result(P2, second, BlocksResult(OK, blocks={cid: blocks[cid] for cid in second[:10]}, unsent=second[10:])) + self.assertEqual(list(pending), walk[42:]) + # A probe asks for first[0] (still in flight with P1) and the next pending CID (still pending) + in_flight[P3] = ([first[0], pending[0]], True) + result(P1, first, BlocksResult(BUSY)) # first[0] stays out: the probe still has it + self.assertEqual(list(pending), first[1:] + walk[42:]) + result(P3, [first[0], walk[42]], BlocksResult(FAILED, unsent=[first[0], walk[42]])) + # Back in the walk's order, each CID once, and only the ones not stored + self.assertEqual(list(pending), walk[:32] + walk[42:]) + self.assertEqual(len(pending), len(walk) - 10) + self.assertNotIn(second[0], pending) + self.assertIn(walk[-1], pending) + drained = [pending.popleft() for _ in range(len(pending))] + self.assertEqual(drained, walk[:32] + walk[42:]) + self.assertFalse(any(cid in pending for cid in walk)) + + +class DagBudgetTest(unittest.TestCase): + """_DagBudget, the pure bookkeeping of a fetch's DAG byte limits, and the budgets of its read_range calls.""" + + def setUp(self): + """A fetch's own budget, with no limit yet, a DAG node and a raw chunk.""" + self.budget = fetcher_module._DagBudget(Cid.for_block(RAW, b"root")) + self.node = Cid.for_block(DAG, b"node") + self.chunk = Cid.for_block(RAW, b"chunk") + + def test_nothing_is_counted_without_a_limit(self): + self.assertIsNone(self.budget.limit()) + self.assertIsNone(self.budget.room([self.node], {})) + self.budget.count(self.node, 1000) + self.assertEqual(self.budget.used, 0) + + def test_a_read_range_budget_counts_against_the_fetchs_too(self): + self.budget.max_dag_bytes = 10_000 + first = self.budget.for_range(3000, 1) + first.count(self.node, 2000) + first.count(self.node, 2000) # once per call + first.count(self.chunk, 5000) # a raw chunk counts against the fetch's limits only + self.assertEqual((first.range_used, self.budget.used), (2000, 7000)) + second = self.budget.for_range(3000, 1) + second.count(self.node, 2000) # each call counts the nodes it meets; the fetch, each block once + self.assertEqual((second.range_used, self.budget.used), (2000, 7000)) + with self.assertRaises(TooLarge): + second.count(Cid.for_block(DAG, b"other"), 1001) + with self.assertRaises(TooLarge): + self.budget.count(Cid.for_block(RAW, b"large"), 3001) + + def test_room_reserves_what_is_in_flight(self): + self.budget.max_dag_bytes = 100 * MAX_BLOCK_SIZE + budget = self.budget.for_range(10 * MAX_BLOCK_SIZE, 1) + wanted = {self.chunk: [walker.Expect(self.chunk, 10, walker.FILE_TYPES, 2)]} + # A node not received may be MAX_BLOCK_SIZE; a raw chunk is its declared size, and nothing to the range + room = budget.room([self.node, self.chunk], wanted) + self.assertEqual(room, (None, 99 * MAX_BLOCK_SIZE - 10, 9 * MAX_BLOCK_SIZE)) + self.assertTrue(budget.fits(room)) + self.assertFalse(budget.fits((None, -1, 0))) + + class ValidityTest(FetcherTest): """Reader checks, max_download_size, space.""" @@ -681,6 +713,269 @@ def test_blocks_already_stored_are_not_fetched(self): self.assertEqual(self.client.requests, []) +class ReadAheadTest(FetcherTest): + """read_range asks for the next window while the reader takes this one (A1); a reader closing early closes + the links still bringing it, and never cancels a response (NET-1).""" + + def setUp(self): + """A file of four windows of WINDOW (6) chunks, held by three providers. Each request carries one chunk (no + rate estimate: CHUNK_SIZE), and one for a chunk of window k + 1 is answered only once the reader has had + the first bytes of window k (self.gates[k] set), so what is in flight when it gets them is seen.""" + super().setUp() + patcher = mock.patch.object(fetcher_module, "WINDOW", 6) + patcher.start() + self.addCleanup(patcher.stop) + self.data = os.urandom(4 * 6 * CHUNK_SIZE) + self.built = self.file(self.data) + self.chunks = dag.decode(self.network[self.built.cid]).children + self.window_of = {child.cid: i // 6 for i, child in enumerate(self.chunks)} + self.gates = [threading.Event() for _ in range(4)] + for gate in self.gates: + self.addCleanup(gate.set) # so no request thread outlives the test + + def gated(cids): + """Serve as usual once the reader has had the previous window's first bytes.""" + window = min(self.window_of.get(cid, 0) for cid in cids) + if window: + self.gates[window - 1].wait(5) + return None + for node_id in (P1, P2, P3): + self.hold(node_id) + self.client.behaviour[node_id] = gated + self.runs.append([P1, P2, P3]) + + def test_every_provider_is_busy_across_window_boundaries(self): + busy = [] + pieces = [] + with self.fetcher.open(self.built.cid) as fetch: + for i, piece in enumerate(fetch.read_range(fetch.resolve())): + if i % 6 == 0: + # The reader has this window's first bytes: the next window's requests are already out + busy.append(sorted(node for node, (_, probe) in fetch._in_flight.items() if not probe)) + self.gates[i // 6].set() + pieces.append(piece) + self.assertEqual(fetch._expects, {}) # nothing left over once read + self.assertEqual(b"".join(pieces), self.data) + self.assertEqual(busy, [[P1, P2, P3]] * 3 + [[]]) # the last window has none after it + asked = [cid for node in (P1, P2, P3) for cid in self.client.asked(node)] + self.assertEqual(sorted(asked), sorted([self.built.cid] + [child.cid for child in self.chunks])) + + def test_early_closing_reader_closes_its_links(self): + def read(): + """What node.read does: the fetch ends when the reader closes its iterator.""" + with self.fetcher.open(self.built.cid) as fetch: + self.fetch = fetch + yield from fetch.read_range(fetch.resolve()) + + reader = read() + self.assertEqual(next(reader), self.data[:CHUNK_SIZE]) + in_flight = sorted(self.fetch._in_flight) + self.assertEqual(in_flight, [P1, P2, P3]) + reader.close() + self.assertEqual(sorted(self.client.closed), in_flight) + # It fetched at most the window after the one it read + asked = [cid for node in (P1, P2, P3) for cid in self.client.asked(node)] + self.assertLessEqual(len(asked), 1 + 2 * 6) + self.assertNotIn(self.chunks[2 * 6].cid, asked) + + def test_a_reader_left_unread_closes_its_links_when_the_fetch_ends(self): + with self.fetcher.open(self.built.cid) as fetch: + reader = fetch.read_range(fetch.resolve()) + next(reader) + in_flight = sorted(fetch._in_flight) + self.assertEqual(sorted(self.client.closed), in_flight) + self.assertTrue(in_flight) + reader.close() + + def test_a_block_arriving_after_its_call_is_stored(self): + first, second, third = walker.expand(walker.root_expect(self.built.cid), self.network[self.built.cid])[:3] + gate = threading.Event() + self.addCleanup(gate.set) + + def held(cids): + """Serve second only once the gate opens.""" + if second.cid in cids: + gate.wait(5) + return None + for node_id in (P1, P2, P3): + self.client.behaviour[node_id] = held + with self.fetcher.open(self.built.cid) as fetch: + fetch.resolve() + fetch.ensure([first], ahead=[second]) # returns with second still in flight + self.assertFalse(self.storage.has(second.cid)) + gate.set() + deadline = time.monotonic() + 5 + while fetch._events.empty() and time.monotonic() < deadline: + time.sleep(0.01) + fetch.ensure([third]) # second's result arrives during a call that doesn't want it + self.assertTrue(self.storage.has(second.cid)) + asked = [cid for node in (P1, P2, P3) for cid in self.client.asked(node)] + self.assertEqual(asked.count(second.cid), 1) + self.assertEqual(self.client.closed, []) + + +class ReadAheadLinkTest(unittest.TestCase): + """An early-closing reader over the real PeerClient and LinkPool (RNS faked as in test_links): its link is torn + down, which stops the response, unless another fetch has a request on it; it never cancels the response's + Resource (a responder would cool it down).""" + + def setUp(self): + """A fetcher using a PeerClient whose links are FakeLinks; one provider serving a file of three windows of + two chunks, which holds back its answers past the first window until self.release is set (5 s at most), + and another file of as many chunks, which it serves at once.""" + patcher = mock.patch.object(links, "RNS", link_fakes.fake_rns()) + patcher.start() + self.addCleanup(patcher.stop) + window = mock.patch.object(fetcher_module, "WINDOW", 2) + window.start() + self.addCleanup(window.stop) + link_fakes.FakeLink.created = [] + link_fakes.FakeLink.establish_delay = 0.0 + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + config = Config(rnfs_dir=tmp.name, slow="no") + storage = Storage(os.path.join(tmp.name, "blocks"), os.path.join(tmp.name, "state"), 10**9) + self.addCleanup(storage.close) + self.pool = links.LinkPool(object(), config) + self.addCleanup(self.pool.close_all) + network: dict[Cid, bytes] = {} + self.data = os.urandom(6 * CHUNK_SIZE) + self.root = builder.build_bytes(self.data, network.__setitem__).cid + self.other_data = os.urandom(6 * CHUNK_SIZE) + self.other = builder.build_bytes(self.other_data, network.__setitem__).cid + held = {child.cid for child in dag.decode(network[self.root]).children[2:]} + self.release = threading.Event() + self.addCleanup(self.release.set) + + def respond(link, path, data): + """Serve at once, but the first file's chunks past its first window only once released, as a slow + response would arrive.""" + cids = [Cid.from_bytes(binary) for binary in data[1]] + if held.intersection(cids): + self.release.wait(5) + return [network[cid] for cid in cids] + link_fakes.FakeLink.responder = staticmethod(respond) + self.fetcher = Fetcher(storage, client_module.PeerClient(self.pool, config), config, OWN, + RoutingTable(OWN, config), lambda key: FakeLookup([[P1]])) + self.addCleanup(self.fetcher.close) + self.fetches = [] + + def read(self, root: Cid): + """What node.read does: the fetch ends when the reader closes its iterator.""" + with self.fetcher.open(root) as fetch: + self.fetches.append(fetch) + yield from fetch.read_range(fetch.resolve()) + + def test_closing_the_reader_tears_the_link_down(self): + reader = self.read(self.root) + self.assertEqual(next(reader), self.data[:CHUNK_SIZE]) + link = link_fakes.FakeLink.created[0] + response = mock.Mock() # the next window's response Resource, arriving + link.incoming_resources = [response] + self.assertEqual(link.status, link_fakes.FakeLink.ACTIVE) + reader.close() + self.assertEqual(link.status, link_fakes.FakeLink.CLOSED) + response.cancel.assert_not_called() + + def test_closing_a_reader_leaves_a_link_another_fetch_uses(self): + other = self.read(self.other) + self.assertEqual(next(other), self.other_data[:CHUNK_SIZE]) # its provider found, its link open + reader = self.read(self.root) + self.assertEqual(next(reader), self.data[:CHUNK_SIZE]) # its next window now holds the link's /blocks slot + rest = [] + thread = threading.Thread(target=lambda: rest.append(b"".join(other)), daemon=True) + thread.start() + peer = self.pool.shared_peer(P1) + deadline = time.monotonic() + 5 + while len(peer.users) < 2 and time.monotonic() < deadline: # the other read's request waits for the slot + time.sleep(0.01) + self.assertEqual(len(peer.users), 2) + link = link_fakes.FakeLink.created[0] + response = mock.Mock() + link.incoming_resources = [response] + reader.close() # gives up mid-window: its request is left to finish, the other's isn't failed + self.assertEqual(link.status, link_fakes.FakeLink.ACTIVE) + self.release.set() + thread.join(10) + self.assertEqual(rest, [self.other_data[CHUNK_SIZE:]]) + provider = self.fetches[0]._providers[P1] + self.assertEqual((provider.failed, provider.stalls, provider.busy), (False, 0, 0)) + self.assertEqual(len(link_fakes.FakeLink.created), 1) # the one link, never closed + self.assertEqual(link.status, link_fakes.FakeLink.ACTIVE) + response.cancel.assert_not_called() + + +class WalkTest(FetcherTest): + """fetch_dag and fetch_listing queue a block's children as soon as it is met (A4), keeping parents before their + children, so an interrupted fetch resumes from what it stored.""" + + def setUp(self): + """A directory of three subdirectories of four small files each; each request carries one DAG node (as + if each were a full block), so the subdirectories go to three providers.""" + super().setUp() + patcher = mock.patch.object(fetcher_module, "DAG_NODE_GUESS", CHUNK_SIZE) + patcher.start() + self.addCleanup(patcher.stop) + self.subdirs, self.files = [], [] + for i in range(3): + files = [self.file(os.urandom(100 + 10 * i + k)) for k in range(4)] + self.files.append([built.cid for built in files]) + self.subdirs.append(builder.build_directory([Entry(f"f{k}", built.cid, built.size, 0) + for k, built in enumerate(files)], self.put)) + self.root = builder.build_directory([Entry(f"s{i}", sub.cid, sub.size, 2) + for i, sub in enumerate(self.subdirs)], self.put) + + def test_children_are_asked_for_before_the_level_ends(self): + asked_below = threading.Event() + seen = [] + early = {cid for files in self.files[:2] for cid in files} + + def answer(cids): + """Hold the last subdirectory until files below the others are asked for (up to 3 s).""" + if early.intersection(cids): + asked_below.set() + if self.subdirs[2].cid in cids: + seen.append(asked_below.wait(3)) + return None + for node_id in (P1, P2, P3): + self.hold(node_id) + self.client.behaviour[node_id] = answer + self.runs.append([P1, P2, P3]) + self.fetch_all(self.root.cid) + self.assertTrue(self.storage.is_complete(self.root.cid)) + self.assertEqual(seen, [True], "nothing below the level was asked for while its last block was held") + asked = [cid for node_id in (P1, P2, P3) for cid in self.client.asked(node_id)] + self.assertEqual(sorted(asked), sorted(self.network)) # each block once + + def test_interrupted_walk_stores_no_child_without_its_parent(self): + self.hold(P1, [cid for cid in self.network if cid != self.subdirs[1].cid]) + self.runs.append([P1]) + with self.assertRaises(NotFound): # P1 fails once it answers None for the second subdirectory + self.fetch_all(self.root.cid) + self.assertFalse(any(self.storage.has(cid) for cid in self.files[1])) + missing = [cid for cid in self.network if not self.storage.has(cid)] + self.hold(P2) + self.runs[:] = [[P2]] + self.fetch_all(self.root.cid) + self.assertTrue(self.storage.is_complete(self.root.cid)) + self.assertEqual(sorted(self.client.asked(P2)), sorted(missing)) # resumed from what it stored + + def test_listing_fetches_index_nodes_as_they_are_met(self): + with mock.patch("rnfs.constants.MAX_CHILDREN", 4): + empty = self.file(b"") + root = builder.build_directory([Entry(f"{i:04d}", empty.cid, 0, 0) for i in range(100)], self.put) + self.hold(P1) + self.hold(P2) + self.runs.append([P1, P2]) + with self.fetcher.open(root.cid) as fetch: + fetch.fetch_listing(fetch.resolve()) + entries = list(walker.list_directory(fetch.target, fetch.get, 1000)) + self.assertEqual([entry.name for entry in entries], [f"{i:04d}" for i in range(100)]) + asked = self.client.asked(P1) + self.client.asked(P2) + self.assertEqual(len(asked), len(set(asked))) + self.assertNotIn(empty.cid, asked) # the entries' contents aren't fetched + + class LimitTest(FetcherTest): """Bounds on what a hostile DAG or provider can make a fetch cost (security findings B1-B3, B5, B7, B9).""" @@ -752,7 +1047,9 @@ def test_hung_request_ends_the_fetch(self): self.config.lookup_timeout, self.config.stall_timeout = 0.3, 0.2 built = self.file(b"never arrives") self.hold(P1) - self.client.behaviour[P1] = lambda cids: (time.sleep(5), BlocksResult(FAILED, unsent=list(cids)))[1] + release = threading.Event() # set as the test ends, so its request thread doesn't outlive it + self.addCleanup(release.set) + self.client.behaviour[P1] = lambda cids: (release.wait(5), BlocksResult(FAILED, unsent=list(cids)))[1] self.runs.append([P1]) start = time.monotonic() with self.assertRaises(NotFound): diff --git a/tests/test_gateway.py b/tests/test_gateway.py index e82a0c4..e902e0d 100644 --- a/tests/test_gateway.py +++ b/tests/test_gateway.py @@ -5,10 +5,11 @@ import unittest from unittest import mock +from rnfs import errors from rnfs.config import Config from rnfs.errors import NotFound -from rnfs.gateway import (CSP_NO_SCRIPTS, CSP_SCRIPTS, IMMUTABLE, GatewayServer, etag_matches, listing_html, - parse_range, sniff) +from rnfs.gateway import (CSP_NO_SCRIPTS, CSP_SCRIPTS, ERROR_MESSAGES, IMMUTABLE, GatewayServer, etag_matches, + listing_html, parse_range, sniff) from tests.support import wait_for from tests.test_api import FakeNode, free_slots, held_connections, raw_request, read_until_closed @@ -210,6 +211,18 @@ def test_500_body_names_no_local_path(self): with mock.patch.object(self.node, "info", side_effect=ValueError("bad target")): self.assertEqual(self.get(f"/rnfs/{self.text}")[2], b"invalid request\n") # a fixed message per status + def test_a_fixed_message_for_every_status(self): + """ERROR_MESSAGES has a message for every status of rnfs.errors.STATUSES (XC4), and for 500; a node error + gets its status's, never its own text.""" + self.assertLessEqual({status for _, status in errors.STATUSES} | {500}, set(ERROR_MESSAGES)) + for error_class, status in errors.STATUSES: + if error_class is NotFound: + continue # its own text is shown (a 404 says what wasn't found) + with self.subTest(status), mock.patch.object(self.node, "info", side_effect=error_class("/home/alice")), \ + mock.patch("rnfs.gateway.log"): + code, _, body = self.get(f"/rnfs/{self.text}") + self.assertEqual((code, body), (status, ERROR_MESSAGES[status].encode() + b"\n")) + def test_http_0_9_gets_headers(self): # A bare body for an HTTP/0.9 request would have no CSP: it is refused, with the usual headers for line in (f"GET /rnfs/{self.page}\r\n", f"GET /rnfs/{self.page} HTTP/0.9\r\n", "GARBAGE\r\n"): diff --git a/tests/test_get_partial.py b/tests/test_get_partial.py index 54f882f..e4a6934 100644 --- a/tests/test_get_partial.py +++ b/tests/test_get_partial.py @@ -32,9 +32,9 @@ def setUp(self): def test_failed_write_out_leaves_nothing(self): real = walker.write_out - def write_then_fail(expect, get, dest, max_entries): + def write_then_fail(expect, get, dest, max_entries, **kwargs): """Write the whole tree, then fail as a late error would.""" - real(expect, get, dest, max_entries) + real(expect, get, dest, max_entries, **kwargs) raise TooLarge("failed after writing") with mock.patch.object(walker, "write_out", write_then_fail): diff --git a/tests/test_ids.py b/tests/test_ids.py new file mode 100644 index 0000000..6172e9c --- /dev/null +++ b/tests/test_ids.py @@ -0,0 +1,88 @@ +"""Tests for the 32-hex-digit ID parser (rnfs/ids.py) and the parsers built on it, which must agree.""" +import unittest + +from rnfs import ids +from rnfs.api import parse_hex_id +from rnfs.features.contacts import parse_hex +from rnfs.features.lxmf import parse_address +from rnfs.names import parse_name +from rnfs.network.identity import parse_node_id + +VALUE = bytes(range(16)) +HEX = VALUE.hex() +# Text holding 32 hex digits that isn't one ID: bytes.fromhex alone would skip the whitespace between them +INNER_WHITESPACE = ["0011 2233 4455 6677 8899 aabb ccdd eeff", HEX[:14] + " " + HEX[14:], HEX[:2] + "\t" + HEX[2:], + f"< {HEX}>", f"<{HEX[:10]}\n{HEX[10:]}>"] +NEVER = ["", HEX[:-1], HEX + "0", "zz" * 16, "0x" + HEX[2:], f"{HEX}{HEX}", "٠" * 32, None, 5, VALUE] + + +class ParseHex16Test(unittest.TestCase): + + def test_plain_forms(self): + for text in (HEX, HEX.upper(), f" {HEX}\n", f"\t{HEX.upper()} "): + self.assertEqual(ids.parse_hex16(text, "ID"), VALUE) + self.assertTrue(ids.is_hex16(text)) + + def test_brackets_only_when_allowed(self): + for text in (f"<{HEX}>", f" <{HEX.upper()}> ", f"<{HEX}", f"{HEX}>"): + self.assertEqual(ids.parse_hex16(text, "ID", brackets=True), VALUE) + with self.assertRaises(ValueError): + ids.parse_hex16(text, "ID") + + def test_refused(self): + for text in INNER_WHITESPACE + NEVER: + for brackets in (False, True): + self.assertFalse(ids.is_hex16(text, brackets=brackets), text) + with self.assertRaises(ValueError, msg=text): + ids.parse_hex16(text, "ID", brackets=brackets) + + def test_lowercase_only(self): + self.assertEqual(ids.parse_hex16(f" {HEX} ", "ID", upper=False), VALUE) + with self.assertRaises(ValueError): + ids.parse_hex16(HEX.upper(), "ID", upper=False) + self.assertFalse(ids.is_hex16("A" + HEX[1:], upper=False)) + + def test_error_names_what(self): + with self.assertRaisesRegex(ValueError, r"invalid LXMF address 'x': must be 32 hex characters"): + ids.parse_hex16("x", "LXMF address") + + def test_normal_form(self): + self.assertEqual(ids.normal_hex16(f" <{HEX.upper()}>\n"), HEX) + + def test_patterns_are_unstripped(self): + """Persisted rows are checked against the patterns as they are: what loaded before must load, nothing more.""" + self.assertTrue(ids.HEX16.fullmatch(HEX.upper())) + self.assertIsNone(ids.HEX16_LOWER.fullmatch(HEX.upper())) + for pattern in (ids.HEX16, ids.HEX16_LOWER): + self.assertIsNone(pattern.fullmatch(f" {HEX}")) + self.assertIsNone(pattern.fullmatch(f"<{HEX}>")) + + +class CallersAgreeTest(unittest.TestCase): + """Every 32-hex parser takes the same text, differing only in whether are allowed.""" + + BRACKETED = (parse_node_id, parse_name, parse_hex, parse_address) + + def test_bracketed_parsers(self): + for parse in self.BRACKETED: + for text in (HEX, HEX.upper(), f" {HEX}\n", f"<{HEX}>", f" <{HEX.upper()}> "): + self.assertEqual(parse(text), VALUE, (parse.__name__, text)) + for text in INNER_WHITESPACE + NEVER: + with self.assertRaises(ValueError, msg=(parse.__name__, text)): + parse(text) + + def test_api_parser_refuses_brackets(self): + self.assertEqual(parse_hex_id(f" {HEX.upper()} "), VALUE) + for text in [f"<{HEX}>"] + INNER_WHITESPACE + NEVER: + with self.assertRaises(ValueError, msg=text): + parse_hex_id(text) + + def test_node_id_inner_whitespace_narrowed(self): + """parse_node_id (bootstrap_nodes) used bytes.fromhex, which took whitespace between byte pairs; now the + same 32 digits as every other ID. Such a bootstrap node is skipped with a warning, as malformed ones were.""" + with self.assertRaisesRegex(ValueError, "invalid node ID"): + parse_node_id("0011 2233 4455 6677 8899 aabb ccdd eeff") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 72b50dd..895c6d1 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -20,7 +20,7 @@ from rnfs.content.cid import RAW, Cid from rnfs.content.storage import USER from rnfs.features import jobs -from rnfs.node import RnfsNode +from rnfs.node import STOP_TIMEOUT, RnfsNode from tests.support import failing_commits, real, state_execute, state_rows, wait_for, write_legacy from tests.test_api import FakeNode from tests.test_names import FakeNetwork @@ -212,6 +212,21 @@ def test_submit_saves_once_whether_the_job_starts_or_waits(self): self.assertEqual(self.finished(started)["state"], "done") self.assertEqual(self.finished(queued)["state"], "done") + def test_a_stalled_running_job_isnt_saved_again(self): + """The periodic save writes a running job only when its record changed: a stalled one costs no commit.""" + cid = self.share(b"stalled") + self.net.gate.clear() + job_id = self.jobs.submit("get", cid, self.out("stalled"))["id"] + self.assertTrue(wait_for(lambda: self.jobs.get(job_id)["state"] == "running")) + self.jobs.save() + with mock.patch.object(self.jobs._db, "transaction", wraps=self.jobs._db.transaction) as write: + self.jobs.save() + self.jobs.save() + self.assertEqual(write.call_count, 0) + self.net.gate.set() + self.assertEqual(self.finished(job_id)["state"], "done") + self.assertTrue(wait_for(lambda: [r["state"] for r in self.saved()] == ["done"])) # its outcome saved + def test_cancel_queued_job_and_concurrency_limit(self): self.b.config.max_jobs = 1 cids = [self.share(f"file {i}".encode()) for i in range(3)] @@ -471,7 +486,7 @@ def test_failed_save_leaves_the_job_to_the_next(self): with failing_commits(): job_id = self.jobs.submit("pin", cid)["id"] self.assertEqual(self.saved(), []) - self.assertTrue(self.jobs._dirty) + self.assertEqual(self.jobs._store.pending, {job_id}) # left for the next save self.jobs.save() self.assertEqual([(r["id"], r["state"]) for r in self.saved()], [(job_id, "queued")]) @@ -490,7 +505,8 @@ def test_stop_never_hangs(self): self.assertTrue(wait_for(lambda: self.net.waiting > 0)) started = time.monotonic() self.b.stop() - self.assertLess(time.monotonic() - started, jobs.STOP_WAIT + 2) + # Its fetch notices the stop at its next step, well within the node's stop budget + self.assertLess(time.monotonic() - started, STOP_TIMEOUT - 1) class ApiBase(JobsBase): @@ -736,6 +752,26 @@ def test_rate_is_smoothed_and_decays(self): job._samples.append((start + jobs.RATE_WINDOW + 50, 10_000)) self.assertAlmostEqual(job.rate(start + jobs.RATE_WINDOW + 60), 0.0) + def test_rate_samples_pruned_without_polling(self): + """S1: a running job nobody polls keeps only the window's samples (plus the base), and its rate is + the one an unpruned job would give.""" + clock = [1000.0] + with mock.patch.object(jobs.time, "monotonic", lambda: clock[0]): + job = jobs.Job("0123abcd", "get", "x", "/o") + job.begin_run() + seconds = 3 * int(jobs.RATE_WINDOW) + for n in range(seconds): + clock[0] += 1.0 + job.block(Cid.for_block(RAW, n.to_bytes(4, "big")), 100 if n < seconds - 60 else 400, True) + # One a second over the window, plus the base sample at its start + self.assertEqual(len(job._samples), int(jobs.RATE_WINDOW / jobs.SAMPLE_INTERVAL) + 1) + self.assertLessEqual(job._samples[0][0], clock[0] - jobs.RATE_WINDOW) + self.assertGreater(job._samples[1][0], clock[0] - jobs.RATE_WINDOW) + # The last RATE_WINDOW seconds: 240 at 100 B/s, 60 at 400 B/s + self.assertAlmostEqual(job.rate(), (240 * 100 + 60 * 400) / jobs.RATE_WINDOW) + self.assertEqual(job.downloaded_bytes, (seconds - 60) * 100 + 60 * 400) + self.assertEqual(job.blocks, seconds) + def test_settings_validated(self): for settings in ({"max_jobs": 0}, {"max_jobs": True}, {"max_jobs": 1.5}, {"max_finished_jobs": -1}, {"max_finished_jobs": 0}, {"max_finished_jobs": True}): diff --git a/tests/test_keepers.py b/tests/test_keepers.py new file mode 100644 index 0000000..92f3401 --- /dev/null +++ b/tests/test_keepers.py @@ -0,0 +1,222 @@ +"""Direct tests of a keeper query (rnfs/features/keepers.py, _KeeperQuery): the search for a name's keepers, the +choice of each batch, the answers kept, when they settle the name, and when the search stops and resumes. + +The keepers resolution as a whole (resolve, kept records, forgeries) is tested through offline nodes in +tests/test_names_keep.py and tests/security/test_m3*.py; here resolver B is an offline node over KeeperNet, whose +/name answers each test sets per keeper. +""" +import os +import tempfile +import time +import unittest +from unittest import mock + +import RNS + +from rnfs.config import Config +from rnfs.features.keepers import _UNVOUCHED, SEARCH_SHARE, _KeeperQuery, name_key +from rnfs.names import name_for_identity +from rnfs.node import RnfsNode +from tests.test_names import FakeNetwork +from tests.test_names_keep import CID1, CID2, KeeperNet, signed + +B_ID, S_ID = bytes([0xB] * 16), bytes([0x5] * 16) +K1, K2, K3, K4 = (bytes([0x10 + i] * 16) for i in range(4)) # keepers: /name answers only (no node behind them) +SOURCE1, SOURCE2 = bytes([0x21] * 16), bytes([0x22] * 16) + + +class KeeperQueryTest(unittest.TestCase): + """Resolver B (alpha 3) and the records of a name at sequences 1 and 2, signed by its key.""" + + def setUp(self): + """The network, B, and the name's records.""" + saved = RNS.loglevel + RNS.loglevel = RNS.LOG_CRITICAL + self.addCleanup(setattr, RNS, "loglevel", saved) + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dir = tmp.name + self.net = KeeperNet() + self.b = self.make_node("b", B_ID) + identity = RNS.Identity() + self.name = name_for_identity(identity) + self.key = name_key(self.name) + self.old, self.new = signed(identity, 1, CID1).wire(), signed(identity, 2, CID2).wire() + + def make_node(self, label: str, node_id: bytes, slow: bool = False, **settings) -> RnfsNode: + """An offline node with its own directory; heard servers are established at once unless settings say.""" + settings.setdefault("provider_seniority", 0) + config = Config(rnfs_dir=os.path.join(self.dir, label), slow="no", lookup_timeout=5, path_timeout=0.2, + **settings) + node = RnfsNode.offline(config, node_id, self.net, FakeNetwork(), slow=slow) + self.net.nodes[node_id] = node + self.addCleanup(node.stop) + return node + + def query(self, node: RnfsNode | None = None) -> _KeeperQuery: + """A keeper query of node (B by default) for the name.""" + return _KeeperQuery((node or self.b).feature("keepers"), self.name) + + def with_candidates(self, vouched: dict[bytes, set[bytes]], node: RnfsNode | None = None) -> _KeeperQuery: + """A query whose search found vouched's keys, ranked in that order, each listed by the given sources.""" + query = self.query(node) + query.candidates = list(vouched) + query.vouched = dict(vouched) + return query + + # The search + + def test_search_ranks_providers_and_records_who_vouches(self): + """A record stored here vouches with this node; one an established server lists, with that server.""" + s = self.make_node("s", S_ID) + self.b.routing.heard(S_ID) + self.assertTrue(s.records.store(self.key, K1)) + self.assertTrue(self.b.records.store(self.key, K2)) + self.b.routing.heard(K2) # in the table: ranks before K1, which isn't + query = self.query() + self.assertEqual(query.search(), [K2, K1]) + self.assertEqual(query.vouched, {K1: {S_ID}, K2: {B_ID}}) + self.assertIsNone(query.floor) + self.assertEqual(self.net.name_requests, []) # searching asks no keeper + + def test_listing_by_a_server_without_standing_vouches_as_nobody(self): + b = self.make_node("b2", bytes([0xB2] * 16), provider_seniority=86400) + s = self.make_node("s", S_ID) + b.routing.heard(S_ID) # healthy, but not in the table long enough to be established + self.assertTrue(s.records.store(self.key, K1)) + query = self.query(b) + self.assertEqual(query.search(), [K1]) + self.assertEqual(query.vouched, {K1: {_UNVOUCHED}}) + + def test_search_finds_nothing(self): + query = self.query() + self.assertEqual(query.search(), []) + self.assertEqual(query.candidates, []) + + def test_search_takes_known_sequence_as_floor(self): + self.assertTrue(self.b.records.store(self.key, K1)) + self.b.names.offer(self.name, 2, CID2, 0.0) + query = self.query() + query.search() + self.assertEqual(query.floor, 2) + + def test_soft_budget_stops_a_search_with_candidates(self): + """Once lookup_timeout / SEARCH_SHARE has passed, any candidate in hand stops the search (cut short).""" + query = self.query() + query._search_started = time.monotonic() - self.b.config.lookup_timeout / SEARCH_SHARE + self.assertFalse(query.should_stop()) # nothing found yet + query.add(K1) + self.assertTrue(query.should_stop()) + self.assertTrue(query.cut_short) + query._search_started = time.monotonic() + query.cut_short = False + self.assertFalse(query.should_stop()) # too early for one candidate + + def test_resume_only_after_a_search_stopped_early_and_a_batch_asked(self): + self.assertTrue(self.b.records.store(self.key, K1)) + query = self.query() + with mock.patch.object(query.lookup, "find_providers") as find: + self.assertEqual(query.search(), [K1]) + self.assertFalse(query.resume()) # the search ended by itself (the lookup ended it) + query.cut_short = True + self.assertFalse(query.resume()) # none of its candidates asked yet + self.assertEqual(query.next_batch(), [K1]) + self.assertFalse(query.resume()) # resumed, but nothing new found + self.assertEqual(find.call_count, 2) + + def test_resumed_search_asks_only_new_candidates(self): + self.assertTrue(self.b.records.store(self.key, K1)) + query = self.query() + with mock.patch.object(query.lookup, "find_providers"): + query.search() + self.assertEqual(query.next_batch(), [K1]) + + def found_more(on_provider, should_stop): + """A resumed run finding K1 again, and K2.""" + on_provider(K1) + on_provider(K2) + should_stop() + return [K2] + + query.cut_short = True + with mock.patch.object(query.lookup, "find_providers", side_effect=found_more): + self.assertTrue(query.resume()) + self.assertEqual(query.candidates, [K2]) + self.assertEqual(query.next_batch(), [K2]) + + # Batches + + def test_batch_takes_keepers_of_unused_sources_first_then_rank_order(self): + query = self.with_candidates({K1: {SOURCE1}, K2: {SOURCE1}, K3: {SOURCE2}, K4: {SOURCE1, SOURCE2}}) + self.assertEqual(query.next_batch(), [K1, K3, K2]) + self.assertEqual(query.next_batch(), [K4]) + self.assertEqual(query.next_batch(), []) + self.assertEqual(query.asked, 4) + + def test_slow_node_asks_one_at_a_time(self): + b = self.make_node("slow", bytes([0xB3] * 16), slow=True) + query = self.with_candidates({K1: {SOURCE1}, K2: {SOURCE2}}, node=b) + self.assertEqual((query.next_batch(), query.next_batch(), query.next_batch()), ([K1], [K2], [])) + + def test_unknown_keepers_get_a_budget_of_alpha(self): + """Each unknown keeper costs a path request: past alpha of them, they are passed over (and dropped).""" + unknown = {K1, K2, K3, K4} + with mock.patch.object(self.b, "_is_known", lambda node_id: node_id not in unknown): + query = self.with_candidates({K1: {SOURCE1}, K2: {SOURCE1}, K3: {SOURCE1}, K4: {SOURCE1}, S_ID: {SOURCE1}}) + self.assertEqual(query.next_batch(), [K1, K2, K3]) + self.assertEqual(query.next_batch(), [S_ID]) # K4 is over the budget + self.assertEqual(query.candidates, []) + self.assertEqual(query.asked, 4) + + # Answers + + def test_only_verifying_answers_are_kept(self): + other = signed(RNS.Identity(), 3, CID2).wire() # validly signed, but by another name's key + self.net.forged.update({K1: self.old, K2: other, K3: False, K4: None}) + query = self.with_candidates({K1: {SOURCE1}, K2: {SOURCE1}, K3: {SOURCE1}, K4: {SOURCE1}}) + for keeper in (K1, K2, K3, K4): + query.ask(keeper) + self.assertEqual([(record.sequence, record.cid) for record in query.records()], [(1, CID1)]) + + def test_failed_request_is_no_answer(self): + query = self.with_candidates({K1: {SOURCE1}}) + with mock.patch.object(self.net, "name_record", side_effect=OSError("link failed")): + query.ask(K1) + self.assertEqual(query.records(), []) + + # Settling + + def test_nothing_known_settles_on_two_established_keepers(self): + """With no sequence known, only established keepers confirm each other.""" + self.b.routing.heard(K1) + self.b.routing.heard(K2) + self.net.forged.update({K1: self.old, K2: self.old, K3: self.old}) + query = self.with_candidates({K1: {SOURCE1}, K2: {SOURCE1}, K3: {SOURCE1}}) + self.assertFalse(query.settled()) + query.ask(K1) + query.ask(K3) # not in B's table: no standing + self.assertFalse(query.settled()) + query.ask(K2) + self.assertTrue(query.settled()) + + def test_known_sequence_settles_on_an_answer_reaching_it_and_a_second_keeper(self): + self.net.forged.update({K1: self.old, K2: self.old, K3: self.new}) + query = self.with_candidates({K1: {SOURCE1}, K2: {SOURCE1}, K3: {SOURCE1}}) + query.floor = 2 + query.ask(K1) + query.ask(K2) + self.assertFalse(query.settled()) # two answers, both older than what is known + query.ask(K3) + self.assertTrue(query.settled()) # any keepers count once an answer reaches the known sequence + + def test_slow_node_settles_on_one_answer_reaching_known_sequence(self): + b = self.make_node("slow", bytes([0xB3] * 16), slow=True) + self.net.forged[K1] = self.new + query = self.with_candidates({K1: {SOURCE1}}, node=b) + query.floor = 2 + query.ask(K1) + self.assertTrue(query.settled()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_links.py b/tests/test_links.py index fcb5d10..e9a789b 100644 --- a/tests/test_links.py +++ b/tests/test_links.py @@ -150,7 +150,7 @@ def fake_rns(): Identity=types.SimpleNamespace(recall=lambda node_id: types.SimpleNamespace(node_id=node_id)), Transport=types.SimpleNamespace(await_path=lambda node_id, timeout: True, has_path=lambda node_id: True, request_path=lambda node_id: None), - log=lambda *args: None, prettyhexrep=lambda b: b.hex(), LOG_INFO=4, LOG_VERBOSE=5, + log=lambda *args: None, prettyhexrep=lambda b: b.hex(), LOG_INFO=4, LOG_VERBOSE=5, loglevel=4, ) @@ -238,6 +238,27 @@ def test_unreachable(self): links.RNS.Transport.await_path = lambda node_id, timeout: False self.assertIsNone(run(self, lambda: self.pool.get(NODE))) + def test_awaited_identity_marked_used_once(self): + # Polls for an identity still on its way don't mark the destination used; contacting it does, once (C2) + calls = [] + + def recall(node_id, _no_use=False): + """Unknown for the first call and two polls, then known.""" + calls.append(_no_use) + return types.SimpleNamespace(node_id=node_id) if len(calls) > 3 else None + links.RNS.Identity.recall = recall + self.assertIsNotNone(run(self, lambda: self.pool.get(NODE))) + self.assertEqual(calls, [False, True, True, True, False]) + + def test_path_described_only_for_a_kept_debug_message(self): + # The description costs RNS lookups: built only when a LOG_DEBUG message goes somewhere (C2) + with mock.patch.object(links, "path_description", return_value="") as describe: + run(self, lambda: self.pool.get(NODE)) + describe.assert_not_called() + links.RNS.loglevel = links.LOG_DEBUG + run(self, lambda: self.pool.get(OTHER)) + describe.assert_called_once_with(OTHER) + def test_no_link_to_another_destination_of_the_identity(self): # The identity recalled for NODE has another rnfs.dht hash: NODE is some other destination of it links.RNS.Identity.recall = lambda node_id: types.SimpleNamespace(node_id=node_id, dht_hash=OTHER) diff --git a/tests/test_log.py b/tests/test_log.py index 057f44b..b9b54b0 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -1,8 +1,11 @@ -"""Tests for rnfs.log: every log line escapes non-printable characters (remote names reach the log).""" +"""Tests for rnfs.log: every log line escapes non-printable characters (remote names reach the log); ThrottledLog.""" +import threading +import types import unittest from unittest import mock from rnfs import api, cli, fsutil, log +from tests.support import FakeClock class LogTest(unittest.TestCase): @@ -37,5 +40,78 @@ def test_cap_warnings(self): self.assertEqual(len(capped["warnings"]), log.MAX_WARNINGS + 1) +class ThrottledLogTest(unittest.TestCase): + """log.ThrottledLog: a line at most every interval, counting the ones left out.""" + + def setUp(self): + """A throttled log on a fake clock, its lines recorded.""" + self.clock = FakeClock() + self.throttled = log.ThrottledLog(60.0, self.clock) + self.lines = [] + patcher = mock.patch.object(log, "log", lambda message, level: self.lines.append((message, level))) + patcher.start() + self.addCleanup(patcher.stop) + + def test_one_line_per_interval_counting_the_rest(self): + self.throttled("refused one", log.LOG_WARNING) + for _ in range(5): + self.clock.now += 10 + self.throttled("refused another", log.LOG_WARNING) + self.assertEqual(self.lines, [("refused one", log.LOG_WARNING)]) + self.clock.now += 10 # 60 s after the first line + self.throttled("refused a seventh", log.LOG_VERBOSE) + self.assertEqual(self.lines[1], ("refused a seventh (5 more since the last)", log.LOG_VERBOSE)) + self.clock.now += 59 + self.throttled("not yet", log.LOG_WARNING) + self.clock.now += 120 + self.throttled("again", log.LOG_WARNING) + self.assertEqual(self.lines[2:], [("again (1 more since the last)", log.LOG_WARNING)]) + + def test_due(self): + """due() is the check alone, for a caller that words its own line (RequestServer._over_rate).""" + self.assertEqual(self.throttled.due(), 0) + self.assertIsNone(self.throttled.due()) + self.assertIsNone(self.throttled.due()) + self.clock.now += 60 + self.assertEqual(self.throttled.due(), 2) + self.assertEqual(self.lines, []) + + def test_thread_safe(self): + throttled = log.ThrottledLog(3600.0) + threads = [threading.Thread(target=lambda: [throttled("x", log.LOG_WARNING) for _ in range(2000)]) + for _ in range(4)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.assertEqual(len(self.lines), 1) + self.assertEqual(throttled.skipped, 7999) + + + +class ThrottledTracebackTest(unittest.TestCase): + """What the unexpected-exception sites (XC4) rely on.""" + + def test_traceback_reaches_the_debug_log_inside_an_except(self): + """Logged inside an except at LOG_WARNING, a throttled line carries the traceback to the debug log, as + rnfs.log does for any line; the lines it leaves out write nothing.""" + from rnfs import debuglog + throttled = log.ThrottledLog(60.0, FakeClock()) + written = [] + debug = types.SimpleNamespace(write=lambda level, source, text, trace: written.append((text, trace))) + with mock.patch.object(debuglog, "_active", debug), mock.patch("RNS.log"): + self.addCleanup(log.set_debug_sink, log._debug_sink) + log.set_debug_sink(debuglog._rnfs_message) + for _ in range(2): + try: + raise TypeError("a bug") + except TypeError: + throttled("unexpected", log.LOG_WARNING) + self.assertEqual(len(written), 1) + text, trace = written[0] + self.assertEqual(text, "unexpected") + self.assertIn("TypeError: a bug", trace) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_lookup.py b/tests/test_lookup.py index 5d0bc3e..6259623 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -6,7 +6,7 @@ from rnfs.config import Config from rnfs.constants import ID_LENGTH, K -from rnfs.dht.lookup import Lookup +from rnfs.dht.lookup import MAX_QUERIES, STALL_WINDOWS, Lookup from rnfs.dht.providers import ProviderStore from rnfs.dht.routing import AVOID_AFTER, RoutingTable, distance from tests.support import FakeClock, wait_for @@ -586,5 +586,94 @@ def test_rerun_gives_fresh_random_half(self): self.assertIn(holder, self.net.queried[len(self.net.queried) - 20:]) # re-queried +class BoundsTest(LookupTestCase): + """Nodes that keep answering with ever closer IDs (made up, or ground for the key) can't hold a run open: it + ends at the query cap, or after STALL_WINDOWS windows of answers in a row that bring the key no nearer.""" + + def grinding(self, next_distance) -> tuple[list[bytes], object]: + """A query function for key ident(0), and the nodes it was asked: every node answers with K fresh IDs at + distances next_distance(n) + 0..K-1, n counting the answers so far.""" + asked: list[bytes] = [] + lock = threading.Lock() + + def query(node_id, key): + """A made-up answer, nearer the key as next_distance says.""" + with lock: + asked.append(node_id) + distance_ = next_distance(len(asked)) + return [], [ident(distance_ + i) for i in range(K)] + return asked, query + + def run_grinding(self, next_distance, start: int = 1 << 120, **kwargs) -> tuple[list[bytes], str]: + """find_closest over grinding nodes from three start nodes at distance start; the nodes asked, and how the + run ended.""" + asked, query = self.grinding(next_distance) + reports: list[str] = [] + lookup = Lookup(ident(0), [ident(start + i) for i in range(3)], query, ident(1 << 127), timeout=30, + poll_interval=0.05, report=reports.append, **kwargs) + lookup.find_closest() + ends = [text for text in reports if "s; asked" in text] + self.assertEqual(len(ends), 1, reports) + return asked, ends[0] + + def test_ever_closer_answers_end_at_the_query_cap(self): + asked, end = self.run_grinding(lambda n: 1 << max(8, 119 - n)) # each answer a bit nearer + self.assertIn(": query cap after", end) + self.assertEqual(len(asked), MAX_QUERIES) + self.assertEqual(MAX_QUERIES, 8 * K) + + def test_query_cap_is_a_parameter(self): + asked, end = self.run_grinding(lambda n: 1 << max(8, 119 - n), max_queries=10) + self.assertIn(": query cap after", end) + self.assertEqual(len(asked), 10) + + def test_answers_that_bring_the_key_no_nearer_end_the_run(self): + """Each answer is nearer, but within the same power of two: the beta-th closest distance keeps its bit + length, so the run ends after 3 windows (9 answers), with at most alpha more queries out.""" + start = (1 << 120) + (1 << 119) + asked, end = self.run_grinding(lambda n: start - 1000 * n, start=start) + self.assertIn(": no progress after", end) + self.assertGreaterEqual(len(asked), STALL_WINDOWS * 3) + self.assertLessEqual(len(asked), STALL_WINDOWS * 3 + 3) + + def test_progress_resets_the_count(self): + """A bit nearer every 5th answer is progress enough: such a run goes on to the query cap.""" + asked, end = self.run_grinding(lambda n: (1 << max(8, 119 - n // 5)) + (1 << max(7, 118 - n // 5)) - n) + self.assertIn(": query cap after", end) + self.assertEqual(len(asked), MAX_QUERIES) + + +class HonestBoundsTest(LookupTestCase): + """The bounds never end an honest lookup: over a 2000-node network every run completes, as it did before them, so + its result is unchanged (only completion, a deadline, or running out of nodes ends a run the bounds don't).""" + + @classmethod + def setUpClass(cls): + """One 2000-node network for the class (it takes seconds to build).""" + cls.net = FakeNetwork(2000, seed=21) + + def test_lookups_in_2000_nodes_complete(self): + rng = random.Random(22) + ids = list(self.net.nodes) + for alpha, beta in ((3, 3), (1, 3), (5, 3), (3, 1)): + for _ in range(25): + key = rng.randbytes(ID_LENGTH) + reports: list[str] = [] + lock = threading.Lock() + + def report(text, reports=reports, lock=lock): + """Collect reports (query threads report too).""" + with lock: + reports.append(text) + + result = Lookup(key, rng.sample(ids, 3), self.net.query, ident(0), alpha=alpha, beta=beta, timeout=30, + poll_interval=0.05, report=report).find_closest() + with lock: + ends = [text for text in reports if "s; asked" in text] + self.assertEqual(len(ends), 1) + self.assertIn(": complete after", ends[0], f"alpha {alpha}, beta {beta}") + self.assertEqual(result[:beta], self.net.true_closest(key, beta)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_lxmf.py b/tests/test_lxmf.py index fbdefc3..a476c84 100644 --- a/tests/test_lxmf.py +++ b/tests/test_lxmf.py @@ -8,6 +8,8 @@ import subprocess import sys import tempfile +import threading +import time import unittest from unittest import mock @@ -23,7 +25,7 @@ from rnfs.features.lxmf import FIELD_CUSTOM_DATA, FIELD_CUSTOM_TYPE, Incoming, Link, MessageLog from rnfs.node import RnfsNode -from tests.support import FakeClock, failing_commits, state_execute, state_rows, write_legacy +from tests.support import FakeClock, failing_commits, state_execute, state_rows, wait_for, write_legacy from tests.test_names import FakeNetwork from tests.test_node import NetClient @@ -415,6 +417,10 @@ def poll(self): """Nothing changes by itself.""" return [] + def idle(self): + """Nothing to poll.""" + return True + def sync(self, nearest=False): """Started if there is a propagation node to use.""" return self.propagation_node(nearest) is not None @@ -830,24 +836,29 @@ def test_refusals(self): self.assertEqual(self.network.sent, []) def test_contact_nick(self): + # The node's own address book (contacts.resolve_id finds the recipient) feature = self.feature_node() - contacts = mock.Mock() - contacts.lookup.side_effect = lambda nick, kind=None: SENDER if nick == "alice" and kind in ("lxmf", None) \ - else (OTHER if nick == "bob" else None) - self.addCleanup(self.b.features["contacts"].close) # the node closes the stand-in instead - self.b.features["contacts"] = contacts + contacts = self.b.feature("contacts") + contacts.add("alice", SENDER.hex(), "lxmf") + contacts.add("bob", OTHER.hex(), "node") + contacts.add("site", NAME.hex(), "name") feature.send("alice", str(self.cid)) self.assertEqual(self.network.sent[0][0], SENDER) - contacts.lookup.side_effect = lambda nick, kind=None: OTHER if kind is None else None - with self.assertRaisesRegex(ValueError, "not an LXMF address"): + with self.assertRaisesRegex(ValueError, "contact bob is not an LXMF address"): feature.send("bob", str(self.cid)) + with self.assertRaisesRegex(ValueError, "no LXMF contact carol"): + feature.send("carol", str(self.cid)) # A name contact as the target: the link follows the name - contacts.lookup.side_effect = lambda nick, kind=None: {("alice", "lxmf"): SENDER, - ("site", "name"): NAME}.get((nick, kind)) feature.send("alice", "site/docs/a.txt") self.assertEqual(lxmf.decode_field(self.network.sent[1][2]), [Link(name=NAME, path=("docs", "a.txt"), from_node=B_ID)]) + def test_sync(self): + """POST /api/lxmf/sync's answer: the propagation node asked, and whether a download started.""" + feature = self.feature_node(FakeLxmf(heard=OTHER)) + self.assertEqual(feature.sync(), {"propagation_node": None, "started": False}) + self.assertEqual(feature.sync(nearest=True), {"propagation_node": OTHER.hex(), "started": True}) + def test_unfinished_messages_fail_after_restart(self): feature = self.feature_node() feature.send(SENDER.hex(), str(self.cid)) @@ -858,6 +869,93 @@ def test_unfinished_messages_fail_after_restart(self): self.assertEqual(self.reloaded().outbox_list()[0]["state"], "failed") +class LoopNetwork(FakeLxmf): + """FakeLxmf counting the loop's announces, downloads and polls; idle unless the test queues a message.""" + + def __init__(self): + """Nothing queued; feature is set by the test, for wake().""" + super().__init__() + self.announces = self.syncs = self.polls = 0 + self.pending = 0 # polls until the queued message is done with + self.feature = None + + def announce(self): + """Count it.""" + self.announces += 1 + + def sync(self, nearest=False): + """Count it.""" + self.syncs += 1 + return super().sync(nearest) + + def poll(self): + """Count it; the queued message is done with after its polls.""" + self.polls += 1 + self.pending = max(0, self.pending - 1) + return [] + + def idle(self): + """Idle once the queued message is done with.""" + return not self.pending + + def queue(self, polls: int): + """Queue a message followed for polls polls, waking the loop as LxmfNetwork.send does.""" + self.pending = polls + self.feature.wake() + + +class LoopTest(FeatureTestCase): + """S7: the announce/sync/poll loop polls each TICK only while the network has something to poll.""" + + def test_idle_loop_waits_until_the_next_announce_or_download(self): + network = LoopNetwork() + feature = self.feature_node(network, startup_announce_delay=30, lxmf_sync_interval=3600) + timeouts = [] + + class Recording(threading.Event): + """Records each wait's timeout and returns at once; queues a message at the third, stops at the sixth.""" + + def wait(self, timeout=None): + timeouts.append(timeout) + if len(timeouts) == 3: + network.pending = 2 + if len(timeouts) == 6: + feature._stop.set() + return super().wait(0) + + feature._wake = Recording() + feature._run() # here, until the sixth wait stops it + self.assertEqual((network.announces, network.syncs, network.polls), (1, 0, 6)) + # Idle: until the second announce (startup_announce_delay); the queued message: each TICK + self.assertEqual(timeouts[3], lxmf.TICK) + for timeout in timeouts[:3] + timeouts[4:]: + self.assertTrue(29 < timeout <= 30, timeouts) + + def test_a_queued_message_wakes_the_idle_loop(self): + network = LoopNetwork() + feature = self.feature_node(network) # the default startup_announce_delay: idle, it waits 30 s + network.feature = feature + thread = threading.Thread(target=feature._run, daemon=True) + thread.start() + self.addCleanup(thread.join, 5) + self.addCleanup(feature.wake) + self.addCleanup(feature._stop.set) + wait_for(lambda: network.polls, 10, "the first poll") + time.sleep(1.2) + self.assertEqual(network.polls, 1) # no polls each second while idle + network.queue(2) + # Polled at once, then each TICK until it is done with; then idle again + wait_for(lambda: network.polls >= 3, 10, "the queued message's polls") + time.sleep(1.2) + self.assertEqual(network.polls, 3) + self.assertEqual(network.announces, 1) + # stop() wakes the idle loop + feature._stop.set() + feature.wake() + thread.join(5) + self.assertFalse(thread.is_alive()) + + class DisabledTest(FeatureTestCase): """Without the LXMF package, or switched off, the feature stays off cleanly.""" @@ -868,12 +966,14 @@ def test_lxmf_missing(self): feature.start() self.assertIsNone(feature.network) self.assertIn("not installed", feature.off_reason) - self.assertEqual(feature._threads, []) + self.assertEqual(feature._spawned, []) self.assertIsNone(self.b.status()["lxmf_address"]) with self.assertRaisesRegex(ValueError, "not installed"): feature.send(SENDER.hex(), str(self.cid)) with self.assertRaisesRegex(ValueError, "not installed"): feature.address() + with self.assertRaisesRegex(ValueError, "not installed"): + feature.sync() self.assertEqual(feature.inbox_list(), []) # stored messages stay readable feature.stop() @@ -991,6 +1091,8 @@ def test_parser(self): self.assertEqual(parser.parse_args(["inbox"]).run.__name__, "_run_inbox") args = parser.parse_args(["inbox", "get", "4", "--link", "2", "-o", "x"]) self.assertEqual((args.run.__name__, args.n, args.link, args.output), ("_run_inbox_get", 4, 2, "x")) + self.assertFalse(args.use_sent_cid) + self.assertTrue(parser.parse_args(["inbox", "get", "4", "--use-sent-cid"]).use_sent_cid) self.assertTrue(parser.parse_args(["inbox", "rm", "--all"]).all) self.assertEqual(parser.parse_args(["outbox"]).run.__name__, "_run_outbox") self.assertEqual(parser.parse_args(["lxmf", "address"]).run.__name__, "_run_address") @@ -1058,7 +1160,7 @@ def test_inbox_bytes_are_bounded_and_saved_as_utf8(self): for number in range(20): feature.received(self.link_message(number=number, source=bytes([number]) * 16, text=("é" * 3000).encode())) - sizes = [lxmf._json_bytes(e) for e in feature.inbox_list()] + sizes = [lxmf._json_bytes(e) for e in feature.inbox.entries()] self.assertLessEqual(sum(sizes), 20_000) self.assertEqual(feature.inbox_list()[-1]["id"], 20) feature.save() @@ -1086,7 +1188,7 @@ def test_inbox_get_prints_escaped(self): client.post.return_value = results out = io.StringIO() with mock.patch("rnfs.cli.client", return_value=client), mock.patch("sys.stdout", out): - lxmf._run_inbox_get(SimpleNamespace(output=None, n=1, link=None)) + lxmf._run_inbox_get(SimpleNamespace(output=None, n=1, link=None, use_sent_cid=False)) self.assertNotIn("\x1b", out.getvalue()) self.assertNotIn("\x07", out.getvalue()) @@ -1235,7 +1337,7 @@ def test_inbox_get_prints_errors_after_the_jobs(self): out, err = io.StringIO(), io.StringIO() with mock.patch("rnfs.cli.client", return_value=client), mock.patch("sys.stdout", out), \ mock.patch("sys.stderr", err), self.assertRaises(SystemExit): - lxmf._run_inbox_get(SimpleNamespace(output=None, n=1, link=None)) + lxmf._run_inbox_get(SimpleNamespace(output=None, n=1, link=None, use_sent_cid=False)) self.assertIn("job 0123abcd: /x/a", out.getvalue()) self.assertIn("/x/b: too many jobs", err.getvalue()) self.assertNotIn("\x1b", err.getvalue()) @@ -1269,6 +1371,30 @@ def test_failed_construction_releases_the_state_database(self): lxmf.LxmfFeature(self.b) self.assertEqual(database.references, before) + def test_constructor_sets_every_attribute_before_its_start_up_steps(self): + """FT1: building the feature over an inbox that isn't clean records an unrecorded lxmf: pin in a new + message (_release_unrecorded -> _record_kept, which takes the lock and adds to the inbox); every attribute + the finished feature has is already set when that runs.""" + self.feature_node() + target = Cid.parse(self.a.info(f"{self.cid}/file.txt")["cid"]) + self.b.storage.pin(target, f"lxmf:{SENDER.hex()}") # the inbox was never saved, so it isn't clean + seen = [] + original = lxmf.LxmfFeature._release_unrecorded + + def release_unrecorded(feature): + """Note the attributes set so far, then run the real start-up step.""" + seen.append(set(vars(feature))) + original(feature) + + with mock.patch.object(lxmf.LxmfFeature, "_release_unrecorded", release_unrecorded): + again = self.reloaded() + self.assertEqual(len(seen), 1) + self.assertEqual(set(vars(again)) - seen[0], set(), "attributes set only after the start-up steps") + records = again.inbox_list() + self.assertEqual([(e["from"], e["auto"]) for e in records], + [(SENDER.hex(), [{"state": "pinned", "cid": str(target)}])]) + self.assertEqual(self.b.storage.pins()[target], {f"lxmf:{SENDER.hex()}"}) + class MessageLogBoundsTest(unittest.TestCase): """MessageLog's protection of pin holders and its byte bound.""" @@ -1330,6 +1456,7 @@ def network(self) -> lxmf.LxmfNetwork: network._propagation_nodes = {} network._configured_pn = network._current_pn = None network._sync_started = float("-inf") + network._outbound = {} network.identity = object() return network @@ -1448,6 +1575,40 @@ def test_sync_only_with_a_configured_node_unless_nearest(self): network.router.request_messages_from_propagation_node.assert_called_once_with( network.identity, lxmf.SYNC_MAX_MESSAGES) + def test_idle_and_wakes(self): + # S7: the feature's loop polls only while a message is followed or held or a download runs, and the + # network wakes it when one starts + network = self.network() + self.network_lxmf = LXMF = network.LXMF + router, wake = network.router, network.feature.wake + router.PR_PATH_REQUESTED, router.PR_COMPLETE = LXMF.LXMRouter.PR_PATH_REQUESTED, LXMF.LXMRouter.PR_COMPLETE + router.propagation_transfer_state = LXMF.LXMRouter.PR_IDLE + self.assertTrue(network.idle()) + with mock.patch.object(RNS.Transport, "request_path"): + network._delivered(self.forged(os.urandom(16), b"HELD")) + wake.assert_called_once_with() + self.assertFalse(network.idle()) + network._held = [] + self.assertTrue(network.idle()) + for state, idle in ((LXMF.LXMRouter.PR_PATH_REQUESTED, False), (LXMF.LXMRouter.PR_RECEIVING, False), + (LXMF.LXMRouter.PR_COMPLETE, True), (LXMF.LXMRouter.PR_FAILED, True)): + router.propagation_transfer_state = state + self.assertEqual(network.idle(), idle, state) + router.propagation_transfer_state = LXMF.LXMRouter.PR_IDLE + network._configured_pn = OTHER + wake.reset_mock() + self.assertTrue(network.sync()) + wake.assert_called_once_with() + # Sending: followed (and the loop woken) from when it is queued + wake.reset_mock() + network.LXMF, network.destination = mock.Mock(), mock.Mock() + network.LXMF.LXMessage.return_value = mock.Mock(hash=b"m" * 32) + with mock.patch.object(RNS.Identity, "recall", return_value=mock.Mock()), \ + mock.patch.object(RNS, "Destination"): + self.assertEqual(network.send(SENDER, "hi", {}, False), b"m" * 32) + wake.assert_called_once_with() + self.assertFalse(network.idle()) + def test_settings(self): # BG-L7 and the new lxmf_send_from_hint for settings in ({"lxmf_enabled": 0}, {"lxmf_enabled": 1}, {"lxmf_send_from_hint": "no"}, diff --git a/tests/test_mesh.py b/tests/test_mesh.py index c1a4f77..5768eca 100644 --- a/tests/test_mesh.py +++ b/tests/test_mesh.py @@ -2,18 +2,21 @@ seniority, hops and bitrate; the upload budget, interface rules and quiet hours on /blocks; traffic counters and `rnfs peers` on an offline node.""" import os +import sys import tempfile import threading import types import unittest +from unittest import mock import RNS -from rnfs import mesh +from rnfs import log, mesh from rnfs.config import Config from rnfs.constants import BLOCKS_PATH, FIND_PATH, KIB, MAX_RESPONSE_SIZE from rnfs.features import peers as peers_feature from rnfs.network import protocol, server +from rnfs.network.announces import DhtAnnounceHandler, encode_mode from rnfs.network.client import BUSY, FAILED, OK, BlocksResult, PeerClient from rnfs.node import RnfsNode from tests.support import FakeClock @@ -23,6 +26,14 @@ P1, P2, P3, P4 = (bytes([i]) * 16 for i in (1, 2, 3, 4)) + +def recorder(logged: list): + """A stand-in for rnfs.log.log appending (message, level, the type of the exception being handled) to logged.""" + def record(message, level=log.LOG_INFO): + """Record the line and whether it was logged inside an except.""" + logged.append((message, level, sys.exc_info()[0])) + return record + _saved_loglevel = RNS.loglevel @@ -102,6 +113,36 @@ def test_open_links_ranked(self): probed = {node for node, _ in self.client.requests[:2]} self.assertEqual(probed, {P3, P1}) # P2's path is unknown: last + def test_open_links_paths_looked_up_together(self): + """The open links' paths come from one path_infos call (one RPC on a shared instance), ranked as one by one; + a node it doesn't answer for falls back to path_info, and a failing path_infos to path_info alone.""" + self.config.max_parallel_providers = 2 + self.client.links = [P1, P2, P3] + one_by_one = paths({P1: (3, None), P3: (1, 9600)}) + calls = [] + + def together(node_ids): + """path_infos, answering for all but P3.""" + calls.append(list(node_ids)) + return {node_id: one_by_one(node_id) for node_id in node_ids if node_id != P3} + + self.fetcher.path_info = one_by_one + self.fetcher.path_infos = together + self.fetch_all(self.block.cid) + self.assertEqual(calls, [[P1, P2, P3]]) + self.assertEqual({node for node, _ in self.client.requests[:2]}, {P3, P1}) + + def test_path_infos_errors_fall_back_to_path_info(self): + def broken(node_ids): + """A batch lookup that fails.""" + raise RuntimeError("no RNS") + self.config.max_parallel_providers = 2 + self.client.links = [P1, P2, P3] + self.fetcher.path_info = paths({P1: (3, None), P3: (1, 9600)}) + self.fetcher.path_infos = broken + self.fetch_all(self.block.cid) + self.assertEqual({node for node, _ in self.client.requests[:2]}, {P3, P1}) + def test_path_info_errors_mean_unknown(self): def broken(node_id): """A lookup that fails.""" @@ -112,18 +153,147 @@ def broken(node_id): self.assertTrue(self.storage.is_complete(self.block.cid)) +HUB = "TCPInterface[Hub/10.0.0.1:4242]" +RADIO = "RNodeInterface[LoRa]" + + +class FakeInstance: + """A Reticulum attached to a shared instance: path answers come by RPC, counted in calls.""" + + is_connected_to_shared_instance = True + + def __init__(self, names: dict): + """The instance's next-hop interface name per node ({node_id: str(interface)}).""" + self.names = names + self.calls = {"next_hop": 0, "path_table": 0, "stats": 0} + self.table_fails = False + + def get_next_hop_if_name(self, node_id: bytes) -> str: + """One RPC: str() of the next-hop interface, "None" for a node the instance has no path to.""" + self.calls["next_hop"] += 1 + return self.names.get(node_id, "None") + + def get_path_table(self, max_hops=None) -> list[dict]: + """One RPC: the instance's whole path table.""" + self.calls["path_table"] += 1 + if self.table_fails: + raise ConnectionError("rpc failed") + return [{"hash": node_id, "timestamp": 0, "via": b"\0" * 16, "hops": 2, "expires": 0, "interface": name} + for node_id, name in self.names.items()] + + def get_interface_stats(self) -> dict: + """One RPC: the instance's interfaces.""" + self.calls["stats"] += 1 + return {"interfaces": [{"name": HUB, "short_name": "Hub", "bitrate": 10_000_000}, + {"name": RADIO, "short_name": "LoRa", "bitrate": 1200}]} + + +class RnsPathsTest(unittest.TestCase): + """RnsPaths attached to a shared instance: next-hop interfaces cached for at most PATH_CACHE (C1).""" + + def setUp(self): + """Local paths to P1-P3 (P4 has none), and an instance knowing P1 and P2's interfaces.""" + self.local = {P1: 1, P2: 3, P3: 2} # node -> hops in this process's own path table + for name, fake in (("has_path", lambda node_id: node_id in self.local), + ("hops_to", lambda node_id: self.local.get(node_id, mesh.PATHFINDER_M))): + patcher = mock.patch.object(RNS.Transport, name, fake) + patcher.start() + self.addCleanup(patcher.stop) + self.instance = FakeInstance({P1: HUB, P2: RADIO}) + self.clock = FakeClock() + self.paths = mesh.RnsPaths(self.instance, clock=self.clock) + + def test_info(self): + self.assertEqual(self.paths.info(P1), mesh.PathInfo(1, "Hub", 10_000_000.0)) + self.assertEqual(self.paths.info(P2), mesh.PathInfo(3, "LoRa", 1200.0)) + self.assertEqual(self.paths.info(P3), mesh.PathInfo(2, None, None)) # the instance has no path + self.assertEqual(self.paths.info(P4), mesh.NO_PATH) + self.assertEqual(self.instance.calls["next_hop"], 3) # none for P4: no local path, no question + + def test_rns_failure_logged_with_its_traceback(self): + """RNS failing to answer is a bug or an RNS change: logged at LOG_WARNING inside the except (so the debug + log has the traceback), at most once a minute; the path is unknown.""" + self.paths._failure_log = log.ThrottledLog(60.0, self.clock) + logged = [] + with mock.patch.object(RNS.Transport, "has_path", side_effect=TypeError("renamed")), \ + mock.patch("rnfs.log.log", recorder(logged)): + self.assertEqual(self.paths.info(P1), mesh.NO_PATH) + self.assertEqual(self.paths.info(P2), mesh.NO_PATH) + self.assertEqual([(level, raised) for _, level, raised in logged], [(RNS.LOG_WARNING, TypeError)]) + self.assertIn("renamed", logged[0][0]) + + def test_interface_reused_for_at_most_path_cache(self): + self.paths.info(P1) + self.instance.names[P1] = RADIO # the path moved to another interface + self.clock.now += mesh.PATH_CACHE + self.assertEqual(self.paths.info(P1).interface, "Hub") # stale, within PATH_CACHE + self.assertEqual(self.instance.calls["next_hop"], 1) + self.clock.now += 0.001 + self.assertEqual(self.paths.info(P1).interface, "LoRa") # never older than PATH_CACHE + self.assertEqual(self.instance.calls["next_hop"], 2) + + def test_path_and_hops_always_fresh(self): + self.paths.info(P1) + self.local[P1] = 4 + self.assertEqual(self.paths.info(P1).hops, 4) + del self.local[P1] + self.assertEqual(self.paths.info(P1), mesh.NO_PATH) + self.assertEqual(self.instance.calls["next_hop"], 1) + + def test_cache_bounded(self): + many = [bytes([5, i]) * 8 for i in range(20)] + self.local.update(dict.fromkeys(many, 1)) + with mock.patch.object(mesh, "MAX_CACHED_PATHS", 8): + for node_id in many: + self.paths.info(node_id) + self.assertEqual(len(self.paths._names), 8) + self.assertEqual(list(self.paths._names), many[-8:]) # the oldest are forgotten first + self.clock.now += mesh.PATH_CACHE + 1 + self.paths.info(P1) + self.assertEqual(list(self.paths._names), [P1]) # expired entries are dropped too + + def test_infos_reads_one_path_table(self): + ids = [P1, P2, P3, P4] + expected = {node_id: mesh.RnsPaths(FakeInstance(dict(self.instance.names))).info(node_id) for node_id in ids} + self.assertEqual(self.paths.infos(ids), expected) + # P3 has a local path the instance's table lacks: asked on its own, as info() would + self.assertEqual(self.instance.calls, {"next_hop": 1, "path_table": 1, "stats": 1}) + self.assertEqual(self.paths.infos(ids), expected) + self.assertEqual(self.instance.calls, {"next_hop": 1, "path_table": 1, "stats": 1}) # all cached + self.clock.now += mesh.PATH_CACHE + 1 + self.instance.names[P1] = RADIO + self.assertEqual(self.paths.infos(ids)[P1].interface, "LoRa") + self.assertEqual(self.instance.calls["path_table"], 2) + + def test_infos_without_the_table_asks_each_node(self): + self.instance.table_fails = True + self.assertEqual(self.paths.infos([P1, P2, P4]), + {P1: mesh.PathInfo(1, "Hub", 10_000_000.0), P2: mesh.PathInfo(3, "LoRa", 1200.0), + P4: mesh.NO_PATH}) + self.assertEqual(self.instance.calls["next_hop"], 2) + + def test_own_transport_needs_no_rpc(self): + interface = types.SimpleNamespace(name="LoRa", bitrate=1200) + self.instance.is_connected_to_shared_instance = False + with mock.patch.object(RNS.Transport, "next_hop_interface", lambda node_id: interface): + self.assertEqual(self.paths.infos([P1, P2]), {P1: mesh.PathInfo(1, "LoRa", 1200.0), + P2: mesh.PathInfo(3, "LoRa", 1200.0)}) + self.assertEqual(self.instance.calls, {"next_hop": 0, "path_table": 0, "stats": 0}) + + # Airtime policy class TokenBucketTest(unittest.TestCase): - """Continuous refill; a response goes if the bucket holds its size or is full.""" + """Continuous refill; a response goes if the bucket holds its size (beyond what is committed) or is full.""" - def test_spend_and_refill(self): + def test_reserve_and_refill(self): clock = FakeClock() bucket = mesh.TokenBucket(3600, 3600, clock) # 1 byte/s - self.assertTrue(bucket.take(3000)) - self.assertFalse(bucket.take(1000)) # 600 left + self.assertTrue(bucket.reserve(3000, 3000)) + self.assertFalse(bucket.reserve(1000, 1000)) # 600 left + self.assertAlmostEqual(bucket.available(), 600) # a refusal spends nothing clock.now += 400 - self.assertTrue(bucket.take(1000)) + self.assertTrue(bucket.reserve(1000, 1000)) self.assertAlmostEqual(bucket.available(), 0) clock.now += 10_000 self.assertEqual(bucket.available(), 3600) # never above capacity @@ -131,28 +301,55 @@ def test_spend_and_refill(self): def test_large_response_allowed_when_full(self): clock = FakeClock() bucket = mesh.TokenBucket(1000, 3600, clock) - self.assertTrue(bucket.take(5000)) # bigger than the whole budget, but the bucket was full + self.assertTrue(bucket.reserve(5000, 5000)) # bigger than the whole budget, but the bucket was full self.assertLess(bucket.available(), 0) - self.assertFalse(bucket.take(1)) # in debt + self.assertFalse(bucket.reserve(1, 1)) # in debt clock.now += 3600 * 4.9 - self.assertFalse(bucket.take(1000)) + self.assertFalse(bucket.reserve(1000, 1000)) clock.now += 3600 * 1.2 - self.assertTrue(bucket.take(5000)) # full again + self.assertTrue(bucket.reserve(5000, 5000)) # full again + + def test_reservation_then_charges(self): + """Admitted on its whole size, a response is charged only its reservation up front, then as it is sent, + into debt if need be.""" + clock = FakeClock() + bucket = mesh.TokenBucket(1000, 3600, clock) + self.assertTrue(bucket.reserve(800, 100)) + self.assertEqual(bucket.available(), 900) + bucket.charge(700) + self.assertEqual(bucket.available(), 200) + bucket.charge(500) + self.assertEqual(bucket.available(), -300) + self.assertFalse(bucket.reserve(1, 1)) + + def test_committed_counts_as_gone(self): + clock = FakeClock() + bucket = mesh.TokenBucket(1000, 3600, clock) + self.assertFalse(bucket.reserve(100, 10, committed=950)) # full, but others are owed most of it + self.assertEqual(bucket.available(), 1000) + self.assertTrue(bucket.reserve(5000, 10, committed=0)) # full and nothing owed: any one response + self.assertEqual(bucket.available(), 990) + self.assertFalse(bucket.reserve(50, 10, committed=950)) + self.assertTrue(bucket.reserve(40, 10, committed=950)) def test_give_back(self): clock = FakeClock() bucket = mesh.TokenBucket(1000, 3600, clock) - bucket.take(800) + bucket.reserve(800, 800) bucket.give_back(800) self.assertEqual(bucket.available(), 1000) + bucket.reserve(800, 100) + bucket.charge(700) + bucket.give_back(5000) + self.assertEqual(bucket.available(), 1000) # never above capacity def test_thread_safe(self): bucket = mesh.TokenBucket(10_000, 1e12) taken = [] def spend(): - """Take 1 byte many times.""" - taken.append(sum(bucket.take(1) for _ in range(5000))) + """Reserve 1 byte many times.""" + taken.append(sum(bucket.reserve(1, 1) for _ in range(5000))) threads = [threading.Thread(target=spend) for _ in range(4)] for thread in threads: @@ -224,7 +421,9 @@ def test_defaults_refuse_nothing(self): policy = mesh.AirtimePolicy(Config()) self.assertIsNone(policy.refusal([])) self.assertEqual(policy.limit(12345), 12345) - self.assertTrue(policy.spend(10**12)) + self.assertTrue(policy.reserve(10**12, 10**12, committed=10**12)) + policy.charge(10**12) + policy.refund(10**12) self.assertIsNone(policy.remaining()) def test_quiet_hours(self): @@ -238,13 +437,33 @@ def test_budget_limit(self): clock = FakeClock() policy = mesh.AirtimePolicy(Config(upload_budget=100 * KIB), clock) self.assertEqual(policy.limit(500 * KIB), 500 * KIB) # full: any one response may go - self.assertTrue(policy.spend(60 * KIB)) + self.assertTrue(policy.reserve(60 * KIB, 60 * KIB)) self.assertEqual(policy.limit(500 * KIB), 40 * KIB - 1 * KIB) # what's left, less framing room - self.assertTrue(policy.spend(40 * KIB)) + self.assertEqual(policy.limit(500 * KIB, committed=10 * KIB), 30 * KIB - 1 * KIB) # less what's owed + self.assertFalse(policy.reserve(41 * KIB, 41 * KIB)) + self.assertTrue(policy.reserve(40 * KIB, 40 * KIB)) + self.assertEqual(policy.remaining(), 0) self.assertEqual(policy.refusal([]), mesh.REFUSED_BUDGET) clock.now += 36 # 1% of an hour self.assertIsNone(policy.refusal([])) + def test_reserve_charge_refund(self): + """A response is admitted on its whole size, charged its reservation, then as it is sent; what wasn't + sent is refunded.""" + clock = FakeClock() + policy = mesh.AirtimePolicy(Config(upload_budget=100 * KIB), clock) + self.assertTrue(policy.reserve(60 * KIB, 1 * KIB)) + self.assertEqual(policy.remaining(), 99 * KIB) + policy.charge(29 * KIB) + self.assertEqual(policy.remaining(), 70 * KIB) + policy.charge(0) + policy.charge(-5) # nothing: only what was sent is charged + self.assertEqual(policy.remaining(), 70 * KIB) + policy.refund(10 * KIB) + self.assertEqual(policy.remaining(), 80 * KIB) + self.assertFalse(policy.reserve(50 * KIB, 1 * KIB, committed=40 * KIB)) # 80 KiB less 40 KiB owed + self.assertEqual(policy.remaining(), 80 * KIB) # refused: nothing charged + def test_validate(self): for settings in ({"upload_budget": 0}, {"upload_budget": True}, {"upload_budget": 1.5}, {"serve_interfaces": "LoRa"}, {"no_serve_interfaces": [""]}, {"quiet_hours": "late"}): @@ -339,7 +558,7 @@ def test_upload_budget(self): def test_missing_blocks_cost_nothing(self): srv = self.make_server(upload_budget=150 * KIB) - srv.airtime.bucket.take(149 * KIB) + srv.airtime.bucket.charge(149 * KIB) answer = self.ask(BLOCKS_PATH, protocol.encode_blocks(MAX_RESPONSE_SIZE, [block_of(10, 99)[0]]), self.connect()) self.assertEqual(protocol.parse_blocks_response(answer, 1), [None]) @@ -439,6 +658,77 @@ def test_lists_table_links_and_counted_nodes(self): lines = peers_feature.format_peers(list(listed.values()), 0) self.assertEqual(len(lines), 5) + def test_listing_failures_logged_with_their_traceback(self): + """What a listing can't read is left out and logged at LOG_WARNING inside the except (so the debug log has + the traceback), at most once a minute for the feature.""" + clock = FakeClock() + self.feature._failure_log = log.ThrottledLog(60.0, clock) + logged = [] + + def broken(*args): + """A bug or an RNS change.""" + raise TypeError("renamed") + self.node.client.open_links = broken + with mock.patch("rnfs.log.log", recorder(logged)): + self.assertEqual(self.feature.peers(), []) + self.feature.peers() # within the minute: counted, not logged + self.assertEqual([(level, raised) for _, level, raised in logged], [(RNS.LOG_WARNING, TypeError)]) + self.assertIn("open links", logged[0][0]) + del self.node.client.open_links + self.node.client.rate_estimate = broken + clock.now += 60 + with mock.patch("rnfs.log.log", recorder(logged)): + (listed,) = self.feature.peers() + self.assertIsNone(listed["rate"]) + self.assertEqual([(level, raised) for _, level, raised in logged[1:]], [(RNS.LOG_WARNING, TypeError)]) + self.assertIn("rate estimate", logged[1][0]) + self.assertIn("(1 more since the last)", logged[1][0]) + + def test_paths_from_one_lookup(self): + asked = [] + + class Paths: + """RnsPaths' bulk lookup, recording what it is asked.""" + + def infos(self, node_ids): + """A path to P1 only.""" + asked.append(list(node_ids)) + return {node_id: mesh.PathInfo(2, "LoRa", 1200.0) if node_id == P1 else mesh.NO_PATH + for node_id in node_ids} + self.node.paths = Paths() + self.node.routing.heard(P1) + listed = {peer["node_id"]: peer for peer in self.feature.peers()} + self.assertEqual(asked, [[P1, P3]]) + self.assertEqual([listed[P1.hex()][key] for key in ("hops", "interface", "bitrate")], [2, "LoRa", 1200.0]) + self.assertIsNone(listed[P3.hex()]["hops"]) + self.assertEqual(list(listed), [P1.hex(), P3.hex()]) # nearest first + + def test_announces_counted_once_by_the_node(self): + # S8: the node's rnfs.dht announce handler counts each announce; the feature registers none of its own + self.node.reticulum = object() + self.addCleanup(setattr, self.node, "reticulum", None) + with mock.patch.object(RNS.Transport, "register_announce_handler") as register: + self.feature.start() + register.assert_not_called() + handler = DhtAnnounceHandler(self.node._heard_server, self.node._heard_client, self.node.routing.is_banned, + self.node.node_id) # as RnfsNode._start_network wires it + self.node.routing.ban(P4) + with mock.patch.object(self.feature.traffic, "heard", wraps=self.feature.traffic.heard) as heard: + for node_id, server in ((P1, True), (P2, False), (P4, True), (OWN, True)): + handler.received_announce(node_id, None, encode_mode(server)) + self.assertEqual(heard.call_args_list, [mock.call(P1, "server"), mock.call(P2, "client")]) + listed = {peer["node_id"]: peer for peer in self.feature.peers()} + self.assertEqual((listed[P1.hex()]["mode"], listed[P1.hex()]["in_table"]), ("server", True)) + self.assertEqual((listed[P2.hex()]["mode"], listed[P2.hex()]["in_table"]), ("client", False)) + self.assertIsNotNone(listed[P2.hex()]["last_heard"]) + + def test_recalled_mode_doesnt_mark_used(self): + # A listing is a check, not a contact (C2) + self.node.reticulum = object() + with mock.patch.object(RNS.Identity, "recall_app_data", return_value=None) as recall: + self.assertIsNone(self.feature._recalled_mode(P1)) + recall.assert_called_once_with(P1, _no_use=True) + def test_status_and_stats(self): status = self.node.status() for key in ("bytes_sent", "bytes_received", "blocks_refused_budget", "blocks_refused_interface", diff --git a/tests/test_migrate_features.py b/tests/test_migrate_features.py index 2f43501..89c3349 100644 --- a/tests/test_migrate_features.py +++ b/tests/test_migrate_features.py @@ -384,7 +384,7 @@ def test_reconcile_after_the_upgrade(self): write_legacy(self.state, "pins.json", {"pins": {str(A): [source], str(C): [source]}}) write_legacy(self.state, self.FILE, text=inbox) node = self.node() - messages = node.feature("lxmf").inbox_list() + messages = node.feature("lxmf").inbox.entries() # the records, as imported self.assertEqual(node.storage.pinned_by(source), {A, C} if kept else {A}) if kept: self.assertEqual([m["auto"] for m in messages], [[{"state": "pinned", "cid": str(A)}, diff --git a/tests/test_names.py b/tests/test_names.py index 9fb1b93..4e6dfc6 100644 --- a/tests/test_names.py +++ b/tests/test_names.py @@ -84,6 +84,7 @@ def __init__(self): self.app_data: dict[bytes, bytes] = {} self.heard: dict[bytes, float] = {} self.emitted: dict[bytes, float] = {} + self.stored_emitted: dict[bytes, float] = {} # when the announce RNS 'stored' last was emitted, if known self.path_requests = [] self.on_request_path = None @@ -126,12 +127,19 @@ def emitted_time(self, name): """When the newest announce RNS 'took a path from' was emitted (None: tests not setting emitted).""" return self.emitted.get(name) + def recalled_emission(self, name, app_data): + """When the announce RNS 'stored' last was emitted (None if the test didn't say), and that nothing shows it + isn't the path's record: an older one is a suspected replay all the same (DHT-2).""" + return self.stored_emitted.get(name), True + def store(self, name, app_data, heard, emitted=None): """Make RNS 'remember' an announce, as validate_announce does, without calling handlers. An announce emitted at emitted joins the path's, which keeps the newest emission: repeating an old one changes nothing.""" self.app_data[name] = app_data self.heard[name] = heard + self.stored_emitted.pop(name, None) if emitted is not None: + self.stored_emitted[name] = emitted self.emitted[name] = max(emitted, self.emitted.get(name, emitted)) def announce(self, name, app_data, heard, emitted=None): @@ -190,8 +198,10 @@ def setUp(self): self.manager = self.make_manager() def make_manager(self) -> NameManager: - """A manager over the same directory (as after a restart), sharing the fakes; closed after the test.""" - manager = NameManager(self.config, self.pins, network=self.network, clock=self.clock, monotonic=self.clock) + """A manager over the same directory (as after a restart), sharing the fakes; closed after the test. Its + refreshes wait in virtual time: path_timeout passes at once unless an answer has already arrived.""" + manager = NameManager(self.config, self.pins, network=self.network, clock=self.clock, monotonic=self.clock, + wait=lambda event, timeout: event.is_set()) self.addCleanup(manager.close) return manager @@ -780,6 +790,7 @@ def test_heard_time_in_the_future_isnt_fresh(self): name = other_name(1) self.network.announce(name, encode_record(1, CID_A), self.clock.now) self.manager.resolve(name, persist=True) + self.clock.now += 60 # past the first resolve's refresh, which a resolve reuses for max_age (E1) with self.manager._lock: # as loaded from a record saved while the clock ran ahead self.manager._held[name] = names.Record(1, CID_A, self.clock.now + 10**6) self.network.path_requests.clear() @@ -801,6 +812,115 @@ def test_resolve_rejects_bad_names(self): self.manager.resolve(b"short", persist=False) +class RefreshCacheTest(NamesTestCase): + """A resolve's network refresh is reused for max_age and shared by the resolves running alongside it (E1): a + name whose owner is offline, with keepers answering.""" + + AGE = 60.0 # the gateway's max_age (node.GATEWAY_NAME_AGE) + + def setUp(self): + """A started manager with keepers holding sequence 4; RNS remembers the owner's last announce, of 3, + emitted two days ago (it never shows current).""" + super().setUp() + self.manager.start() + self.name = other_name(1) + emitted = self.clock.now - 2 * 24 * 3600 + self.network.store(self.name, encode_record(3, CID_A), emitted, emitted=emitted) + self.keepers = self.manager.keepers = FakeKeepers() + self.keepers.answers[self.name] = [(4, CID_B, emitted + 10)] + + def gateway(self) -> dict | None: + """A gateway resolve.""" + return self.manager.resolve(self.name, persist=False, max_age=self.AGE) + + def work(self) -> tuple[int, int]: + """(path requests, keeper queries) so far.""" + return len(self.network.path_requests), len(self.keepers.queried) + + def test_gateway_resolves_reuse_a_refresh_for_max_age(self): + """Gateway resolves within max_age of a refresh go to neither the network nor keepers; after it they do.""" + for _ in range(5): + self.assertEqual(self.gateway()["sequence"], 4) + self.clock.now += self.AGE / 5 - 1 + self.assertEqual(self.work(), (1, 1)) + self.clock.now += 5 + self.assertEqual(self.gateway()["sequence"], 4) + self.assertEqual(self.work(), (2, 2)) + + def test_cli_resolves_always_refresh(self): + """A resolve with max_age 0 (the CLI's) never reuses a refresh, nor does it leave one the gateway can't.""" + for _ in range(3): + self.assertEqual(self.manager.resolve(self.name, persist=True)["sequence"], 4) + self.assertEqual(self.work(), (3, 3)) + self.assertEqual(self.gateway()["sequence"], 4) + self.assertEqual(self.work(), (3, 3)) + + def test_nothing_found_is_reused_too(self): + """A name nothing answers for resolves to None, and again to None without the network for max_age.""" + self.keepers.answers.clear() + self.network.app_data.clear() + self.assertIsNone(self.gateway()) + self.assertIsNone(self.gateway()) + self.assertEqual(self.work(), (1, 1)) + + def test_a_record_evicted_since_is_looked_for_again(self): + """A refresh that found a record isn't reused once the bounded table has dropped it.""" + self.assertEqual(self.gateway()["sequence"], 4) + for i in range(2, 2 + self.config.name_table_size): + self.manager.offer(other_name(i), 1, CID_A, self.clock.now) + self.assertIsNone(self.manager.record(self.name)) + self.assertEqual(self.gateway()["sequence"], 4) + self.assertEqual(self.work(), (2, 2)) + + def test_a_suspected_replay_is_not_reused(self): + """A refresh leaving a record that is a suspected replay (DHT-2: older than the path's announce) isn't + reused, so each resolve asks keepers again.""" + self.keepers.answers.clear() + self.network.store(self.name, encode_record(3, CID_A), self.clock.now, emitted=self.clock.now - 3600) + self.network.emitted[self.name] = self.clock.now - 60 # the path holds a newer announce + for _ in range(3): + self.assertEqual(self.gateway()["sequence"], 3) + self.assertEqual(self.work(), (3, 3)) + + def test_simultaneous_resolves_share_one_refresh(self): + """Resolves of a name being refreshed, the CLI's included, wait for that refresh and take its record: one + path request and one keeper query for all of them. Each persists as it asked.""" + gate = threading.Lock() + gate.acquire() + keepers = self.keepers + real_query = keepers.query + + def query(name): + """The keepers' answer, once the test opens the gate.""" + with gate: + return real_query(name) + + keepers.query = query + joined, real_wait = [], threading.Event.wait + + def wait(event, timeout=None): + """Event.wait, noting a resolve waiting for a refresh running.""" + if event in self.manager._refreshing.values(): + joined.append(event) + return real_wait(event, timeout) + + results = {} + persist = {i: i == 0 for i in range(8)} # resolve 0 is the CLI's; the rest the gateway's + threads = [threading.Thread(target=lambda i=i: results.__setitem__(i, self.manager.resolve( + self.name, persist=persist[i], max_age=0.0 if persist[i] else self.AGE))) for i in range(8)] + with mock.patch.object(threading.Event, "wait", wait): + for thread in threads: + thread.start() + wait_for(lambda: len(joined) == 7, what="seven resolves waiting for the eighth's refresh") + gate.release() + for thread in threads: + thread.join(10) + self.assertEqual(self.work(), (1, 1)) + self.assertEqual({i: result["sequence"] for i, result in results.items()}, {i: 4 for i in range(8)}) + self.assertEqual(self.manager._held[self.name].sequence, 4) # the CLI's resolve persisted it + self.assertEqual(self.manager._refreshing, {}) + + class StateTest(NamesTestCase): """The published and held records in the state database: what a save writes, failed commits, damaged rows.""" @@ -983,6 +1103,30 @@ def test_order_and_raises_survive_a_restart(self): self.assertEqual([(name, sequence) for name, (sequence, _) in restarted._floor.items()], [(other_name(2), 9), (other_name(3), 1), (other_name(1), 2)]) + def test_round_trip(self): + """A restart loads the floor as it was saved, each name with its (sequence, resolved, used) value in the + same order, after resolves, raises, a name held for the user and the highest sequence; a resolve after it + goes last.""" + for i in (1, 2, 3, 4): + self.clock.now += 1 + self.resolve_for_gateway(i, sequence=i) + self.resolve_for_gateway(2, sequence=5) # resolved again: last, raised + self.network.announce(other_name(3), encode_record(9, CID_B), self.clock.now) # raised in place + self.manager.offer(other_name(1), names.MAX_SEQUENCE, CID_B, self.clock.now) # a keeper's answer + self.manager.resolve(other_name(4), persist=True, max_age=60) # held: leaves the floor + self.manager.stop() + saved = list(self.manager._floor._entries.items()) + self.assertEqual([(name, sequence) for name, (sequence, _, _) in saved], + [(other_name(1), names.MAX_SEQUENCE), (other_name(3), 9), (other_name(2), 5)]) + restarted = self.make_manager() + self.assertEqual(list(restarted._floor._entries.items()), saved) + self.assertEqual(dict(restarted._floor), {name: (sequence, at) for name, (sequence, at, _) in saved}) + self.network.announce(other_name(5), encode_record(1, CID_A), self.clock.now) + restarted.resolve(other_name(5), persist=False, max_age=60) + self.assertEqual(list(restarted._floor), [other_name(1), other_name(3), other_name(2), other_name(5)]) + restarted.stop() + self.assertEqual(list(self.make_manager()._floor), [other_name(1), other_name(3), other_name(2), other_name(5)]) + def test_save_writes_only_changed_rows(self): """A resolve adding a name to the floor writes its row alone, as a save after an announce raised one does.""" for i in range(1, 6): @@ -1030,7 +1174,7 @@ def test_a_raise_soon_after_a_save_waits_for_the_interval(self): self.resolve_for_gateway(1) self.network.announce(other_name(1), encode_record(4, CID_B), self.clock.now) self.assertEqual(self.saved_sequence(1), [("1",)]) - self.assertIsNotNone(self.manager._floor_timer) + self.assertIsNotNone(self.manager._floor._timer) wait_for(lambda: self.saved_sequence(1) == [("4",)], timeout=5, what="the timer's save") def test_no_timer_after_close(self): @@ -1040,7 +1184,7 @@ def test_no_timer_after_close(self): self.manager.close() self.clock.now += names.FLOOR_SAVE_INTERVAL self.manager.offer(other_name(1), 5, CID_B, self.clock.now) - self.assertIsNone(self.manager._floor_timer) + self.assertIsNone(self.manager._floor._timer) def test_written_without_the_lock(self): """The floor's commit runs without the manager's lock: resolves and announces needn't wait for it.""" @@ -1098,7 +1242,7 @@ def test_name_resolved_for_the_user_leaves_the_floor(self): self.assertEqual(self.saved(), [other_name(1), other_name(2)]) self.manager.resolve(other_name(1), persist=True, max_age=60) self.assertNotIn(other_name(1), self.manager._floor) - self.assertNotIn(other_name(1), self.manager._floor_used) + self.assertNotIn(other_name(1), self.manager._floor._entries) self.manager.announce_due() self.assertEqual(self.saved(), [other_name(2)]) diff --git a/tests/test_node.py b/tests/test_node.py index 13e67fd..742e3dc 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -15,7 +15,9 @@ from rnfs.config import Config from rnfs.constants import CHUNK_SIZE, DEFAULT_BOOTSTRAP_NODES, K +from rnfs.content import dag, seal from rnfs.content.cid import RAW, Cid +from rnfs.content.seal import SEGMENT_SIZE from rnfs.content.storage import remote_source from rnfs import fetcher from rnfs.errors import Denied, InvalidDag, NotFound, RnfsError, StorageFull, TooLarge @@ -61,6 +63,10 @@ def rate_estimate(self, node_id): """No estimates.""" return None + def link_mdu(self, node_id): + """No open links, so no MDU known.""" + return None + def find(self, node_id, key): """Ask a node's backend as /find would.""" node = self.nodes.get(node_id) @@ -341,6 +347,185 @@ def test_invalid_root_reported(self): self.b.info(str(Cid.for_block(RAW, b"bad"))) +class SealedCheckTest(NodeTest): + """Whether a file is sealed is read from its first bytes once: plain files are remembered (_not_sealed), and + read_file passes what its info opened on to its read.""" + + def setUp(self): + """Node A, with an identity to seal for itself, counting seal.open_sealed's calls (each reads a file's + first bytes).""" + super().setUp() + self.a.identity = RNS.Identity() + opened = mock.patch.object(seal, "open_sealed", wraps=seal.open_sealed) + self.opened = opened.start() + self.addCleanup(opened.stop) + + def test_plain_files_are_checked_once(self): + """A plain file's first bytes are read once, whatever reads it afterwards; each target separately.""" + data = os.urandom(3 * CHUNK_SIZE) + cid = self.a.add_stream(io.BytesIO(data))["cid"] + self.assertEqual(b"".join(self.a.read(cid, CHUNK_SIZE, 10)), data[CHUNK_SIZE:CHUNK_SIZE + 10]) + self.assertEqual(self.opened.call_count, 1) + info, chunks = self.a.read_file(cid, 5, 10) + self.assertEqual((info["size"], b"".join(chunks)), (len(data), data[5:15])) + self.assertNotIn("sealed", self.a.info(cid)) + self.assertEqual(b"".join(self.a.read(cid)), data) + self.assertEqual(self.opened.call_count, 1) + # The same file under a directory is another target (its expectation differs): checked once too + root = self.a.add_stream(io.BytesIO(data), name="f.bin")["cid"] + for _ in range(2): + self.assertEqual(b"".join(self.a.read(f"{root}/f.bin", 0, 4)), data[:4]) + self.assertEqual(self.opened.call_count, 2) + + def test_sealed_files_are_opened_each_time_once_per_read_file(self): + """A sealed file isn't remembered; read_file opens it once (for its info), not again for its bytes.""" + data = os.urandom(SEGMENT_SIZE + 5) + cid = self.a.add_stream(io.BytesIO(data), to_self=True)["cid"] + for _ in range(2): + self.assertEqual(b"".join(self.a.read(cid, 3, 4)), data[3:7]) + self.assertEqual(self.opened.call_count, 2) + self.assertEqual(len(self.a._not_sealed), 0) + for count in (3, 4): + info, chunks = self.a.read_file(cid, SEGMENT_SIZE - 1, 3) + self.assertEqual((info["sealed"], info["recipient"], info["size"]), (True, True, len(data))) + self.assertEqual(b"".join(chunks), data[SEGMENT_SIZE - 1:SEGMENT_SIZE + 2]) + self.assertEqual(self.opened.call_count, count) + + def test_read_file_of_a_file_sealed_for_others(self): + """What info opened isn't unlocked for a non-recipient: the read raises NotRecipient, and for a denied + file Denied first, as when the read opened it itself.""" + other = RNS.Identity() + data = b"for someone else" * 10 + cid = self.a.add_stream(io.BytesIO(data), to=[other.get_public_key().hex()])["cid"] + info, chunks = self.a.read_file(cid) + self.assertEqual((info["sealed"], info["recipient"]), (True, False)) + with self.assertRaises(seal.NotRecipient): + next(chunks) + self.a.identity = None + info, chunks = self.a.read_file(cid) + with self.assertRaisesRegex(seal.NotRecipient, "no identity"): + next(chunks) + stored = b"".join(self.a.read(cid, raw=True)) + self.a.deny.add(["sha256:" + hashlib.sha256(stored).hexdigest()]) + info, chunks = self.a.read_file(cid) + with self.assertRaises(Denied): + next(chunks) + + def test_the_memo_is_bounded(self): + """At most NOT_SEALED_MEMO targets are remembered, the least recently used forgotten first.""" + cids = [self.a.add_stream(io.BytesIO(b"file %d" % i))["cid"] for i in range(3)] + with mock.patch.object(node_module, "NOT_SEALED_MEMO", 2): + for cid in cids[:2]: + b"".join(self.a.read(cid)) + b"".join(self.a.read(cids[0])) # used again: now the most recent + b"".join(self.a.read(cids[2])) + self.assertEqual([str(expect.cid) for expect in self.a._not_sealed], [cids[0], cids[2]]) + self.assertEqual(self.opened.call_count, 3) + b"".join(self.a.read(cids[1])) # forgotten: read again + self.assertEqual(self.opened.call_count, 4) + + def test_a_deny_list_change_empties_the_memo(self): + """Reading the first bytes checks the deny list: once their chunk is denied, a read of a remembered file + raises Denied as it did before the file was remembered, though the range read doesn't cover it.""" + data = os.urandom(3 * CHUNK_SIZE) + cid = self.a.add_stream(io.BytesIO(data))["cid"] + self.assertEqual(b"".join(self.a.read(cid, 2 * CHUNK_SIZE, 4)), data[2 * CHUNK_SIZE:2 * CHUNK_SIZE + 4]) + first = dag.decode(self.a.storage.get(Cid.parse(cid))).children[0].cid + self.a.deny.add([str(first)]) + with self.assertRaises(Denied): + b"".join(self.a.read(cid, 2 * CHUNK_SIZE, 4)) + self.assertEqual(self.opened.call_count, 2) + self.assertEqual(len(self.a._not_sealed), 0) + + +class PinLsMemoTest(NodeTest): + """pin_ls remembers each complete pin's type, name from inside and sealed status, rather than reading its + blocks at every call; what it gives is what reading them gives.""" + + def setUp(self): + """Node A, with an identity, counting _pin_shape's calls (each reads the pin's root block).""" + super().setUp() + self.a.identity = RNS.Identity() + shape = mock.patch.object(self.a, "_pin_shape", wraps=self.a._pin_shape) + self.shape = shape.start() + self.addCleanup(shape.stop) + + def fresh(self) -> list[dict]: + """pin_ls as it is without the memo.""" + with mock.patch.object(node_module, "PIN_MEMO", 0): + saved, self.a._pin_memo = self.a._pin_memo, {} + try: + return self.a.pin_ls() + finally: + self.a._pin_memo = saved + + def test_complete_pins_are_read_once(self): + """A second pin_ls reads no pin's blocks again and answers the same, for files, directories and sealed + files; pin_ls(cid) uses the memo too.""" + cids = [self.a.add_stream(io.BytesIO(b"small"))["cid"], + self.a.add_stream(io.BytesIO(os.urandom(2 * CHUNK_SIZE)))["cid"], + self.a.add_stream(io.BytesIO(b"inside"), name="one.txt")["cid"], + self.a.add_stream(io.BytesIO(b"secret"), to_self=True)["cid"]] + first = self.a.pin_ls() + self.assertEqual(self.shape.call_count, 4) + self.assertEqual({entry["cid"]: (entry["type"], entry["sealed"]) for entry in first}, + {cids[0]: ("file", False), cids[1]: ("file", False), cids[2]: ("directory", False), + cids[3]: ("file", True)}) + self.assertEqual(self.a.pin_ls(), first) + self.assertEqual(self.a.pin_ls(cids[2]), [entry for entry in first if entry["cid"] == cids[2]]) + self.assertEqual(self.shape.call_count, 4) + self.assertEqual(first, self.fresh()) + + def test_released_pins_are_forgotten(self): + """pin_ls forgets the pins released since the last.""" + keep, drop = (self.a.add_stream(io.BytesIO(b"%d" % i))["cid"] for i in range(2)) + self.a.pin_ls() + self.assertEqual({str(c) for c in self.a._pin_memo}, {keep, drop}) + self.a.pin_rm(drop) + self.assertEqual([entry["cid"] for entry in self.a.pin_ls()], [keep]) + self.assertEqual({str(c) for c in self.a._pin_memo}, {keep}) + + def test_incomplete_pins_are_read_each_time(self): + """A pin missing blocks isn't remembered: once they arrive it is seen complete, its sealed status known.""" + data = os.urandom(2 * CHUNK_SIZE) + cid = Cid.parse(self.b.add_stream(io.BytesIO(data))["cid"]) + self.a.storage.put(cid, self.b.storage.get(cid)) + self.a.storage.pin(cid, "user") + for _ in range(2): + self.assertEqual([(e["complete"], e["type"], e["sealed"]) for e in self.a.pin_ls()], [(False, "file", None)]) + self.assertEqual(self.a._pin_memo, {}) + for child in dag.decode(self.b.storage.get(cid)).child_cids: + self.a.storage.put(child, self.b.storage.get(child)) + self.assertEqual([(e["complete"], e["type"], e["sealed"]) for e in self.a.pin_ls()], [(True, "file", False)]) + self.assertIn(cid, self.a._pin_memo) + + def test_a_pin_that_loses_a_block_is_read_again(self): + """A remembered pin whose first chunk is found damaged (and deleted) is incomplete again: its sealed + status is unknown, as reading its blocks says.""" + cid = Cid.parse(self.a.add_stream(io.BytesIO(os.urandom(2 * CHUNK_SIZE)))["cid"]) + self.assertEqual([(e["complete"], e["sealed"]) for e in self.a.pin_ls()], [(True, False)]) + first = dag.decode(self.a.storage.get(cid)).child_cids[0] + path = self.a.storage.blocks.path(first) + os.chmod(path, 0o600) + with open(path, "r+b") as f: + f.write(b"damaged") + self.assertIsNone(self.a.storage.get(first, verify=True)) # deleted + self.assertEqual([(e["complete"], e["type"], e["sealed"]) for e in self.a.pin_ls()], [(False, "file", None)]) + self.assertEqual(self.a.pin_ls(), self.fresh()) + + def test_the_memo_is_bounded(self): + """At most PIN_MEMO pins are remembered (the first met); the others are read each time, answering the + same.""" + for i in range(3): + self.a.add_stream(io.BytesIO(b"%d" % i)) + with mock.patch.object(node_module, "PIN_MEMO", 2): + first = self.a.pin_ls() + self.assertEqual(len(self.a._pin_memo), 2) + self.assertEqual(self.a.pin_ls(), first) + self.assertEqual(self.shape.call_count, 4) # 3, then the one past the bound again + self.assertEqual(first, self.fresh()) + + class BackendTest(NodeTest): """What the request server asks of the node.""" @@ -388,6 +573,15 @@ def test_is_member(self): self.assertTrue(node.backend.is_member(IDENTITY)) self.assertFalse(node.backend.is_member(bytes(16))) + def test_identity_checks_dont_mark_destinations_used(self): + # Checks, not contacts (C2): recalled with _no_use, which skips RNS's "used" RPC on a shared instance + node = self.make_node("p", bytes([0xD] * 16), private=True, members=[IDENTITY.hex()]) + member = mock.Mock(hash=IDENTITY) + with mock.patch.object(RNS.Identity, "recall", return_value=member) as recall: + self.assertTrue(node._is_member_node(A_ID)) + self.assertTrue(node_module._rns_is_known(B_ID)) + self.assertEqual(recall.call_args_list, [mock.call(A_ID, _no_use=True), mock.call(B_ID, _no_use=True)]) + def wait_for(self, condition, timeout=10): """Poll until condition() is true.""" deadline = time.monotonic() + timeout @@ -471,7 +665,7 @@ def test_rnfs_dir_of_another_user_is_refused(self): os.mkdir(theirs) with mock.patch("os.geteuid", return_value=os.geteuid() + 1): with self.assertRaisesRegex(RnfsError, "belongs to another user"): - node_module._make_private_dirs(Config(rnfs_dir=theirs)) + node_module._make_private_dirs(theirs) self.assertFalse(os.path.exists(os.path.join(theirs, "state"))) # nothing written there @unittest.skipUnless(POSIX_MODES, "Unix permission bits") @@ -481,7 +675,7 @@ def test_rnfs_dir_that_cant_be_made_private_is_refused(self): os.chmod(loose, 0o777) with mock.patch("os.chmod", side_effect=PermissionError("not permitted")): with self.assertRaisesRegex(RnfsError, "couldn't make .* private"): - node_module._make_private_dirs(Config(rnfs_dir=loose)) + node_module._make_private_dirs(loose) def test_no_permission_fix_on_windows(self): # Windows' st_mode always shows others allowed: no chmod and no notice at every start @@ -490,7 +684,7 @@ def test_no_permission_fix_on_windows(self): with mock.patch.object(fsutil, "IS_WINDOWS", True), mock.patch("os.chmod") as chmod, \ mock.patch.object(node_module, "log") as log, \ mock.patch("os.stat", return_value=mock.Mock(st_mode=0o40777)): - node_module._make_private_dirs(Config(rnfs_dir=loose)) + node_module._make_private_dirs(loose) chmod.assert_not_called() log.assert_not_called() @@ -746,6 +940,90 @@ def test_failed_fetched_save_is_tried_again(self): [(Cid.for_block(RAW, b"other").binary,)]) +class StopTest(NodeTest): + """stop() runs every step whatever raises, and waits for the threads it ends (audit XC5).""" + + def test_a_raising_service_still_closes_the_state_and_releases_the_destination(self): + """An embedded node whose announcer fails to stop: its state database is still closed and its rnfs.dht + destination still released, so a later node in the process can use both.""" + node = self.make_node("embedded", N1) + node.embedded = True + node.destination = mock.Mock(links=[]) + node.announcer = mock.Mock(**{"stop.side_effect": RuntimeError("announcer broke")}) + with mock.patch.object(RNS.Transport, "deregister_destination") as deregister, \ + mock.patch("rnfs.node.log") as log: + node.stop() + deregister.assert_called_once_with(node.destination) + self.assertIsNone(node._db._database) + self.assertTrue(any("announcer broke" in call.args[0] for call in log.call_args_list)) + self.make_node("embedded", N1).stop() + + def test_links_close_before_the_threads_are_waited_for(self): + """A background thread stuck on the network (the publisher connecting to a node that has gone) is freed by + closing the links, and its last save lands while the state database is still open (found by the interop + harness: `publishing failed: StateDbError(... is closed)` at stop).""" + node = self.make_node("stuck", N1) + freed = threading.Event() + saved = [] + + def stuck_publisher(): + """Waits on the network until the links close, then saves, as the publisher's loop does.""" + freed.wait(10) + node.publisher.save() + saved.append(node._db._database is not None) + + node.links = mock.Mock(**{"close_all.side_effect": freed.set}) + thread = threading.Thread(target=stuck_publisher, daemon=True) + thread.start() + node.threads.append(thread) + with mock.patch("rnfs.node.log") as log: + node.stop() + self.assertEqual(saved, [True]) + self.assertFalse([call for call in log.call_args_list if "failed" in call.args[0]]) + + def test_a_raising_feature_or_save_skips_nothing(self): + """A feature's stop and a store's save raising something other than OSError: the other stores are still + saved and the state closed.""" + node = self.make_node("broken", N1) + cid = node.add_stream(io.BytesIO(b"cached"), pin=False)["cid"] + node._note_fetched(Cid.parse(cid)) + with mock.patch.object(node.feature("follow"), "stop", side_effect=RuntimeError("follow broke")), \ + mock.patch.object(node.storage, "save", side_effect=ValueError("bad row")), \ + mock.patch("rnfs.node.log") as log: + node.stop() + logged = [call.args[0] for call in log.call_args_list] + self.assertTrue(any("follow broke" in line for line in logged), logged) + self.assertTrue(any("bad row" in line for line in logged), logged) + self.assertEqual(state_rows(node.config.state_dir, "SELECT cid FROM fetched"), [(Cid.parse(cid).binary,)]) + self.assertIsNone(node._db._database) + + def test_stop_cancels_remote_pin_fetches_and_waits_for_their_workers(self): + """A remote pin being fetched when the node stops: its fetch watches stop_event, stop waits for its worker, + and the pin is saved as waiting, for the next start.""" + cid = Cid.parse(self.a.add_stream(io.BytesIO(os.urandom(2 * CHUNK_SIZE)))["cid"]) + self.advertise(str(cid)) + asked, go = threading.Event(), threading.Event() + self.addCleanup(go.set) + + def held(node_id, cids, on_progress=None): + """Hold every /blocks until the test ends, then fail it.""" + asked.set() + go.wait(30) + return BlocksResult("failed", unsent=list(cids)) + + self.net.blocks = held + before = set(threading.enumerate()) + self.assertTrue(self.b.backend.remote_pin(IDENTITY, cid, True)) + self.assertTrue(asked.wait(10)) + workers = [t for t in threading.enumerate() if t not in before and t.name == "rnfs-remote-pin"] + self.assertEqual(len(workers), 1) + started = time.monotonic() + self.b.stop() + self.assertFalse(workers[0].is_alive(), "stop didn't wait for the remote-pin worker") + self.assertLess(time.monotonic() - started, node_module.STOP_TIMEOUT) + self.assertEqual(state_rows(self.b.config.state_dir, "SELECT cid FROM remote_pins"), [(cid.binary,)]) + + class MakeRoomTest(NodeTest): """Eviction for room, as each stored response batch asks for.""" diff --git a/tests/test_pacing.py b/tests/test_pacing.py index 0f1510c..fca2f0a 100644 --- a/tests/test_pacing.py +++ b/tests/test_pacing.py @@ -15,21 +15,12 @@ from rnfs.fetcher import Fetcher from rnfs.network import protocol, rates, server from rnfs.network.client import BUSY, OK, BlocksResult +from tests.fakes.log import quiet_log from tests.support import FakeClock from tests.test_server import FakeBackend, FakeDestination, FakeLink, FakeResource OWN, PROVIDER = bytes(16), bytes([1]) * 16 -_saved_loglevel = RNS.loglevel - - -def setUpModule(): - """Silence the fetcher's and server's logs.""" - RNS.loglevel = RNS.LOG_CRITICAL - - -def tearDownModule(): - """Restore RNS's loglevel.""" - RNS.loglevel = _saved_loglevel +setUpModule, tearDownModule = quiet_log() class PacingTest(unittest.TestCase): @@ -44,8 +35,9 @@ def setUp(self): self.tmp = tmp.name def fetch(self, rate: float, rtt: float, chunks: int, server_config: Config | None = None, - server_measures: float = 1.0) -> tuple[int, float]: - """Fetch a file of chunks chunks from one provider over a link of rate bits/s and rtt seconds. + server_measures: float = 1.0, mtu: int = 500, mdu: int = 431) -> tuple[int, float]: + """Fetch a file of chunks chunks from one provider over a link of rate bits/s and rtt seconds, whose + MTU and MDU are mtu and mdu bytes. The responder measures server_measures × the rate (RNS's sender-side span includes a round trip). Returns (refusals, virtual seconds taken); fails the test if the provider is dropped. @@ -62,7 +54,7 @@ def pace(response, wait): """The responder holding a response back, in virtual time.""" clock.now += wait responder._pace = pace - link = FakeLink(b"L" * 16, mtu=500, mdu=431) + link = FakeLink(b"L" * 16, mtu=mtu, mdu=mdu) link.rtt = rtt destination.links.append(link) destination.established(link) @@ -115,7 +107,8 @@ def pace(response, wait): result = BlocksResult(OK, unsent=batch[len(entries):]) for cid, entry in zip(batch, entries): result.blocks[cid] = entry - fetch._handle(PROVIDER, batch, result, False, wanted, children, outstanding, sent_at, sent_rate) + fetch._handle(PROVIDER, batch, result, False, wanted, children, outstanding, fetch.budget, sent_at, + sent_rate) pending = [cid for cid in pending if cid in outstanding] self.assertTrue(storage.is_complete(built.cid)) return refusals, clock.now - start @@ -149,6 +142,15 @@ def test_responder_with_a_lower_upload_cap(self): refusals, _ = self.fetch(50_000, 0.5, 12, Config(max_upload_rate=12_500)) self.assertGreater(refusals, 0) + def test_large_mtu_single_packet_responses_are_paced(self): + # A fast path whose MTU holds a whole response (RNS: 256 KiB at 750 Mbit/s): every response goes as one + # packet, so the responder never gets a rate estimate, but an explicit max_upload_rate applies all the + # same (NET-2); its waits are within the bound, so they are held, not refused + cap = 2_000_000 + refusals, taken = self.fetch(10_000_000, 0.05, 30, Config(max_upload_rate=cap), mtu=262144, mdu=262000) + self.assertEqual(refusals, 0) + self.assertGreater(taken, 0.9 * 30 * CHUNK_SIZE * 8 / cap) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 80d8392..23694eb 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -1,10 +1,13 @@ -"""Static checks of the packaging files: install.sh's isolation and unit quoting, and the dependency lock.""" +"""Static checks of the packaging files: install.sh's isolation and unit quoting, the dependency locks, the workflows, +the supported Pythons and the CHANGELOG's version.""" +import ast import json import os import re import shutil import subprocess import sys +import tempfile import tomllib import unittest @@ -105,16 +108,81 @@ def test_only_its_own_unit_is_replaced_or_removed(self): write = shell_function("write_unit") self.assertIn('$UNIT_MARK', write) + @unittest.skipUnless(shutil.which("sh"), "needs a POSIX shell") + def test_release_wheel_is_of_the_tag(self): + """A release install takes only a wheel of the tag's version: an older release's sdist renamed to this + release's file name builds an older one (SUP-2). Without a release (RNFS_SOURCE, --main) any rnfs wheel.""" + def find(directory, tag): + script = shell_function("find_wheel") + '\nfind_wheel "$1"' + return subprocess.run(["sh", "-c", script, "sh", directory], capture_output=True, text=True, check=True, + env={**os.environ, "RELEASE_TAG": tag}).stdout + with tempfile.TemporaryDirectory() as wheels: + # The script joins with "/" whatever the platform (sh on Windows is Git Bash), so the expected paths do too + older = wheels + "/rnfs-0.1.4-py3-none-any.whl" + open(older, "w").close() + self.assertEqual(find(wheels, "v0.1.6"), "") + self.assertEqual(find(wheels, ""), older + "\n") + wheel = wheels + "/rnfs-0.1.6-py3-none-any.whl" + open(wheel, "w").close() + self.assertEqual(find(wheels, "v0.1.6"), wheel + "\n") + install = shell_function("do_install") + self.assertIn('wheel=$(find_wheel "$wheels")', install) + self.assertLess(install.index('build_wheel "$wheels"'), install.index('wheel=$(find_wheel "$wheels")')) + + def test_provenance_names_the_tag(self): + """The provenance check asks for this tag's build on a GitHub-hosted runner, where gh can (SUP-2; the + check itself is run in tests/security/test_l15_install_release_check.py).""" + verify = shell_function("verify_release") + self.assertIn('*--source-ref*) options="$options --source-ref=refs/tags/$RELEASE_TAG"', verify) + self.assertIn('*--deny-self-hosted-runners*) options="$options --deny-self-hosted-runners"', verify) + def test_unit_matches_contrib(self): - """install.sh's unit has the same [Unit] limits and [Service] settings as contrib/rnfs.service.""" + """install.sh's unit has every [Unit] and [Service] line of contrib/rnfs.service, in order, except + ExecStart (the launcher's path) and comments (SUP-6).""" def settings(text): - return [line for line in text.splitlines() - if re.match(r"(StartLimit|Restart|NoNewPrivileges|Restrict|LockPersonality|KillSignal)", line)] + section, lines = None, [] + for line in text.splitlines(): + line = line.strip() + if re.fullmatch(r"\[\w+\]", line): + section = line + elif section in ("[Unit]", "[Service]") and line and not line.startswith(("#", "ExecStart=")): + lines.append((section, line)) + return lines contrib = settings(read(UNIT)) - self.assertIn("StartLimitBurst=5", contrib) - self.assertIn("NoNewPrivileges=yes", contrib) + for setting in ("StartLimitBurst=5", "NoNewPrivileges=yes", "After=rnsd.service", "TimeoutStopSec=30", + "SystemCallFilter=@system-service"): + self.assertIn(setting, [line for _, line in contrib]) self.assertEqual(settings(shell_function("write_unit")), contrib) + def test_unit_hardening_is_an_opt_in_drop_in(self): + """ProtectSystem and PrivateTmp, which need unprivileged user namespaces in a user unit, are in a documented + drop-in only; no unit sets UMask, since what `get` writes takes the user's umask (SUP-6, Q16).""" + drop_in = read(os.path.join(REPO, "contrib", "rnfs.service.d", "hardening.conf")) + self.assertIn("\n[Service]\n", drop_in) + self.assertRegex(drop_in, r"(?m)^ProtectSystem=full$") + self.assertRegex(drop_in, r"(?m)^PrivateTmp=yes$") + for unit in (read(UNIT), shell_function("write_unit")): + self.assertNotRegex(unit, r"(?m)^(ProtectSystem|PrivateTmp|UMask)=") + self.assertIn("contrib/rnfs.service.d/hardening.conf", unit) + self.assertNotRegex(drop_in, r"(?m)^UMask=") + + @unittest.skipUnless(shutil.which("sh"), "needs a POSIX shell") + def test_plain_http_source_warns(self): + """An RNFS_SOURCE fetched over plain http:// (a URL, or a requirement naming one) gets a warning (SUP-6).""" + script = "\n".join([re.search(r"(?m)^warn\(\) \{.*\}$", read(INSTALL_SH)).group(0), + shell_function("warn_plain_http"), 'SOURCE="$1"', "warn_plain_http"]) + + def warning(source): + return subprocess.run(["sh", "-c", script, "sh", source], capture_output=True, text=True, + check=True).stderr + for source in ("http://example.org/rnfs.zip", "HTTP://example.org/rnfs.zip", + "rnfs @ http://example.org/rnfs-0.1.6-py3-none-any.whl"): + self.assertIn("plain http://", warning(source), source) + for source in ("https://example.org/rnfs.zip", "/home/a/rnfs", "", "rnfs==0.1.6"): + self.assertEqual(warning(source), "", source) + install = shell_function("do_install") + self.assertLess(install.index(" choose_source\n"), install.index(" warn_plain_http\n")) + class BuildScriptsTest(unittest.TestCase): """build-deb.sh and the workflows install nothing unhashed and fail on a missing cffi backend.""" @@ -128,6 +196,14 @@ def test_deb_cffi_download_errors_are_fatal(self): self.assertIn("--no-build-isolation", text) self.assertIn('python3 (<< $PYTHON_BELOW)', text) + def test_deb_postinst_python_is_isolated(self): + """The .deb's postinst, run as root, byte-compiles with python3 -I (SUP-6).""" + postinst = re.search(r"(?ms)^cat > \"\$ROOT/DEBIAN/postinst\" <<'EOF'\n(.*?)^EOF$", read(BUILD_DEB)) + self.assertTrue(postinst) + calls = re.findall(r"python3( -\S+)", postinst.group(1)) + self.assertTrue(calls) + self.assertEqual(set(calls), {" -I"}) + def test_workflow_pip_installs_are_hashed(self): """Every pip install in the workflows uses a hash lock or --no-deps (with no build isolation).""" for name in os.listdir(WORKFLOWS): @@ -148,6 +224,69 @@ def test_release_publishes_only_on_a_pushed_tag(self): self.assertFalse([line for line in uploads if "--clobber" in line]) self.assertRegex(text, r"actions/attest-build-provenance@[0-9a-f]{40}") + def test_e2e_step_runs_every_gated_module(self): + """Every test module gated on RNFS_E2E is run by the workflow's end-to-end step (else it never runs in CI).""" + step = re.search(r"(?m)^ - name: End-to-end tests.*\n(?: .*\n)*? run: >-\n((?: .*\n)+)", + read(os.path.join(WORKFLOWS, "tests.yml"))) + self.assertTrue(step) + listed = set(step.group(1).split()) + gated = set() + for top, _dirs, files in os.walk(os.path.join(REPO, "tests")): + for name in files: + path = os.path.join(top, name) + if name.endswith(".py") and path != os.path.abspath(__file__) and 'environ.get("RNFS_E2E")' in read(path): + gated.add(os.path.relpath(path, REPO)[:-3].replace(os.sep, ".")) + self.assertIn("tests.test_e2e", gated) + self.assertEqual(gated - listed, set()) + + def test_flatpak_builds_skip_the_actions_cache(self): + """Every flatpak-builder step turns the Actions cache off: a cache any workflow run can write must not + reach a release bundle.""" + steps = 0 + for name in os.listdir(WORKFLOWS): + text = read(os.path.join(WORKFLOWS, name)) + for step in re.finditer(r"(?m)^( *)- uses: [^\n]*/flatpak-builder@.*\n((?:\1 .*\n)*)", text): + steps += 1 + self.assertRegex(step.group(2), r"(?m)^ +cache: false$", f"{name}: {step.group(0)}") + self.assertTrue(steps) + + def test_flatpak_sources_are_checked_by_the_tests(self): + """The check that the Flatpak's wheels match the lock runs in tests.yml (on every push, not only at a + release, TS5), and the release builds the committed files rather than an artifact of its own.""" + tests = read(os.path.join(WORKFLOWS, "tests.yml")) + job = re.search(r"(?m)^ flatpak-sources:\n((?: .*\n|\n)+?)(?=^ \S|\Z)", tests) + self.assertTrue(job) + self.assertIn("python -I packaging/flatpak/generate-sources.py\n", job.group(1)) + self.assertIn("git diff --exit-code -- packaging/flatpak ||", job.group(1)) + release = read(os.path.join(WORKFLOWS, "release.yml")) + self.assertNotIn("flatpak-sources", release.replace("tests.yml's flatpak-sources job", "")) + + def test_flatpak_copies_no_local_debris(self): + """The Flatpak's copy of the checkout skips what local builds, tests and lint leave: every .gitignore entry + that isn't a pattern (flatpak-builder's skip entries are paths), and the egg-info (TS12).""" + manifest = read(os.path.join(REPO, "packaging", "flatpak", "io.github.afit21.rnfs.yml")) + block = re.search(r"(?m)^ path: \.\./\.\.\n skip:\n((?: - .*\n)+)", manifest) + self.assertTrue(block) + skipped = set(re.findall(r"- (\S+)", block.group(1))) + ignored = {line.strip().rstrip("/") for line in read(os.path.join(REPO, ".gitignore")).splitlines() + if line.strip() and not line.startswith("#") and not set("*?[") & set(line)} + self.assertTrue(ignored) + self.assertEqual(ignored - skipped, set()) + self.assertIn("rnfs.egg-info", skipped) + + def test_flatpak_sandbox_is_narrow(self): + """The Flatpak reaches only rnfs's and Reticulum's directories and Downloads, and no device (SUP-5).""" + manifest = read(os.path.join(REPO, "packaging", "flatpak", "io.github.afit21.rnfs.yml")) + block = re.search(r"(?m)^finish-args:\n((?: +.*\n)+)", manifest) + self.assertTrue(block) + args = re.findall(r"(?m)^ +- (--\S+)$", block.group(1)) + self.assertEqual(args, ["--share=network", "--filesystem=~/.rnfs:create", "--filesystem=~/.reticulum:create", + "--filesystem=xdg-download"]) + # The opt-ins are documented in the manifest and the README + for text in (block.group(1), read(os.path.join(REPO, "README.md"))): + for override in ("--device=all", "--filesystem="): + self.assertIn(f"flatpak override --user {override}", text) + def test_release_waits_for_the_tests(self): """The release job needs the whole tests workflow, called from release.yml.""" release = read(os.path.join(WORKFLOWS, "release.yml")) @@ -178,7 +317,7 @@ def test_build_locks(self): """The build locks are hashed, agree with each other and with requirements.lock, and fit pyproject.toml.""" backend, tools, runtime = read_lock(BACKEND_LOCK), read_lock(TOOLS_LOCK), read_lock() self.assertEqual(list(backend), ["setuptools"]) - for name in ("setuptools", "build", "twine", "pyinstaller", "pyinstaller-hooks-contrib", "ruff"): + for name in ("setuptools", "build", "twine", "pyinstaller", "pyinstaller-hooks-contrib", "ruff", "coverage"): self.assertIn(name, tools) self.assertEqual(backend["setuptools"], tools["setuptools"]) for name, (version, hashes) in [*backend.items(), *tools.items()]: @@ -210,6 +349,164 @@ def test_flatpak_wheels_are_locked(self): self.assertIn(source["sha256"], lock[name][1], filename) +def version_tuple(text: str) -> tuple[int, ...]: + """(1, 2, 3) from "1.2.3".""" + return tuple(int(part) for part in text.split(".")) + + +def python_constants(path: str) -> dict[str, object]: + """The module-level constants (literals) a Python script assigns, read without running it.""" + with open(path, encoding="utf-8") as f: + tree = ast.parse(f.read(), path) + constants = {} + for node in tree.body: + if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + try: + constants[node.targets[0].id] = ast.literal_eval(node.value) + except ValueError: + pass # not a literal + return constants + + +def shell_assignment(path: str, name: str) -> str: + """The value of a NAME=value (or NAME="value") line in a shell script.""" + match = re.search(r'(?m)^%s="?([^"\n#]*?)"?(?:\s+#.*)?$' % re.escape(name), read(path)) + assert match, (path, name) + return match.group(1) + + +class SupportedPythonsTest(unittest.TestCase): + """The supported Pythons are written in several places; they all agree with the tests.yml matrix (TS7), so + adding one (or dropping one) in only some of them fails here rather than in a release build.""" + + @classmethod + def setUpClass(cls): + """The tests.yml matrix's Pythons, oldest first.""" + matrix = re.search(r'(?m)^ +python: \[(.*)\]$', read(os.path.join(WORKFLOWS, "tests.yml"))) + assert matrix + cls.pythons = sorted(re.findall(r'"(3\.\d+)"', matrix.group(1)), key=version_tuple) + cls.oldest, cls.newest = cls.pythons[0], cls.pythons[-1] + + def test_matrix_is_consecutive(self): + """The matrix is every minor version from the oldest to the newest.""" + major, low = version_tuple(self.oldest) + self.assertEqual(self.pythons, [f"{major}.{minor}" for minor in range(low, version_tuple(self.newest)[1] + 1)]) + + def test_requires_python(self): + """pyproject.toml's requires-python is the oldest.""" + with open(os.path.join(REPO, "pyproject.toml"), "rb") as f: + self.assertEqual(tomllib.load(f)["project"]["requires-python"], f">={self.oldest}") + + def test_update_lock(self): + """build-tools.lock has wheels for every matrix Python, and requirements.lock is resolved for the oldest.""" + constants = python_constants(os.path.join(REPO, "packaging", "update-lock.py")) + self.assertEqual(constants["BUILD_PYTHONS"], self.pythons) + args = constants["RESOLVE_ARGS"] + self.assertEqual(args[args.index("--python-version") + 1], self.oldest) + args = constants["BUILD_RESOLVE_ARGS"] + self.assertIn(args[args.index("--python-version") + 1], self.pythons) + + def test_flatpak_runtime_pythons(self): + """The Flatpak's wheels are for supported Pythons, up to the newest (the runtime's Python and its + neighbours).""" + runtime = python_constants(os.path.join(REPO, "packaging", "flatpak", "generate-sources.py"))["RUNTIME_PYTHONS"] + self.assertLessEqual(set(runtime), set(self.pythons)) + self.assertEqual(max(runtime, key=version_tuple), self.newest) + + def test_build_deb(self): + """The .deb bundles a cffi backend for every matrix Python, and its python3 dependency is within them.""" + self.assertEqual([self.oldest, *shell_assignment(BUILD_DEB, "EXTRA_PYTHONS").split()], self.pythons) + major, minor = version_tuple(self.newest) + self.assertEqual(shell_assignment(BUILD_DEB, "PYTHON_BELOW"), f"{major}.{minor + 1}") + text = read(BUILD_DEB) + self.assertIn(f"for version in {self.oldest} $EXTRA_PYTHONS; do", text) + self.assertIn(f"--python-version {self.oldest} ", text) + self.assertIn(f"Depends: python3 (>= {self.oldest}),", text) + + def test_install_sh(self): + """install.sh accepts the oldest and looks for each matrix Python by name, newest first.""" + candidates = re.search(r"(?m)^ for candidate in (.*); do$", shell_function("find_python")) + self.assertTrue(candidates) + self.assertEqual(candidates.group(1).split(), + ["python3", *(f"python{v}" for v in reversed(self.pythons))]) + major, minor = version_tuple(self.oldest) + self.assertIn(f"sys.version_info >= ({major}, {minor})", shell_function("python_ok")) + self.assertIn(f"older than Python {self.oldest}, which rnfs needs", shell_function("find_python")) + + def test_readme_pyinstaller_is_locked(self): + """README's pyinstaller==X (building the binary by hand), where it gives one, is the version build-tools.lock + pins.""" + versions = re.findall(r"\bpyinstaller==([\w.]+)", read(os.path.join(REPO, "README.md"))) + self.assertLessEqual(set(versions), {read_lock(TOOLS_LOCK)["pyinstaller"][0]}) + + +class ChangelogTest(unittest.TestCase): + """CHANGELOG.md's first section is the current version's, or the next one's while it is unreleased, and the + release workflow publishes that section as the release notes (TS6).""" + + def notes_step(self) -> str: + """The run: script of release.yml's step that checks CHANGELOG.md and writes release-notes.md.""" + step = re.search(r"(?m)^ - name: Release notes from CHANGELOG\.md.*\n(?: .*\n)*? run: \|\n" + r"((?: .*\n|\n)+?)(?=^ - )", read(os.path.join(WORKFLOWS, "release.yml"))) + self.assertTrue(step) + return "\n".join(line[10:] for line in step.group(1).splitlines()) + + def test_first_section_is_the_version(self): + """The first `## ` heading is `## <__version__>`, or `## X.Y.Z (unreleased)` for a later X.Y.Z while + __version__ still has a section of its own (it is bumped at the release).""" + from rnfs import __version__ + headings = re.findall(r"(?m)^## (.*)$", read(os.path.join(REPO, "CHANGELOG.md"))) + self.assertTrue(headings) + first = re.fullmatch(r"(\d+\.\d+\.\d+)( \(unreleased\))?", headings[0]) + self.assertTrue(first, headings[0]) + if first.group(2): + self.assertGreater(version_tuple(first.group(1)), version_tuple(__version__), headings[0]) + self.assertIn(__version__, headings[1:]) + else: + self.assertEqual(first.group(1), __version__) + self.assertEqual(len(headings), len(set(headings)), "a version has two sections") + + def test_specification_names_the_version(self): + """specification.md's version note names the version CHANGELOG.md's first section is for (TS11).""" + first = re.search(r"(?m)^## (\d+\.\d+\.\d+)", read(os.path.join(REPO, "CHANGELOG.md"))) + note = re.search(r"(?m)^\*\*Version\.\*\* This document describes rnfs (\d+\.\d+\.\d+)\.", + read(os.path.join(REPO, "specification.md"))) + self.assertTrue(first and note) + self.assertEqual(note.group(1), first.group(1)) + + def test_release_notes_are_the_changelog_section(self): + """The release is created with --notes-file, from the notes the version job extracted, not + --generate-notes.""" + release = read(os.path.join(WORKFLOWS, "release.yml")) + self.assertNotIn("--generate-notes", release) + self.assertIn("--notes-file notes/release-notes.md", release) + self.assertIn('[ "${GITHUB_REF_TYPE}" = tag ] && [ "$first" != "## $VERSION" ]', self.notes_step()) + + @unittest.skipUnless(shutil.which("sh") and shutil.which("awk"), "needs a POSIX shell and awk") + def test_release_notes_step(self): + """The step refuses a tag whose section is still unreleased or not first, and writes the version's section + (without its heading or the next one) as the notes.""" + changelog = ("# Changelog\n\nIntro.\n\n## 0.2.0 (unreleased)\n\nNext.\n\n## 0.1.9\n\nNotes of 0.1.9.\n\n" + "### Security\n- A fix.\n\n## 0.1.8\n\nOlder.\n") + + def run(text, version, ref_type): + with tempfile.TemporaryDirectory() as tmp: + with open(os.path.join(tmp, "CHANGELOG.md"), "w", encoding="utf-8") as f: + f.write(text) + done = subprocess.run(["sh", "-ec", self.notes_step()], cwd=tmp, capture_output=True, text=True, + env={**os.environ, "VERSION": version, "GITHUB_REF_TYPE": ref_type}) + notes = read(os.path.join(tmp, "release-notes.md")) if done.returncode == 0 else None + return done.returncode, notes + section = "\nNotes of 0.1.9.\n\n### Security\n- A fix.\n\n" + self.assertEqual(run(changelog, "0.1.9", "branch"), (0, section)) + self.assertNotEqual(run(changelog, "0.1.9", "tag")[0], 0) # 0.2.0 is still above it + self.assertNotEqual(run(changelog, "0.2.0", "tag")[0], 0) # unreleased + self.assertEqual(run(changelog, "0.2.0", "branch"), (0, "\nNext.\n\n")) + released = changelog.replace("## 0.2.0 (unreleased)\n\nNext.\n\n", "") + self.assertEqual(run(released, "0.1.9", "tag"), (0, section)) + self.assertNotEqual(run(released, "0.1.7", "branch")[0], 0) # no section + + class ReadmeTest(unittest.TestCase): """README.md's release links and file names are for the current version, so a version bump updates them.""" diff --git a/tests/test_pin_names.py b/tests/test_pin_names.py index a469c05..748b450 100644 --- a/tests/test_pin_names.py +++ b/tests/test_pin_names.py @@ -3,6 +3,8 @@ import os import tempfile import unittest +from collections.abc import Iterator +from contextlib import contextmanager from unittest import mock from rnfs.cli import display_name @@ -13,6 +15,22 @@ from tests.support import failing_commits, state_execute, state_rows, write_legacy +@contextmanager +def recorded_transactions(db, label: str, opened: list | None = None) -> Iterator[list]: + """Within the with block, each transaction opened through the StateDb reference db appends (label, durable) + to opened (a new list if None), which is yielded.""" + opened = [] if opened is None else opened + real = db.transaction + + def transaction(durable: bool = False): + """The real transaction, noted.""" + opened.append((label, durable)) + return real(durable) + + with mock.patch.object(db, "transaction", transaction): + yield opened + + class PinNamesStoreTest(unittest.TestCase): """The store itself: record, keep the first for pin add, prune, persist, ignore damage.""" @@ -81,6 +99,25 @@ def test_failed_save_is_written_with_the_next_change(self): names.record(self.other, "second") self.assertEqual(self.rows(), {self.other: '"second"'}) + def test_failed_save_is_written_by_save(self): + """save() writes a name whose write failed (the node saves its stores at stop).""" + names = self.open() + with failing_commits(), self.assertRaises(OSError): + names.record(self.cid, "first") + self.assertEqual(self.rows(), {}) + names.save() + self.assertEqual(self.rows(), {self.cid: '"first"'}) + + def test_names_are_written_at_once_but_not_durably(self): + """Each change is committed as it is made, with synchronous=NORMAL (S3): the pin it names is durable.""" + names = self.open() + with recorded_transactions(names._db, "name") as opened: + names.record(self.cid, "a") + self.assertEqual(self.rows(), {self.cid: '"a"'}) + names.prune(set()) + self.assertEqual(self.rows(), {}) + self.assertEqual(opened, [("name", False), ("name", False)]) + class PinLsNamesTest(unittest.TestCase): """The node's pin_ls reports type and name for each way a pin can be made.""" @@ -105,6 +142,25 @@ def write(self, name: str, data: bytes) -> str: f.write(data) return path + def test_an_add_commits_durably_once_pin_first(self): + """S3: an add makes one durable commit, the pin's; the name is written after it, not durably, so a power + loss can lose at most the name, never the pin.""" + with recorded_transactions(self.node.storage._db, "pin") as opened, \ + recorded_transactions(self.node.pin_names._db, "name", opened): + cid = self.node.add_path(self.write("notes.txt", b"hello"))["cid"] + self.node.add_stream(io.BytesIO(b"streamed"), name="report.pdf") + self.assertEqual(opened, [("pin", True), ("name", False)] * 2) + self.assertEqual(self.entry(cid)["name"], os.path.abspath(os.path.join(self.dir, "notes.txt"))) + + def test_stop_saves_a_name_whose_write_failed(self): + """The node saves pin names at stop: a name whose write failed is written then.""" + cid = Cid.for_block("raw", b"x") + with failing_commits(), self.assertRaises(OSError): + self.node.pin_names.record(cid, "/home/u/a.txt") + self.node.stop() + rows = state_rows(self.node.config.state_dir, "SELECT cid, name FROM pin_names") + self.assertEqual(rows, [(cid.binary, '"/home/u/a.txt"')]) + def test_added_file_shows_its_path(self): path = self.write("notes.txt", b"hello") cid = self.node.add_path(path)["cid"] diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 6d7a982..5e87f7d 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -1,13 +1,19 @@ -"""Tests for encoding and strictly validating rnfs.dht messages, and for request batch sizes.""" +"""Tests for encoding and strictly validating rnfs.dht messages (keepers' /name responses too), and for request +batch sizes.""" import unittest import RNS from RNS.vendor import umsgpack +from rnfs.config import Config from rnfs.constants import (BLOCKS_PATH, CHUNK_SIZE, FIND_PATH, ID_LENGTH, K, KIB, MAX_BLOCK_SIZE, MAX_REQUEST_SIZE, MAX_RESPONSE_SIZE, PROVIDE_PATH) from rnfs.content.cid import DAG, RAW, Cid -from rnfs.network import protocol +from rnfs.features import keepers +from rnfs.names import encode_record, name_for_identity +from rnfs.network import protocol, server +from tests.support import FakeClock +from tests.test_server import FakeBackend, FakeDestination, FakeLink KEY = bytes(range(ID_LENGTH)) RAW_CID = Cid.for_block(RAW, b"raw block") @@ -208,5 +214,68 @@ def test_rejects_malformed(self): self.assertIsNone(protocol.parse_pin_request(data)) +class BodyRefusalTest(unittest.TestCase): + """protocol.body_refusal: the shape check on a response alone, which response_refusal applies after the + framing and the request server applies to its features' responses.""" + + ACCEPTED = (True, False, None, [], [b"block", None], [[KEY] * K, [KEY]], [None] * protocol.MAX_BLOCKS_CIDS, + [bytes(MAX_BLOCK_SIZE)]) + REFUSED = ({"a": 1}, "text", 1, [1], [None] * (protocol.MAX_BLOCKS_CIDS + 1), [bytes(MAX_BLOCK_SIZE + 1)], + [[KEY] * (K + 1), []], [{}]) + + def test_agrees_with_response_refusal(self): + for accepted, responses in ((True, self.ACCEPTED), (False, self.REFUSED)): + for response in responses: + with self.subTest(response=repr(response)[:40]): + body = protocol.body_refusal(umsgpack.packb(response)) + framed = protocol.response_refusal(umsgpack.packb([bytes(protocol.REQUEST_ID_LENGTH), response])) + self.assertEqual(body, framed) + self.assertEqual(body is None, accepted) + + def test_at_a_position_and_nothing_after(self): + packed = b"junk" + umsgpack.packb([b"block"]) + self.assertIsNone(protocol.body_refusal(packed, 4)) + self.assertIsNotNone(protocol.body_refusal(packed)) + self.assertEqual(protocol.body_refusal(umsgpack.packb(True) + b"\x00"), "1 bytes after the response") + self.assertEqual(protocol.body_refusal(b""), "no response") + self.assertEqual(protocol.body_refusal(packed, len(packed)), "no response") + + +class NameResponseTest(unittest.TestCase): + """Every /name response keepers sends is a shape requesters accept (/name is in the module's table; nodes since + 0.1.4 refuse other shapes before decoding them), and the request server sends it as it is.""" + + def responses(self) -> dict: + """Each kind of /name answer: a kept record (real, and of the largest fields), none kept, malformed.""" + identity = RNS.Identity() + record = keepers.sign_record(identity, name_for_identity(identity), encode_record(2 ** 63, DAG_CID)) + largest = keepers.SignedRecord(KEY, bytes(keepers.PUBLIC_KEY_LENGTH), bytes(keepers.RANDOM_HASH_LENGTH), + bytes(keepers.RATCHET_LENGTH), bytes(keepers.SIGNATURE_LENGTH), + bytes(keepers.MAX_APP_DATA), 1, DAG_CID) + return {"record": keepers.encode_response(record), "largest": keepers.encode_response(largest), + "none kept": keepers.encode_response(None), "malformed": False} + + def test_accepted_by_requesters(self): + for kind, response in self.responses().items(): + with self.subTest(kind): + framed = umsgpack.packb([bytes(keepers.REQUEST_ID_LENGTH), response]) + self.assertIsNone(protocol.body_refusal(umsgpack.packb(response))) + self.assertIsNone(protocol.response_refusal(framed)) + self.assertLessEqual(len(framed), keepers.max_response_size()) + + def test_sent_as_they_are(self): + for kind, response in self.responses().items(): + with self.subTest(kind): + backend = FakeBackend() + backend.feature_requests = lambda answer=response: {keepers.NAME_PATH: lambda data, identity: answer} + destination = FakeDestination() + server.RequestServer(destination, Config(), backend, clock=FakeClock(), start_monitor=False) + link = FakeLink(b"l" * 16) + destination.links.append(link) + destination.established(link) + handler = destination.handlers[keepers.NAME_PATH][0] + self.assertEqual(handler(keepers.NAME_PATH, KEY, bytes(16), link.link_id, None, 0.0), response) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_providers.py b/tests/test_providers.py index aaecccb..142288c 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -23,14 +23,26 @@ def ident(n: int) -> bytes: class StoreTestCase(unittest.TestCase): - """Base: an in-memory store with a fake clock and seeded rng.""" + """Base: an in-memory store with a fake clock and seeded rng; every store's indexes are checked after each test.""" max_records = 1000 def setUp(self): """Create the store.""" self.clock = FakeClock(1_700_000_000.0) - self.store = ProviderStore(self.max_records, None, clock=self.clock, rng=random.Random(1)) + self.stores: list[ProviderStore] = [] + self.store = self.new_store(self.max_records, rng=random.Random(1)) + + def new_store(self, max_records: int, **settings) -> ProviderStore: + """An in-memory store on the fake clock, checked for consistency after the test.""" + store = ProviderStore(max_records, None, clock=self.clock, **settings) + self.stores.append(store) + return store + + def tearDown(self): + """Every index of every store agrees with its records.""" + for store in self.stores: + store._check_consistency() class RecordTest(StoreTestCase): @@ -58,6 +70,34 @@ def test_future_times_clamped_at_expiry(self): self.clock.now += PROVIDER_TTL self.assertEqual(self.store.expire(), 1) + def test_expiry_clamps_only_after_the_clock_went_back(self): + """The clamping pass runs only when the clock reads earlier than a time it gave before (S5).""" + with mock.patch.object(self.store, "_clamp", wraps=self.store._clamp) as clamp: + self.store.expire() + self.store.store(KEY, ident(1)) + self.store.expire() + self.clock.now += 60 + self.store.expire() + self.assertEqual(clamp.call_count, 0) + self.clock.now -= 100 # corrected backwards: the record was stored in the future + self.store.expire() + self.assertEqual(clamp.call_count, 1) + self.assertEqual(self.store.first_seen(ident(1)), self.clock.now) + self.store.expire() # every time is now at most the clock's: nothing more to clamp + self.clock.now += 5 + self.store.expire() + self.assertEqual(clamp.call_count, 1) + + def test_departure_in_the_future_clamped_at_expiry(self): + """A departure time, taken when the publisher's last record went, is clamped too (S5): the withdrawal's own + clock reading counts as one the store has seen.""" + self.store.store(KEY, ident(1)) + self.clock.now += DAY + self.store.withdraw(KEY, ident(1)) # departs a day ahead of the corrected clock + self.clock.now -= DAY + self.store.expire() + self.assertEqual(self.store._departed[ident(1)], (self.clock.now, self.clock.now)) + def test_expires_ttl_after_last_stored(self): self.store.store(KEY, ident(1)) self.clock.now += PROVIDER_TTL - 1 @@ -262,7 +302,7 @@ def test_refresh_at_the_limit(self): self.assertTrue(self.store.store(ident(100), ident(1))) # republishing is never refused def test_own_records_are_not_limited(self): - store = ProviderStore(self.max_records, None, clock=self.clock, own_id=ident(1)) + store = self.new_store(self.max_records, own_id=ident(1)) for i in range(3 * store.max_per_publisher): self.assertTrue(store.store(ident(100 + i), ident(1))) self.assertTrue(store.store(KEY, ident(1))) @@ -280,7 +320,7 @@ def test_expired_records_do_not_count_toward_the_limit(self): self.assertEqual(self.store.record_count(), 1) def test_full_key_evicts_a_random_non_senior_publisher(self): - store = ProviderStore(1000, None, clock=self.clock, rng=random.Random(1), max_per_key=20) + store = self.new_store(1000, rng=random.Random(1), max_per_key=20) accepted = [store.store(KEY, ident(i)) for i in range(1, 31)] self.assertEqual(accepted, [True] * 30) # newcomers are never locked out providers = store.providers(KEY) @@ -295,7 +335,7 @@ def test_full_key_evicts_a_random_non_senior_publisher(self): self.assertEqual(store.providers(KEY), [ident(31)]) def test_sybils_holding_a_key_cannot_lock_out_later_providers(self): - store = ProviderStore(100_000, None, clock=self.clock, rng=random.Random(1)) + store = self.new_store(100_000, rng=random.Random(1)) early = ident(1) store.store(KEY, early) for i in range(store.max_per_key - 1): @@ -306,7 +346,7 @@ def test_sybils_holding_a_key_cannot_lock_out_later_providers(self): self.assertIn(early, store.for_reply(KEY)[:K // 2]) # long-standing providers are never evicted def test_reply_asks_exclude_only_about_chosen_candidates(self): - store = ProviderStore(10_000, None, clock=self.clock, rng=random.Random(1)) + store = self.new_store(10_000, rng=random.Random(1)) for i in range(store.max_per_key): store.store(KEY, ident(i)) self.assertEqual(len(store.providers(KEY)), store.max_per_key) @@ -399,13 +439,20 @@ def setUp(self): self.addCleanup(self.directory.cleanup) self.state = os.path.join(self.directory.name, "state") self.clock = FakeClock(1_700_000_000.0) + self.stores: list[ProviderStore] = [] def open(self, max_records: int = 100, **settings) -> ProviderStore: - """A store on self.state, closed when the test ends.""" + """A store on self.state, checked for consistency and closed when the test ends.""" store = ProviderStore(max_records, self.state, clock=self.clock, **settings) self.addCleanup(store.close) + self.stores.append(store) return store + def tearDown(self): + """Every index of every store opened agrees with its records (before cleanups close them).""" + for store in self.stores: + store._check_consistency() + def test_round_trip(self): """Records and first-seen times are saved, and loaded by the next store.""" store = self.open() @@ -423,6 +470,18 @@ def test_round_trip(self): self.assertEqual(loaded.first_seen(ident(1)), store.first_seen(ident(1))) self.assertEqual(loaded.first_seen(ident(2)), store.first_seen(ident(2))) + def test_own_record_beyond_a_full_key_survives_restart(self): + """The node's own record doesn't count toward max_per_key on load either (DHT-5).""" + own = ident(1) + store = self.open(1000, max_per_key=5, own_id=own) + for i in range(5): + store.store(KEY, ident(100 + i)) + self.clock.now += 1 + store.store(KEY, own) + store.save() + loaded = self.open(1000, max_per_key=5, own_id=own) + self.assertEqual(loaded.providers(KEY), [ident(100 + i) for i in range(5)] + [own]) + def test_first_seen_order_survives_restart(self): """A publisher seen first still evicts the newest one's records after a restart.""" store = self.open(3) @@ -635,6 +694,8 @@ def check(): store.save() loaded = ProviderStore(60, self.state, clock=self.clock, **settings) try: + store._check_consistency() + loaded._check_consistency() self.assertEqual(held(loaded), held(store)) self.assertEqual(saved(self.state), held(store)) self.assertFalse(loaded._dirty_records or loaded._dirty_publishers or loaded._dirty_gone) @@ -674,6 +735,65 @@ def check(): self.assertTrue(store._departed or store._evicted) check() + def test_random_operations_keep_indexes_consistent(self): + """Random stores, withdrawals, expiries, clock corrections and reloads (some with smaller limits) in a store + kept full: every index agrees with the records after each step, and after expire() no time is later than + now.""" + rng = random.Random(11) + own = ident(1) + publishers = [ident(1 + n) for n in range(25)] + keys = [ident(1000 + n) for n in range(20)] + limits = [{"max_records": 40}, {"max_records": 40}, {"max_records": 30, "max_per_publisher": 5}, + {"max_records": 20, "max_per_key": 3}] + settings = {"max_per_key": 5, "own_id": own, "seniority": DAY, "rng": random.Random(2)} + store = self.open(**(settings | limits[0])) + counts = {"pooled": 0, "reloads": 0, "clamped": 0} + + def no_future_times(store: ProviderStore): + """Nothing store holds is later than now.""" + now = self.clock.now + state = held(store) + self.assertTrue(all(seen <= now for seen in state["publishers"].values())) + self.assertTrue(all(first <= now and last <= now for first, last, _ in state["records"].values())) + for memory in ("departed", "evicted"): + self.assertTrue(all(seen <= now and at <= now for seen, at in state[memory].values())) + + for _ in range(3000): + action = rng.random() + publisher = rng.choice(publishers) + if action < 0.45: + store.store(rng.choice(keys), publisher) + elif action < 0.6: + store.withdraw(rng.choice(keys), publisher) + elif action < 0.75: + self.clock.now += rng.choice((1, 600, 3 * 3600, DAY, 3 * DAY)) + elif action < 0.8: + self.clock.now -= rng.choice((1, 600, DAY)) # corrected backwards: times are now in the future + counts["clamped"] += 1 + elif action < 0.88: + store.expire() + no_future_times(store) + elif action < 0.9: + store.retain(lambda p: p not in {publisher}) + elif action < 0.95: + # A restart, sometimes with expired records and future times still held (the load drops and clamps + # them): after an expire(), the reloaded store holds what this one did unless its limits are smaller + expired = rng.random() < 0.5 + if expired: + store.expire() + store.save() + store.close() + setting = rng.choice(limits) + reloaded = self.open(**(settings | setting)) + if expired and setting == limits[0] and store.max_records == 40: + self.assertEqual(held(reloaded), held(store)) + no_future_times(reloaded) + store = reloaded + counts["reloads"] += 1 + counts["pooled"] = max(counts["pooled"], store._pool_count) + store._check_consistency() + self.assertTrue(counts["pooled"] and counts["reloads"] > 50 and counts["clamped"] > 50) + class ThreadSafetyTest(StoreTestCase): """Concurrent stores, withdrawals and replies.""" diff --git a/tests/test_public.py b/tests/test_public.py index 269736a..302e7d2 100644 --- a/tests/test_public.py +++ b/tests/test_public.py @@ -15,6 +15,7 @@ import threading import time import unittest +import warnings from unittest import mock import RNS @@ -177,6 +178,48 @@ def spy(target, offset, length, raw, from_node): self.assertEqual(f.read(), data) self.assertEqual(reads, [(0, len(data))]) # the rest in one read, not one per 8 KiB + def test_open_reads_ahead_while_sequential(self): + """Sequential refills double up to READ_AHEAD; a seek starts again with a refill of only what it covers.""" + data = os.urandom(1_000_000) + cid = self.share(data, "f.bin") + reads = [] + original = self.rnfs._backend.read + + def spy(target, offset, length, raw, from_node): + """Record each read the file makes.""" + reads.append((offset, length)) + return original(target, offset, length, raw, from_node) + buffer = interface.FILE_BUFFER + with mock.patch.object(self.rnfs._backend, "read", spy), \ + mock.patch.object(interface, "READ_AHEAD", 4 * buffer), self.rnfs.open(f"{cid}/f.bin") as f: + self.assertEqual(b"".join(iter(lambda: f.read(1000), b"")), data) + self.assertEqual(f.tell(), len(data)) + self.assertEqual(reads, [(0, buffer), (buffer, 2 * buffer), (3 * buffer, 4 * buffer), + (7 * buffer, 4 * buffer), (11 * buffer, 4 * buffer), + (15 * buffer, len(data) - 15 * buffer)]) + # Random reads: each refill reads only the chunks it covers, however far the read-ahead had grown + reads.clear() + for offset in (500_000, 10, 999_990, 300_000): + f.seek(offset) + self.assertEqual(f.read(10), data[offset:offset + 10]) + self.assertEqual(reads, [(500_000, buffer), (10, buffer), (999_990, 10), (300_000, buffer)]) + # Reading on from a seek grows again from FILE_BUFFER, and tell() counts what was read ahead as unread + reads.clear() + self.assertEqual(f.read(3 * buffer), data[300_010:300_010 + 3 * buffer]) + self.assertEqual(f.tell(), 300_010 + 3 * buffer) + self.assertEqual(reads, [(300_000 + buffer, 2 * buffer), (300_000 + 3 * buffer, 4 * buffer)]) + # A seek into what was read ahead (past the File's own buffer) reads it again, from there only + reads.clear() + f.seek(300_000 + 5 * buffer) + self.assertEqual(f.read(10), data[300_000 + 5 * buffer:300_010 + 5 * buffer]) + self.assertEqual(reads, [(300_000 + 5 * buffer, buffer)]) + # The rest: what was read ahead, then one read + reads.clear() + self.assertEqual(f.read(buffer), data[300_010 + 5 * buffer:300_010 + 6 * buffer]) + self.assertEqual(f.read(), data[300_010 + 6 * buffer:]) + self.assertEqual(reads, [(300_000 + 6 * buffer, 2 * buffer), (300_000 + 8 * buffer, + len(data) - 300_000 - 8 * buffer)]) + def test_cat_get_and_uri_targets(self): """cat and get, rnfs: URI targets (from= and a name), and block().""" data = os.urandom(100_000) @@ -217,6 +260,30 @@ def test_pins(self): self.assertIsInstance(self.rnfs.gc(), rnfs.Collected) self.assertEqual(self.rnfs.verify().bad, ()) + def test_dir_set_and_rm(self): + """dir_set adds an entry, dir_rm removes it again; dir_rm of a root is ValueError (a 400 for a Client), + refused by the node itself before it fetches anything (ND3).""" + directory = self.share(b"inside", name="a.txt") + child = self.rnfs.add(b"child").cid + edited = self.rnfs.dir_set(directory, "b.txt", child, executable=True) + self.assertEqual([(e.name, e.kind) for e in self.rnfs.ls(edited)], [("a.txt", "file"), ("b.txt", "executable")]) + self.assertEqual(self.rnfs.dir_rm(f"{edited}/b.txt"), directory) + remote = self.share(b"not here", name="c.txt") + for target in (remote, f"{remote}/"): + with self.subTest(target), self.assertRaisesRegex(ValueError, "the entry to remove"): + self.rnfs.dir_rm(target) + self.assertFalse(self.b.storage.has(remote)) # not fetched + + def test_remote_pin_providers_and_ping(self): + """pin_remote asks the node (as /pin), providers finds the provider, ping times a node or answers None.""" + cid = self.share(b"provided") + self.assertIs(self.rnfs.pin_remote(A_ID, cid), True) + self.assertIs(self.rnfs.pin_remote(A_ID, cid, pin=False), True) + self.assertEqual(self.net.pins, [(A_ID, cid, True), (A_ID, cid, False)]) + self.assertIn(A_ID, self.rnfs.providers(cid)) + self.assertEqual(self.rnfs.ping(A_ID), 0.1) + self.assertIsNone(self.rnfs.ping(OTHER)) + def test_errors_are_one_hierarchy(self): """The same exception classes whichever class raises them.""" data = os.urandom(1000) @@ -248,6 +315,24 @@ def test_names(self): with self.assertRaises(rnfs.NotFound): self.rnfs.names.get("site") self.assertEqual(self.rnfs.names.import_key("again", key).name, name.name) + + def test_publish_warning_is_a_runtime_warning(self): + """The node's private-mode warning on publish (DHT-4) reaches the program as a RuntimeWarning, and the + record is returned as usual; without one, nothing is warned.""" + cid = self.rnfs.add(b"site").cid + name = self.rnfs.names.create("site") + original = self.rnfs._backend.name_publish + + def warning_publish(alias, target): + return {**original(alias, target), "warning": "visible outside the private network"} + + with mock.patch.object(self.rnfs._backend, "name_publish", warning_publish), \ + self.assertWarnsRegex(RuntimeWarning, "visible outside the private network"): + record = name.publish(cid) + self.assertEqual((record.cid, record.sequence), (cid, 1)) + with warnings.catch_warnings(): + warnings.simplefilter("error") + self.assertEqual(name.publish(cid).sequence, 2) self.assertEqual(rnfs.Name.from_dict(name.to_dict()), name) def test_watch_hears_newer_records_on_the_event_thread(self): @@ -783,7 +868,8 @@ def run_python(self, code: str) -> str: def test_client_loads_no_rns(self): """rnfs.Client, the objects and errors load no RNS.""" code = ("import sys, rnfs; rnfs.Client(port=1); rnfs.Cid, rnfs.Record, rnfs.NotFound, rnfs.ApiError, " - "rnfs.Job, rnfs.File; print(any(m == 'RNS' or m.startswith('RNS.') for m in sys.modules))") + "rnfs.Job, rnfs.File, rnfs.SealError, rnfs.NotRecipient, rnfs.BlockMismatch; " + "print(any(m == 'RNS' or m.startswith('RNS.') for m in sys.modules))") self.assertEqual(self.run_python(code), "False") def test_node_loads_rns_when_used(self): @@ -795,6 +881,11 @@ def test_names_and_status(self): """API_STATUS, the exported names, and an unknown one.""" self.assertEqual(rnfs.API_STATUS, "provisional") self.assertIs(rnfs.NotFound, __import__("rnfs.errors").errors.NotFound) + # The errors programs could only catch as ValueError before (XC4): the classes rnfs raises + from rnfs.content import blockstore, seal + self.assertIs(rnfs.SealError, seal.SealError) + self.assertIs(rnfs.NotRecipient, seal.NotRecipient) + self.assertIs(rnfs.BlockMismatch, blockstore.BlockMismatch) self.assertIs(rnfs.Client, remote.Client) self.assertIn("Node", dir(rnfs)) with self.assertRaises(AttributeError): diff --git a/tests/test_publisher.py b/tests/test_publisher.py index 6ff0f28..1ebd7de 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -6,8 +6,6 @@ import unittest from unittest import mock -import RNS - from rnfs import publisher as publisher_module from rnfs.config import Config from rnfs.constants import PROVIDER_TTL @@ -17,40 +15,15 @@ from rnfs.denylist import DenyList from rnfs.dht.providers import ProviderStore from rnfs.publisher import PUBLISH_LOOKUPS, SLOW_LOOKUP_PACE, SLOW_ROUND_LOOKUPS, DhtKey, Publisher +from tests.fakes.client import FakeClient +from tests.fakes.log import quiet_log from tests.support import FakeClock, failing_commits, state_execute, state_rows, write_legacy from tests.test_providers import recorded_writes, rows_written OWN = bytes(16) N1, N2, N3 = (bytes([i]) * 16 for i in (1, 2, 3)) -_saved_loglevel = RNS.loglevel - - -def setUpModule(): - """Silence the publisher's log.""" - RNS.loglevel = RNS.LOG_CRITICAL - - -def tearDownModule(): - """Restore RNS's loglevel.""" - RNS.loglevel = _saved_loglevel - - -class FakeClient: - """Records /provide requests; nodes in refuse record nothing, a node in limit records only its first keys.""" - - def __init__(self): - """No requests yet.""" - self.calls: list[tuple[bytes, list[bytes], list[bytes]]] = [] - self.refuse: set[bytes] = set() - self.limit: dict[bytes, int] = {} - - def provide(self, node_id, keys, withdrawn): - """Record one batched /provide; the keys the node may have recorded, as PeerClient.provide returns.""" - self.calls.append((node_id, list(keys), list(withdrawn))) - if node_id in self.refuse: - return set() - return set(list(keys)[:self.limit.get(node_id, len(keys))]) +setUpModule, tearDownModule = quiet_log() class FakeLookup: @@ -480,6 +453,25 @@ def test_times_clamped_and_earlier_nodes_expired_at_load_are_saved(self): restarted.save() self.assertEqual(self.saved()["earlier"], {}) + def test_provided_again_cancels_only_its_queued_withdrawals(self): + """A CID provided (again) cancels the withdrawals queued for its key at every node, whose nodes join its + earlier ones; the other keys' stay queued, and a CID with none queued cancels nothing.""" + a, b = self.pinned(b"a"), self.pinned(b"b") + other = bytes([7]) * 16 + with self.publisher._lock: # as if queued earlier, and saved + self.publisher._withdrawals = {N1: {a.key, other}, N2: {a.key}, N3: {other}} + for node in (N1, N2, N3): + self.publisher._touch_node(node) + self.publisher.save() + self.publisher.refresh() + self.assertEqual(self.publisher.provided(), {a, b}) + self.assertEqual(self.publisher._withdrawals, {N1: {other}, N3: {other}}) + self.assertEqual(set(self.publisher._earlier[a]), {N1, N2}) + self.assertNotIn(b, self.publisher._earlier) + self.assertEqual(self.saved()["withdraw"], {N1: {other}, N3: {other}}) + self.assertEqual(self.records.providers(a.key), [OWN]) + self.assertEqual(self.records.providers(b.key), [OWN]) # recorded locally, nothing to cancel + def test_served_blocks_are_the_provided_dags(self): cid = self.pinned(os.urandom(200_000)) self.publisher.refresh() @@ -604,18 +596,13 @@ def due_keys(self, count: int) -> list[Cid]: def test_lookups_run_at_once(self): """A round's lookups run PUBLISH_LOOKUPS at once, on threads of their own, so a round of many keys whose - lookups take time finishes in a fraction of the time one at a time takes.""" + lookups take time finishes in a fraction of the time one at a time takes (the peak count shows it).""" self.due_keys(20) self.lookup_delay = lambda key: 0.1 # 2 s one at a time - started = time.monotonic() result = self.publisher.publish_due() - elapsed = time.monotonic() - started self.assertEqual(len(result["published"]), 20) self.assertEqual(self.running.peak, PUBLISH_LOOKUPS) self.assertEqual(self.running.threads, {"rnfs-publish-lookup"}) - # About 20 / PUBLISH_LOOKUPS × 0.1 = 0.5 s; CI runners have taken 0.4 s more, so the bound only rules out - # lookups one at a time - self.assertLess(elapsed, 1.5) self.assertEqual(len(self.client.calls), 2) # still one /provide per node, every key in it self.assertTrue(all(len(keys) == 20 for _, keys, _ in self.client.calls)) @@ -702,8 +689,8 @@ def fail(key): # Refreshes that change nothing def counting(self) -> dict[str, int]: - """Count the calls of storage.pins and dag_blocks from now until the test ends: {name: calls}.""" - counts = {"pins": 0, "dag_blocks": 0} + """Count the calls of storage.pinned_cids and dag_blocks from now until the test ends: {name: calls}.""" + counts = {"pinned_cids": 0, "dag_blocks": 0} for name in counts: original = getattr(self.storage, name) @@ -729,16 +716,16 @@ def test_refresh_skipped_when_nothing_changed(self): counts = self.counting() for _ in range(3): self.publisher.refresh() - self.assertEqual(counts, {"pins": 0, "dag_blocks": 0}) + self.assertEqual(counts, {"pinned_cids": 0, "dag_blocks": 0}) self.assertIs(self.publisher._provided, provided) self.assertIs(self.publisher._served, served) self.assertEqual(self.publisher.provided(), {cid}) self.assertEqual(self.publisher.provided_keys(), keys) publisher = self.make_publisher() publisher.refresh() - counts["pins"] = 0 + counts["pinned_cids"] = 0 publisher.refresh() - self.assertEqual(counts["pins"], 1) + self.assertEqual(counts["pinned_cids"], 1) def test_refresh_computes_after_each_kind_of_change(self): """With a generation, each change a refresh is computed from makes the next refresh compute, and what @@ -756,9 +743,9 @@ def test_refresh_computes_after_each_kind_of_change(self): def refresh(provided: set, withdrawn: Cid | bytes | None = None): """Refresh: it computes, provides provided (CIDs and keys) and has queued, at once, the withdrawal of withdrawn (a CID or key published to N1 and N2). Then publish, and a refresh computes nothing.""" - counts["pins"] = 0 + counts["pinned_cids"] = 0 self.publisher.refresh() - self.assertEqual(counts["pins"], 1) + self.assertEqual(counts["pinned_cids"], 1) self.assertEqual(self.publisher.provided() | self.publisher.provided_keys(), provided) if withdrawn is not None: key = withdrawn if isinstance(withdrawn, bytes) else withdrawn.key @@ -766,9 +753,9 @@ def refresh(provided: set, withdrawn: Cid | bytes | None = None): self.assertEqual(self.held(self.publisher)["withdraw"], {N1: {key}, N2: {key}}) self.publisher.publish_due() # also published, so a later withdrawal has nodes to go to self.assertEqual(self.held(self.publisher)["withdraw"], {}) - counts["pins"] = 0 + counts["pinned_cids"] = 0 self.publisher.refresh() - self.assertEqual(counts["pins"], 0) + self.assertEqual(counts["pinned_cids"], 0) with self.subTest("pin"): a = self.pinned(b"a") @@ -834,16 +821,16 @@ def test_change_made_while_refreshing_is_seen_by_the_next(self): self.publisher = self.make_publisher(generation=True) a = self.pinned(b"a") built = builder.build_bytes(b"pinned meanwhile", self.storage.put) - pins = self.storage.pins + pinned_cids = self.storage.pinned_cids def pins_then_pin(): - """The pins as they were, then (once) another pin.""" - result = pins() + """The pinned CIDs as they were, then (once) another pin.""" + result = pinned_cids() if not self.storage.is_pinned(built.cid): self.storage.pin(built.cid, USER) return result - with mock.patch.object(self.storage, "pins", pins_then_pin): + with mock.patch.object(self.storage, "pinned_cids", pins_then_pin): self.publisher.refresh() self.assertEqual(self.publisher.provided(), {a}) self.publisher.refresh() @@ -874,6 +861,18 @@ def test_failed_save_is_written_next_time(self): restarted.refresh() self.assertEqual(restarted.publish_due()["published"], []) + def test_saves_are_one_transaction_not_durable(self): + """A save writes published, published_earlier and withdrawals in one transaction, not durably (NORMAL).""" + cid = self.pinned(b"normal") + self.publisher.refresh() + self.publisher.publish_due() + self.storage.unpin(cid, USER) + real = self.publisher._db.transaction + with mock.patch.object(self.publisher._db, "transaction", side_effect=real) as opened: + self.publisher.refresh() # the item's rows deleted and its withdrawals queued, in one save + self.assertEqual([call.args for call in opened.call_args_list], [(False,)]) + self.assertEqual(self.saved(), self.held(self.publisher)) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_rates.py b/tests/test_rates.py index ca4758b..0d223b2 100644 --- a/tests/test_rates.py +++ b/tests/test_rates.py @@ -3,7 +3,7 @@ import unittest from rnfs.config import Config -from rnfs.constants import CHUNK_SIZE, KIB +from rnfs.constants import CHUNK_SIZE, KIB, MAX_RESPONSE_SIZE, RESPONSE_OVERHEAD from rnfs.network import rates LINK = b"L" * 16 @@ -119,5 +119,34 @@ def test_requester_max_bytes(self): self.assertEqual(rates.requester_max_bytes(8000, config), 60_000) +class SizeRulesTest(unittest.TestCase): + """The /blocks size rules both ends share: response_capacity and the default upload cap.""" + + def test_response_capacity(self): + carried = MAX_RESPONSE_SIZE - RESPONSE_OVERHEAD + self.assertEqual(rates.response_capacity(10 * CHUNK_SIZE, None, None, 60), CHUNK_SIZE) # no estimate yet + self.assertEqual(rates.response_capacity(10 * 1024 * KIB, 1e9, None, 60), carried) + self.assertEqual(rates.response_capacity(1_000_000, 8000, None, 60), 60_000) + self.assertEqual(rates.response_capacity(1_000_000, 8000, 4000, 60), 30_000) # the cap's rate, if lower + self.assertEqual(rates.response_capacity(1_000_000, 8000, 16000, 60), 60_000) + self.assertEqual(rates.response_capacity(5000, 1e9, None, 60), 5000) + + def test_requester_capacity_is_its_max_bytes_up_to_one_response(self): + # What the fetcher estimates a response carries: max_bytes, at most what one response can hold + config = Config(response_target=60) + for rate in (None, 0, 1200, 8000, 50_000, 1e6, 1e9, 123_456.789): + with self.subTest(rate=rate): + max_bytes = rates.requester_max_bytes(rate, config) + self.assertEqual(rates.response_capacity(max_bytes, rate, None, config.response_target), + min(max_bytes, MAX_RESPONSE_SIZE - RESPONSE_OVERHEAD)) + + def test_default_upload_cap(self): + self.assertEqual(rates.default_upload_cap(True, 50_000), 25_000) + self.assertIsNone(rates.default_upload_cap(True, None)) # no estimate: no derived cap (NET-2) + self.assertIsNone(rates.default_upload_cap(False, 50_000)) + self.assertEqual(Config().effective_max_upload_rate(True, 50_000), 25_000) + self.assertEqual(Config(max_upload_rate=1000).effective_max_upload_rate(False, None), 1000) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_rns_compat.py b/tests/test_rns_compat.py index 1cec2e0..3a4b1b3 100644 --- a/tests/test_rns_compat.py +++ b/tests/test_rns_compat.py @@ -159,9 +159,20 @@ class Internal(NamedTuple): "random_blobs.append(random_blob)", "path_table_entry = [now, received_from, announce_hops, expires, random_blobs, " "packet.receiving_interface, packet.packet_hash]"))), - Internal("node_content.py", "Identity.recall(hash, from_identity_hash=, _no_use=), raising AttributeError " - "without a Reticulum instance unless _no_use", lambda: _recall()), - Internal("features/keepers.py", "Transport.get_cached_packet(packet_hash, packet_type=\"announce\") and the " + Internal("names.py", "Transport.path_table entries holding, at IDX_PT_PACKET, the packet hash of the announce " + "the path was taken from; Identity.remember storing each announce's packet hash at entry[1], a " + "known destination's too; the announce cache written, in Transport._inbound, only for an announce " + "that takes the path (should_add)", + lambda: (_true(sys.modules["RNS.Transport"].IDX_PT_PACKET == names._PATH_PACKET, "IDX_PT_PACKET moved"), + _source(RNS.Identity.remember, "entry[1] = packet_hash"), + _announce_cache_on_path())), + Internal("node_content.py, node.py, features/peers.py, network/links.py", + "Identity.recall(hash, from_identity_hash=, _no_use=), raising AttributeError without a Reticulum " + "instance unless _no_use (read-only checks pass _no_use=True so they don't mark the destination used)", + lambda: _recall()), + Internal("features/peers.py", "Identity.recall_app_data(hash, _no_use=)", + lambda: _params(RNS.Identity.recall_app_data, "_no_use")), + Internal("features/keepers.py, names.py", "Transport.get_cached_packet(packet_hash, packet_type=\"announce\") and the " "signed announce in it: public key, name hash, random hash (emission time in its last 5 bytes), " "ratchet, signature, app_data", lambda: _cached_announce()), Internal("names.py", "announce handlers with five parameters get announce_packet_hash and is_path_response", @@ -178,6 +189,12 @@ class Internal(NamedTuple): _has(RNS.Transport, "hops_to", "next_hop_interface"), _has(RNS.Reticulum, "get_next_hop_if_name", "get_instance", "configdir"), _assigns(RNS.Reticulum.__init__, "is_connected_to_shared_instance"))), + Internal("mesh.py", "Reticulum.get_path_table() entries {hash, interface}, interface being the str() of the " + "interface that get_next_hop_if_name gives (RnsPaths.infos)", + lambda: (_source(RNS.Reticulum.get_path_table, "'hash': dst_hash", + "'interface': str(RNS.Transport.path_table[dst_hash][5])"), + _source(RNS.Reticulum.get_next_hop_if_name, + "return str(RNS.Transport.next_hop_interface(destination))"))), Internal("mesh.py, network/rates.py", "Reticulum.get_interface_stats() {\"interfaces\": [{name, short_name, " "type, status, bitrate}]}, with LocalServerInterface and LocalClientInterface types", lambda: (_source(RNS.Reticulum.get_interface_stats, "ifstats['name'] = str(interface)", @@ -438,6 +455,20 @@ def _known_destinations(): "the public key doesn't make the destination hash") +def _announce_cache_on_path(): + """Transport._inbound caches announces (packet_type "announce") only under `if should_add:`.""" + tree = _tree(RNS.Transport._inbound) + guarded = {id(node) for branch in ast.walk(tree) + if isinstance(branch, ast.If) and ast.unparse(branch.test) == "should_add" + for statement in branch.body for node in ast.walk(statement)} + caches = [node for node in ast.walk(tree) if isinstance(node, ast.Call) + and ast.unparse(node.func) == "Transport.cache" + and any(keyword.arg == "packet_type" and ast.unparse(keyword.value) == "'announce'" + for keyword in node.keywords)] + _true(caches, "Transport._inbound no longer caches announces") + _true(all(id(node) in guarded for node in caches), "Transport._inbound caches an announce outside should_add") + + def _recall(): """Identity.recall with _no_use, as node_content does without a Reticulum instance.""" destination = _destination("rnfs", "dht") @@ -457,7 +488,8 @@ def _recall(): def _cached_announce(): - """An rnfs.name announce, cached as Transport caches it, read back and verified as keepers does.""" + """An rnfs.name announce, cached as Transport caches it, read back as names reads it and verified as keepers + does.""" _true((keepers.PUBLIC_KEY_LENGTH, keepers.NAME_HASH_LENGTH, keepers.RATCHET_LENGTH, keepers.SIGNATURE_LENGTH) == (RNS.Identity.KEYSIZE // 8, RNS.Identity.NAME_HASH_LENGTH // 8, RNS.Identity.RATCHETSIZE // 8, RNS.Identity.SIGLENGTH // 8), "announce field sizes changed") @@ -471,6 +503,9 @@ def _cached_announce(): os.mkdir(os.path.join(cache, "announces")) RNS.Transport.cache(packet, force_cache=True, packet_type="announce") cached = RNS.Transport.get_cached_packet(packet.get_hash(), packet_type="announce") + # As RnsNetwork.recalled_emission reads it + emitted, app_data_read = names.RnsNetwork._cached_announce(destination.hash, packet.get_hash()) + _true(app_data_read == app_data and abs(emitted - time.time()) < 60, f"names read {emitted}, {app_data_read}") # As KeeperFeature._from_rns_cache reads it _true(cached is not None and cached.unpack() is not False, "the cached packet doesn't unpack") _true(cached.destination_hash == destination.hash and cached.packet_type == RNS.Packet.ANNOUNCE, diff --git a/tests/test_routing.py b/tests/test_routing.py index 26bf314..9ec6b4a 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -408,6 +408,76 @@ def test_ban_of_unknown_node(self): self.assertFalse(self.table.is_banned(node_id(43))) + +class PruneTest(RoutingTestCase): + """Expired bans and dropped nodes are forgotten from the front of their expiry order (S4), and the outcome is + the same when an entry is out of order.""" + + def drop(self, n: bytes): + """Add n and fail it until it is dropped.""" + self.table.heard(n) + self.assertTrue(self.table.contains(n)) + while not self.table.failed(n): + pass + + def test_expired_entries_are_forgotten_when_a_node_is_heard(self): + dropped, banned = [node_id(100 + i) for i in range(3)], [node_id(200 + i) for i in range(3)] + for d, b in zip(dropped, banned): + self.drop(d) + self.table.ban(b) + self.clock.now += 10 + self.clock.now = 1000 + 3600 + 15 # the first two bans have ended + self.table.heard(node_id(5)) + self.assertEqual(list(self.table._bans), banned[2:]) + self.assertEqual(list(self.table._dropped), dropped) + self.clock.now = 1000 + 2 * ANNOUNCE + 5 # the first drop has expired, and every ban + self.table.heard(node_id(5)) + self.assertEqual(list(self.table._bans), []) + self.assertEqual(list(self.table._dropped), dropped[1:]) + + def test_setting_again_moves_an_entry_to_the_end(self): + first, second = node_id(42), node_id(43) + self.table.ban(first) + self.clock.now += 10 + self.table.ban(second) + self.clock.now += 10 + self.table.ban(first) # ends after second's ban now + self.assertEqual(list(self.table._bans), [second, first]) + self.clock.now += 3600 - 5 # second's ban has ended, not first's + self.assertTrue(self.table.heard(second)) + self.assertNotIn(second, self.table._bans) + self.assertFalse(self.table.heard(first)) + first, second = node_id(44), node_id(45) + self.drop(first) + self.clock.now += 10 + self.drop(second) + self.clock.now += 10 + self.drop(first) # back, then dropped again: after second + self.assertEqual(list(self.table._dropped), [second, first]) + + def test_out_of_order_entries_still_expire(self): + # A changed ban_time (or a clock step) can put a later-ending entry first: it then holds the ones behind it + # in memory, but they no longer count + long_ban, short_ban = node_id(42), node_id(43) + self.table.ban(long_ban) + self.config.ban_time = 10 + self.table.ban(short_ban) + self.clock.now += 11 + self.assertTrue(self.table.heard(short_ban)) + self.assertFalse(self.table.heard(long_ban)) + late, early = node_id(44), node_id(45) + self.clock.now = 5000 + self.drop(late) + self.clock.now = 1000 # stepped back + self.drop(early) + self.clock.now = 1000 + 2 * ANNOUNCE + 1 # early's seniority has expired, late's hasn't + self.table.heard(node_id(5)) + self.assertEqual(list(self.table._dropped), [late, early]) # held behind late + self.assertTrue(self.table.heard(early)) + self.assertEqual(self.table.seniority(early), 0) + self.assertTrue(self.table.heard(late)) + self.assertEqual(self.table.seniority(late), 1000 + 2 * ANNOUNCE + 1 - 5000) + class ThreadSafetyTest(RoutingTestCase): """Concurrent use from announce, request and lookup threads.""" diff --git a/tests/test_security_index.py b/tests/test_security_index.py new file mode 100644 index 0000000..7480718 --- /dev/null +++ b/tests/test_security_index.py @@ -0,0 +1,50 @@ +"""tests/security/README.md, the index from audit ID to test module, code and fix commit, stays in step with the +security tests: every tests/security/test_*.py is listed, and every test module a row names exists.""" +import os +import re +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SECURITY = os.path.join(REPO, "tests", "security") +INDEX = os.path.join(SECURITY, "README.md") + +# A row of one of the index's tables: "| ID | Test modules | Code guarded | Fix commits |" (not the header or rule). +ROW = re.compile(r"(?m)^\| (?!ID \||---)(.+?) \| (.+?) \| (.+?) \| (.+?) \|$") +# A test module in tests/security, named bare, and one elsewhere in tests/, named by its path. +SECURITY_TEST = re.compile(r"(? list[tuple[str, str]]: + """Each row of the index as (ID, test modules column).""" + with open(INDEX, encoding="utf-8") as f: + return [(match.group(1), match.group(2)) for match in ROW.finditer(f.read())] + + +class SecurityIndexTest(unittest.TestCase): + """The index lists every security test and names only ones that exist.""" + + def test_every_security_test_is_indexed(self): + """Each tests/security/test_*.py appears in some row's test modules.""" + listed = {name for _, tests in index_rows() for name in SECURITY_TEST.findall(tests)} + present = {name for name in os.listdir(SECURITY) if re.fullmatch(r"test_\w+\.py", name)} + self.assertEqual(present - listed, set(), "add these to tests/security/README.md") + + def test_every_indexed_test_exists(self): + """Every test module a row names exists: bare names in tests/security, tests/... paths in tests/.""" + rows = index_rows() + self.assertGreater(len(rows), 50) + for audit_id, tests in rows: + for name in SECURITY_TEST.findall(tests): + self.assertTrue(os.path.isfile(os.path.join(SECURITY, name)), f"{audit_id}: {name}") + for name in OTHER_TEST.findall(tests): + self.assertTrue(os.path.isfile(os.path.join(REPO, "tests", name)), f"{audit_id}: tests/{name}") + + def test_ids_are_unique(self): + """One row per ID, so a reader looking one up finds all of it in one place.""" + ids = [audit_id for audit_id, _ in index_rows() if not audit_id.startswith("none")] + self.assertEqual(sorted(set(ids)), sorted(ids)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_server.py b/tests/test_server.py index 2720366..e346ca5 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -11,12 +11,12 @@ from RNS.vendor import umsgpack from rnfs.config import Config -from rnfs.constants import (BLOCKS_PATH, CHUNK_SIZE, FIND_PATH, ID_LENGTH, K, KIB, MAX_REQUEST_SIZE, MAX_RESPONSE_SIZE, - PIN_PATH, PROVIDE_PATH, RESPONSE_OVERHEAD) +from rnfs.constants import (BLOCKS_PATH, CHUNK_SIZE, FIND_PATH, ID_LENGTH, K, KIB, MAX_REQUEST_SIZE, + MAX_REQUESTS_PER_LINK, MAX_RESPONSE_SIZE, PIN_PATH, PROVIDE_PATH, RESPONSE_OVERHEAD) from rnfs.content.cid import RAW, Cid from rnfs.network import guards, protocol, server from rnfs.network.identity import node_id_for_identity -from tests.support import FakeClock +from tests.support import FakeClock, wait_for KEY = bytes(range(ID_LENGTH)) @@ -362,6 +362,10 @@ def served_block(self, cid): """The served block for cid, if any.""" return self.blocks.get(cid) + def served_block_size(self, cid): + """Not known: the server reads each block to size it.""" + return None + def remote_pin(self, identity_hash, cid, pin): """Accept or refuse a remote pin.""" self.pins.append((identity_hash, cid, pin)) @@ -371,6 +375,10 @@ def is_member(self, identity_hash): """Whether the identity is a member.""" return identity_hash in self.members + def feature_requests(self): + """No feature request paths.""" + return {} + class RequestServerTest(unittest.TestCase): """Handlers and limits, driven directly with fake links.""" @@ -457,6 +465,70 @@ def test_pin_needs_listed_identity(self): self.assertIs(self.ask(PIN_PATH, [cid.binary, True], link, self.identity), True) self.assertEqual(self.backend.pins, [(self.identity.hash, cid, True)]) + def test_handler_that_raises_frees_its_slot(self): + """A handler whose work raises is answered False and frees its request slot, so the link isn't refused + from then on (MAX_REQUESTS_PER_LINK + 1 failures in a row would otherwise leave it with none).""" + def broken(*args): + """A backend or feature bug.""" + raise RuntimeError("bug") + self.backend.find = self.backend.provide = self.backend.remote_pin = broken + self.backend.feature_requests = lambda: {"/feature": broken} + self.make_server(self.config) + link = self.connect(b"a") + cid = self.big[0][0] + for path, data in ((FIND_PATH, KEY), (PROVIDE_PATH, [[KEY], []]), (PIN_PATH, [cid.binary, True]), + ("/feature", b"")): + with self.subTest(path=path): + for _ in range(MAX_REQUESTS_PER_LINK + 1): + self.assertIs(self.ask(path, data, link, self.identity), False) + self.assertEqual(self.server.incoming.in_progress(link.link_id), (0, 0)) + del self.backend.find, self.backend.provide, self.backend.remote_pin + self.assertEqual(len(self.ask(FIND_PATH, KEY, link)[0]), K) + self.assertIs(self.ask(PROVIDE_PATH, [[KEY], []], link, self.identity), True) + self.assertIs(self.ask(PIN_PATH, [cid.binary, True], link, self.identity), True) + + def test_feature_responses_requesters_refuse_are_answered_false(self): + """A feature's response is checked with protocol.body_refusal, as every requester since 0.1.4 checks it: + one of a shape they'd refuse is answered False (and logged as the bug it is) instead.""" + answers = {} + self.backend.feature_requests = lambda: {"/feature": lambda data, identity: answers[data]} + self.make_server(self.config) + link = self.connect(b"a") + too_many = [None] * (protocol.MAX_BLOCKS_CIDS + 1) + for response in ({"a": 1}, "text", 7, too_many, [bytes(protocol.MAX_BLOCK_SIZE + 1)], [[KEY], [KEY], [KEY]]): + with self.subTest(response=repr(response)[:40]): + answers[b"x"] = response + with mock.patch.object(server, "log") as log: + self.assertIs(self.ask("/feature", b"x", link), False) + log.assert_called_once_with(mock.ANY, server.LOG_ERROR) + self.assertIn("/feature", log.call_args.args[0]) + for response in (True, False, [], [b"a", None], [[KEY], [KEY]]): + with self.subTest(response=response): + answers[b"x"] = response + self.assertEqual(self.ask("/feature", b"x", link), response) + self.assertEqual(self.server.incoming.in_progress(link.link_id), (0, 0)) + + def test_busy_link_refused_after_the_checks(self): + """With both of a link's request slots taken, a request is refused without running its work, and a + refusal by a handler's own checks takes no slot.""" + calls = [] + self.backend.feature_requests = lambda: {"/feature": lambda data, identity: calls.append(data) or True} + self.make_server(self.config) + link = self.connect(b"a") + for _ in range(MAX_REQUESTS_PER_LINK): + self.assertTrue(self.server.incoming.start_request(link.link_id, blocks=False)) + with mock.patch.object(self.backend, "find") as find: + self.assertIs(self.ask(FIND_PATH, KEY, link), False) + find.assert_not_called() + self.assertIs(self.ask("/feature", b"x", link), False) + self.assertEqual(calls, []) + self.assertIs(self.ask(PIN_PATH, [b"x", True], link, self.identity), False) # malformed: refused anyway + self.assertEqual(self.server.incoming.in_progress(link.link_id), (MAX_REQUESTS_PER_LINK, 0)) + self.server.incoming.finish_request(link.link_id, blocks=False) + self.assertIs(self.ask("/feature", b"x", link), True) + self.assertEqual(calls, [b"x"]) + self.assertEqual(self.server.incoming.in_progress(link.link_id), (MAX_REQUESTS_PER_LINK - 1, 0)) + def test_private_mode_refuses_non_members(self): self.backend.members = {self.identity.hash} self.make_server(Config(private=True)) @@ -570,9 +642,17 @@ def test_second_blocks_refused_until_previous_concludes(self): self.assertIs(self.blocks(link, request_id=b"s" * 16), False) resource = self.send(link) self.assertIs(self.blocks(link, request_id=b"s" * 16), False) - # All parts sent, proof withheld: still refused (after a short grace), and the slots stay held + # All parts sent, proof withheld: still refused (after a short grace, on the server's clock), and the + # slots stay held resource.status = RNS.Resource.AWAITING_PROOF - self.assertIs(self.blocks(link, request_id=b"s" * 16), False) + results = [] + thread = threading.Thread(target=lambda: results.append(self.blocks(link, request_id=b"s" * 16))) + thread.start() + time.sleep(0.05) + self.assertEqual(results, []) # waiting out the grace + self.clock.now += server.PROOF_GRACE + thread.join(5) + self.assertEqual(results, [False]) self.assertEqual(len(self.server.pools.responses()), 1) # The proof arriving while the next request waits: accepted threading.Timer(0.1, resource.complete).start() @@ -583,22 +663,22 @@ def test_only_one_request_per_link_waits_for_a_withheld_proof(self): self.blocks(link) self.send(link).status = RNS.Resource.AWAITING_PROOF # all parts sent, proof withheld # A flood of /blocks on that link: one waits out the grace, the rest are refused at once - finished, results = [], [] + results = [] def ask(i): - """One /blocks request, noting when it was answered.""" + """One /blocks request, noting its answer.""" results.append(self.blocks(link, request_id=bytes([i]) * 16)) - finished.append(time.monotonic()) - start = time.monotonic() threads = [threading.Thread(target=ask, args=(i,)) for i in range(1, 9)] for thread in threads: thread.start() + wait_for(lambda: len(results) == 7, timeout=5, what="seven requests to be refused") + time.sleep(0.05) + self.assertEqual(len(results), 7) # the grace (2 × RTT + 1 s, on the server's clock) hasn't passed + self.clock.now += server.PROOF_GRACE for thread in threads: thread.join(5) self.assertEqual(results, [False] * 8) - quick = sum(1 for t in finished if t - start < 0.5) # the grace here is 2 × RTT + 1 s - self.assertEqual(quick, 7) self.assertEqual(len(self.server.pools.responses()), 1) self.assertEqual(self.server.incoming.in_progress(link.link_id), (1, 1)) # The waiter's mark is cleared, so the next request may wait for the proof again @@ -667,8 +747,11 @@ def test_upload_pacing_between_responses(self): bound = server.MAX_PACING_WAIT sleeps = [] with mock.patch.object(self.server, "_pace", lambda response, wait: sleeps.append(wait)): - self.blocks(link) # no rate estimate yet: no cap, no pacing + self.blocks(link) # no rate estimate yet: the explicit cap applies all the same (NET-2) self.send(link).complete() + self.assertGreater(self.server.pacer.wait_time(link.link_id), bound) + self.assertIs(self.blocks(link, request_id=b"q" * 16), False) + self.clock.now += 11 self.assertEqual(self.server.pacer.wait_time(link.link_id), 0) self.server.rates.record(link.link_id, 64 * KIB, 1_000_000) self.assertNotEqual(self.blocks(link, request_id=b"s" * 16), False) diff --git a/tests/test_shared_folder.py b/tests/test_shared_folder.py index da7fa7e..445b065 100644 --- a/tests/test_shared_folder.py +++ b/tests/test_shared_folder.py @@ -1,17 +1,25 @@ """Tests for the shared folder (specification section 8): entries imported and pinned with source "folder".""" +import hashlib import os import tempfile import threading +import time import unittest from unittest import mock from rnfs import fsutil, statedb -from rnfs.content import builder, shared_folder +from rnfs.config import Config +from rnfs.content import builder, chunker, shared_folder +from rnfs.content.builder import Built from rnfs.content.cid import RAW, Cid -from rnfs.content.shared_folder import SharedFolder +from rnfs.content.shared_folder import FolderFiles, SharedFolder from rnfs.content.storage import FOLDER, USER, Storage +from rnfs.errors import BlockMismatch, Denied, TooLarge +from rnfs.node import RnfsNode from tests.support import CAN_SYMLINK, HAS_FIFO, POSIX_MODES, state_execute, state_rows, write_legacy from tests.test_builder import swap_on_open, within +from tests.test_names import FakeNetwork +from tests.test_node import A_ID, NetClient SECRET = b"private key" @@ -130,11 +138,12 @@ def test_import_failing_for_a_reason_that_may_pass_is_tried_again(self): self.assertEqual(self.storage.pinned_by(FOLDER), {self.cid_of("a.txt")}) def test_import_failing_for_its_content_waits_for_a_change(self): - """A failure of the content itself (here ValueError) isn't tried again, however long, until it changes.""" + """A failure of the content itself (an RnfsError: here TooLarge) isn't tried again, however long, until it + changes.""" clock = [100.0] self.folder.clock = lambda: clock[0] self.write("a.txt", b"a") - with mock.patch.object(builder, "build_open", side_effect=ValueError("bad")), \ + with mock.patch.object(builder, "build_open", side_effect=TooLarge("too deep")), \ mock.patch("rnfs.content.shared_folder.log"): self.folder.scan() clock[0] += 100 * shared_folder.RETRY_AFTER @@ -143,6 +152,26 @@ def test_import_failing_for_its_content_waits_for_a_change(self): self.write("a.txt", b"changed") self.assertEqual(self.folder.scan(), {self.cid_of("a.txt")}) + def test_unexpected_value_error_is_tried_again(self): + """A ValueError isn't the content's (invalid names and special files inside are skipped): a bug, such as a + BlockMismatch, so the entry is logged with the error's type and tried again RETRY_AFTER later, not parked + until it changes (XC4).""" + for error in (ValueError("bug"), BlockMismatch("block does not match")): + with self.subTest(type(error).__name__): + clock = [100.0] + self.folder.clock = lambda: clock[0] + name = f"{type(error).__name__}.txt" + self.write(name, name.encode()) + with mock.patch.object(builder, "build_open", side_effect=error), \ + mock.patch("rnfs.content.shared_folder.log") as log: + self.folder.scan() + self.assertIn(type(error).__name__, log.call_args_list[0].args[0]) + self.assertEqual(log.call_args_list[0].args[1], shared_folder.LOG_WARNING) + with mock.patch.object(builder, "build_open", side_effect=AssertionError("tried again")): + self.folder.scan() # not yet: no retry before RETRY_AFTER + clock[0] += shared_folder.RETRY_AFTER + self.assertIn(self.cid_of(name), self.folder.scan()) + def test_unchanged_entry_whose_pin_fails_doesnt_stop_the_scan(self): """A failed pin of an unchanged entry is logged; the other entries are scanned and it is kept.""" self.write("a.txt", b"a") @@ -194,6 +223,253 @@ def slow_build_open(*args, **kwargs): scan.join(10) self.assertEqual(self.folder.names(), {a: "a.txt", b: "b.txt"}) + # The per-file cache (audit D1): inside an entry that changed, files that didn't aren't read again + + def tree(self, count: int, entry: str = "dir") -> list[str]: + """entry/f0 ... with content "file 0" ...: paths.""" + return [self.write(os.path.join(entry, f"f{i}"), f"file {i}".encode()) for i in range(count)] + + def rewrite(self, path: str, data: bytes): + """Write data over path (the same inode), a second later by its modification time (a rewrite within the + filesystem's time resolution would keep it).""" + st = os.stat(path) + with open(path, "wb") as f: + f.write(data) + os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns + 10**9)) + + def scan_counting(self, options: builder.BuildOptions | None = None) -> tuple[set[Cid], int]: + """A scan's pins, and how many files it read.""" + with mock.patch.object(builder, "build_file", wraps=builder.build_file) as build_file: + pins = self.folder.scan(options) + return pins, build_file.call_count + + def test_changing_one_file_reads_only_that_file(self): + paths = self.tree(5) + self.assertEqual(self.scan_counting(), ({self.cid_of("dir")}, 5)) + self.rewrite(paths[2], b"changed, and longer") + pins, read = self.scan_counting() + self.assertEqual(read, 1) + self.assertEqual(pins, {self.cid_of("dir")}) + self.assertEqual(self.storage.pinned_by(FOLDER), pins) + self.assertTrue(self.storage.is_complete(self.cid_of("dir"))) + + def test_same_size_rewrite_and_touch_are_read_again(self): + """A file rewritten with content of the same size, or only touched, has a new modification time: read.""" + paths = self.tree(3) + self.folder.scan() + self.rewrite(paths[0], b"FILE 0") + pins, read = self.scan_counting() + self.assertEqual((pins, read), ({self.cid_of("dir")}, 1)) + st = os.stat(paths[1]) + os.utime(paths[1], ns=(st.st_atime_ns, st.st_mtime_ns + 10**9)) + self.assertEqual(self.scan_counting(), (pins, 1)) # read again; the same content, so the same CID + + def test_renamed_files_are_not_read_again(self): + """A file renamed in place, over another, or into another entry is the same inode, size and time.""" + paths = self.tree(3) + self.tree(1, "other") + self.write("top.txt", b"a top-level file") + self.folder.scan() + os.rename(paths[0], os.path.join(self.files, "dir", "renamed")) + os.replace(paths[1], paths[2]) + os.rename(os.path.join(self.files, "top.txt"), os.path.join(self.files, "moved.txt")) + pins, read = self.scan_counting() + self.assertEqual(read, 0) + self.assertEqual(pins, {self.cid_of("dir"), self.cid_of("other"), self.cid_of("moved.txt")}) + os.rename(os.path.join(self.files, "dir", "renamed"), os.path.join(self.files, "other", "from dir")) + pins, read = self.scan_counting() + self.assertEqual(read, 0) + self.assertEqual(pins, {self.cid_of("dir"), self.cid_of("other"), self.cid_of("moved.txt")}) + self.assertEqual(self.storage.pinned_by(FOLDER), pins) + + def test_a_newly_denied_hash_is_still_refused(self): + """A remembered file's content hash goes through the deny check as if it were read.""" + paths = self.tree(3) + self.folder.scan(builder.BuildOptions(check=lambda digest: None)) # hashed: the deny list has hashes + denied = hashlib.sha256(b"file 1").digest() + + def deny(digest: bytes): + """The deny list's check, denying file 1's content.""" + if digest == denied: + raise Denied("denied") + + self.rewrite(paths[0], b"changed") + with mock.patch("rnfs.content.shared_folder.log") as log: + pins, read = self.scan_counting(builder.BuildOptions(check=deny)) + self.assertEqual(read, 1) # only the changed file: file 1's remembered hash was checked + self.assertEqual(pins, set()) + self.assertEqual(self.storage.pinned_by(FOLDER), set()) + self.assertIn("couldn't import", log.call_args.args[0]) + + def test_a_file_remembered_without_its_hash_is_read_when_there_is_a_check(self): + """Files built while the deny list had no hashes were never hashed: with a check, they are read again.""" + paths = self.tree(3) + self.folder.scan() + checked = [] + self.rewrite(paths[0], b"changed") + pins, read = self.scan_counting(builder.BuildOptions(check=checked.append)) + self.assertEqual((pins, read), ({self.cid_of("dir")}, 3)) + self.assertEqual(len(checked), 3) + self.rewrite(paths[0], b"changed again") + checked.clear() + self.assertEqual(self.scan_counting(builder.BuildOptions(check=checked.append)), ({self.cid_of("dir")}, 1)) + self.assertEqual(sorted(checked), sorted(hashlib.sha256(data).digest() + for data in (b"changed again", b"file 1", b"file 2"))) + + def test_a_denied_block_under_a_remembered_file_is_refused(self): + """A file one of whose blocks is denied since is read again, so storing that block refuses the import.""" + paths = self.tree(3) + self.folder.scan() + denied = Cid.for_block(RAW, b"file 1") + self.storage.deny = mock.Mock(cids=frozenset({denied.binary}), denies=lambda cid: cid == denied) + self.rewrite(paths[0], b"changed") + with mock.patch("rnfs.content.shared_folder.log"): + pins, read = self.scan_counting() + self.assertEqual(read, 2) + self.assertEqual(pins, set()) + + def test_a_block_gone_under_a_remembered_file_is_read_again(self): + """A remembered file whose block was deleted (damaged), or collected once its entry's pin went, is read + again and stored.""" + paths = self.tree(3) + (first,) = self.folder.scan() + self.storage._delete({Cid.for_block(RAW, b"file 1")}) + self.rewrite(paths[0], b"changed") + pins, read = self.scan_counting() + self.assertEqual((pins, read), ({self.cid_of("dir")}, 2)) + self.assertTrue(self.storage.is_complete(self.cid_of("dir"))) + (second,) = pins + self.storage.unpin(second, FOLDER) # as `rnfs gc` after the pin went: every block of it is collected + self.storage.gc() + self.assertFalse(self.storage.is_complete(Cid.for_block(RAW, b"file 2"))) + self.rewrite(paths[0], b"changed again") + pins, read = self.scan_counting() + self.assertEqual((pins, read), ({self.cid_of("dir")}, 3)) + self.assertTrue(self.storage.is_complete(self.cid_of("dir"))) + self.assertNotEqual(first, second) + + def test_a_file_written_while_read_is_not_remembered(self): + """A file that changed while it was read may have given neither version: it is read again next time.""" + paths = self.tree(2) + build_file = builder.build_file + before = builder.file_state(os.stat(paths[0])) + + def writing(stream, *args, **kwargs): + """build_file, appending to f0 once it is read.""" + built = build_file(stream, *args, **kwargs) + if built.cid == Cid.for_block(RAW, b"file 0"): + with open(paths[0], "ab") as f: + f.write(b" and more") + return built + + with mock.patch.object(builder, "build_file", side_effect=writing): + self.folder.scan() + self.assertIsNone(self.folder.files.get(before)) + self.assertIsNotNone(self.folder.files.get(builder.file_state(os.stat(paths[1])))) + self.assertEqual(self.scan_counting(), ({self.cid_of("dir")}, 1)) + + def test_removed_entries_are_forgotten(self): + """The files of an entry removed, or of one changed, that it no longer holds, are forgotten.""" + paths = self.tree(2) + other = self.tree(1, "other") + self.folder.scan() + states = [builder.file_state(os.stat(path)) for path in [*paths, *other]] + os.remove(paths[0]) + os.remove(other[0]) + os.rmdir(os.path.join(self.files, "other")) + self.folder.scan() + self.assertEqual([self.folder.files.get(state) is not None for state in states], [False, True, False]) + + def test_only_fixed_chunks_are_remembered(self): + """What files were built as stands only for the folder's chunker (fixed): a scan with another reads all.""" + paths = self.tree(2) + self.folder.scan(builder.BuildOptions(chunker.FASTCDC)) + self.assertIsNone(self.folder.files.get(builder.file_state(os.stat(paths[0])))) + + def file_rows(self) -> dict: + """What the shared_folder_files table holds: (entry name, state) -> (CID, height, digest).""" + return {(statedb.load(name), tuple(statedb.load(state))): (Cid.from_bytes(cid), height, digest) + for name, state, cid, height, digest in state_rows( + self.state, "SELECT name, state, cid, height, digest FROM shared_folder_files")} + + def test_the_file_cache_survives_a_restart(self): + """What files were built as is saved (by save(), with the node's batched stores), and a restart keeps it; + rows follow the entries: a file replaced or removed, and an entry removed, take their rows with them.""" + paths = self.tree(3) + hashing = builder.BuildOptions(check=lambda digest: None) + self.folder.scan(hashing) + self.assertEqual(self.file_rows(), {}) # saved by save(), not by the scan + self.folder.files.save() + states = [builder.file_state(os.stat(path)) for path in paths] + self.assertEqual(self.file_rows(), {("dir", state): (Cid.for_block(RAW, f"file {i}".encode()), 0, + hashlib.sha256(f"file {i}".encode()).digest()) + for i, state in enumerate(states)}) + self.folder.close() + self.folder = self.open() + self.rewrite(paths[0], b"changed") + self.assertEqual(self.scan_counting(hashing), ({self.cid_of("dir")}, 1)) + self.folder.files.save() + rows = self.file_rows() + self.assertEqual(len(rows), 3) + self.assertNotIn(("dir", states[0]), rows) + self.assertIn(("dir", builder.file_state(os.stat(paths[0]))), rows) + os.remove(paths[1]) + self.folder.scan(hashing) + self.folder.files.save() + self.assertEqual(len(self.file_rows()), 2) + for path in (paths[0], paths[2]): + os.remove(path) + os.rmdir(os.path.dirname(paths[0])) + self.folder.scan(hashing) + self.folder.files.save() + self.assertEqual(self.file_rows(), {}) + + def test_damaged_file_rows_are_deleted(self): + """File rows that don't parse are skipped at load and deleted; the others are kept.""" + (path,) = self.tree(1) + self.folder.scan() + self.folder.files.save() + ((name, state), (cid, _, _)), = self.file_rows().items() + good = statedb.dump(list(state)) + bad = [("not json", good, cid.binary, 0, None), ("7", good, cid.binary, 0, None), + ('"x"', "not json", cid.binary, 0, None), ('"x"', "[1, 2]", cid.binary, 0, None), + ('"x"', "[1, 2, -3, 4, false]", cid.binary, 0, None), ('"x"', "[true, 2, 3, 4, false]", cid.binary, 0, None), + ('"x"', "[1, 2, 3, 4, 1]", cid.binary, 0, None), ('"a"', good, b"not a cid", 0, None), + ('"b"', good, cid.binary, -1, None), ('"c"', good, cid.binary, "zero", None), + ('"d"', good, cid.binary, 0, b"short")] + for row in bad: + state_execute(self.state, "INSERT INTO shared_folder_files (name, state, cid, height, digest) " + "VALUES (?, ?, ?, ?, ?)", row) + with mock.patch("rnfs.content.shared_folder.log"): + folder = self.open() + self.assertEqual(set(self.file_rows()), {(name, state)}) + self.assertEqual(folder.files.get(builder.file_state(os.stat(path))), Built(cid, 6, 0, 0)) + + def test_failed_file_save_is_written_by_the_next(self): + """A save whose commit failed raises, keeps the changes, and the next save writes them.""" + self.tree(2) + self.folder.scan() + with mock.patch.object(self.folder.files._rows.db, "transaction", + side_effect=statedb.StateDbError("disk I/O error")): + with self.assertRaises(OSError): + self.folder.files.save() + self.assertEqual(self.file_rows(), {}) + self.folder.files.save() + self.assertEqual(len(self.file_rows()), 2) + + def test_file_entry_names_that_are_not_unicode_round_trip(self): + """An entry name with a lone surrogate is saved and loaded as it was.""" + files = FolderFiles(self.state) + self.addCleanup(files.close) + state = (1, 2, 1, 4, False) + built = Built(Cid.for_block(RAW, b"x"), 1, 0, 0, hashlib.sha256(b"x").digest()) + files.update("caf\udce9", {state: built}, complete=True) + files.save() + loaded = FolderFiles(self.state) + self.addCleanup(loaded.close) + self.assertEqual(loaded._entries, {"caf\udce9": {state: built}}) + self.assertEqual(loaded.get(state).digest, built.digest) + def test_creates_missing_directory(self): self.assertEqual(self.folder.scan(), set()) self.assertTrue(os.path.isdir(self.files)) @@ -364,5 +640,39 @@ def swap(): self.assertEqual(within(5, self.folder.scan), pins) +class NodeFileCacheTest(unittest.TestCase): + """The node saves the shared folder's per-file cache with its other stores (D1, XC5).""" + + def test_saved_at_stop_and_used_after_a_restart(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + config = Config(rnfs_dir=os.path.join(tmp.name, "rnfs"), slow="no") + paths = [] + for i in range(3): + path = os.path.join(config.files_dir, "dir", f"f{i}") + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(f"file {i}".encode()) + paths.append(path) + node = RnfsNode.offline(config, A_ID, NetClient(), FakeNetwork()) + self.addCleanup(node.stop) + self.assertIn(node.shared_folder.files, node._every_store(batched=True)) + node.maintain(time.monotonic(), {}) + self.assertEqual(state_rows(config.state_dir, "SELECT COUNT(*) FROM shared_folder_files"), [(0,)]) + node.stop() + self.assertEqual(state_rows(config.state_dir, "SELECT COUNT(*) FROM shared_folder_files"), [(3,)]) + st = os.stat(paths[0]) + with open(paths[0], "wb") as f: + f.write(b"changed") + os.utime(paths[0], ns=(st.st_atime_ns, st.st_mtime_ns + 10**9)) + node = RnfsNode.offline(config, A_ID, NetClient(), FakeNetwork()) + self.addCleanup(node.stop) + with mock.patch.object(builder, "build_file", wraps=builder.build_file) as build_file: + node.maintain(time.monotonic(), {}) + self.assertEqual(build_file.call_count, 1) + self.assertEqual(node.storage.pinned_by(FOLDER), + {builder.add_path(os.path.join(config.files_dir, "dir"), builder.NO_PUT)[0].cid}) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_statedb.py b/tests/test_statedb.py index 08cd389..3ed63c1 100644 --- a/tests/test_statedb.py +++ b/tests/test_statedb.py @@ -1,12 +1,15 @@ """Tests for the state database (rnfs.statedb): opening and checking it, transactions, references, and the migration of the JSON state files earlier versions wrote.""" import builtins +import hashlib import json import os import sqlite3 import tempfile import threading import unittest +from collections.abc import Iterator +from contextlib import contextmanager from unittest import mock import RNS @@ -15,6 +18,11 @@ from rnfs.errors import RnfsError from tests.support import POSIX_MODES, failing_commits, state_execute, state_rows, write_legacy +# The schema steps rnfs 0.1.5 released: a database 0.1.5 wrote has them already, so an edit to one would never +# reach it. Never change this digest: append a step instead. +RELEASED_0_1_5 = "50bd4ec20bbf10e408a9313cb168d35915af0f9d2afcf850d812dd1a7b7fb72b" +RELEASED_0_1_5_STEPS = 16 + _saved_loglevel = RNS.loglevel @@ -103,6 +111,43 @@ def test_older_schema_is_upgraded_step_by_step(self): self.assertEqual(db.query("SELECT cid, source FROM pins"), [(b"x", "user")]) self.assertEqual(db.query("SELECT count(*) FROM lxmf_messages"), [(0,)]) + def test_released_steps_are_unchanged(self): + """The steps 0.1.5 released are exactly as released (statedb.SCHEMA's comment): new steps are appended.""" + released = statedb.SCHEMA[:RELEASED_0_1_5_STEPS] + self.assertTrue(all(isinstance(step, str) for _, steps in released for step in steps)) + digest = hashlib.sha256(json.dumps([[label, list(steps)] for label, steps in released]).encode()).hexdigest() + self.assertEqual(digest, RELEASED_0_1_5) + + def test_callable_step_runs_in_the_upgrade_transaction(self): + """A step's callable gets the upgrade's transaction: its writes commit with the step, and a raise rolls the + whole step back (user_version included) and stops startup.""" + calls = [] + + def copy_pins(tx): + calls.append(tx) + tx.execute("INSERT INTO extra (cid) SELECT cid FROM pins") + + with mock.patch.object(statedb, "SCHEMA", statedb.SCHEMA[:2]): + db = statedb.connect(self.state) + with db.transaction() as tx: + tx.execute("INSERT INTO pins (cid, source) VALUES (?, ?)", (b"x", "user")) + db.close() + extra = ("extra", ("CREATE TABLE extra (cid BLOB PRIMARY KEY) WITHOUT ROWID", copy_pins)) + + def failing(tx): + raise RuntimeError("migration failed") + + with mock.patch.object(statedb, "SCHEMA", (*statedb.SCHEMA[:2], ("broken", ( + "CREATE TABLE broken (x)", failing)))), self.assertRaises(RuntimeError): + statedb.connect(self.state) + self.assertEqual(state_rows(self.state, "PRAGMA user_version"), [(2,)]) + self.assertEqual(state_rows(self.state, "SELECT name FROM sqlite_master WHERE name = 'broken'"), []) + with mock.patch.object(statedb, "SCHEMA", (*statedb.SCHEMA[:2], extra)): + self.connect().close() + self.assertEqual(len(calls), 1) + self.assertEqual(state_rows(self.state, "PRAGMA user_version"), [(3,)]) + self.assertEqual(state_rows(self.state, "SELECT cid FROM extra"), [(b"x",)]) + def test_file_that_isnt_a_database_stops_startup(self): """A file that isn't an SQLite database is refused with what to do, and left as it was.""" os.makedirs(self.state) @@ -276,6 +321,220 @@ def test_dump_round_trips_any_text(self): self.assertIn("é \U0001f600", text) +class RowStoreTest(StateDbTest): + """RowStore, on the contacts table (nick -> (kind, id)), and the other store helpers.""" + + def setUp(self): + """A state database and an address book kept in memory under its own lock.""" + super().setUp() + self.db = self.connect() + self.lock = threading.Lock() + self.memory: dict[str, tuple[str, bytes]] = {} + + def store(self, mode: str, durable: bool = True, **options) -> statedb.RowStore: + """A RowStore for self.memory in mode.""" + return statedb.RowStore(self.db, "contacts", ("nick",), durable=durable, mode=mode, lock=self.lock, + row=self.row, **options) + + def row(self, nick: str) -> tuple | None: + """nick's contacts row, or None if it is gone.""" + entry = self.memory.get(nick) + return None if entry is None else (nick, *entry) + + def set(self, store: statedb.RowStore, nick: str, entry: tuple[str, bytes] | None): + """Change nick in memory (None: delete it) and tell store, as a store does.""" + with self.lock: + if entry is None: + self.memory.pop(nick, None) + else: + self.memory[nick] = entry + store.changed(nick) + + def saved(self) -> dict: + """What the contacts table holds.""" + return {nick: (kind, ident) for nick, kind, ident in state_rows(self.state, "SELECT * FROM contacts")} + + @contextmanager + def transactions(self) -> Iterator[list[bool]]: + """Within the with block, the durable flag of each transaction opened on self.db, in order.""" + opened, real = [], self.db.transaction + + def transaction(durable: bool = False): + """The real transaction, noted.""" + opened.append(durable) + return real(durable) + + with mock.patch.object(self.db, "transaction", transaction): + yield opened + + def test_write_through_writes_each_change_at_once(self): + """WRITE_THROUGH: each change is committed by changed() itself, durable as asked; a key gone is deleted.""" + for durable in (True, False): + with self.subTest(durable=durable): + store = self.store(statedb.WRITE_THROUGH, durable) + with self.transactions() as opened: + self.set(store, "alice", ("name", b"a" * 16)) + self.assertEqual(self.saved(), {"alice": ("name", b"a" * 16)}) + self.set(store, "alice", None) + self.assertEqual(self.saved(), {}) + self.assertEqual(opened, [durable, durable]) + store.save() # nothing left to write + self.assertEqual(store.pending, frozenset()) + + def test_write_through_failed_transaction_raises_and_is_written_next(self): + """WRITE_THROUGH: a failing transaction raises OSError and changes nothing on disk; the change stays in + memory, and the next change, or a save, writes it.""" + store = self.store(statedb.WRITE_THROUGH) + self.set(store, "alice", ("name", b"a" * 16)) + with failing_commits(), self.assertRaises(OSError): + self.set(store, "bob", ("node", b"b" * 16)) + self.assertEqual(self.saved(), {"alice": ("name", b"a" * 16)}) + self.assertIn("bob", self.memory) + self.assertEqual(store.pending, {"bob"}) + self.set(store, "alice", None) + self.assertEqual(self.saved(), {"bob": ("node", b"b" * 16)}) + with failing_commits(), self.assertRaises(OSError): + self.set(store, "carol", ("lxmf", b"c" * 16)) + store.save() + self.assertEqual(set(self.saved()), {"bob", "carol"}) + self.assertEqual(store.pending, frozenset()) + + def test_snapshot_writes_at_save_outside_the_lock(self): + """SNAPSHOT: changed() writes nothing; save() writes every change since the last save in one transaction, + durable as asked, with the store's lock free while it commits.""" + store = self.store(statedb.SNAPSHOT, durable=False) + self.set(store, "alice", ("name", b"a" * 16)) + self.set(store, "bob", ("node", b"b" * 16)) + self.assertEqual(self.saved(), {}) + free = [] + real = statedb._Database._commit + + def commit(database, conn): + """Note whether the store's lock is free, then commit.""" + free.append(self.lock.acquire(blocking=False) and (self.lock.release() or True)) + real(database, conn) + + with self.transactions() as opened, mock.patch.object(statedb._Database, "_commit", commit): + store.save() + store.save() # nothing changed since + self.assertEqual((opened, free), ([False], [True])) + self.assertEqual(set(self.saved()), {"alice", "bob"}) + + def test_snapshot_failed_transaction_is_written_by_the_next_save(self): + """SNAPSHOT: a failing transaction raises OSError and the keys it held are noted again, with changes made + since; the next save writes them all.""" + store = self.store(statedb.SNAPSHOT) + self.set(store, "alice", ("name", b"a" * 16)) + with failing_commits(), self.assertRaises(OSError): + store.save() + self.assertEqual(self.saved(), {}) + self.assertEqual(store.pending, {"alice"}) + self.set(store, "bob", ("node", b"b" * 16)) + store.save() + self.assertEqual(set(self.saved()), {"alice", "bob"}) + self.assertEqual(store.pending, frozenset()) + + def test_snapshot_an_older_snapshot_never_lands_last(self): + """SNAPSHOT: a save taking its snapshot while another writes waits for it, so the newer row lands last.""" + store = self.store(statedb.SNAPSHOT) + self.set(store, "alice", ("name", b"1" * 16)) + later = [] + real = statedb._Database._commit + + def commit(database, conn): + """As the first save commits, change alice and start a second save on another thread.""" + if not later: + self.set(store, "alice", ("name", b"2" * 16)) + later.append(threading.Thread(target=store.save)) + later[0].start() + later[0].join(0.2) # it waits for this save to end + self.assertTrue(later[0].is_alive()) + real(database, conn) + + with mock.patch.object(statedb._Database, "_commit", commit): + store.save() + later[0].join(5) + self.assertEqual(self.saved(), {"alice": ("name", b"2" * 16)}) + + def test_skip_unchanged_writes_only_rows_that_changed(self): + """skip_unchanged: a key whose row is as last written is left out, and no transaction opens for none.""" + for mode in (statedb.WRITE_THROUGH, statedb.SNAPSHOT): + with self.subTest(mode=mode): + store = self.store(mode, skip_unchanged=True) + self.set(store, "alice", ("name", b"a" * 16)) + store.save() + with self.transactions() as opened: + self.set(store, "alice", ("name", b"a" * 16)) + store.save() + self.assertEqual(opened, []) + self.set(store, "alice", None) + store.save() + self.assertEqual(self.saved(), {}) + + def test_unknown_mode_is_refused(self): + with self.assertRaises(ValueError): + self.store("sometimes") + + def test_open_store_releases_the_database_if_the_load_fails(self): + """open_store gives the reference once load has read it; a load that raises releases it.""" + db = statedb.open_store(self.state, lambda db: db.query("SELECT * FROM contacts")) + db.close() + self.db.close() + with self.assertRaises(KeyError): + statedb.open_store(self.state, lambda db: {}["missing"]) + self.assertNotIn(self.path, statedb.open_databases()) + + def test_delete_rows(self): + """delete_rows deletes the rows with those keys, in one transaction, and nothing for none.""" + with self.db.transaction() as tx: + tx.executemany("INSERT INTO contacts VALUES (?, 'name', ?)", [("a", b"a"), ("b", b"b"), ("c", b"c")]) + with self.transactions() as opened: + statedb.delete_rows(self.db, "contacts", ("nick",), []) + statedb.delete_rows(self.db, "contacts", ("nick",), [("a",), ("c",)]) + self.assertEqual((opened, set(self.saved())), ([False], {"b"})) + + +class ReconcilePinsTest(unittest.TestCase): + """reconcile_pins: a feature's pins against its saved state.""" + + PINS = {"cid1": {"user", "lxmf:aa"}, "cid2": {"lxmf:aa", "lxmf:bb"}, "cid3": {"follow:aa"}, "cid4": {"lxmf:cc"}} + + def reconcile(self, clean: bool, release=None) -> tuple[list, list]: + """reconcile_pins over PINS for "lxmf:", with lxmf:bb's pin of cid2 recorded: (adopted, released).""" + adopted, released = [], [] + statedb.reconcile_pins(self.PINS, "lxmf:", lambda owner, cid: (owner, cid) == ("bb", "cid2"), clean, + adopted.append, release or (lambda owner, cid: released.append((owner, cid)))) + return adopted, released + + def test_clean_state_releases_the_pins_it_doesnt_record(self): + adopted, released = self.reconcile(clean=True) + self.assertEqual((adopted, sorted(released)), ([], [("aa", "cid1"), ("aa", "cid2"), ("cc", "cid4")])) + + def test_state_not_clean_adopts_them(self): + adopted, released = self.reconcile(clean=False) + self.assertEqual((adopted, released), ([{"aa": ["cid1", "cid2"], "cc": ["cid4"]}], [])) + + def test_a_failed_release_is_logged_and_the_rest_released(self): + released = [] + + def release(owner, cid): + """Fail for cid1, as a read-only state directory would.""" + if cid == "cid1": + raise OSError("read-only") + released.append(cid) + + with mock.patch.object(statedb, "log") as log: + self.reconcile(clean=True, release=release) + self.assertEqual(sorted(released), ["cid2", "cid4"]) + self.assertIn("could not release cid1", log.call_args.args[0]) + + def test_nothing_unrecorded_does_nothing(self): + adopted = [] + statedb.reconcile_pins({"cid1": {"lxmf:aa"}}, "lxmf:", lambda owner, cid: True, False, adopted.append, + None) + self.assertEqual(adopted, []) + + def _import_rows(value, tx) -> int: """A test importer: value's strings become pins of source "user"; returns how many.""" tx.execute("DELETE FROM pins") @@ -397,7 +656,7 @@ def connect(): """Connect, keeping the reference or the error.""" try: dbs.append(statedb.connect(self.state)) - except BaseException as e: # noqa: BLE001 - reported below + except BaseException as e: # reported below errors.append(e) with mock.patch.object(statedb, "LEGACY", TEST_LEGACY), \ diff --git a/tests/test_storage.py b/tests/test_storage.py index 4141202..4b73249 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -1,12 +1,13 @@ """Tests for Storage (specification section 8): pins and sources, cached fetches, protection, eviction, GC.""" import os +import random import shutil import tempfile import threading import unittest from unittest import mock -from rnfs import fsutil, statedb +from rnfs import constants, fsutil, statedb from rnfs.content import builder, dag, storage as storage_module, walker from rnfs.content.cid import DAG, RAW, Cid from rnfs.content.dag import Child, Entry, FileNode @@ -291,6 +292,80 @@ def test_pin_count(self): self.storage.unpin(a, FOLDER) self.assertEqual(self.storage.pin_count(), 1) + def test_has_pin(self): + """has_pin(cid, source) agrees with pins() through pins, unpins, a failed unpin and a reopen.""" + a, b = self.add(b"a").cid, self.add(b"b").cid + lxmf = "lxmf:" + IDENTITY + + def check(storage): + pins = storage.pins() + for cid in (a, b): + for source in (USER, FOLDER, lxmf): + self.assertEqual(storage.has_pin(cid, source), source in pins.get(cid, set()), (cid, source)) + + check(self.storage) + self.assertFalse(self.storage.has_pin(a, USER)) + self.storage.pin(a, USER) + self.storage.pin_many([a, b], lxmf) + self.assertTrue(self.storage.has_pin(a, lxmf)) + self.assertFalse(self.storage.has_pin(b, USER)) + check(self.storage) + with failing_commits(): + with self.assertRaises(OSError): + self.storage.unpin(b, lxmf) + self.assertTrue(self.storage.has_pin(b, lxmf)) # still pinned, as the database says + self.storage.unpin(a, USER) + self.assertFalse(self.storage.has_pin(a, USER)) + self.assertTrue(self.storage.has_pin(a, lxmf)) + self.storage.unpin(b, lxmf) # its last source: b is no longer pinned at all + self.assertFalse(self.storage.has_pin(b, lxmf)) + check(self.storage) + check(self.open()) + + def test_pinned_cids(self): + """pinned_cids() is pins()'s CIDs, the same frozenset while pins don't change, through pins, a second + source, unpins, failed saves (asked for while saving, too) and a reopen.""" + a, b, c = (self.add(b"%d" % i).cid for i in range(3)) + + def check(storage): + self.assertEqual(storage.pinned_cids(), frozenset(storage.pins())) + + empty = self.storage.pinned_cids() + self.assertEqual(empty, frozenset()) + self.assertIs(self.storage.pinned_cids(), empty) + self.storage.pin(a, USER) + pinned = self.storage.pinned_cids() + self.assertEqual(pinned, {a}) + self.assertIs(self.storage.pinned_cids(), pinned) + self.storage.pin_many([a, b], FOLDER) + self.assertEqual(self.storage.pinned_cids(), {a, b}) + check(self.storage) + self.storage.pin(a, FOLDER) # nothing changes + self.assertIs(self.storage.pinned_cids(), self.storage.pinned_cids()) + + def failing_save(*args, **kwargs): + """Ask for the set while the change is in memory but not saved, then fail as a full disk does.""" + self.storage.pinned_cids() + raise OSError("database or disk is full") + + with mock.patch.object(self.storage, "_save_pins", side_effect=failing_save): + with self.assertRaises(OSError): + self.storage.pin(c, USER) + self.assertEqual(self.storage.pinned_cids(), {a, b}) + with self.assertRaises(OSError): + self.storage.unpin(b, FOLDER) + self.assertEqual(self.storage.pinned_cids(), {a, b}) + with failing_commits(): + with self.assertRaises(OSError): + self.storage.pin(c, USER) + check(self.storage) + self.storage.unpin(a, USER) + self.assertEqual(self.storage.pinned_cids(), {a, b}) # still pinned by the folder + self.storage.unpin(a, FOLDER) + self.assertEqual(self.storage.pinned_cids(), {b}) + check(self.storage) + check(self.open()) + def test_damaged_state_is_moved_aside(self): with open(os.path.join(self.tmp.name, "state", "pins.json"), "w", encoding="utf-8") as f: f.write("{not json") @@ -451,6 +526,49 @@ def test_cached_fetch_keeps_path_nodes_and_target(self): self.assertEqual(lost, set()) # a is still complete; b was never a target self.assertEqual(self.storage.cached_fetches(), [(root.cid, ("a",), a.cid)]) + def test_warm_reads_what_the_path_walk_reaches(self): + """_warm and _path_cids share one path walk: warming reads exactly the stored nodes a record's path + reaches, through index nodes, missing names, files and nodes not stored, and the locked walk then reads + nothing.""" + with mock.patch.multiple(constants, MAX_CHILDREN=3): + files = [self.add(b"%02d" % i * 20) for i in range(10)] + [self.add(b"G" * 70000)] # raw blocks; a node + names = [f"f{i:02d}" for i in range(10)] + ["g"] + big = builder.build_directory([Entry(n, f.cid, f.size, 0) for n, f in zip(names, files)], + self.storage.put) # index nodes over groups of 3 entries + inner = builder.wrap("big", big, self.storage.put) + root = builder.wrap("inner", inner, self.storage.put) + absent = builder.wrap("x", builder.build_bytes(b"absent", builder.NO_PUT), builder.NO_PUT) + missing = builder.wrap("absent", absent, self.storage.put) # its one entry isn't stored + paths = [(root.cid, ("inner", "big", "f05")), (root.cid, ("inner", "big", "g")), + (root.cid, ("inner", "big", "a")), (root.cid, ("inner", "big", "zz")), + (root.cid, ("inner", "nothing")), (root.cid, ("inner", "big", "f00", "below a raw block")), + (root.cid, ("inner", "big", "g", "below a file node")), (missing.cid, ("absent", "x")), + (root.cid, ()), (files[0].cid, ("x",))] + for path in paths: + with self.subTest(path=path): + self.storage = self.open() # nothing memoised + read = set() + real = Storage._read_node + with mock.patch.object(Storage, "_read_node", lambda s, cid: (read.add(cid), real(s, cid))[1]): + self.storage._warm(set(), [path]) + stored = {cid for cid in read if self.storage.has(cid)} + read.clear() + with self.storage._lock: + reached = self.storage._path_cids(*path) + self.assertEqual(reached, stored) + self.assertEqual(read, set()) # warmed: the walk under the lock read nothing + # What the walk reaches is what walker.step passes through + expected, expect = set(), walker.root_expect(root.cid) + for name in ("inner", "big", "f05"): + while True: + expected.add(expect.cid) + expect = walker.step(expect, self.storage.get(expect.cid), name) + if not walker.is_index_step(expect): + break + self.assertGreater(len(expected), 4) # through index nodes + with self.storage._lock: + self.assertEqual(self.storage._path_cids(root.cid, ("inner", "big", "f05")), expected) + def test_lru_eviction(self): first = self.add(b"1" * 1000) second = self.add(b"2" * 1000) @@ -621,6 +739,40 @@ def test_verify_checks_completeness_without_the_lock(self): self.assertEqual(set(stats["bad"]), damaged) self.assertEqual(stats["incomplete"], {pinned.cid}) # not partial: it was incomplete before + def test_every_record_removal_is_saved_counted_and_collected(self): + """The cap, eviction, drop_denied and gc each remove records with the same bookkeeping (_drop_records): + the cache generation moves, the next save deletes their rows, and their blocks are garbage.""" + files = [self.add(b"%d" % i * 1000) for i in range(6)] + for built in files: + self.storage.record_fetch(built.cid, [], built.cid) + + def removed(operation) -> set[Cid]: + """The targets operation removed from the records, checked against a reopened storage.""" + before = self.storage.cached_targets() + generation = self.storage.cache_generation() + operation() + gone = before - self.storage.cached_targets() + self.assertGreater(self.storage.cache_generation(), generation) + self.storage.save() + self.assertEqual(self.open().cached_fetches(), self.storage.cached_fetches()) + if any(self.storage.has(cid) for cid in gone): + self.assertTrue(self.storage._maybe_garbage) # the next collection takes them + self.storage.collect() + self.assertFalse(any(self.storage.has(cid) for cid in gone)) + return gone + + with mock.patch.object(storage_module, "MAX_CACHE_RECORDS", 5): + # Touching files[0] leaves files[1] the oldest, past the cap + self.assertEqual(removed(lambda: self.storage.record_fetch(files[0].cid, [], files[0].cid)), + {files[1].cid}) + self.storage.max_storage = self.storage.used_bytes() - 500 + self.assertEqual(removed(self.storage.evict), {files[2].cid}) + stats = {} + self.assertEqual(removed(lambda: stats.update(self.storage.drop_denied(frozenset({files[3].cid.binary})))), + {files[3].cid}) + self.assertEqual((stats["uncached"], stats["incomplete"]), ({files[3].cid}, {files[3].cid})) + self.assertEqual(removed(self.storage.gc), {files[0].cid, files[4].cid, files[5].cid}) + def test_cache_generation(self): files = [self.add(b"%d" % i * 1000) for i in range(4)] generation = self.storage.cache_generation() @@ -805,5 +957,213 @@ def test_size_queries_read_nodes_without_the_lock(self): self.assertFalse(any(cid.is_raw for cid in self.storage._links)) # raw blocks list nothing +def walked_reach(storage: Storage, roots, known=frozenset(), damaged=None, partial=frozenset()) -> set[Cid]: + """Storage._reach as it was before D3: always a walk, never a complete DAG taken whole (the reference).""" + reached: set[Cid] = set() + stack = [cid for cid in roots if cid in storage._sizes and cid not in known] + while stack: + cid = stack.pop() + if cid in reached: + continue + reached.add(cid) + listed = storage._children(cid) + if listed is None: + if damaged is not None: + damaged.add(cid) + continue + stack.extend(child for child in listed if child in storage._sizes and child not in reached and child not in known) + return reached + + +class CompletenessMemoTest(StorageTest): + """D3: complete DAGs taken whole, their byte totals and incomplete DAGs' missing blocks remembered; each memo + exact through every way blocks go (deletion, verify, eviction, unpin and gc) and come back.""" + + def scenario(self, seed: int) -> list[builder.Built]: + """Files shared between pinned and cached directories, records through paths, a block missing; the files.""" + rng = random.Random(seed) + files = [self.add(b"%d.%d." % (seed, i) * rng.choice((10, 2000, 30000))) for i in range(12)] + dirs = [] + for _ in range(5): + chosen = rng.sample(range(len(files)), rng.randint(1, 5)) + dirs.append(builder.build_directory([Entry(f"f{i:02d}", files[i].cid, files[i].size, 0) for i in chosen], + self.storage.put)) + top = builder.build_directory([Entry(f"d{i}", d.cid, d.size, 2) for i, d in enumerate(dirs)], self.storage.put) + # Older, another directory over some of the same: its DAG holds the path nodes of newer records + other = builder.build_directory([Entry(f"x{i}", dirs[i].cid, dirs[i].size, 2) for i in rng.sample(range(5), 3)], + self.storage.put) + self.storage.record_fetch(other.cid, [], other.cid) + self.storage.pin(dirs[0].cid, USER) + self.storage.pin(files[rng.randrange(len(files))].cid, USER) + self.storage.record_fetch(top.cid, [], top.cid) + for d in rng.sample(dirs[1:], 2): + self.storage.record_fetch(d.cid, [], d.cid) + for i, d in enumerate(dirs): + entries = dag.decode(self.storage.get(d.cid)).entries + entry = entries[rng.randrange(len(entries))] + self.storage.record_fetch(top.cid, [f"d{i}", entry.name], entry.cid) + for root, names, _ in rng.sample(self.storage.cached_fetches(), 3): + self.storage.touch(root, names) + self.storage._delete({files[rng.randrange(len(files))].cid}) # a file incomplete or gone + return files + + def copy(self, name: str) -> Storage: + """A storage over a copy of this one's blocks and state.""" + self.storage.save() + for part in ("blocks", "state"): + shutil.copytree(os.path.join(self.tmp.name, part), os.path.join(self.tmp.name, name, part)) + storage = Storage(os.path.join(self.tmp.name, name, "blocks"), os.path.join(self.tmp.name, name, "state"), + self.storage.max_storage) + self.addCleanup(storage.close) + return storage + + def test_collections_decide_as_the_walk_does(self): + """Eviction, gc and drop_denied delete, keep and report exactly what they did when every DAG was walked, + including records whose target holds a newer record's path node (kept without its other children).""" + for seed in range(16): + for operation in ("evict", "gc", "drop_denied"): + with self.subTest(seed=seed, operation=operation): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.storage = self.open() + files = self.scenario(seed) + results = [] + for name, reach in (("memo", Storage._reach), ("walk", walked_reach)): + storage = self.copy(name) + for cid in storage.cached_targets() | set(storage.pins()): + storage.is_complete(cid) # memoised, as the publisher and earlier evictions leave them + storage.max_storage = storage.used_bytes() * (seed + 2) // 18 + with mock.patch.object(Storage, "_reach", reach): + if operation == "evict": + result = storage.evict() + elif operation == "gc": + result = storage.gc() + else: + result = storage.drop_denied(frozenset({files[seed % 12].cid.binary})) + results.append((result, storage.cached_fetches(), sorted(cid for cid, _ in storage.blocks.scan()))) + self.assertEqual(results[0], results[1]) + + def test_complete_dags_are_taken_whole(self): + root, a, b = self.tree() + self.storage.pin(root.cid, USER) + expected = self.storage.pinned_bytes() + self.storage = self.open() + self.assertTrue(self.storage.is_complete(root.cid)) + visits = [] + real = Storage._children + with mock.patch.object(Storage, "_children", lambda s, cid: (visits.append(cid), real(s, cid))[1]): + self.assertEqual(self.storage.pinned_bytes(), expected) + with self.storage._lock: + self.assertEqual(self.storage._reach([root.cid]), self.blocks_of(root.cid)) + self.assertEqual(self.storage._reach([root.cid], {a.cid} | self.blocks_of(a.cid)), + self.blocks_of(root.cid) - self.blocks_of(a.cid)) + self.assertEqual(visits, []) + # A partial block of known (a path node kept without its other children) stops the DAG being taken whole + with self.storage._lock: + self.assertEqual(self.storage._reach([root.cid], {root.cid}), set()) + self.assertEqual(self.storage._reach([root.cid], {a.cid}, partial={a.cid}), + walked_reach(self.storage, [root.cid], {a.cid})) + self.assertEqual(self.storage._reach([root.cid], {a.cid}, partial={a.cid}), + self.blocks_of(root.cid) - self.blocks_of(a.cid)) + + def test_dag_bytes_memo_follows_every_deletion(self): + """dag_bytes of a complete DAG is summed once, and forgotten whenever one of its blocks goes: deleted, + found bad by verify, evicted with a cached fetch, or collected after an unpin.""" + def check(cid: Cid, remembered: bool): + """dag_bytes is what a walk on a storage that has memoised nothing finds, from the memo or not.""" + self.storage.is_complete(cid) + self.assertEqual(self.storage.dag_bytes(cid), self.open().dag_bytes(cid)) + self.assertEqual(cid in self.storage._complete_bytes, remembered) + + root, a, b = self.tree() + self.storage.pin(root.cid, USER) + check(root.cid, True) + self.storage._delete({dag.decode(self.storage.get(a.cid)).children[0].cid}) + check(root.cid, False) + self.storage.unpin(root.cid, USER) + + root, a, b = self.tree() # a's lost chunk is back + self.storage.pin(root.cid, USER) + check(root.cid, True) + with open(self.storage.blocks.path(b.cid), "wb") as f: + f.write(b"damaged") + self.storage.verify() + check(root.cid, False) + self.storage.unpin(root.cid, USER) + + cached = self.add(b"C" * 70000) + self.storage.record_fetch(cached.cid, [], cached.cid) + check(cached.cid, True) + self.storage.max_storage = 1 + self.assertEqual(self.storage.evict(), {cached.cid}) + check(cached.cid, False) + self.storage.max_storage = 10**9 + + pinned = self.add(b"P" * 70000) + self.storage.pin(pinned.cid, USER) + check(pinned.cid, True) + self.storage.unpin(pinned.cid, USER) + self.storage.gc() + check(pinned.cid, False) + self.assertEqual(self.storage.dag_bytes(pinned.cid), 0) + + def test_missing_block_is_remembered(self): + """An incomplete DAG stays incomplete while the block found missing is: asked again it reads nothing, + and it is walked again once that block is back.""" + built = self.add(os.urandom(300 * 64 * 1024)) # a root over two file nodes + nodes = dag.decode(self.storage.get(built.cid)).children + first = dag.decode(self.storage.get(nodes[0].cid)).children[0].cid # the walk meets it last + last = dag.decode(self.storage.get(nodes[1].cid)).children[-1].cid + blocks = {cid: self.storage.get(cid) for cid in (first, last)} + self.storage._delete({first}) + self.assertFalse(self.storage.is_complete(built.cid)) + self.assertEqual(self.storage._missing, {built.cid: first}) + with mock.patch.object(self.storage.blocks, "get", side_effect=AssertionError("read")), \ + mock.patch.object(Storage, "block_size", side_effect=AssertionError("sized")): + self.assertFalse(self.storage.is_complete(built.cid)) + with self.assertRaises(NotFound): + self.storage.dag_blocks(built.cid) + self.storage._delete({last}) # another block goes too: still incomplete, still remembered + self.assertFalse(self.storage.is_complete(built.cid)) + self.storage.put(first, blocks[first]) # back: walked again, finding the other + self.assertFalse(self.storage.is_complete(built.cid)) + self.assertEqual(self.storage._missing, {built.cid: last}) + self.storage.put(last, blocks[last]) + self.assertTrue(self.storage.is_complete(built.cid)) + self.assertEqual(self.storage._missing, {}) + # A root that isn't stored + absent = Cid.for_block(RAW, b"absent") + self.assertFalse(self.storage.is_complete(absent)) + self.storage.put(absent, b"absent") + self.assertTrue(self.storage.is_complete(absent)) + + def test_missing_block_memo_is_bounded_and_notes_deletions_during_the_walk(self): + files = [self.add(b"%d" % i * 70000) for i in range(4)] + for f in files: + self.storage._delete({dag.decode(self.storage.get(f.cid)).children[0].cid}) + with mock.patch.object(storage_module, "MISSING_MEMO", 2): + for f in files: + self.assertFalse(self.storage.is_complete(f.cid)) + self.assertEqual(list(self.storage._missing), [files[2].cid, files[3].cid]) + self.assertFalse(self.storage.is_complete(files[0].cid)) # forgotten: walked again + # A block of the DAG deleted while the walk runs, after the walk counted it, is remembered as missing + whole = self.add(b"W" * 70000) + chunks = dag.decode(self.storage.get(whole.cid)).children + real = self.storage.block_size + + def block_size(cid): + """Storage.block_size; the last chunk's (met first) deletes it once counted.""" + size = real(cid) + if cid == chunks[-1].cid and size is not None: + self.storage._delete({cid}) + return size + + with mock.patch.object(self.storage, "block_size", block_size): + with self.assertRaises(NotFound) as caught: + self.storage.dag_blocks(whole.cid) + self.assertEqual(str(caught.exception), f"a block of {whole.cid} was deleted") + self.assertEqual(self.storage._missing[whole.cid], chunks[-1].cid) + self.assertFalse(self.storage.is_complete(whole.cid)) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_unexpected_errors.py b/tests/test_unexpected_errors.py new file mode 100644 index 0000000..3ee230c --- /dev/null +++ b/tests/test_unexpected_errors.py @@ -0,0 +1,101 @@ +"""Unexpected exceptions caught by a broad except are logged at LOG_WARNING inside it, so the debug log records +their traceback (XC4): the fetcher's /blocks worker, the lookup's routing-table callbacks and LinkPool.request. +The mesh and peers sites are tested in test_mesh, ThrottledLog in test_log.""" +import os +import sys +import unittest +from unittest import mock + +from rnfs.config import Config +from rnfs.dht.lookup import Lookup +from rnfs.log import LOG_INFO, LOG_WARNING +from rnfs.network import links +from tests.test_fetcher import FetcherTest +from tests.test_links import FakeLink, fake_rns, run + +P1, P2 = bytes([1]) * 16, bytes([2]) * 16 + + +def recorder(logged: list): + """A stand-in for rnfs.log.log appending (message, level, the type of the exception being handled) to logged.""" + def record(message, level=LOG_INFO): + """Record the line and whether it was logged inside an except.""" + logged.append((message, level, sys.exc_info()[0])) + return record + + +def warnings(logged: list) -> list: + """(message, exception type) of each line logged at LOG_WARNING.""" + return [(message, raised) for message, level, raised in logged if level == LOG_WARNING] + + +class FetcherWorkerTest(FetcherTest): + """A /blocks request whose client raises (a bug, not a network failure) is a failed request, logged.""" + + def test_client_exception_logged_and_fetched_elsewhere(self): + built = self.file(os.urandom(3000)) + + def broken(cids): + """A bug in the client.""" + raise TypeError("a bug") + self.client.behaviour[P1] = broken + self.hold(P2) + self.runs.append([P1, P2]) + logged = [] + with mock.patch("rnfs.fetcher.log", recorder(logged)): + self.fetch_all(built.cid) + self.assertTrue(self.storage.is_complete(built.cid)) + (message, raised), = [line for line in warnings(logged) if "raised" in line[0]] + self.assertIs(raised, TypeError) + self.assertIn("a bug", message) + + +class LookupCallbackTest(unittest.TestCase): + """A routing-table callback that raises doesn't end the lookup, and is logged.""" + + def test_callback_exception_logged(self): + def on_answer(node_id): + """A bug in the routing table (private mode's admit, say).""" + raise TypeError("a bug") + lookup = Lookup(bytes(16), [P1], lambda node_id, key: ([], []), bytes([9]) * 16, on_answer=on_answer) + logged = [] + with mock.patch("rnfs.log.log", recorder(logged)): + self.assertEqual(lookup.find_closest(), [P1]) + (message, raised), = warnings(logged) + self.assertIs(raised, TypeError) + self.assertIn("a bug", message) + + +class LinkPoolSendTest(unittest.TestCase): + """Link.request raising (it answers False when it can't send) is a failed request, logged.""" + + def setUp(self): + """A LinkPool over fake_rns().""" + patcher = mock.patch.object(links, "RNS", fake_rns()) + patcher.start() + self.addCleanup(patcher.stop) + FakeLink.created = [] + FakeLink.establish_delay = 0.0 + FakeLink.responder = staticmethod(lambda link, path, data: True) + self.pool = links.LinkPool(object(), Config()) + self.addCleanup(self.pool.close_all) + + def test_request_exception_logged(self): + peer = run(self, lambda: self.pool.get(P1)) + + def broken(*args, **kwargs): + """A bug in RNS or in what rnfs passed it.""" + raise TypeError("a bug") + peer.link.request = broken + logged = [] + with mock.patch.object(links, "log", recorder(logged)): + exchange = run(self, lambda: self.pool.request(peer, "/find", bytes(16))) + self.assertFalse(exchange.answered) + (message, raised), = warnings(logged) + self.assertIs(raised, TypeError) + self.assertIn("a bug", message) + self.assertEqual(peer.requests, 0) # its slot was freed + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_upgrade_015.py b/tests/test_upgrade_015.py new file mode 100644 index 0000000..d58fdb1 --- /dev/null +++ b/tests/test_upgrade_015.py @@ -0,0 +1,204 @@ +"""Upgrading from rnfs 0.1.5: the current code started on an rnfs directory exactly as 0.1.5 left it +(tests/fixtures/rnfs-0.1.5, its state database holding rows in every table; see its README) keeps every row. + +The expectation is the fixture's expected.json, every table's rows as 0.1.5 left them. The current code may add +rows and tables (its own schema steps), but none of 0.1.5's rows may change or go: the database upgrades, passes +quick_check, sets nothing aside, and a second start changes nothing. The tables 0.1.5 created must also be +exactly those a fresh database gets, so a released schema step edited in place (which a database 0.1.5 wrote +would never see) fails here as well as in test_statedb's digest. + +The clock is the fixture's (expected.json's "now"), so no record has expired.""" +import functools +import importlib.util +import io +import json +import os +import shutil +import sqlite3 +import tempfile +import time +import unittest +from unittest import mock + +import RNS + +from rnfs import node as node_module +from rnfs import statedb +from rnfs.config import Config +from rnfs.content import builder +from rnfs.content.blockstore import BlockStore +from rnfs.node import RnfsNode +from tests.support import FakeClock, state_rows +from tests.test_names import FakeNetwork +from tests.test_node import NetClient +from tests.test_statedb import RELEASED_0_1_5_STEPS +from tests.test_upgrade_014 import NODE_ID, make_blocks + +FIXTURE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "rnfs-0.1.5") + +_saved_loglevel = RNS.loglevel + + +def setUpModule(): + """Silence the log while these tests run.""" + RNS.loglevel = RNS.LOG_CRITICAL + + +def tearDownModule(): + """Restore RNS's loglevel.""" + RNS.loglevel = _saved_loglevel + + +def _expected() -> dict: + """expected.json.""" + with open(os.path.join(FIXTURE, "expected.json"), encoding="utf-8") as f: + return json.load(f) + + +EXPECTED = _expected() + + +def _content(): + """The fixture's fixture_content module: the files 0.1.5 added on top of the 0.1.4 fixture's.""" + spec = importlib.util.spec_from_file_location("fixture_content_015", os.path.join(FIXTURE, "fixture_content.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def row(values: list) -> tuple: + """A row of expected.json as sqlite3 gives it.""" + return tuple(bytes.fromhex(v["blob"]) if isinstance(v, dict) else v for v in values) + + +def loadable(table: str, values: tuple) -> bool: + """Whether this platform keeps the row: a get job's output must be absolute, and /home/... isn't on Windows + (Python 3.13 and later), so the jobs store drops that row there (as test_upgrade_014's loadable_jobs).""" + if table != "jobs": + return True + output = statedb.load(values[2]).get("output") + return output is None or os.path.isabs(output) + + +def tables(state_dir: str) -> dict[str, set[tuple]]: + """Every row of every table of the state database.""" + names = [r[0] for r in state_rows(state_dir, "SELECT name FROM sqlite_master WHERE type = 'table'")] + return {name: set(state_rows(state_dir, f"SELECT * FROM {name}")) for name in names} + + +def schema_sql(path: str) -> dict[str, str]: + """{name: sql} of every table and index in the database at path.""" + conn = sqlite3.connect(path) + try: + return dict(conn.execute("SELECT name, sql FROM sqlite_master WHERE sql IS NOT NULL")) + finally: + conn.close() + + +class UpgradeTest(unittest.TestCase): + """A copy of the fixture under a temporary directory, with its blocks, started on with the clock at "now".""" + + def setUp(self): + """Copy the fixture and make its blocks; fake the wall clock, also for the stores that take it as an + argument.""" + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = os.path.join(self.tmp.name, "rnfs") + for part in ("state", "names", "files"): + shutil.copytree(os.path.join(FIXTURE, part), os.path.join(self.root, part)) + scratch = os.path.join(self.tmp.name, "scratch") + os.makedirs(scratch) + blocks = os.path.join(self.root, "blocks") + self.built = make_blocks(blocks, scratch) + store = BlockStore(blocks) + for data, name in _content().STREAMS: + self.built.add(builder.add_stream(io.BytesIO(data), store.put, len(data), name).cid) + self.state = os.path.join(self.root, "state") + self.clock = FakeClock(EXPECTED["now"]) + for patch in (mock.patch.object(time, "time", self.clock), + *(mock.patch.object(node_module, cls, functools.partial(getattr(node_module, cls), + clock=self.clock)) + for cls in ("ProviderStore", "Publisher", "NameManager"))): + patch.start() + self.addCleanup(patch.stop) + + def start(self) -> RnfsNode: + """An offline node on the copy (stopped before the directory is removed).""" + config = Config(rnfs_dir=self.root, **EXPECTED["settings"]) + config.validate() + node = RnfsNode.offline(config, NODE_ID, NetClient(), FakeNetwork()) + self.addCleanup(node.stop) + return node + + def test_fixture_is_complete(self): + """The fixture is what its README says: 0.1.5's schema, rows in every table, the 0.1.5-only ones included, + and the deny list's blocks made again by the current code (no CID changed) and pinned "deny".""" + self.assertEqual(EXPECTED["user_version"], RELEASED_0_1_5_STEPS) + self.assertEqual(state_rows(self.state, "PRAGMA user_version"), [(RELEASED_0_1_5_STEPS,)]) + self.assertEqual([name for name, rows in EXPECTED["tables"].items() if not rows], []) + for table in ("deny_entries", "deny_lists", "deny_hashes", "meta"): + self.assertIn(table, EXPECTED["tables"]) + deny_list = row(EXPECTED["tables"]["deny_lists"][0])[1] + self.assertIn(deny_list, {cid.binary for cid in self.built}) + self.assertIn((deny_list, "deny"), {row(values) for values in EXPECTED["tables"]["pins"]}) + + def test_schema_is_upgraded(self): + """user_version is the current schema's, and the upgraded database passes quick_check.""" + self.start().stop() + self.assertEqual(state_rows(self.state, "PRAGMA user_version"), [(len(statedb.SCHEMA),)]) + self.assertEqual(state_rows(self.state, "PRAGMA quick_check"), [("ok",)]) + # 0.1.6's steps were applied: the shared folder's per-file cache (D1) has its table + self.assertEqual(state_rows(self.state, "SELECT name FROM sqlite_master WHERE name = 'shared_folder_files'"), + [("shared_folder_files",)]) + + def test_every_row_is_kept(self): + """Every row 0.1.5 left is still there, unchanged, in every table.""" + node = self.start() + node.stop() + found = tables(self.state) + for table, rows in EXPECTED["tables"].items(): + with self.subTest(table=table): + self.assertIn(table, found) + expected = {row(values) for values in rows if loadable(table, row(values))} + self.assertEqual(expected - found[table], set()) + + def test_every_pin_is_complete(self): + """Every pin, the deny list's own included, is complete: the stores agree with the blocks.""" + node = self.start() + pins = node.storage.pins() + self.assertEqual({cid.binary for cid in pins}, {row(values)[0] for values in EXPECTED["tables"]["pins"]}) + for cid in pins: + self.assertTrue(node.storage.is_complete(cid), cid) + + def test_nothing_is_set_aside(self): + """No file in the state directory is set aside as damaged, and the .migrated files are left alone.""" + listing = sorted(os.listdir(self.state)) + self.start().stop() + self.assertEqual([name for name in os.listdir(self.state) if ".damaged" in name], []) + migrated = [name for name in listing if name.endswith(statedb.MIGRATED_SUFFIX)] + self.assertEqual(len(migrated), len(statedb.LEGACY)) + for name in migrated: + with open(os.path.join(self.state, name), "rb") as kept, \ + open(os.path.join(FIXTURE, "state", name), "rb") as original: + self.assertEqual(kept.read(), original.read(), name) + + def test_second_start_changes_nothing(self): + """A second start changes no row, nor anything in the state directory.""" + self.start().stop() + before, listing = tables(self.state), sorted(os.listdir(self.state)) + self.start().stop() + self.assertEqual(tables(self.state), before) + self.assertEqual(sorted(os.listdir(self.state)), listing) + + def test_released_tables_match_a_fresh_database(self): + """The tables and indexes 0.1.5 created are exactly those the current schema creates in a fresh database: + a released step edited in place would make them differ.""" + released = schema_sql(os.path.join(FIXTURE, "state", statedb.DB_FILE)) + db = statedb.connect(os.path.join(self.tmp.name, "fresh")) + db.close() + fresh = schema_sql(os.path.join(self.tmp.name, "fresh", statedb.DB_FILE)) + self.assertEqual({name: fresh.get(name) for name in released}, released) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_walker.py b/tests/test_walker.py index d61adc1..1837a9a 100644 --- a/tests/test_walker.py +++ b/tests/test_walker.py @@ -336,6 +336,14 @@ def test_resolve_through_index(self): walker.resolve(root, [missing], blocks.get) self.assertEqual(str(caught.exception), f"{root}/{missing} does not exist") + def test_index_child(self): + """The child with the greatest first_name <= name, or None before the first.""" + index = IndexNode([IndexChild(n, Cid.for_block(DAG, n.encode()), 0) for n in ("b", "d", "f")]) + for name, position in (("a", None), ("b", 0), ("c", 0), ("d", 1), ("e", 1), ("f", 2), ("zz", 2)): + with self.subTest(name=name): + self.assertEqual(walker.index_child(index, name), position) + self.assertIsNone(walker.index_child(IndexNode([]), "a")) + def test_resolve_nested_and_into_file(self): """Nested names resolve, no names give the root, and a name below a file is NotFound, naming the path.""" blocks = Blocks() @@ -372,12 +380,107 @@ def test_count_entries_repeated_empty_subtrees(self): walker.count_entries(root_expect(cid), blocks.get, 1109) self.assertEqual(len(blocks.reads), 4) + def test_leaf_bytes(self): + """Each distinct raw chunk counted once, each distinct file node read once, no raw block read.""" + blocks = Blocks() + a, b = blocks.raw(b"aaaa"), blocks.raw(b"bbbbbb") + lower = blocks.node(FileNode([Child(a, 4), Child(b, 6), Child(a, 4)])) + root = blocks.node(FileNode([Child(lower, 14)] * 3 + [Child(b, 6)])) + self.assertEqual(walker.leaf_bytes(root_expect(root), blocks.get), 10) + self.assertEqual(blocks.reads, {root: 1, lower: 1}) + self.assertEqual(walker.leaf_bytes(root_expect(a), blocks.get), 4) # a one-block file: its own length + # Stops once stop_at is met, before reading further nodes + blocks.reads.clear() + other = blocks.node(FileNode([Child(blocks.raw(b"c"), 1)])) + early = blocks.node(FileNode([Child(a, 4), Child(other, 1)])) + self.assertEqual(walker.leaf_bytes(root_expect(early), blocks.get, stop_at=4), 4) + self.assertNotIn(other, blocks.reads) + self.assertEqual(walker.leaf_bytes(root_expect(early), blocks.get, stop_at=100), 5) + + def test_leaf_bytes_refuses_impossible_chunks_and_directories(self): + blocks = Blocks() + missing = Cid.for_block(RAW, b"never stored") # not read, so only its claimed size counts + big = constants.MAX_BLOCK_SIZE + 1 + with self.assertRaises(InvalidDag): + walker.leaf_bytes(root_expect(blocks.node(FileNode([Child(missing, big)]))), blocks.get) + directory = blocks.node(DirectoryNode([Entry("f", blocks.raw(b"x"), 1, 0)])) + with self.assertRaises(ValueError): + walker.leaf_bytes(root_expect(directory), blocks.get) + def test_height(self): with small_tree(): blocks = Blocks() built = builder.build_bytes(bytes(50), blocks.put) self.assertEqual(walker.height(root_expect(built.cid), blocks.get), built.height) + def chain(self, blocks: Blocks, depth: int) -> builder.Built: + """depth directories, each the one entry of the next, over a one-chunk file: height depth.""" + built = builder.build_bytes(b"leaf %d" % depth, blocks.put) + for _ in range(depth): + built = builder.wrap("d", built, blocks.put) + return built + + def test_height_remembers_checked_subtrees(self): + """A subtree measured before isn't read again, so an edited tree costs only its new nodes (D2).""" + with mock.patch.object(walker, "_heights", walker._Heights()): + blocks = Blocks() + subtrees = [self.chain(blocks, 3), self.chain(blocks, 5), builder.build_bytes(bytes(70000), blocks.put)] + entries = [Entry(f"e{i}", b.cid, b.size, b.kind) for i, b in enumerate(subtrees)] + root = builder.build_directory(entries, blocks.put) + self.assertEqual(walker.height(root_expect(root.cid), blocks.get), 6) + self.assertEqual(walker.height(root_expect(root.cid), blocks.get), 6) + self.assertEqual(sum(blocks.reads.values()), 10) # each DAG node once: the second call read nothing + blocks.reads.clear() + edited = builder.build_directory(entries[1:] + [Entry("new", subtrees[0].cid, subtrees[0].size, 2)], + blocks.put) + self.assertEqual(walker.height(root_expect(edited.cid), blocks.get), 6) + self.assertEqual(blocks.reads, {edited.cid: 1}) + # A changed file is a new CID, so it is read + changed = builder.build_bytes(bytes(70001), blocks.put) + edited = builder.build_directory(entries[:2] + [Entry("e2", changed.cid, changed.size, 0)], blocks.put) + blocks.reads.clear() + self.assertEqual(walker.height(root_expect(edited.cid), blocks.get), 6) + self.assertEqual(set(blocks.reads), {edited.cid, changed.cid}) + + def test_height_memo_is_per_expectation(self): + """A subtree remembered under one expectation is still checked under another.""" + with mock.patch.object(walker, "_heights", walker._Heights()): + blocks = Blocks() + inner = self.chain(blocks, 2) + self.assertEqual(walker.height(root_expect(inner.cid), blocks.get), 2) + lying = blocks.node(DirectoryNode([Entry("x", inner.cid, inner.size + 1, dag.KIND_DIRECTORY)])) + for _ in range(2): # and an invalid tree isn't remembered + with self.assertRaises(InvalidDag): + walker.height(root_expect(lying), blocks.get) + as_file = blocks.node(DirectoryNode([Entry("x", inner.cid, inner.size, dag.KIND_FILE)])) + with self.assertRaises(InvalidDag): + walker.height(root_expect(as_file), blocks.get) + + def test_height_past_max_depth_remembers_nothing_short(self): + """A walk cut off at MAX_DEPTH keeps no heights (they may be short), and a remembered subtree that + makes a tree too deep still gives MAX_DEPTH + 1.""" + with mock.patch.object(walker, "_heights", walker._Heights()): + blocks = Blocks() + deep = self.chain(blocks, 6) + with mock.patch.object(constants, "MAX_DEPTH", 3): + self.assertEqual(walker.height(root_expect(deep.cid), blocks.get), 4) + self.assertEqual(walker.height(root_expect(deep.cid), blocks.get), 6) # not a short memo + with mock.patch.object(constants, "MAX_DEPTH", 3): + blocks.reads.clear() + self.assertEqual(walker.height(root_expect(deep.cid), blocks.get), 4) + self.assertEqual(blocks.reads, {}) # remembered, and capped at MAX_DEPTH + 1 + outer = builder.wrap("o", self.chain(blocks, 2), blocks.put) + self.assertEqual(walker.height(root_expect(outer.cid), blocks.get), 3) + + def test_height_memo_is_bounded(self): + with mock.patch.object(walker, "_heights", walker._Heights()), mock.patch.object(walker, "HEIGHT_MEMO", 2): + blocks = Blocks() + built = [self.chain(blocks, depth) for depth in (3, 4, 5)] + for _ in range(2): + for b in built: + self.assertEqual(walker.height(root_expect(b.cid), blocks.get), b.height) + self.assertEqual(len(walker._heights._heights), 2) + def test_parse_target(self): cid = Cid.for_block(RAW, b"x") self.assertEqual(walker.parse_target(str(cid)), (cid, [])) @@ -506,26 +609,32 @@ def refusing(real, *names_and_errors): """real(path, ...), but OSError(error) for the names given.""" refused = dict(names_and_errors) - def operation(path, *args): + def operation(path, *args, **kwargs): """Refuse some names as the filesystem would.""" error = refused.get(os.path.basename(path)) if error is not None: raise OSError(error, os.strerror(error), path) - return real(path, *args) + return real(path, *args, **kwargs) return operation - with mock.patch.object(walker, "open", refusing(open, ("a?b", errno.EINVAL), ("long", errno.ENAMETOOLONG)), - create=True), \ - mock.patch("os.mkdir", refusing(os.mkdir, ("d?", errno.EINVAL))): - warnings = walker.write_out(root_expect(root.cid), self.blocks.get, self.path("out"), 10) - self.assertEqual(os.listdir(self.path("out")), ["good"]) - self.assertEqual(len(warnings), 3) - self.assertTrue(all("can't represent" in warning for warning in warnings)) - # Any other error (a full disk) still fails the whole get - with mock.patch.object(walker, "open", refusing(open, ("good", errno.ENOSPC)), create=True), \ - self.assertRaises(OSError): - walker.write_out(root_expect(root.cid), self.blocks.get, self.path("out2"), 10) + def files_refusing(*names_and_errors): + """Patch how write_out creates files (os.open by descriptor, or open by path) to refuse some names.""" + if walker.FD_WRITE and not fsutil.IS_WINDOWS: + return mock.patch("os.open", refusing(os.open, *names_and_errors)) + return mock.patch.object(walker, "open", refusing(open, *names_and_errors), create=True) + + for fd_write in dict.fromkeys((walker.FD_WRITE, False)): # by descriptor (Linux, macOS), by path (Windows) + with self.subTest(fd_write=fd_write), mock.patch.object(walker, "FD_WRITE", fd_write): + with files_refusing(("a?b", errno.EINVAL), ("long", errno.ENAMETOOLONG)), \ + mock.patch("os.mkdir", refusing(os.mkdir, ("d?", errno.EINVAL))): + warnings = walker.write_out(root_expect(root.cid), self.blocks.get, self.path(f"out{fd_write}"), 10) + self.assertEqual(os.listdir(self.path(f"out{fd_write}")), ["good"]) + self.assertEqual(len(warnings), 3) + self.assertTrue(all("can't represent" in warning for warning in warnings)) + # Any other error (a full disk) still fails the whole get + with files_refusing(("good", errno.ENOSPC)), self.assertRaises(OSError): + walker.write_out(root_expect(root.cid), self.blocks.get, self.path(f"err{fd_write}"), 10) @unittest.skipUnless(POSIX_MODES, "Windows has no execute bit") def test_execute_bit_is_set_through_the_descriptor(self): diff --git a/tests/test_wire.py b/tests/test_wire.py new file mode 100644 index 0000000..313b8f2 --- /dev/null +++ b/tests/test_wire.py @@ -0,0 +1,73 @@ +"""Wire known-answer tests: the current code's encoders give, byte for byte, what rnfs 0.1.3 and 0.1.4 put on the +wire, and its parsers read those bytes back (tests/fixtures/wire: /find, /provide, /blocks, /pin, keepers' /name, +announce app_data, name records, the LXMF field). The vectors were made by those releases' own code, so a change +that still round-trips with itself (an int width, a list for a tuple, a field order) fails here.""" +import importlib.util +import json +import os +import unittest + +FIXTURE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "wire") + + +def _cases(): + """tests/fixtures/wire/wire_cases.py, which generate.py ran on the old trees.""" + spec = importlib.util.spec_from_file_location("wire_cases", os.path.join(FIXTURE, "wire_cases.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +wire_cases = _cases() + + +def _vectors() -> dict: + """vectors.json.""" + with open(os.path.join(FIXTURE, "vectors.json"), encoding="utf-8") as f: + return json.load(f) + + +class WireVectorTest(unittest.TestCase): + """Every vector, encoded and parsed by the current code.""" + + @classmethod + def setUpClass(cls): + """The vectors, and the current code's modules.""" + cls.vectors = _vectors() + cls.m = wire_cases.modules() + + def test_vectors_come_from_released_code(self): + """The vectors were made by 0.1.3 and 0.1.4, which agreed, and cover every case and kind.""" + self.assertEqual(self.vectors["generated_by"], ["0.1.3", "0.1.4"]) + names = [vector["name"] for vector in self.vectors["vectors"]] + self.assertEqual(names, [name for name, _, _ in wire_cases.cases(self.m)]) + kinds = {vector["kind"] for vector in self.vectors["vectors"]} + self.assertEqual(kinds, {"find_request", "find_response", "provide_request", "accept_response", + "blocks_request", "blocks_response", "pin_request", "name_request", + "name_response", "announce_mode", "name_record", "lxmf_field"}) + + def test_encode_gives_the_released_bytes(self): + """packb(encode(inputs)) is the vector's bytes (app_data: encode(inputs) is).""" + for vector in self.vectors["vectors"]: + with self.subTest(vector["name"]): + inputs = wire_cases.from_json(self.m, vector["inputs"]) + self.assertEqual(wire_cases.wire(self.m, vector["kind"], inputs).hex(), vector["hex"]) + + def test_parse_reads_the_released_bytes(self): + """parse(unpackb(bytes)) is the vector's inputs.""" + for vector in self.vectors["vectors"]: + with self.subTest(vector["name"]): + inputs = wire_cases.from_json(self.m, vector["inputs"]) + data = wire_cases.unwire(self.m, vector["kind"], bytes.fromhex(vector["hex"])) + self.assertEqual(wire_cases.parse(self.m, vector["kind"], data, inputs), inputs) + + def test_inputs_are_the_fixed_ones(self): + """The inputs in vectors.json are those wire_cases builds now (so a changed case can't pass silently).""" + built = {name: wire_cases.to_json(inputs) for name, _, inputs in wire_cases.cases(self.m)} + for vector in self.vectors["vectors"]: + with self.subTest(vector["name"]): + self.assertEqual(vector["inputs"], built[vector["name"]]) + + +if __name__ == "__main__": + unittest.main()