diff --git a/.github/RELEASE_NOTES.md b/.github/RELEASE_NOTES.md index 276b304..0155249 100644 --- a/.github/RELEASE_NOTES.md +++ b/.github/RELEASE_NOTES.md @@ -1,7 +1,15 @@ -**deed** is a command line for nostr, built in Zig. It runs on macOS and Linux, on both Intel and ARM. The Linux binaries are statically linked, so there is no glibc version to satisfy. The macOS binaries are **ad-hoc signed, not notarized**, which means a download that came through a browser needs `xattr -dr com.apple.quarantine` before it will run. +**deed** is a command line for nostr, built in Zig. It runs on macOS, Linux and Windows, on both Intel and ARM. The Linux binaries are statically linked, so there is no glibc version to satisfy. The macOS binaries are **ad-hoc signed, not notarized**, which means a download that came through a browser needs `xattr -dr com.apple.quarantine` before it will run. The Windows builds are zips holding `deed.exe`, which is not code-signed. Every artifact below is published with a `.sha256` beside it, so the download can be checked against a digest that was written by the same job that built it. +### What's new in v0.4.0 + +**deed runs on Windows.** Each release now carries a zip for x86_64 and one for ARM, holding `deed.exe`, with a `.sha256` beside each. The nostr library underneath builds and passes its tests on Windows too, including the read deadline that `req` and `fetch --timeout` rely on. + +**A new store's directories are made for you, and a store that will not open says why.** `--store ~/.deed/db` used to need `~/.deed` to exist already, and a store that failed to open printed only an error name. deed now creates the missing directories and names the reason: no permission, not a store, or a directory where the file should be. `--local` only reads, so it never creates an empty store at a mistyped path. + +**Codes in capitals decode.** An all-uppercase `NPUB1...` or `NEVENT1...` decodes, as BIP-173 allows, and so does a `NOSTR:` scheme in any case, which is what a QR code carries. A code in mixed case is still refused. + ### What's new in v0.3.2 **deed works well with AI agents.** A skill in `skills/deed` teaches an agent the commands, working recipes, and the two rules that matter: publishing is public and permanent, so it asks first, and a secret key stays in `NOSTR_SECRET_KEY`. Add it with `npx skills add zig-nostr/deed`, or in Claude Code as a plugin. `AGENTS.md` covers changing deed itself. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55a57ae..cab975a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,3 +116,47 @@ jobs: fi [ ! -e "$work/out/bin/deed" ] || { echo "it refused and installed anyway" >&2; exit 1; } echo "a tampered archive is refused and nothing is installed" + + # Windows gets its own job because most of the steps above are bash scripts + # for the macOS and Linux installer. What matters here is that deed builds, + # that its tests (which talk to relays over real loopback sockets) pass, and + # that the binary that was built runs. Formatting does not depend on the + # platform and is checked above. + build-test-windows: + runs-on: windows-latest + timeout-minutes: 20 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + + - name: Read Zig version + id: zig-version + run: echo "version=$(cat .zigversion)" >> "$GITHUB_OUTPUT" + + - uses: mlugg/setup-zig@v2 + with: + version: ${{ steps.zig-version.outputs.version }} + + - name: Build + run: zig build + + - name: Test + run: zig build test + + # A binary that links is not a binary that runs. This makes a key, signs + # an event with it and checks the signature, through a pipe, which covers + # the argument parsing, the environment, and standard input and output + # that differ most on Windows. + - name: The binary runs + run: | + set -euo pipefail + bin=zig-out/bin/deed.exe + [ -f "$bin" ] || { echo "the build produced no deed.exe" >&2; exit 1; } + got="$("$bin" version)" + want="deed $(sed -n 's/^pub const version = "\(.*\)";$/\1/p' src/main.zig)" + [ "$got" = "$want" ] || { echo "the binary says '$got', expected '$want'" >&2; exit 1; } + sk="$("$bin" key generate --hex)" + NOSTR_SECRET_KEY="$sk" "$bin" event -c "windows check" | "$bin" verify + echo "deed runs on Windows" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cf790b5..8807e33 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,6 @@ name: Release -# On a version tag, build deed for four targets from that commit, publish each +# On a version tag, build deed for six targets from that commit, publish each # with a checksum beside it, and make the release visible only once every claim # about it has been checked by a command that fails the job. # @@ -82,18 +82,26 @@ jobs: v="${{ steps.version.outputs.version }}" mkdir -p dist - for target in aarch64-macos-none x86_64-macos-none x86_64-linux-musl aarch64-linux-musl; do + for target in aarch64-macos-none x86_64-macos-none x86_64-linux-musl aarch64-linux-musl x86_64-windows-gnu aarch64-windows-gnu; do case "$target" in - aarch64-macos-none) name=macos-aarch64 ;; - x86_64-macos-none) name=macos-x86_64 ;; - x86_64-linux-musl) name=linux-x86_64 ;; - aarch64-linux-musl) name=linux-aarch64 ;; + aarch64-macos-none) name=macos-aarch64 ;; + x86_64-macos-none) name=macos-x86_64 ;; + x86_64-linux-musl) name=linux-x86_64 ;; + aarch64-linux-musl) name=linux-aarch64 ;; + x86_64-windows-gnu) name=windows-x86_64 ;; + aarch64-windows-gnu) name=windows-aarch64 ;; esac rm -rf "out-$target" zig build -Dtarget="$target" -Doptimize=ReleaseSafe -Dstrip=true --prefix "out-$target" - bin="out-$target/bin/deed" + # Windows binaries carry an .exe suffix and ship in a zip, which is + # what Expand-Archive and Explorer open. Every other target ships a + # tar.gz holding a binary named deed. + case "$target" in + *-windows-*) bin="out-$target/bin/deed.exe"; ext=zip ;; + *) bin="out-$target/bin/deed"; ext=tar.gz ;; + esac [ -s "$bin" ] || { echo "$target produced no binary" >&2; exit 1; } # The linker ad-hoc signs the arm64 Mach-O and leaves the x86_64 one @@ -107,14 +115,27 @@ jobs: ;; esac - asset="deed-$v-$name.tar.gz" - tar -czf "dist/$asset" -C "out-$target/bin" deed -C "$GITHUB_WORKSPACE" LICENSE README.md - [ -s "dist/$asset" ] || { echo "$asset is empty" >&2; exit 1; } - - # An archive holding only LICENSE and README unpacks perfectly and - # installs nothing, so the binary is confirmed to be in there. - tar -tzf "dist/$asset" | grep -qx deed \ - || { echo "$asset carries no member named deed" >&2; exit 1; } + asset="deed-$v-$name.$ext" + case "$ext" in + zip) + # -j keeps the files at the top of the archive, and -X leaves + # out the extra attributes that make a zip differ by machine. + zip -q -X -j "dist/$asset" "$bin" LICENSE README.md + [ -s "dist/$asset" ] || { echo "$asset is empty" >&2; exit 1; } + unzip -Z1 "dist/$asset" | grep -qx deed.exe \ + || { echo "$asset carries no member named deed.exe" >&2; exit 1; } + ;; + *) + tar -czf "dist/$asset" -C "out-$target/bin" deed -C "$GITHUB_WORKSPACE" LICENSE README.md + [ -s "dist/$asset" ] || { echo "$asset is empty" >&2; exit 1; } + + # An archive holding only LICENSE and README unpacks perfectly + # and installs nothing, so the binary is confirmed to be in + # there. + tar -tzf "dist/$asset" | grep -qx deed \ + || { echo "$asset carries no member named deed" >&2; exit 1; } + ;; + esac ( cd dist && shasum -a 256 "$asset" > "$asset.sha256" ) [ -s "dist/$asset.sha256" ] || { echo "$asset.sha256 is empty" >&2; exit 1; } @@ -192,9 +213,65 @@ jobs: ./deed event -c "release check" --sec "$sk" | ./deed verify echo "$asset runs on ${{ matrix.runner }}" + # The same check for the Windows zips, on Windows. Extraction is PowerShell's + # own Expand-Archive, which is what the README tells a reader to use, and the + # rest runs in bash so that the pipe between the two deeds carries bytes and + # not the re-encoded text PowerShell would make of it. + verify-windows: + name: run ${{ matrix.asset }} on ${{ matrix.runner }} + needs: build + strategy: + fail-fast: false + matrix: + include: + - runner: windows-latest + asset: windows-x86_64 + - runner: windows-11-arm + asset: windows-aarch64 + runs-on: ${{ matrix.runner }} + defaults: + run: + shell: bash + steps: + - name: Download it and check it + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + set -euo pipefail + v="${GITHUB_REF_NAME#v}" + asset="deed-$v-${{ matrix.asset }}.zip" + + gh release download "$GITHUB_REF_NAME" \ + --pattern "$asset" --pattern "$asset.sha256" --dir . + + sha256sum -c "$asset.sha256" + echo "asset=$asset" >> "$GITHUB_ENV" + + - name: Unpack it + shell: pwsh + run: Expand-Archive -Path $env:asset -DestinationPath unpacked + + - name: Run it + run: | + set -euo pipefail + v="${GITHUB_REF_NAME#v}" + bin=./unpacked/deed.exe + [ -f "$bin" ] || { echo "the archive holds no deed.exe" >&2; exit 1; } + + got="$("$bin" version)" + [ "$got" = "deed $v" ] || { echo "the binary says '$got', expected 'deed $v'" >&2; exit 1; } + + # The same gesture as on the other platforms: libsecp256k1, the + # bech32 encoder and the JSON writer and reader in one line, on the + # artifact as published. + sk="$("$bin" key generate --hex)" + "$bin" event -c "release check" --sec "$sk" | "$bin" verify + echo "${{ matrix.asset }} runs on ${{ matrix.runner }}" + publish: name: check the page, then lift the draft - needs: verify + needs: [verify, verify-windows] runs-on: ubuntu-latest steps: - name: Everything the page must carry, before anyone can see it @@ -214,10 +291,12 @@ jobs: "deed-$v-linux-aarch64.tar.gz" "deed-$v-linux-aarch64.tar.gz.sha256" \ "deed-$v-linux-x86_64.tar.gz" "deed-$v-linux-x86_64.tar.gz.sha256" \ "deed-$v-macos-aarch64.tar.gz" "deed-$v-macos-aarch64.tar.gz.sha256" \ - "deed-$v-macos-x86_64.tar.gz" "deed-$v-macos-x86_64.tar.gz.sha256" | sort)" + "deed-$v-macos-x86_64.tar.gz" "deed-$v-macos-x86_64.tar.gz.sha256" \ + "deed-$v-windows-aarch64.zip" "deed-$v-windows-aarch64.zip.sha256" \ + "deed-$v-windows-x86_64.zip" "deed-$v-windows-x86_64.zip.sha256" | sort)" actual="$(ls dist | sort)" diff <(printf '%s\n' "$expected") <(printf '%s\n' "$actual") \ - || { echo "the release assets are not the eight expected files" >&2; exit 1; } + || { echo "the release assets are not the twelve expected files" >&2; exit 1; } for f in dist/*; do [ -s "$f" ] || { echo "$f is empty" >&2; exit 1; } @@ -230,9 +309,9 @@ jobs: done # A glob that matched nothing would leave the loop body unrun and the # step green, having verified precisely nothing. - [ "$checked" = "4" ] || { echo "verified $checked checksums, expected 4" >&2; exit 1; } + [ "$checked" = "6" ] || { echo "verified $checked checksums, expected 6" >&2; exit 1; } - echo "eight assets, all present, none empty, every digest matching" + echo "twelve assets, all present, none empty, every digest matching" - name: Lift the draft, then read the page back env: @@ -248,6 +327,6 @@ jobs: # Read the list again from the published release. What was checked in # the step above was the draft, and this is the page a reader opens. count="$(gh release view "$GITHUB_REF_NAME" --json assets --jq '.assets | length')" - [ "$count" = "8" ] || { echo "the published release carries $count assets, expected 8" >&2; exit 1; } + [ "$count" = "12" ] || { echo "the published release carries $count assets, expected 12" >&2; exit 1; } echo "published $GITHUB_REF_NAME" diff --git a/AGENTS.md b/AGENTS.md index dbf8a0a..842fe61 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,22 +11,27 @@ A guide to this repository for anyone changing it, people and coding agents alik ```sh zig build # binary at zig-out/bin/deed zig build test # unit tests, including real relays on loopback -zig fmt --check src build.zig # CI fails on unformatted code +zig fmt --check . # CI fails on unformatted code zig build -Doptimize=ReleaseSafe -Dstrip=true # the release build python3 bench/run.py zig-out/bin/deed # benchmarks, see BENCHMARKS.md +zig build test -Dtarget=x86_64-windows-gnu # compile the tests for Windows; the run step then fails, the host cannot run them ``` -Use the Zig version in `.zigversion`. The `nostr` dependency is pinned by URL and hash in `build.zig.zon`; move it with `zig fetch --save=nostr ` and check the diff is only the url and hash lines. +Use the Zig version in `.zigversion`. Building only the executable for Windows is not enough to catch a break there: code the executable never reaches, the test relay for one, only gets analysed when the test binary is built, which is what the Windows line above does. The `nostr` dependency is pinned by URL and hash in `build.zig.zon`; move it with `zig fetch --save=nostr ` and check the diff is only the url and hash lines. ## Layout +[`ARCHITECTURE.md`](ARCHITECTURE.md) explains how the pieces fit: where each command lives, how a run flows, the store, the exit codes and how it is tested. The tree below is the short version. + ``` src/ main.zig # dispatch, the stdout/stderr writers, exit codes cli.zig # exit code constants, the stdin record reader + console.zig # Windows only: UTF-8 console code pages for the length of a run cmd_*.zig # one file per verb relayset.zig # the shared relay query behind req and fetch dial.zig # dialling every relay at once under one deadline + storepath.zig # opening the store a --store path names, and saying why one will not open testrelay.zig # a websocket relay on loopback, for tests only bench/ # the benchmark script and its relay scripts/ # the one-line installer (pure ASCII, CI checks it) @@ -37,6 +42,8 @@ skills/deed/ # the skill for agents that operate deed - Every verb takes its inputs as arguments or, given none, as newline-delimited records on stdin, and writes one result per line on stdout. Diagnostics go to stderr. Keep it that way: it is what makes the verbs compose. - Exit codes are an interface: 0 success, 1 ran and failed, 2 not understood, 141 the reader went away. Do not add new ones casually. +- Match a NIP-19 prefix with `keyinput.hasPrefix` and strip `nostr:` with `keyinput.stripUri`, never `std.mem.startsWith`: bech32 allows all upper case, and a code in upper case has to reach the decoder. +- Open a `--store` path with `storepath.open`, so a new store's directories are made and a store that will not open says why. - Anything a relay sends is untrusted: events are verified and matched against the filter before they are printed or stored, and relay text is escaped before it reaches a terminal. - Tests that dial use `testrelay.DialAllocator`, not `std.testing.allocator`: on macOS a stack-capturing allocator can swallow a cancel and hang the test. - `zig build test` caches passing runs. To check for flakiness, run the test binary directly (`ls -t .zig-cache/o/*/test | head -1`) several times. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..6319020 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,112 @@ +# Architecture + +How `deed` is put together, and where to look first. For how to use it, read the [README](README.md) and `deed help `. For how to change it, read [`AGENTS.md`](AGENTS.md). The skill in [`skills/deed`](skills/deed/SKILL.md) is for programs that drive deed. + +## What it is + +A command line for nostr, written in Zig 0.16 on top of the [`nostr`](https://github.com/zig-nostr/nostr) library. It makes keys, builds and signs events, converts NIP-19 codes, encrypts with NIP-44, verifies signatures, asks relays for events, publishes to them, and keeps what it fetches in a local LMDB store. + +Everything with real protocol content comes from the library: keys and signatures, the event model, filters, relay connections, the store. deed is the command line around it: argument parsing, reading and writing records, running a query or a publish under a deadline, and deciding what to say and what to exit with. Nothing in `src/` implements cryptography. + +deed is one short-lived process per command. There is no daemon, no config file and no default location for the store. State lives in the environment (`NOSTR_SECRET_KEY`), in arguments, and in the store path when one is given. + +## Where each command lives + +`src/main.zig` reads the arguments, calls the verb, and turns its result into an exit status. Each verb is one file with a `usage` string and a `run` function that takes the allocator, `std.Io`, its arguments, and two writers, one for stdout and one for stderr. + +| command | file | what it does | +| --- | --- | --- | +| `key` | `cmd_key.zig` | `generate` a key, or derive the public key from a secret one | +| `event` | `cmd_event.zig` | build one event from flags, or sign drafts read from stdin | +| `decode` | `cmd_decode.zig` | NIP-19 code to a JSON object, one per code | +| `encode` | `cmd_encode.zig` | hex parts to a NIP-19 code | +| `encrypt`, `decrypt` | `cmd_crypt.zig` | NIP-44, both directions in one file | +| `verify` | `cmd_verify.zig` | check signatures, silent on success | +| `req` | `cmd_req.zig` | build a filter and run it against relays or against the store | +| `fetch` | `cmd_fetch.zig` | turn a code into a filter plus relay hints, then run it | +| `publish` | `cmd_publish.zig` | offer signed events to relays and print the accepted ones | + +Shared pieces: + +- `cli.zig`: the exit code constants, `isOneOf`, and `Input`, the reader every verb uses for its records. +- `relayset.zig`: the query that `req` and `fetch` share. +- `dial.zig`: dialling many relays at once under one deadline. +- `keyinput.zig`: reading a key as `nsec1...`, `npub1...` or hex, and the two helpers every code reader uses to match an entity prefix and strip `nostr:` ignoring ASCII case. Use them and not `startsWith`, or an uppercase code is refused as unknown. +- `storepath.zig`: opening the store a `--store` path names, see the store section. +- `jsonout.zig`: escaping strings for JSON output. +- `console.zig`: Windows only, see Platforms. +- `testrelay.zig`: a websocket relay on loopback, used by tests only. + +## How a run flows + +The verbs that need no network (`key`, `event`, `decode`, `encode`, `encrypt`, `decrypt`, `verify`) do the same three things: parse arguments, take records from the arguments or from stdin, write one result line per record. `Input` in `cli.zig` is the one place that rule lives. Positional arguments win over stdin, a line ending is removed and nothing else is trimmed, an empty line separates records, and a record too long for the buffer fails alone and does not take the rest of the stream with it. A bad record is reported on stderr, the stream carries on, and the exit code says something failed. + +`req` and `fetch` go on to the network: + +1. Parse the arguments into a `Filter`. `req` builds it from flags. `fetch` decodes the code, builds the filter it names, and adds the code's relay hints to the relays given on the command line. +2. With no relay named, `req` prints the `REQ` it would send and stops. With `--local` it answers from the store and dials nothing. +3. Open the store if `--store` was given. +4. `relayset.query` dials every relay at once (`dial.all`), subscribes on each one that connected, and reads until every relay has sent EOSE, the deadline passes, or, with `--stream`, until the deadline alone. +5. Each event is checked before anything else happens to it: duplicates by id are dropped, the signature is verified, and the event must match the filter that was asked. What fails is counted on stderr and never printed or stored. +6. What passes is held in a batch of up to 512, written to the store in one transaction, and then printed. It is stored before it is printed, so a run cut short keeps what it already showed. Without a store it is printed straight away. + +`publish` runs differently because it answers per event. It reads records, parses and verifies each one before anything is sent, dials the relays when the first valid event arrives, sends the event to every connected relay at once, and waits for their `OK` answers against one deadline. An event is printed to stdout only if at least one relay accepted it, so the output is a record of what was published. A relay that dropped an idle connection is dialled again for the next event. + +### Relays, deadlines and cancellation + +Name lookup is the one step that cannot be cut short. Every other wait is bounded. + +`dial.all` starts one task per relay with `Io.Select`, starts a timer beside them, and cancels whatever is still dialling when the timer fires. The bound is five seconds, or `--timeout` if that is shorter. A relay that does not answer is named on stderr and left out, and the run continues with the others. + +`publish` holds one reader task per connected relay for the whole run (`Set` in `cmd_publish.zig`). A reader answers pings and passes `OK` answers and notices to the main task through a queue, and reports the moment its connection goes away. Each message is tagged with the connection it came from, so what a dropped connection left in the queue is recognised and ignored. Sends are concurrent too, so a relay that has stopped reading holds up only its own send, which is cancelled at the deadline. + +`req` and `fetch` do not use a reader per relay. `relayset.query` reads from one thread and takes each relay in turn with a 100 ms read timeout, so a quiet relay costs the loop at most that long before it moves to the next one. + +Relay text is untrusted. `publish` escapes the notices and refusal reasons a relay sends before they reach a terminal, and an event is never taken on a relay's word: `req` and `fetch` verify it and match it against the filter first. + +## The store + +`--store ` names an LMDB file, opened with `nostr.store.Store`. `req` and `fetch` write to it and `req --local` reads from it. `publish` does not touch it. deed picks no default path. On Windows LMDB sizes the file to its whole map (the library's default, 1 GiB) when it is opened, so a new store takes that much disk at once. + +The store comes from the library, along with its indexes, replaceable event rules and deletion handling. deed's part is small: events are handed over in batches already verified, so the store is not asked to verify again, and the open goes through `storepath.zig`. + +`storepath.open` takes a mode. A run that writes (`req` and `fetch` with `--store`) uses `create`: it makes the directories above the path first, as `mkdir -p` does, and then lets LMDB make the file and its `-lock` file beside it. `req --local` only reads, so it uses `existing`: with no file there it says "there is no store at" and creates nothing. A leading `~` is read from `HOME`, because a quoted one is not expanded by the shell and would otherwise make a directory named `~`; `~user` is refused, and so is an empty path. LMDB reports every reason an open failed as one status, so when it fails `storepath` looks at the path itself and names the cause: a directory, a part of the path that is a file, no permission to read and write the file or its directory, a read-only file system, or a file that is not a store, which is left as it was. A failed open exits 1. + +## Exit codes + +They are part of the interface. `cli.zig` defines them. + +| | | +| --- | --- | +| `0` | it worked | +| `1` | the command ran and failed: a bad signature, an unreadable key, a malformed code, an event no relay accepted, no relay answering a `req` or `fetch` | +| `2` | the command was not understood: unknown verb or flag, missing argument. Nothing was attempted | +| `141` | the reader on the other end of the pipe went away | + +`req` and `fetch` exit 0 when some relay answered, even if nothing matched, because an empty answer is an answer. `publish` exits 0 only when every record was accepted by at least one relay. `main` also turns a failed flush of stdout into a nonzero status, so a result that was lost is never reported as success. + +## Platforms + +macOS, Linux and Windows, on x86_64 and aarch64. Linux release builds are static (musl). Most of the platform work is in the library. What deed handles itself is small: + +- Windows needs an allocator to split the command line into arguments, so `main` uses `Args.Iterator.initAllocator`, which is free elsewhere. +- A Windows console decodes text with its own code page. `console.zig` switches it to UTF-8 for the length of the run, the output page only when stdout is the console and the input page only when stdin is, so the deeds in a pipeline do not switch it back under each other. It puts the pages back before exit and on Ctrl-C. It does nothing on other systems. +- The test relay shrinks a socket's receive window. On Windows std's sockets are AFD handles Winsock does not know, so it sends the option as the same AFD request std uses for its own socket options, and that is best effort. + +Building only the executable for a target does not analyse code the executable never reaches. `zig build test -Dtarget=x86_64-windows-gnu` builds the test binary too, which is the check that finds a break in the test relay. The host cannot run the result, so the command ends with an error saying so once the compile has succeeded. + +## How it is tested + +`zig build test` runs the unit tests, which sit beside the code they test. They cover argument parsing and every verb's output and exit code, the record reader, the dispatcher, and the dial and query logic. + +The network tests talk to `testrelay.zig` over real sockets on loopback. It answers the websocket upgrade and then behaves according to a mode: accepting, refusing, serving stored events, going silent, never reading, sending pings or notices without stopping, closing mid-answer, sending hostile text with terminal escapes. Each mode exists for one failure a real relay can cause, and the test that uses it checks that the run ends on time with the right output and exit code. Nothing leaves the machine. + +Two details matter when adding a test that dials. Use `testrelay.DialAllocator` and not `std.testing.allocator`, because on macOS a stack-capturing allocator can swallow a cancel and hang the test. And do not take the stdin branch of a verb in a unit test, because it would block on the test runner's own stdin. `zig build test` caches passing runs, so to look for flakiness run the test binary directly several times. + +`bench/` is not a test suite. `bench/run.py` measures size, startup, memory, signing and verifying throughput, store speed, and relays over loopback against a build, using `bench/relay.py` as the relay. [`BENCHMARKS.md`](BENCHMARKS.md) has the numbers and how to reproduce them. + +CI (`.github/workflows/ci.yml`) builds, tests and checks formatting on Linux and macOS, builds and tests on Windows, checks that the installer script is pure ASCII and parses, and installs the latest release and runs it. The release workflow (`release.yml`) builds every target from the tag, publishes each as a draft with a SHA-256 beside it, downloads the artifacts back and runs them on matching runners, and lifts the draft only after every check passes. + +## Where to start reading + +`src/main.zig` for dispatch and exit codes. `src/cli.zig` for the record contract. Then one small verb such as `cmd_verify.zig`, and after it `relayset.zig` and `dial.zig`, which hold the part of deed that is not argument handling. `cmd_publish.zig` is the largest file because of the reader-per-relay machinery in it. diff --git a/README.md b/README.md index 22a65d7..1356370 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ A deed is two things at once: a signed instrument, and a thing done. So is a nos curl -fsSL https://raw.githubusercontent.com/zig-nostr/deed/main/scripts/install.sh | bash ``` -macOS and Linux, Intel and ARM. It works out which build this machine wants, checks the download against the SHA-256 published beside it, and installs into `~/.local/bin`, so nothing needs root and nothing lands outside your home directory. If the digest does not match, it installs nothing and says so. +For macOS and Linux, Intel and ARM; Windows is [below](#windows). It works out which build this machine wants, checks the download against the SHA-256 published beside it, and installs into `~/.local/bin`, so nothing needs root and nothing lands outside your home directory. If the digest does not match, it installs nothing and says so. `--prefix ` puts it somewhere else, `--version ` installs a particular release, and `--archive ` installs from a tarball you already have, which still wants its `.sha256` beside it. Pass `--help` for the list. @@ -19,7 +19,7 @@ macOS and Linux, Intel and ARM. It works out which build this machine wants, che The script is short and worth reading before you pipe anything into bash. If you would rather do it yourself: ```sh -VERSION=0.3.2 +VERSION=0.4.0 PLATFORM=macos-aarch64 # or macos-x86_64, linux-x86_64, linux-aarch64 BASE=https://github.com/zig-nostr/deed/releases/download/v$VERSION @@ -38,6 +38,26 @@ The macOS builds are ad-hoc signed and not notarized. A copy that arrived throug xattr -dr com.apple.quarantine deed ``` +### Windows + +Windows takes a zip rather than the script. Download `deed--windows-x86_64.zip` (or `windows-aarch64` on an ARM machine) and the `.sha256` beside it from the [releases page](https://github.com/zig-nostr/deed/releases), then in PowerShell, from the folder they are in: + +```powershell +$zip = (Get-Item deed-*-windows-*.zip).Name +$want = (Get-Content "$zip.sha256").Split(' ')[0] +$got = (Get-FileHash $zip -Algorithm SHA256).Hash +if ($got -ne $want) { throw "the SHA-256 does not match, do not run this" } + +$dir = "$env:LOCALAPPDATA\deed" +Expand-Archive $zip -DestinationPath $dir -Force +$user = [Environment]::GetEnvironmentVariable('Path', 'User') +if (($user -split ';') -notcontains $dir) { + [Environment]::SetEnvironmentVariable('Path', "$user;$dir".TrimStart(';'), 'User') +} +``` + +That checks the download, unpacks `deed.exe`, with the `LICENSE` and `README.md`, into `%LOCALAPPDATA%\deed`, and puts that folder on your `PATH` unless it is there already, so the same lines install a newer version over an older one. Keep only the zip you mean to install in the folder. Open a new terminal and `deed version` confirms it. PowerShell compares the two digests without regard to case, so the capitals `Get-FileHash` prints are fine. The binary is not code-signed, so Windows may show a warning the first time it runs. A store made with `--store` takes 1 GiB of disk on Windows from the first run, because LMDB sizes the file to its whole map up front there; on macOS and Linux it grows with what is kept. deed reads a leading `~` in a `--store` path from `HOME`, which Windows does not usually set, so give a full path there. + ## What it does deed reaches relays and keeps what it finds. Every verb that does not need a socket still works without one, so what comes out can be read before any of it leaves the machine. @@ -55,7 +75,7 @@ deed reaches relays and keeps what it finds. Every verb that does not need a soc | `fetch` | get the events a code names | | `publish` | offer signed events to relays, and print the ones they accepted | -`deed help ` explains any of them. +`deed help ` explains any of them. Wherever a verb takes an `npub`, `nsec` or `note` it also takes it in upper case, the form a QR code carries, and `decode` and `fetch` take a code with a `nostr:` prefix in either case. A code that mixes cases is refused, as bech32 requires. ## It keeps what it fetches @@ -66,6 +86,8 @@ deed req -k 1 -l 50 --store ~/.deed/db wss://relay.example # once, over the ne deed req -k 1 -l 50 --store ~/.deed/db --local # again, dialling nothing ``` +`--store` creates the store and any directories above it the first time, so the path above works on a machine that has no `~/.deed` yet, and a leading `~` means the home directory even when the shell did not expand it. `--local` only reads: it opens a store that exists, and says there is none rather than leaving an empty one behind a mistyped path. When a store will not open, the message says why: the path is a directory, a part of it is a file, there is no permission, or the file is not a store. + The second command opens no socket. The events came out of a local store that the first command filled, and they are the same events: `deed verify` is as happy with them as it was the first time, because what is stored is what was signed. Events are checked before they are stored or printed. A relay can send anything, so a signature that does not verify, and an event that does not answer the question that was asked, are both dropped and reported. @@ -74,8 +96,6 @@ Events are checked before they are stored or printed. A relay can send anything, **Relay selection.** A code with no relay hints is not looked up: `deed fetch npub1...` asks you to name a relay rather than going to find the author's relay list first. Doing that properly means a second round trip and a cache with its own staleness rules, and doing it badly is worse than saying so. -**Windows.** deed does not compile there yet, and the protocol library cannot resolve a hostname on Windows ([nostr#59](https://github.com/zig-nostr/nostr/issues/59)). - ## How the verbs fit together Each verb takes its inputs as arguments and, given none, reads them as newline-delimited records on standard input, writing one result per line. That is the whole reason they compose: @@ -135,6 +155,8 @@ zig build run -- --help Uses the Zig version pinned in `.zigversion`. The protocol library is pinned by URL and digest in `build.zig.zon`, so a build here and a build in CI are the same build. +[ARCHITECTURE.md](ARCHITECTURE.md) explains how deed is put together: where each command lives, how a run flows, the store, the exit codes and how it is tested. + ## License MIT. See [LICENSE](LICENSE). diff --git a/build.zig.zon b/build.zig.zon index 75f0b92..853f69e 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = .deed, - .version = "0.3.2", + .version = "0.4.0", .fingerprint = 0x89498c2094a1a4a3, .minimum_zig_version = "0.16.0", .dependencies = .{ @@ -10,8 +10,8 @@ // themselves later, and "whatever main happened to be that afternoon" // is not that. .nostr = .{ - .url = "https://github.com/zig-nostr/nostr/archive/refs/tags/v0.14.7.tar.gz", - .hash = "nostr-0.14.7-CMyPzf41CQDLOqp_yXGvqnbCen3jPgf63PpqDXZVxxEC", + .url = "https://github.com/zig-nostr/nostr/archive/refs/tags/v0.14.8.tar.gz", + .hash = "nostr-0.14.8-CMyPzRVbCgAb0DJX7YWTKSS1MQvD5o-wVv6QxcDCBdk0", }, }, .paths = .{ diff --git a/scripts/install.sh b/scripts/install.sh index ffd4de4..1b51ae2 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -9,7 +9,8 @@ # outside your home directory, and nothing is installed that did not verify. # # One script for both systems rather than two, because deed publishes the same -# thing four times: a tar.gz holding one static binary, named for the platform. +# thing four times for them: a tar.gz holding one static binary, named for the +# platform. Windows builds are zips and are installed by hand, see the README. # Two scripts would share every line that matters and drift in the ones that do # not. # @@ -86,7 +87,8 @@ detect() { case "$os" in darwin) os=macos ;; linux) os=linux ;; - *) die "deed publishes macOS and Linux builds. This machine reports '$os'." ;; + mingw*|msys*|cygwin*) die "this script installs the macOS and Linux builds. On Windows, download the .zip from https://github.com/$repo/releases and follow the README." ;; + *) die "deed publishes macOS, Linux and Windows builds. This machine reports '$os'." ;; esac case "$arch" in diff --git a/skills/deed/SKILL.md b/skills/deed/SKILL.md index 9eee0dd..a312ac6 100644 --- a/skills/deed/SKILL.md +++ b/skills/deed/SKILL.md @@ -5,7 +5,7 @@ description: Use deed, the nostr command line, to work with nostr from a shell. # deed -A fast command line for nostr. One binary, macOS and Linux. Every command takes its input as arguments or as newline-delimited records on stdin, and writes one result per line on stdout, so commands pipe into each other and into `jq`. +A fast command line for nostr. One binary, on macOS, Linux and Windows. Every command takes its input as arguments or as newline-delimited records on stdin, and writes one result per line on stdout, so commands pipe into each other and into `jq`. ## Install @@ -13,7 +13,7 @@ A fast command line for nostr. One binary, macOS and Linux. Every command takes curl -fsSL https://raw.githubusercontent.com/zig-nostr/deed/main/scripts/install.sh | bash ``` -Installs into `~/.local/bin` after checking the download's SHA-256. `deed version` confirms it. `deed help ` prints the exact usage of any command; check it before guessing a flag. +Installs into `~/.local/bin` after checking the download's SHA-256. That is for macOS and Linux; on Windows, download the `.zip` from the releases page and put `deed.exe` on `PATH`. `deed version` confirms it. `deed help ` prints the exact usage of any command; check it before guessing a flag. ## Before you act @@ -58,6 +58,7 @@ deed fetch npub1... wss://relay.example # a code with no relay hints needs # NIP-19 deed decode nprofile1... # {"pubkey":"...","relays":[...]} +deed decode NOSTR:NPUB1... # upper case and a nostr: prefix are both fine deed encode nevent --relay wss://relay.example --kind 1 # NIP-44 @@ -70,11 +71,11 @@ Filters for `req`: `-k` kind, `-a` author, `-i` id, `-e` / `-p` / `-t` tag value ## Things that trip people up - `deed event -` needs the `-` to read drafts from stdin; without it, it signs one empty-content event. +- `--store` creates the file and any directories above it, so `~/.deed/db` works on a fresh machine. `--local` only reads: with no store at the path it says so and exits 1. - `deed verify` is silent on success. Check the exit code, not the output. - Events from relays are verified and matched against the filter before they are printed, so a relay cannot slip in forged or unrelated events. Anything dropped is reported on stderr. - A relay that does not accept the connection within five seconds (or `--timeout`, if shorter) is named on stderr and left out; the run carries on with the others. - deed does not yet find an author's relays on its own: name the relays to ask. -- There is no Windows build yet. ## Judging what you fetch diff --git a/src/cmd_decode.zig b/src/cmd_decode.zig index b762a98..c18afdc 100644 --- a/src/cmd_decode.zig +++ b/src/cmd_decode.zig @@ -6,6 +6,7 @@ const std = @import("std"); const nostr = @import("nostr"); const cli = @import("cli.zig"); +const keyinput = @import("keyinput.zig"); const nip19 = nostr.nip19; const hex = nostr.hex; @@ -22,8 +23,9 @@ pub const usage = \\ deed decode [...] \\ \\Accepts npub, nsec, note, nprofile, nevent, naddr and nrelay, with or - \\without a leading `nostr:`. Reads codes one per line on stdin when given - \\no arguments. Prints one JSON object per code. + \\without a leading `nostr:`, in lower case or all upper case (the form a + \\QR code carries). Reads codes one per line on stdin when given no + \\arguments. Prints one JSON object per code. \\ \\ deed decode npub1… {"pubkey":"…"} \\ deed decode nevent1… {"id":"…","relays":[…],"kind":1} @@ -107,38 +109,38 @@ fn reason(e: anyerror) []const u8 { /// Decodes one code into a JSON object. Caller owns the returned bytes. fn decodeOne(gpa: std.mem.Allocator, raw: []const u8) ![]u8 { // `nostr:npub1…` and `npub1…` are the same code wearing different clothes. - const code = nip19.fromNostrUri(std.mem.trim(u8, raw, " \t\r\n")); + const code = keyinput.stripUri(std.mem.trim(u8, raw, " \t\r\n")); var buf: std.ArrayList(u8) = .empty; errdefer buf.deinit(gpa); try buf.append(gpa, '{'); - if (std.mem.startsWith(u8, code, "npub1")) { + if (keyinput.hasPrefix(code, "npub1")) { try appendHexField(&buf, gpa, "pubkey", &try nip19.decodeNpub(gpa, code), true); - } else if (std.mem.startsWith(u8, code, "nsec1")) { + } else if (keyinput.hasPrefix(code, "nsec1")) { try appendHexField(&buf, gpa, "seckey", &try nip19.decodeNsec(gpa, code), true); - } else if (std.mem.startsWith(u8, code, "note1")) { + } else if (keyinput.hasPrefix(code, "note1")) { try appendHexField(&buf, gpa, "id", &try nip19.decodeNote(gpa, code), true); - } else if (std.mem.startsWith(u8, code, "nprofile1")) { + } else if (keyinput.hasPrefix(code, "nprofile1")) { var p = try nip19.decodeNprofile(gpa, code); defer p.deinit(gpa); try appendHexField(&buf, gpa, "pubkey", &p.pubkey, true); try appendRelays(&buf, gpa, p.relays, false); - } else if (std.mem.startsWith(u8, code, "nevent1")) { + } else if (keyinput.hasPrefix(code, "nevent1")) { var p = try nip19.decodeNevent(gpa, code); defer p.deinit(gpa); try appendHexField(&buf, gpa, "id", &p.id, true); try appendRelays(&buf, gpa, p.relays, false); if (p.author) |a| try appendHexField(&buf, gpa, "author", &a, false); if (p.kind) |k| try appendIntField(&buf, gpa, "kind", k, false); - } else if (std.mem.startsWith(u8, code, "naddr1")) { + } else if (keyinput.hasPrefix(code, "naddr1")) { var p = try nip19.decodeNaddr(gpa, code); defer p.deinit(gpa); try appendStringField(&buf, gpa, "identifier", p.identifier, true); try appendHexField(&buf, gpa, "pubkey", &p.pubkey, false); try appendIntField(&buf, gpa, "kind", p.kind, false); try appendRelays(&buf, gpa, p.relays, false); - } else if (std.mem.startsWith(u8, code, "nrelay1")) { + } else if (keyinput.hasPrefix(code, "nrelay1")) { const url = try nip19.decodeNrelay(gpa, code); defer gpa.free(url); try appendStringField(&buf, gpa, "url", url, true); @@ -265,3 +267,47 @@ test "an unknown prefix is refused" { const gpa = std.testing.allocator; try std.testing.expectError(error.InvalidPrefix, decodeOne(gpa, "nwhat1abc")); } + +test "an uppercase code, with either scheme case, decodes like the lowercase one" { + const gpa = std.testing.allocator; + const id = [_]u8{0x05} ** 32; + const hints = [_][]const u8{"wss://relay.example"}; + const codes = [_][]u8{ + try nip19.encodeNpub(gpa, id), + try nip19.encodeNsec(gpa, id), + try nip19.encodeNote(gpa, id), + try nip19.encodeNprofile(gpa, id, &hints), + try nip19.encodeNevent(gpa, id, &hints, id, 1), + try nip19.encodeNaddr(gpa, "slug", id, 30023, &hints), + try nip19.encodeNrelay(gpa, "wss://relay.example"), + }; + defer for (codes) |c| gpa.free(c); + + for (codes) |lower| { + const want = try decodeOne(gpa, lower); + defer gpa.free(want); + const upper = try std.ascii.allocUpperString(gpa, lower); + defer gpa.free(upper); + const forms = [_][]const u8{ "", "NOSTR:", "nostr:" }; + for (forms) |scheme| { + const input = try std.fmt.allocPrint(gpa, "{s}{s}", .{ scheme, upper }); + defer gpa.free(input); + const got = try decodeOne(gpa, input); + defer gpa.free(got); + try std.testing.expectEqualStrings(want, got); + } + const loud = try std.fmt.allocPrint(gpa, "NOSTR:{s}", .{lower}); + defer gpa.free(loud); + const got = try decodeOne(gpa, loud); + defer gpa.free(got); + try std.testing.expectEqualStrings(want, got); + } +} + +test "a code that mixes cases is still refused" { + const gpa = std.testing.allocator; + const npub = try nip19.encodeNpub(gpa, [_]u8{0x06} ** 32); + defer gpa.free(npub); + npub[0] = 'N'; + try std.testing.expectError(error.MixedCase, decodeOne(gpa, npub)); +} diff --git a/src/cmd_fetch.zig b/src/cmd_fetch.zig index fbe3b6b..bd5ad85 100644 --- a/src/cmd_fetch.zig +++ b/src/cmd_fetch.zig @@ -6,7 +6,9 @@ const std = @import("std"); const nostr = @import("nostr"); const cli = @import("cli.zig"); +const keyinput = @import("keyinput.zig"); const relayset = @import("relayset.zig"); +const storepath = @import("storepath.zig"); const nip19 = nostr.nip19; const filter = nostr.filter; @@ -19,8 +21,8 @@ pub const usage = \\ deed fetch [...] \\ \\Accepts note, nevent, naddr, nprofile and npub, with or without a leading - \\`nostr:`. A code carrying relay hints is looked for at those relays as - \\well as any named here. + \\`nostr:`, in lower case or all upper case. A code carrying relay hints is + \\looked for at those relays as well as any named here. \\ \\Options: \\ --store keep every event this receives in a local store @@ -30,6 +32,9 @@ pub const usage = \\within --timeout if that is shorter, is named on stderr and left out. \\Looking up a relay's name is the one step that cannot be cut short. \\ + \\A store is one file, with a -lock file beside it. --store creates them, and + \\any directories above them, the first time. + \\ \\A bare npub fetches that person's profile rather than everything they have \\ever written, which is what `npub` on its own can sensibly mean. \\ @@ -119,12 +124,7 @@ pub fn run( var store: ?nostr.store.Store = null; defer if (store) |*st| st.deinit(); if (store_path) |path| { - const z = try gpa.dupeZ(u8, path); - defer gpa.free(z); - store = nostr.store.Store.open(z, .{}) catch |e| { - try err.print("deed fetch: cannot open the store at {s}: {s}\n", .{ path, @errorName(e) }); - return cli.exit_fail; - }; + store = (try storepath.open(gpa, io, "fetch", path, .create, err)) orelse return cli.exit_fail; } const outcome = try relayset.query(gpa, io, urls.items, &.{target.filter}, out, err, .{ @@ -171,18 +171,18 @@ fn resolve( d_values: *[1][]const u8, tags: *[1]filter.TagFilter, ) !Resolved { - const s = if (std.mem.startsWith(u8, raw, "nostr:")) raw["nostr:".len..] else raw; + const s = keyinput.stripUri(raw); - if (std.mem.startsWith(u8, s, "nevent1")) { + if (keyinput.hasPrefix(s, "nevent1")) { const p = try nip19.decodeNevent(gpa, s); ids[0] = p.id; return .{ .filter = .{ .ids = ids[0..1] }, .hints = p.relays, .owned = p.relays }; } - if (std.mem.startsWith(u8, s, "note1")) { + if (keyinput.hasPrefix(s, "note1")) { ids[0] = try nip19.decodeNote(gpa, s); return .{ .filter = .{ .ids = ids[0..1] }, .hints = &.{} }; } - if (std.mem.startsWith(u8, s, "naddr1")) { + if (keyinput.hasPrefix(s, "naddr1")) { const p = try nip19.decodeNaddr(gpa, s); authors[0] = p.pubkey; kinds[0] = @intCast(p.kind); @@ -195,7 +195,7 @@ fn resolve( .owned_identifier = p.identifier, }; } - if (std.mem.startsWith(u8, s, "nprofile1")) { + if (keyinput.hasPrefix(s, "nprofile1")) { const p = try nip19.decodeNprofile(gpa, s); authors[0] = p.pubkey; kinds[0] = 0; @@ -205,7 +205,7 @@ fn resolve( .owned = p.relays, }; } - if (std.mem.startsWith(u8, s, "npub1")) { + if (keyinput.hasPrefix(s, "npub1")) { authors[0] = try nip19.decodeNpub(gpa, s); // Kind 0, not everything. A bare npub names a person, and the thing a // person's code most usefully resolves to is who they say they are. @@ -326,6 +326,47 @@ test "a nostr: prefix is stripped, and nonsense is refused" { try testing.expectError(error.InvalidPrefix, resolveFor("not-a-code", bufs)); } +test "an uppercase code resolves like the lowercase one, with or without NOSTR:" { + var ids: [1][32]u8 = undefined; + var authors: [1][32]u8 = undefined; + var kinds: [1]u16 = undefined; + var d: [1][]const u8 = undefined; + var tags: [1]filter.TagFilter = undefined; + const bufs = .{ .ids = &ids, .authors = &authors, .kinds = &kinds, .d = &d, .tags = &tags }; + + const key = [_]u8{0x44} ** 32; + const hints = [_][]const u8{"wss://one.example"}; + const codes = [_][]u8{ + try nip19.encodeNote(testing.allocator, key), + try nip19.encodeNevent(testing.allocator, key, &hints, null, null), + try nip19.encodeNaddr(testing.allocator, "my-article", key, 30023, &.{}), + try nip19.encodeNprofile(testing.allocator, key, &hints), + try nip19.encodeNpub(testing.allocator, key), + }; + defer for (codes) |c| testing.allocator.free(c); + + for (codes) |lower| { + var want = try resolveFor(lower, bufs); + defer want.deinit(testing.allocator); + const want_hints = want.hints.len; + const want_id: ?[32]u8 = if (want.filter.ids) |v| v[0] else null; + const want_author: ?[32]u8 = if (want.filter.authors) |v| v[0] else null; + + const upper = try std.ascii.allocUpperString(testing.allocator, lower); + defer testing.allocator.free(upper); + const forms = [_][]const u8{ "", "NOSTR:" }; + for (forms) |scheme| { + const input = try std.fmt.allocPrint(testing.allocator, "{s}{s}", .{ scheme, upper }); + defer testing.allocator.free(input); + var got = try resolveFor(input, bufs); + defer got.deinit(testing.allocator); + try testing.expectEqual(want_hints, got.hints.len); + try testing.expectEqual(want_id, if (got.filter.ids) |v| v[0] else null); + try testing.expectEqual(want_author, if (got.filter.authors) |v| v[0] else null); + } + } +} + test "64 hex characters are taken as an event id" { var ids: [1][32]u8 = undefined; var authors: [1][32]u8 = undefined; diff --git a/src/cmd_publish.zig b/src/cmd_publish.zig index 1b42784..40dcb5d 100644 --- a/src/cmd_publish.zig +++ b/src/cmd_publish.zig @@ -497,6 +497,9 @@ const Set = struct { timeout_ms: i64, stalled: State, waiting: *[dial.max_relays]bool, + /// Where a send that failed outright is handed back instead of being + /// reported, so the caller can try it once more. Null reports it. + send_failed: ?*[dial.max_relays]?anyerror, err: *std.Io.Writer, ) !void { var results: [dial.max_relays]?Sent = @splat(null); @@ -556,6 +559,9 @@ const Set = struct { // socket is half a frame, so this connection is done. try err.print("deed publish: {s}: {s}: could not send within {d} ms\n", .{ id, self.urls[i], timeout_ms }); try self.drop(i, stalled, err); + } else if (send_failed) |f| { + f[i] = e; + try self.drop(i, .closed, err); } else { try err.print("deed publish: {s}: {s}: {s}\n", .{ id, self.urls[i], @errorName(e) }); try self.drop(i, .closed, err); @@ -585,7 +591,26 @@ const Set = struct { which[n] = i; n += 1; } - try self.sendSome(which[0..n], ev, id, deadline, timeout_ms, .gone, &waiting, err); + var send_failed: [dial.max_relays]?anyerror = @splat(null); + try self.sendSome(which[0..n], ev, id, deadline, timeout_ms, .gone, &waiting, &send_failed, err); + // A relay that dropped an idle connection can also be found out by the + // send itself rather than by the read after it: Windows refuses the + // write at once where POSIX takes it and reports the close next. Either + // way it gets the one fresh connection and one more try that a close + // while waiting gets below. + for (0..self.urls.len) |i| { + const e = send_failed[i] orelse continue; + if (msLeft(io, deadline) >= min_resend_ms) { + resent[i] = true; + try self.dialSome(&.{i}, @min(dial.default_timeout_ms, msLeft(io, deadline)), .closed, err); + if (self.conns[i] != null) { + // Reports its own failure this time. + try self.sendSome(&.{i}, ev, id, deadline, timeout_ms, .closed, &waiting, null, err); + continue; + } + } + try err.print("deed publish: {s}: {s}: {s}\n", .{ id, self.urls[i], @errorName(e) }); + } self.seq += 1; var timer = try io.concurrent(deadlineAfter, .{ &self.heard, io, msLeft(io, deadline), self.seq }); @@ -639,7 +664,7 @@ const Set = struct { resent[i] = true; try self.dialSome(&.{i}, @min(dial.default_timeout_ms, msLeft(io, deadline)), .closed, err); if (self.conns[i] != null) { - try self.sendSome(&.{i}, ev, id, deadline, timeout_ms, .closed, &waiting, err); + try self.sendSome(&.{i}, ev, id, deadline, timeout_ms, .closed, &waiting, null, err); if (waiting[i]) continue; } } diff --git a/src/cmd_req.zig b/src/cmd_req.zig index 3c16272..9b5f680 100644 --- a/src/cmd_req.zig +++ b/src/cmd_req.zig @@ -8,7 +8,9 @@ const std = @import("std"); const nostr = @import("nostr"); const cli = @import("cli.zig"); +const keyinput = @import("keyinput.zig"); const relayset = @import("relayset.zig"); +const storepath = @import("storepath.zig"); const filter = nostr.filter; const message = nostr.message; @@ -40,6 +42,10 @@ pub const usage = \\within --timeout if that is shorter, is named on stderr and left out. \\Looking up a relay's name is the one step that cannot be cut short. \\ + \\A store is one file, with a -lock file beside it. --store creates them, and + \\any directories above them, the first time; --local reads only a store that + \\already exists. + \\ \\Given no relay, it prints what it would send and stops, so a filter can be \\read before it is asked of anybody: \\ @@ -91,8 +97,8 @@ fn Collected(comptime T: type) type { fn idOrKey(gpa: std.mem.Allocator, s: []const u8) ?[32]u8 { const t = std.mem.trim(u8, s, " \t\r\n"); if (t.len == 64) return hex.decodeFixed(32, t) catch null; - if (std.mem.startsWith(u8, t, "npub1")) return nostr.nip19.decodeNpub(gpa, t) catch null; - if (std.mem.startsWith(u8, t, "note1")) return nostr.nip19.decodeNote(gpa, t) catch null; + if (keyinput.hasPrefix(t, "npub1")) return nostr.nip19.decodeNpub(gpa, t) catch null; + if (keyinput.hasPrefix(t, "note1")) return nostr.nip19.decodeNote(gpa, t) catch null; return null; } @@ -281,12 +287,7 @@ pub fn run( try err.writeAll("deed req: --local needs --store to read from\n"); return cli.exit_usage; }; - const z = try gpa.dupeZ(u8, path); - defer gpa.free(z); - var st = nostr.store.Store.open(z, .{}) catch |e| { - try err.print("deed req: cannot open the store at {s}: {s}\n", .{ path, @errorName(e) }); - return cli.exit_fail; - }; + var st = (try storepath.open(gpa, io, "req", path, .existing, err)) orelse return cli.exit_fail; defer st.deinit(); var result = st.query(gpa, req.filter) catch |e| { @@ -319,15 +320,7 @@ pub fn run( var store: ?nostr.store.Store = null; defer if (store) |*st| st.deinit(); if (req.store_path) |path| { - const z = gpa.dupeZ(u8, path) catch { - try err.writeAll("deed req: out of memory opening the store\n"); - return cli.exit_fail; - }; - defer gpa.free(z); - store = nostr.store.Store.open(z, .{}) catch |e| { - try err.print("deed req: cannot open the store at {s}: {s}\n", .{ path, @errorName(e) }); - return cli.exit_fail; - }; + store = (try storepath.open(gpa, io, "req", path, .create, err)) orelse return cli.exit_fail; } const outcome = try relayset.query(gpa, io, req.relays, &.{req.filter}, out, err, .{ @@ -397,6 +390,30 @@ test "an author is taken as an npub or as hex, and means the same thing" { try std.testing.expect(std.mem.indexOf(u8, from_hex.out, hex_key) != null); } +test "an uppercase npub or note is taken the same as a lowercase one" { + const gpa = std.testing.allocator; + const key = [_]u8{0x55} ** 32; + const npub = try nostr.nip19.encodeNpub(gpa, key); + defer gpa.free(npub); + const note = try nostr.nip19.encodeNote(gpa, key); + defer gpa.free(note); + const npub_up = try std.ascii.allocUpperString(gpa, npub); + defer gpa.free(npub_up); + const note_up = try std.ascii.allocUpperString(gpa, note); + defer gpa.free(note_up); + + try std.testing.expectEqual(key, idOrKey(gpa, npub_up).?); + try std.testing.expectEqual(key, idOrKey(gpa, note_up).?); + + var ob: [4096]u8 = undefined; + var eb: [1024]u8 = undefined; + const lower = try runReq(&.{ "-a", npub, "--bare" }, &ob, &eb); + var ob2: [4096]u8 = undefined; + var eb2: [1024]u8 = undefined; + const upper = try runReq(&.{ "-a", npub_up, "--bare" }, &ob2, &eb2); + try std.testing.expectEqualStrings(lower.out, upper.out); +} + test "tag filters land under their own letter" { var ob: [4096]u8 = undefined; var eb: [1024]u8 = undefined; @@ -448,3 +465,45 @@ test "help is printed on stdout and succeeds" { try std.testing.expectEqual(cli.exit_ok, r.code); try std.testing.expect(std.mem.indexOf(u8, r.out, "deed req") != null); } + +test "--local reads only a store that exists, and creates nothing" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var root: [std.fs.max_path_bytes]u8 = undefined; + const root_len = try tmp.dir.realPath(std.testing.io, &root); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/typo/store.mdb", .{root[0..root_len]}); + + var ob: [1024]u8 = undefined; + var eb: [1024]u8 = undefined; + const r = try runReq(&.{ "-k", "1", "--store", path, "--local" }, &ob, &eb); + try std.testing.expectEqual(cli.exit_fail, r.code); + try std.testing.expect(std.mem.indexOf(u8, r.err, "there is no store at ") != null); + try std.testing.expectError(error.FileNotFound, tmp.dir.statFile(std.testing.io, "typo", .{})); +} + +test "--store creates the directories a new store needs" { + const io = std.testing.io; + const testrelay = @import("testrelay.zig"); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var root: [std.fs.max_path_bytes]u8 = undefined; + const root_len = try tmp.dir.realPath(io, &root); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/a/b/store.mdb", .{root[0..root_len]}); + + // A closed port: the store is opened before any relay is dialled, and the + // refused dial ends the run at once. + var closed = try testrelay.listen(io); + const port = closed.socket.address.ip4.port; + closed.deinit(io); + var url_buf: [40]u8 = undefined; + const url = try testrelay.url(&url_buf, port); + + var ob: [1024]u8 = undefined; + var eb: [1024]u8 = undefined; + const r = try runReq(&.{ "-k", "1", "--store", path, url }, &ob, &eb); + try std.testing.expect(std.mem.indexOf(u8, r.err, "the store") == null); + try std.testing.expect(std.mem.indexOf(u8, r.err, "no relay answered") != null); + _ = try tmp.dir.statFile(io, "a/b/store.mdb", .{}); +} diff --git a/src/console.zig b/src/console.zig new file mode 100644 index 0000000..0b27d2b --- /dev/null +++ b/src/console.zig @@ -0,0 +1,117 @@ +//! The Windows console and UTF-8. +//! +//! deed writes UTF-8 and nothing else: JSON, NIP-19 codes and the ellipses in +//! its help text. A Windows console decodes what it is given with its own code +//! page (437 or 850 on most machines), so without this, `deed help event` +//! prints `nsec1ΓǪ`. Switching the console to UTF-8 for the length of the run +//! fixes that, and putting it back afterwards leaves the user's shell as it was. +//! +//! The code page belongs to the console, not to a handle, so every process +//! attached to it shares one. That is why a page is only switched for the +//! standard handle that is the console itself: in `deed key generate | deed +//! event`, only the second deed writes to the console, so only it switches the +//! output page, and the first one exiting early cannot switch it back under the +//! second one's output. The cost is that a deed whose stdout is redirected +//! prints its stderr with the console's own page. +//! +//! Ctrl-C ends a run (`req --stream` has no other way to stop) without reaching +//! the end of `main`, so a console control handler puts the pages back then +//! too. Everything here is a no-op off Windows. + +const std = @import("std"); +const builtin = @import("builtin"); + +const is_windows = builtin.os.tag == .windows; + +const utf8: u32 = 65001; + +const kernel32 = struct { + const std_input: u32 = @bitCast(@as(i32, -10)); + const std_output: u32 = @bitCast(@as(i32, -11)); + + extern "kernel32" fn GetStdHandle(which: u32) callconv(.winapi) ?*anyopaque; + extern "kernel32" fn GetConsoleMode(handle: ?*anyopaque, mode: *u32) callconv(.winapi) c_int; + extern "kernel32" fn GetConsoleCP() callconv(.winapi) u32; + extern "kernel32" fn GetConsoleOutputCP() callconv(.winapi) u32; + extern "kernel32" fn SetConsoleCP(cp: u32) callconv(.winapi) c_int; + extern "kernel32" fn SetConsoleOutputCP(cp: u32) callconv(.winapi) c_int; + extern "kernel32" fn SetConsoleCtrlHandler( + handler: *const fn (u32) callconv(.winapi) c_int, + add: c_int, + ) callconv(.winapi) c_int; +}; + +/// The code pages to put back, 0 where nothing was changed. +const Saved = struct { + input: u32 = 0, + output: u32 = 0, +}; + +/// What `useUtf8` changed. Written once before the control handler is +/// installed, and only read after that. +var saved: Saved = .{}; + +/// Which pages to switch, given whether stdin and stdout are the console and +/// the pages in force now. A page of 0 means there is no console; one already +/// UTF-8 needs nothing. +fn plan(stdin_is_console: bool, stdout_is_console: bool, input: u32, output: u32) Saved { + return .{ + .input = if (stdin_is_console and input != 0 and input != utf8) input else 0, + .output = if (stdout_is_console and output != 0 and output != utf8) output else 0, + }; +} + +/// Switches the console to UTF-8 where deed reads from it or writes to it. +pub fn useUtf8() void { + if (comptime !is_windows) return; + var want = plan( + isConsole(kernel32.std_input), + isConsole(kernel32.std_output), + kernel32.GetConsoleCP(), + kernel32.GetConsoleOutputCP(), + ); + // Only what was actually changed is put back. + if (want.input != 0 and kernel32.SetConsoleCP(utf8) == 0) want.input = 0; + if (want.output != 0 and kernel32.SetConsoleOutputCP(utf8) == 0) want.output = 0; + saved = want; + if (saved.input != 0 or saved.output != 0) _ = kernel32.SetConsoleCtrlHandler(onControl, 1); +} + +/// Puts back the pages `useUtf8` changed. Safe to call more than once. +pub fn restore() void { + if (comptime !is_windows) return; + if (saved.input != 0) _ = kernel32.SetConsoleCP(saved.input); + if (saved.output != 0) _ = kernel32.SetConsoleOutputCP(saved.output); +} + +fn isConsole(which: u32) bool { + var mode: u32 = 0; + return kernel32.GetConsoleMode(kernel32.GetStdHandle(which), &mode) != 0; +} + +/// Runs on a thread of its own on Ctrl-C, Ctrl-Break or a closing window. +/// Returning 0 passes the event on, so the default handler still ends the run. +fn onControl(_: u32) callconv(.winapi) c_int { + restore(); + return 0; +} + +test "off Windows the console is left alone" { + if (is_windows) return error.SkipZigTest; + useUtf8(); + try std.testing.expectEqual(Saved{}, saved); + restore(); +} + +test "a page is switched only for the handle that is the console" { + // Both are the console: both pages switch, and the old ones are kept. + try std.testing.expectEqual(Saved{ .input = 437, .output = 850 }, plan(true, true, 437, 850)); + // `deed key generate | deed event`: the first one writes into a pipe and + // must leave the output page to the one that writes to the console. + try std.testing.expectEqual(Saved{ .input = 437 }, plan(true, false, 437, 437)); + // And the second one reads from a pipe. + try std.testing.expectEqual(Saved{ .output = 437 }, plan(false, true, 437, 437)); + // Already UTF-8, or no console at all: nothing to change or put back. + try std.testing.expectEqual(Saved{}, plan(true, true, utf8, utf8)); + try std.testing.expectEqual(Saved{}, plan(true, true, 0, 0)); +} diff --git a/src/dial.zig b/src/dial.zig index d9a049c..a5af009 100644 --- a/src/dial.zig +++ b/src/dial.zig @@ -10,6 +10,7 @@ //! libc call: a resolver that hangs still holds its dial until it returns. const std = @import("std"); +const builtin = @import("builtin"); const nostr = @import("nostr"); const Io = std.Io; @@ -139,7 +140,13 @@ test "a silent relay times out, a closed port fails, and a live relay connects, }; try std.testing.expect(out[0] == .timed_out); - try std.testing.expect(out[1] == .failed); + // Windows retries a refused connect for about two seconds before it + // gives up, so there the closed port runs into the deadline instead. + if (builtin.os.tag == .windows) { + try std.testing.expect(out[1] == .failed or out[1] == .timed_out); + } else { + try std.testing.expect(out[1] == .failed); + } try std.testing.expect(out[2] == .connected); // Bounded by the deadline, not by the slowest relay. try std.testing.expect(took < 3_000); diff --git a/src/keyinput.zig b/src/keyinput.zig index bda0cd2..5320bd4 100644 --- a/src/keyinput.zig +++ b/src/keyinput.zig @@ -13,10 +13,26 @@ const hex = nostr.hex; pub const Error = error{UnrecognizedKey}; +/// Whether `s` starts with `prefix`, ignoring ASCII case. Bech32 allows a +/// code to be all lowercase or all uppercase (a QR code carries the uppercase +/// form), and only a mix is invalid. This sniffs the entity; the decoder keeps +/// the job of refusing a mix. +pub fn hasPrefix(s: []const u8, prefix: []const u8) bool { + return std.ascii.startsWithIgnoreCase(s, prefix); +} + +/// Strips a leading `nostr:` (any case, URI schemes are case-insensitive). +/// Returns a slice into `s`. +pub fn stripUri(s: []const u8) []const u8 { + const scheme = "nostr:"; + if (hasPrefix(s, scheme)) return s[scheme.len..]; + return s; +} + /// A secret key as `nsec1…` or as 64 hex characters. pub fn secretKey(gpa: std.mem.Allocator, s: []const u8) !keys.SecretKey { const t = std.mem.trim(u8, s, " \t\r\n"); - if (std.mem.startsWith(u8, t, "nsec1")) return nip19.decodeNsec(gpa, t); + if (hasPrefix(t, "nsec1")) return nip19.decodeNsec(gpa, t); if (t.len == 64) return hex.decodeFixed(32, t); return Error.UnrecognizedKey; } @@ -24,7 +40,7 @@ pub fn secretKey(gpa: std.mem.Allocator, s: []const u8) !keys.SecretKey { /// A public key as `npub1…` or as 64 hex characters. pub fn publicKey(gpa: std.mem.Allocator, s: []const u8) !keys.PublicKey { const t = std.mem.trim(u8, s, " \t\r\n"); - if (std.mem.startsWith(u8, t, "npub1")) return nip19.decodeNpub(gpa, t); + if (hasPrefix(t, "npub1")) return nip19.decodeNpub(gpa, t); if (t.len == 64) return hex.decodeFixed(32, t); return Error.UnrecognizedKey; } @@ -45,3 +61,34 @@ test "an unrecognized key is refused rather than guessed at" { const gpa = std.testing.allocator; try std.testing.expectError(Error.UnrecognizedKey, secretKey(gpa, "not-a-key")); } + +test "the entity prefix and the nostr: scheme match in either case" { + try std.testing.expect(hasPrefix("npub1abc", "npub1")); + try std.testing.expect(hasPrefix("NPUB1ABC", "npub1")); + try std.testing.expect(!hasPrefix("nsec1abc", "npub1")); + try std.testing.expect(!hasPrefix("np", "npub1")); + try std.testing.expectEqualStrings("npub1abc", stripUri("nostr:npub1abc")); + try std.testing.expectEqualStrings("NPUB1ABC", stripUri("NOSTR:NPUB1ABC")); + try std.testing.expectEqualStrings("NPUB1ABC", stripUri("NPUB1ABC")); +} + +test "an uppercase nsec or npub is a key, and a mixed-case one is not" { + const gpa = std.testing.allocator; + const raw = [_]u8{0x22} ** 32; + const nsec = try nip19.encodeNsec(gpa, raw); + defer gpa.free(nsec); + const npub = try nip19.encodeNpub(gpa, raw); + defer gpa.free(npub); + const nsec_up = try std.ascii.allocUpperString(gpa, nsec); + defer gpa.free(nsec_up); + const npub_up = try std.ascii.allocUpperString(gpa, npub); + defer gpa.free(npub_up); + + try std.testing.expectEqual(raw, try secretKey(gpa, nsec_up)); + try std.testing.expectEqual(try publicKey(gpa, npub), try publicKey(gpa, npub_up)); + + const mixed = try gpa.dupe(u8, npub); + defer gpa.free(mixed); + mixed[0] = 'N'; + try std.testing.expectError(error.MixedCase, publicKey(gpa, mixed)); +} diff --git a/src/main.zig b/src/main.zig index 648edef..6b28a4c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5,6 +5,7 @@ const std = @import("std"); const cli = @import("cli.zig"); +const console = @import("console.zig"); const cmd_decode = @import("cmd_decode.zig"); const cmd_encode = @import("cmd_encode.zig"); const cmd_crypt = @import("cmd_crypt.zig"); @@ -15,7 +16,7 @@ const cmd_publish = @import("cmd_publish.zig"); const cmd_req = @import("cmd_req.zig"); const cmd_verify = @import("cmd_verify.zig"); -pub const version = "0.3.2"; +pub const version = "0.4.0"; const usage = \\deed: the nostr command line @@ -54,13 +55,25 @@ pub fn main(init: std.process.Init) !void { // decoding a stream of codes spent most of its time in mmap and munmap. const gpa = std.heap.smp_allocator; + // Windows only: the console decodes output with its own code page, and + // everything deed writes is UTF-8. A no-op elsewhere. Every way out of + // `main` puts the code page back: the errdefer for a failed setup, the + // call before `exit` below, and a control handler for Ctrl-C. + console.useUtf8(); + errdefer console.restore(); + var threaded = std.Io.Threaded.init(gpa, .{}); defer threaded.deinit(); const io = threaded.io(); var argv: std.ArrayList([]const u8) = .empty; defer argv.deinit(gpa); - var it = std.process.Args.Iterator.init(init.minimal.args); + // `initAllocator` rather than `init`: on Windows the command line is one + // string that has to be split into arguments, and the iterator owns the + // buffer they point into. It is a no-op everywhere else. The arguments + // live as long as `it`, which is the whole run. + var it = try std.process.Args.Iterator.initAllocator(init.minimal.args, gpa); + defer it.deinit(); _ = it.skip(); // argv[0] while (it.next()) |a| try argv.append(gpa, a); @@ -102,6 +115,7 @@ pub fn main(init: std.process.Init) !void { // still buffered would simply be lost. const final = finish(&stdout, &stderr, code); stderr.interface.flush() catch {}; + console.restore(); std.process.exit(final); } @@ -314,6 +328,7 @@ test "what is not understood exits 2 and says so on stderr" { test { std.testing.refAllDecls(@This()); _ = @import("cli.zig"); + _ = @import("console.zig"); _ = @import("keyinput.zig"); _ = @import("jsonout.zig"); _ = @import("cmd_crypt.zig"); @@ -325,6 +340,7 @@ test { _ = @import("cmd_publish.zig"); _ = @import("cmd_req.zig"); _ = @import("relayset.zig"); + _ = @import("storepath.zig"); _ = @import("dial.zig"); _ = @import("testrelay.zig"); _ = @import("cmd_verify.zig"); diff --git a/src/storepath.zig b/src/storepath.zig new file mode 100644 index 0000000..64383f9 --- /dev/null +++ b/src/storepath.zig @@ -0,0 +1,388 @@ +//! Opening the store a `--store` path names. +//! +//! LMDB turns every reason it could not open a file into one status, and the +//! library reports that as `error.Lmdb`, which tells the reader nothing they can +//! act on. So when an open fails this looks at the path itself and says what is +//! wrong with it. + +const std = @import("std"); +const builtin = @import("builtin"); +const nostr = @import("nostr"); + +pub const Mode = enum { + /// Create the store when it is missing, and every directory above it. + create, + /// Open only a store that already exists. A run that only reads should + /// not leave an empty store, or directories, behind a mistyped path. + existing, +}; + +/// Opens the store at `path`, or writes one line to `err` saying why it could +/// not and returns null. A leading `~` is the home directory: a quoted `~` is +/// never expanded by the shell, and creating a directory literally named `~` +/// is never what was meant. +pub fn open( + gpa: std.mem.Allocator, + io: std.Io, + verb: []const u8, + path: []const u8, + mode: Mode, + err: *std.Io.Writer, +) !?nostr.store.Store { + return openWithHome(gpa, io, verb, path, getHome(), mode, err); +} + +fn openWithHome( + gpa: std.mem.Allocator, + io: std.Io, + verb: []const u8, + path: []const u8, + home: ?[]const u8, + mode: Mode, + err: *std.Io.Writer, +) !?nostr.store.Store { + if (path.len == 0) { + try err.print("deed {s}: --store needs a path, and it was empty\n", .{verb}); + return null; + } + const full = expandHome(gpa, path, home) catch |e| switch (e) { + error.OutOfMemory => return e, + error.NoHome => { + try err.print("deed {s}: cannot open the store at {s}: HOME is not set, so there is no ~ to expand\n", .{ verb, path }); + return null; + }, + error.OtherUser => { + try err.print("deed {s}: cannot open the store at {s}: only ~ and ~/ are expanded, so write that path out in full\n", .{ verb, path }); + return null; + }, + }; + defer gpa.free(full); + + const cwd = std.Io.Dir.cwd(); + switch (mode) { + .create => if (std.fs.path.dirname(full)) |parent| { + cwd.createDirPath(io, parent) catch |e| { + try err.print("deed {s}: cannot create the directory {s} for the store: {s}\n", .{ verb, parent, whyNotCreated(e) }); + return null; + }; + }, + .existing => if (cwd.statFile(io, full, .{})) |_| {} else |e| switch (e) { + error.FileNotFound => { + if (fileInTheWay(io, full)) { + try err.print("deed {s}: cannot open the store at {s}: a part of the path is a file, not a directory\n", .{ verb, full }); + } else { + try err.print("deed {s}: there is no store at {s}\n", .{ verb, full }); + } + return null; + }, + // Anything else is described below, once the open has failed too. + else => {}, + }, + } + + const z = try gpa.dupeZ(u8, full); + defer gpa.free(z); + if (nostr.store.Store.open(z, .{})) |st| return st else |e| switch (e) { + error.OutOfMemory => return e, + error.CorruptRecord => { + try err.print("deed {s}: cannot open the store at {s}: it holds a damaged record\n", .{ verb, full }); + return null; + }, + error.Lmdb => { + try err.print("deed {s}: cannot open the store at {s}: {s}\n", .{ verb, full, whyNotOpened(io, full) }); + return null; + }, + } +} + +fn getHome() ?[]const u8 { + const value = std.c.getenv("HOME") orelse return null; + const span = std.mem.span(value); + return if (span.len == 0) null else span; +} + +/// Whether the nearest part of `full` that exists is a file rather than a +/// directory. Some systems report a path through a file as not found rather +/// than as not a directory, and "there is no store" would hide the real fault. +fn fileInTheWay(io: std.Io, full: []const u8) bool { + var p = full; + while (std.fs.path.dirname(p)) |parent| : (p = parent) { + const stat = std.Io.Dir.cwd().statFile(io, parent, .{}) catch |e| switch (e) { + error.FileNotFound => continue, + error.NotDir => return true, + else => return false, + }; + return stat.kind != .directory; + } + return false; +} + +/// `~` and `~/rest` become the home directory and `home/rest`. Any other path +/// comes back as it was. The caller frees the result. +fn expandHome(gpa: std.mem.Allocator, path: []const u8, home: ?[]const u8) error{ OutOfMemory, NoHome, OtherUser }![]u8 { + if (path.len == 0 or path[0] != '~') return gpa.dupe(u8, path); + if (path.len > 1 and path[1] != '/') return error.OtherUser; + const h = home orelse return error.NoHome; + return std.mem.concat(gpa, u8, &.{ h, path[1..] }); +} + +fn whyNotCreated(e: std.Io.Dir.CreateDirPathError) []const u8 { + return switch (e) { + error.AccessDenied, error.PermissionDenied => "permission denied", + error.NotDir, error.PathAlreadyExists => "a part of it is a file, not a directory", + error.ReadOnlyFileSystem => "the file system is read-only", + error.NoSpaceLeft => "the disk is full", + error.NameTooLong => "the path is too long", + else => @errorName(e), + }; +} + +/// Why LMDB could not open `full`, found by asking the file system directly. +/// LMDB keeps a second file beside the store, `-lock`, and needs to +/// read and write both. +fn whyNotOpened(io: std.Io, full: []const u8) []const u8 { + const cwd = std.Io.Dir.cwd(); + const parent = std.fs.path.dirname(full) orelse "."; + const stat = cwd.statFile(io, full, .{}) catch |e| return switch (e) { + error.FileNotFound => whyNoNewFile(io, parent, "no permission to create the store in its directory") orelse "LMDB could not create it", + error.NotDir => "a part of the path is a file, not a directory", + error.AccessDenied, error.PermissionDenied => "permission denied on a directory above it", + else => @errorName(e), + }; + if (stat.kind == .directory) return "it is a directory, and a store is a single file"; + cwd.access(io, full, .{ .read = true, .write = true }) catch |e| return switch (e) { + error.AccessDenied, error.PermissionDenied => "no permission to read and write it", + error.ReadOnlyFileSystem => "it is on a read-only file system", + else => @errorName(e), + }; + + var buf: [std.fs.max_path_bytes]u8 = undefined; + if (std.fmt.bufPrint(&buf, "{s}-lock", .{full})) |lock| { + if (cwd.access(io, lock, .{ .read = true, .write = true })) |_| {} else |e| switch (e) { + error.FileNotFound => if (whyNoNewFile(io, parent, "no permission to create its lock file in its directory")) |why| return why, + error.AccessDenied, error.PermissionDenied => return "no permission to read and write its lock file", + error.ReadOnlyFileSystem => return "its lock file is on a read-only file system", + else => {}, + } + } else |_| {} + return unknown; +} + +const unknown = "LMDB could not open it, so it is probably not a store"; + +/// Why a file that does not exist yet could not be made in `parent`, or null +/// when nothing about `parent` explains it. +fn whyNoNewFile(io: std.Io, parent: []const u8, denied: []const u8) ?[]const u8 { + std.Io.Dir.cwd().access(io, parent, .{ .write = true }) catch |e| return switch (e) { + error.AccessDenied, error.PermissionDenied => denied, + error.ReadOnlyFileSystem => "its directory is on a read-only file system", + error.FileNotFound => "its directory does not exist", + else => @errorName(e), + }; + return null; +} + +// -- tests ------------------------------------------------------------------ + +const testing = std.testing; + +/// A temporary directory, its absolute path, and somewhere for messages. +const Fixture = struct { + tmp: testing.TmpDir, + root_buf: [std.fs.max_path_bytes]u8 = undefined, + root_len: usize = 0, + err_buf: [512]u8 = undefined, + err: std.Io.Writer = undefined, + + fn init(self: *Fixture) !void { + self.* = .{ .tmp = testing.tmpDir(.{}) }; + self.root_len = try self.tmp.dir.realPath(testing.io, &self.root_buf); + self.err = .fixed(&self.err_buf); + } + + fn deinit(self: *Fixture) void { + self.tmp.cleanup(); + } + + fn path(self: *Fixture, buf: []u8, rest: []const u8) ![]const u8 { + return std.fmt.bufPrint(buf, "{s}/{s}", .{ self.root_buf[0..self.root_len], rest }); + } + + fn open(self: *Fixture, p: []const u8, mode: Mode) !?nostr.store.Store { + return openWithHome(testing.allocator, testing.io, "req", p, null, mode, &self.err); + } + + fn said(self: *Fixture, words: []const u8) !void { + const text = self.err.buffered(); + if (std.mem.indexOf(u8, text, words) == null) { + std.debug.print("expected the message to say \"{s}\", it said: {s}\n", .{ words, text }); + return error.TestUnexpectedResult; + } + } + + fn exists(self: *Fixture, rest: []const u8) bool { + _ = self.tmp.dir.statFile(testing.io, rest, .{}) catch return false; + return true; + } +}; + +fn runningAsRoot() bool { + return std.c.geteuid() == 0; +} + +test "a store under directories that do not exist yet is created, directories and all" { + var f: Fixture = undefined; + try f.init(); + defer f.deinit(); + var buf: [std.fs.max_path_bytes]u8 = undefined; + const p = try f.path(&buf, "a/b/c/store.mdb"); + + var st = (try f.open(p, .create)) orelse return error.TestUnexpectedResult; + st.deinit(); + try testing.expect(f.exists("a/b/c/store.mdb")); + try testing.expectEqualStrings("", f.err.buffered()); +} + +test "a relative path with .. in it is created where it points" { + var f: Fixture = undefined; + try f.init(); + defer f.deinit(); + // tmpDir lives under the working directory, so this path is relative. + var buf: [std.fs.max_path_bytes]u8 = undefined; + const p = try std.fmt.bufPrint(&buf, ".zig-cache/tmp/{s}/x/../y/store.mdb", .{&f.tmp.sub_path}); + + var st = (try f.open(p, .create)) orelse return error.TestUnexpectedResult; + st.deinit(); + try testing.expect(f.exists("y/store.mdb")); +} + +test "a run that only reads leaves nothing behind a path with no store" { + var f: Fixture = undefined; + try f.init(); + defer f.deinit(); + var buf: [std.fs.max_path_bytes]u8 = undefined; + const p = try f.path(&buf, "typo/store.mdb"); + + try testing.expect(try f.open(p, .existing) == null); + try f.said("there is no store at "); + try testing.expect(!f.exists("typo")); +} + +test "a directory where the store should be is named as one" { + var f: Fixture = undefined; + try f.init(); + defer f.deinit(); + try f.tmp.dir.createDir(testing.io, "d", .default_dir); + var buf: [std.fs.max_path_bytes]u8 = undefined; + const p = try f.path(&buf, "d"); + + try testing.expect(try f.open(p, .create) == null); + try f.said("it is a directory, and a store is a single file"); +} + +test "a file in the way of a directory is named as one" { + var f: Fixture = undefined; + try f.init(); + defer f.deinit(); + try f.tmp.dir.writeFile(testing.io, .{ .sub_path = "file", .data = "x" }); + var buf: [std.fs.max_path_bytes]u8 = undefined; + + try testing.expect(try f.open(try f.path(&buf, "file/sub/store.mdb"), .create) == null); + try f.said("cannot create the directory "); + try f.said("a part of it is a file, not a directory"); + + f.err = .fixed(&f.err_buf); + try testing.expect(try f.open(try f.path(&buf, "file/store.mdb"), .existing) == null); + try f.said("a part of the path is a file, not a directory"); +} + +test "a file that is not a store is not opened, and not overwritten" { + var f: Fixture = undefined; + try f.init(); + defer f.deinit(); + const text = "these are my notes, not a database\n" ** 200; + try f.tmp.dir.writeFile(testing.io, .{ .sub_path = "notes.txt", .data = text }); + var buf: [std.fs.max_path_bytes]u8 = undefined; + + try testing.expect(try f.open(try f.path(&buf, "notes.txt"), .create) == null); + try f.said("probably not a store"); + var read_buf: [text.len + 1]u8 = undefined; + try testing.expectEqualStrings(text, try f.tmp.dir.readFile(testing.io, "notes.txt", &read_buf)); +} + +test "a directory that cannot be written is named as the reason" { + // Windows has no mode bits to take away. + if (comptime builtin.os.tag == .windows) return error.SkipZigTest; + if (runningAsRoot()) return error.SkipZigTest; + var f: Fixture = undefined; + try f.init(); + defer f.deinit(); + try f.tmp.dir.createDir(testing.io, "locked", .default_dir); + try f.tmp.dir.setFilePermissions(testing.io, "locked", .fromMode(0o555), .{}); + defer f.tmp.dir.setFilePermissions(testing.io, "locked", .fromMode(0o755), .{}) catch {}; + var buf: [std.fs.max_path_bytes]u8 = undefined; + + // A directory below it has to be made, and cannot be. + try testing.expect(try f.open(try f.path(&buf, "locked/sub/store.mdb"), .create) == null); + try f.said("cannot create the directory "); + try f.said("permission denied"); + + // The directory exists, and the store file cannot be made in it. + f.err = .fixed(&f.err_buf); + try testing.expect(try f.open(try f.path(&buf, "locked/store.mdb"), .create) == null); + try f.said("no permission to create the store in its directory"); +} + +test "a store that cannot be read and written is named as the reason" { + // Windows has no mode bits to take away. + if (comptime builtin.os.tag == .windows) return error.SkipZigTest; + if (runningAsRoot()) return error.SkipZigTest; + var f: Fixture = undefined; + try f.init(); + defer f.deinit(); + var buf: [std.fs.max_path_bytes]u8 = undefined; + const p = try f.path(&buf, "store.mdb"); + var st = (try f.open(p, .create)) orelse return error.TestUnexpectedResult; + st.deinit(); + try f.tmp.dir.setFilePermissions(testing.io, "store.mdb", .fromMode(0o444), .{}); + + try testing.expect(try f.open(p, .existing) == null); + try f.said("no permission to read and write it"); +} + +test "~ is the home directory, and nothing else is touched" { + const gpa = testing.allocator; + const cases = [_]struct { in: []const u8, want: []const u8 }{ + .{ .in = "~", .want = "/home/me" }, + .{ .in = "~/.deed/db", .want = "/home/me/.deed/db" }, + .{ .in = "./~/db", .want = "./~/db" }, + .{ .in = "a/~/db", .want = "a/~/db" }, + .{ .in = "../db", .want = "../db" }, + }; + for (cases) |c| { + const got = try expandHome(gpa, c.in, "/home/me"); + defer gpa.free(got); + try testing.expectEqualStrings(c.want, got); + } + try testing.expectError(error.OtherUser, expandHome(gpa, "~bob/db", "/home/me")); + try testing.expectError(error.NoHome, expandHome(gpa, "~/db", null)); +} + +test "a quoted ~ opens under the home directory, not in a directory named ~" { + var f: Fixture = undefined; + try f.init(); + defer f.deinit(); + var st = (try openWithHome(testing.allocator, testing.io, "req", "~/deed/store.mdb", f.root_buf[0..f.root_len], .create, &f.err)) orelse + return error.TestUnexpectedResult; + st.deinit(); + try testing.expect(f.exists("deed/store.mdb")); + try testing.expectError(error.FileNotFound, std.Io.Dir.cwd().statFile(testing.io, "~", .{})); +} + +test "an empty path is refused rather than opened" { + var f: Fixture = undefined; + try f.init(); + defer f.deinit(); + try testing.expect(try f.open("", .create) == null); + try f.said("--store needs a path"); +} diff --git a/src/testrelay.zig b/src/testrelay.zig index a2e25a0..494700b 100644 --- a/src/testrelay.zig +++ b/src/testrelay.zig @@ -5,6 +5,7 @@ //! the same `nostr.relay.dial` a user's run goes through, over a real socket. const std = @import("std"); +const builtin = @import("builtin"); const nostr = @import("nostr"); const Io = std.Io; @@ -68,6 +69,33 @@ pub const Mode = enum { /// test hangs instead of failing. `std.testing.allocator` captures them. pub const DialAllocator = std.heap.DebugAllocator(.{ .stack_trace_frames = 0 }); +/// Sets a receive buffer of 4096 bytes on `handle`. +/// +/// std's sockets on Windows are AFD handles that it opens without Winsock, so +/// Winsock's `setsockopt` does not know them. The option goes through the same +/// AFD request std uses for its own socket options instead. It is best effort +/// there: if the driver turns it down, the two modes that use it still work, +/// because what they send fills a large window too, only later. +fn shrinkReceiveWindow(io: Io, handle: Io.net.Socket.Handle) !void { + const size: c_int = 4096; + if (comptime builtin.os.tag == .windows) { + const windows = std.os.windows; + _ = try io.operate(.{ .device_io_control = .{ + .file = .{ .handle = handle, .flags = .{ .nonblocking = true } }, + .code = windows.IOCTL.AFD.SOCKOPT, + .in = @ptrCast(&windows.AFD.SOCKOPT_INFO{ + .mode = .set, + .level = windows.ws2_32.SOL.SOCKET, + .optname = windows.ws2_32.SO.RCVBUF, + .optval = &size, + .optlen = @sizeOf(c_int), + }), + } }); + return; + } + try std.posix.setsockopt(handle, std.posix.SOL.SOCKET, std.posix.SO.RCVBUF, std.mem.asBytes(&size)); +} + /// A loopback listener on a port the kernel picks. pub fn listen(io: Io) !Io.net.Server { var address: Io.net.IpAddress = .{ .ip4 = .loopback(0) }; @@ -107,8 +135,7 @@ pub const Relay = struct { // A small window, inherited by the connection it accepts, so what // is sent to it fills the sender's socket after a few hundred // kilobytes however large the system's defaults are. - const size: c_int = 4096; - try std.posix.setsockopt(self.server.socket.handle, std.posix.SOL.SOCKET, std.posix.SO.RCVBUF, std.mem.asBytes(&size)); + try shrinkReceiveWindow(io, self.server.socket.handle); } self.task = try io.concurrent(serve, .{ self, io }); }