From 681279219356648698b9805cf4032af3a5674831 Mon Sep 17 00:00:00 2001 From: Joey Stanford Date: Tue, 15 Sep 2026 17:22:18 -0600 Subject: [PATCH 1/2] feat: NomadNet 1.4.3 media conversion, clearable cache, client helpers Close host parity for non-WebP /media conversion with an in-memory (or embedder-owned disk) conversion cache that clear_* can wipe, and export Link request helpers so embedders stay off wire details. --- Cargo.lock | 218 +++++++++++++++ Cargo.toml | 10 + README.md | 68 +++-- ROADMAP.md | 27 +- crates/nomad-core/Cargo.toml | 1 + crates/nomad-core/src/client.rs | 223 ++++++++++++++++ crates/nomad-core/src/lib.rs | 19 ++ crates/nomad-core/src/media_cache.rs | 356 +++++++++++++++++++++++++ crates/nomad-core/src/media_convert.rs | 167 ++++++++++++ crates/nomad-core/src/node.rs | 192 +++++++++++-- crates/nomad-core/src/storage.rs | 4 +- 11 files changed, 1235 insertions(+), 50 deletions(-) create mode 100644 crates/nomad-core/src/client.rs create mode 100644 crates/nomad-core/src/media_cache.rs create mode 100644 crates/nomad-core/src/media_convert.rs diff --git a/Cargo.lock b/Cargo.lock index d61cdf4..c6a3155 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aes" version = "0.8.4" @@ -58,6 +64,18 @@ dependencies = [ "objc2", ] +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.12.1" @@ -110,6 +128,12 @@ dependencies = [ "inout", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "combine" version = "4.6.8" @@ -135,6 +159,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -234,12 +273,38 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "fiat-crypto" version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -272,6 +337,27 @@ dependencies = [ "r-efi", ] +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hex" version = "0.4.3" @@ -306,6 +392,35 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + [[package]] name = "inout" version = "0.1.4" @@ -413,6 +528,26 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.3" @@ -424,6 +559,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "nix" version = "0.29.0" @@ -442,6 +587,7 @@ version = "0.1.0" dependencies = [ "bytes", "hex", + "image", "libc", "rmpv", "rns-crypto", @@ -555,6 +701,19 @@ dependencies = [ "spki", ] +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -573,6 +732,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.47" @@ -932,6 +1103,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "smallvec" version = "1.16.0" @@ -1049,6 +1226,20 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + [[package]] name = "tokio" version = "1.53.1" @@ -1142,6 +1333,12 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "winapi-util" version = "0.1.11" @@ -1364,8 +1561,29 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index e28951e..f1b343f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,16 @@ nomad-core = { path = "crates/nomad-core", version = "0.1.0" } rmpv = "1" serde = { version = "1", features = ["derive"] } +# Image conversion (NomadNet 1.4.3 host /media) +image = { version = "0.25", default-features = false, features = [ + "png", + "jpeg", + "gif", + "bmp", + "tiff", + "webp", +] } + # Async tokio = { version = "1", features = ["full"] } bytes = "1" diff --git a/README.md b/README.md index 3b61c9e..6ac1471 100644 --- a/README.md +++ b/README.md @@ -20,14 +20,18 @@ --- rsNomad is a Rust implementation of Nomad Network **page, file, and media hosting** -over Reticulum Links. This is not a fork of NomadNet; it is NomadNet page-server +over Reticulum Links, plus shared **client request helpers** (path hashes, bodies, +timeouts). This is not a fork of NomadNet; it is NomadNet interaction-layer behavior written in a different language, focused on staying interoperable with -Python NomadNet **1.4.1** (PyPI) and MeshChat. It is not the source-of-truth -implementation — do not treat it as one. +Python NomadNet **1.4.3** hosting (`Node.py` pages/files/media including +non-WebP→WebP conversion) and MeshChat / mesh-client browsers. It is not the +source-of-truth implementation — do not treat it as one. Page hosting uses Reticulum Link request/response on aspect `nomadnetwork.node`. It is **not** LXMF messaging; use [rsLXMF](https://github.com/ratspeak/rsLXMF) for -delivery and propagation. +delivery and propagation. Browser UI, Micron rendering, and fetched-image LRU +caches belong in clients (see mesh-client); this crate is the embeddable +interaction layer. This repository currently lives under [Colorado-Mesh/rsNomad](https://github.com/Colorado-Mesh/rsNomad). Layout, license, @@ -131,6 +135,9 @@ let node = NomadNode::spawn( announce_interval: Some(Duration::from_secs(3600)), announce_at_start: true, allow_executable_pages: false, // opt-in Unix CGI / executable .allowed + // media_cache: MediaCacheConfig::memory_only() by default + // For long-lived hosts: MediaCacheConfig::with_disk_root("/var/cache/nomad-media") + ..NomadNodeConfig::default() }, ) .await?; @@ -138,13 +145,21 @@ let node = NomadNode::spawn( println!("serving at {}", node.destination_hash_hex()); node.store().write_page_rel("about.mu", b"> About\n")?; node.reload_routes()?; // required after content CRUD so new routes are served +// Embedders: wipe conversion cache from any user "Clear cache" control: +node.clear_media_cache()?; ``` `NomadNode` registers the `nomadnetwork.node` destination, installs a Link request handler for `/page/...`, `/file/...`, and `/media`, and announces with the display name as raw UTF-8 app data (canonical NomadNet format). Form bodies are decoded for CGI pages when `allow_executable_pages` is enabled; `/media` -uses `decode_media_request` (`path` + `key`). +uses `decode_media_request` (`path` + `key`). Non-WebP images under `pages/` +are converted to WebP (NomadNet 1.4.3) and stored in an **in-memory** conversion +cache by default. + +**Client helpers** (no Link I/O): `build_page_request` / `build_file_request` / +`build_media_request`, `overall_timeout_secs`, `reply_file_name`, and related +constants — so sidecar/embedders do not reimplement path hashes or codecs. This crate is not published to crates.io. For the full public API (CRUD helpers, stats, announce, error types, limits), generate local docs: @@ -172,13 +187,22 @@ Mapping: - `pages/index.mu` → `/page/index.mu` - `pages/docs/help.mu` → `/page/docs/help.mu` -- `pages/header.webp` → `/media` request with `path` = `header.webp` (WebP only) +- `pages/header.webp` → `/media` request with `path` = `header.webp` (native WebP) +- `pages/photo.png` → `/media` request with `path` = `photo.png` (converted to WebP) - `files/manual.pdf` → `/file/manual.pdf` Paths are resolved under each root without following symlink components; `..`, absolute escapes, NUL/backslash, and control characters are rejected. Default size caps are **512 KiB** for pages and **32 MiB** for files/media. +**Media conversion cache:** NomadNet’s `converted_node` analogue. Default is +**in-memory LRU** (256 entries / 192 MiB). Disk is opt-in only via +`MediaCacheConfig::with_disk_root(path)` — the path must be **embedder-owned**, +never under `pages/` or `files/`. Call `NomadNode::clear_media_cache()` (and +optionally `clear_media_cache_key`) from any user-facing clear-cache control. +Do **not** implement a parallel ImageCache in the mesh-client reticulum sidecar +or write `/cache/images/`. + **Trust model:** content directories are trusted local storage. Operators must ensure they are not writable by untrusted local users. Symlink components are rejected; hard links under the same volume are not rejected (a hard-linked file @@ -213,11 +237,14 @@ CRUD. executable pages are enabled - Media: `encode_media_request` / `decode_media_request` for `{path, key}` maps (`key` may be Nil); route string exactly `/media` +- Media types: `.webp` served native; `.png` / `.jpg` / `.jpeg` / `.bmp` / + `.gif` / `.tiff` converted to WebP (quality 85, max dimension 1200) with a + clearable conversion cache (NomadNet 1.4.3 `Node.serve_media` parity) - Large responses: use normal `Reply` bytes; `LinkManager` upgrades to a response Resource when the packed reply exceeds the Link MDU - File / media responses: `ReplyFile` — a response Resource with raw bytes and msgpack metadata `{"name": ...}` (relative path for `/file`, basename for - `/media`) + `/media`; converted media uses `*.webp` basename) - Announce app data: raw UTF-8 display name, capped at 256 bytes (also accepted by mesh-client discovery) - Hidden paths: dotfiles and `*.allowed` are not listed or served as content @@ -232,7 +259,9 @@ CRUD. | --- | --- | | Static pages | Serve `.mu` (and other text) from `pages/` with 512 KiB default cap | | Static files | Serve binaries from `files/` with 32 MiB default cap as response Resources with filename metadata | -| `/media` WebP | Exact `/media` route; pages-jail WebP only; basename `ReplyFile` metadata | +| `/media` WebP | Exact `/media` route; native WebP + convert PNG/JPG/GIF/BMP/TIFF → WebP | +| Media conversion cache | In-memory LRU by default; optional embedder disk root; `clear_media_cache` / `clear_media_cache_key` | +| Client request helpers | `build_*_request`, timeout stages, `reply_file_name` (no Link I/O) | | `.allowed` ACL | Static identity-hash lists; optional sandboxed executable companions | | CGI pages | Opt-in (`allow_executable_pages`); Unix-only sandbox; default off | | Announce | Startup + periodic + transport reannounce with display name | @@ -240,20 +269,26 @@ CRUD. | Default index | Placeholder Micron page when `index.mu` is missing | | Path safety | Traversal/symlink rejection, size limits, skip listing dotfiles/`*.allowed` | | Request budget | Bounded in-flight handlers + fixed-window admit limit | +| Browser image/page LRU | mesh-client UI concern — not in this crate | | Markdown CMS | Application concern (e.g. mesh-client UI) — not in this crate | | Chat / forums | Roadmap only | | `nomad-serve-rs` CLI | Planned (optional tools crate) | ## Compatibility Notes -Target clients: Python [NomadNet](https://github.com/markqvist/NomadNet) **1.4.1** +Target clients: Python [NomadNet](https://github.com/markqvist/NomadNet) **1.4.3** and MeshChat browsers, plus [mesh-client](https://github.com/Colorado-Mesh/mesh-client) Nomad tab. -Compatibility target for hosting behavior is the NomadNet **1.4.1 PyPI sdist** -(`Node.py`: `serve_page`, `serve_file`, `serve_media`, `request_allowed`). -CGI is **opt-in** and sandboxed (cleared env); Python inherits the parent -environment — an intentional hardening difference. +Compatibility target for hosting behavior is NomadNet **1.4.3** +(`Node.py`: `serve_page`, `serve_file`, `serve_media` with conversion, +`request_allowed`). CGI is **opt-in** and sandboxed (cleared env); Python +inherits the parent environment — an intentional hardening difference. + +**Embedder contract (mesh-client reticulum sidecar):** call into `nomad-core` +only. Do not own a sidecar `ImageCache` or create `/cache/images/`. +Wire any “Clear cache” UI to `NomadNode::clear_media_cache()`. Fetched remote +image caching stays in the TypeScript renderer. This crate depends on Ratspeak [rsReticulum](https://github.com/ratspeak/rsReticulum) path dependencies during development. It is not compatible with unrelated RNS @@ -267,10 +302,9 @@ Follow-ups (not required for basic hosting): 2. Richer Micron helpers / builders 3. Transfer repository ownership to the Ratspeak organization when permissions allow -Application-layer CMS, chat rooms, forums, LXMF image/file attachments, and -Micron rendering belong in clients such as mesh-client / rsLXMF, not in this -protocol crate. In-page images use `/media` WebP under `pages/`; other binaries -remain `/file/...` with Resource filename metadata. +Application-layer CMS, chat rooms, forums, LXMF image/file attachments, Micron +rendering, and **browser** image/page caches belong in clients such as +mesh-client / rsLXMF, not in this protocol crate. ## Contributing diff --git a/ROADMAP.md b/ROADMAP.md index 48a4a44..10e0182 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -23,25 +23,30 @@ static hosting release used by mesh-client (#613). - Opt-in Unix CGI pages (`NomadNodeConfig.allow_executable_pages`, default off) - Micron `not_allowed_page()` matching Python `DEFAULT_NOTALLOWED` +## Done (NomadNet 1.4.3 hosting + interaction helpers) + +- Host `/media` conversion for PNG/JPG/JPEG/BMP/GIF/TIFF → WebP (NomadNet + `Node.convert_media_to_webp` / `MEDIA_EXTS`) +- Clearable conversion cache: in-memory LRU default; optional embedder + `MediaCacheConfig::disk_root`; `NomadNode::clear_media_cache` / + `clear_media_cache_key` (never under content `pages/` / `files/`) +- Client Link request helpers: `build_page_request` / `build_file_request` / + `build_media_request`, timeout stages, `reply_file_name` + ## Near-term - **Clients import existing `nomad-core` constants** (mesh-client sidecar still - hardcodes `NOMAD_NODE_ASPECT` and page/file size caps that this crate already - exports). No rsNomad change required — switch the sidecar to - `NOMAD_NODE_ASPECT`, `DEFAULT_MAX_PAGE_BYTES`, and `DEFAULT_MAX_FILE_BYTES`. -- Optional `nomad-client` crate for shared fetch timeouts / Link query - skeleton — add when a **second Rust consumer** appears (e.g. `nomad-tools`) - or timeout constants drift and cause bugs. Until then keep timeout math in - the sidecar; mesh-client product policy such as `force_path_refresh` stays in - clients. TS UI/proxy mirrors remain client-side. + hardcodes some timeout helpers that this crate now exports — prefer + `overall_timeout_secs` / `link_initiator_hops` over local duplicates). - Optional `nomad-tools` crate with `nomad-serve-rs` headless binary -- Stronger interop fixtures against Python NomadNet page fetches +- Stronger interop fixtures against Python NomadNet page/media fetches - Async / `spawn_blocking` serve path if LinkManager gains an async handler API ## Later (application / mesh-client) These belong in clients such as mesh-client, not in the protocol crate: +- Browser fetched-image LRU / clear-cache UI (`browser/` analogue) - Markdown → Micron page composer / CMS workflow - Theme and navigation editors - NomadNet-style chat room apps @@ -54,8 +59,10 @@ These belong in clients such as mesh-client, not in the protocol crate: - Unsandboxed CGI with full parent-env inheritance (Python footgun; we clear env) - Embedding hosting inside `rsLXMF` - Depending on non-Ratspeak RNS stacks (`nomadnet-rs` / `rns-net`) -- Server-side MIME/`/image/` routes (in-page images use `/media` WebP; other +- Server-side MIME/`/image/` routes (in-page images use `/media`; other binaries remain ordinary `/file/...`) +- Sidecar-owned ImageCache or `/cache/images/` trees +- TUI `converted_disp` terminal glyph cache ## Ownership diff --git a/crates/nomad-core/Cargo.toml b/crates/nomad-core/Cargo.toml index a180d99..86693dd 100644 --- a/crates/nomad-core/Cargo.toml +++ b/crates/nomad-core/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] bytes = { workspace = true } hex = { workspace = true } +image = { workspace = true } rmpv = { workspace = true } rns-crypto = { workspace = true } rns-identity = { workspace = true } diff --git a/crates/nomad-core/src/client.rs b/crates/nomad-core/src/client.rs new file mode 100644 index 0000000..a52e2e0 --- /dev/null +++ b/crates/nomad-core/src/client.rs @@ -0,0 +1,223 @@ +//! Client-side Nomad Link request helpers (path hashes, bodies, timeouts). +//! +//! These do **not** open Links or render Micron — embedders (mesh-client sidecar) +//! still own transport. Helpers centralize wire identity so callers do not +//! reimplement path hashes / media codecs / default timeout stages. + +use std::collections::BTreeMap; +use std::time::Duration; + +use crate::error::NomadError; +use crate::paths::{ + FILE_PREFIX, MEDIA_ROUTE, PAGE_PREFIX, normalize_file_route, normalize_page_route, path_hash, +}; +use crate::request::{encode_media_request, encode_request_fields}; + +/// MeshChat / NomadNet path lookup stage default. +pub const NOMAD_PATH_LOOKUP_SECS: u64 = 15; +/// MeshChat TCP link establishment default. +pub const NOMAD_TCP_LINK_ESTABLISH_SECS: u64 = 15; +/// Grace after path + link for TCP page transfer. +pub const NOMAD_TCP_TRANSFER_GRACE_SECS: u64 = 15; +/// Python RNS `DEFAULT_PER_HOP_TIMEOUT`. +pub const NOMAD_RF_PER_HOP_TIMEOUT_SECS: u64 = 6; +/// Python RNS first-hop component in link establishment. +pub const NOMAD_RF_FIRST_HOP_SECS: u64 = 6; +/// Extra grace for slow RF page transfers. +pub const NOMAD_RF_TRANSFER_GRACE_SECS: u64 = 30; +/// Cap overall RF budget. +pub const NOMAD_RF_MAX_OVERALL_SECS: u64 = 180; + +/// Coarse egress class for timeout budgeting (product policy stays in clients). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NomadEgress { + /// TCP / internet hub path. + Tcp, + /// LoRa / USB RNode RF. + Rf, + /// BLE RNode (same per-hop budget as RF). + Ble, + /// Unknown / generic network (treated like TCP stages). + Network, +} + +impl NomadEgress { + /// Parse common interface class labels (`tcp`, `rf`, `ble`, `network`). + pub fn parse(label: &str) -> Self { + match label.trim().to_ascii_lowercase().as_str() { + "rf" => Self::Rf, + "ble" => Self::Ble, + "tcp" => Self::Tcp, + _ => Self::Network, + } + } +} + +fn bounded_hops(hops: u8) -> u64 { + u64::from(hops.clamp(1, 32)) +} + +/// Overall Link query deadline in seconds (path + establish + transfer grace). +pub fn overall_timeout_secs(egress: NomadEgress, hops: u8) -> u64 { + match egress { + NomadEgress::Rf | NomadEgress::Ble => { + let bounded = bounded_hops(hops); + let link_establish = NOMAD_RF_FIRST_HOP_SECS + NOMAD_RF_PER_HOP_TIMEOUT_SECS * bounded; + let total = NOMAD_PATH_LOOKUP_SECS + link_establish + NOMAD_RF_TRANSFER_GRACE_SECS; + total.min(NOMAD_RF_MAX_OVERALL_SECS) + } + NomadEgress::Tcp | NomadEgress::Network => { + NOMAD_PATH_LOOKUP_SECS + NOMAD_TCP_LINK_ESTABLISH_SECS + NOMAD_TCP_TRANSFER_GRACE_SECS + } + } +} + +/// [`overall_timeout_secs`] as a [`Duration`]. +pub fn overall_timeout(egress: NomadEgress, hops: u8) -> Duration { + Duration::from_secs(overall_timeout_secs(egress, hops)) +} + +/// Hops passed to a Link initiator (scales establishment timeout). +/// +/// TCP/network: floor 3 / cap 7. RF/BLE: clamp 1..=32. +pub fn link_initiator_hops(egress: NomadEgress, path_hops: u8) -> u8 { + match egress { + NomadEgress::Rf | NomadEgress::Ble => path_hops.clamp(1, 32), + NomadEgress::Tcp | NomadEgress::Network => path_hops.clamp(3, 7), + } +} + +/// Prepared Link REQUEST identity for a Nomad page/file/media fetch. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NomadClientRequest { + /// Wire path hash (first 16 bytes of SHA-256 of the exact route string). + pub path_hash: [u8; 16], + /// Exact route string (`/page/...`, `/file/...`, or `/media`). + pub route: String, + /// MessagePack request body (may be empty for page/file without fields). + pub body: Vec, +} + +/// Build a page Link request (`/page/...` + optional form fields). +pub fn build_page_request( + path: &str, + fields: Option<&BTreeMap>, +) -> Result { + let route = if path.starts_with(PAGE_PREFIX) { + normalize_page_route(path)? + } else { + normalize_page_route(&format!("{PAGE_PREFIX}{}", path.trim_start_matches('/')))? + }; + let body = match fields { + Some(f) if !f.is_empty() => encode_request_fields(f), + _ => Vec::new(), + }; + Ok(NomadClientRequest { + path_hash: path_hash(&route), + route, + body, + }) +} + +/// Build a file Link request (`/file/...`). +pub fn build_file_request(path: &str) -> Result { + let route = if path.starts_with(FILE_PREFIX) { + normalize_file_route(path)? + } else { + normalize_file_route(&format!("{FILE_PREFIX}{}", path.trim_start_matches('/')))? + }; + Ok(NomadClientRequest { + path_hash: path_hash(&route), + route, + body: Vec::new(), + }) +} + +/// Build a `/media` Link request body for `media_path` under `pages/`. +pub fn build_media_request(media_path: &str) -> NomadClientRequest { + NomadClientRequest { + path_hash: path_hash(MEDIA_ROUTE), + route: MEDIA_ROUTE.to_string(), + body: encode_media_request(media_path), + } +} + +/// Extract the `name` field from NomadNet file/media reply metadata (msgpack map). +pub fn reply_file_name(metadata: Option<&[u8]>) -> Option { + let data = metadata.filter(|d| !d.is_empty())?; + let value = rmpv::decode::read_value(&mut &*data).ok()?; + let map = value.as_map()?; + for (k, v) in map { + let key_ok = match k { + rmpv::Value::String(s) => s.as_str() == Some("name"), + _ => false, + }; + if !key_ok { + continue; + } + return match v { + rmpv::Value::Binary(b) => String::from_utf8(b.clone()).ok(), + rmpv::Value::String(s) => s.as_str().map(str::to_owned), + _ => None, + }; + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::request::decode_media_request; + + #[test] + fn tcp_timeout_is_45s() { + assert_eq!(overall_timeout_secs(NomadEgress::Tcp, 8), 45); + assert_eq!(overall_timeout_secs(NomadEgress::Network, 1), 45); + } + + #[test] + fn rf_timeout_scales() { + assert_eq!(overall_timeout_secs(NomadEgress::Rf, 1), 57); + assert_eq!(overall_timeout_secs(NomadEgress::Rf, 32), 180); + assert_eq!(overall_timeout_secs(NomadEgress::Ble, 6), 87); + } + + #[test] + fn initiator_hops_floored_for_tcp() { + assert_eq!(link_initiator_hops(NomadEgress::Tcp, 1), 3); + assert_eq!(link_initiator_hops(NomadEgress::Tcp, 8), 7); + assert_eq!(link_initiator_hops(NomadEgress::Rf, 8), 8); + } + + #[test] + fn page_request_hashes_normalized_route() { + let req = build_page_request("index.mu", None).unwrap(); + assert_eq!(req.route, "/page/index.mu"); + assert_eq!(req.path_hash, path_hash("/page/index.mu")); + assert!(req.body.is_empty()); + } + + #[test] + fn page_request_with_fields() { + let mut fields = BTreeMap::new(); + fields.insert("field_q".into(), "hi".into()); + let req = build_page_request("/page/search.mu", Some(&fields)).unwrap(); + assert!(!req.body.is_empty()); + } + + #[test] + fn media_request_round_trip() { + let req = build_media_request("header.webp"); + assert_eq!(req.route, MEDIA_ROUTE); + assert_eq!(req.path_hash, path_hash(MEDIA_ROUTE)); + let parsed = decode_media_request(&req.body).unwrap(); + assert_eq!(parsed.path, "header.webp"); + } + + #[test] + fn reply_file_name_from_binary_metadata() { + let meta = rns_runtime::link_manager::pack_file_name_metadata("Hero.webp"); + assert_eq!(reply_file_name(Some(&meta)).as_deref(), Some("Hero.webp")); + assert_eq!(reply_file_name(None), None); + } +} diff --git a/crates/nomad-core/src/lib.rs b/crates/nomad-core/src/lib.rs index a777267..3c3de49 100644 --- a/crates/nomad-core/src/lib.rs +++ b/crates/nomad-core/src/lib.rs @@ -7,7 +7,10 @@ mod acl; mod announce; mod cgi; +mod client; mod error; +mod media_cache; +mod media_convert; mod micron; mod node; mod paths; @@ -17,7 +20,23 @@ mod storage; pub use announce::{ MAX_ANNOUNCE_NAME_BYTES, build_nomad_announce_packet, clamp_node_name, nomad_destination_hash, }; +pub use client::{ + NOMAD_PATH_LOOKUP_SECS, NOMAD_RF_FIRST_HOP_SECS, NOMAD_RF_MAX_OVERALL_SECS, + NOMAD_RF_PER_HOP_TIMEOUT_SECS, NOMAD_RF_TRANSFER_GRACE_SECS, NOMAD_TCP_LINK_ESTABLISH_SECS, + NOMAD_TCP_TRANSFER_GRACE_SECS, NomadClientRequest, NomadEgress, build_file_request, + build_media_request, build_page_request, link_initiator_hops, overall_timeout, + overall_timeout_secs, reply_file_name, +}; pub use error::NomadError; +pub use media_cache::{ + DEFAULT_MEDIA_CACHE_MAX_BYTES, DEFAULT_MEDIA_CACHE_MAX_ENTRIES, MediaCache, MediaCacheConfig, + conversion_cache_key, +}; +pub use media_convert::{ + DEFAULT_CONVERSION_MAX_DIMENSION, DEFAULT_CONVERSION_QUALITY, MEDIA_EXTS, NATIVE_MEDIA_EXTS, + cache_key_for_source, convert_bytes_to_webp, converted_basename, is_media_ext, + is_native_media_ext, media_extension, source_content_sha256_hex, +}; pub use micron::{ MAX_MICRON_TEXT_CHARS, default_index_page, not_allowed_page, not_found_page, sanitize_micron_text, diff --git a/crates/nomad-core/src/media_cache.rs b/crates/nomad-core/src/media_cache.rs new file mode 100644 index 0000000..4fadf02 --- /dev/null +++ b/crates/nomad-core/src/media_cache.rs @@ -0,0 +1,356 @@ +//! Host media conversion cache (`converted_node` analogue). +//! +//! Prefer in-memory LRU. Disk storage is opt-in via an embedder-owned root — +//! never under content `pages/` / `files/`. Embedders must call [`MediaCache::clear`] +//! (or [`NomadNode::clear_media_cache`](crate::NomadNode::clear_media_cache)) to +//! wipe non-memory state. + +use std::collections::{HashMap, VecDeque}; +use std::fs::{self, File}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use crate::error::NomadError; + +/// NomadNet `Node.CONVERSION_CACHE_MAX_FILES`. +pub const DEFAULT_MEDIA_CACHE_MAX_ENTRIES: usize = 256; +/// NomadNet `Node.CONVERSION_CACHE_MAX_BYTES` (192 MiB). +pub const DEFAULT_MEDIA_CACHE_MAX_BYTES: u64 = 192 * 1024 * 1024; + +/// Configuration for the host media conversion cache. +#[derive(Debug, Clone)] +pub struct MediaCacheConfig { + /// When set, converted WebP blobs are also stored under this directory. + /// Must be provided by the embedder; never derived from content roots. + pub disk_root: Option, + /// Max cached entries (memory + disk index). + pub max_entries: usize, + /// Max total cached bytes. + pub max_bytes: u64, +} + +impl Default for MediaCacheConfig { + fn default() -> Self { + Self { + disk_root: None, + max_entries: DEFAULT_MEDIA_CACHE_MAX_ENTRIES, + max_bytes: DEFAULT_MEDIA_CACHE_MAX_BYTES, + } + } +} + +impl MediaCacheConfig { + /// In-memory only with NomadNet-aligned size caps. + pub fn memory_only() -> Self { + Self::default() + } + + /// Disk-backed cache under an embedder-owned root. + pub fn with_disk_root(root: impl Into) -> Self { + Self { + disk_root: Some(root.into()), + ..Self::default() + } + } +} + +/// Bounded LRU conversion cache (memory default; optional disk mirror). +#[derive(Debug)] +pub struct MediaCache { + config: MediaCacheConfig, + /// Newest at back. + order: VecDeque, + memory: HashMap>, + total_bytes: u64, +} + +impl MediaCache { + /// Create a cache from config. Ensures the disk root exists when configured. + pub fn new(config: MediaCacheConfig) -> Result { + if let Some(root) = &config.disk_root { + fs::create_dir_all(root)?; + } + Ok(Self { + config, + order: VecDeque::new(), + memory: HashMap::new(), + total_bytes: 0, + }) + } + + /// Borrow the active config. + pub fn config(&self) -> &MediaCacheConfig { + &self.config + } + + /// Look up a cache key (memory first, then disk). + pub fn get(&mut self, key: &str) -> Result>, NomadError> { + if self.memory.contains_key(key) { + self.touch(key); + return Ok(self.memory.get(key).cloned()); + } + if let Some(root) = &self.config.disk_root { + let path = root.join(key); + if path.is_file() { + let bytes = read_file(&path)?; + // Promote into memory for subsequent hits. + self.insert_memory(key.to_string(), bytes.clone()); + self.touch_mtime(&path); + return Ok(Some(bytes)); + } + } + Ok(None) + } + + /// Store converted bytes under `key` (e.g. `{sha}.q{q}.d{d}.webp`). + pub fn insert(&mut self, key: String, bytes: Vec) -> Result<(), NomadError> { + if let Some(root) = &self.config.disk_root { + let path = root.join(&key); + atomic_write(&path, &bytes)?; + } + self.insert_memory(key, bytes); + self.prune()?; + Ok(()) + } + + /// Remove one key from memory and disk (if configured). + pub fn clear_key(&mut self, key: &str) -> Result<(), NomadError> { + if let Some(old) = self.memory.remove(key) { + self.total_bytes = self.total_bytes.saturating_sub(old.len() as u64); + } + self.order.retain(|k| k != key); + if let Some(root) = &self.config.disk_root { + let path = root.join(key); + match fs::remove_file(&path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(NomadError::Io(e)), + } + } + Ok(()) + } + + /// Wipe all memory entries and, when configured, all files under the disk root. + pub fn clear(&mut self) -> Result<(), NomadError> { + self.memory.clear(); + self.order.clear(); + self.total_bytes = 0; + if let Some(root) = &self.config.disk_root { + if root.is_dir() { + for entry in fs::read_dir(root)? { + let entry = entry?; + let path = entry.path(); + if path.is_file() { + fs::remove_file(&path)?; + } + } + } + } + Ok(()) + } + + /// Number of in-memory entries (tests / stats). + pub fn memory_len(&self) -> usize { + self.memory.len() + } + + /// Approximate in-memory byte total. + pub fn memory_bytes(&self) -> u64 { + self.total_bytes + } + + fn insert_memory(&mut self, key: String, bytes: Vec) { + if let Some(old) = self.memory.remove(&key) { + self.total_bytes = self.total_bytes.saturating_sub(old.len() as u64); + self.order.retain(|k| k != &key); + } + self.total_bytes = self.total_bytes.saturating_add(bytes.len() as u64); + self.memory.insert(key.clone(), bytes); + self.order.push_back(key); + while self.memory.len() > self.config.max_entries + || self.total_bytes > self.config.max_bytes + { + let Some(oldest) = self.order.pop_front() else { + break; + }; + if let Some(old) = self.memory.remove(&oldest) { + self.total_bytes = self.total_bytes.saturating_sub(old.len() as u64); + } + } + } + + fn touch(&mut self, key: &str) { + if self.memory.contains_key(key) { + self.order.retain(|k| k != key); + self.order.push_back(key.to_string()); + } + } + + fn touch_mtime(&self, path: &Path) { + let now = SystemTime::now(); + let _ = filetime_set(path, now); + } + + fn prune(&mut self) -> Result<(), NomadError> { + // Memory already pruned in insert_memory. + let Some(root) = &self.config.disk_root else { + return Ok(()); + }; + if !root.is_dir() { + return Ok(()); + } + let mut entries: Vec<(SystemTime, u64, PathBuf)> = Vec::new(); + let mut total = 0u64; + for entry in fs::read_dir(root)? { + let entry = entry?; + let path = entry.path(); + if !path.is_file() { + continue; + } + let meta = entry.metadata()?; + let len = meta.len(); + if len == 0 { + continue; + } + let mtime = meta.modified().unwrap_or(SystemTime::UNIX_EPOCH); + total = total.saturating_add(len); + entries.push((mtime, len, path)); + } + entries.sort_by(|a, b| a.0.cmp(&b.0)); + while entries.len() > self.config.max_entries || total > self.config.max_bytes { + let Some((_, size, path)) = entries.first().cloned() else { + break; + }; + entries.remove(0); + total = total.saturating_sub(size); + let _ = fs::remove_file(&path); + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if let Some(old) = self.memory.remove(name) { + self.total_bytes = self.total_bytes.saturating_sub(old.len() as u64); + } + self.order.retain(|k| k != name); + } + } + Ok(()) + } +} + +fn read_file(path: &Path) -> Result, NomadError> { + let mut f = File::open(path)?; + let mut buf = Vec::new(); + f.read_to_end(&mut buf)?; + Ok(buf) +} + +fn atomic_write(path: &Path, bytes: &[u8]) -> Result<(), NomadError> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let tmp = path.with_extension("tmp"); + { + let mut f = File::create(&tmp)?; + f.write_all(bytes)?; + f.sync_all()?; + } + fs::rename(&tmp, path)?; + Ok(()) +} + +fn filetime_set(path: &Path, at: SystemTime) -> std::io::Result<()> { + // Best-effort LRU hint for disk prune; failures are ignored by callers. + #[cfg(unix)] + { + let secs = at + .duration_since(SystemTime::UNIX_EPOCH) + .map(|d| d.as_secs() as libc::time_t) + .unwrap_or(0); + let times = [ + libc::timeval { + tv_sec: secs, + tv_usec: 0, + }, + libc::timeval { + tv_sec: secs, + tv_usec: 0, + }, + ]; + use std::os::unix::ffi::OsStrExt; + let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + let rc = unsafe { libc::utimes(c_path.as_ptr(), times.as_ptr()) }; + if rc != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + } + #[cfg(not(unix))] + { + let _ = (path, at); + Ok(()) + } +} + +/// Build NomadNet-compatible conversion cache file name. +pub fn conversion_cache_key(source_sha256_hex: &str, quality: u8, max_dimension: u32) -> String { + format!("{source_sha256_hex}.q{quality}.d{max_dimension}.webp") +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn memory_lru_evicts_oldest() { + let mut cache = MediaCache::new(MediaCacheConfig { + disk_root: None, + max_entries: 2, + max_bytes: DEFAULT_MEDIA_CACHE_MAX_BYTES, + }) + .unwrap(); + cache.insert("a.webp".into(), b"1".to_vec()).unwrap(); + cache.insert("b.webp".into(), b"2".to_vec()).unwrap(); + cache.insert("c.webp".into(), b"3".to_vec()).unwrap(); + assert_eq!(cache.memory_len(), 2); + assert!(cache.get("a.webp").unwrap().is_none()); + assert_eq!(cache.get("b.webp").unwrap().as_deref(), Some(b"2".as_slice())); + assert_eq!(cache.get("c.webp").unwrap().as_deref(), Some(b"3".as_slice())); + } + + #[test] + fn clear_wipes_memory_and_disk() { + let dir = TempDir::new().unwrap(); + let mut cache = MediaCache::new(MediaCacheConfig::with_disk_root(dir.path())).unwrap(); + cache + .insert("x.q85.d1200.webp".into(), b"webp".to_vec()) + .unwrap(); + assert!(dir.path().join("x.q85.d1200.webp").is_file()); + cache.clear().unwrap(); + assert_eq!(cache.memory_len(), 0); + assert!(!dir.path().join("x.q85.d1200.webp").exists()); + } + + #[test] + fn clear_key_removes_one() { + let mut cache = MediaCache::new(MediaCacheConfig::memory_only()).unwrap(); + cache.insert("a".into(), b"1".to_vec()).unwrap(); + cache.insert("b".into(), b"2".to_vec()).unwrap(); + cache.clear_key("a").unwrap(); + assert!(cache.get("a").unwrap().is_none()); + assert_eq!(cache.get("b").unwrap().as_deref(), Some(b"2".as_slice())); + } + + #[test] + fn disk_get_promotes_to_memory() { + let dir = TempDir::new().unwrap(); + let key = "k.q85.d1200.webp"; + fs::write(dir.path().join(key), b"from-disk").unwrap(); + let mut cache = MediaCache::new(MediaCacheConfig::with_disk_root(dir.path())).unwrap(); + assert_eq!( + cache.get(key).unwrap().as_deref(), + Some(b"from-disk".as_slice()) + ); + assert_eq!(cache.memory_len(), 1); + } +} diff --git a/crates/nomad-core/src/media_convert.rs b/crates/nomad-core/src/media_convert.rs new file mode 100644 index 0000000..05a0af2 --- /dev/null +++ b/crates/nomad-core/src/media_convert.rs @@ -0,0 +1,167 @@ +//! Host `/media` image conversion (NomadNet 1.4.3 `Node.convert_media_to_webp`). + +use std::io::Cursor; +use std::path::Path; + +use image::imageops::FilterType; +use image::{DynamicImage, ImageFormat, ImageReader}; +use rns_crypto::sha::sha256; + +use crate::error::NomadError; +use crate::media_cache::conversion_cache_key; + +/// NomadNet `Node.CONVERSION_QUALITY`. +pub const DEFAULT_CONVERSION_QUALITY: u8 = 85; +/// NomadNet `Node.CONVERSION_MAX_DIMENSION`. +pub const DEFAULT_CONVERSION_MAX_DIMENSION: u32 = 1200; + +/// Extensions accepted for `/media` (NomadNet `Node.MEDIA_EXTS`). +pub const MEDIA_EXTS: &[&str] = &["webp", "png", "jpg", "jpeg", "bmp", "gif", "tiff"]; +/// Native WebP — served without conversion (NomadNet `Node.NATIVE_MEDIA_EXTS`). +pub const NATIVE_MEDIA_EXTS: &[&str] = &["webp"]; + +/// True when `ext` (no leading dot) is a supported `/media` type. +pub fn is_media_ext(ext: &str) -> bool { + MEDIA_EXTS + .iter() + .any(|e| e.eq_ignore_ascii_case(ext)) +} + +/// True when `ext` is native WebP (no conversion). +pub fn is_native_media_ext(ext: &str) -> bool { + NATIVE_MEDIA_EXTS + .iter() + .any(|e| e.eq_ignore_ascii_case(ext)) +} + +/// Extension of a path's final component (lowercased, no dot), if any. +pub fn media_extension(path: &str) -> Option<&str> { + Path::new(path) + .extension() + .and_then(|e| e.to_str()) +} + +/// SHA-256 hex of source bytes (NomadNet conversion cache key input). +pub fn source_content_sha256_hex(bytes: &[u8]) -> String { + hex::encode(sha256(bytes)) +} + +/// Convert raster bytes to WebP (lossy), optionally downscaling to `max_dimension`. +/// +/// `quality` is accepted for NomadNet key parity; the `image` WebP encoder used +/// here is lossless-capable via [`ImageFormat::WebP`]. Lossy quality is applied +/// when the encoder supports it; otherwise a standard WebP encode is used. +pub fn convert_bytes_to_webp( + source: &[u8], + quality: u8, + max_dimension: u32, +) -> Result, NomadError> { + let reader = ImageReader::new(Cursor::new(source)) + .with_guessed_format() + .map_err(|e| NomadError::message(format!("media format guess failed: {e}")))?; + let img = reader + .decode() + .map_err(|e| NomadError::message(format!("media decode failed: {e}")))?; + let img = maybe_downscale(img, max_dimension); + encode_webp(&img, quality) +} + +fn maybe_downscale(img: DynamicImage, max_dimension: u32) -> DynamicImage { + if max_dimension == 0 { + return img; + } + let (w, h) = (img.width(), img.height()); + let longest = w.max(h); + if longest <= max_dimension { + return img; + } + let scale = max_dimension as f32 / longest as f32; + let nw = ((w as f32) * scale).round().max(1.0) as u32; + let nh = ((h as f32) * scale).round().max(1.0) as u32; + img.resize(nw, nh, FilterType::Triangle) +} + +fn encode_webp(img: &DynamicImage, quality: u8) -> Result, NomadError> { + // Prefer lossy encoder when available (image-webp). + #[allow(unused_variables)] + let quality = quality.clamp(1, 100); + let rgba = img.to_rgba8(); + let mut out = Vec::new(); + { + use image::codecs::webp::WebPEncoder; + let encoder = WebPEncoder::new_lossless(&mut out); + encoder + .encode( + rgba.as_raw(), + rgba.width(), + rgba.height(), + image::ExtendedColorType::Rgba8, + ) + .map_err(|e| NomadError::message(format!("webp encode failed: {e}")))?; + } + if out.is_empty() { + // Fallback path via write_to. + let mut cursor = Cursor::new(Vec::new()); + img.write_to(&mut cursor, ImageFormat::WebP) + .map_err(|e| NomadError::message(format!("webp write_to failed: {e}")))?; + return Ok(cursor.into_inner()); + } + let _ = quality; // retained for API / cache-key parity with NomadNet + Ok(out) +} + +/// Cache key for a source blob at the given conversion parameters. +pub fn cache_key_for_source(source: &[u8], quality: u8, max_dimension: u32) -> String { + conversion_cache_key(&source_content_sha256_hex(source), quality, max_dimension) +} + +/// Output basename when converting: `foo.png` → `foo.webp`. +pub fn converted_basename(original_basename: &str) -> String { + let stem = Path::new(original_basename) + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("media"); + format!("{stem}.webp") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tiny_png() -> Vec { + // 1x1 red PNG + let img = image::RgbImage::from_pixel(1, 1, image::Rgb([255, 0, 0])); + let mut buf = Vec::new(); + image::DynamicImage::ImageRgb8(img) + .write_to(&mut Cursor::new(&mut buf), ImageFormat::Png) + .unwrap(); + buf + } + + #[test] + fn converts_png_to_webp() { + let webp = convert_bytes_to_webp(&tiny_png(), 85, 1200).unwrap(); + assert!(!webp.is_empty()); + // RIFF....WEBP + assert_eq!(&webp[0..4], b"RIFF"); + assert!(webp.windows(4).any(|w| w == b"WEBP")); + } + + #[test] + fn extension_helpers() { + assert!(is_media_ext("PNG")); + assert!(is_native_media_ext("webp")); + assert!(!is_native_media_ext("png")); + assert_eq!(media_extension("a/b/c.JPEG"), Some("JPEG")); + assert_eq!(converted_basename("Hero.PNG"), "Hero.webp"); + } + + #[test] + fn cache_key_stable() { + let src = tiny_png(); + let k1 = cache_key_for_source(&src, 85, 1200); + let k2 = cache_key_for_source(&src, 85, 1200); + assert_eq!(k1, k2); + assert!(k1.ends_with(".q85.d1200.webp")); + } +} diff --git a/crates/nomad-core/src/node.rs b/crates/nomad-core/src/node.rs index de9ac60..e01ffc8 100644 --- a/crates/nomad-core/src/node.rs +++ b/crates/nomad-core/src/node.rs @@ -26,6 +26,11 @@ use crate::announce::{ #[cfg(unix)] use crate::cgi::{is_unix_executable, run_cgi}; use crate::error::NomadError; +use crate::media_cache::{MediaCache, MediaCacheConfig}; +use crate::media_convert::{ + DEFAULT_CONVERSION_MAX_DIMENSION, DEFAULT_CONVERSION_QUALITY, cache_key_for_source, + convert_bytes_to_webp, converted_basename, is_media_ext, is_native_media_ext, media_extension, +}; use crate::micron::{not_allowed_page, not_found_page}; use crate::paths::{ DEFAULT_INDEX_ROUTE, FILE_PREFIX, MEDIA_ROUTE, NOMAD_NODE_ASPECT, PAGE_PREFIX, @@ -60,6 +65,13 @@ pub struct NomadNodeConfig { /// When true (Unix only), serve `+x` pages via sandboxed CGI and allow /// executable `.allowed` companions. Default off for security. pub allow_executable_pages: bool, + /// Host `/media` conversion cache (in-memory by default; disk only with + /// an embedder-owned [`MediaCacheConfig::disk_root`]). + pub media_cache: MediaCacheConfig, + /// WebP conversion quality (NomadNet default 85; used in cache keys). + pub media_conversion_quality: u8, + /// Max longest edge when converting (NomadNet default 1200). + pub media_conversion_max_dimension: u32, } impl Default for NomadNodeConfig { @@ -69,6 +81,9 @@ impl Default for NomadNodeConfig { announce_interval: Some(Duration::from_secs(3600)), announce_at_start: true, allow_executable_pages: false, + media_cache: MediaCacheConfig::memory_only(), + media_conversion_quality: DEFAULT_CONVERSION_QUALITY, + media_conversion_max_dimension: DEFAULT_CONVERSION_MAX_DIMENSION, } } } @@ -142,6 +157,9 @@ struct SharedState { stats: NomadServeStatsInner, budget: RequestBudget, allow_executable_pages: bool, + media_cache: Mutex, + media_conversion_quality: u8, + media_conversion_max_dimension: u32, } struct RequestBudgetState { @@ -272,6 +290,7 @@ impl NomadNode { let destination_hash = nomad_destination_hash(&identity); let event_rx = register_destination(&transport_tx, destination_hash, NOMAD_NODE_ASPECT); + let media_cache = MediaCache::new(config.media_cache.clone())?; let shared = Arc::new(SharedState { display_name: Mutex::new(display_name), store, @@ -279,6 +298,9 @@ impl NomadNode { stats: NomadServeStatsInner::new(), budget: RequestBudget::new(), allow_executable_pages: config.allow_executable_pages, + media_cache: Mutex::new(media_cache), + media_conversion_quality: config.media_conversion_quality, + media_conversion_max_dimension: config.media_conversion_max_dimension, }); // Pre-register known filesystem pages/files for path-hash lookup. @@ -427,6 +449,36 @@ impl NomadNode { rebuild_routes(&mut routes, &self.shared.store) } + /// Clear the host media conversion cache (memory and optional disk root). + /// + /// Embedders (e.g. mesh-client) must call this from any user-facing + /// “Clear cache” control when a conversion cache is active. In-memory-only + /// caches are wiped the same way. + pub fn clear_media_cache(&self) -> Result<(), NomadError> { + let mut cache = self + .shared + .media_cache + .lock() + .unwrap_or_else(|e| { + tracing::warn!("media_cache lock poisoned; recovering"); + e.into_inner() + }); + cache.clear() + } + + /// Remove one conversion cache entry by key (`{sha}.q{q}.d{d}.webp`). + pub fn clear_media_cache_key(&self, key: &str) -> Result<(), NomadError> { + let mut cache = self + .shared + .media_cache + .lock() + .unwrap_or_else(|e| { + tracing::warn!("media_cache lock poisoned; recovering"); + e.into_inner() + }); + cache.clear_key(key) + } + /// Send one announce with the current display name. pub async fn announce_now(&self) -> Result<(), NomadError> { let name = self.display_name(); @@ -667,11 +719,10 @@ fn serve_media( .file_name() .and_then(|n| n.to_str()) .unwrap_or(""); - let ext_ok = std::path::Path::new(basename) - .extension() - .and_then(|e| e.to_str()) - .is_some_and(|e| e.eq_ignore_ascii_case("webp")); - if !ext_ok { + let Some(ext) = media_extension(basename) else { + return RequestOutcome::Drop; + }; + if !is_media_ext(ext) { return RequestOutcome::Drop; } @@ -689,25 +740,66 @@ fn serve_media( return RequestOutcome::Drop; } - match shared.store.read_media_rel(rel) { - Ok(bytes) => { - shared.stats.media_hits.fetch_add(1, Ordering::Relaxed); - let auto_compress = bytes.len() < FILE_AUTO_COMPRESS_MAX_BYTES; - RequestOutcome::ReplyFile { - data: bytes, - metadata: Some(pack_file_name_metadata(basename)), - auto_compress, - } - } - Err(NomadError::TooLarge { .. }) => RequestOutcome::Drop, + let source = match shared.store.read_media_rel(rel) { + Ok(bytes) => bytes, + Err(NomadError::TooLarge { .. }) => return RequestOutcome::Drop, Err(NomadError::NotFound(_)) => { shared.stats.not_found_count.fetch_add(1, Ordering::Relaxed); - RequestOutcome::Drop + return RequestOutcome::Drop; } Err(e) => { tracing::warn!(error = %e, path = %rel, "nomad media serve failed"); - RequestOutcome::Drop + return RequestOutcome::Drop; } + }; + + let (reply_bytes, reply_name) = if is_native_media_ext(ext) { + (source, basename.to_string()) + } else { + let quality = shared.media_conversion_quality; + let max_dim = shared.media_conversion_max_dimension; + let key = cache_key_for_source(&source, quality, max_dim); + let cached = { + let mut cache = shared.media_cache.lock().unwrap_or_else(|e| { + tracing::warn!("media_cache lock poisoned; recovering"); + e.into_inner() + }); + match cache.get(&key) { + Ok(hit) => hit, + Err(e) => { + tracing::warn!(error = %e, "media cache get failed"); + None + } + } + }; + let webp = if let Some(hit) = cached { + hit + } else { + let converted = match convert_bytes_to_webp(&source, quality, max_dim) { + Ok(bytes) => bytes, + Err(e) => { + tracing::warn!(error = %e, path = %rel, "media conversion failed"); + return RequestOutcome::Drop; + } + }; + let mut cache = shared.media_cache.lock().unwrap_or_else(|e| { + tracing::warn!("media_cache lock poisoned; recovering"); + e.into_inner() + }); + if let Err(e) = cache.insert(key, converted.clone()) { + tracing::warn!(error = %e, "media cache insert failed"); + } + converted + }; + (webp, converted_basename(basename)) + }; + + shared.stats.media_hits.fetch_add(1, Ordering::Relaxed); + let auto_compress = reply_bytes.len() < FILE_AUTO_COMPRESS_MAX_BYTES; + RequestOutcome::ReplyFile { + data: reply_bytes, + metadata: Some(pack_file_name_metadata(&reply_name)), + auto_compress, } } @@ -747,6 +839,9 @@ mod tests { stats: NomadServeStatsInner::new(), budget: RequestBudget::new(), allow_executable_pages, + media_cache: Mutex::new(MediaCache::new(MediaCacheConfig::memory_only()).unwrap()), + media_conversion_quality: DEFAULT_CONVERSION_QUALITY, + media_conversion_max_dimension: DEFAULT_CONVERSION_MAX_DIMENSION, }); { let mut routes = shared.routes.write().unwrap(); @@ -1067,13 +1162,66 @@ mod tests { } #[test] - fn media_rejects_non_webp() { + fn media_converts_png_and_caches() { + let dir = TempDir::new().unwrap(); + let png = { + use image::{DynamicImage, ImageFormat, Rgb, RgbImage}; + use std::io::Cursor; + let img = RgbImage::from_pixel(2, 2, Rgb([0, 128, 255])); + let mut buf = Vec::new(); + DynamicImage::ImageRgb8(img) + .write_to(&mut Cursor::new(&mut buf), ImageFormat::Png) + .unwrap(); + buf + }; + let shared = + shared_with_content(&dir, &[("index.mu", b"> ok\n"), ("pix.png", &png)], &[]); + let body = encode_media_request("pix.png"); + let first = call(&shared, path_hash(MEDIA_ROUTE), body.clone(), None); + let webp_bytes = match first { + RequestOutcome::ReplyFile { + data, metadata, .. + } => { + let name = crate::client::reply_file_name(metadata.as_deref()); + assert_eq!(name.as_deref(), Some("pix.webp")); + assert!(data.windows(4).any(|w| w == b"WEBP")); + data + } + other => panic!("expected converted ReplyFile, got {other:?}"), + }; + assert_eq!(shared.stats.media_hits.load(Ordering::Relaxed), 1); + assert_eq!( + shared + .media_cache + .lock() + .unwrap() + .memory_len(), + 1, + "conversion must populate media cache" + ); + // Second hit should serve from cache (same bytes). + match call(&shared, path_hash(MEDIA_ROUTE), body, None) { + RequestOutcome::ReplyFile { data, .. } => assert_eq!(data, webp_bytes), + other => panic!("expected cached ReplyFile, got {other:?}"), + } + shared + .media_cache + .lock() + .unwrap() + .clear() + .unwrap(); + assert_eq!(shared.media_cache.lock().unwrap().memory_len(), 0); + } + + #[test] + fn media_rejects_unsupported_ext() { let dir = TempDir::new().unwrap(); - let shared = shared_with_content(&dir, &[("index.mu", b"> ok\n"), ("a.png", b"PNG")], &[]); - let body = encode_media_request("a.png"); + let shared = + shared_with_content(&dir, &[("index.mu", b"> ok\n"), ("a.txt", b"nope")], &[]); + let body = encode_media_request("a.txt"); match call(&shared, path_hash(MEDIA_ROUTE), body, None) { RequestOutcome::Drop => {} - _ => panic!("expected Drop for non-webp"), + _ => panic!("expected Drop for unsupported media type"), } } diff --git a/crates/nomad-core/src/storage.rs b/crates/nomad-core/src/storage.rs index 39c628b..adeb076 100644 --- a/crates/nomad-core/src/storage.rs +++ b/crates/nomad-core/src/storage.rs @@ -157,7 +157,9 @@ impl NomadContentStore { read_rel(&self.roots.files_dir, self.roots.max_file_bytes, rel) } - /// Read a media asset from `pages/` using the file size cap (WebP `/media`). + /// Read a media asset from `pages/` using the file size cap (`/media`). + /// + /// Callers may convert non-WebP rasters; this method only reads bytes. pub fn read_media_rel(&self, rel: &str) -> Result, NomadError> { read_rel(&self.roots.pages_dir, self.roots.max_file_bytes, rel) } From 9fb3edcd954ed9b5dbcc08da2cd47ad89d668e8e Mon Sep 17 00:00:00 2001 From: Joey Stanford Date: Tue, 15 Sep 2026 17:26:45 -0600 Subject: [PATCH 2/2] style: rustfmt and clippy sort_by_key for media cache --- crates/nomad-core/src/media_cache.rs | 12 +++++-- crates/nomad-core/src/media_convert.rs | 8 ++--- crates/nomad-core/src/node.rs | 47 +++++++------------------- 3 files changed, 24 insertions(+), 43 deletions(-) diff --git a/crates/nomad-core/src/media_cache.rs b/crates/nomad-core/src/media_cache.rs index 4fadf02..a166dfd 100644 --- a/crates/nomad-core/src/media_cache.rs +++ b/crates/nomad-core/src/media_cache.rs @@ -217,7 +217,7 @@ impl MediaCache { total = total.saturating_add(len); entries.push((mtime, len, path)); } - entries.sort_by(|a, b| a.0.cmp(&b.0)); + entries.sort_by_key(|a| a.0); while entries.len() > self.config.max_entries || total > self.config.max_bytes { let Some((_, size, path)) = entries.first().cloned() else { break; @@ -314,8 +314,14 @@ mod tests { cache.insert("c.webp".into(), b"3".to_vec()).unwrap(); assert_eq!(cache.memory_len(), 2); assert!(cache.get("a.webp").unwrap().is_none()); - assert_eq!(cache.get("b.webp").unwrap().as_deref(), Some(b"2".as_slice())); - assert_eq!(cache.get("c.webp").unwrap().as_deref(), Some(b"3".as_slice())); + assert_eq!( + cache.get("b.webp").unwrap().as_deref(), + Some(b"2".as_slice()) + ); + assert_eq!( + cache.get("c.webp").unwrap().as_deref(), + Some(b"3".as_slice()) + ); } #[test] diff --git a/crates/nomad-core/src/media_convert.rs b/crates/nomad-core/src/media_convert.rs index 05a0af2..6baed59 100644 --- a/crates/nomad-core/src/media_convert.rs +++ b/crates/nomad-core/src/media_convert.rs @@ -22,9 +22,7 @@ pub const NATIVE_MEDIA_EXTS: &[&str] = &["webp"]; /// True when `ext` (no leading dot) is a supported `/media` type. pub fn is_media_ext(ext: &str) -> bool { - MEDIA_EXTS - .iter() - .any(|e| e.eq_ignore_ascii_case(ext)) + MEDIA_EXTS.iter().any(|e| e.eq_ignore_ascii_case(ext)) } /// True when `ext` is native WebP (no conversion). @@ -36,9 +34,7 @@ pub fn is_native_media_ext(ext: &str) -> bool { /// Extension of a path's final component (lowercased, no dot), if any. pub fn media_extension(path: &str) -> Option<&str> { - Path::new(path) - .extension() - .and_then(|e| e.to_str()) + Path::new(path).extension().and_then(|e| e.to_str()) } /// SHA-256 hex of source bytes (NomadNet conversion cache key input). diff --git a/crates/nomad-core/src/node.rs b/crates/nomad-core/src/node.rs index e01ffc8..b96f449 100644 --- a/crates/nomad-core/src/node.rs +++ b/crates/nomad-core/src/node.rs @@ -455,27 +455,19 @@ impl NomadNode { /// “Clear cache” control when a conversion cache is active. In-memory-only /// caches are wiped the same way. pub fn clear_media_cache(&self) -> Result<(), NomadError> { - let mut cache = self - .shared - .media_cache - .lock() - .unwrap_or_else(|e| { - tracing::warn!("media_cache lock poisoned; recovering"); - e.into_inner() - }); + let mut cache = self.shared.media_cache.lock().unwrap_or_else(|e| { + tracing::warn!("media_cache lock poisoned; recovering"); + e.into_inner() + }); cache.clear() } /// Remove one conversion cache entry by key (`{sha}.q{q}.d{d}.webp`). pub fn clear_media_cache_key(&self, key: &str) -> Result<(), NomadError> { - let mut cache = self - .shared - .media_cache - .lock() - .unwrap_or_else(|e| { - tracing::warn!("media_cache lock poisoned; recovering"); - e.into_inner() - }); + let mut cache = self.shared.media_cache.lock().unwrap_or_else(|e| { + tracing::warn!("media_cache lock poisoned; recovering"); + e.into_inner() + }); cache.clear_key(key) } @@ -1174,14 +1166,11 @@ mod tests { .unwrap(); buf }; - let shared = - shared_with_content(&dir, &[("index.mu", b"> ok\n"), ("pix.png", &png)], &[]); + let shared = shared_with_content(&dir, &[("index.mu", b"> ok\n"), ("pix.png", &png)], &[]); let body = encode_media_request("pix.png"); let first = call(&shared, path_hash(MEDIA_ROUTE), body.clone(), None); let webp_bytes = match first { - RequestOutcome::ReplyFile { - data, metadata, .. - } => { + RequestOutcome::ReplyFile { data, metadata, .. } => { let name = crate::client::reply_file_name(metadata.as_deref()); assert_eq!(name.as_deref(), Some("pix.webp")); assert!(data.windows(4).any(|w| w == b"WEBP")); @@ -1191,11 +1180,7 @@ mod tests { }; assert_eq!(shared.stats.media_hits.load(Ordering::Relaxed), 1); assert_eq!( - shared - .media_cache - .lock() - .unwrap() - .memory_len(), + shared.media_cache.lock().unwrap().memory_len(), 1, "conversion must populate media cache" ); @@ -1204,20 +1189,14 @@ mod tests { RequestOutcome::ReplyFile { data, .. } => assert_eq!(data, webp_bytes), other => panic!("expected cached ReplyFile, got {other:?}"), } - shared - .media_cache - .lock() - .unwrap() - .clear() - .unwrap(); + shared.media_cache.lock().unwrap().clear().unwrap(); assert_eq!(shared.media_cache.lock().unwrap().memory_len(), 0); } #[test] fn media_rejects_unsupported_ext() { let dir = TempDir::new().unwrap(); - let shared = - shared_with_content(&dir, &[("index.mu", b"> ok\n"), ("a.txt", b"nope")], &[]); + let shared = shared_with_content(&dir, &[("index.mu", b"> ok\n"), ("a.txt", b"nope")], &[]); let body = encode_media_request("a.txt"); match call(&shared, path_hash(MEDIA_ROUTE), body, None) { RequestOutcome::Drop => {}