Add GetHostStorage collector and integrate with SwarmCmd handler pipeline

Cargo.toml

@@ -10,6 +10,8 @@ bitcoin = { version = "0.29.2", default-features = false, features = [
     "std",
 ] }
 bollard = "0.18"
+# safe statvfs wrapper for the /vol host-bind fallback (no raw libc FFI)
+nix = { version = "0.29", features = ["fs"] }
 futures-core = "0.3"
 futures-util = "0.3"
 serde = { version = "1.0", features = ["derive"] }

app/src/api/cmd.ts

@@ -58,6 +58,7 @@ export type Cmd =
   | "ListPays"
   | "ListPayments"
   | "GetStatistics"
+  | "GetHostStorage"
   | "ListPendingChannels"
   | "GetClients"
   | "AddBoltwallAdminPubkey"

app/src/api/swarm.ts

@@ -69,6 +69,10 @@ export async function get_container_stat(name?: string) {
   return await swarmCmd("GetStatistics", name);
 }
 
+export async function get_host_storage() {
+  return await swarmCmd("GetHostStorage");
+}
+
 export async function add_boltwall_admin_pubkey(pubkey: string, name?: string) {
   return await swarmCmd("AddBoltwallAdminPubkey", { pubkey, name });
 }

doc/HostStorage.md

@@ -0,0 +1,134 @@
+# Host Storage Telemetry — `GetHostStorage`
+
+Read-only host-level storage telemetry, exposed through the existing authenticated
+admin command endpoint. No new route, no new endpoint.
+
+```
+curl -H "x-jwt: $JWT" --get https://<swarm>/api/cmd \
+  --data-urlencode 'tag=SWARM' \
+  --data-urlencode 'txt={"cmd":"GetHostStorage"}'
+```
+
+(URL-encoding `txt` is required — the payload is JSON and `/api/cmd` reads it from
+the `txt` query parameter. Note the `--get` flag; the request must be a GET.)
+
+## Sources
+
+The command combines three independent collectors (each capped at **8s**, whole
+collection bounded by **15s** — well inside the 60s `REQUEST_TIMEOUT_DURATION_IN_SEC`):
+
+1. **`node_exporter`** (primary) — scrapes the co-located `node_exporter`
+   sidecar (`network_mode: host`, `--path.rootfs=/host`) at
+   `http://{docker-gateway}:9100/metrics`. The gateway is read from the
+   container's own `/proc/net/route`; overridable with `NODE_EXPORTER_URL`,
+   which is read **once at startup** and must be loopback or the resolved
+   docker gateway (anything else is rejected). Figures genuinely describe the
+   host filesystems.
+2. **`container_bind`** (fallback) — when the scrape fails, a `statvfs` on the
+   container's `/vol` path (a bind of the host's `/home/admin/vol`).
+3. **`none`** — neither is available. `filesystems: []` and
+   `host_visible: false`; volume and Neo4j data still come back.
+
+Docker volume sizes always come from the Docker daemon (`GET /system/df`), and
+the daemon's storage root from `GET /info`.
+
+## Response shape (v1)
+
+```json
+{ "host_visible": true,
+  "source": "node_exporter",
+  "collected_at": 1730000000,
+  "cached": false,
+  "filesystems": [
+    {"mount": "/", "device": "/dev/nvme0n1p1", "fstype": "ext4",
+     "total_bytes": 0, "used_bytes": 0, "free_bytes": 0, "describes_host": true}
+  ],
+  "docker_root_dir": "/var/lib/docker",
+  "docker_root_filesystem": "/",
+  "volumes": [{"name": "neo4j.sphinx", "size_bytes": 0, "size_known": true}],
+  "neo4j": {"volumes": ["neo4j.sphinx"], "size_bytes": 0, "size_known": true},
+  "errors": [{"collector": "volumes", "reason": "docker df timed out after 8s"}] }
+```
+
+### Field semantics
+
+- **`host_visible`** is a testable predicate: `true` **iff** at least one entry
+  in `filesystems[]` has `describes_host: true`. It is never derived from "a
+  mount path exists". An entry is `describes_host: true` only when its figures
+  were read from a rootfs-scoped `node_exporter` (`source: "node_exporter"`) or
+  from a `statvfs` on a path that is a bind of a host directory
+  (`source: "container_bind"`). The container's own overlay `/` is **never**
+  reported as host data.
+- **`source`** is one of `"node_exporter" | "container_bind" | "none"`.
+- **`volumes[].size_bytes` is nullable.** Docker returns `-1` (or omits
+  `usage_data`) for volumes it did not compute; that maps to
+  `size_bytes: null, size_known: false` plus an `errors[]` entry — never a
+  fabricated `0`.
+- **`neo4j` is `null`** when no Neo4j node exists in the stack (a valid,
+  non-error response). When present it lists *all* named volumes attributed to
+  the Neo4j node and sums them. In this repo Neo4j mounts exactly one named
+  volume (`<hostname>.sphinx` at `/data`); the plugins and `apoc.conf` are
+  copied into the container filesystem (`docker cp`), not volumes. **Host-path
+  bind mounts are excluded** from the attribution. `neo4j.size_bytes` is
+  derived by lookup into the same map that produced `volumes[]`, so the two can
+  never disagree; it is summed only when every volume's size is known.
+- **`docker_root_dir` / `docker_root_filesystem`** come from `Docker::info()` and
+  a longest-prefix match against `filesystems[].mount`, so an operator can tell
+  which reported `free_bytes` actually governs the Neo4j volume — the daemon's
+  storage root is frequently a separate device from `/`.
+- **`errors[]` elements are objects**: `{"collector": "filesystems" | "volumes"
+  | "neo4j" | "docker_info", "reason": "<string>"}`.
+- **Partial failures return 200** with a fully-formed object; collector failures
+  are folded into `errors[]` and never fail the request.
+- **`used_bytes` / `free_bytes`**: `used = total - free` (free, not available,
+  so root-reserved space counts as used); `free_bytes` is the *available* figure
+  (`node_filesystem_avail_bytes` / statvfs `bavail`) — what a non-root writer
+  would actually get.
+- `filesystems[]` excludes virtual filesystems (`tmpfs`, `overlay`, `squashfs`,
+  `devtmpfs`) and node_exporter's own rootfs-view mountpoints under
+  `/host/proc`, `/host/sys`, `/host/run`.
+
+## Timing & caching
+
+- **Collection budget: 15s** (each collector individually capped at 8s;
+  collectors run concurrently).
+- **Cache TTL: 60s.** Within the TTL the last result is served as-is with
+  `cached: true` and the **original** `collected_at` — use `collected_at` to
+  detect staleness. Concurrent calls single-flight into one collection rather
+  than stacking volume scans on a host that may already be near disk
+  exhaustion. Cache hits log nothing.
+
+## Compatibility policy
+
+**Additive-only.** New fields may appear; existing field names, types and
+nullability do not change within this major version. Any breaking change ships
+as a future `GetHostStorageV2`.
+
+## Errors
+
+A collector failure is *not* a request failure. Transport-level failures (bad
+JWT, malformed payload, outer timeout, access denied) surface as
+`{"stack_error": "..."}` — handle both shapes. Collector-level failures live in
+`errors[]`.
+
+## Access control
+
+Behind the existing `x-jwt` admin guard (`auth::AdminJwtClaims`). Admins,
+sub-admins and super-admins may call it; unauthenticated callers are denied.
+Host-capacity details are deliberately visible to sub-admins (they already
+read container listings and API tokens).
+
+## Related
+
+`SwarmCmd::GetEc2CpuUtilization` (super-admin binary) reads EC2 CPU utilization
+from CloudWatch — EC2-only and super-admin-only. Disk telemetry lives on the
+per-swarm command surface instead because it must work on **any** deployment
+and be readable with the swarm's **own** admin credential.
+
+## Deployment notes
+
+`node_exporter` is defined in `sphinx.yml`, `second-brain.yml`,
+`second-brain-2.yml`, `docker-compose.yml` and `sphinxv2.yml`. Swarms deployed
+from composes without it still work via the `/vol` fallback (or report
+`host_visible: false` with an explanatory `errors[]` entry). `superadmin.yml`
+and `config.yml` hosts are not swarm data hosts and are out of scope.

docker-compose.yml

@@ -91,6 +91,17 @@ services:
       - AWS_S3_BUCKET_NAME=$AWS_S3_BUCKET_NAME
       - HF_TOKEN=$HF_TOKEN # repo2graph GAIA dataset bootstrap (gated HF repo)
 
+  node_exporter:
+    image: quay.io/prometheus/node-exporter:latest
+    container_name: node_exporter
+    command:
+      - '--path.rootfs=/host'
+    network_mode: host
+    pid: host
+    restart: unless-stopped
+    volumes:
+      - '/:/host:ro,rslave'
+
 networks:
   sphinx-swarm:
     external: true

sphinxv2.yml

@@ -80,6 +80,17 @@ services:
       - VECTOR_FORWARD_URL=$VECTOR_FORWARD_URL
       - VECTOR_FORWARD_TOKEN=$VECTOR_FORWARD_TOKEN
 
+  node_exporter:
+    image: quay.io/prometheus/node-exporter:latest
+    container_name: node_exporter
+    command:
+      - '--path.rootfs=/host'
+    network_mode: host
+    pid: host
+    restart: unless-stopped
+    volumes:
+      - '/:/host:ro,rslave'
+
 networks:
   sphinx-swarm:
     external: true

src/bin/stack/mod.rs

@@ -16,6 +16,10 @@ use tokio::sync::Mutex;
 
 #[rocket::main]
 async fn main() -> Result<()> {
+    // Pin the node_exporter scrape target once at startup (SSRF guard: later
+    // runtime env writes can never repoint the GetHostStorage collector).
+    sphinx_swarm::host_stats::init_node_exporter_target();
+
     dotenv::dotenv().ok();
 
     let docker = dockr();

src/cmd.rs

@@ -208,6 +208,7 @@ pub enum SwarmCmd {
     RestartContainer(String),
     UpdateNode(UpdateNode),
     GetStatistics(Option<String>),
+    GetHostStorage,
     AddBoltwallAdminPubkey(AddAdminRequest),
     GetBoltwallSuperAdmin,
     AddBoltwallUser(AddUserRequest),
@@ -410,12 +411,32 @@ pub enum HsmdCmd {
     GetClients,
 }
 
+#[cfg(test)]
+mod ready_tests {
+    use super::*;
+
+    #[test]
+    fn get_host_storage_can_run_before_ready() {
+        let cmd = Cmd::Swarm(SwarmCmd::GetHostStorage);
+        assert!(cmd.can_run_before_ready());
+        // existing allowlist unaffected
+        assert!(Cmd::Swarm(SwarmCmd::GetConfig).can_run_before_ready());
+        assert!(Cmd::Swarm(SwarmCmd::Login(LoginInfo {
+            username: "u".into(),
+            password: "p".into(),
+        }))
+        .can_run_before_ready());
+        assert!(!Cmd::Swarm(SwarmCmd::ListContainers).can_run_before_ready());
+    }
+}
+
 impl Cmd {
     pub fn can_run_before_ready(&self) -> bool {
         match self {
             Cmd::Swarm(c) => match c {
                 SwarmCmd::GetConfig => true,
                 SwarmCmd::Login(_) => true,
+                SwarmCmd::GetHostStorage => true,
                 _ => false,
             },
             _ => false,

src/handler.rs

@@ -36,6 +36,60 @@ use rocket::tokio::time::Duration;
 use serde::{Deserialize, Serialize};
 use tokio::sync::mpsc;
 
+#[cfg(test)]
+mod access_tests {
+    use super::*;
+
+    fn stack_with_role(role: Role) -> Stack {
+        Stack {
+            network: "regtest".to_string(),
+            nodes: vec![],
+            host: None,
+            users: vec![User {
+                id: 1,
+                username: "u".to_string(),
+                pass_hash: "x".to_string(),
+                pubkey: None,
+                role,
+            }],
+            jwt_key: "k".to_string(),
+            ready: true,
+            ip: None,
+            auto_update: None,
+            auto_restart: None,
+            custom_2b_domain: None,
+            global_mem_limit: None,
+            backup_services: None,
+            backup_files: None,
+            lightning_peers: None,
+            ssl_cert_last_modified: None,
+            instance_id: None,
+        }
+    }
+
+    #[test]
+    fn get_host_storage_allowed_for_all_roles() {
+        for role in [Role::Admin, Role::SubAdmin, Role::Super] {
+            let stack = stack_with_role(role.clone());
+            let cmd = Cmd::Swarm(SwarmCmd::GetHostStorage);
+            assert!(
+                access(&cmd, &stack, &Some(1)),
+                "GetHostStorage must be allowed for {:?} (sub-admin visibility is a documented accept)",
+                role
+            );
+        }
+    }
+
+    #[test]
+    fn get_host_storage_requires_authenticated_user() {
+        let stack = stack_with_role(Role::Admin);
+        let cmd = Cmd::Swarm(SwarmCmd::GetHostStorage);
+        assert!(!access(&cmd, &stack, &None), "no user_id => denied");
+        let stack_unknown = stack_with_role(Role::Super);
+        assert!(!access(&cmd, &stack_unknown, &Some(999)), "unknown user id => denied");
+    }
+}
+
 fn access(cmd: &Cmd, stack: &Stack, user_id: &Option<u32>) -> bool {
     // login needs no auth
     if let Cmd::Swarm(c) = cmd {
@@ -69,6 +123,7 @@ fn access(cmd: &Cmd, stack: &Stack, user_id: &Option<u32>) -> bool {
                 SwarmCmd::ChangePassword(_) => true,
                 SwarmCmd::ChangeUserPasswordBySuperAdmin(_) => true,
                 SwarmCmd::GetApiToken => true,
+                SwarmCmd::GetHostStorage => true,
                 SwarmCmd::ChangeReservedSwarmToActive(_) => true,
                 SwarmCmd::UpdateEvn(_) => true,
                 SwarmCmd::GetEnv(_) => true,
@@ -292,6 +347,24 @@ pub async fn handle(
                 println!("GetStatistics Called");
                 Some(serde_json::to_string(&containers)?)
             }
+            SwarmCmd::GetHostStorage => {
+                let started = std::time::Instant::now();
+                let nodes = config::stack_read(|s| s.nodes.clone()).await;
+                let result = crate::host_stats::get_host_storage(docker, &nodes).await;
+                let root_free = result.docker_root_filesystem.as_deref().and_then(|root| {
+                    result.filesystems.iter().find(|f| f.mount == root).map(|f| f.free_bytes)
+                });
+                log::info!(
+                    "GetHostStorage source={} host_visible={} root_free_bytes={:?} neo4j_bytes={:?} errors={} elapsed_ms={}",
+                    result.source,
+                    result.host_visible,
+                    root_free,
+                    result.neo4j.as_ref().and_then(|n| n.size_bytes),
+                    result.errors.len(),
+                    started.elapsed().as_millis()
+                );
+                Some(serde_json::to_string(&result)?)
+            }
             SwarmCmd::AddBoltwallAdminPubkey(admin) => {
                 log::info!("AddBoltwallAdminPubkey ->pubkey {}, name {:?}", admin.pubkey, admin.name);
                 let boltwall = config::stack_read(|s| find_boltwall(&s.nodes)).await?;