test: make the Docker harness work on a NixOS host

AGENTS.md

@@ -15,12 +15,28 @@ Personal homelab utility tools. Rust workspace, edition **2024**.
 - **`rustfmt` requires nightly** (`+nightly`). `rustfmt.toml` uses unstable features (`imports_granularity = "Item"`, `group_imports = "StdExternalCrate"`).
 - **`.cargo/config.toml` always enables `docker-tests`** via `--cfg feature="docker-tests"`. So `--all-features` in dev commands is redundant but harmless.
 
+### Build environment (NixOS host)
+
+`openssl-sys` cannot find OpenSSL here, so **every** cargo command needs
+these exported first, in the same shell as the cargo call:
+
+```bash
+export OPENSSL_LIB_DIR=/nix/store/7fr737xfi9qw3fzvdsqmnbqid56knndp-openssl-3.6.4/lib
+export OPENSSL_INCLUDE_DIR=/nix/store/0la6k2nj90y1716c1znhdm713ia1qgx8-openssl-3.6.4-dev/include
+```
+
+- `OPENSSL_DIR` alone is **not** enough: the `-dev` store path has the headers, the other has the `.so` files, and `openssl-sys` rejects a libdir without them.
+- `cargo test -p lab-ops_auto-discover` additionally needs
+  `LD_LIBRARY_PATH=/nix/store/7fr737xfi9qw3fzvdsqmnbqid56knndp-openssl-3.6.4/lib`
+  or the test binary dies with `libssl.so.3: cannot open shared object file`. The other three crates do not need it.
+- Do not "fix" this in `Cargo.toml` — it is the environment, not the code.
+
 ## Test Strategy ⚠️
 
-- **Run ONLY relevant tests first.** After a change, run the specific test/crate, not the full suite. E.g. `cargo test -p natmap`, `cargo test -p auto-discover`.
+- **Run targeted tests first.** After a change, run the specific test/crate, not the whole suite — it is faster and most changes do not touch the Docker paths. E.g. `cargo test -p lab-ops_natmap`, `cargo test -p lab-ops_auto-discover`.
 - **If a test fails, fix and rerun only that test.** Use `cargo test <test_name> -p <crate>`.
-- **⚠️ `./dev.sh all` runs the FULL suite including Docker integration tests (122+ tests, ~2-3 min).** Do NOT run `./dev.sh all` or `./dev.sh test` without asking the user first. Notify the user and let them trigger it themselves.
-- **Docker tests require `--test-threads=1`** (enforced by `.cargo/config.toml`). Each test spins up a privileged Ubuntu container with iptables.
+- **⚠️ The full suite may be run without asking.** `./dev.sh all` / `./dev.sh test` run the FULL suite including the Docker integration tests (122+ tests, ~2-3 min) — run them before calling work verified when a change touches the Docker harness, the iptables path, the natmap state file, or the auto_discover bootstrap.
+- **Nothing enforces `--test-threads=1`.** `.cargo/config.toml` sets only `rustflags`, so the Docker suites run with cargo's default parallelism. Pass `-- --test-threads=1` yourself when you need determinism; they are currently flaky under parallel execution (#54). Each test spins up a privileged Ubuntu container with iptables.
 
 ### Quick Test Commands
 

tests/auto_discover/mod.rs

@@ -8,15 +8,50 @@ mod recovery;
 mod registration;
 mod startup_race;
 
+use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
+use std::path::PathBuf;
 use std::process::Command;
 use std::sync::Once;
 
 static INIT: Once = Once::new();
 
+/// A test that hits `exit 1` never reaches the `teardown()` fragment appended to
+/// its script, so its `it-*` container survives on the host and the next run dies
+/// at `docker run --name` with a conflict that masks the real failure. Anchored
+/// `^it-` so a loose match cannot reach names like `audit-it-decoy`. Best effort:
+/// a missing or unhappy `docker` must never turn the suite red.
+fn sweep_leaked_containers() {
+    let Ok(list) = Command::new("docker")
+        .args(["ps", "-aq", "--filter", "name=^it-"])
+        .output()
+    else {
+        return;
+    };
+    let ids: Vec<String> = String::from_utf8_lossy(&list.stdout)
+        .lines()
+        .map(str::trim)
+        .filter(|id| !id.is_empty())
+        .map(String::from)
+        .collect();
+    if ids.is_empty() {
+        return;
+    }
+    eprintln!(
+        "sweeping {} leaked it-* test container(s) from a previous run: {}",
+        ids.len(),
+        ids.join(" ")
+    );
+    let _ = Command::new("docker")
+        .args(["rm", "-f"])
+        .args(&ids)
+        .status();
+}
+
 fn setup_image() -> &'static str {
     let image_name = "lab-ops-auto-discover-test:latest";
     INIT.call_once(|| {
+        sweep_leaked_containers();
         let dockerfile = concat!(
             "FROM ubuntu:24.04\n",
             "RUN apt-get update && apt-get install -y iptables jq curl unzip iproute2 docker.io\n",
@@ -41,6 +76,28 @@ fn setup_image() -> &'static str {
     image_name
 }
 
+/// A NixOS host links lab-ops against a loader and an OpenSSL under
+/// /nix/store that the test image lacks, so the binary cannot exec.
+/// Mounting /nix fixes the loader, but the lib still needs to be on the
+/// loader path, and setting LD_LIBRARY_PATH for the whole container
+/// shadows the image's own OpenSSL-linked tools: nix libcrypto's RUNPATH
+/// pulls nix glibc's libdl into curl, which then fails against the
+/// image's glibc. So put the path on a wrapper around the binary alone.
+/// Returns the wrapper to bind-mount over lab-ops, or None off NixOS.
+fn nix_wrapper(label: &str) -> Option<PathBuf> {
+    if !Path::new("/nix").is_dir() {
+        return None;
+    }
+    let lib_dir = std::env::var("OPENSSL_LIB_DIR").ok()?;
+    let wrapper = std::env::temp_dir().join(format!("lab-ops-{label}-wrapper.sh"));
+    let script = format!(
+        "#!/bin/sh\nexec env LD_LIBRARY_PATH={lib_dir} /usr/local/bin/lab-ops.bin \"$@\"\n"
+    );
+    std::fs::write(&wrapper, script).ok()?;
+    std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o755)).ok()?;
+    Some(wrapper)
+}
+
 pub(crate) fn run(script: &str) -> String {
     let image = setup_image();
     let binary_path = env!("CARGO_BIN_EXE_lab-ops");
@@ -50,18 +107,26 @@ pub(crate) fn run(script: &str) -> String {
         "--rm",
         "--privileged",
         "-v",
-        &format!("{binary_path}:/usr/local/bin/lab-ops"),
-        "-v",
         "/var/run/docker.sock:/var/run/docker.sock",
         "-e",
         "NATMAP_SOCKET=/tmp/natmap.sock",
         "-e",
         "CONSUL_HTTP_ADDR=http://127.0.0.1:8500",
     ]);
-    // A NixOS host links lab-ops against a loader under /nix/store, which the
-    // test image lacks, so the binary cannot exec. No-op elsewhere.
-    if Path::new("/nix").is_dir() {
-        cmd.args(["-v", "/nix:/nix:ro"]);
+    match nix_wrapper("auto-discover-docker") {
+        Some(wrapper) => {
+            cmd.args([
+                "-v",
+                "/nix:/nix:ro",
+                "-v",
+                &format!("{binary_path}:/usr/local/bin/lab-ops.bin"),
+                "-v",
+                &format!("{}:/usr/local/bin/lab-ops:ro", wrapper.display()),
+            ]);
+        }
+        None => {
+            cmd.args(["-v", &format!("{binary_path}:/usr/local/bin/lab-ops")]);
+        }
     }
     cmd.args([image, "sh", "-c"]);
     cmd.arg(script);

tests/natmap_docker.rs

@@ -1,6 +1,8 @@
 #[cfg(feature = "docker-tests")]
 mod natmap_docker {
+    use std::os::unix::fs::PermissionsExt;
     use std::path::Path;
+    use std::path::PathBuf;
     use std::process::Command;
     use std::sync::Once;
 
@@ -31,21 +33,47 @@ mod natmap_docker {
         image_name
     }
 
+    /// A NixOS host links lab-ops against a loader and an OpenSSL under
+    /// /nix/store that the test image lacks, so the binary cannot exec.
+    /// Mounting /nix fixes the loader, but the lib still needs to be on the
+    /// loader path, and setting LD_LIBRARY_PATH for the whole container
+    /// shadows the image's own OpenSSL-linked tools: nix libcrypto's RUNPATH
+    /// pulls nix glibc's libdl into curl, which then fails against the
+    /// image's glibc. So put the path on a wrapper around the binary alone.
+    /// Returns the wrapper to bind-mount over lab-ops, or None off NixOS.
+    fn nix_wrapper(label: &str) -> Option<PathBuf> {
+        if !Path::new("/nix").is_dir() {
+            return None;
+        }
+        let lib_dir = std::env::var("OPENSSL_LIB_DIR").ok()?;
+        let wrapper = std::env::temp_dir().join(format!("lab-ops-{label}-wrapper.sh"));
+        let script = format!(
+            "#!/bin/sh\nexec env LD_LIBRARY_PATH={lib_dir} /usr/local/bin/lab-ops.bin \"$@\"\n"
+        );
+        std::fs::write(&wrapper, script).ok()?;
+        std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o755)).ok()?;
+        Some(wrapper)
+    }
+
     fn run_in_docker(args: &[&str]) -> String {
         let image = setup_docker_image();
         let binary_path = env!("CARGO_BIN_EXE_lab-ops");
         let mut cmd = Command::new("docker");
-        cmd.args([
-            "run",
-            "--rm",
-            "--privileged",
-            "-v",
-            &format!("{binary_path}:/usr/local/bin/lab-ops"),
-        ]);
-        // A NixOS host links lab-ops against a loader under /nix/store, which
-        // the test image lacks, so the binary cannot exec. No-op elsewhere.
-        if Path::new("/nix").is_dir() {
-            cmd.args(["-v", "/nix:/nix:ro"]);
+        cmd.args(["run", "--rm", "--privileged"]);
+        match nix_wrapper("natmap-docker") {
+            Some(wrapper) => {
+                cmd.args([
+                    "-v",
+                    "/nix:/nix:ro",
+                    "-v",
+                    &format!("{binary_path}:/usr/local/bin/lab-ops.bin"),
+                    "-v",
+                    &format!("{}:/usr/local/bin/lab-ops:ro", wrapper.display()),
+                ]);
+            }
+            None => {
+                cmd.args(["-v", &format!("{binary_path}:/usr/local/bin/lab-ops")]);
+            }
         }
         cmd.args([image, "sh", "-c"]);