Command/backend/contract architecture; unify commands under commands/

.github/workflows/rust-binaries.yml

@@ -46,13 +46,19 @@ jobs:
       - name: Build deps
         run: sudo apt-get update && sudo apt-get install -y build-essential cmake
 
+      # Each user-facing command is now a thin shim (`nova-load`, `nova-storm`)
+      # plus a backend executable (`nova-load-qdrant`, `nova-storm-qdrant`); a
+      # worker needs BOTH. `nova-dist` downloads each of these release assets.
       - name: Build (release)
-        run: cargo build --release -p nova-load -p nova-storm
+        run: >
+          cargo build --release
+          -p nova-load -p nova-load-qdrant
+          -p nova-storm -p nova-storm-qdrant
 
       - name: Stage assets as <bin>-<target>
         run: |
           mkdir -p dist
-          for bin in nova-load nova-storm; do
+          for bin in nova-load nova-load-qdrant nova-storm nova-storm-qdrant; do
             cp "target/release/$bin" "dist/$bin-${{ matrix.target }}"
           done
 

Cargo.toml

@@ -1,9 +1,22 @@
 [workspace]
 resolver = "3"
 members = [
-    "crates/nova-storm",
-    "crates/nova-load",
-    "crates/nova-inspect",
+    # Per-language shared interfaces: the Rust contract each Rust backend
+    # implements (compile-time enforcement within Rust).
+    "backends/nova-load/contracts/rust",
+    "backends/nova-storm/contracts/rust",
+    # Backend implementations (execute the real work behind a command contract).
+    "backends/nova-storm/qdrant",
+    "backends/nova-load/qdrant",
+    # User-facing commands: thin shims that dispatch to a backend, plus the
+    # generic contract checker. `nova-shim` is the shared shim library (not a
+    # command) that nova-load/nova-storm both build on.
+    "commands/nova-shim",
+    "commands/nova-load",
+    "commands/nova-storm",
+    "commands/nova-contract",
+    # Standalone dev tool, not backend-dispatched.
+    "commands/nova-inspect",
 ]
 version = "0.0.8"
 

Makefile

@@ -1,10 +1,11 @@
 # supernova — install the polyglot CLI and its sub-tools.
 #
-#   make all      install the `nova` dispatcher + every sub-tool (embed, load, storm, inspect, bf, dist, sweep)
+#   make all      install the `nova` dispatcher + every sub-tool (embed, load, storm, inspect, bf, dist, sweep, contract)
 #   make cli      just the `nova` dispatcher (zero deps, instant)
 #   make embed    the `nova embed` Python tool (heavy: torch, sentence-transformers)
-#   make load     the `nova load` Rust binary
-#   make storm    the `nova storm` Rust binary
+#   make load     the `nova load` shim + `nova-load-qdrant` backend (Rust)
+#   make storm    the `nova storm` shim + `nova-storm-qdrant` backend (Rust)
+#   make contract the `nova contract` conformance checker (Rust)
 #   make inspect  the `nova inspect` Rust binary (count vectors + parquet schema)
 #   make bf       the `nova bf` Python tool (brute-force ground truth; torch)
 #   make dist     the `nova dist` orchestrator (SkyPilot; controller-side only)
@@ -13,50 +14,59 @@
 #   make test     run Rust + Python tests
 #
 # Sub-tools follow the git model: each installs a `nova-<cmd>` on PATH, and the
-# `nova` dispatcher execs it. Install only the ones you need.
+# `nova` dispatcher execs it. `load`/`storm` are now a thin shim command that
+# dispatches to a backend executable (`nova-<cmd>-<backend>`); installing them
+# installs both. Install only the ones you need.
 
-.PHONY: all cli embed load storm inspect bf dist sweep docs docs-build test clean
+.PHONY: all cli embed load storm contract inspect bf dist sweep docs docs-build test clean
 
-all: cli embed load storm inspect bf dist sweep
+all: cli embed load storm contract inspect bf dist sweep
 	@echo
-	@echo "✓ installed nova + embed/load/storm/inspect/bf/dist/sweep. Check with: nova --help"
+	@echo "✓ installed nova + embed/load/storm/contract/inspect/bf/dist/sweep. Check with: nova --help"
 
 # The `nova` dispatcher (root pyproject). Zero deps — installs anywhere instantly.
 cli:
 	uv pip install -e .
 
 # `nova embed` — Python, with the ML stack (torch, sentence-transformers, …).
 embed:
-	uv pip install -e 'python/nova-embed[embed]'
+	uv pip install -e 'commands/nova-embed[embed]'
 
-# `nova load` — Rust binary, into ~/.cargo/bin.
+# `nova load` — the shim (`nova-load`) + the Qdrant backend (`nova-load-qdrant`),
+# both into ~/.cargo/bin. The shim reads `vectorstore.type` and execs the backend.
 load:
-	cargo install --path crates/nova-load
+	cargo install --path commands/nova-load
+	cargo install --path backends/nova-load/qdrant
 
-# `nova storm` — Rust binary, into ~/.cargo/bin.
+# `nova storm` — the shim (`nova-storm`) + the Qdrant backend (`nova-storm-qdrant`).
 storm:
-	cargo install --path crates/nova-storm
+	cargo install --path commands/nova-storm
+	cargo install --path backends/nova-storm/qdrant
+
+# `nova contract` — the language-neutral backend conformance checker.
+contract:
+	cargo install --path commands/nova-contract
 
 # `nova inspect` — Rust binary, into ~/.cargo/bin.
 inspect:
-	cargo install --path crates/nova-inspect
+	cargo install --path commands/nova-inspect
 
 # `nova bf` — Python brute-force ground truth. `[compute]` pulls torch (GPU);
 # drop the extra for a controller that only runs `nova bf merge`.
 bf:
-	uv pip install -e 'python/nova-bf[compute]'
+	uv pip install -e 'commands/nova-bf[compute]'
 
 # `nova dist` — SkyPilot orchestrator. Controller-side only (your laptop / a
 # dispatch box); workers never need it. Still part of `make all` (pulls in
 # skypilot[aws], the heaviest dep in the whole install).
 dist:
-	uv pip install -e python/nova-dist
+	uv pip install -e commands/nova-dist
 
 # `nova sweep` — parameter sweep orchestrator (drives nova-load/nova-storm
 # subprocesses). Controller-side only, same precedent as `dist` — but, like
 # `dist`, still part of `make all`.
 sweep:
-	uv pip install -e python/nova-sweep
+	uv pip install -e commands/nova-sweep
 
 # Live docs at http://localhost:8000 (no install needed; uvx fetches zensical).
 docs:
@@ -67,9 +77,15 @@ docs-build:
 
 test:
 	cargo test
-	uv run --directory python/nova-embed --extra dev pytest -q || true
-	uv run --directory python/nova-bf --extra dev pytest -q || true
-	uv run --directory python/nova-sweep --extra dev pytest -q || true
+	# Build the backends + checker, then run contract conformance at shape/dry-run
+	# level (no live backend needed). These fail the build if a backend drifts
+	# from its contract.
+	cargo build -q -p nova-contract -p nova-load-qdrant -p nova-storm-qdrant
+	./target/debug/nova-contract check ./target/debug/nova-load-qdrant --contract contracts/nova-load/v1.yaml --level dry-run --fixtures tests/contracts/nova-load
+	./target/debug/nova-contract check ./target/debug/nova-storm-qdrant --contract contracts/nova-storm/v1.yaml --level dry-run --fixtures tests/contracts/nova-storm
+	uv run --directory commands/nova-embed --extra dev pytest -q || true
+	uv run --directory commands/nova-bf --extra dev pytest -q || true
+	uv run --directory commands/nova-sweep --extra dev pytest -q || true
 
 clean:
 	cargo clean

README.md

@@ -92,12 +92,20 @@ pool/job YAMLs without launching. Templates live in `configs/skypilot/`.
 supernova/
 ├── pyproject.toml          # the `nova` dispatcher (src/cli/)
 ├── src/cli/                # git-style dispatch: nova <cmd> -> nova-<cmd>
-├── crates/                 # Rust tools
-│   ├── nova-load/          #   nova load
-│   └── nova-storm/         #   nova storm
-├── python/
-│   ├── nova-embed/         # nova embed (ML pipeline; [embed] extra)
-│   └── nova-dist/          # nova dist  (SkyPilot orchestration)
+├── commands/               # ALL user-facing nova-* commands (any language)
+│   ├── nova-load/          #   nova load  (Rust shim → dispatches on vectorstore.type)
+│   ├── nova-storm/         #   nova storm (Rust shim → dispatches on target.type)
+│   ├── nova-contract/      #   nova contract (Rust backend conformance checker)
+│   ├── nova-inspect/       #   nova inspect (Rust dev tool)
+│   ├── nova-embed/         #   nova embed (Python ML pipeline; [embed] extra)
+│   ├── nova-bf/            #   nova bf    (Python brute-force ground truth)
+│   ├── nova-opt/           #   nova opt   (Python tuner; WIP, tracked on another branch)
+│   ├── nova-sweep/         #   nova sweep (Python sweep orchestrator)
+│   └── nova-dist/          #   nova dist  (Python SkyPilot orchestration)
+├── backends/               # backend implementations behind command contracts
+│   ├── nova-load/{contracts/rust, qdrant}    #   → nova-load-qdrant
+│   └── nova-storm/{contracts/rust, qdrant}   #   → nova-storm-qdrant
+├── contracts/              # language-neutral contract specs (nova-load, nova-storm)
 ├── configs/                # example YAML configs (+ skypilot/ resource templates)
 ├── docs/                   # zensical docs site
 └── Makefile

backends/nova-load/contracts/rust/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+# Shared Rust interface for `nova load` backends. Any Rust backend (the Qdrant
+# one today, a future Milvus/Vespa one tomorrow) depends on this crate and
+# implements its `VectorStore` trait — so the compiler enforces the contract at
+# build time within Rust. The language-neutral `contracts/nova-load/v1.yaml` is
+# the canonical cross-language contract; this crate is its Rust embodiment and
+# must be kept in lockstep with it.
+name = "nova-load-contract-rust"
+version = "0.0.8"
+edition = "2024"
+repository = "https://github.com/qdrant-labs/supernova"
+
+[lib]
+name = "nova_load_contract_rust"
+path = "src/lib.rs"
+
+[dependencies]
+async-trait = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+thiserror = { workspace = true }