diff --git a/.github/workflows/rust-binaries.yml b/.github/workflows/rust-binaries.yml index d04f763..6a9fc86 100644 --- a/.github/workflows/rust-binaries.yml +++ b/.github/workflows/rust-binaries.yml @@ -46,13 +46,19 @@ jobs: - name: Build deps run: sudo apt-get update && sudo apt-get install -y build-essential cmake + # Each user-facing command is now a thin shim (`nova-load`, `nova-storm`) + # plus a backend executable (`nova-load-qdrant`, `nova-storm-qdrant`); a + # worker needs BOTH. `nova-dist` downloads each of these release assets. - name: Build (release) - run: cargo build --release -p nova-load -p nova-storm + run: > + cargo build --release + -p nova-load -p nova-load-qdrant + -p nova-storm -p nova-storm-qdrant - name: Stage assets as - run: | mkdir -p dist - for bin in nova-load nova-storm; do + for bin in nova-load nova-load-qdrant nova-storm nova-storm-qdrant; do cp "target/release/$bin" "dist/$bin-${{ matrix.target }}" done diff --git a/Cargo.lock b/Cargo.lock index e598e8b..4304a5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1632,6 +1632,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nova-contract" +version = "0.0.8" +dependencies = [ + "clap", + "serde", + "serde_json", + "serde_yaml", +] + [[package]] name = "nova-inspect" version = "0.0.8" @@ -1643,12 +1653,30 @@ dependencies = [ [[package]] name = "nova-load" version = "0.0.8" +dependencies = [ + "nova-shim", +] + +[[package]] +name = "nova-load-contract-rust" +version = "0.0.8" +dependencies = [ + "async-trait", + "serde", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "nova-load-qdrant" +version = "0.0.8" dependencies = [ "async-trait", "clap", "duckdb", "futures", "indicatif", + "nova-load-contract-rust", "object_store", "qdrant-client", "rstest", @@ -1662,14 +1690,37 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "nova-shim" +version = "0.0.8" +dependencies = [ + "serde_yaml", +] + [[package]] name = "nova-storm" version = "0.0.8" +dependencies = [ + "nova-shim", +] + +[[package]] +name = "nova-storm-contract-rust" +version = "0.0.8" +dependencies = [ + "async-trait", + "thiserror 2.0.18", +] + +[[package]] +name = "nova-storm-qdrant" +version = "0.0.8" dependencies = [ "async-trait", "clap", "duckdb", "indicatif", + "nova-storm-contract-rust", "qdrant-client", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 77f51aa..37c5e8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,22 @@ [workspace] resolver = "3" members = [ - "crates/nova-storm", - "crates/nova-load", - "crates/nova-inspect", + # Per-language shared interfaces: the Rust contract each Rust backend + # implements (compile-time enforcement within Rust). + "backends/nova-load/contracts/rust", + "backends/nova-storm/contracts/rust", + # Backend implementations (execute the real work behind a command contract). + "backends/nova-storm/qdrant", + "backends/nova-load/qdrant", + # User-facing commands: thin shims that dispatch to a backend, plus the + # generic contract checker. `nova-shim` is the shared shim library (not a + # command) that nova-load/nova-storm both build on. + "commands/nova-shim", + "commands/nova-load", + "commands/nova-storm", + "commands/nova-contract", + # Standalone dev tool, not backend-dispatched. + "commands/nova-inspect", ] version = "0.0.8" diff --git a/Makefile b/Makefile index 1a959a9..3d2f592 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,11 @@ # supernova — install the polyglot CLI and its sub-tools. # -# make all install the `nova` dispatcher + every sub-tool (embed, load, storm, inspect, bf, dist, sweep) +# make all install the `nova` dispatcher + every sub-tool (embed, load, storm, inspect, bf, dist, sweep, contract) # make cli just the `nova` dispatcher (zero deps, instant) # make embed the `nova embed` Python tool (heavy: torch, sentence-transformers) -# make load the `nova load` Rust binary -# make storm the `nova storm` Rust binary +# make load the `nova load` shim + `nova-load-qdrant` backend (Rust) +# make storm the `nova storm` shim + `nova-storm-qdrant` backend (Rust) +# make contract the `nova contract` conformance checker (Rust) # make inspect the `nova inspect` Rust binary (count vectors + parquet schema) # make bf the `nova bf` Python tool (brute-force ground truth; torch) # make dist the `nova dist` orchestrator (SkyPilot; controller-side only) @@ -13,13 +14,15 @@ # make test run Rust + Python tests # # Sub-tools follow the git model: each installs a `nova-` on PATH, and the -# `nova` dispatcher execs it. Install only the ones you need. +# `nova` dispatcher execs it. `load`/`storm` are now a thin shim command that +# dispatches to a backend executable (`nova--`); installing them +# installs both. Install only the ones you need. -.PHONY: all cli embed load storm inspect bf dist sweep docs docs-build test clean +.PHONY: all cli embed load storm contract inspect bf dist sweep docs docs-build test clean -all: cli embed load storm inspect bf dist sweep +all: cli embed load storm contract inspect bf dist sweep @echo - @echo "✓ installed nova + embed/load/storm/inspect/bf/dist/sweep. Check with: nova --help" + @echo "✓ installed nova + embed/load/storm/contract/inspect/bf/dist/sweep. Check with: nova --help" # The `nova` dispatcher (root pyproject). Zero deps — installs anywhere instantly. cli: @@ -27,36 +30,43 @@ cli: # `nova embed` — Python, with the ML stack (torch, sentence-transformers, …). embed: - uv pip install -e 'python/nova-embed[embed]' + uv pip install -e 'commands/nova-embed[embed]' -# `nova load` — Rust binary, into ~/.cargo/bin. +# `nova load` — the shim (`nova-load`) + the Qdrant backend (`nova-load-qdrant`), +# both into ~/.cargo/bin. The shim reads `vectorstore.type` and execs the backend. load: - cargo install --path crates/nova-load + cargo install --path commands/nova-load + cargo install --path backends/nova-load/qdrant -# `nova storm` — Rust binary, into ~/.cargo/bin. +# `nova storm` — the shim (`nova-storm`) + the Qdrant backend (`nova-storm-qdrant`). storm: - cargo install --path crates/nova-storm + cargo install --path commands/nova-storm + cargo install --path backends/nova-storm/qdrant + +# `nova contract` — the language-neutral backend conformance checker. +contract: + cargo install --path commands/nova-contract # `nova inspect` — Rust binary, into ~/.cargo/bin. inspect: - cargo install --path crates/nova-inspect + cargo install --path commands/nova-inspect # `nova bf` — Python brute-force ground truth. `[compute]` pulls torch (GPU); # drop the extra for a controller that only runs `nova bf merge`. bf: - uv pip install -e 'python/nova-bf[compute]' + uv pip install -e 'commands/nova-bf[compute]' # `nova dist` — SkyPilot orchestrator. Controller-side only (your laptop / a # dispatch box); workers never need it. Still part of `make all` (pulls in # skypilot[aws], the heaviest dep in the whole install). dist: - uv pip install -e python/nova-dist + uv pip install -e commands/nova-dist # `nova sweep` — parameter sweep orchestrator (drives nova-load/nova-storm # subprocesses). Controller-side only, same precedent as `dist` — but, like # `dist`, still part of `make all`. sweep: - uv pip install -e python/nova-sweep + uv pip install -e commands/nova-sweep # Live docs at http://localhost:8000 (no install needed; uvx fetches zensical). docs: @@ -67,9 +77,15 @@ docs-build: test: cargo test - uv run --directory python/nova-embed --extra dev pytest -q || true - uv run --directory python/nova-bf --extra dev pytest -q || true - uv run --directory python/nova-sweep --extra dev pytest -q || true + # Build the backends + checker, then run contract conformance at shape/dry-run + # level (no live backend needed). These fail the build if a backend drifts + # from its contract. + cargo build -q -p nova-contract -p nova-load-qdrant -p nova-storm-qdrant + ./target/debug/nova-contract check ./target/debug/nova-load-qdrant --contract contracts/nova-load/v1.yaml --level dry-run --fixtures tests/contracts/nova-load + ./target/debug/nova-contract check ./target/debug/nova-storm-qdrant --contract contracts/nova-storm/v1.yaml --level dry-run --fixtures tests/contracts/nova-storm + uv run --directory commands/nova-embed --extra dev pytest -q || true + uv run --directory commands/nova-bf --extra dev pytest -q || true + uv run --directory commands/nova-sweep --extra dev pytest -q || true clean: cargo clean diff --git a/README.md b/README.md index d9e4e87..6c5b62e 100644 --- a/README.md +++ b/README.md @@ -92,12 +92,20 @@ pool/job YAMLs without launching. Templates live in `configs/skypilot/`. supernova/ ├── pyproject.toml # the `nova` dispatcher (src/cli/) ├── src/cli/ # git-style dispatch: nova -> nova- -├── crates/ # Rust tools -│ ├── nova-load/ # nova load -│ └── nova-storm/ # nova storm -├── python/ -│ ├── nova-embed/ # nova embed (ML pipeline; [embed] extra) -│ └── nova-dist/ # nova dist (SkyPilot orchestration) +├── commands/ # ALL user-facing nova-* commands (any language) +│ ├── nova-load/ # nova load (Rust shim → dispatches on vectorstore.type) +│ ├── nova-storm/ # nova storm (Rust shim → dispatches on target.type) +│ ├── nova-contract/ # nova contract (Rust backend conformance checker) +│ ├── nova-inspect/ # nova inspect (Rust dev tool) +│ ├── nova-embed/ # nova embed (Python ML pipeline; [embed] extra) +│ ├── nova-bf/ # nova bf (Python brute-force ground truth) +│ ├── nova-opt/ # nova opt (Python tuner; WIP, tracked on another branch) +│ ├── nova-sweep/ # nova sweep (Python sweep orchestrator) +│ └── nova-dist/ # nova dist (Python SkyPilot orchestration) +├── backends/ # backend implementations behind command contracts +│ ├── nova-load/{contracts/rust, qdrant} # → nova-load-qdrant +│ └── nova-storm/{contracts/rust, qdrant} # → nova-storm-qdrant +├── contracts/ # language-neutral contract specs (nova-load, nova-storm) ├── configs/ # example YAML configs (+ skypilot/ resource templates) ├── docs/ # zensical docs site └── Makefile diff --git a/backends/nova-load/contracts/rust/Cargo.toml b/backends/nova-load/contracts/rust/Cargo.toml new file mode 100644 index 0000000..9d873a9 --- /dev/null +++ b/backends/nova-load/contracts/rust/Cargo.toml @@ -0,0 +1,21 @@ +[package] +# Shared Rust interface for `nova load` backends. Any Rust backend (the Qdrant +# one today, a future Milvus/Vespa one tomorrow) depends on this crate and +# implements its `VectorStore` trait — so the compiler enforces the contract at +# build time within Rust. The language-neutral `contracts/nova-load/v1.yaml` is +# the canonical cross-language contract; this crate is its Rust embodiment and +# must be kept in lockstep with it. +name = "nova-load-contract-rust" +version = "0.0.8" +edition = "2024" +repository = "https://github.com/qdrant-labs/supernova" + +[lib] +name = "nova_load_contract_rust" +path = "src/lib.rs" + +[dependencies] +async-trait = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/nova-load/src/stores/mod.rs b/backends/nova-load/contracts/rust/src/lib.rs similarity index 55% rename from crates/nova-load/src/stores/mod.rs rename to backends/nova-load/contracts/rust/src/lib.rs index c86e5b6..1e1c4b4 100644 --- a/crates/nova-load/src/stores/mod.rs +++ b/backends/nova-load/contracts/rust/src/lib.rs @@ -1,51 +1,87 @@ -mod qdrant; +//! Shared Rust interface for `nova load` backends. +//! +//! This crate is the Rust embodiment of the language-neutral contract in +//! `contracts/nova-load/v1.yaml`. A Rust backend depends on it and implements +//! [`VectorStore`]; the compiler then enforces the method set at build time. A +//! non-Rust backend (or any backend at all) is instead checked at runtime by +//! `nova contract check`, which compares the backend's `capabilities --json` +//! against the same YAML contract. Keep the three in lockstep: +//! +//! - the [`VectorStore`] trait method names here, +//! - the `methods:` list in `contracts/nova-load/v1.yaml`, +//! - the `methods` array a backend advertises from `capabilities --json`. +//! +//! Only the genuinely backend-agnostic surface lives here. Backend-specific +//! config (collection tuning, connection details, the `vectorstore.type` +//! dispatch enum) stays in each backend crate. use std::collections::HashMap; use async_trait::async_trait; use serde::{Deserialize, Serialize}; -use crate::config::VectorSpec; - -/// Errors from a [`VectorStore`](crate::stores::VectorStore) backend. +/// Errors from a [`VectorStore`] backend. /// -/// Backend client errors are boxed: they're large, and an unboxed variant -/// would bloat every `Result` (including the `Ok` path on hot calls like -/// `upsert_batch`). The manual `From` keeps `?` ergonomic. +/// Backend-neutral by construction: a backend renders its own client error to +/// string form at the trait boundary via [`StoreError::backend`], so this crate +/// never depends on any particular vector-DB client. #[derive(Debug, thiserror::Error)] pub enum StoreError { - #[error(transparent)] - Qdrant(Box), + /// A backend client error (the vector DB's own error), captured as its + /// string form so this type stays backend-neutral. + #[error("{0}")] + Backend(String), /// Backend-agnostic failure, e.g. an existing collection whose config /// conflicts with the requested one. #[error("{0}")] Other(String), } -impl From for StoreError { - fn from(err: qdrant_client::QdrantError) -> Self { - StoreError::Qdrant(Box::new(err)) +impl StoreError { + /// Wrap any backend error as a neutral [`StoreError::Backend`]. Use at the + /// trait boundary, e.g. `client.foo().await.map_err(StoreError::backend)?`. + pub fn backend(err: E) -> Self { + StoreError::Backend(err.to_string()) } } -/// Vectorstore backend config, dispatched on `type:`. Each backend owns its -/// config struct in its own module; the variant is gated on the same feature. -#[derive(Debug, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum VectorStoreConfig { - Qdrant(qdrant::QdrantConfig), +/// One named vector's spec. The scalar knobs (distance, datatype, comparator, +/// modifier) are strings interpreted by the store. HNSW/quantization tuning is +/// collection-wide (see each backend's store params), not per-vector. +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct VectorSpec { + #[serde(rename = "type")] + pub kind: VectorKind, + /// Parquet column the reader pulls this vector from. + pub column: String, + /// Dense vector dimensionality. Optional: when omitted the loader infers it + /// from the parquet schema (the column is a fixed-size list). Ignored for + /// sparse vectors, which have no fixed size. Read by the store at + /// collection-creation time; ignored by the reader. + #[serde(default)] + pub size: Option, + /// Read by the store at collection-creation time; ignored by the reader. + #[serde(default)] + pub distance: Option, + /// Multivector comparator (e.g. `max_sim`); only meaningful for `multivector`. + #[serde(default)] + pub comparator: Option, + #[serde(default)] + pub datatype: Option, + #[serde(default)] + pub on_disk: Option, + /// Sparse re-weighting modifier (e.g. `idf`); only meaningful for `sparse`. + #[serde(default)] + pub modifier: Option, } -impl VectorStoreConfig { - /// Connect to the backend, building the live client once. Consumes the - /// config (it's parsed once, then handed straight here) and returns the - /// runtime store as a trait object — connection errors surface here, at - /// startup, rather than mid-load. - pub async fn connect(self) -> Result, StoreError> { - match self { - VectorStoreConfig::Qdrant(c) => Ok(Box::new(c.connect().await?)), - } - } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VectorKind { + Dense, + Sparse, + Multivector, } /// What any backend needs to create or verify a collection: the named vector @@ -69,7 +105,7 @@ pub enum PointId { /// One named vector's value, as read from the source. Covers the three shapes a /// backend like Qdrant accepts; the reader emits the variant matching the -/// vector's configured [`kind`](crate::config::VectorKind). +/// vector's configured [`kind`](VectorKind). #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum VectorValue { @@ -92,6 +128,9 @@ pub struct Point { /// A vector store backend. `Display` is the human-readable name used in logs /// (e.g. `qdrant(my_collection)`). +/// +/// The method set here is the canonical Rust contract for a `nova load` +/// backend and must match `methods:` in `contracts/nova-load/v1.yaml`. #[async_trait] pub trait VectorStore: Send + Sync + std::fmt::Display { /// Create the target collection if absent (or verify it exists), from the diff --git a/crates/nova-load/Cargo.toml b/backends/nova-load/qdrant/Cargo.toml similarity index 63% rename from crates/nova-load/Cargo.toml rename to backends/nova-load/qdrant/Cargo.toml index ec99792..25e0180 100644 --- a/crates/nova-load/Cargo.toml +++ b/backends/nova-load/qdrant/Cargo.toml @@ -1,11 +1,21 @@ [package] -name = "nova-load" +# Qdrant backend for `nova load`. Built as the executable `nova-load-qdrant`; +# the user-facing `nova-load` shim (commands/nova-load) execs it based on +# `vectorstore.type` in the config. The library crate keeps the historical +# name `nova_load` (see `[lib]`) so no internal module paths change. +name = "nova-load-qdrant" version = "0.0.8" edition = "2024" repository = "https://github.com/qdrant-labs/supernova" +# Keep the library name stable so `use nova_load::...` in main.rs and the +# inline tests keeps working after the package rename. +[lib] +name = "nova_load" +path = "src/lib.rs" + [[bin]] -name = "nova-load" +name = "nova-load-qdrant" path = "src/main.rs" # `cargo binstall nova-load` fetches the prebuilt binary the rust-binaries.yml @@ -16,6 +26,9 @@ pkg-url = "{ repo }/releases/download/v{ version }/{ name }-{ target }{ binary-e pkg-fmt = "bin" [dependencies] +# Shared Rust interface this backend implements (the `VectorStore` trait + +# neutral types). See backends/nova-load/contracts/rust. +nova-load-contract-rust = { path = "../contracts/rust" } async-trait = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/nova-load/docs/future-work.md b/backends/nova-load/qdrant/docs/future-work.md similarity index 100% rename from crates/nova-load/docs/future-work.md rename to backends/nova-load/qdrant/docs/future-work.md diff --git a/crates/nova-load/src/config.rs b/backends/nova-load/qdrant/src/config.rs similarity index 88% rename from crates/nova-load/src/config.rs rename to backends/nova-load/qdrant/src/config.rs index b63d07f..b05378c 100644 --- a/crates/nova-load/src/config.rs +++ b/backends/nova-load/qdrant/src/config.rs @@ -6,6 +6,12 @@ use serde::Deserialize; use crate::sources::DataSourceConfig; use crate::stores::VectorStoreConfig; +// Per-vector spec types are part of the backend-agnostic contract; they live in +// the shared `nova-load-contract-rust` crate. Re-export here so existing +// `crate::config::{VectorSpec, VectorKind}` paths across this backend are +// unchanged. +pub use nova_load_contract_rust::{VectorKind, VectorSpec}; + /// The full parsed load config. This is the top-level struct deserialized from the YAML; it references the backend-specific configs in the `stores` and `sources` modules. #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -162,44 +168,9 @@ fn expand_env_with( Ok(out) } -/// One named vector's spec. The scalar knobs (distance, datatype, comparator, -/// modifier) are strings interpreted by the store. HNSW/quantization tuning is -/// collection-wide (see the store params), not per-vector. -#[derive(Debug, Clone, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct VectorSpec { - #[serde(rename = "type")] - pub kind: VectorKind, - /// Parquet column the reader pulls this vector from. - pub column: String, - /// Dense vector dimensionality. Optional: when omitted the loader infers it - /// from the parquet schema (the column is a fixed-size list). Ignored for - /// sparse vectors, which have no fixed size. Read by the store at - /// collection-creation time; ignored by the reader. - #[serde(default)] - pub size: Option, - /// Read by the store at collection-creation time; ignored by the reader. - #[serde(default)] - pub distance: Option, - /// Multivector comparator (e.g. `max_sim`); only meaningful for `multivector`. - #[serde(default)] - pub comparator: Option, - #[serde(default)] - pub datatype: Option, - #[serde(default)] - pub on_disk: Option, - /// Sparse re-weighting modifier (e.g. `idf`); only meaningful for `sparse`. - #[serde(default)] - pub modifier: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum VectorKind { - Dense, - Sparse, - Multivector, -} +// `VectorSpec` and `VectorKind` moved to the shared `nova-load-contract-rust` +// crate (re-exported at the top of this module). They are part of the +// backend-agnostic contract, not qdrant-specific config. /// Collection-wide HNSW index parameters. All fields optional; the store /// applies its own defaults for any left unset. diff --git a/crates/nova-load/src/engine.rs b/backends/nova-load/qdrant/src/engine.rs similarity index 100% rename from crates/nova-load/src/engine.rs rename to backends/nova-load/qdrant/src/engine.rs diff --git a/crates/nova-load/src/lib.rs b/backends/nova-load/qdrant/src/lib.rs similarity index 100% rename from crates/nova-load/src/lib.rs rename to backends/nova-load/qdrant/src/lib.rs diff --git a/crates/nova-load/src/main.rs b/backends/nova-load/qdrant/src/main.rs similarity index 62% rename from crates/nova-load/src/main.rs rename to backends/nova-load/qdrant/src/main.rs index b67aba1..38856d9 100644 --- a/crates/nova-load/src/main.rs +++ b/backends/nova-load/qdrant/src/main.rs @@ -33,6 +33,49 @@ enum Command { Delete(RunArgs), /// Inspect the config and the file list without connecting or loading. Inspect(LoadArgs), + /// Print this backend's capabilities as stable JSON (the machine-readable + /// contract descriptor `nova-contract` validates against + /// `contracts/nova-load/v1.yaml`). Does not connect or load. + Capabilities(CapabilitiesArgs), +} + +/// Args for `capabilities`. `--json` is accepted for forward-compat and CLI +/// symmetry; output is always JSON regardless. +#[derive(Debug, Args)] +struct CapabilitiesArgs { + /// Emit JSON (the default and only format today). + #[arg(long)] + json: bool, +} + +/// The stable capabilities descriptor for this backend. Kept in lockstep with +/// `contracts/nova-load/v1.yaml`: the `commands`, `methods`, `vector_kinds`, +/// and `point_id_types` here are exactly what that contract declares required, +/// and `nova-contract check` fails if this drifts from the contract file. +/// +/// - `commands` mirror the clap subcommands below. +/// - `methods` mirror the `VectorStore` trait in `stores/mod.rs`. +/// - `vector_kinds` mirror `VectorValue` (`stores/mod.rs`). +/// - `point_id_types` mirror `PointId` (`stores/mod.rs`). +fn capabilities_json() -> serde_json::Value { + serde_json::json!({ + "contract": "nova-load-backend/v1", + "backend": "qdrant", + "commands": [ + "capabilities", "run", "prepare", "load", + "finalize", "inspect", "reindex", "delete" + ], + "methods": [ + "ensure_collection", "upsert_batch", "close", "defer_indexing", + "enable_indexing", "wait_for_indexing", "reindex", "delete_collection" + ], + "vector_kinds": ["dense", "sparse", "multivector"], + "point_id_types": ["integer", "string"], + "flags": { + "load": ["--num-jobs", "--job-rank"], + "inspect": ["--num-jobs", "--job-rank"] + } + }) } /// Args for phases that act on the whole dataset (no partitioning). @@ -86,6 +129,16 @@ async fn main() -> ExitCode { /// Dispatch a subcommand. Returns `Err(ExitCode)` for any failure so `main` /// stays a thin shell. async fn run(command: Command) -> Result<(), ExitCode> { + // `capabilities` is a pure descriptor: no config, no connection, always JSON. + if let Command::Capabilities(_) = command { + println!( + "{}", + serde_json::to_string_pretty(&capabilities_json()) + .expect("capabilities descriptor is always serializable") + ); + return Ok(()); + } + let result = match command { Command::Run(a) => nova_load::run(load_config(&a.config)?).await, Command::Prepare(a) => nova_load::prepare(load_config(&a.config)?).await, @@ -106,6 +159,8 @@ async fn run(command: Command) -> Result<(), ExitCode> { })?; nova_load::inspect(load_config(&a.config)?, partition).await } + // Handled above with an early return before this match. + Command::Capabilities(_) => unreachable!("capabilities handled before dispatch"), }; result.map_err(|err| { diff --git a/crates/nova-load/src/plan.rs b/backends/nova-load/qdrant/src/plan.rs similarity index 100% rename from crates/nova-load/src/plan.rs rename to backends/nova-load/qdrant/src/plan.rs diff --git a/crates/nova-load/src/sources/local.rs b/backends/nova-load/qdrant/src/sources/local.rs similarity index 100% rename from crates/nova-load/src/sources/local.rs rename to backends/nova-load/qdrant/src/sources/local.rs diff --git a/crates/nova-load/src/sources/mod.rs b/backends/nova-load/qdrant/src/sources/mod.rs similarity index 100% rename from crates/nova-load/src/sources/mod.rs rename to backends/nova-load/qdrant/src/sources/mod.rs diff --git a/crates/nova-load/src/sources/s3.rs b/backends/nova-load/qdrant/src/sources/s3.rs similarity index 100% rename from crates/nova-load/src/sources/s3.rs rename to backends/nova-load/qdrant/src/sources/s3.rs diff --git a/backends/nova-load/qdrant/src/stores/mod.rs b/backends/nova-load/qdrant/src/stores/mod.rs new file mode 100644 index 0000000..6c0c489 --- /dev/null +++ b/backends/nova-load/qdrant/src/stores/mod.rs @@ -0,0 +1,32 @@ +mod qdrant; + +use serde::Deserialize; + +// The backend-agnostic contract (trait + shared types) lives in the shared +// Rust interface crate `nova-load-contract-rust`. Re-export it so the rest of +// this backend keeps referring to `crate::stores::{VectorStore, Point, ...}` +// unchanged, and so the qdrant module implements exactly that trait. +pub use nova_load_contract_rust::{ + CollectionSchema, Point, PointId, StoreError, VectorStore, VectorValue, +}; + +/// Vectorstore backend config, dispatched on `type:`. Each backend owns its +/// config struct in its own module. This dispatch enum names concrete backends, +/// so it stays here in the backend crate rather than in the neutral contract. +#[derive(Debug, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum VectorStoreConfig { + Qdrant(qdrant::QdrantConfig), +} + +impl VectorStoreConfig { + /// Connect to the backend, building the live client once. Consumes the + /// config (it's parsed once, then handed straight here) and returns the + /// runtime store as a trait object — connection errors surface here, at + /// startup, rather than mid-load. + pub async fn connect(self) -> Result, StoreError> { + match self { + VectorStoreConfig::Qdrant(c) => Ok(Box::new(c.connect().await?)), + } + } +} diff --git a/crates/nova-load/src/stores/qdrant.rs b/backends/nova-load/qdrant/src/stores/qdrant.rs similarity index 93% rename from crates/nova-load/src/stores/qdrant.rs rename to backends/nova-load/qdrant/src/stores/qdrant.rs index 8bff93e..f9f5eec 100644 --- a/crates/nova-load/src/stores/qdrant.rs +++ b/backends/nova-load/qdrant/src/stores/qdrant.rs @@ -472,7 +472,8 @@ impl QdrantConfig { let client = Qdrant::from_url(&self.url) .api_key(self.api_key) // .check_compatibility(false) // skip since the log is annoying - .build()?; + .build() + .map_err(StoreError::backend)?; Ok(QdrantStore { client, collection_name: self.collection_name, @@ -490,46 +491,53 @@ impl fmt::Display for QdrantStore { /// One named vector value → the qdrant wire `Vector` (leans on qdrant-client's /// own `From` impls for each shape). -impl From for Vector { - fn from(value: VectorValue) -> Self { - match value { - VectorValue::Dense(d) => d.into(), - VectorValue::Multi(m) => m.into(), - VectorValue::Sparse { indices, values } => { - indices.into_iter().zip(values).collect::>().into() - } +/// +/// A free function, not a `From` impl: both `VectorValue` (the shared contract +/// crate) and `Vector` (qdrant-client) are now foreign to this crate, so the +/// orphan rule forbids `impl From for Vector` here. +fn vector_value_to_wire(value: VectorValue) -> Vector { + match value { + VectorValue::Dense(d) => d.into(), + VectorValue::Multi(m) => m.into(), + VectorValue::Sparse { indices, values } => { + indices.into_iter().zip(values).collect::>().into() } } } -/// A read point → a qdrant `PointStruct`. Allowed by the orphan rule because -/// `Point` is local; gives `Into` for free. -impl From for PointStruct { - fn from(point: Point) -> Self { - let vectors: HashMap = point - .vectors - .into_iter() - .map(|(name, value)| (name, value.into())) - .collect(); - let payload = Payload::from(point.payload); - match point.id { - PointId::Integer(n) => PointStruct::new(n, vectors, payload), - PointId::String(s) => PointStruct::new(s, vectors, payload), - } +/// A read [`Point`] → a qdrant `PointStruct`. Also a free function (not `From`) +/// for the same orphan-rule reason as [`vector_value_to_wire`]. +fn point_to_wire(point: Point) -> PointStruct { + let vectors: HashMap = point + .vectors + .into_iter() + .map(|(name, value)| (name, vector_value_to_wire(value))) + .collect(); + let payload = Payload::from(point.payload); + match point.id { + PointId::Integer(n) => PointStruct::new(n, vectors, payload), + PointId::String(s) => PointStruct::new(s, vectors, payload), } } #[async_trait] impl VectorStore for QdrantStore { async fn ensure_collection(&self, schema: &CollectionSchema) -> Result<(), StoreError> { - let exists = self.client.collection_exists(self.collection_name.as_str()).await?; + let exists = self + .client + .collection_exists(self.collection_name.as_str()) + .await + .map_err(StoreError::backend)?; if exists { if !self.params.recreate { // We dont confirm that the collection schema matches the config // in the future we could... but for now, just assume the user knows what they're doing if they set recreate=false. return Ok(()); } - self.client.delete_collection(self.collection_name.as_str()).await?; + self.client + .delete_collection(self.collection_name.as_str()) + .await + .map_err(StoreError::backend)?; } let request = build_create_collection( @@ -539,18 +547,22 @@ impl VectorStore for QdrantStore { &schema.dims, ) .map_err(|e| StoreError::Other(e.to_string()))?; - self.client.create_collection(request).await?; + self.client + .create_collection(request) + .await + .map_err(StoreError::backend)?; Ok(()) } async fn upsert_batch(&self, points: Vec) -> Result<(), StoreError> { - let points: Vec = points.into_iter().map(PointStruct::from).collect(); + let points: Vec = points.into_iter().map(point_to_wire).collect(); self.client .upsert_points( UpsertPointsBuilder::new(self.collection_name.as_str(), points) .wait(self.upsert_wait), ) - .await?; + .await + .map_err(StoreError::backend)?; Ok(()) } @@ -566,7 +578,8 @@ impl VectorStore for QdrantStore { UpdateCollectionBuilder::new(self.collection_name.as_str()) .optimizers_config(OptimizersConfigDiffBuilder::default().indexing_threshold(0)), ) - .await?; + .await + .map_err(StoreError::backend)?; Ok(()) } @@ -583,7 +596,8 @@ impl VectorStore for QdrantStore { OptimizersConfigDiffBuilder::default().indexing_threshold(threshold), ), ) - .await?; + .await + .map_err(StoreError::backend)?; Ok(()) } @@ -601,7 +615,11 @@ impl VectorStore for QdrantStore { let mut green_since: Option = None; // TODO: add an overall timeout so a stuck (non-erroring) optimizer can't loop forever. loop { - let resp = self.client.collection_info(self.collection_name.as_str()).await?; + let resp = self + .client + .collection_info(self.collection_name.as_str()) + .await + .map_err(StoreError::backend)?; let result = resp.result; if let Some(status) = result.as_ref().and_then(|r| r.optimizer_status.as_ref()) @@ -630,13 +648,24 @@ impl VectorStore for QdrantStore { async fn reindex(&self) -> Result<(), StoreError> { let builder = build_update_collection(&self.collection_name, &self.params) .map_err(|e| StoreError::Other(e.to_string()))?; - self.client.update_collection(builder).await?; + self.client + .update_collection(builder) + .await + .map_err(StoreError::backend)?; Ok(()) } async fn delete_collection(&self) -> Result<(), StoreError> { - if self.client.collection_exists(self.collection_name.as_str()).await? { - self.client.delete_collection(self.collection_name.as_str()).await?; + if self + .client + .collection_exists(self.collection_name.as_str()) + .await + .map_err(StoreError::backend)? + { + self.client + .delete_collection(self.collection_name.as_str()) + .await + .map_err(StoreError::backend)?; } Ok(()) } @@ -657,7 +686,7 @@ mod tests { fn load_fixture() -> LoadConfig { let path = concat!( env!("CARGO_MANIFEST_DIR"), - "/../../tests/configs/qdrant_all_params.yaml" + "/../../../tests/configs/qdrant_all_params.yaml" ); let yaml = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("read fixture {path}: {e}")); diff --git a/backends/nova-storm/contracts/rust/Cargo.toml b/backends/nova-storm/contracts/rust/Cargo.toml new file mode 100644 index 0000000..51eb871 --- /dev/null +++ b/backends/nova-storm/contracts/rust/Cargo.toml @@ -0,0 +1,18 @@ +[package] +# Shared Rust interface for `nova storm` backends. Any Rust target backend +# depends on this crate and implements its `QueryTarget` trait, so the compiler +# enforces the contract at build time within Rust. The language-neutral +# `contracts/nova-storm/v1.yaml` is canonical across languages; this crate is +# its Rust embodiment and must be kept in lockstep with it. +name = "nova-storm-contract-rust" +version = "0.0.8" +edition = "2024" +repository = "https://github.com/qdrant-labs/supernova" + +[lib] +name = "nova_storm_contract_rust" +path = "src/lib.rs" + +[dependencies] +async-trait = { workspace = true } +thiserror = { workspace = true } diff --git a/backends/nova-storm/contracts/rust/src/lib.rs b/backends/nova-storm/contracts/rust/src/lib.rs new file mode 100644 index 0000000..5d82e35 --- /dev/null +++ b/backends/nova-storm/contracts/rust/src/lib.rs @@ -0,0 +1,86 @@ +//! Shared Rust interface for `nova storm` backends. +//! +//! This crate is the Rust embodiment of the language-neutral contract in +//! `contracts/nova-storm/v1.yaml`. A Rust target backend depends on it and +//! implements [`QueryTarget`]; the compiler then enforces the method set at +//! build time. Any backend is additionally checked at runtime by +//! `nova contract check`, which compares the backend's `capabilities --json` +//! against the same YAML contract. Keep the three in lockstep: +//! +//! - the [`QueryTarget`] method names here, +//! - the `methods:` list in `contracts/nova-storm/v1.yaml`, +//! - the `methods` array a backend advertises from `capabilities --json`. +//! +//! Only the backend-agnostic surface lives here. The `target.type` dispatch +//! enum and each backend's connection/query config stay in the backend crate. + +use std::time::Duration; + +use async_trait::async_trait; + +/// Errors from a [`QueryTarget`] backend. +/// +/// Backend-neutral by construction: a backend renders its own client error to +/// string form at the trait boundary via [`TargetError::backend`], so this +/// crate never depends on any particular vector-DB client. Note this covers +/// setup/teardown only — a *query* failure during the load run is recorded as a +/// non-fatal error sample (see [`BatchOutcome`]), not surfaced here. +#[derive(Debug, thiserror::Error)] +pub enum TargetError { + /// A backend client error (the vector DB's own error), captured as its + /// string form so this type stays backend-neutral. + #[error("{0}")] + Backend(String), + /// Backend-agnostic failure, e.g. a config the backend can't honour. + #[error("{0}")] + Other(String), +} + +impl TargetError { + /// Wrap any backend error as a neutral [`TargetError::Backend`]. Use at the + /// trait boundary, e.g. `builder.build().map_err(TargetError::backend)?`. + pub fn backend(err: E) -> Self { + TargetError::Backend(err.to_string()) + } +} + +/// Outcome of a single batch dispatch (one `query_batch` round-trip, covering +/// `vectors.len()` queries). A failure is recorded here (`ok = false`) rather +/// than aborting the run — a storm measures how a cluster behaves under load, +/// and errors at the limit are a finding, not a crash. `latency`/`ok`/`error` +/// describe the one round-trip, not any individual query inside it — a single +/// gRPC call's timing can't be honestly disaggregated into per-query numbers. +#[derive(Debug, Clone)] +pub struct BatchOutcome { + pub latency: Duration, + pub ok: bool, + /// One entry per submitted query, in the same order as the input + /// `vectors` — the point ids that query actually returned, best-first. + /// `None` at a position means there's nothing meaningful to report for + /// that query: recall tracking wasn't on for this run + /// (`QdrantTarget::collect_ids` is `false`) or the whole dispatch failed + /// (`!ok`). `Some(vec![])` is a real, different thing — recall tracking + /// was on, the dispatch succeeded, and that query just matched nothing. + pub ids: Vec>>, + pub error: Option, +} + +/// A backend a storm sends queries to. `Display` is the name used in logs +/// (e.g. `qdrant(products)`). +/// +/// The method set here is the canonical Rust contract for a `nova storm` +/// backend and must match `methods:` in `contracts/nova-storm/v1.yaml`. +#[async_trait] +pub trait QueryTarget: Send + Sync + std::fmt::Display { + /// Fire one batch dispatch covering all of `vectors` in a single + /// round-trip and return its latency + outcome. The top-k / vector-name + /// knobs are baked into the target at construction, so the hot path is + /// just the vectors. A single-element slice is not a special case — it's + /// the default (`LoadProfile::batch_size == 1`). + async fn query_batch(&self, vectors: &[&[f32]]) -> BatchOutcome; + + /// Tear down connections. Default: nothing (clients close on drop). + async fn close(&self) -> Result<(), TargetError> { + Ok(()) + } +} diff --git a/crates/nova-storm/Cargo.toml b/backends/nova-storm/qdrant/Cargo.toml similarity index 55% rename from crates/nova-storm/Cargo.toml rename to backends/nova-storm/qdrant/Cargo.toml index 656e56a..4b6875c 100644 --- a/crates/nova-storm/Cargo.toml +++ b/backends/nova-storm/qdrant/Cargo.toml @@ -1,11 +1,21 @@ [package] -name = "nova-storm" +# Qdrant backend for `nova storm`. Built as the executable `nova-storm-qdrant`; +# the user-facing `nova-storm` shim (commands/nova-storm) execs it based on +# `target.type` in the config. The library crate keeps the historical name +# `nova_storm` (see `[lib]`) so no internal module paths change. +name = "nova-storm-qdrant" version = "0.0.8" edition = "2024" repository = "https://github.com/qdrant-labs/supernova" +# Keep the library name stable so `use nova_storm::...` in main.rs and the +# inline tests keeps working after the package rename. +[lib] +name = "nova_storm" +path = "src/lib.rs" + [[bin]] -name = "nova-storm" +name = "nova-storm-qdrant" path = "src/main.rs" # `cargo binstall nova-storm` fetches the prebuilt binary from the matching @@ -15,6 +25,9 @@ pkg-url = "{ repo }/releases/download/v{ version }/{ name }-{ target }{ binary-e pkg-fmt = "bin" [dependencies] +# Shared Rust interface this backend implements (the `QueryTarget` trait + +# `BatchOutcome`/`TargetError`). See backends/nova-storm/contracts/rust. +nova-storm-contract-rust = { path = "../contracts/rust" } async-trait = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/nova-storm/src/config.rs b/backends/nova-storm/qdrant/src/config.rs similarity index 100% rename from crates/nova-storm/src/config.rs rename to backends/nova-storm/qdrant/src/config.rs diff --git a/crates/nova-storm/src/errors.rs b/backends/nova-storm/qdrant/src/errors.rs similarity index 53% rename from crates/nova-storm/src/errors.rs rename to backends/nova-storm/qdrant/src/errors.rs index 7589a1d..8e72cd6 100644 --- a/crates/nova-storm/src/errors.rs +++ b/backends/nova-storm/qdrant/src/errors.rs @@ -3,27 +3,10 @@ //! Like `nova-load`, each layer has its own enum; [`StormError`] aggregates them //! for the binary. -/// Errors from a [`QueryTarget`](crate::targets::QueryTarget) backend. -/// -/// Backend client errors are boxed: `QdrantError` is large and an unboxed -/// variant would bloat every `Result`. Note this covers setup/teardown only — -/// a *query* failure during the load run is recorded as a non-fatal error -/// sample (see [`BatchOutcome`](crate::targets::BatchOutcome)), not surfaced here. -#[derive(Debug, thiserror::Error)] -pub enum TargetError { - #[error(transparent)] - Qdrant(Box), - - /// Backend-agnostic failure, e.g. a config the backend can't honour. - #[error("{0}")] - Other(String), -} - -impl From for TargetError { - fn from(e: qdrant_client::QdrantError) -> Self { - TargetError::Qdrant(Box::new(e)) - } -} +// `TargetError` is part of the backend-agnostic contract and lives in the +// shared `nova-storm-contract-rust` crate. Re-export it so `crate::errors:: +// TargetError` still resolves and `StormError`'s `#[from]` below keeps working. +pub use nova_storm_contract_rust::TargetError; /// Errors loading the query-vector set from parquet. #[derive(Debug, thiserror::Error)] diff --git a/crates/nova-storm/src/lib.rs b/backends/nova-storm/qdrant/src/lib.rs similarity index 100% rename from crates/nova-storm/src/lib.rs rename to backends/nova-storm/qdrant/src/lib.rs diff --git a/crates/nova-storm/src/main.rs b/backends/nova-storm/qdrant/src/main.rs similarity index 58% rename from crates/nova-storm/src/main.rs rename to backends/nova-storm/qdrant/src/main.rs index 9b51757..c7b66fb 100644 --- a/crates/nova-storm/src/main.rs +++ b/backends/nova-storm/qdrant/src/main.rs @@ -18,8 +18,41 @@ struct Cli { json: bool, } +/// The stable capabilities descriptor for this backend. Kept in lockstep with +/// `contracts/nova-storm/v1.yaml`: `commands`, `methods`, `search_modes`, and +/// `features` here are exactly what that contract declares required, and +/// `nova-contract check` fails if this drifts from the contract file. +/// +/// - `commands`: `run` is the default positional-config invocation +/// (`nova-storm-qdrant [--json]`); `capabilities` is this descriptor. +/// - `methods` mirror the `QueryTarget` trait in `targets/mod.rs`. +/// - `search_modes` mirror `runner.rs`'s closed-loop / open-loop modes. +fn capabilities_json() -> serde_json::Value { + serde_json::json!({ + "contract": "nova-storm-backend/v1", + "backend": "qdrant", + "commands": ["run", "capabilities"], + "methods": ["query_batch", "close"], + "search_modes": ["closed_loop", "open_loop"], + "features": ["recall", "percentiles", "exact_search", "hnsw_ef"] + }) +} + #[tokio::main] async fn main() -> ExitCode { + // `capabilities` is a pure descriptor handled before clap so the legacy + // `nova-storm [--json]` positional CLI is untouched. Accepts an + // optional `--json` (output is always JSON regardless). + let raw: Vec = std::env::args().skip(1).collect(); + if raw.first().map(String::as_str) == Some("capabilities") { + println!( + "{}", + serde_json::to_string_pretty(&capabilities_json()) + .expect("capabilities descriptor is always serializable") + ); + return ExitCode::SUCCESS; + } + tracing_subscriber::fmt() .with_env_filter( tracing_subscriber::EnvFilter::try_from_default_env() diff --git a/crates/nova-storm/src/queries.rs b/backends/nova-storm/qdrant/src/queries.rs similarity index 100% rename from crates/nova-storm/src/queries.rs rename to backends/nova-storm/qdrant/src/queries.rs diff --git a/crates/nova-storm/src/runner.rs b/backends/nova-storm/qdrant/src/runner.rs similarity index 100% rename from crates/nova-storm/src/runner.rs rename to backends/nova-storm/qdrant/src/runner.rs diff --git a/backends/nova-storm/qdrant/src/targets/mod.rs b/backends/nova-storm/qdrant/src/targets/mod.rs new file mode 100644 index 0000000..eba2608 --- /dev/null +++ b/backends/nova-storm/qdrant/src/targets/mod.rs @@ -0,0 +1,44 @@ +//! Query targets — the backends a storm fires at. +//! +//! A [`QueryTarget`] is a thin adapter: "fire one batch dispatch of nearest- +//! neighbour queries and report its latency." A batch of 1 is not a special +//! case — it's just the default. The load *shape* (concurrency, duration, +//! rate, batch size) lives in the [runner](crate::runner), so a backend stays +//! minimal and the same runner drives any store. Targets are built once and +//! shared across every concurrent request via `Arc`, so the trait is +//! `Send + Sync` (the gRPC client multiplexes concurrent calls over one +//! connection). + +use std::sync::Arc; + +use serde::Deserialize; + +use crate::config::QueryConfig; +use crate::errors::TargetError; + +// The backend-agnostic contract (trait + `BatchOutcome`) lives in the shared +// Rust interface crate `nova-storm-contract-rust`. Re-export it so the rest of +// this backend keeps referring to `crate::targets::{QueryTarget, BatchOutcome}` +// unchanged, and so the qdrant module implements exactly that trait. +pub use nova_storm_contract_rust::{BatchOutcome, QueryTarget}; + +pub mod qdrant; + +/// Target backend config, dispatched on `type:`. Each backend owns its config +/// struct in its own module. This dispatch enum names concrete backends, so it +/// stays here in the backend crate rather than in the neutral contract. +#[derive(Debug, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum TargetConfig { + Qdrant(qdrant::QdrantConfig), +} + +impl TargetConfig { + /// Connect and build the shared target. `query` carries the vector name and + /// top-k the backend bakes in. + pub fn into_target(self, query: &QueryConfig) -> Result, TargetError> { + match self { + TargetConfig::Qdrant(c) => Ok(Arc::new(c.into_target(query)?)), + } + } +} diff --git a/crates/nova-storm/src/targets/qdrant.rs b/backends/nova-storm/qdrant/src/targets/qdrant.rs similarity index 99% rename from crates/nova-storm/src/targets/qdrant.rs rename to backends/nova-storm/qdrant/src/targets/qdrant.rs index 5448f9c..8d3044b 100644 --- a/crates/nova-storm/src/targets/qdrant.rs +++ b/backends/nova-storm/qdrant/src/targets/qdrant.rs @@ -74,7 +74,7 @@ impl QdrantConfig { if let Some(key) = self.api_key { builder = builder.api_key(key); } - let client = builder.build()?; + let client = builder.build().map_err(TargetError::backend)?; Ok(QdrantTarget { client, diff --git a/commands/README.md b/commands/README.md new file mode 100644 index 0000000..59570e7 --- /dev/null +++ b/commands/README.md @@ -0,0 +1,85 @@ +# supernova — commands + +Every user-facing `nova-*` command lives here, **organized by command, not by +language**. A command may be a Rust binary or a Python console script; the +git-style `nova` dispatcher treats them identically. Commands are not +code-coupled across the language boundary — they meet only at runtime through +the dispatcher and shared data contracts (the YAML config, parquet/point +formats). Backend implementations behind the load/storm command contracts live +one level up, in `../backends`. + +## The dispatcher model + +`nova [args...]` finds an executable named `nova-` on `PATH` and +replaces itself with it (`os.execv`). A command can be implemented in any +language: + +- **Rust** — a binary like `nova-load`, installed by `cargo install`. +- **Python** — a console script like `nova-embed`, installed by `pip`/`uv`. + +The dispatcher doesn't know or care which. To add a command, just put a +`nova-` executable on `PATH`. + +## Commands + +| Command | Language | Provides | Install | +|----------------|----------|--------------------------------|------------------------------------------------| +| `nova` (root) | Python | the dispatcher | `uv pip install -e .` (repo root) | +| `nova-load` | Rust | shim → `nova-load-qdrant` | `cargo install --path commands/nova-load` | +| `nova-storm` | Rust | shim → `nova-storm-qdrant` | `cargo install --path commands/nova-storm` | +| `nova-contract`| Rust | backend conformance checker | `cargo install --path commands/nova-contract` | +| `nova-inspect` | Rust | parquet schema / vector count | `cargo install --path commands/nova-inspect` | +| `nova-embed` | Python | embedding pipeline (heavy ML) | `uv pip install -e 'commands/nova-embed[embed]'` | +| `nova-bf` | Python | brute-force ground truth | `uv pip install -e 'commands/nova-bf[compute]'` | +| `nova-opt` | Python | cost/recall BO tuner (WIP) | `uv pip install -e commands/nova-opt` | +| `nova-sweep` | Python | parameter-sweep orchestrator | `uv pip install -e commands/nova-sweep` | +| `nova-dist` | Python | SkyPilot fleet orchestration | `uv pip install -e commands/nova-dist` | + +The **dispatcher lives at the repo root** (`pyproject.toml` + `src/cli/`), so +`uv pip install -e .` from the root installs `nova`. It's the project's front +door and the spine of the polyglot tool, so it sits at the top rather than +buried as just-another-command. It's deliberately dependency-free. Every other +command is installed only where needed — like `git-*` subcommands. (`nova-opt` +is a work-in-progress tracked on another branch; it may be absent from a given +checkout.) + +Each `Makefile` target installs one command (`make embed`, `make bf`, …); +`make load`/`make storm` install both the shim and its Qdrant backend. See the +top-level `AGENTS.md` for the command/backend/contract architecture. + +## nova-embed + +Embedding generation (chunkers → embedders → storage), streamed from a dataset +source and written as parquet. Honors the same `--num-jobs` / `--job-rank` +distributed contract as `nova-load`: each rank computes its own `offset`/`limit` +slice of the dataset (from `--job-rank`, or `$SKYPILOT_JOB_RANK`). + +Config is validated with **pydantic** (`nova_embed.config`): `pipeline` knobs are +typed with defaults in one place, while `source`/`*_embedder`/`storage` carry a +`type` plus flexible backend-specific kwargs. `${VAR}` / `${VAR:-default}` +references are env-expanded, matching the Rust crates. + +The base package is light (pydantic, pyarrow, …); the actual ML stack (torch, +sentence-transformers, …) is the `embed` extra: + +```sh +uv pip install -e 'commands/nova-embed[embed]' +nova embed configs/embedder/test.yaml --num-jobs 50 --job-rank $SKYPILOT_JOB_RANK +nova embed configs/embedder/test.yaml --dry-run +``` + +## Dev setup + +```sh +uv pip install -e . # the `nova` dispatcher (from repo root) +uv pip install -e commands/nova-embed # embedding command (heavy) +cargo install --path commands/nova-load # the `nova-load` shim +cargo install --path backends/nova-load/qdrant # the `nova-load-qdrant` backend + +nova --help # lists discovered nova-* commands +nova load inspect configs/loader/test.yaml +nova embed ... +``` + +> Ensure your Python user-scripts dir (e.g. `~/.local/bin` or +> `~/Library/Python/X.Y/bin`) and `~/.cargo/bin` are on `PATH`. diff --git a/python/nova-bf/pyproject.toml b/commands/nova-bf/pyproject.toml similarity index 100% rename from python/nova-bf/pyproject.toml rename to commands/nova-bf/pyproject.toml diff --git a/python/nova-bf/src/nova_bf/__init__.py b/commands/nova-bf/src/nova_bf/__init__.py similarity index 100% rename from python/nova-bf/src/nova_bf/__init__.py rename to commands/nova-bf/src/nova_bf/__init__.py diff --git a/python/nova-bf/src/nova_bf/cli.py b/commands/nova-bf/src/nova_bf/cli.py similarity index 100% rename from python/nova-bf/src/nova_bf/cli.py rename to commands/nova-bf/src/nova_bf/cli.py diff --git a/python/nova-bf/src/nova_bf/compute.py b/commands/nova-bf/src/nova_bf/compute.py similarity index 100% rename from python/nova-bf/src/nova_bf/compute.py rename to commands/nova-bf/src/nova_bf/compute.py diff --git a/python/nova-bf/src/nova_bf/config.py b/commands/nova-bf/src/nova_bf/config.py similarity index 100% rename from python/nova-bf/src/nova_bf/config.py rename to commands/nova-bf/src/nova_bf/config.py diff --git a/python/nova-bf/src/nova_bf/dates.py b/commands/nova-bf/src/nova_bf/dates.py similarity index 100% rename from python/nova-bf/src/nova_bf/dates.py rename to commands/nova-bf/src/nova_bf/dates.py diff --git a/python/nova-bf/src/nova_bf/filters.py b/commands/nova-bf/src/nova_bf/filters.py similarity index 100% rename from python/nova-bf/src/nova_bf/filters.py rename to commands/nova-bf/src/nova_bf/filters.py diff --git a/python/nova-bf/src/nova_bf/ids.py b/commands/nova-bf/src/nova_bf/ids.py similarity index 100% rename from python/nova-bf/src/nova_bf/ids.py rename to commands/nova-bf/src/nova_bf/ids.py diff --git a/python/nova-bf/src/nova_bf/io.py b/commands/nova-bf/src/nova_bf/io.py similarity index 100% rename from python/nova-bf/src/nova_bf/io.py rename to commands/nova-bf/src/nova_bf/io.py diff --git a/python/nova-bf/src/nova_bf/merge.py b/commands/nova-bf/src/nova_bf/merge.py similarity index 100% rename from python/nova-bf/src/nova_bf/merge.py rename to commands/nova-bf/src/nova_bf/merge.py diff --git a/python/nova-bf/src/nova_bf/results.py b/commands/nova-bf/src/nova_bf/results.py similarity index 100% rename from python/nova-bf/src/nova_bf/results.py rename to commands/nova-bf/src/nova_bf/results.py diff --git a/python/nova-bf/src/nova_bf/tokenize.py b/commands/nova-bf/src/nova_bf/tokenize.py similarity index 100% rename from python/nova-bf/src/nova_bf/tokenize.py rename to commands/nova-bf/src/nova_bf/tokenize.py diff --git a/python/nova-bf/tests/test_compute.py b/commands/nova-bf/tests/test_compute.py similarity index 100% rename from python/nova-bf/tests/test_compute.py rename to commands/nova-bf/tests/test_compute.py diff --git a/python/nova-bf/tests/test_compute_dates.py b/commands/nova-bf/tests/test_compute_dates.py similarity index 100% rename from python/nova-bf/tests/test_compute_dates.py rename to commands/nova-bf/tests/test_compute_dates.py diff --git a/python/nova-bf/tests/test_compute_multi.py b/commands/nova-bf/tests/test_compute_multi.py similarity index 100% rename from python/nova-bf/tests/test_compute_multi.py rename to commands/nova-bf/tests/test_compute_multi.py diff --git a/python/nova-bf/tests/test_compute_sparse.py b/commands/nova-bf/tests/test_compute_sparse.py similarity index 100% rename from python/nova-bf/tests/test_compute_sparse.py rename to commands/nova-bf/tests/test_compute_sparse.py diff --git a/python/nova-bf/tests/test_config_dates.py b/commands/nova-bf/tests/test_config_dates.py similarity index 100% rename from python/nova-bf/tests/test_config_dates.py rename to commands/nova-bf/tests/test_config_dates.py diff --git a/python/nova-bf/tests/test_config_searches.py b/commands/nova-bf/tests/test_config_searches.py similarity index 100% rename from python/nova-bf/tests/test_config_searches.py rename to commands/nova-bf/tests/test_config_searches.py diff --git a/python/nova-bf/tests/test_dates.py b/commands/nova-bf/tests/test_dates.py similarity index 100% rename from python/nova-bf/tests/test_dates.py rename to commands/nova-bf/tests/test_dates.py diff --git a/python/nova-bf/tests/test_filter_optimizations.py b/commands/nova-bf/tests/test_filter_optimizations.py similarity index 100% rename from python/nova-bf/tests/test_filter_optimizations.py rename to commands/nova-bf/tests/test_filter_optimizations.py diff --git a/python/nova-bf/tests/test_filters.py b/commands/nova-bf/tests/test_filters.py similarity index 100% rename from python/nova-bf/tests/test_filters.py rename to commands/nova-bf/tests/test_filters.py diff --git a/python/nova-bf/tests/test_merge.py b/commands/nova-bf/tests/test_merge.py similarity index 100% rename from python/nova-bf/tests/test_merge.py rename to commands/nova-bf/tests/test_merge.py diff --git a/python/nova-bf/tests/test_qdrant_datetime_parity.py b/commands/nova-bf/tests/test_qdrant_datetime_parity.py similarity index 100% rename from python/nova-bf/tests/test_qdrant_datetime_parity.py rename to commands/nova-bf/tests/test_qdrant_datetime_parity.py diff --git a/python/nova-bf/uv.lock b/commands/nova-bf/uv.lock similarity index 100% rename from python/nova-bf/uv.lock rename to commands/nova-bf/uv.lock diff --git a/commands/nova-contract/Cargo.toml b/commands/nova-contract/Cargo.toml new file mode 100644 index 0000000..8ef1e3b --- /dev/null +++ b/commands/nova-contract/Cargo.toml @@ -0,0 +1,24 @@ +[package] +# `nova contract` — a generic, language-neutral conformance checker. Runs a +# backend executable's `capabilities --json` and validates it against a shared +# contract spec (contracts//vN.yaml). This is the cross-language half +# of the two-layer contract model: the native trait gives compile-time checking +# within one language; this gives runtime conformance across any language. +name = "nova-contract" +version = "0.0.8" +edition = "2024" +repository = "https://github.com/qdrant-labs/supernova" + +[[bin]] +name = "nova-contract" +path = "src/main.rs" + +[package.metadata.binstall] +pkg-url = "{ repo }/releases/download/v{ version }/{ name }-{ target }{ binary-ext }" +pkg-fmt = "bin" + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +serde_yaml = { workspace = true } +clap = { workspace = true } diff --git a/commands/nova-contract/src/main.rs b/commands/nova-contract/src/main.rs new file mode 100644 index 0000000..71662fd --- /dev/null +++ b/commands/nova-contract/src/main.rs @@ -0,0 +1,349 @@ +//! `nova-contract` — a generic, language-neutral backend conformance checker. +//! +//! Usage: +//! nova contract check --contract [--level ...] [--json] +//! nova-contract check --contract ... +//! +//! It runs the backend's `capabilities --json`, then validates that descriptor +//! against a shared contract spec (`contracts//vN.yaml`). This is the +//! cross-language half of the two-layer model: the per-language interface crate +//! (`backends//contracts//`) enforces the contract at compile +//! time within one language; this checker enforces it at runtime for *any* +//! backend, whatever language it's written in, by only ever talking to the +//! backend's executable. + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::process::{Command, ExitCode}; + +use clap::{Args, Parser, Subcommand, ValueEnum}; +use serde::Deserialize; + +#[derive(Debug, Parser)] +#[command(name = "nova-contract", version, about = "Language-neutral backend contract checker")] +struct Cli { + #[command(subcommand)] + command: Cmd, +} + +#[derive(Debug, Subcommand)] +enum Cmd { + /// Check a backend executable against a contract spec. + Check(CheckArgs), +} + +#[derive(Debug, Args)] +struct CheckArgs { + /// The backend executable to check: a path (e.g. `target/debug/nova-load-qdrant`) + /// or a name resolved on `PATH` (e.g. `nova-load-qdrant`). NOTE: point this at + /// a *backend*, not a user-facing shim like `nova-load`. + backend: String, + /// Path to the language-neutral contract YAML (e.g. contracts/nova-load/v1.yaml). + #[arg(long)] + contract: PathBuf, + /// How hard to check. `shape`: capabilities vs contract only. `dry-run` + /// (default): + behavioral checks needing no live backend. `live`: + run a + /// declared live check (requires `--config`). + #[arg(long, value_enum, default_value_t = Level::DryRun)] + level: Level, + /// Directory of conformance fixtures (reserved for fixture-driven checks). + #[arg(long)] + fixtures: Option, + /// Config file for `--level live` (substituted into the contract's live_check). + #[arg(long)] + config: Option, + /// Emit the report as a single JSON object instead of a human-readable table. + #[arg(long)] + json: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] +enum Level { + Shape, + #[value(name = "dry-run")] + DryRun, + Live, +} + +/// The language-neutral contract spec. Unknown keys (description, behavior +/// notes, etc.) are ignored on purpose — they're human docs, not machine rules. +#[derive(Debug, Deserialize)] +struct Contract { + #[allow(dead_code)] + id: String, + #[allow(dead_code)] + version: u32, + /// The full `id/version` string a conforming backend must advertise. + contract: String, + #[serde(default)] + required_commands: Vec, + #[serde(default)] + required_methods: Vec, + #[serde(default)] + required_vector_kinds: Vec, + #[serde(default)] + required_point_id_types: Vec, + #[serde(default)] + required_search_modes: Vec, + #[serde(default)] + required_features: Vec, + #[serde(default)] + required_flags: BTreeMap>, + #[serde(default)] + live_check: Option, +} + +/// A backend command the `live` level runs against a real target. `{config}` in +/// `args` is replaced by the `--config` path. +#[derive(Debug, Deserialize)] +struct LiveCheck { + args: Vec, +} + +/// One validation result. +struct Check { + name: &'static str, + ok: bool, + detail: String, +} + +impl Check { + fn pass(name: &'static str, detail: impl Into) -> Self { + Check { name, ok: true, detail: detail.into() } + } + fn fail(name: &'static str, detail: impl Into) -> Self { + Check { name, ok: false, detail: detail.into() } + } +} + +fn main() -> ExitCode { + match Cli::parse().command { + Cmd::Check(args) => check(args), + } +} + +fn check(args: CheckArgs) -> ExitCode { + let contract = match load_contract(&args.contract) { + Ok(c) => c, + Err(e) => { + eprintln!("nova-contract: {e}"); + return ExitCode::FAILURE; + } + }; + + let mut checks: Vec = Vec::new(); + + // --- Run `capabilities --json` (the one thing every backend must do). --- + let caps_json = match run_capabilities(&args.backend) { + Ok((true, stdout, _)) => { + checks.push(Check::pass("capabilities-runs", "`capabilities --json` exited 0")); + stdout + } + Ok((false, stdout, stderr)) => { + checks.push(Check::fail( + "capabilities-runs", + format!("`capabilities --json` exited non-zero; stderr: {}", stderr.trim()), + )); + stdout + } + Err(e) => { + // Can't even spawn the backend — report and stop. + checks.push(Check::fail("capabilities-runs", e)); + return report(&args, &contract, &checks); + } + }; + + // --- Parse the descriptor. --- + let caps: serde_json::Value = match serde_json::from_str(&caps_json) { + Ok(v) => { + checks.push(Check::pass("capabilities-json-parses", "valid JSON")); + v + } + Err(e) => { + checks.push(Check::fail("capabilities-json-parses", format!("invalid JSON: {e}"))); + return report(&args, &contract, &checks); + } + }; + + // --- SHAPE checks: descriptor vs contract. --- + shape_checks(&contract, &caps, &mut checks); + + // --- DRY-RUN checks: cheap behavioral checks, no live backend. --- + if args.level >= Level::DryRun { + dry_run_checks(&args.backend, &caps_json, &mut checks); + } + + // --- LIVE checks: run a declared command against a real target. --- + if args.level == Level::Live { + live_checks(&args, &contract, &mut checks); + } + + report(&args, &contract, &checks) +} + +fn shape_checks(contract: &Contract, caps: &serde_json::Value, checks: &mut Vec) { + // Contract id/version must match exactly. + let advertised = caps.get("contract").and_then(|v| v.as_str()).unwrap_or(""); + if advertised == contract.contract { + checks.push(Check::pass("contract-id-matches", format!("`{advertised}`"))); + } else { + checks.push(Check::fail( + "contract-id-matches", + format!("backend advertises `{advertised}`, contract expects `{}`", contract.contract), + )); + } + + subset_check("commands", &contract.required_commands, caps, "commands", checks); + subset_check("methods", &contract.required_methods, caps, "methods", checks); + subset_check("vector-kinds", &contract.required_vector_kinds, caps, "vector_kinds", checks); + subset_check("point-id-types", &contract.required_point_id_types, caps, "point_id_types", checks); + subset_check("search-modes", &contract.required_search_modes, caps, "search_modes", checks); + subset_check("features", &contract.required_features, caps, "features", checks); + + // Required flags: `flags` is a map of command -> advertised flags. + if !contract.required_flags.is_empty() { + let advertised_flags = caps.get("flags"); + for (cmd, required) in &contract.required_flags { + let have: Vec = advertised_flags + .and_then(|m| m.get(cmd)) + .map(json_str_array) + .unwrap_or_default(); + let missing: Vec<&String> = required.iter().filter(|f| !have.contains(f)).collect(); + let name: &'static str = Box::leak(format!("flags[{cmd}]").into_boxed_str()); + if missing.is_empty() { + checks.push(Check::pass(name, format!("advertises {required:?}"))); + } else { + checks.push(Check::fail(name, format!("missing {missing:?} (has {have:?})"))); + } + } + } +} + +/// Assert every entry in `required` appears in `caps[caps_key]`. +fn subset_check( + name: &'static str, + required: &[String], + caps: &serde_json::Value, + caps_key: &str, + checks: &mut Vec, +) { + if required.is_empty() { + return; // contract declares nothing here — skip silently + } + let have = caps.get(caps_key).map(json_str_array); + match have { + None => checks.push(Check::fail(name, format!("backend advertises no `{caps_key}`"))), + Some(have) => { + let missing: Vec<&String> = required.iter().filter(|r| !have.contains(r)).collect(); + if missing.is_empty() { + checks.push(Check::pass(name, format!("all {} present", required.len()))); + } else { + checks.push(Check::fail(name, format!("missing {missing:?}"))); + } + } + } +} + +fn dry_run_checks(backend: &str, first: &str, checks: &mut Vec) { + // capabilities must be deterministic — a moving descriptor can't be a + // contract. Run it again and compare byte-for-byte. + match run_capabilities(backend) { + Ok((_, second, _)) if second == first => { + checks.push(Check::pass("capabilities-deterministic", "identical across two runs")); + } + Ok((_, _, _)) => { + checks.push(Check::fail("capabilities-deterministic", "descriptor changed between runs")); + } + Err(e) => checks.push(Check::fail("capabilities-deterministic", e)), + } +} + +fn live_checks(args: &CheckArgs, contract: &Contract, checks: &mut Vec) { + let Some(live) = &contract.live_check else { + checks.push(Check::fail( + "live-check", + "contract declares no `live_check`; nothing to run at --level live", + )); + return; + }; + let Some(config) = &args.config else { + checks.push(Check::fail("live-check", "--level live requires --config ")); + return; + }; + // Substitute {config} in the declared args. + let subbed: Vec = live + .args + .iter() + .map(|a| a.replace("{config}", &config.display().to_string())) + .collect(); + match Command::new(&args.backend).args(&subbed).output() { + Ok(out) if out.status.success() => { + checks.push(Check::pass("live-check", format!("`{}` exited 0", subbed.join(" ")))); + } + Ok(out) => checks.push(Check::fail( + "live-check", + format!( + "`{}` exited non-zero; stderr: {}", + subbed.join(" "), + String::from_utf8_lossy(&out.stderr).trim() + ), + )), + Err(e) => checks.push(Check::fail("live-check", format!("failed to spawn: {e}"))), + } +} + +/// Read ` capabilities --json`. Returns (exit-ok, stdout, stderr). +fn run_capabilities(backend: &str) -> Result<(bool, String, String), String> { + let out = Command::new(backend) + .arg("capabilities") + .arg("--json") + .output() + .map_err(|e| format!("failed to spawn `{backend} capabilities --json`: {e}"))?; + Ok(( + out.status.success(), + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + )) +} + +fn json_str_array(v: &serde_json::Value) -> Vec { + v.as_array() + .map(|a| a.iter().filter_map(|x| x.as_str().map(str::to_string)).collect()) + .unwrap_or_default() +} + +fn load_contract(path: &PathBuf) -> Result { + let text = std::fs::read_to_string(path) + .map_err(|e| format!("failed to read contract `{}`: {e}", path.display()))?; + serde_yaml::from_str(&text) + .map_err(|e| format!("failed to parse contract `{}`: {e}", path.display())) +} + +/// Print the report (human or JSON) and return the process exit code. +fn report(args: &CheckArgs, contract: &Contract, checks: &[Check]) -> ExitCode { + let ok = checks.iter().all(|c| c.ok); + + if args.json { + let arr: Vec = checks + .iter() + .map(|c| serde_json::json!({ "name": c.name, "ok": c.ok, "detail": c.detail })) + .collect(); + let obj = serde_json::json!({ + "backend": args.backend, + "contract": contract.contract, + "level": format!("{:?}", args.level).to_lowercase(), + "ok": ok, + "checks": arr, + }); + println!("{}", serde_json::to_string_pretty(&obj).expect("report serializes")); + } else { + println!("contract check: {} vs {}", args.backend, contract.contract); + for c in checks { + let mark = if c.ok { "✓" } else { "✗" }; + println!(" {mark} {:<26} {}", c.name, c.detail); + } + println!("{}", if ok { "PASS" } else { "FAIL" }); + } + + if ok { ExitCode::SUCCESS } else { ExitCode::FAILURE } +} diff --git a/python/nova-dist/pyproject.toml b/commands/nova-dist/pyproject.toml similarity index 100% rename from python/nova-dist/pyproject.toml rename to commands/nova-dist/pyproject.toml diff --git a/python/nova-dist/src/nova_dist/__init__.py b/commands/nova-dist/src/nova_dist/__init__.py similarity index 100% rename from python/nova-dist/src/nova_dist/__init__.py rename to commands/nova-dist/src/nova_dist/__init__.py diff --git a/python/nova-dist/src/nova_dist/cli.py b/commands/nova-dist/src/nova_dist/cli.py similarity index 100% rename from python/nova-dist/src/nova_dist/cli.py rename to commands/nova-dist/src/nova_dist/cli.py diff --git a/python/nova-dist/src/nova_dist/sky.py b/commands/nova-dist/src/nova_dist/sky.py similarity index 93% rename from python/nova-dist/src/nova_dist/sky.py rename to commands/nova-dist/src/nova_dist/sky.py index 5edb375..53ad951 100644 --- a/python/nova-dist/src/nova_dist/sky.py +++ b/commands/nova-dist/src/nova_dist/sky.py @@ -69,20 +69,26 @@ def forward_env(config_path: str, extra: list[str] | None = None) -> dict[str, s _REPO = "https://github.com/qdrant-labs/supernova" -def _rust_worker_setup(binary: str) -> str: +def _rust_worker_setup(*binaries: str) -> str: """ - Install a prebuilt Rust binary on a worker (seconds), falling back to a - source compile if no release asset matches the arch (e.g. before the first - release). Installs to `/usr/local/bin` so it's on `PATH` in both SkyPilot's - `setup` and `run` shells (they're separate). + Install one or more prebuilt Rust binaries on a worker (seconds each), + falling back to a source compile if no release asset matches the arch (e.g. + before the first release). Installs to `/usr/local/bin` so they're on `PATH` + in both SkyPilot's `setup` and `run` shells (they're separate). + + A `nova ` is now a thin shim (`nova-`) that execs a backend + executable (`nova--`), so a worker needs BOTH — pass both + (e.g. `nova-load` and `nova-load-qdrant`). """ - return ( + header = ( "set -e\n" 'case "$(uname -m)" in\n' " x86_64) t=x86_64-unknown-linux-gnu ;;\n" " aarch64|arm64) t=aarch64-unknown-linux-gnu ;;\n" " *) t= ;;\n" "esac\n" + ) + blocks = [ f'if [ -n "$t" ] && curl -fsSL "{_REPO}/releases/latest/download/{binary}-$t" -o /tmp/{binary}; then\n' f" sudo install -m 0755 /tmp/{binary} /usr/local/bin/{binary}\n" "else\n" @@ -92,7 +98,9 @@ def _rust_worker_setup(binary: str) -> str: f" cargo install --git {_REPO} {binary}\n" f' sudo install -m 0755 "$HOME/.cargo/bin/{binary}" /usr/local/bin/{binary}\n' "fi" - ) + for binary in binaries + ] + return header + "\n".join(blocks) def _python_worker_setup(binary: str, pip_spec: str) -> str: @@ -153,17 +161,17 @@ def _python_worker_setup(binary: str, pip_spec: str) -> str: }, "setup": _python_worker_setup( "nova-embed", - f"nova-embed[embed] @ git+{_REPO}@master#subdirectory=python/nova-embed", + f"nova-embed[embed] @ git+{_REPO}@master#subdirectory=commands/nova-embed", ), "envs": {"HF_HUB_ENABLE_HF_TRANSFER": "1"}, }, "load": { "resources": {"cloud": "aws", "cpus": "8+", "use_spot": False, "disk_size": 100}, - "setup": _rust_worker_setup("nova-load"), + "setup": _rust_worker_setup("nova-load", "nova-load-qdrant"), }, "storm": { "resources": {"cloud": "aws", "cpus": "4+", "use_spot": False}, - "setup": _rust_worker_setup("nova-storm"), + "setup": _rust_worker_setup("nova-storm", "nova-storm-qdrant"), }, "bf": { # GPU brute force on the same real-GPU AMI as embed (see _GPU_IMAGE note). @@ -176,7 +184,7 @@ def _python_worker_setup(binary: str, pip_spec: str) -> str: }, "setup": _python_worker_setup( "nova-bf", - f"nova-bf[compute] @ git+{_REPO}@master#subdirectory=python/nova-bf", + f"nova-bf[compute] @ git+{_REPO}@master#subdirectory=commands/nova-bf", ), }, } diff --git a/python/nova-embed/pyproject.toml b/commands/nova-embed/pyproject.toml similarity index 100% rename from python/nova-embed/pyproject.toml rename to commands/nova-embed/pyproject.toml diff --git a/python/nova-embed/src/nova_embed/__init__.py b/commands/nova-embed/src/nova_embed/__init__.py similarity index 100% rename from python/nova-embed/src/nova_embed/__init__.py rename to commands/nova-embed/src/nova_embed/__init__.py diff --git a/python/nova-embed/src/nova_embed/chunkers/__init__.py b/commands/nova-embed/src/nova_embed/chunkers/__init__.py similarity index 100% rename from python/nova-embed/src/nova_embed/chunkers/__init__.py rename to commands/nova-embed/src/nova_embed/chunkers/__init__.py diff --git a/python/nova-embed/src/nova_embed/chunkers/base.py b/commands/nova-embed/src/nova_embed/chunkers/base.py similarity index 100% rename from python/nova-embed/src/nova_embed/chunkers/base.py rename to commands/nova-embed/src/nova_embed/chunkers/base.py diff --git a/python/nova-embed/src/nova_embed/chunkers/fixed_char.py b/commands/nova-embed/src/nova_embed/chunkers/fixed_char.py similarity index 100% rename from python/nova-embed/src/nova_embed/chunkers/fixed_char.py rename to commands/nova-embed/src/nova_embed/chunkers/fixed_char.py diff --git a/python/nova-embed/src/nova_embed/chunkers/passthrough.py b/commands/nova-embed/src/nova_embed/chunkers/passthrough.py similarity index 100% rename from python/nova-embed/src/nova_embed/chunkers/passthrough.py rename to commands/nova-embed/src/nova_embed/chunkers/passthrough.py diff --git a/python/nova-embed/src/nova_embed/chunkers/semantic.py b/commands/nova-embed/src/nova_embed/chunkers/semantic.py similarity index 100% rename from python/nova-embed/src/nova_embed/chunkers/semantic.py rename to commands/nova-embed/src/nova_embed/chunkers/semantic.py diff --git a/python/nova-embed/src/nova_embed/cli/__init__.py b/commands/nova-embed/src/nova_embed/cli/__init__.py similarity index 100% rename from python/nova-embed/src/nova_embed/cli/__init__.py rename to commands/nova-embed/src/nova_embed/cli/__init__.py diff --git a/python/nova-embed/src/nova_embed/cli/run_embedder.py b/commands/nova-embed/src/nova_embed/cli/run_embedder.py similarity index 100% rename from python/nova-embed/src/nova_embed/cli/run_embedder.py rename to commands/nova-embed/src/nova_embed/cli/run_embedder.py diff --git a/python/nova-embed/src/nova_embed/config.py b/commands/nova-embed/src/nova_embed/config.py similarity index 100% rename from python/nova-embed/src/nova_embed/config.py rename to commands/nova-embed/src/nova_embed/config.py diff --git a/python/nova-embed/src/nova_embed/embedders/__init__.py b/commands/nova-embed/src/nova_embed/embedders/__init__.py similarity index 100% rename from python/nova-embed/src/nova_embed/embedders/__init__.py rename to commands/nova-embed/src/nova_embed/embedders/__init__.py diff --git a/python/nova-embed/src/nova_embed/embedders/backends/__init__.py b/commands/nova-embed/src/nova_embed/embedders/backends/__init__.py similarity index 100% rename from python/nova-embed/src/nova_embed/embedders/backends/__init__.py rename to commands/nova-embed/src/nova_embed/embedders/backends/__init__.py diff --git a/python/nova-embed/src/nova_embed/embedders/backends/bge_m3.py b/commands/nova-embed/src/nova_embed/embedders/backends/bge_m3.py similarity index 100% rename from python/nova-embed/src/nova_embed/embedders/backends/bge_m3.py rename to commands/nova-embed/src/nova_embed/embedders/backends/bge_m3.py diff --git a/python/nova-embed/src/nova_embed/embedders/backends/device.py b/commands/nova-embed/src/nova_embed/embedders/backends/device.py similarity index 100% rename from python/nova-embed/src/nova_embed/embedders/backends/device.py rename to commands/nova-embed/src/nova_embed/embedders/backends/device.py diff --git a/python/nova-embed/src/nova_embed/embedders/backends/fastembed.py b/commands/nova-embed/src/nova_embed/embedders/backends/fastembed.py similarity index 100% rename from python/nova-embed/src/nova_embed/embedders/backends/fastembed.py rename to commands/nova-embed/src/nova_embed/embedders/backends/fastembed.py diff --git a/python/nova-embed/src/nova_embed/embedders/backends/openai.py b/commands/nova-embed/src/nova_embed/embedders/backends/openai.py similarity index 100% rename from python/nova-embed/src/nova_embed/embedders/backends/openai.py rename to commands/nova-embed/src/nova_embed/embedders/backends/openai.py diff --git a/python/nova-embed/src/nova_embed/embedders/backends/sentence_transformer.py b/commands/nova-embed/src/nova_embed/embedders/backends/sentence_transformer.py similarity index 100% rename from python/nova-embed/src/nova_embed/embedders/backends/sentence_transformer.py rename to commands/nova-embed/src/nova_embed/embedders/backends/sentence_transformer.py diff --git a/python/nova-embed/src/nova_embed/embedders/backends/vllm.py b/commands/nova-embed/src/nova_embed/embedders/backends/vllm.py similarity index 100% rename from python/nova-embed/src/nova_embed/embedders/backends/vllm.py rename to commands/nova-embed/src/nova_embed/embedders/backends/vllm.py diff --git a/python/nova-embed/src/nova_embed/embedders/base.py b/commands/nova-embed/src/nova_embed/embedders/base.py similarity index 100% rename from python/nova-embed/src/nova_embed/embedders/base.py rename to commands/nova-embed/src/nova_embed/embedders/base.py diff --git a/python/nova-embed/src/nova_embed/embedders/buffer.py b/commands/nova-embed/src/nova_embed/embedders/buffer.py similarity index 100% rename from python/nova-embed/src/nova_embed/embedders/buffer.py rename to commands/nova-embed/src/nova_embed/embedders/buffer.py diff --git a/python/nova-embed/src/nova_embed/embedders/engine.py b/commands/nova-embed/src/nova_embed/embedders/engine.py similarity index 100% rename from python/nova-embed/src/nova_embed/embedders/engine.py rename to commands/nova-embed/src/nova_embed/embedders/engine.py diff --git a/python/nova-embed/src/nova_embed/embedders/runner.py b/commands/nova-embed/src/nova_embed/embedders/runner.py similarity index 100% rename from python/nova-embed/src/nova_embed/embedders/runner.py rename to commands/nova-embed/src/nova_embed/embedders/runner.py diff --git a/python/nova-embed/src/nova_embed/embedders/worker.py b/commands/nova-embed/src/nova_embed/embedders/worker.py similarity index 100% rename from python/nova-embed/src/nova_embed/embedders/worker.py rename to commands/nova-embed/src/nova_embed/embedders/worker.py diff --git a/python/nova-embed/src/nova_embed/media/__init__.py b/commands/nova-embed/src/nova_embed/media/__init__.py similarity index 100% rename from python/nova-embed/src/nova_embed/media/__init__.py rename to commands/nova-embed/src/nova_embed/media/__init__.py diff --git a/python/nova-embed/src/nova_embed/media/image.py b/commands/nova-embed/src/nova_embed/media/image.py similarity index 100% rename from python/nova-embed/src/nova_embed/media/image.py rename to commands/nova-embed/src/nova_embed/media/image.py diff --git a/python/nova-embed/src/nova_embed/media/text.py b/commands/nova-embed/src/nova_embed/media/text.py similarity index 100% rename from python/nova-embed/src/nova_embed/media/text.py rename to commands/nova-embed/src/nova_embed/media/text.py diff --git a/python/nova-embed/src/nova_embed/models.py b/commands/nova-embed/src/nova_embed/models.py similarity index 100% rename from python/nova-embed/src/nova_embed/models.py rename to commands/nova-embed/src/nova_embed/models.py diff --git a/python/nova-embed/src/nova_embed/registry.py b/commands/nova-embed/src/nova_embed/registry.py similarity index 100% rename from python/nova-embed/src/nova_embed/registry.py rename to commands/nova-embed/src/nova_embed/registry.py diff --git a/python/nova-embed/src/nova_embed/sources/__init__.py b/commands/nova-embed/src/nova_embed/sources/__init__.py similarity index 100% rename from python/nova-embed/src/nova_embed/sources/__init__.py rename to commands/nova-embed/src/nova_embed/sources/__init__.py diff --git a/python/nova-embed/src/nova_embed/sources/base.py b/commands/nova-embed/src/nova_embed/sources/base.py similarity index 100% rename from python/nova-embed/src/nova_embed/sources/base.py rename to commands/nova-embed/src/nova_embed/sources/base.py diff --git a/python/nova-embed/src/nova_embed/sources/huggingface.py b/commands/nova-embed/src/nova_embed/sources/huggingface.py similarity index 100% rename from python/nova-embed/src/nova_embed/sources/huggingface.py rename to commands/nova-embed/src/nova_embed/sources/huggingface.py diff --git a/python/nova-embed/src/nova_embed/storage/__init__.py b/commands/nova-embed/src/nova_embed/storage/__init__.py similarity index 100% rename from python/nova-embed/src/nova_embed/storage/__init__.py rename to commands/nova-embed/src/nova_embed/storage/__init__.py diff --git a/python/nova-embed/src/nova_embed/storage/base.py b/commands/nova-embed/src/nova_embed/storage/base.py similarity index 100% rename from python/nova-embed/src/nova_embed/storage/base.py rename to commands/nova-embed/src/nova_embed/storage/base.py diff --git a/python/nova-embed/src/nova_embed/storage/huggingface.py b/commands/nova-embed/src/nova_embed/storage/huggingface.py similarity index 100% rename from python/nova-embed/src/nova_embed/storage/huggingface.py rename to commands/nova-embed/src/nova_embed/storage/huggingface.py diff --git a/python/nova-embed/src/nova_embed/storage/local.py b/commands/nova-embed/src/nova_embed/storage/local.py similarity index 100% rename from python/nova-embed/src/nova_embed/storage/local.py rename to commands/nova-embed/src/nova_embed/storage/local.py diff --git a/python/nova-embed/src/nova_embed/storage/object_store.py b/commands/nova-embed/src/nova_embed/storage/object_store.py similarity index 100% rename from python/nova-embed/src/nova_embed/storage/object_store.py rename to commands/nova-embed/src/nova_embed/storage/object_store.py diff --git a/python/nova-embed/src/nova_embed/storage/writer.py b/commands/nova-embed/src/nova_embed/storage/writer.py similarity index 100% rename from python/nova-embed/src/nova_embed/storage/writer.py rename to commands/nova-embed/src/nova_embed/storage/writer.py diff --git a/python/nova-embed/tests/fake_backends.py b/commands/nova-embed/tests/fake_backends.py similarity index 100% rename from python/nova-embed/tests/fake_backends.py rename to commands/nova-embed/tests/fake_backends.py diff --git a/python/nova-embed/tests/test_config.py b/commands/nova-embed/tests/test_config.py similarity index 100% rename from python/nova-embed/tests/test_config.py rename to commands/nova-embed/tests/test_config.py diff --git a/python/nova-embed/tests/test_engine.py b/commands/nova-embed/tests/test_engine.py similarity index 100% rename from python/nova-embed/tests/test_engine.py rename to commands/nova-embed/tests/test_engine.py diff --git a/python/nova-embed/tests/test_media.py b/commands/nova-embed/tests/test_media.py similarity index 100% rename from python/nova-embed/tests/test_media.py rename to commands/nova-embed/tests/test_media.py diff --git a/python/nova-embed/tests/test_pipeline.py b/commands/nova-embed/tests/test_pipeline.py similarity index 100% rename from python/nova-embed/tests/test_pipeline.py rename to commands/nova-embed/tests/test_pipeline.py diff --git a/python/nova-embed/tests/test_storage.py b/commands/nova-embed/tests/test_storage.py similarity index 100% rename from python/nova-embed/tests/test_storage.py rename to commands/nova-embed/tests/test_storage.py diff --git a/python/nova-embed/uv.lock b/commands/nova-embed/uv.lock similarity index 100% rename from python/nova-embed/uv.lock rename to commands/nova-embed/uv.lock diff --git a/crates/nova-inspect/Cargo.toml b/commands/nova-inspect/Cargo.toml similarity index 100% rename from crates/nova-inspect/Cargo.toml rename to commands/nova-inspect/Cargo.toml diff --git a/crates/nova-inspect/src/main.rs b/commands/nova-inspect/src/main.rs similarity index 100% rename from crates/nova-inspect/src/main.rs rename to commands/nova-inspect/src/main.rs diff --git a/commands/nova-load/Cargo.toml b/commands/nova-load/Cargo.toml new file mode 100644 index 0000000..0b5fdb0 --- /dev/null +++ b/commands/nova-load/Cargo.toml @@ -0,0 +1,24 @@ +[package] +# The user-facing `nova load` command. This is a thin front controller: it +# reads only `vectorstore.type` from the config, then `exec`s the matching +# backend executable (`nova-load-`, e.g. `nova-load-qdrant`) with the +# original args unchanged. The real work lives in backends/nova-load/*. +name = "nova-load" +version = "0.0.8" +edition = "2024" +repository = "https://github.com/qdrant-labs/supernova" + +[[bin]] +name = "nova-load" +path = "src/main.rs" + +# `cargo binstall nova-load` fetches the prebuilt shim from each `v*` release +# (see .github/workflows/rust-binaries.yml). Matches the asset naming there. +[package.metadata.binstall] +pkg-url = "{ repo }/releases/download/v{ version }/{ name }-{ target }{ binary-ext }" +pkg-fmt = "bin" + +[dependencies] +# Shared shim front-controller logic (config-type dispatch + exec). See +# commands/nova-shim; keeps this shim identical to the nova-storm shim. +nova-shim = { path = "../nova-shim" } diff --git a/commands/nova-load/src/main.rs b/commands/nova-load/src/main.rs new file mode 100644 index 0000000..08cc350 --- /dev/null +++ b/commands/nova-load/src/main.rs @@ -0,0 +1,19 @@ +//! `nova-load` — the user-facing front controller for `nova load`. +//! +//! A thin shim: it reads only `vectorstore.type` from the config and `exec`s the +//! matching backend (`nova-load-`, e.g. `nova-load-qdrant`) with argv +//! unchanged. All the dispatch logic lives in the shared [`nova_shim`] crate so +//! it stays identical to the `nova-storm` shim; only the [`nova_shim::Spec`] +//! below differs. + +use std::process::ExitCode; + +fn main() -> ExitCode { + nova_shim::dispatch(&nova_shim::Spec { + program: "nova-load", + dispatch_key: &["vectorstore", "type"], + backend_prefix: "nova-load-", + default_type: "qdrant", + install_hint: "`make load` or `cargo install --path backends/nova-load/qdrant`", + }) +} diff --git a/commands/nova-shim/Cargo.toml b/commands/nova-shim/Cargo.toml new file mode 100644 index 0000000..7dcd2ad --- /dev/null +++ b/commands/nova-shim/Cargo.toml @@ -0,0 +1,16 @@ +[package] +# Shared front-controller logic for the `nova-*` command shims. NOT itself a +# command — a library the shim crates (commands/nova-load, commands/nova-storm) +# depend on so their dispatch behavior (config-type lookup, backend resolution, +# exec semantics, argv passthrough) is defined once and can't drift between them. +name = "nova-shim" +version = "0.0.8" +edition = "2024" +repository = "https://github.com/qdrant-labs/supernova" + +[lib] +name = "nova_shim" +path = "src/lib.rs" + +[dependencies] +serde_yaml = { workspace = true } diff --git a/commands/nova-shim/src/lib.rs b/commands/nova-shim/src/lib.rs new file mode 100644 index 0000000..d5acffd --- /dev/null +++ b/commands/nova-shim/src/lib.rs @@ -0,0 +1,126 @@ +//! Shared front-controller logic for the `nova-*` command shims. +//! +//! A shim (`commands/nova-load`, `commands/nova-storm`) reads only enough of the +//! config to learn which backend to use, maps that to a backend executable, and +//! **replaces this process** with it via `execv`, passing the original args +//! through untouched. Because it execs rather than spawns, the backend inherits +//! stdin/stdout/stderr and its exit code / signals surface directly — the shim +//! adds no layer at runtime. (This is why storm's single `--json` summary line +//! is emitted straight from the backend, unwrapped.) +//! +//! All of that is identical across shims; only the config key, the backend-name +//! prefix, and a few labels differ. Those live in [`Spec`], so the behavior is +//! defined here exactly once and the load/storm shims cannot drift apart. +//! +//! Backend resolution prefers an executable sitting next to the shim (so a +//! `target/debug` dev build finds its sibling backend without install), then +//! falls back to a normal `PATH` lookup (so `cargo install` layouts work). + +use std::ffi::OsString; +use std::os::unix::process::CommandExt; +use std::path::Path; +use std::process::{Command, ExitCode}; + +/// What distinguishes one shim from another. Everything else in [`dispatch`] is +/// shared. +pub struct Spec { + /// This shim's program name, used in error messages, e.g. `"nova-load"`. + pub program: &'static str, + /// Nested config key that selects the backend, e.g. + /// `&["vectorstore", "type"]` (load) or `&["target", "type"]` (storm). + pub dispatch_key: &'static [&'static str], + /// Backend executables are named ``, e.g. `"nova-load-"`. + pub backend_prefix: &'static str, + /// Backend type assumed when no config file is present in the args (so + /// `--help`, `--version`, and `capabilities` still reach a real backend). + pub default_type: &'static str, + /// Human hint appended to the "backend not found" error, e.g. + /// ``"`make load` or `cargo install --path backends/nova-load/qdrant`"``. + pub install_hint: &'static str, +} + +/// Resolve the backend from the config type in the process args and `exec` it, +/// preserving argv. On success this never returns (the process is replaced); +/// the returned [`ExitCode`] is only reached on a resolution/exec failure. +pub fn dispatch(spec: &Spec) -> ExitCode { + let args: Vec = std::env::args_os().skip(1).collect(); + + let backend_type = match resolve_backend_type(spec, &args) { + Ok(t) => t, + Err(msg) => { + eprintln!("{}: {msg}", spec.program); + return ExitCode::from(2); + } + }; + + let backend = format!("{}{backend_type}", spec.backend_prefix); + let program = resolve_backend_program(&backend); + + // Replace this process with the backend. On success this never returns. + let err = Command::new(&program).args(&args).exec(); + + // Only reached if exec itself failed (e.g. backend not installed). + eprintln!( + "{}: failed to exec backend `{}` (for {}=`{backend_type}`): {err}\n\ + hint: install it, e.g. {}", + spec.program, + program.to_string_lossy(), + spec.dispatch_key.join("."), + spec.install_hint, + ); + ExitCode::from(127) +} + +/// Inspect the args for a config file and return its dispatch type. Falls back +/// to `spec.default_type` when no config file is present (help/version/capabilities). +fn resolve_backend_type(spec: &Spec, args: &[OsString]) -> Result { + for arg in args { + // Skip flags; the config is always a bare positional path. + if arg.to_string_lossy().starts_with('-') { + continue; + } + let path = Path::new(arg); + if !path.is_file() { + continue; // subcommand tokens (`load`, `prepare`, `capabilities`, …) are not files + } + // First existing file among the args is the config. + return read_dispatch_type(spec, path); + } + Ok(spec.default_type.to_string()) +} + +/// Parse `path` as YAML and read the nested `spec.dispatch_key` string. `${VAR}` +/// references elsewhere in the config parse fine as plain scalars — the shim does +/// not need to expand them, only to read the (literal) backend type. +fn read_dispatch_type(spec: &Spec, path: &Path) -> Result { + let key = spec.dispatch_key.join("."); + let text = std::fs::read_to_string(path) + .map_err(|e| format!("failed to read config `{}`: {e}", path.display()))?; + let value: serde_yaml::Value = serde_yaml::from_str(&text) + .map_err(|e| format!("failed to parse config `{}` as YAML: {e}", path.display()))?; + + let mut node = &value; + for k in spec.dispatch_key { + node = node + .get(k) + .ok_or_else(|| format!("config `{}` is missing `{key}`", path.display()))?; + } + node.as_str() + .map(str::to_string) + .ok_or_else(|| format!("`{key}` in `{}` must be a string", path.display())) +} + +/// Resolve the backend executable: prefer a sibling next to this shim (dev +/// builds, and installs where both land in the same dir), else let `exec` +/// resolve `backend` on `PATH`. +fn resolve_backend_program(backend: &str) -> OsString { + if let Ok(exe) = std::env::current_exe() { + if let Some(dir) = exe.parent() { + let candidate = dir.join(backend); + if candidate.is_file() { + return candidate.into_os_string(); + } + } + } + backend.into() +} diff --git a/commands/nova-storm/Cargo.toml b/commands/nova-storm/Cargo.toml new file mode 100644 index 0000000..a13b656 --- /dev/null +++ b/commands/nova-storm/Cargo.toml @@ -0,0 +1,24 @@ +[package] +# The user-facing `nova storm` command. Thin front controller: reads only +# `target.type` from the config, then `exec`s the matching backend executable +# (`nova-storm-`, e.g. `nova-storm-qdrant`) with the original args +# unchanged. The real work lives in backends/nova-storm/*. +name = "nova-storm" +version = "0.0.8" +edition = "2024" +repository = "https://github.com/qdrant-labs/supernova" + +[[bin]] +name = "nova-storm" +path = "src/main.rs" + +# `cargo binstall nova-storm` fetches the prebuilt shim from each `v*` release +# (see .github/workflows/rust-binaries.yml). Matches the asset naming there. +[package.metadata.binstall] +pkg-url = "{ repo }/releases/download/v{ version }/{ name }-{ target }{ binary-ext }" +pkg-fmt = "bin" + +[dependencies] +# Shared shim front-controller logic (config-type dispatch + exec). See +# commands/nova-shim; keeps this shim identical to the nova-load shim. +nova-shim = { path = "../nova-shim" } diff --git a/commands/nova-storm/src/main.rs b/commands/nova-storm/src/main.rs new file mode 100644 index 0000000..1e8fe54 --- /dev/null +++ b/commands/nova-storm/src/main.rs @@ -0,0 +1,20 @@ +//! `nova-storm` — the user-facing front controller for `nova storm`. +//! +//! A thin shim: it reads only `target.type` from the config and `exec`s the +//! matching backend (`nova-storm-`, e.g. `nova-storm-qdrant`) with argv +//! unchanged. All the dispatch logic lives in the shared [`nova_shim`] crate so +//! it stays identical to the `nova-load` shim; only the [`nova_shim::Spec`] +//! below differs. (Because it execs, storm's single `--json` summary line is +//! emitted straight from the backend, unwrapped.) + +use std::process::ExitCode; + +fn main() -> ExitCode { + nova_shim::dispatch(&nova_shim::Spec { + program: "nova-storm", + dispatch_key: &["target", "type"], + backend_prefix: "nova-storm-", + default_type: "qdrant", + install_hint: "`make storm` or `cargo install --path backends/nova-storm/qdrant`", + }) +} diff --git a/python/nova-sweep/pyproject.toml b/commands/nova-sweep/pyproject.toml similarity index 100% rename from python/nova-sweep/pyproject.toml rename to commands/nova-sweep/pyproject.toml diff --git a/python/nova-sweep/src/nova_sweep/__init__.py b/commands/nova-sweep/src/nova_sweep/__init__.py similarity index 100% rename from python/nova-sweep/src/nova_sweep/__init__.py rename to commands/nova-sweep/src/nova_sweep/__init__.py diff --git a/python/nova-sweep/src/nova_sweep/backends/__init__.py b/commands/nova-sweep/src/nova_sweep/backends/__init__.py similarity index 100% rename from python/nova-sweep/src/nova_sweep/backends/__init__.py rename to commands/nova-sweep/src/nova_sweep/backends/__init__.py diff --git a/python/nova-sweep/src/nova_sweep/backends/base.py b/commands/nova-sweep/src/nova_sweep/backends/base.py similarity index 100% rename from python/nova-sweep/src/nova_sweep/backends/base.py rename to commands/nova-sweep/src/nova_sweep/backends/base.py diff --git a/python/nova-sweep/src/nova_sweep/backends/qdrant.py b/commands/nova-sweep/src/nova_sweep/backends/qdrant.py similarity index 100% rename from python/nova-sweep/src/nova_sweep/backends/qdrant.py rename to commands/nova-sweep/src/nova_sweep/backends/qdrant.py diff --git a/python/nova-sweep/src/nova_sweep/cli.py b/commands/nova-sweep/src/nova_sweep/cli.py similarity index 100% rename from python/nova-sweep/src/nova_sweep/cli.py rename to commands/nova-sweep/src/nova_sweep/cli.py diff --git a/python/nova-sweep/src/nova_sweep/config.py b/commands/nova-sweep/src/nova_sweep/config.py similarity index 100% rename from python/nova-sweep/src/nova_sweep/config.py rename to commands/nova-sweep/src/nova_sweep/config.py diff --git a/python/nova-sweep/src/nova_sweep/grid.py b/commands/nova-sweep/src/nova_sweep/grid.py similarity index 100% rename from python/nova-sweep/src/nova_sweep/grid.py rename to commands/nova-sweep/src/nova_sweep/grid.py diff --git a/python/nova-sweep/src/nova_sweep/report.py b/commands/nova-sweep/src/nova_sweep/report.py similarity index 100% rename from python/nova-sweep/src/nova_sweep/report.py rename to commands/nova-sweep/src/nova_sweep/report.py diff --git a/python/nova-sweep/src/nova_sweep/runner.py b/commands/nova-sweep/src/nova_sweep/runner.py similarity index 100% rename from python/nova-sweep/src/nova_sweep/runner.py rename to commands/nova-sweep/src/nova_sweep/runner.py diff --git a/python/nova-sweep/src/nova_sweep/slices.py b/commands/nova-sweep/src/nova_sweep/slices.py similarity index 100% rename from python/nova-sweep/src/nova_sweep/slices.py rename to commands/nova-sweep/src/nova_sweep/slices.py diff --git a/python/nova-sweep/tests/test_config.py b/commands/nova-sweep/tests/test_config.py similarity index 100% rename from python/nova-sweep/tests/test_config.py rename to commands/nova-sweep/tests/test_config.py diff --git a/python/nova-sweep/tests/test_grid.py b/commands/nova-sweep/tests/test_grid.py similarity index 100% rename from python/nova-sweep/tests/test_grid.py rename to commands/nova-sweep/tests/test_grid.py diff --git a/python/nova-sweep/tests/test_qdrant_backend.py b/commands/nova-sweep/tests/test_qdrant_backend.py similarity index 100% rename from python/nova-sweep/tests/test_qdrant_backend.py rename to commands/nova-sweep/tests/test_qdrant_backend.py diff --git a/python/nova-sweep/tests/test_runner.py b/commands/nova-sweep/tests/test_runner.py similarity index 100% rename from python/nova-sweep/tests/test_runner.py rename to commands/nova-sweep/tests/test_runner.py diff --git a/python/nova-sweep/tests/test_slices.py b/commands/nova-sweep/tests/test_slices.py similarity index 100% rename from python/nova-sweep/tests/test_slices.py rename to commands/nova-sweep/tests/test_slices.py diff --git a/python/nova-sweep/uv.lock b/commands/nova-sweep/uv.lock similarity index 100% rename from python/nova-sweep/uv.lock rename to commands/nova-sweep/uv.lock diff --git a/contracts/nova-load/README.md b/contracts/nova-load/README.md new file mode 100644 index 0000000..8047363 --- /dev/null +++ b/contracts/nova-load/README.md @@ -0,0 +1,98 @@ +# `nova-load` backend contract + +This directory holds the **language-neutral** contract for `nova load` backends: +[`v1.yaml`](v1.yaml). It is the canonical description of what any +`nova-load-` executable must do, regardless of the language it's +written in. + +## The command / backend / contract model + +`supernova` is organized by **command and backend, not by language**: + +``` +commands/ + nova-load/ # user-facing shim: `nova load` → picks a backend +backends/ + nova-load/ + contracts/rust/ # shared Rust interface (compile-time enforcement) + qdrant/ # the Qdrant backend → builds `nova-load-qdrant` +contracts/ + nova-load/ + v1.yaml # canonical language-neutral contract (this dir) + README.md +``` + +A backend lives at `backends/nova-load//` and is named for the +**store it targets** (`qdrant`, and one day `milvus`, `vespa`, …), not the +language it happens to be written in. Backends that share a language share a +per-language interface package under `backends/nova-load/contracts//`. + +## The two-layer contract model + +There are two complementary layers of enforcement: + +1. **Native per-language interface — compile-time, within one language.** + `backends/nova-load/contracts/rust` is a Rust crate defining the + `VectorStore` trait plus the neutral data types (`Point`, `VectorValue`, + `PointId`, `CollectionSchema`, `VectorSpec`, `StoreError`). Every Rust + backend depends on this crate and `impl VectorStore for …`. If a Rust + backend is missing a method or has the wrong signature, **it doesn't + compile**. A future Go backend would get an analogous `contracts/go/` + package. + +2. **Language-neutral contract + `nova contract check` — runtime, across + languages.** [`v1.yaml`](v1.yaml) declares the required commands, flags, + method names, vector kinds, and point-id types. Each backend advertises what + it actually supports via `capabilities --json`. `nova contract check` + compares the two. This works for a backend in *any* language, because it only + ever talks to the executable — never to source code. + +Keep the three in lockstep when you change the contract: the trait in +`contracts/rust`, the `methods:`/`commands:` in `v1.yaml`, and the +`capabilities --json` each backend prints. + +## The shim / backend-executable model + +`nova load …` still works exactly as before. Under the hood: + +- `commands/nova-load` builds the executable **`nova-load`** — a thin *shim*. + It reads only `vectorstore.type` from the config, maps it to a backend + executable (`qdrant` → `nova-load-qdrant`), and **`exec`s** that backend with + the original args unchanged. Because it execs (replaces the process) rather + than spawns, stdin/stdout/stderr and the exit code pass straight through — the + shim adds no runtime layer. +- `backends/nova-load/qdrant` builds **`nova-load-qdrant`** — the real backend. + +The `nova` dispatcher finds `nova-load` on `PATH` exactly as before; the shim +then finds `nova-load-qdrant` (preferring a sibling next to itself, else `PATH`). + +Contract checks target the **backend** executable, never the shim: + +```bash +nova contract check "$(command -v nova-load-qdrant)" --contract contracts/nova-load/v1.yaml +# or against a dev build: +nova-contract check ./target/debug/nova-load-qdrant --contract contracts/nova-load/v1.yaml +``` + +Levels: `--level shape` (capabilities vs contract only), `dry-run` (default; +adds cheap behavioral checks like capabilities-determinism), `live` (runs the +contract's `live_check` — for load, `inspect ` — needs `--config`). + +## Adding a new backend (e.g. Milvus) + +1. Create `backends/nova-load/milvus/` as a new crate building the executable + **`nova-load-milvus`**. +2. If it's Rust, depend on `nova-load-contract-rust` and `impl VectorStore` — + the compiler enforces the method set. If it's another language, implement the + equivalent interface (add `backends/nova-load/contracts//` if it's + the first backend in that language). +3. Implement `capabilities --json` advertising `contract: nova-load-backend/v1` + and the commands/methods/kinds it supports. +4. `nova contract check nova-load-milvus --contract contracts/nova-load/v1.yaml` + must pass. +5. No change to `commands/nova-load` is needed: a config with + `vectorstore.type: milvus` dispatches to `nova-load-milvus` automatically. + +Do **not** teach the shim to pretend to be a backend, and do not give +orchestrators (`nova-sweep`) their own backend layer — they call `nova load` / +`nova storm`, which do the dispatch. diff --git a/contracts/nova-load/v1.yaml b/contracts/nova-load/v1.yaml new file mode 100644 index 0000000..1da153f --- /dev/null +++ b/contracts/nova-load/v1.yaml @@ -0,0 +1,100 @@ +# Language-neutral contract for `nova load` backends — v1. +# +# This is the CANONICAL, cross-language contract. Every `nova-load-` +# executable must satisfy it, whatever language it's written in. Conformance is +# checked at runtime by `nova contract check --contract this-file`, +# which reads the backend's `capabilities --json` and validates it below. +# +# Rust backends additionally get compile-time enforcement by implementing the +# `VectorStore` trait in `backends/nova-load/contracts/rust`. Keep this file, +# that trait, and each backend's advertised `capabilities` in lockstep. + +id: nova-load-backend +version: 1 +# The exact `contract` string a conforming backend must advertise. +contract: nova-load-backend/v1 +description: > + Bulk-load pre-embedded vectors from a datasource (parquet on local disk or + S3) into a vector store, with a distributed prepare/load/finalize phase model. + +# CLI subcommands the executable must expose (checked against the `commands` +# array in `capabilities --json`). +required_commands: + - capabilities # print this descriptor as JSON + - run # single-node: prepare + load-all + finalize + - prepare # master: create collection, defer indexing + - load # worker: load this rank's partition + - finalize # master: re-enable indexing and wait + - inspect # dry-run: config + file list, no connect + - reindex # patch index settings in place on an existing collection + - delete # delete the collection if it exists + +# Flags each listed subcommand must accept (checked against `flags` in +# `capabilities --json`). These are the distributed-work contract. +required_flags: + load: + - --num-jobs + - --job-rank + inspect: + - --num-jobs + - --job-rank + +# Backend method names — the internal store abstraction every backend +# implements. Matches the Rust `VectorStore` trait in +# backends/nova-load/contracts/rust and the `methods` array in capabilities. +required_methods: + - ensure_collection # create/verify the collection from a neutral schema + - upsert_batch # upsert a batch of points + - close # tear down connections + - defer_indexing # disable indexing for fast bulk load + - enable_indexing # re-enable indexing after load + - wait_for_indexing # block until indexing converges + - reindex # patch index settings in place + - delete_collection # delete the collection if present + +# Vector shapes a backend must be able to accept (matches `VectorValue`). +required_vector_kinds: + - dense + - sparse + - multivector + +# Point id shapes the reader can produce and the backend must accept +# (matches `PointId`). +required_point_id_types: + - integer + - string + +# `nova contract check --level live` runs this against a real backend. `{config}` +# is replaced by the `--config` path. `inspect` connects to nothing and mutates +# nothing, so it's a safe live-ish smoke check that config + source resolve. +live_check: + args: ["inspect", "{config}"] + +# --------------------------------------------------------------------------- +# Shared behavior notes (prose; not machine-checked, but part of the contract). +# --------------------------------------------------------------------------- +behavior_notes: | + Config env expansion: raw YAML is expanded before parsing. `${VAR}` requires + the variable; `${VAR:-default}` supplies a fallback; `$$` is a literal `$`. + + Distributed partitioning: `load`/`inspect` split the file list by STRIDE, not + by chunking — `files.skip(job_rank).step_by(num_jobs)`. This interleaves + large/small files across workers with zero coordination and MUST be preserved + by every backend for fleet-wide balance and completeness. + + Phase model: `prepare` (master creates the collection and defers indexing) → + `load` (each worker loads its stride) → `finalize` (master re-enables indexing + and waits for convergence). `run` is the single-node all-in-one. Workers never + coordinate with each other; the split is what makes that safe. + + Point ids: integer or string. A backend must accept both shapes; how it maps + them onto its own id space is backend-specific. + + Vectors: dense, sparse (indices+values), and multivector, per the collection's + declared `vectors:` specs. + + Failure semantics: a file that fails to download/read is skipped after its + retries, up to `max_failed_files` (unbounded by default); exceeding the cap + aborts. A failed *upsert*, by contrast, aborts the run after its retries — a + persistent upsert failure means the store is down/misconfigured, not a bad + file. These two philosophies are deliberately different. diff --git a/contracts/nova-storm/README.md b/contracts/nova-storm/README.md new file mode 100644 index 0000000..582a057 --- /dev/null +++ b/contracts/nova-storm/README.md @@ -0,0 +1,84 @@ +# `nova-storm` backend contract + +This directory holds the **language-neutral** contract for `nova storm` +backends: [`v1.yaml`](v1.yaml). It is the canonical description of what any +`nova-storm-` executable must do, regardless of language. + +See [`../nova-load/README.md`](../nova-load/README.md) for the full explanation +of the command/backend/contract model and the two-layer (compile-time + +runtime) enforcement scheme — it applies identically here. This file covers only +what's specific to `nova storm`. + +## Layout + +``` +commands/ + nova-storm/ # user-facing shim: `nova storm` → picks a backend +backends/ + nova-storm/ + contracts/rust/ # shared Rust interface (compile-time enforcement) + qdrant/ # the Qdrant backend → builds `nova-storm-qdrant` +contracts/ + nova-storm/ + v1.yaml # canonical language-neutral contract (this dir) + README.md +``` + +## The two layers, for storm + +1. **Native Rust interface.** `backends/nova-storm/contracts/rust` defines the + `QueryTarget` trait plus `BatchOutcome` and `TargetError`. Every Rust storm + backend depends on this crate and `impl QueryTarget for …`; a wrong or + missing method is a compile error. + +2. **Language-neutral contract.** [`v1.yaml`](v1.yaml) declares the required + commands, method names, load-generation modes, and features. Backends + advertise theirs via `capabilities --json`; `nova contract check` compares. + +## Shim / backend-executable model + +- `commands/nova-storm` builds the shim **`nova-storm`**. It reads only + `target.type` from the config, maps it (`qdrant` → `nova-storm-qdrant`), and + `exec`s the backend with the original args. Crucially, because it execs, the + backend's stdout — including the single `--json` summary line a caller like + `nova sweep` parses — is emitted directly, with no wrapper on it. +- `backends/nova-storm/qdrant` builds the backend **`nova-storm-qdrant`**. + +Note storm's CLI is `nova-storm-qdrant [--json]` (a positional config, +not subcommands) plus the `capabilities` descriptor command. The contract's +`run` command names that default positional invocation. + +## Running contract checks + +```bash +nova contract check "$(command -v nova-storm-qdrant)" --contract contracts/nova-storm/v1.yaml +# dev build: +nova-contract check ./target/debug/nova-storm-qdrant --contract contracts/nova-storm/v1.yaml +``` + +`--level live` runs the storm itself (the contract's `live_check` is just +`{config}`), so point `--config` at a throwaway collection with a short +`load.duration_s`. + +## Storm-specific invariants a backend must honor + +These are in `v1.yaml`'s `behavior_notes` and are load-bearing for correctness +across a fleet — do not "simplify" them away in a new backend: + +- **Closed-loop vs open-loop** are distinct load shapes; open-loop paces on a + virtual schedule to avoid coordinated omission. +- **A failed dispatch is a recorded sample** (`ok:false`), never a hard abort. +- **Percentiles come from raw samples**, nearest-rank, computed once — never + averaged per-worker (so fleet-wide merges stay correct). +- **Work is replicated** across fleet workers (every worker runs the same query + mix), in contrast to nova-load's stride partitioning. +- **Recall** is `hits/top_k` per query when ground truth is provided, aggregated + across queries; absent (not `0.0`) when there's no ground truth. + +## Adding a new backend + +Same recipe as nova-load: create `backends/nova-storm//` building +`nova-storm-`, implement `QueryTarget` (Rust) or the equivalent, advertise +`capabilities --json` with `contract: nova-storm-backend/v1`, and make +`nova contract check` pass. A config with `target.type: ` then dispatches +automatically — no shim change. diff --git a/contracts/nova-storm/v1.yaml b/contracts/nova-storm/v1.yaml new file mode 100644 index 0000000..c15b684 --- /dev/null +++ b/contracts/nova-storm/v1.yaml @@ -0,0 +1,76 @@ +# Language-neutral contract for `nova storm` backends — v1. +# +# This is the CANONICAL, cross-language contract. Every `nova-storm-` +# executable must satisfy it, whatever language it's written in. Conformance is +# checked at runtime by `nova contract check --contract this-file`, +# which reads the backend's `capabilities --json` and validates it below. +# +# Rust backends additionally get compile-time enforcement by implementing the +# `QueryTarget` trait in `backends/nova-storm/contracts/rust`. Keep this file, +# that trait, and each backend's advertised `capabilities` in lockstep. + +id: nova-storm-backend +version: 1 +# The exact `contract` string a conforming backend must advertise. +contract: nova-storm-backend/v1 +description: > + Sustained query-load generator: fire nearest-neighbour queries at a vector + store under a configured load shape and report latency percentiles (and recall + when ground truth is provided). + +# CLI surface the executable must expose (checked against `commands` in +# `capabilities --json`). `run` is the default positional-config invocation +# (`nova-storm- [--json]`); `capabilities` prints this +# descriptor. +required_commands: + - run + - capabilities + +# Backend method names — the internal target abstraction every backend +# implements. Matches the Rust `QueryTarget` trait in +# backends/nova-storm/contracts/rust and the `methods` array in capabilities. +required_methods: + - query_batch # fire one batch dispatch (all vectors, one round-trip) + - close # tear down connections + +# Load-generation modes the runner supports (matches `search_modes`). +required_search_modes: + - closed_loop # fixed concurrency, loop until deadline + - open_loop # paced virtual schedule + semaphore ceiling + +# Capabilities the backend must report supporting. +required_features: + - recall # compare returned ids to ground truth when provided + - percentiles # latency percentiles from raw samples + +# `nova contract check --level live` runs the storm itself against a live target. +# `{config}` is replaced by the `--config` path; keep its `load.duration_s` +# short. This is a real load run, so point it at a throwaway collection. +live_check: + args: ["{config}"] + +# --------------------------------------------------------------------------- +# Shared behavior notes (prose; not machine-checked, but part of the contract). +# --------------------------------------------------------------------------- +behavior_notes: | + Config env expansion: same as nova-load — `${VAR}`, `${VAR:-default}`, `$$`. + + Load modes: CLOSED-LOOP runs a fixed number of concurrent workers looping + until a deadline. OPEN-LOOP paces dispatches on a virtual schedule with a + semaphore ceiling, deliberately avoiding coordinated omission (a slow server + must not slow the offered rate). A backend's timing must not conflate these. + + Failed dispatch = data, not abort: a query batch that errors is recorded as a + BatchOutcome{ok:false} sample, NOT a hard error that stops the run. A storm + measures behavior under load, so errors at the limit are a finding. + + Percentiles: computed ONCE, nearest-rank, from the raw per-dispatch samples — + never averaged per-worker. This is required for correct fleet-wide aggregation + when many workers' samples are merged. + + Replicated work: every fleet worker runs the SAME query mix (contrast with + nova-load's stride partitioning). Storm replicates; it does not partition. + + Recall / ground truth: when the query source declares a ground-truth column, + recall is hits/top_k per query, aggregated (mean/median/min) across queries. + With no ground truth, recall fields are absent — never reported as 0.0. diff --git a/crates/nova-storm/src/targets/mod.rs b/crates/nova-storm/src/targets/mod.rs deleted file mode 100644 index 38028c0..0000000 --- a/crates/nova-storm/src/targets/mod.rs +++ /dev/null @@ -1,77 +0,0 @@ -//! Query targets — the backends a storm fires at. -//! -//! A [`QueryTarget`] is a thin adapter: "fire one batch dispatch of nearest- -//! neighbour queries and report its latency." A batch of 1 is not a special -//! case — it's just the default. The load *shape* (concurrency, duration, -//! rate, batch size) lives in the [runner](crate::runner), so a backend stays -//! minimal and the same runner drives any store. Targets are built once and -//! shared across every concurrent request via `Arc`, so the trait is -//! `Send + Sync` (the gRPC client multiplexes concurrent calls over one -//! connection). - -use std::sync::Arc; -use std::time::Duration; - -use async_trait::async_trait; -use serde::Deserialize; - -use crate::config::QueryConfig; -use crate::errors::TargetError; - -pub mod qdrant; - -/// Outcome of a single batch dispatch (one `query_batch` round-trip, covering -/// `vectors.len()` queries). A failure is recorded here (`ok = false`) rather -/// than aborting the run — a storm measures how a cluster behaves under load, -/// and errors at the limit are a finding, not a crash. `latency`/`ok`/`error` -/// describe the one round-trip, not any individual query inside it — a single -/// gRPC call's timing can't be honestly disaggregated into per-query numbers. -#[derive(Debug, Clone)] -pub struct BatchOutcome { - pub latency: Duration, - pub ok: bool, - /// One entry per submitted query, in the same order as the input - /// `vectors` — the point ids that query actually returned, best-first. - /// `None` at a position means there's nothing meaningful to report for - /// that query: recall tracking wasn't on for this run - /// (`QdrantTarget::collect_ids` is `false`) or the whole dispatch failed - /// (`!ok`). `Some(vec![])` is a real, different thing — recall tracking - /// was on, the dispatch succeeded, and that query just matched nothing. - pub ids: Vec>>, - pub error: Option, -} - -/// A backend a storm sends queries to. `Display` is the name used in logs -/// (e.g. `qdrant(products)`). -#[async_trait] -pub trait QueryTarget: Send + Sync + std::fmt::Display { - /// Fire one batch dispatch covering all of `vectors` in a single - /// round-trip and return its latency + outcome. The top-k / vector-name - /// knobs are baked into the target at construction, so the hot path is - /// just the vectors. A single-element slice is not a special case — it's - /// the default (`LoadProfile::batch_size == 1`). - async fn query_batch(&self, vectors: &[&[f32]]) -> BatchOutcome; - - /// Tear down connections. Default: nothing (clients close on drop). - async fn close(&self) -> Result<(), TargetError> { - Ok(()) - } -} - -/// Target backend config, dispatched on `type:`. Each backend owns its config -/// struct in its own module. -#[derive(Debug, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum TargetConfig { - Qdrant(qdrant::QdrantConfig), -} - -impl TargetConfig { - /// Connect and build the shared target. `query` carries the vector name and - /// top-k the backend bakes in. - pub fn into_target(self, query: &QueryConfig) -> Result, TargetError> { - match self { - TargetConfig::Qdrant(c) => Ok(Arc::new(c.into_target(query)?)), - } - } -} diff --git a/python/README.md b/python/README.md deleted file mode 100644 index f02821f..0000000 --- a/python/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# supernova — Python side - -The Python half of the polyglot `supernova` toolset. The Rust half lives in -`../crates`. The two are **not** code-coupled: they meet only at runtime through -the git-style `nova` dispatcher and shared data contracts (the YAML config, -parquet/point formats). - -## The dispatcher model - -`nova [args...]` finds an executable named `nova-` on `PATH` and -replaces itself with it (`os.execv`). A command can be implemented in any -language: - -- **Rust** — a binary like `nova-load`, installed by `cargo install`. -- **Python** — a console script like `nova-embed`, installed by `pip`. - -The dispatcher doesn't know or care which. To add a command, just put a -`nova-` executable on `PATH`. - -## Packages - -| Package | Location | Provides | Deps | Install where | -|-------------|------------------|--------------|---------------------|----------------------| -| `nova-cli` | repo root | `nova` | none | everywhere (instant) | -| `nova-embed`| `python/nova-embed` | `nova-embed` | torch, sentence-transformers | embedding machines | - -The **dispatcher lives at the repo root** (`pyproject.toml` + `src/nova_cli/`), -so `uv pip install -e .` from the root installs `nova`. It's the project's front -door and the spine of the polyglot tool, so it sits at the top rather than buried -as just-another-package. It's deliberately dependency-free. - -Heavy commands are separate packages under `python/`, installed only where -needed — like `git-*` subcommands. - -### nova-embed - -Embedding generation (chunkers → embedders → storage), streamed from a dataset -source and written as parquet. Honors the same `--num-jobs` / `--job-rank` -distributed contract as `nova-load`: each rank computes its own `offset`/`limit` -slice of the dataset (from `--job-rank`, or `$SKYPILOT_JOB_RANK`). - -Config is validated with **pydantic** (`nova_embed.config`): `pipeline` knobs are -typed with defaults in one place, while `source`/`*_embedder`/`storage` carry a -`type` plus flexible backend-specific kwargs. `${VAR}` / `${VAR:-default}` -references are env-expanded, matching the Rust crates. - -The base package is light (pydantic, pyarrow, …); the actual ML stack (torch, -sentence-transformers, …) is the `embed` extra: - -```sh -uv pip install -e 'python/nova-embed[embed]' -nova embed configs/embedder/test.yaml --num-jobs 50 --job-rank $SKYPILOT_JOB_RANK -nova embed configs/embedder/test.yaml --dry-run -``` - -## Dev setup - -```sh -uv pip install -e . # the `nova` dispatcher (from repo root) -uv pip install -e python/nova-embed # embedding command (heavy) -cargo install --path crates/nova-load # the `nova-load` Rust binary - -nova --help # lists discovered nova-* commands -nova load inspect configs/loader/test.yaml -nova embed ... -``` - -> Ensure your Python user-scripts dir (e.g. `~/.local/bin` or -> `~/Library/Python/X.Y/bin`) and `~/.cargo/bin` are on `PATH`. diff --git a/tests/contracts/nova-load/README.md b/tests/contracts/nova-load/README.md new file mode 100644 index 0000000..5107f20 --- /dev/null +++ b/tests/contracts/nova-load/README.md @@ -0,0 +1,17 @@ +# nova-load conformance fixtures + +Fixtures for `nova contract check` against `nova-load-` executables. + +`make test` runs, at `shape`/`dry-run` level (no live backend needed): + +```bash +nova-contract check --contract contracts/nova-load/v1.yaml +``` + +This validates the backend's `capabilities --json` against the canonical +contract in `contracts/nova-load/v1.yaml`. The `--fixtures ` flag points +here and is reserved for future fixture-driven dry-run checks (e.g. sample +configs a backend must accept/reject); the checker tolerates its absence. + +Live conformance (actually loading into a store) is exercised separately against +a real Qdrant — see the "Testing against a live Qdrant" section in `AGENTS.md`. diff --git a/tests/contracts/nova-storm/README.md b/tests/contracts/nova-storm/README.md new file mode 100644 index 0000000..51efec7 --- /dev/null +++ b/tests/contracts/nova-storm/README.md @@ -0,0 +1,17 @@ +# nova-storm conformance fixtures + +Fixtures for `nova contract check` against `nova-storm-` executables. + +`make test` runs, at `shape`/`dry-run` level (no live backend needed): + +```bash +nova-contract check --contract contracts/nova-storm/v1.yaml +``` + +This validates the backend's `capabilities --json` against the canonical +contract in `contracts/nova-storm/v1.yaml`. The `--fixtures ` flag points +here and is reserved for future fixture-driven dry-run checks; the checker +tolerates its absence. + +Live conformance (actually storming a store) is exercised separately against a +real Qdrant — see the "Testing against a live Qdrant" section in `AGENTS.md`.