VINDEX3: c9 all-layer container, and a real model running from it

crates/larql-cli/src/commands/primary/run_cmd.rs

@@ -137,6 +137,19 @@ pub struct RunArgs {
     #[arg(long, value_name = "URL")]
     pub ffn: Option<String>,
 
+    /// Serve the routed expert banks from a VINDEX3 container, keeping the
+    /// rest of the model (tokenizer, config, embeddings, attention, norms,
+    /// routers, dense/shared FFN, LM head) from the VINDEX2 `MODEL` argument.
+    ///
+    /// Exactly one operand source is replaced — spec §4 classes 4 and 5 —
+    /// so a comparison against the same prompt without this flag is a
+    /// statement about the routed bytes and nothing else.
+    ///
+    /// Never falls back: if the container cannot serve every routed layer the
+    /// model needs, the run is refused before the prompt is encoded.
+    #[arg(long, value_name = "DIR")]
+    pub routed_from: Option<String>,
+
     /// HTTP timeout in seconds for --ffn.
     #[arg(long, default_value = "60")]
     pub ffn_timeout_secs: u64,
@@ -308,6 +321,14 @@ pub fn run(args: RunArgs) -> Result<(), Box<dyn std::error::Error>> {
         return run_bitnet(&vindex_path, &args);
     }
 
+    if let Some(ref routed_dir) = args.routed_from {
+        let prompt = args
+            .prompt
+            .as_deref()
+            .ok_or("--routed-from requires a prompt argument (chat mode not yet supported)")?;
+        return run_with_routed_container(&vindex_path, routed_dir, prompt, args.max_tokens);
+    }
+
     if let Some(ref ffn_url) = args.ffn {
         let prompt = args.prompt.as_deref().ok_or(
             "--ffn requires a prompt argument (chat mode not yet supported with --ffn-dispatch batch)",
@@ -758,14 +779,17 @@ fn run_with_moe_shards(
                 );
             }
             let started = std::time::Instant::now();
+            // Fatal by policy: a shard failure aborts the run rather than
+            // finishing the sentence from a model missing an expert layer.
             let toks = generate_kquant_cpu_remote(
                 &mut weights,
                 &tokenizer,
                 &prompt_ids,
                 max_tokens,
                 &index,
                 &remote,
-            );
+            )
+            .map_err(|e| format!("remote MoE dispatch failed, generation aborted: {e}"))?;
             let total_ms = started.elapsed().as_secs_f64() * 1000.0;
             let strings: Vec<String> = toks.into_iter().map(|(s, _)| s).collect();
             let n = strings.len();
@@ -887,6 +911,82 @@ fn run_with_moe_shards(
     Ok(())
 }
 
+/// `--routed-from DIR` — routed expert banks served from a VINDEX3 container.
+///
+/// The composition, precisely:
+///
+/// ```text
+/// VINDEX2 model   tokenizer, config, embeddings, attention, norms,
+///                 routers, dense/shared FFN, LM head
+/// VINDEX3 dir     routed gate/up and routed down banks  (spec §4 classes 4-5)
+/// ```
+///
+/// Everything but the routed banks is read exactly as an ordinary run reads
+/// it, so the same prompt without the flag is a controlled comparison: the
+/// only variable is where the expert bytes came from.
+///
+/// This is a *composed* run, not a VINDEX3 model. A container holding only
+/// routed banks has no tokenizer and no spine; `larql run <vindex3-dir>` is
+/// still correctly refused. Container completeness is a separate rung.
+fn run_with_routed_container(
+    vindex_path: &std::path::Path,
+    routed_dir: &str,
+    prompt: &str,
+    max_tokens: usize,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let routed_path = std::path::Path::new(routed_dir);
+
+    let mut cb = larql_vindex::SilentLoadCallbacks;
+    let mut weights = larql_vindex::load_model_weights_kquant(vindex_path, &mut cb)
+        .map_err(|e| format!("failed to load spine weights: {e}"))?;
+    let tokenizer = larql_vindex::load_vindex_tokenizer(vindex_path)
+        .map_err(|e| format!("failed to load tokenizer: {e}"))?;
+    let mut index = larql_vindex::VectorIndex::load_vindex(vindex_path, &mut cb)
+        .map_err(|e| format!("failed to load vindex: {e}"))?;
+    index
+        .load_attn_kquant(vindex_path)
+        .map_err(|e| format!("failed to load attn Q4K: {e}"))?;
+    index
+        .load_interleaved_kquant(vindex_path)
+        .map_err(|e| format!("failed to load interleaved Q4K: {e}"))?;
+    let _ = index.load_lm_head_kquant(vindex_path);
+
+    // Compose *before* the prompt is encoded. Every shape, count and region is
+    // checked here, so a mismatch is reported against two named artifacts
+    // rather than surfacing as a wrong number seventeen layers into a forward
+    // pass that has already printed part of an answer.
+    let routed = larql_inference::ffn::ContainerRoutedBackend::open(routed_path, &weights, true)
+        .map_err(|e| format!("--routed-from refused: {e}"))?;
+    eprintln!("{}", routed.describe(vindex_path));
+
+    let wrapped_prompt =
+        larql_inference::chat::render_user_prompt(vindex_path, weights.arch.family(), prompt)?;
+    let prompt_ids = larql_inference::encode_prompt(&tokenizer, &*weights.arch, &wrapped_prompt)
+        .map_err(|e| format!("failed to tokenise prompt: {e}"))?;
+
+    let started = std::time::Instant::now();
+    let toks = larql_inference::vindex::generate_kquant_cpu_routed(
+        &mut weights,
+        &tokenizer,
+        &prompt_ids,
+        max_tokens,
+        &index,
+        &routed,
+    )
+    .map_err(|e| format!("routed container dispatch failed, generation aborted: {e}"))?;
+    let total_ms = started.elapsed().as_secs_f64() * 1000.0;
+
+    let text: String = toks.iter().map(|(t, _)| t.as_str()).collect();
+    println!("{text}");
+    let n = toks.len();
+    eprintln!(
+        "\n  {n} token(s) in {:.0} ms ({:.0} ms/token)",
+        total_ms,
+        if n == 0 { 0.0 } else { total_ms / n as f64 }
+    );
+    Ok(())
+}
+
 /// `--ffn URL` dispatch path for dense models.
 ///
 /// Metal runs attention on the local GPU. Every layer's FFN is a round trip

crates/larql-cli/src/main.rs

@@ -317,6 +317,7 @@ struct ChatArgs {
     /// Route FFN to a remote larql-server.
     #[arg(long, value_name = "URL")]
     ffn: Option<String>,
+    routed_from: Option<String>,
 
     /// HTTP timeout in seconds for --ffn.
     #[arg(long, default_value = "60")]
@@ -338,6 +339,7 @@ impl From<ChatArgs> for run_cmd::RunArgs {
             context_window: 0,
             engine: None,
             ffn: c.ffn,
+            routed_from: c.routed_from,
             ffn_timeout_secs: c.ffn_timeout_secs,
             metal: false,
             verbose: c.verbose,

crates/larql-inference/src/ffn/local_moe.rs

@@ -80,17 +80,23 @@ impl<'a> FfnBackend for LocalMoeFfn<'a> {
     ) -> Result<Option<Array2<f32>>, larql_execution::BoxRefusal> {
         // Local dispatch over resident weights: there is no operand this could
         // fail to reach, so it never refuses.
-        Ok(Some(moe_ffn_block_cpu_with_index(
-            self.weights,
-            h_post_attn,
-            layer,
-            &WeightFfn {
-                weights: self.weights,
-            },
-            None,
-            None,
-            self.index,
-        )))
+        Ok(Some(
+            moe_ffn_block_cpu_with_index(
+                self.weights,
+                h_post_attn,
+                layer,
+                &WeightFfn {
+                    weights: self.weights,
+                },
+                None,
+                None,
+                self.index,
+            )
+            // No route is bound, so the refusal branch is unreachable rather than
+            // ignored — stated so a future caller that *does* bind one here has to
+            // decide what a failure means instead of inheriting silence.
+            .expect("no MoE route is bound, so no refusal is reachable"),
+        ))
     }
 }
 

crates/larql-inference/src/ffn/mod.rs

@@ -17,6 +17,7 @@ pub mod graph_backend;
 pub mod local_moe;
 pub mod moe_backend;
 pub mod moe_bound;
+pub mod moe_container;
 pub mod moe_remote;
 pub mod remote;
 pub mod sparse;
@@ -34,8 +35,11 @@ pub use larql_compute::ffn::{
 // ── Re-exports ──
 
 pub use local_moe::LocalMoeFfn;
-pub use moe_backend::{InProcessMoeBackend, MoeBackendError, MoeExpertBackend};
+pub use moe_backend::{
+    InProcessMoeBackend, MoeBackendError, MoeExpertBackend, MoeFailurePolicy, MoeRoute,
+};
 pub use moe_bound::BoundMoeBackend;
+pub use moe_container::{CompositionError, ContainerRoutedBackend};
 pub use moe_remote::{
     MoeFfn, MoeRouterWeights, RecordedRefusal, RefusalPolicy, RemoteMoeBackend, RemoteMoeError,
     RemoteMoeFfn, ShardConfig,

crates/larql-inference/src/ffn/moe_backend.rs

@@ -54,6 +54,71 @@ pub enum MoeBackendError {
     Remote(#[from] RemoteMoeError),
     #[error("bound expert execution failed: {0}")]
     Bound(#[from] larql_vindex::runtime::ExecutionError),
+    /// A routed operand could not be sourced from its VINDEX3 container.
+    ///
+    /// Distinct from [`Self::Bound`]: that is the executor rejecting operands
+    /// it was given, this is not having them. Collapsing the two would report
+    /// a missing bank as a kernel failure and send the reader to the wrong
+    /// half of the system. Reaching this at generation time is itself a bug —
+    /// composition is validated when the container is opened — so the message
+    /// names the layer and expert rather than assuming a retry.
+    #[error("routed container operand unavailable: {0}")]
+    Container(String),
+}
+
+/// What an operation does when a MoE route refuses.
+///
+/// This is a property of the **operation**, not of the backend. The same
+/// backend is legitimately used both to generate (where a failed layer must
+/// abort) and to probe (where a refusal is the measurement). A backend cannot
+/// distinguish those callers, so asking it would bake operation policy into an
+/// operand provider — and would imply a false taxonomy in which some backends'
+/// failures are tolerable. They are not: a remote shard failing mid-generation
+/// and contributing zero is exactly as invalid as a missing container region.
+///
+/// The default for anything that produces a continuation, a score, a parity
+/// number or a benchmark is [`Self::Fatal`].
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum MoeFailurePolicy {
+    /// Abort the operation. No token, score or measurement is produced from a
+    /// forward pass in which any expert contribution was not computed.
+    Fatal,
+    /// Record the refusal and continue with an incomplete result.
+    ///
+    /// Analysis-only. A caller selecting this is declaring that it *intends*
+    /// to inspect partial execution and will report the incompleteness; it may
+    /// not present the outcome as an ordinary model continuation.
+    RecordRefusal,
+}
+
+/// A route plus the policy the calling operation applies to its refusals.
+///
+/// Paired rather than passed separately so a caller cannot supply a backend
+/// and forget to state what a failure means — the omission that let a failed
+/// layer contribute silent zeros for as long as this branch has existed.
+#[derive(Clone, Copy)]
+pub struct MoeRoute<'a> {
+    pub backend: &'a dyn MoeExpertBackend,
+    pub policy: MoeFailurePolicy,
+}
+
+impl<'a> MoeRoute<'a> {
+    /// A route whose failures abort the operation. The correct choice for
+    /// generation, scoring, parity and benchmarking.
+    pub fn fatal(backend: &'a dyn MoeExpertBackend) -> Self {
+        Self {
+            backend,
+            policy: MoeFailurePolicy::Fatal,
+        }
+    }
+
+    /// A route whose failures are recorded and tolerated. Analysis only.
+    pub fn recording(backend: &'a dyn MoeExpertBackend) -> Self {
+        Self {
+            backend,
+            policy: MoeFailurePolicy::RecordRefusal,
+        }
+    }
 }
 
 /// A route that computes one hybrid-MoE layer's expert contribution.