feat(embeddings): add native MLX executor

crates/surreal-memory/src/embeddings/candle.rs

@@ -31,6 +31,7 @@ struct CandleEmbeddingsInner {
 pub struct CandleEmbeddings {
     inner: OnceCell<Arc<Mutex<CandleEmbeddingsInner>>>,
     model_id: String,
+    model_revision: String,
     cache_dir: String,
     expected_dimensions: usize,
 }
@@ -52,6 +53,8 @@ impl CandleEmbeddings {
         Ok(Self {
             inner: OnceCell::new(),
             model_id: model_id.to_string(),
+            model_revision: std::env::var("LOCAL_EMBEDDING_MODEL_REVISION")
+                .unwrap_or_else(|_| "main".to_string()),
             cache_dir: cache_dir.to_string(),
             expected_dimensions,
         })
@@ -98,7 +101,7 @@ impl CandleEmbeddings {
                 tracing::info!("Using device: {:?}", device);
 
                 let (config_path, tokenizer_path, weights_path) =
-                    Self::download_model(&self.model_id, &self.cache_dir)
+                    Self::download_model(&self.model_id, &self.model_revision, &self.cache_dir)
                         .await
                         .context("Failed to download model files")?;
 
@@ -195,6 +198,12 @@ impl CandleEmbeddings {
     }
 
     fn get_device() -> Result<Device> {
+        let device_preference = std::env::var("LOCAL_EMBEDDING_DEVICE").ok();
+        if force_cpu(device_preference.as_deref())? {
+            tracing::warn!("LOCAL_EMBEDDING_DEVICE=cpu: using the explicit degraded CPU backend");
+            return Ok(Device::Cpu);
+        }
+
         #[cfg(feature = "cuda")]
         {
             if candle_core::utils::cuda_is_available() {
@@ -237,6 +246,7 @@ impl CandleEmbeddings {
 
     async fn download_model(
         model_id: &str,
+        model_revision: &str,
         cache_dir: &str,
     ) -> Result<(PathBuf, PathBuf, PathBuf)> {
         // hf-hub keeps repositories under `<hf-home>/hub`: both
@@ -251,8 +261,9 @@ impl CandleEmbeddings {
         let hub_dir = Self::hub_cache_dir(cache_dir);
 
         tracing::info!(
-            "Resolving model from Hugging Face: {} (cache: {})",
+            "Resolving model from Hugging Face: {}@{} (cache: {})",
             model_id,
+            model_revision,
             hub_dir.display()
         );
 
@@ -263,7 +274,11 @@ impl CandleEmbeddings {
             .with_cache_dir(hub_dir)
             .build()
             .context("build Hugging Face API client")?;
-        let repo = api.repo(Repo::new(model_id.to_string(), RepoType::Model));
+        let repo = api.repo(Repo::with_revision(
+            model_id.to_string(),
+            RepoType::Model,
+            model_revision.to_string(),
+        ));
 
         // Download config, tokenizer, and weights concurrently. The weights
         // future tries safetensors first and falls back to the PyTorch file.
@@ -444,6 +459,14 @@ impl CandleEmbeddings {
     }
 }
 
+fn force_cpu(value: Option<&str>) -> Result<bool> {
+    match value.unwrap_or("auto").trim().to_ascii_lowercase().as_str() {
+        "auto" => Ok(false),
+        "cpu" => Ok(true),
+        other => anyhow::bail!("LOCAL_EMBEDDING_DEVICE must be 'auto' or 'cpu', got '{other}'"),
+    }
+}
+
 #[async_trait]
 impl EmbeddingService for CandleEmbeddings {
     async fn embed(&self, text: &str) -> Result<Embedding> {
@@ -643,6 +666,14 @@ mod tests {
         assert_eq!(estimate("sentence-transformers/all-MiniLM-L6-v2"), 384);
     }
 
+    #[test]
+    fn device_preference_is_explicit_and_fail_closed() {
+        assert!(!force_cpu(None).unwrap());
+        assert!(!force_cpu(Some("auto")).unwrap());
+        assert!(force_cpu(Some("CPU")).unwrap());
+        assert!(force_cpu(Some("metal")).is_err());
+    }
+
     #[test]
     fn model_boundary_guard_accepts_below_and_at_capacity_only() {
         assert!(validate_model_input_len(510, 512).is_ok());

docs/lessons.md

@@ -37,6 +37,11 @@ Format: `YYYY-MM-DD — Rule — *(context: what went wrong)*`
 - 2026-08-26 — Check `uptime` before diagnosing a timing bug. Metal init measured 1.2s, 14s, 36s, then never — I attributed that to code before noticing load average 269 and a static `/health` handler taking 21s. Timing symptoms on a saturated machine are not evidence about code.
 - 2026-08-26 — A watchdog must not cover a phase in which the watched process is structurally incapable of reporting progress. The executor child could not heartbeat until after its first request read, yet the 30s per-request watchdog was armed at spawn. Startup needs its own budget and an explicit readiness signal.
 - 2026-08-26 — Never stack a PR onto another PR's branch when the base will merge first. PR #9 targeted PR #8's branch; #8 merged to main, then #9 merged into an orphaned branch and GitHub reported "MERGED" while main had none of it. Target `main` and rebase, or the fix silently never ships.
+- 2026-08-26 — When adding a typed field to a struct that derives Serde, derive the same Serde direction on the field type before the first compile. *(Context: `LocalEmbeddingBackend` was added to deserializable `Config` without `Deserialize`.)*
+- 2026-08-26 — Do not run the aggregate `prometheus-rust-auditor audit` command in this repository: its pipeline generates a GitHub Actions workflow, which violates the local-only validation policy. Run the read-only enforcement, format, dependency, inventory, and partition checks individually.
+- 2026-08-26 — A copied SwiftPM executable is not self-contained when a C target ships resources: install `mlx-swift_Cmlx.bundle` beside the executable and smoke-test the copied path, because the build-tree binary can hide a missing `default.metallib` deployment.
+- 2026-08-26 — Persisted executor generations span server processes, but a newly pre-warmed child starts with process-local numbering. Adopt the healthy child into the durable generation sequence; do not kill it merely because its initial number is lower, or queue recovery turns into an unnecessary cold model launch.
+- 2026-08-26 — A liveness heartbeat must run independently of the work it supervises. An async Swift `Task` heartbeat can be starved while MLX/Metal synchronously occupies a cooperative executor; use a dedicated Dispatch timer and synchronously drain it before writing the terminal protocol message.
 
 ## Schema / migrations *(carried forward from CLAUDE.md operational rules)*
 

executors/mlx/.gitignore

@@ -0,0 +1 @@
+.build/

executors/mlx/Package.swift

@@ -0,0 +1,51 @@
+// swift-tools-version: 6.1
+
+import PackageDescription
+
+let package = Package(
+    name: "surreal-memory-mlx-executor",
+    platforms: [.macOS(.v14)],
+    products: [
+        .executable(
+            name: "surreal-memory-mlx-executor",
+            targets: ["SurrealMemoryMLXExecutor"]
+        )
+    ],
+    dependencies: [
+        .package(
+            url: "https://github.com/ml-explore/mlx-swift-lm",
+            exact: "3.31.4"
+        ),
+        .package(
+            url: "https://github.com/ml-explore/mlx-swift",
+            exact: "0.31.4"
+        ),
+        .package(
+            url: "https://github.com/huggingface/swift-huggingface",
+            exact: "0.9.0"
+        ),
+        .package(
+            url: "https://github.com/huggingface/swift-transformers",
+            exact: "1.3.0"
+        )
+    ],
+    targets: [
+        .target(name: "SurrealMemoryMLXExecutorCore"),
+        .executableTarget(
+            name: "SurrealMemoryMLXExecutor",
+            dependencies: [
+                "SurrealMemoryMLXExecutorCore",
+                .product(name: "MLX", package: "mlx-swift"),
+                .product(name: "MLXEmbedders", package: "mlx-swift-lm"),
+                .product(name: "MLXLMCommon", package: "mlx-swift-lm"),
+                .product(name: "MLXHuggingFace", package: "mlx-swift-lm"),
+                .product(name: "HuggingFace", package: "swift-huggingface"),
+                .product(name: "Tokenizers", package: "swift-transformers")
+            ]
+        ),
+        .testTarget(
+            name: "SurrealMemoryMLXExecutorCoreTests",
+            dependencies: ["SurrealMemoryMLXExecutorCore"]
+        )
+    ]
+)