perf: bound embedding dimension validation

crates/surreal-memory/src/storage/surreal.rs

@@ -531,7 +531,7 @@ struct SchemaMetadataRecord {
 #[derive(Clone, Debug, Deserialize, SurrealValue)]
 struct EmbeddingDimensionRecord {
     id: Option<RecordId>,
-    embedding: Option<Vec<f32>>,
+    dimension: Option<i64>,
 }
 
 impl From<Memory> for DbMemory {
@@ -808,11 +808,7 @@ impl SurrealStorage {
         table: &str,
         expected_dimension: usize,
     ) -> Result<()> {
-        let query = match table {
-            "entity" => "SELECT id, embedding FROM entity WHERE embedding IS NOT NONE",
-            "memory" => "SELECT id, embedding FROM memory WHERE embedding IS NOT NONE",
-            _ => anyhow::bail!("Unsupported embedding table: {}", table),
-        };
+        let query = Self::embedding_dimension_query(table)?;
 
         let rows: Vec<EmbeddingDimensionRecord> = db
             .query(query)
@@ -822,11 +818,10 @@ impl SurrealStorage {
             .unwrap_or_default();
 
         for row in rows {
-            let Some(embedding) = row.embedding else {
+            let Some(actual_dimension) = row.dimension else {
                 continue;
             };
-            let actual_dimension = embedding.len();
-            if actual_dimension != expected_dimension {
+            if actual_dimension != expected_dimension as i64 {
                 let record_id = row
                     .id
                     .as_ref()
@@ -845,6 +840,18 @@ impl SurrealStorage {
         Ok(())
     }
 
+    fn embedding_dimension_query(table: &str) -> Result<&'static str> {
+        match table {
+            "entity" => Ok(
+                "SELECT id, array::len(embedding) AS dimension FROM entity WHERE embedding IS NOT NONE",
+            ),
+            "memory" => Ok(
+                "SELECT id, array::len(embedding) AS dimension FROM memory WHERE embedding IS NOT NONE",
+            ),
+            _ => anyhow::bail!("Unsupported embedding table: {}", table),
+        }
+    }
+
     async fn rebuild_embedding_indexes(db: &Surreal<Any>, dimension: usize) -> Result<()> {
         let response = db
             .query(
@@ -3553,6 +3560,23 @@ impl PalaceStorage for SurrealStorage {
 mod retry_tests {
     use super::*;
 
+    #[test]
+    fn embedding_dimension_validation_never_projects_vectors() {
+        for (table, expected) in [
+            (
+                "entity",
+                "SELECT id, array::len(embedding) AS dimension FROM entity WHERE embedding IS NOT NONE",
+            ),
+            (
+                "memory",
+                "SELECT id, array::len(embedding) AS dimension FROM memory WHERE embedding IS NOT NONE",
+            ),
+        ] {
+            let query = SurrealStorage::embedding_dimension_query(table).unwrap();
+            assert_eq!(query, expected);
+        }
+    }
+
     #[test]
     fn test_retry_config_defaults() {
         let config = RetryConfig::default();

openspec/changes/bound-embedding-dimension-validation/.openspec.yaml

@@ -0,0 +1,2 @@
+schema: spec-driven
+created: 2026-09-20

openspec/changes/bound-embedding-dimension-validation/design.md

@@ -0,0 +1,20 @@
+# Design: Database-side dimension projection
+
+## Decision
+
+Use `array::len(embedding) AS dimension` in the existing table-specific startup
+queries and deserialize the result as an integer. SurrealDB 3.2 supports
+`array::len(array) -> number`; the exact query was also exercised against the
+deployed database before implementation.
+
+## Preserved behavior
+
+- Both `entity` and `memory` rows with embeddings are checked.
+- A mismatched row still identifies its record and actual dimension.
+- Unsupported table names still fail before issuing a query.
+- Index metadata and index-definition behavior are unchanged.
+
+## Evidence target
+
+The production process must bind the REST API without transferring full vectors
+during validation, and the learning worker must resume durable receipt recovery.

openspec/changes/bound-embedding-dimension-validation/proposal.md

@@ -0,0 +1,25 @@
+# Proposal: Bound startup embedding-dimension validation
+
+## Problem
+
+Server startup selects every full embedding vector from the `entity` and
+`memory` tables to verify dimensions. The deployed database returned roughly
+50 MB for this validation and held the REST API before bind under swap pressure.
+
+## Change
+
+Project each vector's length inside SurrealDB and transfer only the record ID
+and integer dimension to the server. Preserve the existing mismatch error and
+the validation of both embedding tables.
+
+## Scope
+
+- Change the startup validation projection and response type.
+- Add a regression that forbids full-vector projection in this path.
+- Measure deployed startup and operation-receipt recovery.
+
+## Uncomfortable fact
+
+This removes the transfer amplification, but it does not reduce SurrealDB's
+cost of visiting every embedded record. A future cardinality increase may need
+a persisted dimension invariant or database-side mismatch predicate.

openspec/changes/bound-embedding-dimension-validation/specs/startup-embedding-validation/spec.md

@@ -0,0 +1,21 @@
+# startup-embedding-validation Specification
+
+## ADDED Requirements
+
+### Requirement: Startup dimension validation has bounded response shape
+
+The server SHALL validate stored embedding dimensions by projecting each
+embedding's dimension inside the database. The validation response SHALL NOT
+contain the embedding vectors themselves.
+
+#### Scenario: Existing dimensions match
+
+- **WHEN** the server validates stored entity and memory embeddings at startup
+- **THEN** it requests only record IDs and computed dimensions
+- **AND** startup proceeds without transferring full vectors
+
+#### Scenario: Existing dimension conflicts
+
+- **WHEN** a stored embedding dimension differs from the active provider
+- **THEN** startup fails with the table, record identity, actual dimension, and
+  expected dimension

openspec/changes/bound-embedding-dimension-validation/tasks.md

@@ -0,0 +1,9 @@
+## 1. Implementation
+
+- [x] 1.1 Project embedding lengths in SurrealDB and preserve mismatch reporting
+- [x] 1.2 Add a regression that rejects full-vector startup projection
+
+## 2. Verification
+
+- [x] 2.1 Pass focused Rust tests and strict OpenSpec validation
+- [ ] 2.2 Deploy and measure startup plus worker backlog progress