From b1cdac60638bb1840273740d78a122ee347b2b9b Mon Sep 17 00:00:00 2001 From: Travis James Date: Sun, 20 Sep 2026 13:00:16 -0500 Subject: [PATCH] perf: bound embedding dimension validation --- crates/surreal-memory/src/storage/surreal.rs | 42 +++++++++++++++---- .../.openspec.yaml | 2 + .../design.md | 20 +++++++++ .../proposal.md | 25 +++++++++++ .../startup-embedding-validation/spec.md | 21 ++++++++++ .../tasks.md | 9 ++++ 6 files changed, 110 insertions(+), 9 deletions(-) create mode 100644 openspec/changes/bound-embedding-dimension-validation/.openspec.yaml create mode 100644 openspec/changes/bound-embedding-dimension-validation/design.md create mode 100644 openspec/changes/bound-embedding-dimension-validation/proposal.md create mode 100644 openspec/changes/bound-embedding-dimension-validation/specs/startup-embedding-validation/spec.md create mode 100644 openspec/changes/bound-embedding-dimension-validation/tasks.md diff --git a/crates/surreal-memory/src/storage/surreal.rs b/crates/surreal-memory/src/storage/surreal.rs index fb71f3f..84dad01 100644 --- a/crates/surreal-memory/src/storage/surreal.rs +++ b/crates/surreal-memory/src/storage/surreal.rs @@ -531,7 +531,7 @@ struct SchemaMetadataRecord { #[derive(Clone, Debug, Deserialize, SurrealValue)] struct EmbeddingDimensionRecord { id: Option, - embedding: Option>, + dimension: Option, } impl From for DbMemory { @@ -808,11 +808,7 @@ impl SurrealStorage { table: &str, expected_dimension: usize, ) -> Result<()> { - let query = match table { - "entity" => "SELECT id, embedding FROM entity WHERE embedding IS NOT NONE", - "memory" => "SELECT id, embedding FROM memory WHERE embedding IS NOT NONE", - _ => anyhow::bail!("Unsupported embedding table: {}", table), - }; + let query = Self::embedding_dimension_query(table)?; let rows: Vec = db .query(query) @@ -822,11 +818,10 @@ impl SurrealStorage { .unwrap_or_default(); for row in rows { - let Some(embedding) = row.embedding else { + let Some(actual_dimension) = row.dimension else { continue; }; - let actual_dimension = embedding.len(); - if actual_dimension != expected_dimension { + if actual_dimension != expected_dimension as i64 { let record_id = row .id .as_ref() @@ -845,6 +840,18 @@ impl SurrealStorage { Ok(()) } + fn embedding_dimension_query(table: &str) -> Result<&'static str> { + match table { + "entity" => Ok( + "SELECT id, array::len(embedding) AS dimension FROM entity WHERE embedding IS NOT NONE", + ), + "memory" => Ok( + "SELECT id, array::len(embedding) AS dimension FROM memory WHERE embedding IS NOT NONE", + ), + _ => anyhow::bail!("Unsupported embedding table: {}", table), + } + } + async fn rebuild_embedding_indexes(db: &Surreal, dimension: usize) -> Result<()> { let response = db .query( @@ -3553,6 +3560,23 @@ impl PalaceStorage for SurrealStorage { mod retry_tests { use super::*; + #[test] + fn embedding_dimension_validation_never_projects_vectors() { + for (table, expected) in [ + ( + "entity", + "SELECT id, array::len(embedding) AS dimension FROM entity WHERE embedding IS NOT NONE", + ), + ( + "memory", + "SELECT id, array::len(embedding) AS dimension FROM memory WHERE embedding IS NOT NONE", + ), + ] { + let query = SurrealStorage::embedding_dimension_query(table).unwrap(); + assert_eq!(query, expected); + } + } + #[test] fn test_retry_config_defaults() { let config = RetryConfig::default(); diff --git a/openspec/changes/bound-embedding-dimension-validation/.openspec.yaml b/openspec/changes/bound-embedding-dimension-validation/.openspec.yaml new file mode 100644 index 0000000..cbd245e --- /dev/null +++ b/openspec/changes/bound-embedding-dimension-validation/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-20 diff --git a/openspec/changes/bound-embedding-dimension-validation/design.md b/openspec/changes/bound-embedding-dimension-validation/design.md new file mode 100644 index 0000000..933745c --- /dev/null +++ b/openspec/changes/bound-embedding-dimension-validation/design.md @@ -0,0 +1,20 @@ +# Design: Database-side dimension projection + +## Decision + +Use `array::len(embedding) AS dimension` in the existing table-specific startup +queries and deserialize the result as an integer. SurrealDB 3.2 supports +`array::len(array) -> number`; the exact query was also exercised against the +deployed database before implementation. + +## Preserved behavior + +- Both `entity` and `memory` rows with embeddings are checked. +- A mismatched row still identifies its record and actual dimension. +- Unsupported table names still fail before issuing a query. +- Index metadata and index-definition behavior are unchanged. + +## Evidence target + +The production process must bind the REST API without transferring full vectors +during validation, and the learning worker must resume durable receipt recovery. diff --git a/openspec/changes/bound-embedding-dimension-validation/proposal.md b/openspec/changes/bound-embedding-dimension-validation/proposal.md new file mode 100644 index 0000000..b5389c7 --- /dev/null +++ b/openspec/changes/bound-embedding-dimension-validation/proposal.md @@ -0,0 +1,25 @@ +# Proposal: Bound startup embedding-dimension validation + +## Problem + +Server startup selects every full embedding vector from the `entity` and +`memory` tables to verify dimensions. The deployed database returned roughly +50 MB for this validation and held the REST API before bind under swap pressure. + +## Change + +Project each vector's length inside SurrealDB and transfer only the record ID +and integer dimension to the server. Preserve the existing mismatch error and +the validation of both embedding tables. + +## Scope + +- Change the startup validation projection and response type. +- Add a regression that forbids full-vector projection in this path. +- Measure deployed startup and operation-receipt recovery. + +## Uncomfortable fact + +This removes the transfer amplification, but it does not reduce SurrealDB's +cost of visiting every embedded record. A future cardinality increase may need +a persisted dimension invariant or database-side mismatch predicate. diff --git a/openspec/changes/bound-embedding-dimension-validation/specs/startup-embedding-validation/spec.md b/openspec/changes/bound-embedding-dimension-validation/specs/startup-embedding-validation/spec.md new file mode 100644 index 0000000..533ea57 --- /dev/null +++ b/openspec/changes/bound-embedding-dimension-validation/specs/startup-embedding-validation/spec.md @@ -0,0 +1,21 @@ +# startup-embedding-validation Specification + +## ADDED Requirements + +### Requirement: Startup dimension validation has bounded response shape + +The server SHALL validate stored embedding dimensions by projecting each +embedding's dimension inside the database. The validation response SHALL NOT +contain the embedding vectors themselves. + +#### Scenario: Existing dimensions match + +- **WHEN** the server validates stored entity and memory embeddings at startup +- **THEN** it requests only record IDs and computed dimensions +- **AND** startup proceeds without transferring full vectors + +#### Scenario: Existing dimension conflicts + +- **WHEN** a stored embedding dimension differs from the active provider +- **THEN** startup fails with the table, record identity, actual dimension, and + expected dimension diff --git a/openspec/changes/bound-embedding-dimension-validation/tasks.md b/openspec/changes/bound-embedding-dimension-validation/tasks.md new file mode 100644 index 0000000..9dcd299 --- /dev/null +++ b/openspec/changes/bound-embedding-dimension-validation/tasks.md @@ -0,0 +1,9 @@ +## 1. Implementation + +- [x] 1.1 Project embedding lengths in SurrealDB and preserve mismatch reporting +- [x] 1.2 Add a regression that rejects full-vector startup projection + +## 2. Verification + +- [x] 2.1 Pass focused Rust tests and strict OpenSpec validation +- [ ] 2.2 Deploy and measure startup plus worker backlog progress