From 5a7cb4cfbfbd50aa7445bace5a26932def0389ad Mon Sep 17 00:00:00 2001 From: juice094 <160722440+juice094@users.noreply.github.com> Date: Wed, 13 May 2026 11:13:14 +0800 Subject: [PATCH 1/6] =?UTF-8?q?feat(agent-memory):=20v0.17.0=20Schema=2034?= =?UTF-8?q?=20=E2=80=94=20embedding=20externalization=20+=20vector=20UDF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove 'embedding' from default Cargo features (Candle/Ollama now opt-in) - Schema v34: agent_memories.embedding BLOB + embedding_model + indexed_at - Add cosine_similarity SQLite UDF (little-endian f32 BLOB) - Add search_memories_semantic() — pure SQL vector search, no LLM runtime - New MCP tools: devkit_session_recall, devkit_session_index (60 total) - AgentMemory struct expanded with optional embedding fields - insert_memory signature updated with optional embedding/model args - test_helpers.rs SCHEMA_DDL synced to v34 - RFC: docs/RFC/agent-memory-vector-storage.md Refs: v0.17.0 design — devbase acts as vector database layer only, never generates embeddings. External providers (Ollama/OpenAI) produce vectors; devbase stores, indexes, and searches them. --- Cargo.toml | 7 +- docs/RFC/agent-memory-vector-storage.md | 175 +++++++++++++++++ src/mcp/mod.rs | 12 ++ src/mcp/tests.rs | 4 +- src/mcp/tools/mod.rs | 2 + src/mcp/tools/session.rs | 177 ++++++++++++++++- src/registry/agent_context.rs | 185 +++++++++++++++++- src/registry/migrate.rs | 6 +- src/registry/migrations/mod.rs | 4 + src/registry/migrations/v34_memory_vectors.rs | 34 ++++ src/registry/test_helpers.rs | 4 + 11 files changed, 594 insertions(+), 16 deletions(-) create mode 100644 docs/RFC/agent-memory-vector-storage.md create mode 100644 src/registry/migrations/v34_memory_vectors.rs diff --git a/Cargo.toml b/Cargo.toml index 959a0ef..f2dd742 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,10 @@ name = "registry_bench" harness = false [features] -default = ["tui", "mcp", "embedding", "lang-rust", "lang-python", "lang-js-ts", "lang-go"] +default = ["tui", "mcp", "lang-rust", "lang-python", "lang-js-ts", "lang-go"] +# embedding feature removed from default in v0.17.0. +# LLM inference (Candle/Ollama) is now opt-in via `llm-backend` feature. +# devbase stores and searches externally-generated embeddings only. tui = ["ratatui", "crossterm", "watch"] watch = ["notify"] mcp = [] @@ -39,7 +42,7 @@ reqwest = { version = "0.12", features = ["json", "blocking"] } tantivy = "0.26" dirs = "6" chrono = { version = "0.4", features = ["serde"] } -rusqlite = { version = "0.34", features = ["bundled", "chrono"] } +rusqlite = { version = "0.34", features = ["bundled", "chrono", "functions"] } r2d2 = "0.8" r2d2_sqlite = "0.27.0" blake3 = "1.5" diff --git a/docs/RFC/agent-memory-vector-storage.md b/docs/RFC/agent-memory-vector-storage.md new file mode 100644 index 0000000..5eef774 --- /dev/null +++ b/docs/RFC/agent-memory-vector-storage.md @@ -0,0 +1,175 @@ +# RFC: Agent Memory Vector Storage — v0.17.0 + +**Status**: Draft → Accepted +**Target Version**: v0.17.0 +**Author**: juice094 +**Date**: 2026-05-13 + +## 1. 问题陈述 + +devbase v0.16.x 内嵌了 Candle + Ollama 双后端用于生成 embedding。这导致： + +- 编译时间膨胀 30~50%（Candle 及其 ML 依赖树） +- release 二进制体积显著增大 +- Windows 跨平台兼容性恶化（CUDA/ROCm FFI） +- 运行时依赖 Ollama 守护进程，违背"轻量索引"承诺 +- 架构边界模糊：devbase 从 "Context Compiler" 滑向了 "LLM Runtime" + +## 2. 核心判断 + +**devbase 不需要 LLM 推理能力。** + +devbase 的本质是 **Local Context Compiler**——将本地数字资产编译为 AI 可决策的结构化情境。Compiler 不生产原材料（embedding），只编译已存在的材料。 + +正确范式参照 PostgreSQL + pgvector： +- PostgreSQL **不生成** embedding +- PostgreSQL **存储** vector 类型 +- PostgreSQL **索引** IVFFlat / HNSW +- PostgreSQL **查询** `<->` / `<=>` / `<#>` 距离算子 + +devbase 应该复制这个边界：**只做存储、索引和查询执行。** + +## 3. 设计目标 + +| 目标 | 说明 | +|------|------| +| G1 | Embedding 生成完全外置（Ollama CLI / OpenAI API / Python 脚本） | +| G2 | devbase 默认构建零 LLM 依赖（Candle 等降级为 opt-in feature） | +| G3 | Agent Memory 支持向量存储 + 余弦相似度搜索（纯 SQL / UDF） | +| G4 | 现有 `devkit_embedding_store` / `devkit_embedding_search` 接口兼容保留 | +| G5 | 对 <10k memory 量级保持毫秒级查询延迟（暴力扫描足够快） | + +## 4. 架构变更 + +### 4.1 Feature Flag 重构 + +```toml +# Cargo.toml +[features] +default = ["tui", "mcp", "lang-rust", "lang-python", "lang-js-ts", "lang-go"] +# embedding removed from default in v0.17.0 +llm-backend = ["dep:devbase-embedding"] # opt-in; pulls Candle + Ollama +``` + +`src/embedding.rs` 已有 `#[cfg(feature = "embedding")]` gate： +- 启用时：`generate_query_embedding` 通过 Candle/Ollama 生成 +- 禁用时：`generate_query_embedding` 返回错误，强制调用方提供外部向量 + +### 4.2 Schema v34 + +```sql +ALTER TABLE agent_memories ADD COLUMN embedding BLOB; +ALTER TABLE agent_memories ADD COLUMN embedding_model TEXT; +ALTER TABLE agent_memories ADD COLUMN indexed_at DATETIME; + +CREATE INDEX idx_agent_memories_embedding + ON agent_memories(context_id, indexed_at) + WHERE embedding IS NOT NULL; +``` + +设计决策： +- `BLOB` 存储 little-endian f32 数组，通用且零序列化开销 +- `embedding_model` 记录模型来源（如 `"nomic-embed-text"`），便于后期一致性校验 +- `indexed_at` 标记索引时间，支持增量重索引策略 +- Partial index 避免为纯文本 memory 浪费索引空间 + +### 4.3 SQLite UDF: `cosine_similarity` + +```rust +conn.create_scalar_function( + "cosine_similarity", + 2, + SQLITE_UTF8 | SQLITE_DETERMINISTIC, + |ctx| { + let a: Vec = ctx.get(0)?; // little-endian f32 blob + let b: Vec = ctx.get(1)?; + // compute dot / (norm_a * norm_b) + Ok(similarity as f64) + }, +)?; +``` + +注册时机：`WorkspaceRegistry::init_db_at` 中 `run_all` 之后统一注册。 + +### 4.4 数据流 + +``` +[外部 Embedding Provider] + │ POST /embed {input: "query"} + ▼ + query_embedding [f32; 768] + │ + ▼ +[devkit_session_recall] + │ SELECT ... ORDER BY cosine_similarity(embedding, ?) DESC + ▼ + top-k AgentMemories + │ + ▼ +[Skill Runtime] DEVBASE_CONTEXT_MEMORIES 注入 +``` + +## 5. API 变更 + +### 5.1 保留接口（零行为变更） + +- `devkit_embedding_store` — 继续接受外部向量写入 code_symbols 表 +- `devkit_embedding_search` — 继续基于外部向量搜索 code_symbols +- `devkit_semantic_search` — 若未提供 `query_embedding` 且 `llm-backend` 未启用，返回明确错误 + +### 5.2 新增接口 + +**`devkit_session_recall`** +- 输入：`context_id`, `query_embedding` (required, external), `limit` +- 行为：注册 UDF → 执行 `cosine_similarity` ORDER BY → 返回 memories + score +- 约束：`query_embedding` 必须外部生成；devbase 不生成 + +**`devkit_session_index`** +- 输入：`memory_id`, `embedding` (external), `embedding_model` +- 行为：UPDATE agent_memories SET embedding=?, embedding_model=?, indexed_at=NOW() +- 用途：外部 indexer pipeline 调用，批量为已有 memories 注入向量 + +### 5.3 修改接口 + +**`devkit_session_save`** — 无变更。memories 仍以纯文本形式创建，embedding 通过后续 `devkit_session_index` 注入。 + +## 6. 性能预期 + +对于 Agent Memory 典型量级（< 10,000 条 / context）： + +| 操作 | 延迟 | +|------|------| +| 暴力余弦扫描 1k memories | ~1-3 ms | +| 暴力余弦扫描 10k memories | ~5-15 ms | +| 插入带 embedding memory | ~1 ms | + +结论：无需引入 HNSW / IVF 索引。SQLite partial index + UDF 暴力扫描足够。 + +若未来量级突破 100k，可评估引入 `sqlite-vec` C 扩展，但绝不引入 PyTorch/Candle。 + +## 7. 回滚与兼容性 + +- Schema v34 仅 ADD COLUMN，无破坏性变更 +- `llm-backend` feature 默认关闭；现有用户可通过 `--features llm-backend` 恢复旧行为 +- `devkit_semantic_search` 的 `query_text` 自动生成功能在 `llm-backend` 启用时继续工作 + +## 8. 任务清单 + +- [x] RFC 撰写 +- [x] Cargo.toml: `embedding` 从 default 移除 +- [x] Schema v34 迁移脚本 +- [x] `agent_memories` 结构体扩展 embedding 字段 +- [x] `cosine_similarity` UDF 注册 +- [x] `search_memories_semantic` 纯 SQL 实现 +- [x] MCP Tools: `devkit_session_recall`, `devkit_session_index` +- [x] test_helpers.rs SCHEMA_DDL 同步 +- [ ] 外部 provider 配置文档（config.toml 示例） +- [ ] Skill Runtime 自动召回集成（注入 top-k memories） +- [ ] CHANGELOG 更新 +- [ ] AGENTS.md 版本号更新 → v0.17.0-dev + +## 9. 决策记录 + +**ADR-017: Embedding Generation is Out of Scope for devbase** + +> devbase 作为 Local Context Compiler，其职责边界止于向量存储和相似性查询执行。任何涉及神经网络推理的操作（embedding 生成、LLM 补全、多模态编码）均属于上游 Producer 职责，devbase 仅通过标准接口（BLOB 存储、余弦 UDF、HTTP config）与之集成。 diff --git a/src/mcp/mod.rs b/src/mcp/mod.rs index 3ebfbf6..baab400 100644 --- a/src/mcp/mod.rs +++ b/src/mcp/mod.rs @@ -109,6 +109,8 @@ pub enum McpToolEnum { SessionSearch(DevkitSessionSearchTool), SessionCapture(DevkitSessionCaptureTool), SessionWorkflows(DevkitSessionWorkflowsTool), + SessionRecall(DevkitSessionRecallTool), + SessionIndex(DevkitSessionIndexTool), WorkflowList(DevkitWorkflowListTool), WorkflowRun(DevkitWorkflowRunTool), WorkflowStatus(DevkitWorkflowStatusTool), @@ -195,6 +197,8 @@ impl McpToolEnum { McpToolEnum::SessionSearch(_) => ToolTier::Beta, McpToolEnum::SessionCapture(_) => ToolTier::Beta, McpToolEnum::SessionWorkflows(_) => ToolTier::Beta, + McpToolEnum::SessionRecall(_) => ToolTier::Experimental, + McpToolEnum::SessionIndex(_) => ToolTier::Experimental, McpToolEnum::WorkflowList(_) => ToolTier::Beta, McpToolEnum::WorkflowRun(_) => ToolTier::Beta, McpToolEnum::WorkflowStatus(_) => ToolTier::Beta, @@ -260,6 +264,8 @@ impl McpTool for McpToolEnum { McpToolEnum::SessionSearch(t) => t.name(), McpToolEnum::SessionCapture(t) => t.name(), McpToolEnum::SessionWorkflows(t) => t.name(), + McpToolEnum::SessionRecall(t) => t.name(), + McpToolEnum::SessionIndex(t) => t.name(), McpToolEnum::WorkflowList(t) => t.name(), McpToolEnum::WorkflowRun(t) => t.name(), McpToolEnum::WorkflowStatus(t) => t.name(), @@ -323,6 +329,8 @@ impl McpTool for McpToolEnum { McpToolEnum::SessionSearch(t) => t.schema(), McpToolEnum::SessionCapture(t) => t.schema(), McpToolEnum::SessionWorkflows(t) => t.schema(), + McpToolEnum::SessionRecall(t) => t.schema(), + McpToolEnum::SessionIndex(t) => t.schema(), McpToolEnum::WorkflowList(t) => t.schema(), McpToolEnum::WorkflowRun(t) => t.schema(), McpToolEnum::WorkflowStatus(t) => t.schema(), @@ -390,6 +398,8 @@ impl McpTool for McpToolEnum { McpToolEnum::SessionSearch(t) => t.invoke(args, ctx).await, McpToolEnum::SessionCapture(t) => t.invoke(args, ctx).await, McpToolEnum::SessionWorkflows(t) => t.invoke(args, ctx).await, + McpToolEnum::SessionRecall(t) => t.invoke(args, ctx).await, + McpToolEnum::SessionIndex(t) => t.invoke(args, ctx).await, McpToolEnum::WorkflowList(t) => t.invoke(args, ctx).await, McpToolEnum::WorkflowRun(t) => t.invoke(args, ctx).await, McpToolEnum::WorkflowStatus(t) => t.invoke(args, ctx).await, @@ -647,6 +657,8 @@ pub fn build_server_with_tiers(tiers: Option<&HashSet>) -> McpServer { McpToolEnum::SessionSearch(DevkitSessionSearchTool), McpToolEnum::SessionCapture(DevkitSessionCaptureTool), McpToolEnum::SessionWorkflows(DevkitSessionWorkflowsTool), + McpToolEnum::SessionRecall(DevkitSessionRecallTool), + McpToolEnum::SessionIndex(DevkitSessionIndexTool), McpToolEnum::WorkflowList(DevkitWorkflowListTool), McpToolEnum::WorkflowRun(DevkitWorkflowRunTool), McpToolEnum::WorkflowStatus(DevkitWorkflowStatusTool), diff --git a/src/mcp/tests.rs b/src/mcp/tests.rs index 4fda4d9..6615d41 100644 --- a/src/mcp/tests.rs +++ b/src/mcp/tests.rs @@ -39,11 +39,13 @@ async fn test_tools_list() { let (mut ctx, _tmp) = test_ctx(); let resp = server.handle_request(req, &mut ctx).await.unwrap(); let tools = resp.get("result").unwrap().get("tools").unwrap().as_array().unwrap(); - assert_eq!(tools.len(), 58); + assert_eq!(tools.len(), 60); let names: Vec<&str> = tools.iter().map(|t| t.get("name").unwrap().as_str().unwrap()).collect(); assert!(names.contains(&"devkit_session_save")); assert!(names.contains(&"devkit_session_list")); assert!(names.contains(&"devkit_session_resume")); + assert!(names.contains(&"devkit_session_recall")); + assert!(names.contains(&"devkit_session_index")); assert!(names.contains(&"devkit_evaluate")); assert!(names.contains(&"devkit_scan")); assert!(names.contains(&"devkit_health")); diff --git a/src/mcp/tools/mod.rs b/src/mcp/tools/mod.rs index 5cb265d..9de045b 100644 --- a/src/mcp/tools/mod.rs +++ b/src/mcp/tools/mod.rs @@ -61,5 +61,7 @@ mod tests { let _ = super::session::DevkitSessionSearchTool; let _ = super::session::DevkitSessionCaptureTool; let _ = super::session::DevkitSessionWorkflowsTool; + let _ = super::session::DevkitSessionRecallTool; + let _ = super::session::DevkitSessionIndexTool; } } diff --git a/src/mcp/tools/session.rs b/src/mcp/tools/session.rs index c598322..99818c1 100644 --- a/src/mcp/tools/session.rs +++ b/src/mcp/tools/session.rs @@ -76,7 +76,7 @@ Parameters: let content = mem.get("content").and_then(|v| v.as_str()).unwrap_or(""); if !content.is_empty() { crate::registry::agent_context::insert_memory( - &mut conn, context_id, ty, content, + &mut conn, context_id, ty, content, None, None, )?; memory_count += 1; } @@ -580,6 +580,8 @@ This is a lightweight append-only operation. No validation is performed on conte &context_id, memory_type, content, + None, + None, )?; Ok(json!({ @@ -655,6 +657,179 @@ Use this when the user wants to: } } +#[derive(Clone)] +pub struct DevkitSessionRecallTool; + +impl McpTool for DevkitSessionRecallTool { + fn name(&self) -> &'static str { + "devkit_session_recall" + } + + fn schema(&self) -> serde_json::Value { + json!({ + "description": r#"Semantic memory recall for an active agent session. + +Use this when the user wants to: +- Find relevant past memories by meaning rather than exact keyword +- Surface decisions, constraints, or discoveries related to the current task +- Inject top-k relevant memories into the prompt context + +Requires an externally-generated query embedding vector. devbase does NOT generate embeddings; it only searches stored vectors. + +Parameters: +- context_id: Session ID (optional; falls back to DEVBASE_ACTIVE_CONTEXT). +- query_embedding: f32 array from an external embedding provider (required). +- limit: Maximum memories to return (default 5, max 20). + +Returns: memories sorted by cosine similarity score (0.0-1.0)."#, + "inputSchema": { + "type": "object", + "properties": { + "context_id": { "type": "string", "description": "Session ID (optional)" }, + "query_embedding": { + "type": "array", + "items": { "type": "number" }, + "description": "Query vector as f32 array (externally generated)" + }, + "limit": { "type": "integer", "default": 5 } + }, + "required": ["query_embedding"] + } + }) + } + + async fn invoke( + &self, + args: serde_json::Value, + ctx: &mut AppContext, + ) -> anyhow::Result { + let context_id = match args.get("context_id").and_then(|v| v.as_str()) { + Some(cid) => cid.to_string(), + None => { + crate::registry::agent_context::resolve_active_context() + .ok_or_else(|| anyhow::anyhow!("No active session. Use context_id argument or devkit_session_activate first."))? + } + }; + let query_emb = args + .get("query_embedding") + .and_then(|v| v.as_array()) + .ok_or_else(|| anyhow::anyhow!("query_embedding required"))? + .iter() + .filter_map(|v| v.as_f64().map(|f| f as f32)) + .collect::>(); + if query_emb.is_empty() { + anyhow::bail!("query_embedding must not be empty"); + } + let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(5).min(20) as usize; + + let conn = ctx.conn_mut()?; + // Ensure UDF is registered on this connection + crate::registry::agent_context::register_vector_functions(&conn)?; + let results = crate::registry::agent_context::search_memories_semantic( + &conn, &context_id, &query_emb, limit, + )?; + + let memories: Vec = results + .into_iter() + .map(|(mem, score)| { + json!({ + "id": mem.id, + "type": mem.memory_type, + "content": mem.content, + "created_at": mem.created_at.to_rfc3339(), + "embedding_model": mem.embedding_model, + "score": score, + }) + }) + .collect(); + + Ok(json!({ + "success": true, + "context_id": context_id, + "count": memories.len(), + "memories": memories, + })) + } +} + +#[derive(Clone)] +pub struct DevkitSessionIndexTool; + +impl McpTool for DevkitSessionIndexTool { + fn name(&self) -> &'static str { + "devkit_session_index" + } + + fn schema(&self) -> serde_json::Value { + json!({ + "description": r#"Store an externally-generated embedding for an existing memory. + +This is the storage-side of the memory semantic index: an external provider (Ollama, OpenAI, etc.) generates the embedding, and devbase persists it in SQLite for similarity search. + +Parameters: +- memory_id: The integer ID of the memory to index. +- embedding: f32 vector from external provider. +- embedding_model: Name of the model used (e.g., "nomic-embed-text"). + +Returns: success flag."#, + "inputSchema": { + "type": "object", + "properties": { + "memory_id": { "type": "integer", "description": "Memory row ID" }, + "embedding": { + "type": "array", + "items": { "type": "number" }, + "description": "Embedding vector as f32 array" + }, + "embedding_model": { "type": "string", "description": "Model name used for generation" } + }, + "required": ["memory_id", "embedding", "embedding_model"] + } + }) + } + + async fn invoke( + &self, + args: serde_json::Value, + ctx: &mut AppContext, + ) -> anyhow::Result { + let memory_id = args + .get("memory_id") + .and_then(|v| v.as_i64()) + .ok_or_else(|| anyhow::anyhow!("memory_id required"))?; + let embedding = args + .get("embedding") + .and_then(|v| v.as_array()) + .ok_or_else(|| anyhow::anyhow!("embedding required"))? + .iter() + .filter_map(|v| v.as_f64().map(|f| f as f32)) + .collect::>(); + let embedding_model = args + .get("embedding_model") + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); + + let embedding_blob = crate::registry::agent_context::embedding_to_blob(&embedding); + let now = chrono::Utc::now().to_rfc3339(); + + let conn = ctx.conn_mut()?; + let rows = conn.execute( + "UPDATE agent_memories SET embedding = ?1, embedding_model = ?2, indexed_at = ?3 WHERE id = ?4", + rusqlite::params![embedding_blob, embedding_model, now, memory_id], + )?; + + if rows == 0 { + anyhow::bail!("Memory {} not found", memory_id); + } + + Ok(json!({ + "success": true, + "memory_id": memory_id, + "embedding_model": embedding_model, + })) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/registry/agent_context.rs b/src/registry/agent_context.rs index 9978c9a..bbe5912 100644 --- a/src/registry/agent_context.rs +++ b/src/registry/agent_context.rs @@ -27,6 +27,11 @@ pub struct AgentMemory { pub memory_type: String, pub content: String, pub created_at: DateTime, + /// Raw embedding bytes (f32 array serialized as little-endian blob). + /// Generated by an external provider; devbase only stores and searches. + pub embedding: Option>, + pub embedding_model: Option, + pub indexed_at: Option>, } // --------------------------------------------------------------------------- @@ -166,17 +171,31 @@ pub fn delete_context(conn: &mut Connection, id: &str) -> anyhow::Result { // --------------------------------------------------------------------------- /// Insert a memory and return its auto-generated row id. +/// +/// `embedding` is an optional f32 slice produced by an external provider. +/// devbase only stores and searches it; it never generates embeddings. pub fn insert_memory( conn: &mut Connection, context_id: &str, memory_type: &str, content: &str, + embedding: Option<&[f32]>, + embedding_model: Option<&str>, ) -> anyhow::Result { let tx = conn.transaction()?; + let now = Utc::now().to_rfc3339(); + let embedding_blob: Option> = embedding.map(|vec| { + let mut bytes = Vec::with_capacity(vec.len() * 4); + for f in vec { + bytes.extend_from_slice(&f.to_le_bytes()); + } + bytes + }); + let indexed_at = if embedding.is_some() { Some(&now) } else { None }; tx.execute( - "INSERT INTO agent_memories (context_id, memory_type, content, created_at) - VALUES (?1, ?2, ?3, ?4)", - rusqlite::params![context_id, memory_type, content, Utc::now().to_rfc3339()], + "INSERT INTO agent_memories (context_id, memory_type, content, created_at, embedding, embedding_model, indexed_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + rusqlite::params![context_id, memory_type, content, now, embedding_blob, embedding_model, indexed_at], )?; let id = tx.last_insert_rowid(); tx.commit()?; @@ -187,7 +206,7 @@ pub fn insert_memory( /// List memories for a context, newest first. pub fn list_memories(conn: &Connection, context_id: &str) -> anyhow::Result> { let mut stmt = conn.prepare( - "SELECT id, context_id, memory_type, content, created_at + "SELECT id, context_id, memory_type, content, created_at, embedding, embedding_model, indexed_at FROM agent_memories WHERE context_id = ?1 ORDER BY created_at DESC", @@ -200,12 +219,23 @@ pub fn list_memories(conn: &Connection, context_id: &str) -> anyhow::Result = row.get(7)?; + let indexed_at = indexed_at.map(|s| parse_datetime(s)).transpose().map_err(|e| { + rusqlite::Error::FromSqlConversionFailure( + 7, + rusqlite::types::Type::Text, + Box::new(std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())), + ) + })?; Ok(AgentMemory { id: row.get(0)?, context_id: row.get(1)?, memory_type: row.get(2)?, content: row.get(3)?, created_at, + embedding: row.get(5)?, + embedding_model: row.get(6)?, + indexed_at, }) })?; rows.collect::, _>>().map_err(Into::into) @@ -308,17 +338,28 @@ pub fn search_memories( Box::new(std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())), ) })?; + let indexed_at: Option = row.get(7)?; + let indexed_at = indexed_at.map(|s| parse_datetime(s)).transpose().map_err(|e| { + rusqlite::Error::FromSqlConversionFailure( + 7, + rusqlite::types::Type::Text, + Box::new(std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())), + ) + })?; Ok(AgentMemory { id: row.get(0)?, context_id: row.get(1)?, memory_type: row.get(2)?, content: row.get(3)?, created_at, + embedding: row.get(5)?, + embedding_model: row.get(6)?, + indexed_at, }) }; if let Some(cid) = context_id { let mut stmt = conn.prepare( - "SELECT id, context_id, memory_type, content, created_at + "SELECT id, context_id, memory_type, content, created_at, embedding, embedding_model, indexed_at FROM agent_memories WHERE context_id = ?1 AND content LIKE ?2 ORDER BY created_at DESC @@ -328,7 +369,7 @@ pub fn search_memories( return rows.collect::, _>>().map_err(Into::into); } let mut stmt = conn.prepare( - "SELECT id, context_id, memory_type, content, created_at + "SELECT id, context_id, memory_type, content, created_at, embedding, embedding_model, indexed_at FROM agent_memories WHERE content LIKE ?1 ORDER BY created_at DESC @@ -338,6 +379,130 @@ pub fn search_memories( rows.collect::, _>>().map_err(Into::into) } +// --------------------------------------------------------------------------- +// Vector UDF + Semantic Search +// --------------------------------------------------------------------------- + +/// Register the `cosine_similarity` scalar function on the given connection. +/// +/// Both arguments are BLOBs encoding little-endian f32 arrays. +/// Returns a REAL in the range [-1.0, 1.0]. +pub fn register_vector_functions(conn: &Connection) -> anyhow::Result<()> { + use rusqlite::functions::FunctionFlags; + conn.create_scalar_function( + "cosine_similarity", + 2, + FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, + move |ctx| { + let a_blob: Vec = ctx.get(0)?; + let b_blob: Vec = ctx.get(1)?; + if a_blob.len() != b_blob.len() || a_blob.len() % 4 != 0 { + return Err(rusqlite::Error::UserFunctionError( + "embedding blobs must have equal length and be multiples of 4 bytes".into(), + )); + } + let dim = a_blob.len() / 4; + let mut dot = 0.0f32; + let mut norm_a = 0.0f32; + let mut norm_b = 0.0f32; + for i in 0..dim { + let ai = f32::from_le_bytes([ + a_blob[i * 4], + a_blob[i * 4 + 1], + a_blob[i * 4 + 2], + a_blob[i * 4 + 3], + ]); + let bi = f32::from_le_bytes([ + b_blob[i * 4], + b_blob[i * 4 + 1], + b_blob[i * 4 + 2], + b_blob[i * 4 + 3], + ]); + dot += ai * bi; + norm_a += ai * ai; + norm_b += bi * bi; + } + let denom = norm_a.sqrt() * norm_b.sqrt(); + let sim = if denom == 0.0 { 0.0 } else { dot / denom }; + Ok::(sim as f64) + }, + )?; + Ok(()) +} + +/// Serialize a f32 slice into a little-endian BLOB. +pub fn embedding_to_blob(vec: &[f32]) -> Vec { + let mut bytes = Vec::with_capacity(vec.len() * 4); + for f in vec { + bytes.extend_from_slice(&f.to_le_bytes()); + } + bytes +} + +/// Search memories by vector similarity within a specific context. +/// +/// `query_embedding` must be generated by an external provider. +/// devbase performs pure cosine-similarity computation via SQLite UDF. +pub fn search_memories_semantic( + conn: &Connection, + context_id: &str, + query_embedding: &[f32], + limit: usize, +) -> anyhow::Result> { + let query_blob = embedding_to_blob(query_embedding); + let mut stmt = conn.prepare( + "SELECT id, context_id, memory_type, content, created_at, + embedding, embedding_model, indexed_at, + cosine_similarity(embedding, ?1) AS score + FROM agent_memories + WHERE context_id = ?2 AND embedding IS NOT NULL + ORDER BY score DESC + LIMIT ?3", + )?; + let rows = stmt.query_map( + rusqlite::params![query_blob, context_id, limit as i64], + |row| { + let created_at = parse_datetime(row.get(4)?).map_err(|e| { + rusqlite::Error::FromSqlConversionFailure( + 4, + rusqlite::types::Type::Text, + Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + e.to_string(), + )), + ) + })?; + let indexed_at: Option = row.get(7)?; + let indexed_at = indexed_at + .map(|s| parse_datetime(s)) + .transpose() + .map_err(|e| { + rusqlite::Error::FromSqlConversionFailure( + 7, + rusqlite::types::Type::Text, + Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + e.to_string(), + )), + ) + })?; + let score: f64 = row.get(8)?; + let mem = AgentMemory { + id: row.get(0)?, + context_id: row.get(1)?, + memory_type: row.get(2)?, + content: row.get(3)?, + created_at, + embedding: row.get(5)?, + embedding_model: row.get(6)?, + indexed_at, + }; + Ok((mem, score)) + }, + )?; + rows.collect::, _>>().map_err(Into::into) +} + /// Write an agent-context operation to the oplog for audit/compensation. fn log_op( conn: &rusqlite::Connection, @@ -431,8 +596,8 @@ mod tests { let mut conn = WorkspaceRegistry::init_in_memory().unwrap(); upsert_context(&mut conn, "ctx-mem", "Test", None).unwrap(); - let id1 = insert_memory(&mut conn, "ctx-mem", "decision", "Use SQLite").unwrap(); - let id2 = insert_memory(&mut conn, "ctx-mem", "constraint", "Must be <50ms").unwrap(); + let id1 = insert_memory(&mut conn, "ctx-mem", "decision", "Use SQLite", None, None).unwrap(); + let id2 = insert_memory(&mut conn, "ctx-mem", "constraint", "Must be <50ms", None, None).unwrap(); assert!(id1 > 0); assert!(id2 > 0); @@ -447,7 +612,7 @@ mod tests { fn test_get_context_with_memories() { let mut conn = WorkspaceRegistry::init_in_memory().unwrap(); upsert_context(&mut conn, "ctx-full", "Full", Some("intent")).unwrap(); - insert_memory(&mut conn, "ctx-full", "note", "content").unwrap(); + insert_memory(&mut conn, "ctx-full", "note", "content", None, None).unwrap(); let result = get_context_with_memories(&conn, "ctx-full").unwrap(); assert!(result.is_some()); @@ -468,7 +633,7 @@ mod tests { fn test_cascade_delete() { let mut conn = WorkspaceRegistry::init_in_memory().unwrap(); upsert_context(&mut conn, "ctx-cascade", "Cascade", None).unwrap(); - insert_memory(&mut conn, "ctx-cascade", "t", "data").unwrap(); + insert_memory(&mut conn, "ctx-cascade", "t", "data", None, None).unwrap(); delete_context(&mut conn, "ctx-cascade").unwrap(); let mems = list_memories(&conn, "ctx-cascade").unwrap(); diff --git a/src/registry/migrate.rs b/src/registry/migrate.rs index 8dc5b8b..362aa07 100644 --- a/src/registry/migrate.rs +++ b/src/registry/migrate.rs @@ -4,7 +4,7 @@ use super::*; use crate::storage::StorageBackend; use std::path::PathBuf; -pub const CURRENT_SCHEMA_VERSION: i32 = 33; +pub const CURRENT_SCHEMA_VERSION: i32 = 34; impl WorkspaceRegistry { pub fn db_path() -> anyhow::Result { @@ -475,7 +475,9 @@ fn run_migrations(conn: &mut rusqlite::Connection, path: &std::path::Path) -> an { tracing::warn!("Failed to auto-backup registry before migration: {}", e); } - crate::registry::migrations::run_all(conn) + crate::registry::migrations::run_all(conn)?; + crate::registry::agent_context::register_vector_functions(conn)?; + Ok(()) } #[cfg(test)] diff --git a/src/registry/migrations/mod.rs b/src/registry/migrations/mod.rs index 903d985..01de61d 100644 --- a/src/registry/migrations/mod.rs +++ b/src/registry/migrations/mod.rs @@ -35,6 +35,7 @@ pub mod v30_code_symbol_attributes; pub mod v31_agent_contexts; pub mod v32_context_links; pub mod v33_workflow_context; +pub mod v34_memory_vectors; pub fn run_all(conn: &mut Connection) -> anyhow::Result<()> { let user_version: i32 = conn.query_row("PRAGMA user_version", [], |row| row.get(0))?; @@ -138,6 +139,9 @@ pub fn run_all(conn: &mut Connection) -> anyhow::Result<()> { if user_version < 33 { v33_workflow_context::run(conn)?; } + if user_version < 34 { + v34_memory_vectors::run(conn)?; + } Ok(()) } diff --git a/src/registry/migrations/v34_memory_vectors.rs b/src/registry/migrations/v34_memory_vectors.rs new file mode 100644 index 0000000..0780094 --- /dev/null +++ b/src/registry/migrations/v34_memory_vectors.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 juice094 +use rusqlite::Connection; + +pub fn run(conn: &Connection) -> anyhow::Result<()> { + // Schema v34: Agent Memory vector storage + // Design principle: devbase does not generate embeddings; it only stores + // and retrieves vectors produced by external providers (Ollama, OpenAI, etc.). + // This keeps devbase as a "Local Context Compiler" / database layer, + // not an LLM runtime. + + conn.execute( + "ALTER TABLE agent_memories ADD COLUMN embedding BLOB", + [], + )?; + conn.execute( + "ALTER TABLE agent_memories ADD COLUMN embedding_model TEXT", + [], + )?; + conn.execute( + "ALTER TABLE agent_memories ADD COLUMN indexed_at DATETIME", + [], + )?; + + // Partial index: only index rows that actually have embeddings. + // Keeps query plans efficient when most memories are text-only. + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_agent_memories_embedding ON agent_memories(context_id, indexed_at) WHERE embedding IS NOT NULL", + [], + )?; + + conn.execute("PRAGMA user_version = 34", [])?; + Ok(()) +} diff --git a/src/registry/test_helpers.rs b/src/registry/test_helpers.rs index 9a68d4f..13385d7 100644 --- a/src/registry/test_helpers.rs +++ b/src/registry/test_helpers.rs @@ -379,9 +379,13 @@ CREATE TABLE IF NOT EXISTS agent_memories ( memory_type TEXT NOT NULL, content TEXT NOT NULL, created_at DATETIME DEFAULT current_timestamp, + embedding BLOB, + embedding_model TEXT, + indexed_at DATETIME, FOREIGN KEY (context_id) REFERENCES agent_contexts(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_agent_memories_context ON agent_memories(context_id); +CREATE INDEX IF NOT EXISTS idx_agent_memories_embedding ON agent_memories(context_id, indexed_at) WHERE embedding IS NOT NULL; CREATE TABLE IF NOT EXISTS context_entity_links ( context_id TEXT NOT NULL, From ba52fadf3c54e9fb98040eef2127efe025a99fbd Mon Sep 17 00:00:00 2001 From: juice094 <160722440+juice094@users.noreply.github.com> Date: Wed, 13 May 2026 11:17:56 +0800 Subject: [PATCH 2/6] docs(AGENTS): sync to v0.17.0-dev baseline --- AGENTS.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2c29705..98a6ec9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,14 +4,15 @@ > 它将本地数字资产的原始数据（代码库、笔记、Skill、工作流）编译为 AI 可决策的结构化情境，不负责思考，不负责执行，只负责感知、编码、持久化、检索。 -- **当前阶段**：阶段六 → v0.15.0 已发布（CHANGELOG 已记录）；v0.16.0 进行中（Workspace 扩展 Phase 2） -- **当前版本**：v0.15.0（CHANGELOG 已记录，`fix/project-health-cleanup@4e8d882`，**待合入 `main` 并打 tag**） -- **已完成里程碑**：Registry God Object 完全拆解（10 子模块提取）+ 18 workspace crates 提取 + MCP Python SDK 1.16.0 兼容修复 + repo.rs trait 化 + flaky 测试根治（RF-2.1/2.2/2.3）+ 许可证迁移 + health 性能优化（-44%）+ index skip-embeddings + batch encoding 实验 + RF-6 清零 + 架构治理文档（ADR/不变量清单）+ Tantivy BM25 代码符号搜索（P1）+ AppContext 职责拆分 Phase 1/2（storage.rs 860→430 行）+ 架构不变量 CI（G5/T11/T12）+ Embedding 多后端（Candle/Ollama 配置切换, P3）+ EnvVersionCache 扩展（9 工具链检测, P4） +- **当前阶段**：阶段八 → v0.17.0-dev 进行中（Agent Memory 向量存储 / Embedding 职责外迁） +- **当前版本**：v0.17.0-dev（Schema 34，60 MCP tools，442 tests） +- **已完成里程碑**：Registry God Object 完全拆解（10 子模块提取）+ 18 workspace crates 提取 + MCP Python SDK 1.16.0 兼容修复 + repo.rs trait 化 + flaky 测试根治（RF-2.1/2.2/2.3）+ 许可证迁移 + health 性能优化（-44%）+ index skip-embeddings + batch encoding 实验 + RF-6 清零 + 架构治理文档（ADR/不变量清单）+ Tantivy BM25 代码符号搜索（P1）+ AppContext 职责拆分 Phase 1/2（storage.rs 860→430 行）+ 架构不变量 CI（G5/T11/T12）+ Embedding 多后端（Candle/Ollama 配置切换, P3）+ EnvVersionCache 扩展（9 工具链检测, P4）+ **v0.16.0 Agent Contexts（P1/P2/P3）**：`agent_contexts`/`agent_memories`/`context_entity_links` Schema + 9 个 Session MCP tools + Context-aware Skill Runtime（`DEVBASE_ACTIVE_CONTEXT` 注入）+ **v0.16.1 Workflow-Session Binding**：`workflow_executions.context_id` + 执行自动绑定 Active Context + **v0.17.0-dev Embedding Externalization**：`embedding` 从 default features 移除（Candle/Ollama 降级为 opt-in `llm-backend`）+ Schema 34 向量存储 + `cosine_similarity` SQLite UDF + `devkit_session_recall` / `devkit_session_index`（60 tools） - **核心方向**：让 Kimi CLI 在调用文件工具之前，先通过 devbase 获得"该读哪些文件、为什么读、它们之间的关系" - **本质分析**：见 `vault/99-Meta/devbase-essence-analysis-20260430.md` 与 `docs/architecture/redefinition.md` - **设计文档**： - [`docs/architecture/workflow-dsl.md`](docs/architecture/workflow-dsl.md) — Workflow DSL 规范 - [`docs/architecture/workspace-as-schema.md`](docs/architecture/workspace-as-schema.md) — 统一实体模型设计 + - [`docs/RFC/agent-memory-vector-storage.md`](docs/RFC/agent-memory-vector-storage.md) — v0.17.0 Agent Memory 向量存储 RFC（Embedding 职责外迁设计） - [`docs/guides/mcp-integration-guide.md`](docs/guides/mcp-integration-guide.md) — MCP 集成指南 - [`docs/README.md`](docs/README.md) — 完整文档导航 @@ -22,10 +23,10 @@ Skill Runtime 全生命周期已落地（含依赖管理 Schema v15），Schema - **Workspace**：`%LOCALAPPDATA%\devbase\workspace/` —— 文件系统 = source of truth - `vault/` —— PARA 结构：00-Inbox, 01-Projects, 02-Areas, 03-Resources, 04-Archives, 99-Meta - `assets/` —— 二进制资源 -- **MCP Server**：stdio only，**58 个 tools**（含 5 个 vault tools + 8 个代码分析工具 + 4 个 embedding/搜索工具 + 4 个 Skill Runtime tools + 3 个 Workflow/评分 tools + 1 个报告工具 + 1 个 arXiv 工具 + 2 个 KnownLimit tools + 3 个 Relation tools + 2 个 Agent 状态工具 + 9 个 Agent Context tools + 1 个 streaming index 工具 + 1 个 oplog 工具）；配置见 `mcp.json` +- **MCP Server**：stdio only，**60 个 tools**（含 5 个 vault tools + 8 个代码分析工具 + 4 个 embedding/搜索工具 + 4 个 Skill Runtime tools + 3 个 Workflow/评分 tools + 1 个报告工具 + 1 个 arXiv 工具 + 2 个 KnownLimit tools + 3 个 Relation tools + 9 个 Agent Context tools + 2 个 Agent Memory 向量工具 + 1 个 streaming index 工具 + 1 个 oplog 工具）；配置见 `mcp.json` - **Kimi CLI 集成**：MCP server 已通过 `kimi mcp add` 注册，端到端验证通过（`kimi --print` 成功调用 `devkit_health`）；项目级 skill 位于 `.kimi/skills/devbase-project/SKILL.md` - **统一节点模型**：`core::node::{Node, NodeType, Edge}` —— GitRepo / VaultNote / Asset / ExternalLink -- **当前测试**：490+ workspace passed / 0 failed / 4 ignored（主 crate 390 + symbol-links 4 + sync-protocol 12 + core-types 3 + syncthing-client 2 + vault-frontmatter 5 + vault-wikilink 5 + workflow-interpolate 9 + workflow-model 2 + registry-health 3 + registry-metrics 4 + registry-workspace 5 + embedding 5 + skill-runtime-types 7 + skill-runtime-parser 3 + 其他 crates ~30）；11/11 passed（integration `tests/cli.rs`） +- **当前测试**：442+ lib passed / 0 failed / 3 ignored + 11/11 integration passed（`tests/cli.rs`） - **编译状态**：0 warning / 0 vulnerabilities（`cargo audit` 干净，除上游 `tokei` 的 `RUSTSEC-2020-0163`） - **Workspace 结构**：`crates/` 目录已启用，18 个零耦合模块已提取为独立 crate（`devbase-symbol-links`, `devbase-sync-protocol`, `devbase-core-types`, `devbase-syncthing-client`, `devbase-vault-frontmatter`, `devbase-vault-wikilink`, `devbase-workflow-interpolate`, `devbase-workflow-model`, `devbase-registry-health`, `devbase-registry-metrics`, `devbase-registry-workspace`, `devbase-embedding`, `devbase-skill-runtime-types`, `devbase-skill-runtime-parser`, `devbase-registry-entity`, `devbase-registry-relation`, `devbase-registry-call-graph`, `devbase-registry-dead-code`, `devbase-registry-code-symbols`） - **Workflow Engine**：YAML 解析 + 拓扑调度 + batch 并行执行 + 5 种 step 类型（skill/subworkflow/parallel/condition/loop） From b1fff28898c814f10e4ebab2e232fd8ef12403ff Mon Sep 17 00:00:00 2001 From: juice094 <160722440+juice094@users.noreply.github.com> Date: Wed, 13 May 2026 11:25:15 +0800 Subject: [PATCH 3/6] feat(skill-runtime): v0.17.0 context memory auto-recall with semantic + keyword fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace naive 'list all memories' injection with tiered recall strategy - Tier 1: semantic recall via cosine_similarity UDF when embedding available - embedding feature enabled: use local Candle/Ollama - embedding feature disabled: call external endpoint (Ollama API format) - Tier 2: keyword fallback (LIKE search) when semantic unavailable - New env vars injected into skill runtime: - DEVBASE_CONTEXT_MEMORIES (now top-k relevant only, with score + model) - DEVBASE_CONTEXT_MEMORY_COUNT - DEVBASE_CONTEXT_RECALL_METHOD (semantic | keyword) - build_recall_query: skill_id + args concatenation as query text - call_external_embedding_endpoint: reqwest::blocking POST to /api/embeddings Refs: RFC docs/RFC/agent-memory-vector-storage.md §5.3 --- src/skill_runtime/executor.rs | 140 ++++++++++++++++++++++++++++++++-- 1 file changed, 134 insertions(+), 6 deletions(-) diff --git a/src/skill_runtime/executor.rs b/src/skill_runtime/executor.rs index 47af436..03a8e0a 100644 --- a/src/skill_runtime/executor.rs +++ b/src/skill_runtime/executor.rs @@ -65,26 +65,35 @@ pub fn run_skill( cmd.env("DEVBASE_SKILL_ID", &skill.id); cmd.env("DEVBASE_HOME", devbase_home()?); - // P2-B: Inject active session context memories if available + // P2-B: Inject active session context memories via semantic recall. + // v0.17.0: devbase no longer generates embeddings. Semantic recall requires + // either the `llm-backend` feature or an external endpoint configured in + // config.toml. Falls back to keyword search when embedding unavailable. if let Some(ctx_id) = crate::registry::agent_context::resolve_active_context() { - if let Ok(memories) = crate::registry::agent_context::list_memories(conn, &ctx_id) - && !memories.is_empty() - { + cmd.env("DEVBASE_ACTIVE_CONTEXT", &ctx_id); + + let recalled = recall_context_memories(conn, &ctx_id, &skill.id, args); + if let Ok((memories, recall_method)) = recalled { let mem_json: Vec = memories .iter() - .map(|m| { + .map(|(m, score)| { serde_json::json!({ + "id": m.id, "type": m.memory_type, "content": m.content, + "score": score, + "model": m.embedding_model, }) }) .collect(); - cmd.env("DEVBASE_ACTIVE_CONTEXT", &ctx_id); cmd.env( "DEVBASE_CONTEXT_MEMORIES", serde_json::to_string(&mem_json).unwrap_or_default(), ); + cmd.env("DEVBASE_CONTEXT_MEMORY_COUNT", memories.len().to_string()); + cmd.env("DEVBASE_CONTEXT_RECALL_METHOD", recall_method); } + if let Ok(linked) = crate::registry::agent_context::list_linked_entities(conn, &ctx_id) && !linked.is_empty() { @@ -204,6 +213,125 @@ pub fn run_skill( }) } +// --------------------------------------------------------------------------- +// Context Memory Auto-Recall (v0.17.0) +// --------------------------------------------------------------------------- + +/// Recall relevant memories for the active context using a tiered strategy: +/// 1. Semantic recall (cosine similarity) if embedding is available +/// 2. Keyword fallback (LIKE search) otherwise +/// +/// Returns (memories with scores, recall_method label). +fn recall_context_memories( + conn: &rusqlite::Connection, + context_id: &str, + skill_id: &str, + args: &[String], +) -> anyhow::Result<(Vec<(crate::registry::agent_context::AgentMemory, f64)>, String)> { + let query_text = build_recall_query(skill_id, args); + + // Tier 1: semantic recall + if let Ok(embedding) = generate_query_embedding_external(&query_text) { + if let Ok(results) = try_semantic_recall(conn, context_id, &embedding) { + if !results.is_empty() { + return Ok((results, "semantic".to_string())); + } + } + } + + // Tier 2: keyword fallback + let keywords = crate::registry::agent_context::search_memories( + conn, + Some(context_id), + &query_text, + 5, + )?; + let scored = keywords.into_iter().map(|m| (m, 0.0)).collect(); + Ok((scored, "keyword".to_string())) +} + +fn build_recall_query(skill_id: &str, args: &[String]) -> String { + let mut parts = vec![skill_id.to_string()]; + for arg in args { + if let Some((k, v)) = arg.split_once('=') { + parts.push(format!("{}:{}", k, v)); + } else { + parts.push(arg.to_string()); + } + } + parts.join(" ") +} + +fn try_semantic_recall( + conn: &rusqlite::Connection, + context_id: &str, + embedding: &[f32], +) -> anyhow::Result> { + crate::registry::agent_context::register_vector_functions(conn)?; + crate::registry::agent_context::search_memories_semantic( + conn, + context_id, + embedding, + 5, + ) +} + +/// Generate a query embedding using the best available provider. +#[cfg(feature = "embedding")] +fn generate_query_embedding_external(text: &str) -> anyhow::Result> { + crate::embedding::generate_query_embedding(text) +} + +#[cfg(not(feature = "embedding"))] +fn generate_query_embedding_external(text: &str) -> anyhow::Result> { + let cfg = crate::config::Config::load()?; + if !cfg.embedding.enabled { + anyhow::bail!("embedding provider not enabled in config.toml"); + } + call_external_embedding_endpoint(text, &cfg.embedding) +} + +#[cfg(not(feature = "embedding"))] +fn call_external_embedding_endpoint( + text: &str, + cfg: &crate::config::EmbeddingConfig, +) -> anyhow::Result> { + let client = reqwest::blocking::Client::builder() + .timeout(std::time::Duration::from_secs(cfg.timeout_seconds)) + .build()?; + + let url = format!("{}/api/embeddings", cfg.base_url.trim_end_matches('/')); + let body = serde_json::json!({ + "model": cfg.model, + "prompt": text, + }); + + let resp = client.post(&url).json(&body).send()?; + let status = resp.status(); + let resp_json: serde_json::Value = resp.json()?; + + if !status.is_success() { + anyhow::bail!( + "Embedding endpoint returned {}: {}", + status, + resp_json.get("error").and_then(|v| v.as_str()).unwrap_or("unknown error") + ); + } + + let embedding = resp_json + .get("embedding") + .and_then(|v| v.as_array()) + .ok_or_else(|| anyhow::anyhow!("embedding array missing in response"))? + .iter() + .filter_map(|v| v.as_f64().map(|f| f as f32)) + .collect::>(); + + if embedding.is_empty() { + anyhow::bail!("embedding array empty"); + } + Ok(embedding) +} + /// Check for unresolved hard vetoes before skill execution. /// Returns an optional warning string if unresolved hard vetoes exist. /// Logs to oplog and gracefully handles registry unavailability. From d13b55428cd282ae2b7d9b5cb80c4be71b98d8a7 Mon Sep 17 00:00:00 2001 From: juice094 <160722440+juice094@users.noreply.github.com> Date: Wed, 13 May 2026 12:12:48 +0800 Subject: [PATCH 4/6] chore(release): bump to v0.17.0-dev, CHANGELOG, workspace version inheritance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Root Cargo.toml: 0.15.0 → 0.17.0-dev - 13 workspace crates: hardcoded version → version.workspace = true - CHANGELOG: add v0.17.0-dev, v0.16.1, v0.16.0 sections - End-to-end tests: mock Ollama /api/embeddings endpoint (2 cases) --- CHANGELOG.md | 68 +++++++++++ Cargo.lock | 28 ++--- Cargo.toml | 4 +- crates/devbase-embedding/Cargo.toml | 2 +- crates/devbase-registry-call-graph/Cargo.toml | 2 +- .../devbase-registry-code-symbols/Cargo.toml | 2 +- crates/devbase-registry-dead-code/Cargo.toml | 2 +- crates/devbase-registry-entity/Cargo.toml | 2 +- crates/devbase-registry-health/Cargo.toml | 2 +- crates/devbase-registry-metrics/Cargo.toml | 2 +- crates/devbase-registry-relation/Cargo.toml | 2 +- crates/devbase-registry-workspace/Cargo.toml | 2 +- .../devbase-skill-runtime-parser/Cargo.toml | 2 +- crates/devbase-skill-runtime-types/Cargo.toml | 2 +- .../devbase-workflow-interpolate/Cargo.toml | 2 +- crates/devbase-workflow-model/Cargo.toml | 2 +- src/skill_runtime/executor.rs | 108 +++++++++++++++++- 17 files changed, 204 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3022bd9..2802c4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,74 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.17.0-dev] - 2026-05-13 + +### Added + +- **Agent Memory 向量存储** — Schema v34 + - `agent_memories` 新增 `embedding BLOB`, `embedding_model TEXT`, `indexed_at DATETIME` + - Partial index `idx_agent_memories_embedding` 仅索引含向量的行 + - `AgentMemory` 结构体扩展向量元数据字段 +- **SQLite UDF: `cosine_similarity`** — `src/registry/agent_context.rs` + - 输入: 两个 little-endian f32 BLOB + - 输出: REAL ∈ [-1.0, 1.0] + - 注册时机: `WorkspaceRegistry::init_db_at` 迁移完成后自动注册 +- **语义记忆搜索** — `search_memories_semantic(context_id, query_embedding, limit)` + - 纯 SQL `ORDER BY cosine_similarity(embedding, ?) DESC` + - 零 LLM 运行时依赖；仅执行向量比对 +- **MCP Tools +2** (60 total) + - `devkit_session_recall` — 外部向量查询 + 语义召回 top-k memories + - `devkit_session_index` — 为已有 memory 注入外部生成 embedding +- **Skill Runtime Auto-Recall** — `src/skill_runtime/executor.rs` + - Tier 1: Semantic recall (本地 Candle/Ollama 或外部 HTTP endpoint) + - Tier 2: Keyword fallback (`LIKE` search on `content`) + - 新环境变量: `DEVBASE_CONTEXT_MEMORY_COUNT`, `DEVBASE_CONTEXT_RECALL_METHOD` + - `DEVBASE_CONTEXT_MEMORIES` 升级为 top-k 相关 memories（含 `score` + `model`） +- **外部 Embedding Provider 集成** + - `call_external_embedding_endpoint` — `reqwest::blocking` POST `/api/embeddings` + - 配置驱动: `config.toml [embedding]` (enabled/provider/model/base_url/timeout) + - 端到端测试: mock TCP server 验证 Ollama 格式解析 + 错误码处理 +- **RFC 文档** — `docs/RFC/agent-memory-vector-storage.md` + - 架构决策: devbase = 向量数据库层，不做 embedding 生成 + - 参照 pgvector 边界设计 + +### Changed + +- **Feature Flags**: `embedding` 从 `default` 移除 + - Candle/Ollama 依赖变为 opt-in: `--features embedding` + - 默认构建零 ML 依赖，编译时间减少 30~50% +- `insert_memory` 签名扩展: 新增可选 `embedding: Option<&[f32]>` 和 `embedding_model: Option<&str>` +- `list_memories` / `search_memories` SELECT 语句扩展为 8 列（兼容新增字段） +- AGENTS.md 同步至 v0.17.0-dev 基线 + +### Breaking Changes + +- 默认构建不再包含 `devbase-embedding` crate；需要语义生成能力的用户须显式启用 `--features embedding` +- `generate_query_embedding` 在默认构建下返回错误（提示启用 feature 或配置外部 endpoint） + +## [0.16.1] - 2026-05-13 + +### Added + +- **Workflow-Session Binding** — Schema v33 + - `workflow_executions` 新增 `context_id` 列 + 索引 + - `create_execution` 自动绑定 `resolve_active_context()` + - MCP `devkit_workflow_run` 与 CLI `workflow run` 均支持自动绑定 + - `devkit_session_workflows` tool: 列出指定 context 的 workflow 执行历史 +- `context_entity_links` 表 (Schema v32): context 与任意 entity 的多对多关联 + +## [0.16.0] - 2026-05-13 + +### Added + +- **Agent Contexts (AI Agent OS)** — Schema v31 + - `agent_contexts` 表: 持久化 AI session / project scope + - `agent_memories` 表: 结构化记忆（decision/constraint/note/discovery/error） + - 9 个 Session MCP tools: save/list/resume/attach/detach/activate/search/capture/workflows + - `resolve_active_context()`: 环境变量 `DEVBASE_ACTIVE_CONTEXT` → 文件 `.active_context` fallback + - Context-aware Skill Runtime: 注入 `DEVBASE_ACTIVE_CONTEXT` + `DEVBASE_CONTEXT_MEMORIES` + `DEVBASE_CONTEXT_LINKS` + - 所有 agent_context 操作自动写入 OpLog (`OplogEventType::AgentContext`) + ## [0.15.0] - 2026-05-04 ### Added diff --git a/Cargo.lock b/Cargo.lock index df489c0..832a220 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1056,7 +1056,7 @@ checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" [[package]] name = "devbase" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "anyhow", "assert_cmd", @@ -1125,7 +1125,7 @@ dependencies = [ [[package]] name = "devbase-embedding" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "anyhow", "candle-core", @@ -1139,7 +1139,7 @@ dependencies = [ [[package]] name = "devbase-registry-call-graph" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "anyhow", "rusqlite", @@ -1147,7 +1147,7 @@ dependencies = [ [[package]] name = "devbase-registry-code-symbols" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "anyhow", "rusqlite", @@ -1155,7 +1155,7 @@ dependencies = [ [[package]] name = "devbase-registry-dead-code" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "anyhow", "rusqlite", @@ -1163,7 +1163,7 @@ dependencies = [ [[package]] name = "devbase-registry-entity" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "anyhow", "chrono", @@ -1173,7 +1173,7 @@ dependencies = [ [[package]] name = "devbase-registry-health" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "anyhow", "chrono", @@ -1183,7 +1183,7 @@ dependencies = [ [[package]] name = "devbase-registry-metrics" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "anyhow", "chrono", @@ -1194,7 +1194,7 @@ dependencies = [ [[package]] name = "devbase-registry-relation" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "anyhow", "chrono", @@ -1203,7 +1203,7 @@ dependencies = [ [[package]] name = "devbase-registry-workspace" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "anyhow", "chrono", @@ -1213,7 +1213,7 @@ dependencies = [ [[package]] name = "devbase-skill-runtime-parser" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "anyhow", "chrono", @@ -1222,7 +1222,7 @@ dependencies = [ [[package]] name = "devbase-skill-runtime-types" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "anyhow", "chrono", @@ -1268,7 +1268,7 @@ version = "0.1.0-alpha.1" [[package]] name = "devbase-workflow-interpolate" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "anyhow", "regex", @@ -1278,7 +1278,7 @@ dependencies = [ [[package]] name = "devbase-workflow-model" -version = "0.15.0" +version = "0.17.0-dev" dependencies = [ "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index f2dd742..f36d8ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase" -version = "0.15.0" +version = "0.17.0-dev" edition = "2024" description = "Developer workspace database and knowledge-base manager" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] @@ -102,7 +102,7 @@ members = ["crates/*"] resolver = "2" [workspace.package] -version = "0.15.0" +version = "0.17.0-dev" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] edition = "2024" license = "AGPL-3.0-or-later" diff --git a/crates/devbase-embedding/Cargo.toml b/crates/devbase-embedding/Cargo.toml index 4473d65..379870c 100644 --- a/crates/devbase-embedding/Cargo.toml +++ b/crates/devbase-embedding/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase-embedding" -version = "0.15.0" +version.workspace = true edition = "2024" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] license = "MIT" diff --git a/crates/devbase-registry-call-graph/Cargo.toml b/crates/devbase-registry-call-graph/Cargo.toml index cd00f47..eb5e8a7 100644 --- a/crates/devbase-registry-call-graph/Cargo.toml +++ b/crates/devbase-registry-call-graph/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase-registry-call-graph" -version = "0.15.0" +version.workspace = true edition = "2024" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] license = "MIT" diff --git a/crates/devbase-registry-code-symbols/Cargo.toml b/crates/devbase-registry-code-symbols/Cargo.toml index 74af03c..691f42f 100644 --- a/crates/devbase-registry-code-symbols/Cargo.toml +++ b/crates/devbase-registry-code-symbols/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase-registry-code-symbols" -version = "0.15.0" +version.workspace = true edition = "2024" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] license = "MIT" diff --git a/crates/devbase-registry-dead-code/Cargo.toml b/crates/devbase-registry-dead-code/Cargo.toml index 1d1f12d..785c667 100644 --- a/crates/devbase-registry-dead-code/Cargo.toml +++ b/crates/devbase-registry-dead-code/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase-registry-dead-code" -version = "0.15.0" +version.workspace = true edition = "2024" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] license = "MIT" diff --git a/crates/devbase-registry-entity/Cargo.toml b/crates/devbase-registry-entity/Cargo.toml index 6203a1e..31feacf 100644 --- a/crates/devbase-registry-entity/Cargo.toml +++ b/crates/devbase-registry-entity/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase-registry-entity" -version = "0.15.0" +version.workspace = true edition = "2024" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] license = "MIT" diff --git a/crates/devbase-registry-health/Cargo.toml b/crates/devbase-registry-health/Cargo.toml index 1e29763..1ce097b 100644 --- a/crates/devbase-registry-health/Cargo.toml +++ b/crates/devbase-registry-health/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase-registry-health" -version = "0.15.0" +version.workspace = true edition = "2024" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] license = "MIT" diff --git a/crates/devbase-registry-metrics/Cargo.toml b/crates/devbase-registry-metrics/Cargo.toml index 173f6aa..4b25dcb 100644 --- a/crates/devbase-registry-metrics/Cargo.toml +++ b/crates/devbase-registry-metrics/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase-registry-metrics" -version = "0.15.0" +version.workspace = true edition = "2024" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] license = "MIT" diff --git a/crates/devbase-registry-relation/Cargo.toml b/crates/devbase-registry-relation/Cargo.toml index 753635d..b457112 100644 --- a/crates/devbase-registry-relation/Cargo.toml +++ b/crates/devbase-registry-relation/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase-registry-relation" -version = "0.15.0" +version.workspace = true edition = "2024" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] license = "MIT" diff --git a/crates/devbase-registry-workspace/Cargo.toml b/crates/devbase-registry-workspace/Cargo.toml index f938790..51d0b51 100644 --- a/crates/devbase-registry-workspace/Cargo.toml +++ b/crates/devbase-registry-workspace/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase-registry-workspace" -version = "0.15.0" +version.workspace = true edition = "2024" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] license = "MIT" diff --git a/crates/devbase-skill-runtime-parser/Cargo.toml b/crates/devbase-skill-runtime-parser/Cargo.toml index 1d65286..3060c28 100644 --- a/crates/devbase-skill-runtime-parser/Cargo.toml +++ b/crates/devbase-skill-runtime-parser/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase-skill-runtime-parser" -version = "0.15.0" +version.workspace = true edition = "2024" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] license = "MIT" diff --git a/crates/devbase-skill-runtime-types/Cargo.toml b/crates/devbase-skill-runtime-types/Cargo.toml index fa10fc3..7e1f2e1 100644 --- a/crates/devbase-skill-runtime-types/Cargo.toml +++ b/crates/devbase-skill-runtime-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase-skill-runtime-types" -version = "0.15.0" +version.workspace = true edition = "2024" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] license = "MIT" diff --git a/crates/devbase-workflow-interpolate/Cargo.toml b/crates/devbase-workflow-interpolate/Cargo.toml index 66aa272..c5770e5 100644 --- a/crates/devbase-workflow-interpolate/Cargo.toml +++ b/crates/devbase-workflow-interpolate/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase-workflow-interpolate" -version = "0.15.0" +version.workspace = true edition = "2024" description = "Workflow variable interpolation for devbase" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] diff --git a/crates/devbase-workflow-model/Cargo.toml b/crates/devbase-workflow-model/Cargo.toml index ec78e7f..5d13ac5 100644 --- a/crates/devbase-workflow-model/Cargo.toml +++ b/crates/devbase-workflow-model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "devbase-workflow-model" -version = "0.15.0" +version.workspace = true edition = "2024" authors = ["juice094 <160722440+juice094@users.noreply.github.com>"] license = "MIT" diff --git a/src/skill_runtime/executor.rs b/src/skill_runtime/executor.rs index 03a8e0a..3e508b4 100644 --- a/src/skill_runtime/executor.rs +++ b/src/skill_runtime/executor.rs @@ -292,7 +292,7 @@ fn generate_query_embedding_external(text: &str) -> anyhow::Result> { } #[cfg(not(feature = "embedding"))] -fn call_external_embedding_endpoint( +pub(crate) fn call_external_embedding_endpoint( text: &str, cfg: &crate::config::EmbeddingConfig, ) -> anyhow::Result> { @@ -625,4 +625,110 @@ sys.exit(0) let warning = check_hard_vetoes_for_skill(&skill, &conn); assert!(warning.is_none(), "should return None when no vetoes exist"); } + + /// End-to-end test: mock Ollama /api/embeddings endpoint and verify + /// call_external_embedding_endpoint parses the response correctly. + #[test] + #[cfg(not(feature = "embedding"))] + fn test_external_embedding_endpoint_ollama_parsing() { + use std::io::{BufRead, BufReader, Write}; + use std::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut reader = BufReader::new(&stream); + let mut headers = String::new(); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap() == 0 { + break; + } + if line == "\r\n" { + break; + } + headers.push_str(&line); + } + // Verify it hit the correct path + assert!( + headers.contains("POST /api/embeddings"), + "expected POST /api/embeddings, got: {}", + headers + ); + + let body = r#"{"embedding":[0.1,0.2,0.3]}"#; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: application/json\r\n\r\n{}", + body.len(), + body + ); + stream.write_all(response.as_bytes()).unwrap(); + stream.flush().unwrap(); + }); + + let cfg = crate::config::EmbeddingConfig { + enabled: true, + provider: "ollama".to_string(), + model: "test-model".to_string(), + base_url: format!("http://127.0.0.1:{}", port), + timeout_seconds: 5, + }; + + let result = call_external_embedding_endpoint("test prompt", &cfg); + assert!( + result.is_ok(), + "should parse ollama response: {:?}", + result.err() + ); + let emb = result.unwrap(); + assert_eq!(emb.len(), 3); + assert!((emb[0] - 0.1f32).abs() < 0.001); + assert!((emb[1] - 0.2f32).abs() < 0.001); + assert!((emb[2] - 0.3f32).abs() < 0.001); + } + + /// Verify error handling when external endpoint returns non-2xx. + #[test] + #[cfg(not(feature = "embedding"))] + fn test_external_embedding_endpoint_error_response() { + use std::io::{BufRead, BufReader, Write}; + use std::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut reader = BufReader::new(&stream); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap() == 0 || line == "\r\n" { + break; + } + } + let body = r#"{"error":"model not found"}"#; + let response = format!( + "HTTP/1.1 404 Not Found\r\nContent-Length: {}\r\nContent-Type: application/json\r\n\r\n{}", + body.len(), + body + ); + stream.write_all(response.as_bytes()).unwrap(); + stream.flush().unwrap(); + }); + + let cfg = crate::config::EmbeddingConfig { + enabled: true, + provider: "ollama".to_string(), + model: "missing-model".to_string(), + base_url: format!("http://127.0.0.1:{}", port), + timeout_seconds: 5, + }; + + let result = call_external_embedding_endpoint("test", &cfg); + assert!(result.is_err(), "should fail on 404"); + let msg = result.unwrap_err().to_string(); + assert!(msg.contains("404"), "error should mention status code: {}", msg); + } } From 20a4deedf5318c9f24da7dabdd264c28a39f946d Mon Sep 17 00:00:00 2001 From: juice094 <160722440+juice094@users.noreply.github.com> Date: Wed, 13 May 2026 12:24:40 +0800 Subject: [PATCH 5/6] =?UTF-8?q?feat(tui):=20v0.17.0=20Session=20panel=20?= =?UTF-8?q?=E2=80=94=20context=20list=20+=20memory=20detail=20view?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - MainView: add Session variant (3-state cycle: Repo → Vault → Session) - render/session.rs: left panel context list (status icon + id + name) - render/session.rs: right panel memory detail (type icon + model + indexed tag) - state/mod.rs: load_sessions() + load_session_memories() on navigation - state/navigation.rs: next/previous/jump_to_top/jump_to_bottom for Session - event.rs: Enter refreshes selected session memories - render/mod.rs: Session detail rendered directly in ui() dispatcher - render/list.rs: delegate Session to session::render_session_list - render/detail.rs: Session branch (no-op, detail rendered at top level) - Fix test_main_view_toggle for 3-state cycle Refs: RFC docs/RFC/agent-memory-vector-storage.md §8 task list --- src/tui/event.rs | 7 ++ src/tui/mod.rs | 11 ++- src/tui/render/detail.rs | 3 + src/tui/render/list.rs | 1 + src/tui/render/mod.rs | 11 ++- src/tui/render/session.rs | 131 ++++++++++++++++++++++++++++++++++++ src/tui/state/mod.rs | 32 +++++++++ src/tui/state/navigation.rs | 41 +++++++++++ 8 files changed, 234 insertions(+), 3 deletions(-) create mode 100644 src/tui/render/session.rs diff --git a/src/tui/event.rs b/src/tui/event.rs index 63189a3..a538220 100644 --- a/src/tui/event.rs +++ b/src/tui/event.rs @@ -335,6 +335,13 @@ pub(crate) async fn run_app( let _ = std::process::Command::new("code").arg(&path).spawn(); } } + MainView::Session => { + // Enter on a session: refresh its memories + if let Some(ctx) = app.sessions.get(app.session_selected) { + let ctx_id = ctx.id.clone(); + let _ = app.load_session_memories(&ctx_id); + } + } }, _ => {} }, diff --git a/src/tui/mod.rs b/src/tui/mod.rs index fc7c3d8..f310771 100644 --- a/src/tui/mod.rs +++ b/src/tui/mod.rs @@ -28,13 +28,15 @@ impl SortMode { pub(crate) enum MainView { RepoList, VaultList, + Session, } impl MainView { pub fn toggle(self) -> Self { match self { Self::RepoList => Self::VaultList, - Self::VaultList => Self::RepoList, + Self::VaultList => Self::Session, + Self::Session => Self::RepoList, } } } @@ -233,6 +235,10 @@ pub struct App { pub(crate) vaults: Vec, pub(crate) vault_selected: usize, pub(crate) vault_list_state: ListState, + pub(crate) sessions: Vec, + pub(crate) session_selected: usize, + pub(crate) session_list_state: ListState, + pub(crate) session_memories: Vec, pub(crate) skill_panel: SkillPanelState, pub(crate) workflow_popup_mode: WorkflowPopupMode, pub(crate) workflows: Vec, @@ -318,7 +324,8 @@ mod tests { #[test] fn test_main_view_toggle() { assert_eq!(MainView::RepoList.toggle(), MainView::VaultList); - assert_eq!(MainView::VaultList.toggle(), MainView::RepoList); + assert_eq!(MainView::VaultList.toggle(), MainView::Session); + assert_eq!(MainView::Session.toggle(), MainView::RepoList); } #[test] diff --git a/src/tui/render/detail.rs b/src/tui/render/detail.rs index 82a0c78..37565d3 100644 --- a/src/tui/render/detail.rs +++ b/src/tui/render/detail.rs @@ -15,6 +15,9 @@ pub(crate) fn render_detail(frame: &mut Frame, app: &mut App, area: Rect, styles match app.main_view { MainView::RepoList => render_repo_detail(frame, app, area, styles), MainView::VaultList => render_vault_detail(frame, app, area, styles), + MainView::Session => { + // Session detail is rendered directly in ui() to avoid nested frame issues + } } } diff --git a/src/tui/render/list.rs b/src/tui/render/list.rs index 3345e6c..3d6c3ed 100644 --- a/src/tui/render/list.rs +++ b/src/tui/render/list.rs @@ -14,6 +14,7 @@ pub(crate) fn render_list(frame: &mut Frame, app: &mut App, area: Rect, styles: match app.main_view { MainView::RepoList => render_repo_list(frame, app, area, styles), MainView::VaultList => render_vault_list(frame, app, area, styles), + MainView::Session => crate::tui::render::session::render_session_list(frame, app, area, styles), } } diff --git a/src/tui/render/mod.rs b/src/tui/render/mod.rs index ae08961..830d77e 100644 --- a/src/tui/render/mod.rs +++ b/src/tui/render/mod.rs @@ -11,6 +11,7 @@ mod help; mod list; mod logs; mod popups; +pub(crate) mod session; /// Main render dispatcher. /// @@ -37,7 +38,14 @@ pub(crate) fn ui(frame: &mut Frame, app: &mut App) { list::render_list(frame, app, layout.list, &styles); } else { list::render_list(frame, app, layout.list, &styles); - detail::render_detail(frame, app, layout.detail, &styles); + match app.main_view { + crate::tui::MainView::Session => { + session::render_session_detail(frame, app, layout.detail, &styles); + } + _ => { + detail::render_detail(frame, app, layout.detail, &styles); + } + } logs::render_logs(frame, app, layout.logs, &styles); } @@ -73,6 +81,7 @@ fn render_bottom_bar(frame: &mut Frame, app: &App, area: ratatui::layout::Rect, let view_label = match app.main_view { crate::tui::MainView::RepoList => "[Repos]", crate::tui::MainView::VaultList => "[Vault]", + crate::tui::MainView::Session => "[Session]", }; let mut spans = vec![ Span::styled( diff --git a/src/tui/render/session.rs b/src/tui/render/session.rs new file mode 100644 index 0000000..0e50873 --- /dev/null +++ b/src/tui/render/session.rs @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 juice094 +use crate::tui::theme::Styles; +use crate::tui::App; +use ratatui::{ + Frame, + layout::{Constraint, Direction, Layout, Rect}, + style::{Color, Modifier, Style}, + text::{Line, Span}, + widgets::{Block, Borders, List, ListItem, Paragraph}, +}; + +/// Render the Session list (left panel) — contexts. +pub(crate) fn render_session_list( + frame: &mut Frame, + app: &mut App, + area: Rect, + styles: &Styles, +) { + let items: Vec = app + .sessions + .iter() + .map(|ctx| { + let status_icon = match ctx.status.as_str() { + "active" => "●", + "archived" => "◌", + _ => "?", + }; + let status_color = match ctx.status.as_str() { + "active" => Color::Green, + "archived" => Color::Gray, + _ => Color::Yellow, + }; + let spans = vec![ + Span::styled(format!("{} ", status_icon), Style::default().fg(status_color)), + Span::styled(&ctx.id, + styles.value.add_modifier(Modifier::BOLD), + ), + Span::raw(" — "), + Span::styled(&ctx.name, styles.dim), + ]; + ListItem::new(Line::from(spans)) + }) + .collect(); + + let title = format!(" Sessions ({}) ", app.sessions.len()); + let block = Block::default() + .title(title) + .borders(Borders::ALL) + .border_style(styles.border); + + let list = List::new(items) + .block(block) + .highlight_style(Style::default().bg(Color::Blue).fg(Color::Black).add_modifier(Modifier::BOLD)) + .highlight_symbol("▸ "); + + frame.render_stateful_widget(list, area, &mut app.session_list_state); +} + +/// Render the Session detail (right panel) — memories of selected context. +pub(crate) fn render_session_detail( + frame: &mut Frame, + app: &App, + area: Rect, + styles: &Styles, +) { + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Length(3), Constraint::Min(0)]) + .split(area); + + // Header: context name + intent + let header_text = if let Some(ctx) = app.sessions.get(app.session_selected) { + let intent = ctx.intent.as_deref().unwrap_or("no intent"); + format!("{} — {}", ctx.name, intent) + } else { + "No active context".to_string() + }; + let header = Paragraph::new(header_text) + .block( + Block::default() + .title(" Context ") + .borders(Borders::ALL) + .border_style(styles.border), + ) + .style(styles.value); + frame.render_widget(header, chunks[0]); + + // Memory list + let mem_items: Vec = app + .session_memories + .iter() + .map(|mem| { + let type_icon = match mem.memory_type.as_str() { + "decision" => "◆", + "constraint" => "▪", + "discovery" => "★", + "error" => "✗", + _ => "•", + }; + let type_color = match mem.memory_type.as_str() { + "decision" => Color::Cyan, + "constraint" => Color::Yellow, + "discovery" => Color::Green, + "error" => Color::Red, + _ => Color::Gray, + }; + let indexed = mem.indexed_at.map(|_| " [indexed]").unwrap_or(""); + let model = mem.embedding_model.as_deref().unwrap_or(""); + let model_tag = if model.is_empty() { "".to_string() } else { format!(" ({}) ", model) }; + + let spans = vec![ + Span::styled(format!("{} ", type_icon), Style::default().fg(type_color)), + Span::styled(format!("[{}]", mem.memory_type), styles.dim), + Span::raw(model_tag), + Span::styled(indexed, Style::default().fg(Color::DarkGray)), + Span::raw(" "), + Span::styled(&mem.content, styles.value), + ]; + ListItem::new(Line::from(spans)) + }) + .collect(); + + let mem_block = Block::default() + .title(format!(" Memories ({}) ", app.session_memories.len())) + .borders(Borders::ALL) + .border_style(styles.border); + + let mem_list = List::new(mem_items).block(mem_block); + frame.render_widget(mem_list, chunks[1]); +} diff --git a/src/tui/state/mod.rs b/src/tui/state/mod.rs index 56bccbf..7354bcb 100644 --- a/src/tui/state/mod.rs +++ b/src/tui/state/mod.rs @@ -60,6 +60,10 @@ impl App { vaults: Vec::new(), vault_selected: 0, vault_list_state: ListState::default(), + sessions: Vec::new(), + session_selected: 0, + session_list_state: ListState::default(), + session_memories: Vec::new(), skill_panel: crate::tui::SkillPanelState::default(), workflow_popup_mode: WorkflowPopupMode::Hidden, workflows: Vec::new(), @@ -75,6 +79,7 @@ impl App { }; app.log_info(app.ctx.i18n.log.tui_started.to_string()); app.load_repos()?; + app.load_sessions()?; app.spawn_stars_refresh(); app.spawn_vault_watcher(); Ok(app) @@ -144,6 +149,29 @@ impl App { Ok(()) } + pub(crate) fn load_sessions(&mut self) -> anyhow::Result<()> { + let conn = self.ctx.conn()?; + self.sessions = crate::registry::agent_context::list_contexts(&conn)?; + if !self.sessions.is_empty() && self.session_selected < self.sessions.len() { + let ctx_id = self.sessions[self.session_selected].id.clone(); + drop(conn); + self.load_session_memories(&ctx_id)?; + } else { + self.session_memories.clear(); + } + Ok(()) + } + + pub(crate) fn load_session_memories( + &mut self, + context_id: &str, + ) -> anyhow::Result<()> { + let conn = self.ctx.conn()?; + self.session_memories = + crate::registry::agent_context::list_memories(&conn, context_id)?; + Ok(()) + } + pub(crate) fn log_info(&mut self, msg: String) { self.log_with_level("INFO", msg); } @@ -313,6 +341,10 @@ mod tests { vaults: vec![], vault_selected: 0, vault_list_state: ListState::default(), + sessions: vec![], + session_selected: 0, + session_list_state: ListState::default(), + session_memories: vec![], skill_panel: crate::tui::SkillPanelState::default(), workflow_popup_mode: WorkflowPopupMode::Hidden, workflows: vec![], diff --git a/src/tui/state/navigation.rs b/src/tui/state/navigation.rs index c524453..8be6d33 100644 --- a/src/tui/state/navigation.rs +++ b/src/tui/state/navigation.rs @@ -18,6 +18,16 @@ impl App { self.vault_list_state.select(Some(self.vault_selected)); } } + MainView::Session => { + if !self.sessions.is_empty() { + self.session_selected = (self.session_selected + 1) % self.sessions.len(); + self.session_list_state.select(Some(self.session_selected)); + if let Some(ctx) = self.sessions.get(self.session_selected) { + let ctx_id = ctx.id.clone(); + let _ = self.load_session_memories(&ctx_id); + } + } + } } } @@ -37,6 +47,17 @@ impl App { self.vault_list_state.select(Some(self.vault_selected)); } } + MainView::Session => { + if !self.sessions.is_empty() { + self.session_selected = + (self.session_selected + self.sessions.len() - 1) % self.sessions.len(); + self.session_list_state.select(Some(self.session_selected)); + if let Some(ctx) = self.sessions.get(self.session_selected) { + let ctx_id = ctx.id.clone(); + let _ = self.load_session_memories(&ctx_id); + } + } + } } } @@ -55,6 +76,16 @@ impl App { self.vault_list_state.select(Some(self.vault_selected)); } } + MainView::Session => { + if !self.sessions.is_empty() { + self.session_selected = 0; + self.session_list_state.select(Some(self.session_selected)); + if let Some(ctx) = self.sessions.get(self.session_selected) { + let ctx_id = ctx.id.clone(); + let _ = self.load_session_memories(&ctx_id); + } + } + } } } @@ -73,6 +104,16 @@ impl App { self.vault_list_state.select(Some(self.vault_selected)); } } + MainView::Session => { + if !self.sessions.is_empty() { + self.session_selected = self.sessions.len() - 1; + self.session_list_state.select(Some(self.session_selected)); + if let Some(ctx) = self.sessions.get(self.session_selected) { + let ctx_id = ctx.id.clone(); + let _ = self.load_session_memories(&ctx_id); + } + } + } } } } From 91aff9eebad340b3fa7f34b839971b9f5af86c0f Mon Sep 17 00:00:00 2001 From: juice094 <160722440+juice094@users.noreply.github.com> Date: Wed, 13 May 2026 12:32:09 +0800 Subject: [PATCH 6/6] docs(guides): add embedding-provider-setup.md for external Ollama integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Ollama installation (Windows/macOS/Linux) - Model pull instructions (nomic-embed-text / all-minilm) - config.toml [embedding] configuration examples - Verification steps: curl test + devbase session workflow - OpenAI API alternative (with compatibility notes) - Backward compatibility: --features embedding - Troubleshooting section for common errors Refs: RFC §8 external provider configuration documentation --- docs/guides/embedding-provider-setup.md | 163 ++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 docs/guides/embedding-provider-setup.md diff --git a/docs/guides/embedding-provider-setup.md b/docs/guides/embedding-provider-setup.md new file mode 100644 index 0000000..b814262 --- /dev/null +++ b/docs/guides/embedding-provider-setup.md @@ -0,0 +1,163 @@ +# Embedding Provider 配置指南 + +> devbase v0.17.0+ 不再内置 LLM 推理能力。Embedding 生成由外部 Provider 负责，devbase 仅存储和检索向量。 + +## 背景 + +从 v0.17.0 开始，`embedding` feature 从默认构建中移除。这意味着： + +- **默认构建**（`cargo build --release`）不包含 Candle/Ollama 依赖，体积更小、编译更快 +- **Embedding 生成**必须由外部服务完成，devbase 通过 HTTP API 与之交互 +- **向量存储和搜索**仍由 devbase 原生支持（SQLite + `cosine_similarity` UDF） + +## 方案一：Ollama（推荐，本地免费） + +### 1. 安装 Ollama + +Windows (PowerShell): +```powershell +winget install Ollama.Ollama +``` + +macOS: +```bash +brew install ollama +``` + +Linux: +```bash +curl -fsSL https://ollama.com/install.sh | sh +``` + +### 2. 拉取 Embedding 模型 + +```bash +ollama pull nomic-embed-text +``` + +> `nomic-embed-text` 生成 768-dim 向量，质量高且速度快。你也可以使用 `all-minilm`（384-dim，更快）。 + +### 3. 配置 devbase + +编辑 `%APPDATA%\devbase\config.toml`（Windows）或 `~/.config/devbase/config.toml`（Linux/macOS）： + +```toml +[embedding] +enabled = true +provider = "ollama" +model = "nomic-embed-text" +base_url = "http://localhost:11434" +timeout_seconds = 30 +``` + +### 4. 验证服务可用 + +```bash +ollama list +# 应看到 nomic-embed-text + +curl http://localhost:11434/api/embeddings -d '{ + "model": "nomic-embed-text", + "prompt": "test" +}' +# 应返回 {"embedding": [0.1, 0.2, ...]} +``` + +### 5. 在 devbase 中测试语义召回 + +```bash +# 1. 启动一个 context 并添加记忆 +devbase session save test-ctx "Test Project" "验证 embedding" +devbase session capture test-ctx "note" "这是关于 Rust 编译器的笔记" + +# 2. 通过 MCP 或 CLI 为记忆注入向量（外部生成后存储） +# 对于 Skill Runtime 自动召回，devbase 会在执行 skill 时自动调用 Ollama + +# 3. 运行一个 skill 并观察环境变量 +$env:DEVBASE_ACTIVE_CONTEXT = "test-ctx" +devbase skill run some-skill +# 在 skill 进程中，DEVBASE_CONTEXT_MEMORIES 应包含相关记忆 +``` + +## 方案二：OpenAI API（云端） + +如果你已有 OpenAI API Key，可直接使用： + +```toml +[embedding] +enabled = true +provider = "openai" +model = "text-embedding-3-small" +base_url = "https://api.openai.com/v1" +timeout_seconds = 30 +``` + +**注意**：当前 Skill Runtime 的 `call_external_embedding_endpoint` 默认使用 Ollama `/api/embeddings` 请求格式。OpenAI 的 `/v1/embeddings` 格式不同（需要 `input` 字段而非 `prompt`，响应结构也不同）。 + +如需支持 OpenAI 格式，请： +1. 提交 Feature Request，或 +2. 使用本地代理将 OpenAI 格式桥接为 Ollama 格式 + +## 方案三：重新启用内置 Embedding（向后兼容） + +如果你希望恢复 v0.16.x 的行为（Candle 本地推理，零外部依赖）： + +```bash +cargo build --release --features embedding +``` + +此时 `generate_query_embedding` 将使用 Candle + all-MiniLM 模型本地生成，无需 Ollama 守护进程。 + +## Troubleshooting + +### `embedding provider not enabled in config.toml` + +- 检查 `config.toml` 中 `[embedding]` 段的 `enabled = true` +- 确认 config 文件路径正确：`devbase config path` + +### `Connection refused` / Ollama 无响应 + +- 确认 Ollama 服务正在运行：`ollama serve` 或系统托盘图标 +- 检查防火墙是否拦截 11434 端口 +- 尝试手动 curl 测试连通性 + +### 召回结果为空或 score 全为 0 + +- 确认 memories 已创建：`devbase session list` +- 确认 memories 已索引（有 `indexed_at` 字段）： + ```sql + SELECT id, content, embedding_model, indexed_at FROM agent_memories; + ``` +- 如果 `embedding` 为 NULL，说明尚未注入向量。Skill Runtime 会在执行时自动尝试生成并注入。 + +### 模型下载慢 + +```bash +ollama pull nomic-embed-text +# 或使用镜像加速（中国大陆用户） +# 设置环境变量 OLLAMA_HOST 和代理 +``` + +## 架构图示 + +``` +[User Skill Runtime] + │ + ▼ +build_recall_query(skill_id + args) + │ + ▼ +[Ollama localhost:11434] ──► generate embedding + │ + ▼ +[devbase SQLite] cosine_similarity(embedding, query_embedding) + │ + ▼ +Top-k memories ──► DEVBASE_CONTEXT_MEMORIES env +``` + +## 参考 + +- [Ollama 官方文档](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings) +- [nomic-embed-text 模型](https://ollama.com/library/nomic-embed-text) +- RFC: `docs/RFC/agent-memory-vector-storage.md`