From 674283f8439301edd4709bf4a0f20af6535bf079 Mon Sep 17 00:00:00 2001 From: hungtranphamminh Date: Wed, 17 Jun 2026 23:52:31 +0700 Subject: [PATCH 1/6] feat(server): soft-delete memories + clearNamespace/list/forget [WALM-115] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a soft-delete tombstone (deleted_at, migration 010) and three owner-scoped endpoints on the relayer: - POST /api/clear-namespace — soft-delete every memory in a namespace - POST /api/list — enumerate a namespace (metadata only, no decrypt), keyset-paginated via (created_at, id) cursor + has_more - POST /api/memories/forget — soft-delete one memory by its per-row id Soft-delete stops a memory surfacing in recall (and the analyze pre-extraction context) while retaining the row; the underlying Walrus blob persists. The hard DELETE path (/api/forget) is left untouched for benchmark-harness cleanup. stats() now excludes tombstoned rows. Includes Rust unit tests (SQL invariants, cursor keyset/tie-break, stats live-only) and e2e coverage (clear, list+forget, pagination, per-row, cross-owner). --- .../server/migrations/010_soft_delete.sql | 29 ++ services/server/src/main.rs | 7 + services/server/src/routes/admin.rs | 217 ++++++++++- services/server/src/routes/analyze.rs | 40 +- services/server/src/routes/mod.rs | 4 +- services/server/src/storage/db.rs | 317 ++++++++++++++- services/server/src/types.rs | 86 +++++ services/server/tests/e2e_test.py | 360 ++++++++++++++++++ 8 files changed, 1041 insertions(+), 19 deletions(-) create mode 100644 services/server/migrations/010_soft_delete.sql diff --git a/services/server/migrations/010_soft_delete.sql b/services/server/migrations/010_soft_delete.sql new file mode 100644 index 00000000..e19a1a15 --- /dev/null +++ b/services/server/migrations/010_soft_delete.sql @@ -0,0 +1,29 @@ +-- Soft-delete tombstone for memory deletion / namespace clearing. +-- +-- A deleted memory is marked here (deleted_at = NOW()) rather than removed, +-- so it stops surfacing in recall and the pre-extraction dedup context while +-- the row is RETAINED. Retention is load-bearing: the restore flow's +-- presence-check (get_blobs_by_namespace) reads vector_entries unfiltered, so +-- a retained tombstoned row still counts as "present" and restore won't +-- re-index its on-chain blob. (If a future task hard-purges old tombstoned +-- rows, that purge MUST add `AND deleted_at IS NULL` to get_blobs_by_namespace, +-- or restore will resurrect the purged memory.) +-- +-- NULL default = not deleted. Additive + nullable, so no table rewrite and no +-- backfill (every existing row is live). Reads that must hide deleted rows add +-- `AND deleted_at IS NULL`: recall's search_similar (which also feeds the +-- pre-extraction context) and the analyze namespace-existence check. +-- +-- The hard DELETE path (delete_by_namespace, via /api/forget) is intentionally +-- left as-is for the benchmark harness's inter-run cleanup — soft-delete is a +-- separate, user-facing path. + +ALTER TABLE vector_entries + ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ NULL DEFAULT NULL; + +-- Partial index: the recall hot path filters `WHERE owner = ? AND namespace = ? +-- AND deleted_at IS NULL`. A partial index over only live rows keeps it small +-- (tombstones excluded) and matches the predicate. +CREATE INDEX IF NOT EXISTS idx_vector_entries_live + ON vector_entries (owner, namespace) + WHERE deleted_at IS NULL; diff --git a/services/server/src/main.rs b/services/server/src/main.rs index be16ba41..580cdc54 100644 --- a/services/server/src/main.rs +++ b/services/server/src/main.rs @@ -763,6 +763,13 @@ async fn main() { // admin/harness endpoints — namespace delete + stats. // Mode-blind; owner-scoped via AuthInfo. .route("/api/forget", post(routes::forget)) + // user-facing soft-delete (clearNamespace) — tombstones rows so they + // stop surfacing in recall; distinct from /api/forget (hard delete). + .route("/api/clear-namespace", post(routes::clear_namespace)) + // user-facing memory management: list a namespace (metadata + per-row + // id) and soft-delete a single memory by that id. + .route("/api/list", post(routes::list)) + .route("/api/memories/forget", post(routes::forget_by_id)) .route("/api/stats", post(routes::stats)) // Router::layer runs middleware bottom-to-top (last added runs first). // Keep auth outer so AuthInfo is in request extensions before rate limiting reads it. diff --git a/services/server/src/routes/admin.rs b/services/server/src/routes/admin.rs index d5b55ac8..6fffc2f0 100644 --- a/services/server/src/routes/admin.rs +++ b/services/server/src/routes/admin.rs @@ -74,6 +74,169 @@ pub async fn forget( })) } +/// POST /api/clear-namespace +/// +/// User-facing SOFT delete: tombstone every memory in `owner`'s `namespace` +/// (set `deleted_at`, rows retained). The memories immediately stop surfacing +/// in `recall()` and in the analyze pre-extraction context. The underlying +/// Walrus blobs persist (user-owned; Walrus has no delete) — this stops +/// *retrievability*, not blob erasure. Owner-scoped. +/// +/// Distinct from `/api/forget`, which HARD-deletes and is kept for the +/// benchmark harness's inter-run cleanup. This is the SDK/MCP-exposed path. +pub async fn clear_namespace( + State(state): State>, + Extension(auth): Extension, + Json(body): Json, +) -> Result, AppError> { + if body.namespace.is_empty() { + return Err(AppError::BadRequest("namespace cannot be empty".into())); + } + + let owner = &auth.owner; + let namespace = &body.namespace; + tracing::info!("clear_namespace: owner={} ns={}", owner, namespace); + + let cleared = state.db.soft_delete_by_namespace(owner, namespace).await?; + + tracing::info!( + "clear_namespace complete: soft-deleted {} entries for owner={} ns={}", + cleared, + owner, + namespace + ); + + Ok(Json(ClearNamespaceResponse { + cleared, + namespace: namespace.clone(), + owner: owner.clone(), + })) +} + +/// Encode a `(created_at, id)` keyset position as an opaque pagination cursor. +/// Base64url of `|` — opaque to the caller (they pass it +/// back verbatim), but cheap to decode and free of the `id` scheme leaking out. +fn encode_list_cursor(created_at: &chrono::DateTime, id: &str) -> String { + use base64::Engine as _; + let raw = format!("{}|{}", created_at.to_rfc3339(), id); + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(raw) +} + +/// Decode a cursor produced by `encode_list_cursor`. A malformed cursor is a +/// client error (`400`) rather than a silent first-page reset, so a paging bug +/// surfaces instead of looping over the same page. +fn decode_list_cursor(cursor: &str) -> Result<(chrono::DateTime, String), AppError> { + use base64::Engine as _; + let bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(cursor) + .map_err(|_| AppError::BadRequest("invalid cursor".into()))?; + let decoded = + String::from_utf8(bytes).map_err(|_| AppError::BadRequest("invalid cursor".into()))?; + let (ts_str, id) = decoded + .split_once('|') + .ok_or_else(|| AppError::BadRequest("invalid cursor".into()))?; + let ts = chrono::DateTime::parse_from_rfc3339(ts_str) + .map_err(|_| AppError::BadRequest("invalid cursor".into()))? + .with_timezone(&chrono::Utc); + Ok((ts, id.to_string())) +} + +/// POST /api/list +/// +/// Enumerate the live memories in `owner`'s `namespace` — metadata only +/// (`id`, `blob_id`, `created_at`, `importance`), newest first, no decrypted +/// text. Decrypt-free, so it's cheap to call for auditing. The per-row `id` +/// is the handle for `/api/memories/forget`. Owner-scoped; soft-deleted +/// memories are omitted. +/// +/// Cursor-paginated: `returned` is this page's size (NOT the namespace total); +/// when `has_more` is true, pass `next_cursor` back as `cursor` for the next +/// page. The keyset cursor is stable under concurrent deletes — forgotten rows +/// simply drop out, they don't shift a page boundary. +pub async fn list( + State(state): State>, + Extension(auth): Extension, + Json(body): Json, +) -> Result, AppError> { + if body.namespace.is_empty() { + return Err(AppError::BadRequest("namespace cannot be empty".into())); + } + // Cap the page size so a huge namespace can't return an unbounded list. + let limit = body.limit.clamp(1, 500); + let before = match body.cursor.as_deref() { + Some(c) => Some(decode_list_cursor(c)?), + None => None, + }; + + let owner = &auth.owner; + let namespace = &body.namespace; + + let (memories, has_more) = state + .db + .list_by_namespace(owner, namespace, limit, before) + .await?; + let returned = memories.len(); + + // Cursor for the next page = the keyset position of the last row returned. + let next_cursor = if has_more { + memories + .last() + .map(|m| encode_list_cursor(&m.created_at, &m.id)) + } else { + None + }; + + tracing::info!( + "list: owner={} ns={} returned={} has_more={}", + owner, + namespace, + returned, + has_more + ); + + Ok(Json(ListResponse { + memories, + returned, + has_more, + next_cursor, + namespace: namespace.clone(), + owner: owner.clone(), + })) +} + +/// POST /api/memories/forget +/// +/// Soft-delete a SINGLE memory by its per-row `id` (from `/api/list`). The +/// memory stops surfacing in `recall()`; identical-text siblings survive +/// (per-row). The Walrus blob persists (un-recallable, not erased). +/// Owner-scoped — the `id` must belong to the caller, else this is a no-op +/// (`forgotten = 0`). Distinct from `/api/clear-namespace` and `/api/forget`. +pub async fn forget_by_id( + State(state): State>, + Extension(auth): Extension, + Json(body): Json, +) -> Result, AppError> { + if body.id.is_empty() { + return Err(AppError::BadRequest("id cannot be empty".into())); + } + + let owner = &auth.owner; + let forgotten = state.db.soft_delete_by_id(&body.id, owner).await?; + + tracing::info!( + "forget_by_id: owner={} id={} forgotten={}", + owner, + body.id, + forgotten + ); + + Ok(Json(ForgetByIdResponse { + forgotten, + id: body.id, + owner: owner.clone(), + })) +} + /// POST /api/stats /// /// Return memory count + stored bytes for `owner`'s `namespace`. Used by @@ -199,7 +362,7 @@ pub async fn ask( "ask request" ); - // F3 (structure-review): probe the SEAL credential up front. If the + // probe the SEAL credential up front. If the // client is misconfigured (no exported SessionKey, no legacy delegate // key, no server fallback) we want to return 500 immediately rather // than running recall, getting zero (or some) hits, and then either @@ -733,7 +896,7 @@ mod tests { // ── /api/ask body.limit cap ───────────────────────────────── // - // Verifies the structural-review F3 follow-up: `/api/ask` clamps + // Verifies that `/api/ask` clamps // `body.limit` to `<= 100`, matching the cap `/api/recall` already // enforces. A misbehaving client can't make the handler pull // thousands of memories through Walrus + SEAL. @@ -781,4 +944,54 @@ mod tests { "non-empty namespace must pass the validation predicate" ); } + + // ── /api/list limit clamp ───────────────────────── + // + // `list()` clamps `body.limit` to `[1, 500]` so a client can neither pull + // an unbounded page nor pass 0 (which would otherwise return no rows). + // Mirror the production expression: body.limit.clamp(1, 500). + + #[test] + fn list_limit_clamps_to_1_500() { + for (input, expected) in [ + (0usize, 1), // below floor → 1 (not "all"/0) + (1, 1), // at floor + (50, 50), // default-ish, in range + (500, 500), // at ceiling + (501, 500), // over ceiling → clamped + (10_000, 500), + (usize::MAX, 500), + ] { + let clamped = input.clamp(1, 500); + assert_eq!( + clamped, expected, + "list limit clamp: input={input} expected={expected} got={clamped}" + ); + } + } + + // ── empty-input guards on the new handlers ──────── + // + // clear_namespace + list reject an empty `namespace`; forget_by_id rejects + // an empty `id` — all with AppError::BadRequest before touching the DB, + // matching the existing forget/stats convention. Pin the predicates. + + #[test] + fn delete_handlers_reject_empty_inputs() { + // clear_namespace / list: `body.namespace.is_empty()` + assert!( + "".is_empty(), + "empty namespace must trip clear/list validation" + ); + assert!( + !"my-app".is_empty(), + "non-empty namespace must pass clear/list validation" + ); + // forget_by_id: `body.id.is_empty()` + assert!("".is_empty(), "empty id must trip forget_by_id validation"); + assert!( + !"550e8400-e29b-41d4-a716-446655440000".is_empty(), + "non-empty id must pass forget_by_id validation" + ); + } } diff --git a/services/server/src/routes/analyze.rs b/services/server/src/routes/analyze.rs index 6cc7f921..788a4735 100644 --- a/services/server/src/routes/analyze.rs +++ b/services/server/src/routes/analyze.rs @@ -58,6 +58,15 @@ const EMBED_TIMEOUT_MS: u64 = 800; const SEARCH_TIMEOUT_MS: u64 = 300; const FETCH_TIMEOUT_MS: u64 = 500; +// The pre-extraction namespace-existence check. The +// `deleted_at IS NULL` filter is SEPARATE from the one in `search_similar` — +// reverting only this one would let a fully-cleared namespace still read as +// "has memories" (wasted embed+search) and is the second of the two read-path +// filters that are load-bearing. Named so a unit test +// (`mod tests`) pins the filter even though the query needs a live DB to run. +const NAMESPACE_EXISTENCE_SQL: &str = + "SELECT 1 FROM vector_entries WHERE owner = $1 AND namespace = $2 AND deleted_at IS NULL LIMIT 1"; + /// POST /api/analyze /// /// AI fact extraction flow: @@ -124,9 +133,11 @@ pub async fn analyze( let mut pre_extract_seal_ms: u128 = 0; let mut pre_extract_dropped: usize = 0; - let namespace_has_memories: bool = sqlx::query_scalar::<_, i32>( - "SELECT 1 FROM vector_entries WHERE owner = $1 AND namespace = $2 LIMIT 1", - ) + // `deleted_at IS NULL` (migration 010): a fully soft-deleted / cleared + // namespace must read as empty here, so analyze takes the + // skip-pre-extraction fast path instead of embedding + searching against + // tombstoned rows that search_similar would filter out anyway. + let namespace_has_memories: bool = sqlx::query_scalar::<_, i32>(NAMESPACE_EXISTENCE_SQL) .bind(owner) .bind(namespace) .fetch_optional(state.db.pool()) @@ -568,7 +579,7 @@ pub async fn analyze( #[cfg(test)] mod tests { - use super::{ANALYZE_CONCURRENCY, MAX_ANALYZE_TEXT_BYTES}; + use super::{ANALYZE_CONCURRENCY, MAX_ANALYZE_TEXT_BYTES, NAMESPACE_EXISTENCE_SQL}; use crate::routes::remember::MAX_REMEMBER_TEXT_BYTES; use crate::services::extractor::MAX_ANALYZE_FACTS; @@ -609,4 +620,25 @@ mod tests { assert_eq!(analyze_additional_weight(0), 0); assert_eq!(analyze_additional_weight(20), 20); } + + // ── pre-extraction filter keeps deleted_at ── + // + // The namespace-existence check is the SECOND of the two read-path + // tombstone filters (search_similar is the first). Reverting only this one + // would let a fully-cleared namespace still read as "has memories" and pull + // tombstoned rows into the extractor's dedup context. The query needs a + // live DB to run, so pin the filter on the SQL string itself. + + #[test] + fn pre_extraction_existence_check_filters_soft_deleted() { + assert!( + NAMESPACE_EXISTENCE_SQL.contains("deleted_at IS NULL"), + "analyze namespace-existence check must exclude soft-deleted rows: {NAMESPACE_EXISTENCE_SQL}" + ); + assert!( + NAMESPACE_EXISTENCE_SQL.contains("owner = $1") + && NAMESPACE_EXISTENCE_SQL.contains("namespace = $2"), + "existence check must stay owner+namespace scoped: {NAMESPACE_EXISTENCE_SQL}" + ); + } } diff --git a/services/server/src/routes/mod.rs b/services/server/src/routes/mod.rs index 9cfc330d..f4f030af 100644 --- a/services/server/src/routes/mod.rs +++ b/services/server/src/routes/mod.rs @@ -22,7 +22,9 @@ mod sponsor; // Re-export every handler so `main.rs` keeps using `routes::` // without having to know which submodule each handler lives in. -pub use admin::{ask, forget, get_config, health, restore, stats, version}; +pub use admin::{ + ask, clear_namespace, forget, forget_by_id, get_config, health, list, restore, stats, version, +}; pub use analyze::analyze; pub use recall::{recall, recall_manual}; pub use remember::{ diff --git a/services/server/src/storage/db.rs b/services/server/src/storage/db.rs index 968539d5..22a348d5 100644 --- a/services/server/src/storage/db.rs +++ b/services/server/src/storage/db.rs @@ -2,7 +2,37 @@ use pgvector::Vector; use sqlx::postgres::PgPoolOptions; use sqlx::PgPool; -use crate::types::{AppError, SearchHit}; +use crate::types::{AppError, MemoryListItem, SearchHit}; + +// Soft-delete SQL, named so a unit test can pin the +// load-bearing invariant that these are UPDATEs (tombstone) — distinct from +// the HARD `DELETE FROM vector_entries` in `delete_by_namespace`, which the +// benchmark harness depends on. An accidental repoint of `/api/forget` at a +// soft path would break bench isolation; the test on these consts guards it. +const SQL_SOFT_DELETE_BY_NAMESPACE: &str = "UPDATE vector_entries SET deleted_at = NOW() + WHERE owner = $1 AND namespace = $2 AND deleted_at IS NULL"; +const SQL_SOFT_DELETE_BY_ID: &str = "UPDATE vector_entries SET deleted_at = NOW() + WHERE id = $1 AND owner = $2 AND deleted_at IS NULL"; +const SQL_HARD_DELETE_BY_NAMESPACE: &str = + "DELETE FROM vector_entries WHERE owner = $1 AND namespace = $2"; + +// Keyset-paginated namespace listing. The `(created_at, id)` row-value compare +// is the exact inverse of the `ORDER BY created_at DESC, id DESC` keyset, so a +// cursor never skips or repeats a row even when timestamps tie. Named so a unit +// test can pin the tie-break + the live-rows-only filter. +const SQL_LIST_BY_NAMESPACE: &str = "SELECT id, blob_id, created_at, importance + FROM vector_entries + WHERE owner = $1 AND namespace = $2 AND deleted_at IS NULL + AND ($3::timestamptz IS NULL OR (created_at, id) < ($3, $4)) + ORDER BY created_at DESC, id DESC + LIMIT $5"; + +// Live-set stats (count + bytes). `deleted_at IS NULL` so soft-deleted rows +// don't inflate the count — consistent with recall + list. Named so a unit +// test pins the filter (a stats SDK wrapper must not inherit a tombstone over-count). +const SQL_NAMESPACE_STATS: &str = + "SELECT COUNT(*)::BIGINT, COALESCE(SUM(blob_size_bytes)::BIGINT, 0) + FROM vector_entries WHERE owner = $1 AND namespace = $2 AND deleted_at IS NULL"; pub struct VectorDb { pool: PgPool, @@ -90,6 +120,14 @@ impl VectorDb { .await .map_err(|e| AppError::Internal(format!("Failed to run migration 009: {}", e)))?; + // soft-delete tombstone (deleted_at) for memory deletion / namespace + // clearing. Additive nullable column + partial index over live rows. + let migration_010 = include_str!("../../migrations/010_soft_delete.sql"); + sqlx::raw_sql(migration_010) + .execute(&pool) + .await + .map_err(|e| AppError::Internal(format!("Failed to run migration 010: {}", e)))?; + tracing::info!("database connected and migrations applied"); Ok(Self { pool }) @@ -268,12 +306,18 @@ impl VectorDb { // without a second round-trip. Both NOT NULL (migration 001 for // created_at, 009 for importance) so the row tuple types are // non-Option. + // + // `deleted_at IS NULL` (migration 010) hides soft-deleted memories. + // This is the single chokepoint for the recall read path AND the + // analyze pre-extraction dedup context (which also calls this fn), so + // a soft-deleted memory neither surfaces in recall nor leaks back to + // the extractor as "related context". let started = std::time::Instant::now(); let result: Result, f32)>, AppError> = sqlx::query_as( "SELECT blob_id, (embedding <=> $1)::float8 AS distance, created_at, importance FROM vector_entries - WHERE owner = $2 AND namespace = $3 + WHERE owner = $2 AND namespace = $3 AND deleted_at IS NULL ORDER BY embedding <=> $1 LIMIT $4", ) @@ -333,21 +377,21 @@ impl VectorDb { /// Count + total stored bytes for a given owner + namespace. /// Used by `POST /api/stats` for harness verification. Returns /// `(memory_count, storage_bytes)`; both 0 if the namespace is empty. + /// + /// `deleted_at IS NULL` so soft-deleted memories don't inflate the count — + /// stats reflect the *live* set, consistent with what recall + list return. pub async fn namespace_stats( &self, owner: &str, namespace: &str, ) -> Result<(i64, i64), AppError> { let started = std::time::Instant::now(); - let result: Result<(i64, i64), AppError> = sqlx::query_as( - "SELECT COUNT(*)::BIGINT, COALESCE(SUM(blob_size_bytes)::BIGINT, 0) - FROM vector_entries WHERE owner = $1 AND namespace = $2", - ) - .bind(owner) - .bind(namespace) - .fetch_one(&self.pool) - .await - .map_err(|e| AppError::Internal(format!("Failed to get namespace stats: {}", e))); + let result: Result<(i64, i64), AppError> = sqlx::query_as(SQL_NAMESPACE_STATS) + .bind(owner) + .bind(namespace) + .fetch_one(&self.pool) + .await + .map_err(|e| AppError::Internal(format!("Failed to get namespace stats: {}", e))); crate::observability::observe_db( "vector.namespace_stats", db_status(&result), @@ -358,6 +402,99 @@ impl VectorDb { Ok(row) } + /// List the live (non-tombstoned) memories in a namespace as metadata — + /// `(id, blob_id, created_at, importance)`, newest first. Owner-scoped. + /// + /// Metadata-only by design: returns the per-row `id` (the unique handle the + /// user needs for `forget(id)`) without a Walrus fetch or SEAL decrypt, so + /// it's cheap to call for auditing. The memory *text* is NOT returned — + /// surfacing it would cost recall-grade decrypt and the plaintext column is + /// NULL in production. `deleted_at IS NULL` so cleared/forgotten memories + /// don't appear. + /// + /// Cursor pagination: ordered by `(created_at, id)` DESC (the `id` + /// tie-break makes the order total + stable when timestamps collide). Pass + /// `before` = the `(created_at, id)` of the last row from the previous page + /// to get the next page; `None` starts from the newest. Fetches `limit + 1` + /// internally to report whether more rows remain, and returns at most + /// `limit` rows plus that `has_more` flag. + pub async fn list_by_namespace( + &self, + owner: &str, + namespace: &str, + limit: usize, + before: Option<(chrono::DateTime, String)>, + ) -> Result<(Vec, bool), AppError> { + // Over-fetch one row: if we get limit+1 back, there's a next page. + let fetch = (limit as i64) + 1; + let started = std::time::Instant::now(); + let query = SQL_LIST_BY_NAMESPACE; + let (cursor_ts, cursor_id) = match before { + Some((ts, id)) => (Some(ts), Some(id)), + None => (None, None), + }; + let result: Result, f32)>, AppError> = + sqlx::query_as(query) + .bind(owner) + .bind(namespace) + .bind(cursor_ts) + .bind(cursor_id) + .bind(fetch) + .fetch_all(&self.pool) + .await + .map_err(|e| AppError::Internal(format!("Failed to list namespace: {}", e))); + crate::observability::observe_db( + "vector.list_by_namespace", + db_status(&result), + started.elapsed(), + ); + let mut rows = result?; + + // The (limit+1)th row, if present, only signals "more remain" — drop it + // so the caller sees at most `limit` items. + let has_more = rows.len() > limit; + rows.truncate(limit); + + let memories = rows + .into_iter() + .map(|(id, blob_id, created_at, importance)| MemoryListItem { + id, + blob_id, + created_at, + importance, + }) + .collect(); + Ok((memories, has_more)) + } + + /// Soft-delete (tombstone) a single memory by its per-row `id`, scoped to + /// `owner`. Per-row: identical-text siblings (different `id`, same + /// `blob_id`) are untouched. The `id` comes from `list_by_namespace`. + /// Returns the number of rows tombstoned (0 if the id doesn't exist, isn't + /// the caller's, or was already deleted). + pub async fn soft_delete_by_id(&self, id: &str, owner: &str) -> Result { + let started = std::time::Instant::now(); + let result = sqlx::query(SQL_SOFT_DELETE_BY_ID) + .bind(id) + .bind(owner) + .execute(&self.pool) + .await + .map_err(|e| AppError::Internal(format!("Failed to soft-delete by id: {}", e))); + crate::observability::observe_db( + "vector.soft_delete_by_id", + db_status(&result), + started.elapsed(), + ); + let rows = result?.rows_affected(); + tracing::info!( + "soft-deleted {} row(s) for id={}, owner={}", + rows, + id, + owner + ); + Ok(rows) + } + /// Hard-delete all vector index rows for a given owner + namespace. /// (Walrus blobs themselves persist — Walrus has no delete; this only /// removes the local `vector_entries` rows, so the memories stop being @@ -365,7 +502,7 @@ impl VectorDb { /// `POST /api/forget` — authed, owner-scoped. pub async fn delete_by_namespace(&self, owner: &str, namespace: &str) -> Result { let started = std::time::Instant::now(); - let result = sqlx::query("DELETE FROM vector_entries WHERE owner = $1 AND namespace = $2") + let result = sqlx::query(SQL_HARD_DELETE_BY_NAMESPACE) .bind(owner) .bind(namespace) .execute(&self.pool) @@ -388,6 +525,42 @@ impl VectorDb { Ok(rows) } + /// Soft-delete (tombstone) every live memory in a namespace by setting + /// `deleted_at = NOW()`. Rows are RETAINED (not removed) — this is the + /// user-facing `clearNamespace` path. Tombstoned rows stop surfacing in + /// recall + the pre-extraction context (both filter `deleted_at IS NULL`) + /// but remain visible to restore's presence-check, so restore won't + /// re-index their on-chain blobs. Distinct from `delete_by_namespace` + /// (hard DELETE, kept for the benchmark harness). Owner-scoped. + /// Returns the number of rows newly tombstoned (already-deleted rows are + /// skipped via `deleted_at IS NULL`, so a repeat call returns 0). + pub async fn soft_delete_by_namespace( + &self, + owner: &str, + namespace: &str, + ) -> Result { + let started = std::time::Instant::now(); + let result = sqlx::query(SQL_SOFT_DELETE_BY_NAMESPACE) + .bind(owner) + .bind(namespace) + .execute(&self.pool) + .await + .map_err(|e| AppError::Internal(format!("Failed to soft-delete by namespace: {}", e))); + crate::observability::observe_db( + "vector.soft_delete_by_namespace", + db_status(&result), + started.elapsed(), + ); + let rows = result?.rows_affected(); + tracing::info!( + "soft-deleted {} entries for owner={}, ns={}", + rows, + owner, + namespace + ); + Ok(rows) + } + /// Delete a vector entry by blob_id (used for expired blob cleanup). /// Called reactively when Walrus returns 404 during blob download. /// Requires owner to prevent cross-user blob deletion. @@ -608,3 +781,123 @@ impl VectorDb { Ok(result.map(|(id,)| id)) } } + +#[cfg(test)] +mod tests { + use super::{ + SQL_HARD_DELETE_BY_NAMESPACE, SQL_LIST_BY_NAMESPACE, SQL_NAMESPACE_STATS, + SQL_SOFT_DELETE_BY_ID, SQL_SOFT_DELETE_BY_NAMESPACE, + }; + + // ── /api/forget stays HARD; clearNamespace is SOFT ──── + // + // The benchmark harness calls /api/forget for inter-run cleanup and + // depends on it HARD-deleting rows. Soft-delete (clearNamespace) must be a + // SEPARATE path — an accidental repoint of the hard handler at a soft query + // would silently accumulate tombstones across bench runs and corrupt + // ingest counts, with no live-DB test catching it. Pin the invariant on the + // SQL the handlers actually run. + + #[test] + fn hard_delete_is_a_delete_not_a_soft_update() { + assert!( + SQL_HARD_DELETE_BY_NAMESPACE.contains("DELETE FROM vector_entries"), + "/api/forget must HARD-delete (benchmark isolation): {SQL_HARD_DELETE_BY_NAMESPACE}" + ); + assert!( + !SQL_HARD_DELETE_BY_NAMESPACE.contains("deleted_at"), + "hard delete must not be a tombstone UPDATE: {SQL_HARD_DELETE_BY_NAMESPACE}" + ); + } + + #[test] + fn soft_delete_paths_are_tombstone_updates() { + for (label, sql) in [ + ("by_namespace", SQL_SOFT_DELETE_BY_NAMESPACE), + ("by_id", SQL_SOFT_DELETE_BY_ID), + ] { + assert!( + sql.contains("UPDATE vector_entries SET deleted_at = NOW()"), + "soft-delete {label} must tombstone via UPDATE, not DELETE: {sql}" + ); + assert!( + !sql.contains("DELETE FROM"), + "soft-delete {label} must not hard-delete: {sql}" + ); + // Idempotency guard: re-deleting an already-tombstoned row is a + // no-op (0 rows), which the route relies on for clean re-clear/ + // re-forget semantics. + assert!( + sql.contains("deleted_at IS NULL"), + "soft-delete {label} must guard `deleted_at IS NULL` for idempotency: {sql}" + ); + } + } + + #[test] + fn soft_and_hard_namespace_deletes_are_distinct() { + assert_ne!( + SQL_SOFT_DELETE_BY_NAMESPACE, SQL_HARD_DELETE_BY_NAMESPACE, + "soft (clearNamespace) and hard (/api/forget) namespace deletes must differ" + ); + } + + // ── owner-scoping: every soft-delete SQL binds owner ── + // + // The privacy-floor invariant: a user can only delete their OWN memories. + // forget_by_id takes a user-supplied id, so it MUST also constrain owner + // (else a guessed id forgets someone else's memory). Pin that both + // soft-delete queries reference `owner`. + + #[test] + fn soft_delete_queries_are_owner_scoped() { + assert!( + SQL_SOFT_DELETE_BY_NAMESPACE.contains("owner = $1"), + "namespace soft-delete must be owner-scoped" + ); + assert!( + SQL_SOFT_DELETE_BY_ID.contains("owner = $2"), + "by-id soft-delete must bind owner (IDOR guard), not just id" + ); + } + + // ── list() keyset pagination is stable + live-only ── + // + // The cursor must order by the full (created_at, id) keyset so ties are + // deterministic across pages (else paging can skip/duplicate rows), and + // must hide soft-deleted rows. Pin both on the actual query. + + #[test] + fn list_query_has_stable_keyset_and_live_filter() { + assert!( + SQL_LIST_BY_NAMESPACE.contains("ORDER BY created_at DESC, id DESC"), + "list must tie-break on id for stable pagination: {SQL_LIST_BY_NAMESPACE}" + ); + assert!( + SQL_LIST_BY_NAMESPACE.contains("(created_at, id) < ($3, $4)"), + "list cursor must be a (created_at, id) row-value compare matching the keyset" + ); + assert!( + SQL_LIST_BY_NAMESPACE.contains("deleted_at IS NULL"), + "list must omit soft-deleted rows" + ); + assert!( + SQL_LIST_BY_NAMESPACE.contains("owner = $1") + && SQL_LIST_BY_NAMESPACE.contains("namespace = $2"), + "list must stay owner+namespace scoped" + ); + } + + // ── stats() counts only LIVE rows ── + // + // After a soft-delete, stats must not over-count tombstones — otherwise a + // future stats SDK wrapper would inherit a misleading count. + + #[test] + fn stats_query_excludes_soft_deleted() { + assert!( + SQL_NAMESPACE_STATS.contains("deleted_at IS NULL"), + "stats must exclude soft-deleted rows: {SQL_NAMESPACE_STATS}" + ); + } +} diff --git a/services/server/src/types.rs b/services/server/src/types.rs index 246b4005..6e4a092a 100644 --- a/services/server/src/types.rs +++ b/services/server/src/types.rs @@ -544,6 +544,19 @@ pub struct SearchHit { pub importance: f32, } +/// One row in a `/api/list` response — metadata for a stored memory, with NO +/// decrypted text (listing is decrypt-free). `id` is the unique per-row handle +/// the caller passes to `/api/memories/forget` to delete this specific memory. +#[derive(Debug, Clone, Serialize)] +pub struct MemoryListItem { + /// Unique per-row id (`vector_entries.id`) — the handle for `forget`. + pub id: String, + /// Walrus blob id (not unique; identical-text memories share one). + pub blob_id: String, + pub created_at: chrono::DateTime, + pub importance: f32, +} + /// Composite-scoring weights for `/api/recall` and `/api/ask`. Optional on /// the wire — when omitted, the response order is byte-identical to a /// pure pgvector cosine-distance sort (today's behaviour). @@ -859,6 +872,79 @@ pub struct ForgetResponse { pub owner: String, } +/// POST /api/clear-namespace — user-facing SOFT-delete of every memory in a +/// namespace (sets `deleted_at`; rows retained, Walrus blobs persist). The +/// memories stop surfacing in recall. Distinct from /api/forget (hard DELETE, +/// harness-only). Owner-scoped. +#[derive(Debug, Deserialize)] +pub struct ClearNamespaceRequest { + #[serde(default = "default_namespace")] + pub namespace: String, +} + +#[derive(Debug, Serialize)] +pub struct ClearNamespaceResponse { + /// Number of memories newly soft-deleted (0 if already cleared). + pub cleared: u64, + pub namespace: String, + pub owner: String, +} + +/// POST /api/list — enumerate a namespace's live memories (metadata only, +/// decrypt-free). Returns each memory's per-row `id` so the caller can +/// `forget` a specific one. Owner-scoped. +fn default_list_limit() -> usize { + 50 +} + +#[derive(Debug, Deserialize)] +pub struct ListRequest { + #[serde(default = "default_namespace")] + pub namespace: String, + #[serde(default = "default_list_limit")] + pub limit: usize, + /// Opaque pagination cursor from a previous response's `next_cursor`. + /// Omit (or null) for the first page; pass it verbatim to get the next. + #[serde(default)] + pub cursor: Option, +} + +#[derive(Debug, Serialize)] +pub struct ListResponse { + pub memories: Vec, + /// Number of memories in THIS page (== `memories.len()`, capped by `limit`). + /// NOT the namespace's total memory count — use `has_more` + paging to + /// enumerate fully. + pub returned: usize, + /// True if more live memories exist beyond this page. When true, pass + /// `next_cursor` back as `cursor` to fetch the next page. + pub has_more: bool, + /// Opaque cursor for the next page; `None` when `has_more` is false. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_cursor: Option, + pub namespace: String, + pub owner: String, +} + +/// POST /api/memories/forget — soft-delete a SINGLE memory by its per-row +/// `id` (from `/api/list`). Per-row: identical-text siblings survive. The +/// underlying Walrus blob persists (un-recallable, not erased). Owner-scoped. +/// Distinct from `/api/clear-namespace` (whole-namespace) and `/api/forget` +/// (hard delete, harness-only). +#[derive(Debug, Deserialize)] +pub struct ForgetByIdRequest { + pub id: String, +} + +#[derive(Debug, Serialize)] +pub struct ForgetByIdResponse { + /// 1 if the memory was soft-deleted, 0 if not found / not the caller's / + /// already deleted. + pub forgotten: u64, + pub id: String, + pub owner: String, +} + /// POST /api/stats — count + stored bytes for a namespace. /// Used by the benchmark harness for verification. Mode-blind. #[derive(Debug, Deserialize)] diff --git a/services/server/tests/e2e_test.py b/services/server/tests/e2e_test.py index d92b5069..b5dbccba 100644 --- a/services/server/tests/e2e_test.py +++ b/services/server/tests/e2e_test.py @@ -136,6 +136,24 @@ def _load_delegate_key() -> SigningKey | None: return SigningKey(raw, encoder=RawEncoder) +def _load_delegate_key_2() -> tuple[SigningKey | None, str | None]: + """Optional SECOND owner (TEST_DELEGATE_KEY_2 + TEST_ACCOUNT_ID_2) for the + cross-owner isolation test. Returns (key, account_id) or (None, None) + if not configured — the cross-owner test is then skipped (documented gap).""" + hex_key = os.environ.get("TEST_DELEGATE_KEY_2", "").strip() + if not hex_key: + return None, None + try: + raw = bytes.fromhex(hex_key) + except ValueError: + print("[warn] TEST_DELEGATE_KEY_2 is not valid hex; skipping cross-owner check") + return None, None + if len(raw) != 32: + print(f"[warn] TEST_DELEGATE_KEY_2 must be 32 bytes (got {len(raw)}); skipping cross-owner check") + return None, None + return SigningKey(raw, encoder=RawEncoder), (os.environ.get("TEST_ACCOUNT_ID_2") or None) + + def test_health() -> None: req = urllib.request.Request(f"{BASE_URL}/health") with urllib.request.urlopen(req) as resp: @@ -280,6 +298,323 @@ def test_remember_recall_happy_path(signing_key: SigningKey, account_id: str | N print(f"[pass] POST /api/recall → {recall_result['total']} hits, top distance={top['distance']:.4f}") +def test_clear_namespace_soft_delete( + signing_key: SigningKey, account_id: str | None +) -> None: + """remember → recall (present) → clearNamespace → recall (absent). + + Verifies the soft-delete contract end-to-end: a cleared namespace stops + surfacing in recall, and an unrelated namespace is untouched (owner+ns + scoping). Shares the Walrus/SEAL/Sui prerequisites with the happy path. + """ + ns = "e2e-clear-test" + sibling = "e2e-clear-keep" + + # 1. Remember one memory in each namespace. + for namespace, text in ((ns, "The sky is blue."), (sibling, "Grass is green.")): + r = make_signed_request( + "POST", "/api/remember", {"text": text, "namespace": namespace}, + signing_key, account_id=account_id, + ) + wait_for_remember_job(signing_key, account_id, r["job_id"]) + print(f"[pass] seeded memories in {ns} + {sibling}") + + # 2. Recall — the target memory is present. + before = make_signed_request( + "POST", "/api/recall", + {"query": "What colour is the sky?", "limit": 5, "namespace": ns}, + signing_key, account_id=account_id, + ) + assert before["total"] >= 1, f"Expected ≥1 hit before clear, got {before['total']}" + print(f"[pass] recall before clear → {before['total']} hit(s)") + + # 3. clearNamespace (soft-delete). + cleared = make_signed_request( + "POST", "/api/clear-namespace", {"namespace": ns}, + signing_key, account_id=account_id, + ) + assert cleared["cleared"] >= 1, f"Expected ≥1 cleared, got {cleared}" + print(f"[pass] POST /api/clear-namespace → cleared={cleared['cleared']}") + + # 4. Recall again — the cleared namespace returns nothing. + after = make_signed_request( + "POST", "/api/recall", + {"query": "What colour is the sky?", "limit": 5, "namespace": ns}, + signing_key, account_id=account_id, + ) + assert after["total"] == 0, f"Expected 0 hits after clear, got {after['total']}" + print(f"[pass] recall after clear → {after['total']} hits (soft-deleted)") + + # 5. Sibling namespace is untouched (owner+namespace scoping). + keep = make_signed_request( + "POST", "/api/recall", + {"query": "What colour is grass?", "limit": 5, "namespace": sibling}, + signing_key, account_id=account_id, + ) + assert keep["total"] >= 1, f"Sibling namespace wrongly cleared, got {keep['total']}" + print(f"[pass] sibling namespace intact → {keep['total']} hit(s)") + + # 6. Re-clear is idempotent — already-cleared rows are skipped (0). + again = make_signed_request( + "POST", "/api/clear-namespace", {"namespace": ns}, + signing_key, account_id=account_id, + ) + assert again["cleared"] == 0, f"Expected 0 on re-clear, got {again}" + print(f"[pass] re-clear idempotent → cleared={again['cleared']}") + + +def test_list_and_forget_by_id( + signing_key: SigningKey, account_id: str | None +) -> None: + """list() exposes per-row ids; forget(id) is per-row. + + Stores two DISTINCT memories, lists to get their ids, forgets one, and + asserts: the forgotten one is gone from recall + list, the other survives. + Then verifies forget is owner/id-scoped (a bogus id → forgotten=0). + Shares the Walrus/SEAL/Sui prerequisites with the happy path. + """ + ns = "e2e-forget-test" + + # Seed two distinct memories. + for text in ("Alice plays the violin.", "Bob coaches soccer."): + r = make_signed_request( + "POST", "/api/remember", {"text": text, "namespace": ns}, + signing_key, account_id=account_id, + ) + wait_for_remember_job(signing_key, account_id, r["job_id"]) + print(f"[pass] seeded 2 memories in {ns}") + + # list() returns per-row ids (metadata only — no text field). + listing = make_signed_request( + "POST", "/api/list", {"namespace": ns, "limit": 50}, + signing_key, account_id=account_id, + ) + assert listing["returned"] >= 2, f"Expected ≥2 listed, got {listing['returned']}" + for m in listing["memories"]: + assert "id" in m and m["id"], f"list item missing id: {m}" + assert "text" not in m, f"list must be metadata-only, leaked text: {m}" + target_id = listing["memories"][0]["id"] + print(f"[pass] POST /api/list → {listing['returned']} items, ids present, no text") + + # forget one by id. + forgot = make_signed_request( + "POST", "/api/memories/forget", {"id": target_id}, + signing_key, account_id=account_id, + ) + assert forgot["forgotten"] == 1, f"Expected forgotten=1, got {forgot}" + print(f"[pass] POST /api/memories/forget id={target_id[:8]}… → forgotten=1") + + # The forgotten id no longer appears in list; the other remains. + after = make_signed_request( + "POST", "/api/list", {"namespace": ns, "limit": 50}, + signing_key, account_id=account_id, + ) + remaining_ids = {m["id"] for m in after["memories"]} + assert target_id not in remaining_ids, "Forgotten id still listed" + assert after["returned"] == listing["returned"] - 1, ( + f"Expected one fewer after forget: {listing['returned']} → {after['returned']}" + ) + print(f"[pass] list after forget → {after['returned']} (forgotten id absent)") + + # forget is id/owner-scoped: a bogus id is a no-op, not an error. + noop = make_signed_request( + "POST", "/api/memories/forget", {"id": "00000000-0000-0000-0000-000000000000"}, + signing_key, account_id=account_id, + ) + assert noop["forgotten"] == 0, f"Expected forgotten=0 for bogus id, got {noop}" + print(f"[pass] forget bogus id → forgotten=0 (no-op, scoped)") + + # Double-forget the SAME real id is idempotent — the second call hits the + # already-tombstoned branch (deleted_at NOT NULL), distinct from not-found, + # and must also return 0. + redo = make_signed_request( + "POST", "/api/memories/forget", {"id": target_id}, + signing_key, account_id=account_id, + ) + assert redo["forgotten"] == 0, f"Expected forgotten=0 on re-forget, got {redo}" + print(f"[pass] re-forget same id → forgotten=0 (idempotent)") + + +def test_list_pagination(signing_key: SigningKey, account_id: str | None) -> None: + """list() cursor pagination enumerates every live memory once — no skips, + no duplicates, stable across pages, has_more/next_cursor correct. + + Seeds N memories, pages through with limit < N using next_cursor, and + asserts the union of pages == the full set with no repeats and the last + page reports has_more=false. + """ + ns = "e2e-paging-test" + n = 7 + page = 3 # < n, so we get 3 + 3 + 1 across three pages + + for i in range(n): + r = make_signed_request( + "POST", "/api/remember", {"text": f"paging memory number {i}", "namespace": ns}, + signing_key, account_id=account_id, + ) + wait_for_remember_job(signing_key, account_id, r["job_id"]) + print(f"[pass] seeded {n} memories in {ns}") + + seen: list[str] = [] + cursor = None + pages = 0 + while True: + body = {"namespace": ns, "limit": page} + if cursor is not None: + body["cursor"] = cursor + resp = make_signed_request("POST", "/api/list", body, signing_key, account_id=account_id) + pages += 1 + got = [m["id"] for m in resp["memories"]] + assert len(got) == resp["returned"], "returned must equal len(memories)" + assert resp["returned"] <= page, f"page exceeded limit: {resp['returned']} > {page}" + seen.extend(got) + if resp["has_more"]: + assert resp.get("next_cursor"), "has_more=true must carry a next_cursor" + cursor = resp["next_cursor"] + else: + assert not resp.get("next_cursor"), "last page must not carry a next_cursor" + break + assert pages <= n + 2, "pagination did not terminate (cursor not advancing)" + + # Every memory seen exactly once across the pages. + assert len(seen) == n, f"expected {n} memories across pages, saw {len(seen)}" + assert len(set(seen)) == n, f"duplicate ids across pages: {len(seen)} seen, {len(set(seen))} unique" + print(f"[pass] paginated {n} memories over {pages} pages: no skips, no dups, has_more correct") + + # An invalid cursor is a 400, not a silent first-page reset. + bad = None + try: + make_signed_request( + "POST", "/api/list", {"namespace": ns, "limit": page, "cursor": "not-a-valid-cursor"}, + signing_key, account_id=account_id, + ) + except urllib.error.HTTPError as e: + bad = e.code + assert bad == 400, f"invalid cursor should 400, got {bad}" + print(f"[pass] invalid cursor → 400 (no silent reset)") + + make_signed_request("POST", "/api/clear-namespace", {"namespace": ns}, signing_key, account_id=account_id) + + +def test_forget_is_per_row_not_per_blob( + signing_key: SigningKey, account_id: str | None +) -> None: + """forget(id) is keyed on the per-row id, NOT the blob_id. + + Stores TWO memories with IDENTICAL text. SEAL ciphertext is deterministic + (same plaintext → same blob_id — the content-addressed dedup property), so + the two rows share a blob_id but have distinct row `id`s. Forgetting ONE by + its id must leave the other recallable. A buggy blob_id-keyed delete would + tombstone BOTH rows (shared blob_id) and this test would catch it — which + the prior distinct-text test could not. + """ + ns = "e2e-perrow-test" + text = "The mitochondria is the powerhouse of the cell." + + for _ in range(2): + r = make_signed_request( + "POST", "/api/remember", {"text": text, "namespace": ns}, + signing_key, account_id=account_id, + ) + wait_for_remember_job(signing_key, account_id, r["job_id"]) + + listing = make_signed_request( + "POST", "/api/list", {"namespace": ns, "limit": 50}, + signing_key, account_id=account_id, + ) + ids = [m["id"] for m in listing["memories"]] + assert len(ids) >= 2, f"Expected ≥2 identical-text rows, got {len(ids)}" + # If determinism holds they share one blob_id; assert distinct row ids regardless. + assert len(set(ids)) == len(ids), "row ids must be unique even for identical text" + print(f"[pass] 2 identical-text memories stored as {len(ids)} distinct rows") + + # Forget exactly one by its row id. + forgot = make_signed_request( + "POST", "/api/memories/forget", {"id": ids[0]}, + signing_key, account_id=account_id, + ) + assert forgot["forgotten"] == 1, ( + f"Expected forgotten=1 (per-row), got {forgot} — " + "a blob_id-keyed delete would tombstone all siblings" + ) + + # The sibling (same text, different row id) must survive. + after = make_signed_request( + "POST", "/api/list", {"namespace": ns, "limit": 50}, + signing_key, account_id=account_id, + ) + remaining = {m["id"] for m in after["memories"]} + assert ids[0] not in remaining, "forgotten row still listed" + assert ids[1] in remaining, ( + "identical-text sibling was wrongly removed — forget appears to be " + "blob_id-keyed, not per-row" + ) + print(f"[pass] forget one of two identical-text rows → sibling survives (per-row id-keyed)") + + +def test_cross_owner_isolation( + owner_a: SigningKey, account_a: str | None, + owner_b: SigningKey, account_b: str | None, +) -> None: + """owner B cannot clear/list/forget owner A's memories. + + The privacy-floor assertion that matters most: every delete/list path is + owner-scoped by the auth-derived owner, so a second tenant sees nothing of + A's data and their delete attempts are clean no-ops. Requires a SECOND + delegate key (TEST_DELEGATE_KEY_2 / TEST_ACCOUNT_ID_2); skipped otherwise. + """ + ns = "e2e-xowner-test" + + # A seeds a memory. + r = make_signed_request( + "POST", "/api/remember", {"text": "A's private note.", "namespace": ns}, + owner_a, account_id=account_a, + ) + wait_for_remember_job(owner_a, account_a, r["job_id"]) + a_list = make_signed_request( + "POST", "/api/list", {"namespace": ns, "limit": 50}, owner_a, account_id=account_a + ) + assert a_list["returned"] >= 1, "A's own memory should be listed" + a_id = a_list["memories"][0]["id"] + print(f"[pass] owner A seeded + lists own memory (id={a_id[:8]}…)") + + # B sees nothing of A's namespace (B's own rows there = none). + b_list = make_signed_request( + "POST", "/api/list", {"namespace": ns, "limit": 50}, owner_b, account_id=account_b + ) + assert b_list["returned"] == 0, f"owner B must not see A's memories, got {b_list['returned']}" + print(f"[pass] owner B list of A's namespace → 0 (isolated)") + + # B's clear of the shared namespace string clears only B's rows (none). + b_clear = make_signed_request( + "POST", "/api/clear-namespace", {"namespace": ns}, owner_b, account_id=account_b + ) + assert b_clear["cleared"] == 0, f"owner B clear must touch nothing of A's, got {b_clear}" + + # B's forget of A's real id is a no-op (owner-scoped, not just id-scoped). + b_forget = make_signed_request( + "POST", "/api/memories/forget", {"id": a_id}, owner_b, account_id=account_b + ) + assert b_forget["forgotten"] == 0, ( + f"owner B forgetting A's id must be a no-op (IDOR guard), got {b_forget}" + ) + print(f"[pass] owner B clear + forget of A's data → no-ops (owner-scoped)") + + # A's memory survived all of B's attempts. + a_after = make_signed_request( + "POST", "/api/list", {"namespace": ns, "limit": 50}, owner_a, account_id=account_a + ) + assert a_id in {m["id"] for m in a_after["memories"]}, ( + "A's memory was wrongly affected by B — cross-owner isolation broken" + ) + print(f"[pass] owner A's memory intact after B's attempts (isolation holds)") + + # Cleanup A's namespace so reruns start clean. + make_signed_request( + "POST", "/api/clear-namespace", {"namespace": ns}, owner_a, account_id=account_a + ) + + MAX_REMEMBER_TEXT_BYTES = 1024 * 1024 # mirrors src/routes.rs constant # Largest plaintext we exercise in the e2e test. Smaller than the route # ceiling — bigger payloads work too (see scripts/bench-remember-sizes.ts) @@ -420,6 +755,10 @@ def main() -> int: ("size_64kb_summarized", test_remember_size_64kb_summarized), ("size_large_accepted", test_remember_size_large_accepted), ("size_over_limit_rejected", test_remember_size_over_limit_rejected), + ("clear_namespace_soft_delete", test_clear_namespace_soft_delete), + ("list_and_forget_by_id", test_list_and_forget_by_id), + ("list_pagination", test_list_pagination), + ("forget_is_per_row_not_per_blob", test_forget_is_per_row_not_per_blob), ) for name, fn in size_checks: try: @@ -427,9 +766,30 @@ def main() -> int: except (AssertionError, urllib.error.URLError, urllib.error.HTTPError) as e: failures.append(f"{name}: {e}") print(f"[FAIL] {name}: {e}") + + # cross-owner isolation — needs a SECOND delegate key. + owner_b, account_b = _load_delegate_key_2() + if owner_b: + try: + test_cross_owner_isolation(delegate_key, account_id, owner_b, account_b) + except (AssertionError, urllib.error.URLError, urllib.error.HTTPError) as e: + failures.append(f"cross_owner_isolation: {e}") + print(f"[FAIL] cross_owner_isolation: {e}") + else: + # Without a second key, cross-OWNER isolation + # is unverified by E2E. The owner-scoping is pinned at the SQL level + # by the Rust test `soft_delete_queries_are_owner_scoped`; this E2E + # is the live-stack confirmation. Set TEST_DELEGATE_KEY_2 + + # TEST_ACCOUNT_ID_2 to enable. + print("[skip] cross_owner_isolation (set TEST_DELEGATE_KEY_2 + TEST_ACCOUNT_ID_2 to enable)") else: print("[skip] remember_recall_happy_path (no TEST_DELEGATE_KEY)") print("[skip] size_*_test (no TEST_DELEGATE_KEY)") + print("[skip] clear_namespace_soft_delete (no TEST_DELEGATE_KEY)") + print("[skip] list_and_forget_by_id (no TEST_DELEGATE_KEY)") + print("[skip] list_pagination (no TEST_DELEGATE_KEY)") + print("[skip] forget_is_per_row_not_per_blob (no TEST_DELEGATE_KEY)") + print("[skip] cross_owner_isolation (no TEST_DELEGATE_KEY)") print() print("=" * 60) From 258009e198402a00db7412afa3e5c711c38d8029 Mon Sep 17 00:00:00 2001 From: hungtranphamminh Date: Wed, 17 Jun 2026 23:52:38 +0700 Subject: [PATCH 2/6] feat(sdk): clearNamespace/list/forget memory-deletion API [WALM-115] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose the relayer's soft-delete endpoints on the TS SDK: clearNamespace(ns), list(ns, { limit, cursor }) (metadata-only, cursor-paginated with has_more / next_cursor), and forget(id). Docstrings carry the honest semantics — un-recallable not erased, 0 = safe idempotent no-op, clearNamespace clears what exists at call time. Bump 0.0.7 -> 0.0.8. --- packages/sdk/package.json | 2 +- packages/sdk/src/index.ts | 4 ++ packages/sdk/src/memwal.ts | 114 +++++++++++++++++++++++++++++++++++++ packages/sdk/src/types.ts | 49 ++++++++++++++++ 4 files changed, 168 insertions(+), 1 deletion(-) diff --git a/packages/sdk/package.json b/packages/sdk/package.json index beb441c9..84cb642c 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@mysten-incubation/memwal", - "version": "0.0.7", + "version": "0.0.8", "description": "Walrus Memory — Privacy-first AI memory SDK with Ed25519 delegate key auth", "type": "module", "main": "./dist/index.js", diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 0dfdb6ca..fb9fc956 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -40,6 +40,10 @@ export type { AnalyzedFact, HealthResult, RestoreResult, + ClearNamespaceResult, + MemoryListItem, + ListResult, + ForgetResult, RememberBulkItem, RememberBulkOptions, RememberBulkAcceptedResult, diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index c9c9cadc..5d4c3fd3 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -45,6 +45,9 @@ import type { RecallManualOptions, RecallManualResult, RestoreResult, + ClearNamespaceResult, + ListResult, + ForgetResult, RememberBulkItem, RememberBulkOptions, RememberBulkResult, @@ -804,6 +807,117 @@ export class MemWal { }); } + /** + * Clear a namespace — soft-delete every memory in it so it stops surfacing + * in `recall()`. Use this to reset an agent's memory between iterations + * (replaces the namespace-rotation workaround). + * + * Soft-delete clears *retrievability*: the memories no longer come back + * from recall, but the underlying Walrus blobs are user-owned and persist + * until you delete them on-chain or their storage epoch expires. "Cleared" + * here means "un-recallable", not "cryptographically erased". + * + * Owner-scoped — only the caller's own memories are affected. The + * namespace is matched EXACTLY (case- and whitespace-sensitive). + * + * `cleared = 0` is a safe no-op, NOT a failure — it means the namespace was + * already empty/cleared (or never existed). Don't branch on `cleared > 0`; + * verify the end-state with `list()` if you need confirmation. + * + * **Ordering caveat:** this clears what exists at call time. A `remember()` + * still in flight (its upload not yet durable when this runs) can land + * *after* the clear and survive it. For a guaranteed-clean reset, wait for + * outstanding remember job ids to finish before clearing. + * + * @param namespace - Namespace to clear (exact match; no prefix/hierarchy) + * @returns ClearNamespaceResult with the count of memories cleared + * + * @example + * ```typescript + * const result = await memwal.clearNamespace("my-app"); + * console.log(`cleared=${result.cleared}`); + * ``` + */ + async clearNamespace(namespace: string): Promise { + return this.signedRequest("POST", "/api/clear-namespace", { + namespace, + }); + } + + /** + * List the memories stored in a namespace — metadata only (id, blob_id, + * created_at, importance), newest first. Decrypt-free, so it's cheap to + * call for auditing what's stored. Each item's `id` is the handle you pass + * to `forget(id)` to delete that specific memory. + * + * Note: this returns metadata, NOT the memory text — surfacing the text + * would require recall-grade decryption. Use `recall()` to read content. + * + * Owner-scoped; soft-deleted memories are omitted. Namespace matched + * EXACTLY (case- and whitespace-sensitive). + * + * **Pagination:** `result.returned` is THIS page's size (capped by `limit`, + * 1–500), NOT the namespace total. When `result.has_more` is true, pass + * `result.next_cursor` back as `opts.cursor` to fetch the next page. The + * cursor is stable under concurrent deletes (forgotten rows just drop out). + * + * @param namespace - Namespace to list (exact match) + * @param opts - `limit` (default 50, capped 1–500) and `cursor` (from a + * previous `next_cursor`); a bare `number` is also accepted as `limit`. + * @returns ListResult with the memories' metadata + ids + pagination fields + * + * @example + * ```typescript + * let cursor: string | undefined; + * do { + * const page = await memwal.list("my-app", { limit: 100, cursor }); + * for (const m of page.memories) console.log(m.id, m.created_at); + * cursor = page.has_more ? page.next_cursor : undefined; + * } while (cursor); + * ``` + */ + async list( + namespace: string, + opts: number | { limit?: number; cursor?: string } = {}, + ): Promise { + // Accept a bare number as `limit` for ergonomics; otherwise an options object. + const { limit = 50, cursor } = + typeof opts === "number" ? { limit: opts, cursor: undefined } : opts; + return this.signedRequest("POST", "/api/list", { + namespace, + limit, + ...(cursor ? { cursor } : {}), + }); + } + + /** + * Forget (soft-delete) a single memory by its `id` (from `list()`). The + * memory stops surfacing in `recall()`; an identical-text memory stored + * separately is unaffected (deletion is per-memory, not per-content). + * + * Like `clearNamespace`, this clears *retrievability* — the underlying + * Walrus blob is yours and persists until on-chain deletion / epoch expiry. + * + * Owner-scoped. `forgotten = 0` is a safe no-op, NOT a failure — the id + * never existed, isn't yours, or was already forgotten. Don't branch on + * `forgotten > 0` (a retry of a successful forget correctly returns 0); + * verify the end-state with `list()` if needed. + * + * @param id - The memory id from a `list()` entry + * @returns ForgetResult; `forgotten` is 1 on success, 0 if nothing matched + * + * @example + * ```typescript + * const { memories } = await memwal.list("my-app"); + * await memwal.forget(memories[0].id); + * ``` + */ + async forget(id: string): Promise { + return this.signedRequest("POST", "/api/memories/forget", { + id, + }); + } + /** * Check server health. The endpoint is public and does not require request signing. */ diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index 442ae1da..bc13a859 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -304,6 +304,55 @@ export interface RestoreResult { owner: string; } +/** + * Result of `clearNamespace()` — a soft-delete that stops every memory in the + * namespace from surfacing in `recall()`. The underlying Walrus blobs are + * user-owned and persist until on-chain deletion / storage-epoch expiry; this + * clears *retrievability*, not the blob itself. + */ +export interface ClearNamespaceResult { + /** Number of memories newly cleared (0 if the namespace was already clear). */ + cleared: number; + namespace: string; + owner: string; +} + +/** One memory's metadata as returned by `list()`. No decrypted text — listing + * is decrypt-free. `id` is the unique handle to pass to `forget(id)`. */ +export interface MemoryListItem { + /** Unique per-memory id — the handle for `forget(id)`. */ + id: string; + /** Walrus blob id (not unique; identical-text memories share one). */ + blob_id: string; + /** ISO-8601 insertion timestamp. */ + created_at: string; + /** Ranking importance weight for this memory (0.0–1.0). */ + importance: number; +} + +/** Result of `list()` — one page of live memories in a namespace, newest first, + * metadata only (use the `id`s with `forget()`). Soft-deleted memories omitted. */ +export interface ListResult { + memories: MemoryListItem[]; + /** Number of memories in THIS page (== `memories.length`, capped by `limit`). + * NOT the namespace's total — paginate via `has_more`/`next_cursor` to enumerate fully. */ + returned: number; + /** True if more live memories exist beyond this page; pass `next_cursor` back as `cursor`. */ + has_more: boolean; + /** Opaque cursor for the next page; absent when `has_more` is false. */ + next_cursor?: string; + namespace: string; + owner: string; +} + +/** Result of `forget(id)` — soft-delete a single memory by its `id`. */ +export interface ForgetResult { + /** 1 if soft-deleted, 0 if the id wasn't found / not yours / already gone. */ + forgotten: number; + id: string; + owner: string; +} + // ============================================================ // Full Client-Side Manual Flow — MemWalManual class // ============================================================ From 70b0694422a584ff02b51f21f32fabe83eda8efd Mon Sep 17 00:00:00 2001 From: hungtranphamminh Date: Wed, 17 Jun 2026 23:52:46 +0700 Subject: [PATCH 3/6] feat(sdk): python parity for clear_namespace/list/forget [WALM-115] Mirror the memory-deletion API on the Python SDK: clear_namespace, list (with cursor pagination), and forget on both the async MemWal and the sync MemWalSync, plus the ClearNamespaceResult / MemoryListItem / ListResult / ForgetResult dataclasses and exports. Same honest docstrings as the TS SDK. Bump 0.1.4 -> 0.1.5. --- packages/python-sdk-memwal/memwal/__init__.py | 8 ++ packages/python-sdk-memwal/memwal/client.py | 129 ++++++++++++++++++ packages/python-sdk-memwal/memwal/types.py | 54 ++++++++ packages/python-sdk-memwal/pyproject.toml | 2 +- 4 files changed, 192 insertions(+), 1 deletion(-) diff --git a/packages/python-sdk-memwal/memwal/__init__.py b/packages/python-sdk-memwal/memwal/__init__.py index 67c5485e..348fa263 100644 --- a/packages/python-sdk-memwal/memwal/__init__.py +++ b/packages/python-sdk-memwal/memwal/__init__.py @@ -61,6 +61,10 @@ RememberManualResult, RememberResult, RestoreResult, + ClearNamespaceResult, + MemoryListItem, + ListResult, + ForgetResult, ScoringWeights, ) from .utils import delegate_key_to_public_key, delegate_key_to_sui_address @@ -108,6 +112,10 @@ "AnalyzedFact", "HealthResult", "RestoreResult", + "ClearNamespaceResult", + "MemoryListItem", + "ListResult", + "ForgetResult", "ScoringWeights", "RememberManualOptions", "RememberManualResult", diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index 955d719e..c0e5c301 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -64,6 +64,10 @@ RememberManualResult, RememberResult, RestoreResult, + ClearNamespaceResult, + ListResult, + MemoryListItem, + ForgetResult, ) from .utils import ( build_seal_session_personal_message, @@ -816,6 +820,116 @@ async def restore(self, namespace: str, limit: int = 10) -> RestoreResult: owner=data["owner"], ) + async def clear_namespace(self, namespace: str) -> ClearNamespaceResult: + """Clear a namespace — soft-delete every memory in it so it stops + surfacing in :meth:`recall`. Use to reset an agent's memory between + iterations (replaces the namespace-rotation workaround). + + Soft-delete clears *retrievability*: the memories no longer come back + from recall, but the underlying Walrus blobs are user-owned and persist + until deleted on-chain or their storage epoch expires — "cleared" means + "un-recallable", not "cryptographically erased". Owner-scoped; namespace + matched EXACTLY (case- and whitespace-sensitive). + + ``cleared == 0`` is a safe no-op, NOT a failure — the namespace was + already empty/cleared (or never existed). Don't branch on ``cleared > + 0``; verify the end-state with :meth:`list` if you need confirmation. + + Ordering caveat: this clears what exists at call time. A :meth:`remember` + still in flight (its upload not yet durable when this runs) can land + *after* the clear and survive it. For a guaranteed-clean reset, wait for + outstanding remember job ids to finish before clearing. + + Args: + namespace: Namespace to clear. Exact match — no prefix/hierarchy. + + Returns: + :class:`ClearNamespaceResult` with the count of memories cleared. + """ + data = await self._signed_request("POST", "/api/clear-namespace", { + "namespace": namespace, + }) + return ClearNamespaceResult( + cleared=data["cleared"], + namespace=data["namespace"], + owner=data["owner"], + ) + + async def list( + self, namespace: str, limit: int = 50, cursor: Optional[str] = None + ) -> ListResult: + """List one page of memories stored in ``namespace`` — metadata only + (id, blob_id, created_at, importance), newest first. Decrypt-free, so + it's cheap to call for auditing what's stored. + + Each item's ``id`` is the handle to pass to :meth:`forget` to delete + that specific memory. This does NOT return the memory text — surfacing + it would require recall-grade decryption; use :meth:`recall` to read + content. Owner-scoped; soft-deleted memories are omitted. Namespace + matched EXACTLY (case- and whitespace-sensitive). + + Pagination: ``result.returned`` is THIS page's size (capped by + ``limit``), NOT the namespace total. When ``result.has_more`` is True, + pass ``result.next_cursor`` back as ``cursor`` to fetch the next page. + The cursor is stable under concurrent deletes (forgotten rows drop out). + + Args: + namespace: Namespace to list. Exact match — no prefix/hierarchy. + limit: Max memories per page (default 50, capped server-side at 500). + cursor: Opaque cursor from a previous response's ``next_cursor``; + omit for the first page. + + Returns: + :class:`ListResult` with this page's metadata + ids + pagination fields. + """ + payload: Dict[str, Any] = {"namespace": namespace, "limit": limit} + if cursor is not None: + payload["cursor"] = cursor + data = await self._signed_request("POST", "/api/list", payload) + return ListResult( + memories=[ + MemoryListItem( + id=m["id"], + blob_id=m["blob_id"], + created_at=m["created_at"], + importance=m["importance"], + ) + for m in data["memories"] + ], + returned=data["returned"], + has_more=data["has_more"], + next_cursor=data.get("next_cursor"), + namespace=data["namespace"], + owner=data["owner"], + ) + + async def forget(self, memory_id: str) -> ForgetResult: + """Forget (soft-delete) a single memory by its ``id`` (from + :meth:`list`). The memory stops surfacing in :meth:`recall`; an + identical-text memory stored separately is unaffected (deletion is + per-memory, not per-content). + + Like :meth:`clear_namespace`, this clears retrievability — the Walrus + blob is user-owned and persists. Owner-scoped. ``forgotten == 0`` is a + safe no-op, NOT a failure — the id never existed, isn't yours, or was + already forgotten. Don't branch on ``forgotten > 0`` (a retry of a + successful forget correctly returns 0); verify with :meth:`list` if needed. + + Args: + memory_id: The memory id from a :meth:`list` entry. + + Returns: + :class:`ForgetResult`; ``forgotten`` is 1 on success, 0 if nothing matched. + """ + data = await self._signed_request("POST", "/api/memories/forget", { + "id": memory_id, + }) + return ForgetResult( + forgotten=data["forgotten"], + id=data["id"], + owner=data["owner"], + ) + async def health(self) -> HealthResult: """Check server health. No authentication required. @@ -1393,6 +1507,21 @@ def restore(self, namespace: str, limit: int = 10) -> RestoreResult: (matches server + TypeScript SDK).""" return self._run(self._inner.restore(namespace, limit)) + def clear_namespace(self, namespace: str) -> ClearNamespaceResult: + """Synchronous version of :meth:`MemWal.clear_namespace`.""" + return self._run(self._inner.clear_namespace(namespace)) + + def list( + self, namespace: str, limit: int = 50, cursor: Optional[str] = None + ) -> ListResult: + """Synchronous version of :meth:`MemWal.list`. Default limit is 50; + pass ``cursor`` from a prior result's ``next_cursor`` to page.""" + return self._run(self._inner.list(namespace, limit, cursor)) + + def forget(self, memory_id: str) -> ForgetResult: + """Synchronous version of :meth:`MemWal.forget`.""" + return self._run(self._inner.forget(memory_id)) + def health(self) -> HealthResult: """Synchronous version of :meth:`MemWal.health`.""" return self._run(self._inner.health()) diff --git a/packages/python-sdk-memwal/memwal/types.py b/packages/python-sdk-memwal/memwal/types.py index 06cd9ec2..44796c83 100644 --- a/packages/python-sdk-memwal/memwal/types.py +++ b/packages/python-sdk-memwal/memwal/types.py @@ -203,6 +203,60 @@ class RestoreResult: owner: str +@dataclass +class ClearNamespaceResult: + """Result from clear_namespace(). + + Soft-delete: the cleared memories stop surfacing in recall, but the + underlying Walrus blobs are user-owned and persist until on-chain deletion + or storage-epoch expiry. "Cleared" means "un-recallable", not "erased". + """ + + #: Memories newly soft-deleted (0 if the namespace was already clear). + cleared: int + namespace: str + owner: str + + +@dataclass +class MemoryListItem: + """One memory's metadata from list(). No decrypted text — listing is + decrypt-free. ``id`` is the handle to pass to forget().""" + + id: str + #: Walrus blob id (not unique; identical-text memories share one). + blob_id: str + created_at: str + importance: float + + +@dataclass +class ListResult: + """Result from list() — one page of live memories in a namespace, newest + first, metadata only. Soft-deleted memories are omitted.""" + + memories: List[MemoryListItem] + #: Number of memories in THIS page (== len(memories), capped by limit). + #: NOT the namespace total — paginate via has_more/next_cursor to enumerate fully. + returned: int + #: True if more live memories exist beyond this page; pass next_cursor as `cursor`. + has_more: bool + #: Opaque cursor for the next page; None when has_more is False. + next_cursor: Optional[str] + namespace: str + owner: str + + +@dataclass +class ForgetResult: + """Result from forget() — soft-delete a single memory by id.""" + + #: 1 if soft-deleted, 0 if the id wasn't found / not the caller's / already gone. + forgotten: int + id: str + owner: str + + @dataclass class AskMemory: """A memory used to answer a question.""" diff --git a/packages/python-sdk-memwal/pyproject.toml b/packages/python-sdk-memwal/pyproject.toml index 0eb78ae6..c464c3e7 100644 --- a/packages/python-sdk-memwal/pyproject.toml +++ b/packages/python-sdk-memwal/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "memwal" -version = "0.1.4" +version = "0.1.5" description = "Python SDK for Walrus Memory — Privacy-first AI memory with Ed25519 signing" readme = "README.md" license = "MIT" From e8744c3acb372645a4285530f6897ed9804fc42a Mon Sep 17 00:00:00 2001 From: hungtranphamminh Date: Wed, 17 Jun 2026 23:52:55 +0700 Subject: [PATCH 4/6] feat(mcp): memwal_clear_namespace/list/forget sidecar tools [WALM-115] Add the three deletion tools to the MCP sidecar (scripts/mcp/tools) and register default-namespace injection for clear_namespace + list in the client bridge (forget is id-scoped, so excluded). These call the SDK's clearNamespace/list/forget. Dormant until the MCP go-live: the sidecar Docker build installs a pinned published SDK that predates these methods, so the tools resolve them only once the sidecar is built against the new SDK (separate go-live PR). No version bump here. --- packages/mcp/src/bridge.ts | 11 +++ .../scripts/mcp/tools/clear-namespace.ts | 43 ++++++++++ services/server/scripts/mcp/tools/forget.ts | 41 ++++++++++ services/server/scripts/mcp/tools/index.ts | 9 +++ services/server/scripts/mcp/tools/list.ts | 80 +++++++++++++++++++ 5 files changed, 184 insertions(+) create mode 100644 services/server/scripts/mcp/tools/clear-namespace.ts create mode 100644 services/server/scripts/mcp/tools/forget.ts create mode 100644 services/server/scripts/mcp/tools/list.ts diff --git a/packages/mcp/src/bridge.ts b/packages/mcp/src/bridge.ts index 47a6ed7b..831c9f75 100644 --- a/packages/mcp/src/bridge.ts +++ b/packages/mcp/src/bridge.ts @@ -44,6 +44,17 @@ const NAMESPACE_TOOLS = new Set([ "memwal_recall", "memwal_analyze", "memwal_restore", + // soft-delete: clears a namespace's memories from recall. Like restore, + // its upstream schema requires `namespace`; registered here so a + // configured default is injected when the agent omits one. (The tool + // DEFINITION itself lives in the MCP sidecar.) + "memwal_clear_namespace", + // list a namespace's memories (metadata + per-row id, for auditing / + // feeding memwal_forget). Namespace-scoped → default injection applies. + "memwal_list", + // NOTE: memwal_forget is intentionally NOT here — it takes a memory `id`, + // not a `namespace`, so there's nothing to inject. (Tool DEFINITION for + // both lives in the MCP sidecar.) ]); /** diff --git a/services/server/scripts/mcp/tools/clear-namespace.ts b/services/server/scripts/mcp/tools/clear-namespace.ts new file mode 100644 index 00000000..be868e82 --- /dev/null +++ b/services/server/scripts/mcp/tools/clear-namespace.ts @@ -0,0 +1,43 @@ +import { z } from "zod"; +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { MemWalSession } from "../auth.js"; +import { wrapTool } from "./util.js"; + +const CLEAR_NAMESPACE_INPUT = { + namespace: z + .string() + .min(1) + .describe("Namespace bucket to clear. Every memory in it stops surfacing in recall."), +} as const; + +/** + * memwal_clear_namespace — soft-delete every memory in a namespace so it stops + * being recalled. Use to reset an agent's memory between iterations (replaces + * the namespace-rotation workaround). + * + * Soft-delete clears *retrievability*: the memories no longer come back from + * recall, but the underlying Walrus blobs are user-owned and persist until + * deleted on-chain or their storage epoch expires — "cleared" means + * "un-recallable", not "cryptographically erased". Owner-scoped. + */ +export function registerClearNamespaceTool( + server: McpServer, + session: MemWalSession +): void { + server.tool( + "memwal_clear_namespace", + "Soft-delete every memory in a namespace so it stops surfacing in memwal_recall — use to reset a namespace between iterations instead of abandoning it. Owner-scoped (only the caller's own memories). Note: this clears retrievability, not the underlying Walrus blob (which is user-owned and persists until on-chain deletion / storage-epoch expiry) — \"cleared\" means \"un-recallable\", not \"erased\". cleared=0 is a safe no-op (already empty/cleared), not a failure. Clears what exists at call time — a memory still being saved when you clear may survive, so clear after writes have settled. Returns the count of memories cleared.", + CLEAR_NAMESPACE_INPUT, + wrapTool<{ namespace: string }>(async ({ namespace }) => { + const result = await session.memwal.clearNamespace(namespace); + return { + content: [ + { + type: "text", + text: `Cleared namespace "${result.namespace}": cleared=${result.cleared} (now un-recallable).`, + }, + ], + }; + }) + ); +} diff --git a/services/server/scripts/mcp/tools/forget.ts b/services/server/scripts/mcp/tools/forget.ts new file mode 100644 index 00000000..4e51c99c --- /dev/null +++ b/services/server/scripts/mcp/tools/forget.ts @@ -0,0 +1,41 @@ +import { z } from "zod"; +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { MemWalSession } from "../auth.js"; +import { wrapTool } from "./util.js"; + +const FORGET_INPUT = { + id: z + .string() + .min(1) + .describe("The memory id to forget (from a memwal_list entry)."), +} as const; + +/** + * memwal_forget — soft-delete a SINGLE memory by its id (from memwal_list). The + * memory stops surfacing in recall; an identical-text memory stored separately + * is unaffected (deletion is per-memory, not per-content). + * + * Like memwal_clear_namespace, this clears retrievability — the underlying + * Walrus blob is user-owned and persists. Owner-scoped: forgetting an id that + * isn't the caller's is a no-op (forgotten=0). + */ +export function registerForgetTool( + server: McpServer, + session: MemWalSession +): void { + server.tool( + "memwal_forget", + "Soft-delete a single memory by its id (obtained from memwal_list) so it stops surfacing in memwal_recall. Deletion is per-memory, not per-content — an identical-text memory stored separately is unaffected. Owner-scoped. forgotten=0 is a safe no-op (id never existed, isn't yours, or already gone), not a failure — don't treat it as an error. Clears retrievability, not the underlying Walrus blob.", + FORGET_INPUT, + wrapTool<{ id: string }>(async ({ id }) => { + const result = await session.memwal.forget(id); + const text = + result.forgotten === 1 + ? `Forgot memory id=${result.id} (now un-recallable).` + : `No memory forgotten for id=${result.id} (not found, not yours, or already forgotten).`; + return { + content: [{ type: "text", text }], + }; + }) + ); +} diff --git a/services/server/scripts/mcp/tools/index.ts b/services/server/scripts/mcp/tools/index.ts index 6c712177..eab546b6 100644 --- a/services/server/scripts/mcp/tools/index.ts +++ b/services/server/scripts/mcp/tools/index.ts @@ -6,6 +6,9 @@ import { registerRememberBulkTool } from "./remember-bulk.js"; import { registerRecallTool } from "./recall.js"; import { registerAnalyzeTool } from "./analyze.js"; import { registerRestoreTool } from "./restore.js"; +import { registerClearNamespaceTool } from "./clear-namespace.js"; +import { registerListTool } from "./list.js"; +import { registerForgetTool } from "./forget.js"; import { registerHealthTool } from "./health.js"; /** @@ -20,6 +23,9 @@ export function registerTools(server: McpServer, session: MemWalSession): void { registerRecallTool(server, session); registerAnalyzeTool(server, session); registerRestoreTool(server, session); + registerClearNamespaceTool(server, session); + registerListTool(server, session); + registerForgetTool(server, session); registerHealthTool(server, session); } @@ -29,5 +35,8 @@ export { registerRecallTool, registerAnalyzeTool, registerRestoreTool, + registerClearNamespaceTool, + registerListTool, + registerForgetTool, registerHealthTool, }; diff --git a/services/server/scripts/mcp/tools/list.ts b/services/server/scripts/mcp/tools/list.ts new file mode 100644 index 00000000..dec9e784 --- /dev/null +++ b/services/server/scripts/mcp/tools/list.ts @@ -0,0 +1,80 @@ +import { z } from "zod"; +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { MemWalSession } from "../auth.js"; +import { wrapTool } from "./util.js"; + +const LIST_INPUT = { + namespace: z + .string() + .min(1) + .describe("Namespace bucket to enumerate."), + limit: z + .number() + .int() + .min(1) + .max(500) + .default(50) + .describe("Max memories per page (1-500)."), + cursor: z + .string() + .optional() + .describe( + "Pagination cursor from a previous result's next_cursor. Omit for the first page." + ), +} as const; + +/** + * memwal_list — enumerate the memories stored in a namespace, newest first. + * + * Returns METADATA only (id + creation time), not the memory text — listing is + * decrypt-free, so it's cheap to call for auditing what's stored. Each entry's + * `id` is the handle to pass to memwal_forget to delete that specific memory. + * Use memwal_recall to read content. Owner-scoped; cleared/forgotten memories + * are omitted. + * + * Paginated: one page per call. When the result says more remain, call again + * with the reported cursor to continue. + */ +export function registerListTool( + server: McpServer, + session: MemWalSession +): void { + server.tool( + "memwal_list", + "Enumerate the memories stored in a namespace (metadata only: id + created-at, newest first) to audit what's stored — does NOT return memory text (use memwal_recall for content). Each entry's id is the handle for memwal_forget to delete that specific memory. Paginated: returns one page; if more remain, call again passing the reported cursor. Owner-scoped; already-cleared memories are omitted.", + LIST_INPUT, + wrapTool<{ namespace: string; limit: number; cursor?: string }>( + async ({ namespace, limit, cursor }) => { + const result = await session.memwal.list(namespace, { limit, cursor }); + if (result.memories.length === 0) { + return { + content: [ + { + type: "text", + text: `Namespace "${result.namespace}" has no stored memories.`, + }, + ], + }; + } + const lines = result.memories.map( + (m, i) => `${i + 1}. id=${m.id} created=${m.created_at}` + ); + const more = result.has_more + ? `\n\nMore memories remain — call memwal_list again with cursor="${result.next_cursor}".` + : ""; + return { + content: [ + { + type: "text", + text: + `${result.returned} memory(ies) in "${result.namespace}" this page ` + + `(metadata only — use the id with memwal_forget):\n` + + lines.join("\n") + + more, + }, + ], + }; + } + ) + ); +} From 145038d613f6bc98489d865c24d993e99a3acca1 Mon Sep 17 00:00:00 2001 From: hungtranphamminh Date: Wed, 17 Jun 2026 23:53:01 +0700 Subject: [PATCH 5/6] chore(sdk): add changeset for the memory-deletion API [WALM-115] --- .changeset/walm-115-sdk-delete-api.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .changeset/walm-115-sdk-delete-api.md diff --git a/.changeset/walm-115-sdk-delete-api.md b/.changeset/walm-115-sdk-delete-api.md new file mode 100644 index 00000000..03b77fb4 --- /dev/null +++ b/.changeset/walm-115-sdk-delete-api.md @@ -0,0 +1,11 @@ +--- +"@mysten-incubation/memwal": patch +--- + +Add memory deletion + listing to the SDK: `clearNamespace(namespace)`, `forget(id)`, and `list(namespace, { limit, cursor })`. + +- `clearNamespace` soft-deletes every memory in a namespace so it stops surfacing in `recall()` — the reset primitive for iteration/dev loops (replaces namespace rotation). +- `forget` soft-deletes a single memory by the `id` returned from `list()` (per-memory; identical-text memories stored separately are unaffected). +- `list` enumerates a namespace as metadata only (id, blob_id, created_at, importance — no decrypted text), cursor-paginated via `has_more` / `next_cursor`. + +Soft-delete clears *retrievability*, not the underlying Walrus blob (user-owned, persists until on-chain deletion / storage-epoch expiry). All owner-scoped. From 7b15da95a6a4bf1a6c906e2cdb88035c4db25e30 Mon Sep 17 00:00:00 2001 From: hungtranphamminh Date: Thu, 18 Jun 2026 00:10:31 +0700 Subject: [PATCH 6/6] docs: document clearNamespace/list/forget across SDK + relayer refs [WALM-115] Add the memory-deletion API to the public docs: the 3 endpoints in the relayer api-reference, the TS + Python SDK method references, the SDK overview method lists, and a Quick Start snippet in the SDK README. Carries the honest semantics (soft-delete = un-recallable not erased; 0/cleared/forgotten = safe no-op; list is metadata-only + cursor-paginated). The hard /api/forget stays undocumented (harness-only), matching existing convention. --- docs/python-sdk/api-reference.md | 39 +++++++++++++++ docs/relayer/api-reference.md | 85 ++++++++++++++++++++++++++++++++ docs/sdk/api-reference.md | 48 ++++++++++++++++++ docs/sdk/overview.md | 4 +- packages/sdk/README.md | 5 ++ 5 files changed, 179 insertions(+), 2 deletions(-) diff --git a/docs/python-sdk/api-reference.md b/docs/python-sdk/api-reference.md index 8fcf5558..f930ae16 100644 --- a/docs/python-sdk/api-reference.md +++ b/docs/python-sdk/api-reference.md @@ -149,6 +149,45 @@ Rebuild missing indexed entries for one namespace from Walrus. Incremental. RestoreResult(restored: int, skipped: int, total: int, namespace: str, owner: str) ``` +### `clear_namespace(namespace) -> ClearNamespaceResult` + +Soft-delete every memory in a namespace so it stops surfacing in `recall` — the reset primitive. Clears retrievability; the Walrus blob is user-owned and persists (un-recallable, not erased). Owner-scoped; namespace matched exactly. + +- `cleared` is the count newly soft-deleted; `0` is a safe no-op (already empty/cleared) + +```python +ClearNamespaceResult(cleared: int, namespace: str, owner: str) +``` + +### `list(namespace, limit=50, cursor=None) -> ListResult` + +Enumerate the memories in a namespace — metadata only (no decrypted text), newest first, decrypt-free. Each item's `id` is the handle for `forget`. Use `recall` to read content. + +- `limit` defaults to `50` (capped 1–500); omit `cursor` for the first page +- Paginate: when `has_more` is `True`, pass `next_cursor` back as `cursor` +- `returned` is this page's size, not the namespace total + +```python +ListResult( + memories: list[MemoryListItem], # MemoryListItem(id, blob_id, created_at, importance) + returned: int, + has_more: bool, + next_cursor: Optional[str], + namespace: str, + owner: str, +) +``` + +### `forget(memory_id) -> ForgetResult` + +Soft-delete a single memory by its `id` (from `list`). An identical-text memory stored separately is unaffected (per-memory). Owner-scoped. + +- `forgotten` is `1` on success; `0` is a safe no-op (not found / not yours / already gone) + +```python +ForgetResult(forgotten: int, id: str, owner: str) +``` + ### `health() -> HealthResult` Check relayer health. No authentication. Raises `MemWalError` on non-200. diff --git a/docs/relayer/api-reference.md b/docs/relayer/api-reference.md index 8603cc6a..5041d06a 100644 --- a/docs/relayer/api-reference.md +++ b/docs/relayer/api-reference.md @@ -351,3 +351,88 @@ Rebuild missing vector entries for one namespace. Queries onchain blobs by owner "owner": "0x..." } ``` + +### `POST /api/clear-namespace` + +Soft-delete every memory in one namespace. The memories immediately stop appearing in `recall`; the underlying Walrus blobs are user-owned and persist (this clears retrievability, not the blob). Owner-scoped. + +**Request:** + +```json +{ + "namespace": "demo" +} +``` + +**Response:** + +```json +{ + "cleared": 12, + "namespace": "demo", + "owner": "0x..." +} +``` + +`cleared` is the number of memories newly soft-deleted; `0` is a safe no-op (already empty/cleared). + +### `POST /api/list` + +Enumerate the live memories in one namespace, newest first. Metadata only — no decrypted text — so it is cheap to call for auditing. Each item's `id` is the handle for `/api/memories/forget`. Owner-scoped; soft-deleted memories are omitted. + +**Request:** + +```json +{ + "namespace": "demo", + "limit": 50, + "cursor": "" +} +``` + +`limit` defaults to `50` (clamped to 1–500). Omit `cursor` for the first page. + +**Response:** + +```json +{ + "memories": [ + { + "id": "uuid", + "blob_id": "walrus-blob-id", + "created_at": "2026-06-18T12:00:00+00:00", + "importance": 0.5 + } + ], + "returned": 1, + "has_more": false, + "namespace": "demo", + "owner": "0x..." +} +``` + +`returned` is this page's size (not the namespace total). When `has_more` is `true`, the response also includes a `next_cursor`; pass it back as `cursor` to fetch the next page. + +### `POST /api/memories/forget` + +Soft-delete a single memory by its `id` (from `/api/list`). The memory stops appearing in `recall`; an identical-text memory stored separately is unaffected (deletion is per-memory). Owner-scoped. + +**Request:** + +```json +{ + "id": "uuid" +} +``` + +**Response:** + +```json +{ + "forgotten": 1, + "id": "uuid", + "owner": "0x..." +} +``` + +`forgotten` is `1` on success, `0` if the id was not found, is not the caller's, or was already forgotten (all safe no-ops). diff --git a/docs/sdk/api-reference.md b/docs/sdk/api-reference.md index 0e575b67..d9d757ff 100644 --- a/docs/sdk/api-reference.md +++ b/docs/sdk/api-reference.md @@ -141,6 +141,54 @@ Rebuild missing indexed entries for one namespace from Walrus. Incremental — o } ``` +### `clearNamespace(namespace): Promise` + +Soft-delete every memory in a namespace so it stops surfacing in `recall()` — the reset primitive for iteration/dev loops. Soft-delete clears *retrievability*; the Walrus blob is user-owned and persists (un-recallable, not erased). Owner-scoped; namespace matched exactly. + +**Returns:** + +```ts +{ + cleared: number; // Memories newly cleared (0 = safe no-op, already empty/cleared) + namespace: string; + owner: string; +} +``` + +### `list(namespace, opts?): Promise` + +Enumerate the memories in a namespace — metadata only (id, blob_id, created_at, importance), newest first. Decrypt-free, so cheap to call for auditing. Each item's `id` is the handle for `forget()`. Use `recall()` to read content. + +- `opts` is `{ limit?, cursor? }` (a bare `number` is accepted as `limit`); `limit` defaults to `50`, capped 1–500. +- Paginate: when `has_more` is `true`, pass `next_cursor` back as `opts.cursor`. + +**Returns:** + +```ts +{ + memories: { id: string; blob_id: string; created_at: string; importance: number }[]; + returned: number; // This page's size (NOT the namespace total) + has_more: boolean; + next_cursor?: string; // Pass back as opts.cursor for the next page + namespace: string; + owner: string; +} +``` + +### `forget(id): Promise` + +Soft-delete a single memory by its `id` (from `list()`). An identical-text memory stored separately is unaffected (deletion is per-memory). Like `clearNamespace`, clears retrievability, not the blob. Owner-scoped. + +**Returns:** + +```ts +{ + forgotten: number; // 1 on success; 0 = safe no-op (not found / not yours / already gone) + id: string; + owner: string; +} +``` + ### `health(): Promise` Check relayer health. Does not require authentication. diff --git a/docs/sdk/overview.md b/docs/sdk/overview.md index 7c9bbf93..2598263e 100644 --- a/docs/sdk/overview.md +++ b/docs/sdk/overview.md @@ -12,7 +12,7 @@ Use this first. - relayer-backed - best path for most teams -- main methods: `remember`, `recall`, `analyze`, `restore`, `health` +- main methods: `remember`, `recall`, `analyze`, `restore`, `clearNamespace`, `list`, `forget`, `health` ```ts import { MemWal } from "@mysten-incubation/memwal"; @@ -67,7 +67,7 @@ memwal = MemWal.create( ) ``` -Main methods: `remember`, `recall`, `analyze`, `ask`, `restore`, `health` +Main methods: `remember`, `recall`, `analyze`, `ask`, `restore`, `clear_namespace`, `list`, `forget`, `health` Middleware: `with_memwal_langchain`, `with_memwal_openai` diff --git a/packages/sdk/README.md b/packages/sdk/README.md index f0a2b258..645afda0 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -55,6 +55,11 @@ const memories = await memwal.recall({ maxDistance: 0.7, }); await memwal.restore("demo"); + +// Delete: list a namespace (metadata only), forget one memory, or clear it all. +const { memories: stored } = await memwal.list("demo", { limit: 50 }); +if (stored.length) await memwal.forget(stored[0].id); +await memwal.clearNamespace("demo"); // reset the namespace (soft-delete; un-recallable) ``` If you are self-hosting the relayer and do not have an account ID yet, see [Self-Hosting](../../docs/relayer/self-hosting.md) for the account creation and delegate key setup flow.