diff --git a/app/mcp_server.py b/app/mcp_server.py index 644db7c..c6b8802 100644 --- a/app/mcp_server.py +++ b/app/mcp_server.py @@ -57,9 +57,24 @@ def search_memories( return rerank_by_recency(results, recency_weight) @mcp.tool - def list_memories() -> dict: - """List all stored memories for the user (shared across all agents).""" - return memory.get_all(filters={"user_id": default_user}) + def list_memories(limit: int = 50, offset: int = 0) -> dict: + """List stored memories for the user (shared across all agents), paged. + + Returns at most `limit` memories (1-100, default 50) starting at + `offset`. The response's `pagination.has_more` tells you whether + another page exists — call again with `offset` advanced by `limit` + to continue. Prefer `search_memories` over paging through everything + when you're looking for something specific. + """ + if not 1 <= limit <= 100: + raise ValueError(f"limit must be between 1 and 100, got {limit}") + if not 0 <= offset <= memory_mod.MAX_LIST_OFFSET: + raise ValueError( + f"offset must be between 0 and {memory_mod.MAX_LIST_OFFSET}, got {offset}" + ) + return memory_mod.list_paginated( + filters={"user_id": default_user}, limit=limit, offset=offset + ) @mcp.tool def get_memory(memory_id: str) -> dict: diff --git a/app/memory.py b/app/memory.py index 05697a2..f2a391a 100644 --- a/app/memory.py +++ b/app/memory.py @@ -191,6 +191,34 @@ def keyword_search( return {"results": [_point_to_result(p) for p in matches[:limit]]} +# Upper bound on the list-pagination offset. Offset paging is emulated by +# over-fetching offset+limit+1 items and slicing (mem0's get_all has no native +# offset), so the bound caps the per-request fetch size. 10k is far beyond a +# single-user store's realistic page depth. +MAX_LIST_OFFSET = 10_000 + + +def list_paginated(*, filters: dict, limit: int, offset: int = 0) -> dict: + """List memories one page at a time, preserving mem0's result shaping. + + Fetches `offset + limit + 1` items via mem0's get_all (which has no offset + parameter) and slices: the extra item only signals `has_more`, so a page is + never silently truncated at the store's default cap. Ordering comes from + the vector store's scroll (stable by point ID, not chronological) — pages + are consistent across calls as long as the store isn't being written + concurrently. + """ + memory = get_memory() + raw = memory.get_all(filters=filters, top_k=offset + limit + 1) + items = raw.get("results") if isinstance(raw, dict) else raw + items = list(items or []) + has_more = len(items) > offset + limit + return { + "results": items[offset : offset + limit], + "pagination": {"limit": limit, "offset": offset, "has_more": has_more}, + } + + def _result_expiry(item) -> datetime | None: """Parse an `expires_at` from a result item (top-level or nested metadata).""" if not isinstance(item, dict): diff --git a/app/rest.py b/app/rest.py index 013ee1c..c7aad95 100644 --- a/app/rest.py +++ b/app/rest.py @@ -114,13 +114,15 @@ def list_memories( review_status: str | None = None, exclude_expired: bool = False, limit: int = Query(default=50, ge=1, le=100), + offset: int = Query(default=0, ge=0, le=memory_mod.MAX_LIST_OFFSET), ) -> dict: - memory = memory_mod.get_memory() filters = { **_scope_kwargs(user_id, agent_id, run_id), **_provenance_filters(source, confidence, review_status), } - results = memory.get_all(filters=filters, top_k=limit) + results = memory_mod.list_paginated(filters=filters, limit=limit, offset=offset) + # Expiry filtering happens after pagination, so a page may carry fewer than + # `limit` items; `pagination.has_more` still reflects the unfiltered store. return memory_mod.drop_expired(results) if exclude_expired else results diff --git a/docs/DEVELOPER_GUIDE.md b/docs/DEVELOPER_GUIDE.md index ddb96b0..a66d470 100644 --- a/docs/DEVELOPER_GUIDE.md +++ b/docs/DEVELOPER_GUIDE.md @@ -66,9 +66,13 @@ app/ keyword_search() is the substring-match fallback behind search mode="keyword": it scans the user's memories via vector_store.list() and matches the query as a case-insensitive substring of the `data` payload (fail-open). drop_expired() - removes results whose provenance `expires_at` is past. The most tweak-prone file. + removes results whose provenance `expires_at` is past. list_paginated() + implements offset paging for list reads: mem0's get_all has no offset, so it + over-fetches offset+limit+1 (capped by MAX_LIST_OFFSET) and slices, using the + extra item as the has_more signal. The most tweak-prone file. mcp_server.py build_mcp(): the six MCP tools, each thinly wrapping a mem0 op with - user_id defaulted to MEM0_DEFAULT_USER_ID. + user_id defaulted to MEM0_DEFAULT_USER_ID. list_memories pages (default 50, + max 100 per call) so the whole store is never returned in one response. rest.py REST router under /api/v1 (mounted with prefix in main.py). Pydantic request models, _scope_kwargs() for user/agent/run scoping, _provenance_filters() for the source/confidence/review_status metadata convention, check_qdrant() helper. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 2d37127..f567479 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -598,15 +598,26 @@ it's a literal-match fallback, not a replacement for semantic retrieval. ### List memories — `GET /api/v1/memories` -Query params: `agent_id`, `run_id`, `user_id`, `limit` (1–100, default 50), plus the -provenance/review filters `source`, `confidence`, `review_status` (exact match), and -`exclude_expired` (drop memories whose `expires_at` is in the past). See +Query params: `agent_id`, `run_id`, `user_id`, `limit` (1–100, default 50), `offset` +(0–10000, default 0), plus the provenance/review filters `source`, `confidence`, +`review_status` (exact match), and `exclude_expired` (drop memories whose +`expires_at` is in the past). See [Provenance and review metadata](#provenance-and-review-metadata-convention). +The response carries a `pagination` object — `{"limit": …, "offset": …, "has_more": …}` — +so the full store can be enumerated by advancing `offset` by `limit` while +`has_more` is `true`. Ordering is stable (by internal ID) but **not** chronological. +With `exclude_expired=true`, expired items are dropped *after* the page is cut, so a +page may contain fewer than `limit` items while `has_more` is still `true`. + ```bash curl https://mem0.your-domain.com/api/v1/memories?limit=20 \ -H "Authorization: Bearer $MEM0_API_KEY" +# Next page: +curl "https://mem0.your-domain.com/api/v1/memories?limit=20&offset=20" \ + -H "Authorization: Bearer $MEM0_API_KEY" + # Only approved, non-expired memories imported from ChatGPT: curl "https://mem0.your-domain.com/api/v1/memories?source=import:chatgpt&review_status=approved&exclude_expired=true" \ -H "Authorization: Bearer $MEM0_API_KEY" diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 4308a8d..ef0e740 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -108,6 +108,29 @@ async def test_list_memories_tool(mcp, mem): await client.call_tool("list_memories", {}) _, kwargs = mem.get_all.call_args assert kwargs["filters"] == {"user_id": "default-user"} + # Default page of 50 plus the extra has_more sentinel — never the whole store. + assert kwargs["top_k"] == 51 + + +async def test_list_memories_tool_pagination(mcp, mem): + mem.get_all.return_value = { + "results": [{"id": f"m{i}", "memory": f"fact {i}"} for i in range(4)] + } + async with Client(mcp) as client: + result = await client.call_tool("list_memories", {"limit": 2, "offset": 1}) + body = result.data + assert [i["id"] for i in body["results"]] == ["m1", "m2"] + assert body["pagination"] == {"limit": 2, "offset": 1, "has_more": True} + _, kwargs = mem.get_all.call_args + assert kwargs["top_k"] == 4 # offset + limit + 1 + + +async def test_list_memories_tool_rejects_bad_paging(mcp, mem): + async with Client(mcp) as client: + for args in ({"limit": 0}, {"limit": 101}, {"offset": -1}, {"offset": 10001}): + with pytest.raises(ToolError): + await client.call_tool("list_memories", args) + mem.get_all.assert_not_called() async def test_delete_memory_tool(mcp, mem): diff --git a/tests/test_memory.py b/tests/test_memory.py index 73525c7..4d36b81 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -10,6 +10,7 @@ content_fingerprint, drop_expired, keyword_search, + list_paginated, ) EXPIRY_NOW = datetime(2026, 6, 7, tzinfo=UTC) @@ -275,3 +276,29 @@ def test_keyword_search_fails_open(monkeypatch): fake.vector_store.list.side_effect = RuntimeError("qdrant down") monkeypatch.setattr(m, "get_memory", lambda: fake) assert keyword_search("x", user_id="ian") == {"results": []} + + +def test_list_paginated_shapes_and_slices(mem): + mem.get_all.return_value = { + "results": [{"id": f"m{i}"} for i in range(7)] + } + out = list_paginated(filters={"user_id": "u"}, limit=3, offset=3) + assert [i["id"] for i in out["results"]] == ["m3", "m4", "m5"] + assert out["pagination"] == {"limit": 3, "offset": 3, "has_more": True} + _, kwargs = mem.get_all.call_args + assert kwargs == {"filters": {"user_id": "u"}, "top_k": 7} + + +def test_list_paginated_tolerates_bare_list_return(mem): + # Some mem0 versions/stores return a bare list instead of {"results": [...]}. + mem.get_all.return_value = [{"id": "a"}, {"id": "b"}] + out = list_paginated(filters={"user_id": "u"}, limit=10, offset=0) + assert [i["id"] for i in out["results"]] == ["a", "b"] + assert out["pagination"]["has_more"] is False + + +def test_list_paginated_tolerates_none_results(mem): + mem.get_all.return_value = {"results": None} + out = list_paginated(filters={"user_id": "u"}, limit=10, offset=0) + assert out["results"] == [] + assert out["pagination"]["has_more"] is False diff --git a/tests/test_rest.py b/tests/test_rest.py index 8309b8b..6314213 100644 --- a/tests/test_rest.py +++ b/tests/test_rest.py @@ -83,7 +83,8 @@ def test_list(app_instance, mem, auth_header): _, kwargs = mem.get_all.call_args assert kwargs["filters"]["agent_id"] == "n8n" assert kwargs["filters"]["user_id"] == "default-user" - assert kwargs["top_k"] == 50 # default list limit must reach mem0 as top_k + # Default limit 50, offset 0: one extra item is fetched to signal has_more. + assert kwargs["top_k"] == 51 def test_search_default_does_not_rerank(app_instance, mem, auth_header): @@ -244,6 +245,64 @@ def test_list_limit_out_of_range_rejected(app_instance, mem, auth_header): assert c.get("/api/v1/memories?limit=1000", headers=auth_header).status_code == 422 +def _numbered_results(n): + return {"results": [{"id": f"m{i}", "memory": f"fact {i}"} for i in range(n)]} + + +def test_list_paginates_and_reports_has_more(app_instance, mem, auth_header): + # Store holds 5 items; ask for the middle page of 2. + mem.get_all.return_value = _numbered_results(5) + c = _client(app_instance) + resp = c.get("/api/v1/memories?limit=2&offset=2", headers=auth_header) + assert resp.status_code == 200 + body = resp.json() + assert [i["id"] for i in body["results"]] == ["m2", "m3"] + assert body["pagination"] == {"limit": 2, "offset": 2, "has_more": True} + _, kwargs = mem.get_all.call_args + assert kwargs["top_k"] == 5 # offset + limit + 1 + + +def test_list_last_page_has_more_false(app_instance, mem, auth_header): + mem.get_all.return_value = _numbered_results(5) + c = _client(app_instance) + body = c.get("/api/v1/memories?limit=10&offset=0", headers=auth_header).json() + assert len(body["results"]) == 5 + assert body["pagination"]["has_more"] is False + + +def test_list_offset_past_end_is_empty(app_instance, mem, auth_header): + mem.get_all.return_value = _numbered_results(3) + c = _client(app_instance) + body = c.get("/api/v1/memories?limit=10&offset=5", headers=auth_header).json() + assert body["results"] == [] + assert body["pagination"]["has_more"] is False + + +def test_list_offset_out_of_range_rejected(app_instance, mem, auth_header): + c = _client(app_instance) + assert c.get("/api/v1/memories?offset=-1", headers=auth_header).status_code == 422 + assert ( + c.get("/api/v1/memories?offset=10001", headers=auth_header).status_code == 422 + ) + + +def test_list_exclude_expired_filters_page_not_has_more(app_instance, mem, auth_header): + mem.get_all.return_value = { + "results": [ + {"id": "stale", "metadata": {"expires_at": "2000-01-01T00:00:00+00:00"}}, + {"id": "fresh", "metadata": {}}, + {"id": "extra", "metadata": {}}, + ] + } + c = _client(app_instance) + body = c.get( + "/api/v1/memories?exclude_expired=true&limit=2&offset=0", headers=auth_header + ).json() + # Expiry filtering happens after slicing: the page shrinks, has_more doesn't. + assert [i["id"] for i in body["results"]] == ["fresh"] + assert body["pagination"]["has_more"] is True + + def test_delete(app_instance, mem, auth_header): c = _client(app_instance) resp = c.delete("/api/v1/memories/abc", headers=auth_header)