diff --git a/README.md b/README.md index 16d39b0..d14261e 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,44 @@ python -m benchmarks.beam.run \ --chat-sizes 100K --conversations 0-9 ``` +### Option C: GoodMemory (local, no Docker) + +Runs against [GoodMemory](https://github.com/hjqcan/GoodMemory)'s packaged +HTTP bridge (local-first; requires [Bun](https://bun.sh)). Ingestion is +deterministic (every turn is written verbatim, no LLM extractor needed), and +session timestamps are preserved as UTC observation prefixes. However, +**representative recall needs the bridge started with an embedding endpoint, +the `recommended` retrieval preset (GoodMemory's semantic candidate union), +and in-memory storage** (its pure-JS vector index needs no native libraries). +Without these the bridge falls back to a lexical-only floor and under-reports. + +```bash +npm install -g goodmemory + +# Bridge with the semantic recall stack (any OpenAI-compatible embedding +# endpoint works — OpenAI, OpenRouter, Azure, a local Ollama, …): +GOODMEMORY_HTTP_BRIDGE_TOKEN=your-token \ +GOODMEMORY_STORAGE_PROVIDER=memory \ +GOODMEMORY_HTTP_BRIDGE_RETRIEVAL_PRESET=recommended \ +GOODMEMORY_EMBEDDING_PROVIDER=openai \ +GOODMEMORY_EMBEDDING_MODEL=text-embedding-3-small \ +GOODMEMORY_EMBEDDING_API_KEY=$OPENAI_API_KEY \ + goodmemory-http-bridge # serves http://localhost:8739 + +# Run a benchmark against it +GOODMEMORY_HTTP_BRIDGE_TOKEN=your-token python -m benchmarks.locomo.run \ + --project-name my-goodmemory-test \ + --backend goodmemory +``` + +The client requests `hybrid` recall by default (the bridge coerces any other +strategy to the rules-only floor); override with `GOODMEMORY_RECALL_STRATEGY`. +Point at a non-default bridge with `--goodmemory-host` or `GOODMEMORY_HOST`. +Each run should use a fresh `--project-name`: GoodMemory isolates state by +scope (the user id embeds the project name) and the bridge intentionally has +no bulk delete endpoint. In-memory storage is fresh per bridge process, which +suits a benchmark run. + ### Option B: Mem0 OSS (Self-Hosted) Requires Docker and Docker Compose. This starts a local Mem0 server backed by Qdrant. diff --git a/benchmarks/beam/run.py b/benchmarks/beam/run.py index b277261..3bb5061 100644 --- a/benchmarks/beam/run.py +++ b/benchmarks/beam/run.py @@ -47,6 +47,7 @@ from benchmarks.common.llm_client import LLMClient from benchmarks.common.mem0_client import Mem0Client, format_search_results +from benchmarks.common.goodmemory_client import GoodMemoryClient from benchmarks.common.metrics import compute_kendall_tau_b, compute_overall_metrics from benchmarks.common.schema import ( CutoffResult, @@ -955,10 +956,12 @@ def parse_args() -> argparse.Namespace: help="Comma-separated question types to evaluate (default: all)", ) parser.add_argument("--rpm", type=int, default=200, help="Requests per minute for LLM") - parser.add_argument("--backend", default="oss", choices=["oss", "cloud"], - help="Mem0 backend: 'oss' for self-hosted server (default), 'cloud' for api.mem0.ai") + parser.add_argument("--backend", default="oss", choices=["oss", "cloud", "goodmemory"], + help="Mem0 backend: 'oss' for self-hosted server (default), 'cloud' for api.mem0.ai, 'goodmemory' for a GoodMemory HTTP bridge") parser.add_argument("--mem0-host", default=None, help="Mem0 server URL") + parser.add_argument("--goodmemory-host", default=None, + help="GoodMemory HTTP bridge URL (default: GOODMEMORY_HOST env or http://localhost:8739)") parser.add_argument("--mem0-api-key", default=None, help="Mem0 API key (cloud mode only)") return parser.parse_args() @@ -1021,12 +1024,18 @@ async def async_main() -> None: # Init clients backend = os.getenv("MEM0_BACKEND", args.backend) - mem0 = Mem0Client( - mode=backend, - host=args.mem0_host, - api_key=args.mem0_api_key if backend == "cloud" else None, - rpm=args.rpm, - ) + if backend == "goodmemory": + mem0 = GoodMemoryClient( + host=args.goodmemory_host, + rpm=args.rpm, + ) + else: + mem0 = Mem0Client( + mode=backend, + host=args.mem0_host, + api_key=args.mem0_api_key if backend == "cloud" else None, + rpm=args.rpm, + ) answerer = LLMClient( model=args.answerer_model, provider=args.provider, rpm=args.rpm ) diff --git a/benchmarks/common/goodmemory_client.py b/benchmarks/common/goodmemory_client.py new file mode 100644 index 0000000..853200e --- /dev/null +++ b/benchmarks/common/goodmemory_client.py @@ -0,0 +1,298 @@ +""" +GoodMemory Client +================= + +Async client for `GoodMemory `_, a +local-first memory layer for AI products and coding agents, exposed to this +suite through its packaged HTTP bridge. + +Setup (no Docker required):: + + npm install -g goodmemory # requires Bun for the bridge runtime + goodmemory-http-bridge # serves http://localhost:8739 by default + +The bridge port can be changed with ``GOODMEMORY_HTTP_BRIDGE_PORT``; point +this client at it with ``GOODMEMORY_HOST`` or ``--goodmemory-host``. The +bridge requires a bearer token by default (``GOODMEMORY_HTTP_BRIDGE_TOKEN``, +picked up automatically) or start it with ``--allow-insecure`` locally. + +The client exposes the same async interface as :class:`Mem0Client` so the +benchmark runners can switch backends without code changes: + + client.add(messages, user_id, ...) + client.search(query, user_id, top_k=..., ...) + client.delete_user(user_id) + +Notes +----- +- ``add`` maps to ``POST /memory/remember`` with ``scope.userId`` isolation + and GoodMemory's selective write pipeline (``mode="sync"``). +- ``search`` maps to ``POST /memory/recall-context``; the bridge returns + recalled memory items plus a prompt-ready context string. Items are + normalised to the ``{"memory", "score", "id"}`` shape the runners expect. +- GoodMemory scopes runs by ``userId``, so each benchmark run should use a + fresh ``--project-name`` (user ids embed it). ``delete_user`` clears the + scope's recalled items via the bridge's forget endpoint when available and + otherwise logs and returns True, relying on scope isolation. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +from datetime import datetime, timezone +from typing import Any + +import aiohttp +from aiolimiter import AsyncLimiter + +logger = logging.getLogger(__name__) + + +def format_observed_content( + content: str, + observation_date: str | None = None, + timestamp: int | None = None, +) -> str: + """Preserve benchmark event time in the text GoodMemory indexes.""" + if timestamp is not None: + observed_at = datetime.fromtimestamp(timestamp, tz=timezone.utc).isoformat() + observed_at = observed_at.replace("+00:00", "Z") + elif observation_date: + observed_at = observation_date + else: + return content + return f"[Observed at {observed_at}] {content}" + + +class GoodMemoryClient: + """Async client for the GoodMemory HTTP bridge. + + Args: + host: Bridge URL. Defaults to GOODMEMORY_HOST env or + http://localhost:8739. + max_retries: Maximum retry attempts for bridge calls. + retry_delay: Base delay in seconds between retries (linear backoff). + rpm: Requests per minute rate limit. + timeout: HTTP request timeout in seconds. + recall_strategy: GoodMemory recall strategy ("hybrid", "rules-only", + or "auto"). Defaults to "hybrid" — the bridge coerces any + non-"hybrid" strategy (including "auto") to rules-only, which + cannot surface semantically-relevant facts, so hybrid is required + for representative recall (it needs an embedding endpoint + the + recommended retrieval preset on the bridge; see the README). + Set "rules-only" to measure the lexical floor with no embedding. + """ + + def __init__( + self, + host: str | None = None, + token: str | None = None, + max_retries: int = 5, + retry_delay: float = 5.0, + rpm: int = 120, + timeout: float = 300.0, + recall_strategy: str | None = None, + ): + self.host = (host or os.getenv("GOODMEMORY_HOST", "http://localhost:8739")).rstrip("/") + self.token = token or os.getenv("GOODMEMORY_HTTP_BRIDGE_TOKEN", "") + self.max_retries = max_retries + self.retry_delay = retry_delay + self.recall_strategy = ( + recall_strategy or os.getenv("GOODMEMORY_RECALL_STRATEGY") or "hybrid" + ) + self.timeout = aiohttp.ClientTimeout(total=timeout) + self.limiter = AsyncLimiter(rpm, 60) + self._session: aiohttp.ClientSession | None = None + + def _headers(self, user_id: str) -> dict[str, str]: + # The bridge authenticates the CALLER separately from the request + # scope: every request must carry x-goodmemory-user-id (plus the + # bearer token when the bridge was started with one). + headers = { + "Content-Type": "application/json", + "x-goodmemory-user-id": user_id, + } + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + return headers + + async def _get_session(self) -> aiohttp.ClientSession: + if self._session is None or self._session.closed: + self._session = aiohttp.ClientSession(timeout=self.timeout) + return self._session + + async def close(self) -> None: + if self._session and not self._session.closed: + await self._session.close() + + async def __aenter__(self) -> "GoodMemoryClient": + return self + + async def __aexit__(self, *exc: Any) -> None: + await self.close() + + # ------------------------------------------------------------------ + # add + # ------------------------------------------------------------------ + + async def add( + self, + messages: list[dict[str, str]], + user_id: str, + observation_date: str | None = None, + timestamp: int | None = None, + custom_instructions: str | None = None, + metadata: dict | None = None, + ) -> dict | None: + """Write a message batch into the user's memory scope. + + ``observation_date``/``timestamp`` are folded into message content so + the timestamp survives the bridge's intentionally small remember + contract and remains visible to retrieval and answer generation. + """ + session = await self._get_session() + # Benchmark seeding must ingest every turn deterministically. This + # mirrors how GoodMemory's own benchmark harnesses seed conversations: + # every turn is written as a verbatim user-scoped fact with the + # original role preserved in the content (assistant-authored writes + # are otherwise blocked by the product write policy), forced past the + # selective write heuristics with verified fact annotations, under the + # deterministic rules-only extraction strategy. + kept = [ + { + "role": "user", + "content": format_observed_content( + ( + m["content"] + if m.get("role", "user") == "user" + else f"[role={m.get('role', 'user')}] {m['content']}" + ), + observation_date=observation_date, + timestamp=timestamp, + ), + } + for m in messages + if m.get("content") + ] + if not kept: + return {"skipped": True} + payload: dict[str, Any] = { + "scope": {"userId": user_id}, + "messages": kept, + "mode": "sync", + "extractionStrategy": "rules-only", + "annotations": [ + { + "remember": "always", + "confirmed": True, + "verified": True, + "kindHint": "fact", + "messageIndex": i, + } + for i in range(len(kept)) + ], + } + + for attempt in range(self.max_retries): + try: + async with self.limiter: + async with session.post( + f"{self.host}/memory/remember", json=payload, headers=self._headers(user_id) + ) as resp: + if resp.status >= 500: + raise aiohttp.ClientResponseError( + resp.request_info, resp.history, status=resp.status + ) + resp.raise_for_status() + return await resp.json() + except Exception as exc: + logger.warning( + "ADD attempt %d/%d failed (user=%s): %s", + attempt + 1, self.max_retries, user_id, str(exc)[:200], + ) + if attempt < self.max_retries - 1: + await asyncio.sleep(self.retry_delay * (attempt + 1)) + else: + logger.error("ADD failed after %d attempts for user=%s", self.max_retries, user_id) + return None + return None + + # ------------------------------------------------------------------ + # search + # ------------------------------------------------------------------ + + async def search( + self, + query: str, + user_id: str, + top_k: int = 20, + rerank: bool = False, + score_debug: bool = False, + ) -> list[dict]: + """Recall memories for a query; returns Mem0-shaped result dicts.""" + session = await self._get_session() + payload: dict[str, Any] = { + "scope": {"userId": user_id}, + "query": query, + } + if self.recall_strategy: + payload["strategy"] = self.recall_strategy + + for attempt in range(self.max_retries): + try: + async with self.limiter: + async with session.post( + f"{self.host}/memory/recall-context", json=payload, headers=self._headers(user_id) + ) as resp: + if resp.status >= 500: + raise aiohttp.ClientResponseError( + resp.request_info, resp.history, status=resp.status + ) + resp.raise_for_status() + data = await resp.json() + + items = data.get("items", []) if isinstance(data, dict) else [] + normalised: list[dict[str, Any]] = [] + for rank, item in enumerate(items[:top_k]): + content = item.get("content", "") + if not content: + continue + normalised.append( + { + # The bridge returns recall-ordered items without + # a numeric score; preserve that order. + "memory": content, + "score": max(0.0, 1.0 - rank * (1.0 / max(top_k, 1))), + "id": item.get("memoryId", ""), + } + ) + return normalised + except Exception as exc: + logger.warning( + "SEARCH attempt %d/%d failed (user=%s): %s", + attempt + 1, self.max_retries, user_id, str(exc)[:200], + ) + if attempt < self.max_retries - 1: + await asyncio.sleep(self.retry_delay * (attempt + 1)) + else: + logger.error("SEARCH failed after %d attempts for user=%s", self.max_retries, user_id) + return [] + return [] + + # ------------------------------------------------------------------ + # delete_user + # ------------------------------------------------------------------ + + async def delete_user(self, user_id: str) -> bool: + """Benchmark runs isolate state by scope (user id embeds the project + name), so a fresh project never sees another run's memories. The + bridge intentionally exposes no bulk delete-all endpoint; log and + rely on scope isolation.""" + logger.info( + "GoodMemory delete_user(%s): no bulk delete endpoint on the bridge; " + "runs are isolated by scope - use a fresh --project-name per run.", + user_id, + ) + return True diff --git a/benchmarks/locomo/run.py b/benchmarks/locomo/run.py index 8e4bdb9..0908d68 100644 --- a/benchmarks/locomo/run.py +++ b/benchmarks/locomo/run.py @@ -44,6 +44,7 @@ from benchmarks.common.llm_client import LLMClient from benchmarks.common.mem0_client import Mem0Client, format_search_results +from benchmarks.common.goodmemory_client import GoodMemoryClient from benchmarks.common.metrics import compute_overall_metrics from benchmarks.common.schema import ( CutoffResult, @@ -709,10 +710,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--user-profile", action="store_true", help="Fetch user profiles") parser.add_argument("--max-questions", type=int, default=None, help="Max questions to process (for quick testing)") parser.add_argument("--rpm", type=int, default=200, help="Requests per minute for LLM") - parser.add_argument("--backend", default="oss", choices=["oss", "cloud"], - help="Mem0 backend: 'oss' for self-hosted server (default), 'cloud' for api.mem0.ai") + parser.add_argument("--backend", default="oss", choices=["oss", "cloud", "goodmemory"], + help="Mem0 backend: 'oss' for self-hosted server (default), 'cloud' for api.mem0.ai, 'goodmemory' for a GoodMemory HTTP bridge") parser.add_argument("--mem0-host", default=None, help="Mem0 server URL (default: http://localhost:8888 for oss, https://api.mem0.ai for cloud)") + parser.add_argument("--goodmemory-host", default=None, + help="GoodMemory HTTP bridge URL (default: GOODMEMORY_HOST env or http://localhost:8739)") parser.add_argument("--mem0-api-key", default=None, help="Mem0 API key (cloud mode only)") return parser.parse_args() @@ -829,12 +832,18 @@ async def judge_one(qid: str, conv_idx: int, qi: int, qa: dict) -> None: # Init Mem0 (not used for --evaluate-only) backend = os.getenv("MEM0_BACKEND", args.backend) - mem0 = Mem0Client( - mode=backend, - host=args.mem0_host, - api_key=args.mem0_api_key if backend == "cloud" else None, - rpm=args.rpm, - ) + if backend == "goodmemory": + mem0 = GoodMemoryClient( + host=args.goodmemory_host, + rpm=args.rpm, + ) + else: + mem0 = Mem0Client( + mode=backend, + host=args.mem0_host, + api_key=args.mem0_api_key if backend == "cloud" else None, + rpm=args.rpm, + ) shutdown = GracefulShutdown() checkpoint = Checkpoint(output_dir) diff --git a/benchmarks/longmemeval/run.py b/benchmarks/longmemeval/run.py index 396788d..4fa226a 100644 --- a/benchmarks/longmemeval/run.py +++ b/benchmarks/longmemeval/run.py @@ -53,6 +53,7 @@ from benchmarks.common.llm_client import LLMClient from benchmarks.common.mem0_client import Mem0Client, format_search_results +from benchmarks.common.goodmemory_client import GoodMemoryClient from benchmarks.common.metrics import compute_overall_metrics from benchmarks.common.schema import ( CutoffResult, @@ -1033,13 +1034,17 @@ def parse_args() -> argparse.Namespace: help="Requests per minute for LLM", ) parser.add_argument( - "--backend", default="oss", choices=["oss", "cloud"], - help="Mem0 backend: 'oss' for self-hosted server (default), 'cloud' for api.mem0.ai", + "--backend", default="oss", choices=["oss", "cloud", "goodmemory"], + help="Mem0 backend: 'oss' for self-hosted server (default), 'cloud' for api.mem0.ai, 'goodmemory' for a GoodMemory HTTP bridge", ) parser.add_argument( "--mem0-host", default=None, help="Mem0 server URL", ) + parser.add_argument( + "--goodmemory-host", default=None, + help="GoodMemory HTTP bridge URL (default: GOODMEMORY_HOST env or http://localhost:8739)", + ) parser.add_argument( "--mem0-api-key", default=None, help="Mem0 API key (cloud mode only)", @@ -1231,12 +1236,18 @@ async def judge_one(question: dict) -> None: return backend = os.getenv("MEM0_BACKEND", args.backend) - mem0 = Mem0Client( - mode=backend, - host=args.mem0_host, - api_key=args.mem0_api_key if backend == "cloud" else None, - rpm=args.rpm, - ) + if backend == "goodmemory": + mem0 = GoodMemoryClient( + host=args.goodmemory_host, + rpm=args.rpm, + ) + else: + mem0 = Mem0Client( + mode=backend, + host=args.mem0_host, + api_key=args.mem0_api_key if backend == "cloud" else None, + rpm=args.rpm, + ) shutdown = GracefulShutdown() checkpoint = Checkpoint(output_dir) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_goodmemory_client.py b/tests/test_goodmemory_client.py new file mode 100644 index 0000000..1308ab2 --- /dev/null +++ b/tests/test_goodmemory_client.py @@ -0,0 +1,30 @@ +import unittest + +from benchmarks.common.goodmemory_client import format_observed_content + + +class GoodMemoryClientTimestampTest(unittest.TestCase): + def test_formats_unix_timestamp_as_utc_observation_prefix(self) -> None: + self.assertEqual( + format_observed_content("Deployed the API.", timestamp=1683504000), + "[Observed at 2023-05-08T00:00:00Z] Deployed the API.", + ) + + def test_uses_observation_date_when_timestamp_is_absent(self) -> None: + self.assertEqual( + format_observed_content( + "Reviewed the launch plan.", + observation_date="2023-05-09", + ), + "[Observed at 2023-05-09] Reviewed the launch plan.", + ) + + def test_leaves_undated_content_unchanged(self) -> None: + self.assertEqual( + format_observed_content("No date was supplied."), + "No date was supplied.", + ) + + +if __name__ == "__main__": + unittest.main()