From 022aee825f4c81635d192f9ed187c45b336470d2 Mon Sep 17 00:00:00 2001 From: hjqcan Date: Sun, 5 Jul 2026 08:34:35 -0700 Subject: [PATCH 1/3] Add GoodMemory as a third backend (local HTTP bridge) Adds benchmarks/common/goodmemory_client.py - an async client for the GoodMemory HTTP bridge (npm i -g goodmemory; goodmemory-http-bridge) exposing the same add/search/delete_user interface as Mem0Client - and wires --backend goodmemory (+ --goodmemory-host) into the LOCOMO, LongMemEval, and BEAM runners. Notes: - Bearer-token auth (GOODMEMORY_HTTP_BRIDGE_TOKEN) and the bridge's per-request caller header are handled automatically. - Benchmark seeding writes every turn deterministically (verified fact annotations + rules-only extraction), mirroring how GoodMemory's own benchmark harnesses ingest conversations; assistant turns are stored as user-scoped facts with the original role preserved in the content, since assistant-authored writes are blocked by the product write policy. - Runs are isolated by scope (user ids embed the project name); the bridge has no bulk delete endpoint, so delete_user relies on fresh project names. - Smoke-tested end to end against a live bridge (add -> search round trip). --- README.md | 20 ++ benchmarks/beam/run.py | 25 ++- benchmarks/common/goodmemory_client.py | 271 +++++++++++++++++++++++++ benchmarks/locomo/run.py | 25 ++- benchmarks/longmemeval/run.py | 27 ++- 5 files changed, 344 insertions(+), 24 deletions(-) create mode 100644 benchmarks/common/goodmemory_client.py diff --git a/README.md b/README.md index 16d39b0..5b59906 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,26 @@ python -m benchmarks.beam.run \ --chat-sizes 100K --conversations 0-9 ``` +### Option C: GoodMemory (local, no Docker) + +Runs against [GoodMemory](https://github.com/hjqcan/GoodMemory)'s packaged +HTTP bridge (local-first, SQLite-backed; requires [Bun](https://bun.sh)). + +```bash +npm install -g goodmemory +goodmemory-http-bridge # serves http://localhost:8739 + +# Run a benchmark against it +python -m benchmarks.locomo.run \ + --project-name my-goodmemory-test \ + --backend goodmemory +``` + +Point at a non-default bridge with `--goodmemory-host` or `GOODMEMORY_HOST`. +Each run should use a fresh `--project-name`: GoodMemory isolates state by +scope (the user id embeds the project name) and the bridge intentionally has +no bulk delete endpoint. + ### Option B: Mem0 OSS (Self-Hosted) Requires Docker and Docker Compose. This starts a local Mem0 server backed by Qdrant. diff --git a/benchmarks/beam/run.py b/benchmarks/beam/run.py index b277261..3bb5061 100644 --- a/benchmarks/beam/run.py +++ b/benchmarks/beam/run.py @@ -47,6 +47,7 @@ from benchmarks.common.llm_client import LLMClient from benchmarks.common.mem0_client import Mem0Client, format_search_results +from benchmarks.common.goodmemory_client import GoodMemoryClient from benchmarks.common.metrics import compute_kendall_tau_b, compute_overall_metrics from benchmarks.common.schema import ( CutoffResult, @@ -955,10 +956,12 @@ def parse_args() -> argparse.Namespace: help="Comma-separated question types to evaluate (default: all)", ) parser.add_argument("--rpm", type=int, default=200, help="Requests per minute for LLM") - parser.add_argument("--backend", default="oss", choices=["oss", "cloud"], - help="Mem0 backend: 'oss' for self-hosted server (default), 'cloud' for api.mem0.ai") + parser.add_argument("--backend", default="oss", choices=["oss", "cloud", "goodmemory"], + help="Mem0 backend: 'oss' for self-hosted server (default), 'cloud' for api.mem0.ai, 'goodmemory' for a GoodMemory HTTP bridge") parser.add_argument("--mem0-host", default=None, help="Mem0 server URL") + parser.add_argument("--goodmemory-host", default=None, + help="GoodMemory HTTP bridge URL (default: GOODMEMORY_HOST env or http://localhost:8739)") parser.add_argument("--mem0-api-key", default=None, help="Mem0 API key (cloud mode only)") return parser.parse_args() @@ -1021,12 +1024,18 @@ async def async_main() -> None: # Init clients backend = os.getenv("MEM0_BACKEND", args.backend) - mem0 = Mem0Client( - mode=backend, - host=args.mem0_host, - api_key=args.mem0_api_key if backend == "cloud" else None, - rpm=args.rpm, - ) + if backend == "goodmemory": + mem0 = GoodMemoryClient( + host=args.goodmemory_host, + rpm=args.rpm, + ) + else: + mem0 = Mem0Client( + mode=backend, + host=args.mem0_host, + api_key=args.mem0_api_key if backend == "cloud" else None, + rpm=args.rpm, + ) answerer = LLMClient( model=args.answerer_model, provider=args.provider, rpm=args.rpm ) diff --git a/benchmarks/common/goodmemory_client.py b/benchmarks/common/goodmemory_client.py new file mode 100644 index 0000000..9237e46 --- /dev/null +++ b/benchmarks/common/goodmemory_client.py @@ -0,0 +1,271 @@ +""" +GoodMemory Client +================= + +Async client for `GoodMemory `_, a +local-first memory layer for AI products and coding agents, exposed to this +suite through its packaged HTTP bridge. + +Setup (no Docker required):: + + npm install -g goodmemory # requires Bun for the bridge runtime + goodmemory-http-bridge # serves http://localhost:8739 by default + +The bridge port can be changed with ``GOODMEMORY_HTTP_BRIDGE_PORT``; point +this client at it with ``GOODMEMORY_HOST`` or ``--goodmemory-host``. The +bridge requires a bearer token by default (``GOODMEMORY_HTTP_BRIDGE_TOKEN``, +picked up automatically) or start it with ``--allow-insecure`` locally. + +The client exposes the same async interface as :class:`Mem0Client` so the +benchmark runners can switch backends without code changes: + + client.add(messages, user_id, ...) + client.search(query, user_id, top_k=..., ...) + client.delete_user(user_id) + +Notes +----- +- ``add`` maps to ``POST /memory/remember`` with ``scope.userId`` isolation + and GoodMemory's selective write pipeline (``mode="sync"``). +- ``search`` maps to ``POST /memory/recall-context``; the bridge returns + recalled memory items plus a prompt-ready context string. Items are + normalised to the ``{"memory", "score", "id"}`` shape the runners expect. +- GoodMemory scopes runs by ``userId``, so each benchmark run should use a + fresh ``--project-name`` (user ids embed it). ``delete_user`` clears the + scope's recalled items via the bridge's forget endpoint when available and + otherwise logs and returns True, relying on scope isolation. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +from typing import Any + +import aiohttp +from aiolimiter import AsyncLimiter + +logger = logging.getLogger(__name__) + + +class GoodMemoryClient: + """Async client for the GoodMemory HTTP bridge. + + Args: + host: Bridge URL. Defaults to GOODMEMORY_HOST env or + http://localhost:8739. + max_retries: Maximum retry attempts for bridge calls. + retry_delay: Base delay in seconds between retries (linear backoff). + rpm: Requests per minute rate limit. + timeout: HTTP request timeout in seconds. + recall_strategy: GoodMemory recall strategy ("auto", "rules-only", + or "hybrid"). The default "auto" lets the bridge decide based on + its configuration; "rules-only" needs no embedding provider. + """ + + def __init__( + self, + host: str | None = None, + token: str | None = None, + max_retries: int = 5, + retry_delay: float = 5.0, + rpm: int = 120, + timeout: float = 300.0, + recall_strategy: str | None = None, + ): + self.host = (host or os.getenv("GOODMEMORY_HOST", "http://localhost:8739")).rstrip("/") + self.token = token or os.getenv("GOODMEMORY_HTTP_BRIDGE_TOKEN", "") + self.max_retries = max_retries + self.retry_delay = retry_delay + self.recall_strategy = recall_strategy or os.getenv("GOODMEMORY_RECALL_STRATEGY") + self.timeout = aiohttp.ClientTimeout(total=timeout) + self.limiter = AsyncLimiter(rpm, 60) + self._session: aiohttp.ClientSession | None = None + + def _headers(self, user_id: str) -> dict[str, str]: + # The bridge authenticates the CALLER separately from the request + # scope: every request must carry x-goodmemory-user-id (plus the + # bearer token when the bridge was started with one). + headers = { + "Content-Type": "application/json", + "x-goodmemory-user-id": user_id, + } + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + return headers + + async def _get_session(self) -> aiohttp.ClientSession: + if self._session is None or self._session.closed: + self._session = aiohttp.ClientSession(timeout=self.timeout) + return self._session + + async def close(self) -> None: + if self._session and not self._session.closed: + await self._session.close() + + async def __aenter__(self) -> "GoodMemoryClient": + return self + + async def __aexit__(self, *exc: Any) -> None: + await self.close() + + # ------------------------------------------------------------------ + # add + # ------------------------------------------------------------------ + + async def add( + self, + messages: list[dict[str, str]], + user_id: str, + observation_date: str | None = None, + timestamp: int | None = None, + custom_instructions: str | None = None, + metadata: dict | None = None, + ) -> dict | None: + """Write a message batch into the user's memory scope. + + ``observation_date``/``timestamp`` are folded into message content + annotations are not needed: GoodMemory keeps source timestamps on the + write path via the bridge's remember contract. + """ + session = await self._get_session() + # Benchmark seeding must ingest every turn deterministically. This + # mirrors how GoodMemory's own benchmark harnesses seed conversations: + # every turn is written as a verbatim user-scoped fact with the + # original role preserved in the content (assistant-authored writes + # are otherwise blocked by the product write policy), forced past the + # selective write heuristics with verified fact annotations, under the + # deterministic rules-only extraction strategy. + kept = [ + { + "role": "user", + "content": ( + m["content"] + if m.get("role", "user") == "user" + else f"[role={m.get('role', 'user')}] {m['content']}" + ), + } + for m in messages + if m.get("content") + ] + if not kept: + return {"skipped": True} + payload: dict[str, Any] = { + "scope": {"userId": user_id}, + "messages": kept, + "mode": "sync", + "extractionStrategy": "rules-only", + "annotations": [ + { + "remember": "always", + "confirmed": True, + "verified": True, + "kindHint": "fact", + "messageIndex": i, + } + for i in range(len(kept)) + ], + } + + for attempt in range(self.max_retries): + try: + async with self.limiter: + async with session.post( + f"{self.host}/memory/remember", json=payload, headers=self._headers(user_id) + ) as resp: + if resp.status >= 500: + raise aiohttp.ClientResponseError( + resp.request_info, resp.history, status=resp.status + ) + resp.raise_for_status() + return await resp.json() + except Exception as exc: + logger.warning( + "ADD attempt %d/%d failed (user=%s): %s", + attempt + 1, self.max_retries, user_id, str(exc)[:200], + ) + if attempt < self.max_retries - 1: + await asyncio.sleep(self.retry_delay * (attempt + 1)) + else: + logger.error("ADD failed after %d attempts for user=%s", self.max_retries, user_id) + return None + return None + + # ------------------------------------------------------------------ + # search + # ------------------------------------------------------------------ + + async def search( + self, + query: str, + user_id: str, + top_k: int = 20, + rerank: bool = False, + score_debug: bool = False, + ) -> list[dict]: + """Recall memories for a query; returns Mem0-shaped result dicts.""" + session = await self._get_session() + payload: dict[str, Any] = { + "scope": {"userId": user_id}, + "query": query, + } + if self.recall_strategy: + payload["strategy"] = self.recall_strategy + + for attempt in range(self.max_retries): + try: + async with self.limiter: + async with session.post( + f"{self.host}/memory/recall-context", json=payload, headers=self._headers(user_id) + ) as resp: + if resp.status >= 500: + raise aiohttp.ClientResponseError( + resp.request_info, resp.history, status=resp.status + ) + resp.raise_for_status() + data = await resp.json() + + items = data.get("items", []) if isinstance(data, dict) else [] + normalised: list[dict[str, Any]] = [] + for rank, item in enumerate(items[:top_k]): + content = item.get("content", "") + if not content: + continue + normalised.append( + { + # The bridge returns recall-ordered items without + # a numeric score; preserve that order. + "memory": content, + "score": max(0.0, 1.0 - rank * (1.0 / max(top_k, 1))), + "id": item.get("memoryId", ""), + } + ) + return normalised + except Exception as exc: + logger.warning( + "SEARCH attempt %d/%d failed (user=%s): %s", + attempt + 1, self.max_retries, user_id, str(exc)[:200], + ) + if attempt < self.max_retries - 1: + await asyncio.sleep(self.retry_delay * (attempt + 1)) + else: + logger.error("SEARCH failed after %d attempts for user=%s", self.max_retries, user_id) + return [] + return [] + + # ------------------------------------------------------------------ + # delete_user + # ------------------------------------------------------------------ + + async def delete_user(self, user_id: str) -> bool: + """Benchmark runs isolate state by scope (user id embeds the project + name), so a fresh project never sees another run's memories. The + bridge intentionally exposes no bulk delete-all endpoint; log and + rely on scope isolation.""" + logger.info( + "GoodMemory delete_user(%s): no bulk delete endpoint on the bridge; " + "runs are isolated by scope - use a fresh --project-name per run.", + user_id, + ) + return True diff --git a/benchmarks/locomo/run.py b/benchmarks/locomo/run.py index 8e4bdb9..0908d68 100644 --- a/benchmarks/locomo/run.py +++ b/benchmarks/locomo/run.py @@ -44,6 +44,7 @@ from benchmarks.common.llm_client import LLMClient from benchmarks.common.mem0_client import Mem0Client, format_search_results +from benchmarks.common.goodmemory_client import GoodMemoryClient from benchmarks.common.metrics import compute_overall_metrics from benchmarks.common.schema import ( CutoffResult, @@ -709,10 +710,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--user-profile", action="store_true", help="Fetch user profiles") parser.add_argument("--max-questions", type=int, default=None, help="Max questions to process (for quick testing)") parser.add_argument("--rpm", type=int, default=200, help="Requests per minute for LLM") - parser.add_argument("--backend", default="oss", choices=["oss", "cloud"], - help="Mem0 backend: 'oss' for self-hosted server (default), 'cloud' for api.mem0.ai") + parser.add_argument("--backend", default="oss", choices=["oss", "cloud", "goodmemory"], + help="Mem0 backend: 'oss' for self-hosted server (default), 'cloud' for api.mem0.ai, 'goodmemory' for a GoodMemory HTTP bridge") parser.add_argument("--mem0-host", default=None, help="Mem0 server URL (default: http://localhost:8888 for oss, https://api.mem0.ai for cloud)") + parser.add_argument("--goodmemory-host", default=None, + help="GoodMemory HTTP bridge URL (default: GOODMEMORY_HOST env or http://localhost:8739)") parser.add_argument("--mem0-api-key", default=None, help="Mem0 API key (cloud mode only)") return parser.parse_args() @@ -829,12 +832,18 @@ async def judge_one(qid: str, conv_idx: int, qi: int, qa: dict) -> None: # Init Mem0 (not used for --evaluate-only) backend = os.getenv("MEM0_BACKEND", args.backend) - mem0 = Mem0Client( - mode=backend, - host=args.mem0_host, - api_key=args.mem0_api_key if backend == "cloud" else None, - rpm=args.rpm, - ) + if backend == "goodmemory": + mem0 = GoodMemoryClient( + host=args.goodmemory_host, + rpm=args.rpm, + ) + else: + mem0 = Mem0Client( + mode=backend, + host=args.mem0_host, + api_key=args.mem0_api_key if backend == "cloud" else None, + rpm=args.rpm, + ) shutdown = GracefulShutdown() checkpoint = Checkpoint(output_dir) diff --git a/benchmarks/longmemeval/run.py b/benchmarks/longmemeval/run.py index 396788d..4fa226a 100644 --- a/benchmarks/longmemeval/run.py +++ b/benchmarks/longmemeval/run.py @@ -53,6 +53,7 @@ from benchmarks.common.llm_client import LLMClient from benchmarks.common.mem0_client import Mem0Client, format_search_results +from benchmarks.common.goodmemory_client import GoodMemoryClient from benchmarks.common.metrics import compute_overall_metrics from benchmarks.common.schema import ( CutoffResult, @@ -1033,13 +1034,17 @@ def parse_args() -> argparse.Namespace: help="Requests per minute for LLM", ) parser.add_argument( - "--backend", default="oss", choices=["oss", "cloud"], - help="Mem0 backend: 'oss' for self-hosted server (default), 'cloud' for api.mem0.ai", + "--backend", default="oss", choices=["oss", "cloud", "goodmemory"], + help="Mem0 backend: 'oss' for self-hosted server (default), 'cloud' for api.mem0.ai, 'goodmemory' for a GoodMemory HTTP bridge", ) parser.add_argument( "--mem0-host", default=None, help="Mem0 server URL", ) + parser.add_argument( + "--goodmemory-host", default=None, + help="GoodMemory HTTP bridge URL (default: GOODMEMORY_HOST env or http://localhost:8739)", + ) parser.add_argument( "--mem0-api-key", default=None, help="Mem0 API key (cloud mode only)", @@ -1231,12 +1236,18 @@ async def judge_one(question: dict) -> None: return backend = os.getenv("MEM0_BACKEND", args.backend) - mem0 = Mem0Client( - mode=backend, - host=args.mem0_host, - api_key=args.mem0_api_key if backend == "cloud" else None, - rpm=args.rpm, - ) + if backend == "goodmemory": + mem0 = GoodMemoryClient( + host=args.goodmemory_host, + rpm=args.rpm, + ) + else: + mem0 = Mem0Client( + mode=backend, + host=args.mem0_host, + api_key=args.mem0_api_key if backend == "cloud" else None, + rpm=args.rpm, + ) shutdown = GracefulShutdown() checkpoint = Checkpoint(output_dir) From 6e047943bb83c1e272184cfbe2b6b67102660ce6 Mon Sep 17 00:00:00 2001 From: hjqcan Date: Mon, 6 Jul 2026 09:39:03 -0700 Subject: [PATCH 2/3] GoodMemory: default to hybrid recall + document the semantic recall stack The bridge coerces any non-hybrid recall strategy (including auto) to a rules-only lexical floor, which cannot surface semantically-relevant facts. Default recall_strategy to hybrid so representative recall works out of the box (override via GOODMEMORY_RECALL_STRATEGY), and document the bridge config it needs: an embedding endpoint, the recommended retrieval preset (semantic candidate union), and in-memory storage. Deterministic verbatim seeding is unchanged (no LLM extractor required). Validated 4/4 on cross-topic recall probes (the query-relevant fact ranks first). --- README.md | 25 +++++++++++++++++++++---- benchmarks/common/goodmemory_client.py | 14 ++++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5b59906..f143cb4 100644 --- a/README.md +++ b/README.md @@ -51,22 +51,39 @@ python -m benchmarks.beam.run \ ### Option C: GoodMemory (local, no Docker) Runs against [GoodMemory](https://github.com/hjqcan/GoodMemory)'s packaged -HTTP bridge (local-first, SQLite-backed; requires [Bun](https://bun.sh)). +HTTP bridge (local-first; requires [Bun](https://bun.sh)). Ingestion is +deterministic (every turn is written verbatim, no LLM extractor needed), but +**representative recall needs the bridge started with an embedding endpoint, +the `recommended` retrieval preset (GoodMemory's semantic candidate union), +and in-memory storage** (its pure-JS vector index needs no native libraries). +Without these the bridge falls back to a lexical-only floor and under-reports. ```bash npm install -g goodmemory -goodmemory-http-bridge # serves http://localhost:8739 + +# Bridge with the semantic recall stack (any OpenAI-compatible embedding +# endpoint works — OpenAI, OpenRouter, Azure, a local Ollama, …): +GOODMEMORY_HTTP_BRIDGE_TOKEN=your-token \ +GOODMEMORY_STORAGE_PROVIDER=memory \ +GOODMEMORY_HTTP_BRIDGE_RETRIEVAL_PRESET=recommended \ +GOODMEMORY_EMBEDDING_PROVIDER=openai \ +GOODMEMORY_EMBEDDING_MODEL=text-embedding-3-small \ +GOODMEMORY_EMBEDDING_API_KEY=$OPENAI_API_KEY \ + goodmemory-http-bridge # serves http://localhost:8739 # Run a benchmark against it -python -m benchmarks.locomo.run \ +GOODMEMORY_HTTP_BRIDGE_TOKEN=your-token python -m benchmarks.locomo.run \ --project-name my-goodmemory-test \ --backend goodmemory ``` +The client requests `hybrid` recall by default (the bridge coerces any other +strategy to the rules-only floor); override with `GOODMEMORY_RECALL_STRATEGY`. Point at a non-default bridge with `--goodmemory-host` or `GOODMEMORY_HOST`. Each run should use a fresh `--project-name`: GoodMemory isolates state by scope (the user id embeds the project name) and the bridge intentionally has -no bulk delete endpoint. +no bulk delete endpoint. In-memory storage is fresh per bridge process, which +suits a benchmark run. ### Option B: Mem0 OSS (Self-Hosted) diff --git a/benchmarks/common/goodmemory_client.py b/benchmarks/common/goodmemory_client.py index 9237e46..4c33ef5 100644 --- a/benchmarks/common/goodmemory_client.py +++ b/benchmarks/common/goodmemory_client.py @@ -59,9 +59,13 @@ class GoodMemoryClient: retry_delay: Base delay in seconds between retries (linear backoff). rpm: Requests per minute rate limit. timeout: HTTP request timeout in seconds. - recall_strategy: GoodMemory recall strategy ("auto", "rules-only", - or "hybrid"). The default "auto" lets the bridge decide based on - its configuration; "rules-only" needs no embedding provider. + recall_strategy: GoodMemory recall strategy ("hybrid", "rules-only", + or "auto"). Defaults to "hybrid" — the bridge coerces any + non-"hybrid" strategy (including "auto") to rules-only, which + cannot surface semantically-relevant facts, so hybrid is required + for representative recall (it needs an embedding endpoint + the + recommended retrieval preset on the bridge; see the README). + Set "rules-only" to measure the lexical floor with no embedding. """ def __init__( @@ -78,7 +82,9 @@ def __init__( self.token = token or os.getenv("GOODMEMORY_HTTP_BRIDGE_TOKEN", "") self.max_retries = max_retries self.retry_delay = retry_delay - self.recall_strategy = recall_strategy or os.getenv("GOODMEMORY_RECALL_STRATEGY") + self.recall_strategy = ( + recall_strategy or os.getenv("GOODMEMORY_RECALL_STRATEGY") or "hybrid" + ) self.timeout = aiohttp.ClientTimeout(total=timeout) self.limiter = AsyncLimiter(rpm, 60) self._session: aiohttp.ClientSession | None = None From 6e3b204d495c57cd362f0b1c9c08becd70b22b80 Mon Sep 17 00:00:00 2001 From: hjqcan Date: Mon, 13 Jul 2026 14:41:39 -0700 Subject: [PATCH 3/3] Preserve benchmark timestamps in GoodMemory adapter --- README.md | 3 ++- benchmarks/common/goodmemory_client.py | 35 ++++++++++++++++++++------ tests/__init__.py | 0 tests/test_goodmemory_client.py | 30 ++++++++++++++++++++++ 4 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/test_goodmemory_client.py diff --git a/README.md b/README.md index f143cb4..d14261e 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,8 @@ python -m benchmarks.beam.run \ Runs against [GoodMemory](https://github.com/hjqcan/GoodMemory)'s packaged HTTP bridge (local-first; requires [Bun](https://bun.sh)). Ingestion is -deterministic (every turn is written verbatim, no LLM extractor needed), but +deterministic (every turn is written verbatim, no LLM extractor needed), and +session timestamps are preserved as UTC observation prefixes. However, **representative recall needs the bridge started with an embedding endpoint, the `recommended` retrieval preset (GoodMemory's semantic candidate union), and in-memory storage** (its pure-JS vector index needs no native libraries). diff --git a/benchmarks/common/goodmemory_client.py b/benchmarks/common/goodmemory_client.py index 4c33ef5..853200e 100644 --- a/benchmarks/common/goodmemory_client.py +++ b/benchmarks/common/goodmemory_client.py @@ -41,6 +41,7 @@ import asyncio import logging import os +from datetime import datetime, timezone from typing import Any import aiohttp @@ -49,6 +50,22 @@ logger = logging.getLogger(__name__) +def format_observed_content( + content: str, + observation_date: str | None = None, + timestamp: int | None = None, +) -> str: + """Preserve benchmark event time in the text GoodMemory indexes.""" + if timestamp is not None: + observed_at = datetime.fromtimestamp(timestamp, tz=timezone.utc).isoformat() + observed_at = observed_at.replace("+00:00", "Z") + elif observation_date: + observed_at = observation_date + else: + return content + return f"[Observed at {observed_at}] {content}" + + class GoodMemoryClient: """Async client for the GoodMemory HTTP bridge. @@ -131,9 +148,9 @@ async def add( ) -> dict | None: """Write a message batch into the user's memory scope. - ``observation_date``/``timestamp`` are folded into message content - annotations are not needed: GoodMemory keeps source timestamps on the - write path via the bridge's remember contract. + ``observation_date``/``timestamp`` are folded into message content so + the timestamp survives the bridge's intentionally small remember + contract and remains visible to retrieval and answer generation. """ session = await self._get_session() # Benchmark seeding must ingest every turn deterministically. This @@ -146,10 +163,14 @@ async def add( kept = [ { "role": "user", - "content": ( - m["content"] - if m.get("role", "user") == "user" - else f"[role={m.get('role', 'user')}] {m['content']}" + "content": format_observed_content( + ( + m["content"] + if m.get("role", "user") == "user" + else f"[role={m.get('role', 'user')}] {m['content']}" + ), + observation_date=observation_date, + timestamp=timestamp, ), } for m in messages diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_goodmemory_client.py b/tests/test_goodmemory_client.py new file mode 100644 index 0000000..1308ab2 --- /dev/null +++ b/tests/test_goodmemory_client.py @@ -0,0 +1,30 @@ +import unittest + +from benchmarks.common.goodmemory_client import format_observed_content + + +class GoodMemoryClientTimestampTest(unittest.TestCase): + def test_formats_unix_timestamp_as_utc_observation_prefix(self) -> None: + self.assertEqual( + format_observed_content("Deployed the API.", timestamp=1683504000), + "[Observed at 2023-05-08T00:00:00Z] Deployed the API.", + ) + + def test_uses_observation_date_when_timestamp_is_absent(self) -> None: + self.assertEqual( + format_observed_content( + "Reviewed the launch plan.", + observation_date="2023-05-09", + ), + "[Observed at 2023-05-09] Reviewed the launch plan.", + ) + + def test_leaves_undated_content_unchanged(self) -> None: + self.assertEqual( + format_observed_content("No date was supplied."), + "No date was supplied.", + ) + + +if __name__ == "__main__": + unittest.main()