diff --git a/.env.example b/.env.example index e985ba5..c5f645b 100644 --- a/.env.example +++ b/.env.example @@ -2,3 +2,19 @@ DATABASE_URL=postgresql://memorybase:memorybase@localhost:5432/memorybase_db BACKEND_HOST=127.0.0.1 BACKEND_PORT=8000 FRONTEND_PORT=5173 +EMBEDDING_PROVIDER=local +SILICONFLOW_API_KEY= +SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1 +SILICONFLOW_EMBEDDING_MODEL=Qwen/Qwen3-Embedding-0.6B +SILICONFLOW_EMBEDDING_DIMENSIONS=1024 +LLM_PROVIDER= +LLM_ANALYSIS_API_KEY= +LLM_ANALYSIS_BASE_URL=https://api.openai.com/v1 +LLM_ANALYSIS_MODEL=gpt-4o-mini +LLM_ANALYSIS_PROVIDER=openai-compatible +DEEPSEEK_API_KEY= +DEEPSEEK_BASE_URL=https://api.deepseek.com/v1 +DEEPSEEK_CHAT_MODEL=deepseek-chat +SILICONFLOW_CHAT_MODEL= +LLM_TEMPERATURE=0.2 +LLM_MAX_TOKENS=800 diff --git a/.gitignore b/.gitignore index 40b01e3..441f19a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ Thumbs.db .vscode/ .idea/ +.claude/ # Python __pycache__/ @@ -27,6 +28,7 @@ dist/ *.log logs/ data/logs/ +!docs/final-assets/screenshots/logs/ # Environment .env @@ -45,7 +47,10 @@ data/markdown_wiki/ .coverage coverage/ htmlcov/ -evaluation/outputs/*.csv -evaluation/outputs/*.md +evaluation/outputs/** +evaluation/external/*/raw/* +!evaluation/external/*/raw/.gitkeep +evaluation/external/*/processed/* +!evaluation/external/*/processed/.gitkeep docs/prompt.md docs/summary.md diff --git a/README.md b/README.md index cad4153..d65d8cf 100644 --- a/README.md +++ b/README.md @@ -26,19 +26,25 @@ SourceDocument - Frontend: React + Vite - Database: PostgreSQL - Graph Database: Neo4j（可选，用于知识图谱同步与可视化） -- Search: PostgreSQL Full Text Search +- Search: PostgreSQL Full Text Search（GIN tsvector / trigram）+ 可选 embedding 的 hybrid recall +- AI（可选）: OpenAI-compatible LLM 用于候选记忆抽取与 QA；本地 hashing embedding cache，未配置外部 provider 时透明 fallback 到 keyword +- CLI: `mb` / `memorybase`（sessions / observe / remember / search / recall） - SQL: views, triggers, indexes - Deployment: Docker Compose -## P0 MVP +## 功能模块 -- SourceDocument / SourceChunk -- MemoryItem / MemoryEvidence -- MemoryRevision / AuditLog -- RecallLog -- WikiPage / WikiPageRevision -- TimelineEntry -- Dashboard / Source / Memory / Recall / Wiki 页面 +- Source 导入与 chunk 切分：SourceDocument / SourceChunk +- 记忆与证据链：MemoryItem / MemoryEvidence / MemoryRevision +- 治理与溯源：AuditLog、ConflictRecord 冲突治理、ForgetRequest 遗忘/归档审批、TimelineEntry +- 检索：lexical（FTS / trigram）+ 可选 embedding 的 hybrid recall（无 embedding 时透明 fallback 到 keyword），并投影为 Context Pack +- 权限与可见性：AccessPolicy、agent-aware visibility +- 候选记忆抽取：rule-based + 可选 LLM（结果写入 `status='candidate'`，需人工审批后进入 active） +- Wiki 投影：WikiPage / WikiPageRevision，可导出 Markdown +- Graph Explorer：PostgreSQL preview + 可选 Neo4j 同步的 provenance 图谱 +- CLI / Agent Runtime：`mb` sessions / observe / remember / search / recall +- 评测：LoCoMo / LongMemEval / MemoryAgentBench adapters 与 evaluation framework +- 前端页面：Dashboard / Source / Memory / Recall / Governance / Wiki / Runtime / Graph ## 本地运行 @@ -271,29 +277,16 @@ psql postgresql://memorybase:memorybase@localhost:5432/memorybase_db -c "\dt" npm run db:check ``` -## 文档目录 - -- docs/00-project-overview\.md -- docs/01-requirements.md -- docs/02-data-flow\.md -- docs/03-data-dictionary.md -- docs/04-er-design.md -- docs/05-logical-design.md -- docs/06-physical-design.md -- docs/07-system-architecture.md -- docs/08-api-design.md -- docs/09-module-ipo.md -- docs/10-test-plan.md -- docs/11-demo-script.md -- docs/12-github-workflow\.md -- docs/13-final-report-outline.md -- docs/14-initial-issues.md -- docs/15-api-contract-plan.md -- docs/16-agent-runtime-gap-analysis.md -- docs/17-agent-runtime-plan.md -- docs/18-pr4-lexical-search-design.md -- docs/19-repo-session-aware-context-design.md -- docs/20-course-alignment-risk-and-recovery-plan.md +## 文档与材料 + +项目的设计文档、演示材料、源程序说明与参考资料统一放在 [`docs/`](docs/) 目录下： + +- 整合后的主报告：[`docs/final-report.md`](docs/final-report.md) / [`docs/final-report.pdf`](docs/final-report.pdf) +- 答辩 PPT：[`docs/final-assets/slides/final-defense.pdf`](docs/final-assets/slides/final-defense.pdf) +- 流程图 / ER / 时序图与系统演示截图：[`docs/final-assets/`](docs/final-assets/) +- 需求 / 概念 / 逻辑 / 物理设计、范式、索引、API、分工等专项文档：见 [`docs/`](docs/) + +完整的文档导览与主题索引见 [`docs/README.md`](docs/README.md)；开发过程中的规划与记录归档在 [`docs/process/`](docs/process/)。 ## GitHub Workflows diff --git a/backend/app/api/deps.py b/backend/app/api/deps.py index 3d8278b..4692879 100644 --- a/backend/app/api/deps.py +++ b/backend/app/api/deps.py @@ -20,6 +20,7 @@ PostgresGraphRepository, PostgresGraphVisibilityRepository, ) +from ..services.llm_analysis import LlmAnalysisDefaults, OpenAICompatibleAnalysisClient from ..services.llm_service import AnswerService, ChatProvider, OpenAICompatibleChatProvider from ..services.memory_extraction_service import MemoryExtractionService from ..services.memory_service import MemoryService, PostgresMemoryRepository @@ -31,6 +32,8 @@ from ..services.wiki_service import PostgresWikiRepository, WikiService +# Dependency builders keep FastAPI route files thin. The graph service is cached +# because it can own a Neo4j driver pool; ordinary repositories remain cheap wrappers. @lru_cache(maxsize=1) def get_database() -> Database: settings = get_settings() @@ -53,9 +56,19 @@ def get_memory_service() -> MemoryService: def get_memory_extraction_service() -> MemoryExtractionService: + settings = get_settings() return MemoryExtractionService( database=get_database(), memory_service=get_memory_service(), + llm_client=OpenAICompatibleAnalysisClient(), + llm_defaults=LlmAnalysisDefaults( + api_key=settings.llm_analysis_api_key, + base_url=settings.llm_analysis_base_url, + model=settings.llm_analysis_model, + provider=settings.llm_analysis_provider, + temperature=settings.llm_temperature, + max_tokens=settings.llm_max_tokens, + ), ) @@ -138,6 +151,8 @@ def get_app_settings() -> Settings: def _build_recall_repository(settings: Settings) -> PostgresRecallRepository: + # Recall and QA share this builder so keyword/vector/hybrid configuration stays + # consistent across direct recall, context packs, and optional answer generation. return PostgresRecallRepository( get_database(), embedding_provider=_build_embedding_provider(settings), @@ -148,6 +163,8 @@ def _build_recall_repository(settings: Settings) -> PostgresRecallRepository: def _build_embedding_provider(settings: Settings) -> EmbeddingProvider: + # Local hashing is the deterministic course-demo default; SiliconFlow is an + # optional provider path when an external embedding API is configured. provider = _embedding_provider_name(settings) if provider == "siliconflow": return SiliconFlowEmbeddingProvider( diff --git a/backend/app/api/memories.py b/backend/app/api/memories.py index e9060a4..f6eac59 100644 --- a/backend/app/api/memories.py +++ b/backend/app/api/memories.py @@ -8,6 +8,8 @@ from ..models.memory import ( ActorContext, EditorType, + MemoryBatchCreateRequest, + MemoryBatchCreateResponse, MemoryCreateRequest, MemoryDeleteResponse, MemoryDetailResponse, @@ -41,6 +43,29 @@ def create_memory( raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc +@router.post( + "/batch", + response_model=MemoryBatchCreateResponse, + status_code=status.HTTP_201_CREATED, +) +def create_memories( + payload: MemoryBatchCreateRequest, + x_actor_type: EditorType = Header(default="user", alias="X-Actor-Type"), + x_actor_id: UUID | None = Header(default=None, alias="X-Actor-Id"), + x_revision_reason: str = Header(default="memory batch create", alias="X-Revision-Reason"), + service: MemoryService = Depends(get_memory_service), +) -> MemoryBatchCreateResponse: + try: + actor = ActorContext( + actor_type=x_actor_type, + actor_id=x_actor_id, + revision_reason=x_revision_reason, + ) + return service.create_memories(payload, actor) + except MemoryValidationError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + @router.get("", response_model=MemoryListResponse) def list_memories( workspace_id: UUID | None = Query(default=None), diff --git a/backend/app/api/memory_extraction.py b/backend/app/api/memory_extraction.py index fc60b80..0735805 100644 --- a/backend/app/api/memory_extraction.py +++ b/backend/app/api/memory_extraction.py @@ -36,13 +36,14 @@ def extract_from_chunks( actor = ActorContext( actor_type=x_actor_type, actor_id=x_actor_id, - revision_reason="rule-based memory extraction", + revision_reason=f"{payload.method} memory extraction", ) candidates = service.extract_from_chunks(payload, actor) except MemoryExtractionValidationError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc return MemoryExtractionResponse( workspace_id=payload.workspace_id, + method=payload.method, created_count=len(candidates), candidates=candidates, ) diff --git a/backend/app/cli/client.py b/backend/app/cli/client.py index d03e6f9..621ed92 100644 --- a/backend/app/cli/client.py +++ b/backend/app/cli/client.py @@ -63,6 +63,15 @@ def create_memory( ) -> dict[str, Any]: ... + def extract_candidates( + self, + payload: dict[str, Any], + *, + actor_type: str, + actor_id: str | None, + ) -> dict[str, Any]: + ... + class HttpMemoryBaseClient: def __init__(self, api_base_url: str) -> None: @@ -133,6 +142,23 @@ def create_memory( headers["X-Actor-Id"] = actor_id return self._request("POST", "/api/memories", json=payload, headers=headers) + def extract_candidates( + self, + payload: dict[str, Any], + *, + actor_type: str, + actor_id: str | None, + ) -> dict[str, Any]: + headers = {"X-Actor-Type": actor_type} + if actor_id: + headers["X-Actor-Id"] = actor_id + return self._request( + "POST", + "/api/memory-extraction/from-chunks", + json=payload, + headers=headers, + ) + def _request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: url = f"{self._api_base_url}{path}" try: diff --git a/backend/app/cli/commands/extract.py b/backend/app/cli/commands/extract.py new file mode 100644 index 0000000..108802c --- /dev/null +++ b/backend/app/cli/commands/extract.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import os +from pathlib import Path +from uuid import UUID + +import typer + +from ..client import MemoryBaseClientError, MemoryBaseServerError, build_client +from ..config import load_config +from ..output import ( + EXIT_CLIENT_ERROR, + EXIT_OK, + EXIT_SERVER_ERROR, + error, + validate_format, + write_result, +) +from ..runtime import resolve_workspace_and_agent + +EXTRACTION_METHODS = {"rule_based", "llm"} + + +def extract( + config_path: Path | None = typer.Option(None, "--config", help="Config file to read."), + api_base_url: str | None = typer.Option(None, "--api-base", help="MemoryBase API base URL."), + workspace: str | None = typer.Option(None, "--workspace", help="Workspace slug or UUID."), + agent: str | None = typer.Option(None, "--agent", help="Agent name or UUID."), + chunk_ids: list[UUID] | None = typer.Option( + None, + "--chunk", + help="Source chunk UUID. Repeat for multiple chunks.", + ), + max_candidates: int = typer.Option(10, "--max-candidates", min=1, max=50), + method: str = typer.Option("rule_based", "--method", help="rule_based or llm."), + llm_api_key: str | None = typer.Option( + None, + "--llm-api-key", + help="LLM API key. Defaults to LLM_ANALYSIS_API_KEY or OPENAI_API_KEY.", + ), + llm_base_url: str | None = typer.Option( + None, + "--llm-base-url", + help="OpenAI-compatible base URL.", + ), + llm_model: str | None = typer.Option(None, "--llm-model", help="LLM model name."), + llm_provider: str | None = typer.Option( + None, + "--llm-provider", + help="Provider label stored in run audit metadata.", + ), + output_format: str = typer.Option("json", "--format", help="json, markdown, or table."), +) -> None: + validate_format(output_format) + if method not in EXTRACTION_METHODS: + error("Unsupported extraction method. Use rule_based or llm.") + raise typer.Exit(EXIT_CLIENT_ERROR) + if not chunk_ids: + error("At least one --chunk is required.") + raise typer.Exit(EXIT_CLIENT_ERROR) + + config = load_config(config_path).with_overrides( + api_base_url=api_base_url, + workspace=workspace, + agent=agent, + ) + try: + client = build_client(config) + workspace_id, agent_id = resolve_workspace_and_agent(client, config) + payload = { + "workspace_id": workspace_id, + "chunk_ids": [str(chunk_id) for chunk_id in chunk_ids], + "max_candidates": max_candidates, + "method": method, + } + if method == "llm": + payload["llm"] = { + "api_key": llm_api_key + or os.getenv("LLM_ANALYSIS_API_KEY") + or os.getenv("OPENAI_API_KEY"), + "base_url": llm_base_url, + "model": llm_model, + "provider": llm_provider, + } + payload["llm"] = { + key: value for key, value in payload["llm"].items() if value is not None + } + result = client.extract_candidates( + payload, + actor_type=config.actor_type, + actor_id=config.actor_id or agent_id, + ) + except MemoryBaseClientError as exc: + error(str(exc)) + raise typer.Exit(EXIT_CLIENT_ERROR) from exc + except MemoryBaseServerError as exc: + error(str(exc)) + raise typer.Exit(EXIT_SERVER_ERROR) from exc + + write_result(result, output_format=output_format) + raise typer.Exit(EXIT_OK) diff --git a/backend/app/cli/main.py b/backend/app/cli/main.py index f7d9b33..ab6ee6a 100644 --- a/backend/app/cli/main.py +++ b/backend/app/cli/main.py @@ -6,6 +6,7 @@ from .commands.configure import configure from .commands.context import context from .commands.eval import app as eval_app +from .commands.extract import extract from .commands.health import health from .commands.observe import observe from .commands.recall import recall @@ -41,6 +42,7 @@ def main( app.command("configure", short_help="Configure CLI defaults")(configure) app.command("context", short_help="Render agent context")(context) +app.command("extract", short_help="Extract candidate memories from chunks")(extract) app.command("health", short_help="Check API, workspace, and agent health")(health) app.command("observe", short_help="Write conversation messages")(observe) app.command("recall", short_help="Recall governed memories")(recall) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index b571f1f..d39fd20 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -40,6 +40,14 @@ class Settings(BaseModel): llm_provider: str = os.getenv("LLM_PROVIDER", "") llm_temperature: float = float(os.getenv("LLM_TEMPERATURE", "0.2")) llm_max_tokens: int = int(os.getenv("LLM_MAX_TOKENS", "800")) + llm_analysis_api_key: str = os.getenv("LLM_ANALYSIS_API_KEY", "") or os.getenv( + "OPENAI_API_KEY", "" + ) + llm_analysis_base_url: str = os.getenv( + "LLM_ANALYSIS_BASE_URL", "https://api.openai.com/v1" + ) + llm_analysis_model: str = os.getenv("LLM_ANALYSIS_MODEL", "gpt-4o-mini") + llm_analysis_provider: str = os.getenv("LLM_ANALYSIS_PROVIDER", "openai-compatible") deepseek_api_key: str = os.getenv("DEEPSEEK_API_KEY", "") deepseek_base_url: str = os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1") deepseek_chat_model: str = os.getenv("DEEPSEEK_CHAT_MODEL", "deepseek-chat") diff --git a/backend/app/main.py b/backend/app/main.py index ad68634..9f94fb7 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -24,6 +24,8 @@ from .core.config import get_settings +# FastAPI lifespan owns process-level resources. Most services are stateless per +# request, but the graph service may hold a Neo4j driver pool that should be closed. @asynccontextmanager async def app_lifespan(app: FastAPI): try: @@ -39,6 +41,9 @@ def create_app() -> FastAPI: version=settings.app_version, lifespan=app_lifespan, ) + + # All routers are mounted under /api so the React frontend can use one proxy + # prefix while backend modules stay grouped by domain responsibility. app.include_router(health_router, prefix="/api") app.include_router(sources_router, prefix="/api") app.include_router(memories_router, prefix="/api") diff --git a/backend/app/models/memory.py b/backend/app/models/memory.py index ed5bbf9..1d017f9 100644 --- a/backend/app/models/memory.py +++ b/backend/app/models/memory.py @@ -48,9 +48,15 @@ class MemoryCreateRequest(BaseModel): created_from_doc_id: UUID | None = None owner_user_id: UUID | None = None owner_agent_id: UUID | None = None + valid_from: datetime | None = None + supersedes_memory_id: UUID | None = None evidence: list["MemoryEvidenceInput"] = Field(default_factory=list) +class MemoryBatchCreateRequest(BaseModel): + items: list[MemoryCreateRequest] = Field(min_length=1, max_length=500) + + class MemoryUpdateRequest(BaseModel): canonical_text: str | None = Field(default=None, min_length=1) summary: str | None = None @@ -134,6 +140,12 @@ class MemoryListResponse(PageResponse[MemorySummaryResponse]): pass +class MemoryBatchCreateResponse(BaseModel): + workspace_id: UUID + count: int + items: list[MemorySummaryResponse] + + class MemoryDetailResponse(MemorySummaryResponse): owner_user_id: UUID | None = None owner_agent_id: UUID | None = None diff --git a/backend/app/models/memory_extraction.py b/backend/app/models/memory_extraction.py index 9224ff7..2311127 100644 --- a/backend/app/models/memory_extraction.py +++ b/backend/app/models/memory_extraction.py @@ -1,20 +1,42 @@ from __future__ import annotations +from typing import Literal from uuid import UUID -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator from .memory import MemoryListResponse, MemorySummaryResponse +MemoryExtractionMethod = Literal["rule_based", "llm"] + + +class LlmAnalysisOptions(BaseModel): + api_key: str | None = Field(default=None, min_length=1) + base_url: str | None = Field(default=None, min_length=1) + model: str | None = Field(default=None, min_length=1) + provider: str | None = Field(default="openai-compatible", min_length=1) + temperature: float | None = Field(default=None, ge=0, le=2) + max_tokens: int | None = Field(default=None, ge=128, le=8000) + + @field_validator("api_key", "base_url", "model", "provider", mode="before") + @classmethod + def _blank_to_none(cls, value: object) -> object: + if isinstance(value, str) and not value.strip(): + return None + return value + class MemoryExtractionFromChunksRequest(BaseModel): workspace_id: UUID chunk_ids: list[UUID] = Field(min_length=1) max_candidates: int = Field(default=10, ge=1, le=50) + method: MemoryExtractionMethod = "rule_based" + llm: LlmAnalysisOptions | None = None class MemoryExtractionResponse(BaseModel): workspace_id: UUID + method: MemoryExtractionMethod created_count: int candidates: list[MemorySummaryResponse] diff --git a/backend/app/models/qa.py b/backend/app/models/qa.py index 8730591..71667a9 100644 --- a/backend/app/models/qa.py +++ b/backend/app/models/qa.py @@ -33,6 +33,9 @@ class AnswerResponse(BaseModel): citation_map: dict[str, Any] token_count: int token_budget: int + prompt_tokens: int | None = None + completion_tokens: int | None = None + total_tokens: int | None = None selected_memories: list[dict[str, Any]] supporting_evidence: list[dict[str, Any]] created_at: datetime | None = None diff --git a/backend/app/services/chunking.py b/backend/app/services/chunking.py index ae91f3e..49ceba5 100644 --- a/backend/app/services/chunking.py +++ b/backend/app/services/chunking.py @@ -19,6 +19,10 @@ def build_chunks( overlap_lines: int = 1, ) -> list[ChunkDraft]: lines = raw_text.splitlines() or [raw_text] + natural_chunks = _line_level_chunks(lines, max_chars=max_chars) + if natural_chunks: + return natural_chunks + chunks: list[ChunkDraft] = [] index = 0 chunk_no = 1 @@ -60,5 +64,27 @@ def build_chunks( return chunks +def _line_level_chunks(lines: list[str], *, max_chars: int) -> list[ChunkDraft]: + chunks: list[ChunkDraft] = [] + chunk_no = 1 + for index, line in enumerate(lines): + text = line.strip() + if not text: + continue + if len(text) > max_chars: + return [] + chunks.append( + ChunkDraft( + chunk_no=chunk_no, + chunk_text=text, + start_line=index + 1, + end_line=index + 1, + token_count=_estimate_token_count(text), + ) + ) + chunk_no += 1 + return chunks + + def _estimate_token_count(text: str) -> int: return max(1, len(text.split())) diff --git a/backend/app/services/context_pack_service.py b/backend/app/services/context_pack_service.py index 10c0632..3ae8dd7 100644 --- a/backend/app/services/context_pack_service.py +++ b/backend/app/services/context_pack_service.py @@ -106,6 +106,7 @@ def build_markdown( "ref": memory_ref, "memory_id": str(memory.memory_id), "memory_type": memory.memory_type, + "canonical_text": memory.canonical_text, "score": memory.score, "selection_reason": selection_reason, "status": memory.status, diff --git a/backend/app/services/embedding_service.py b/backend/app/services/embedding_service.py index 12021e2..db6a5b7 100644 --- a/backend/app/services/embedding_service.py +++ b/backend/app/services/embedding_service.py @@ -4,6 +4,7 @@ import json import math import re +import time from dataclasses import dataclass from typing import Protocol from uuid import UUID @@ -185,12 +186,18 @@ def __init__( api_key: str, base_url: str = "https://api.siliconflow.cn/v1", timeout: float = 30.0, + max_attempts: int = 3, + retry_backoff_seconds: float = 0.25, ) -> None: if not api_key: raise ValueError("SILICONFLOW_API_KEY is required for SiliconFlow embeddings.") + if max_attempts < 1: + raise ValueError("max_attempts must be at least 1.") self._api_key = api_key self._base_url = base_url.rstrip("/") self._timeout = timeout + self._max_attempts = max_attempts + self._retry_backoff_seconds = retry_backoff_seconds def embed(self, payload: EmbeddingGenerateRequest) -> EmbeddingGenerateResponse: request_body: dict[str, object] = { @@ -201,16 +208,7 @@ def embed(self, payload: EmbeddingGenerateRequest) -> EmbeddingGenerateResponse: if payload.dimension > 0: request_body["dimensions"] = payload.dimension - with httpx.Client(timeout=self._timeout) as client: - response = client.post( - f"{self._base_url}/embeddings", - headers={ - "Authorization": f"Bearer {self._api_key}", - "Content-Type": "application/json", - }, - json=request_body, - ) - response.raise_for_status() + response = self._post_with_retry(request_body) data = response.json() embedding = data.get("data", [{}])[0].get("embedding") if not isinstance(embedding, list): @@ -224,6 +222,34 @@ def embed(self, payload: EmbeddingGenerateRequest) -> EmbeddingGenerateResponse: text_hash=hashlib.sha256(payload.text.encode("utf-8")).hexdigest(), ) + def _post_with_retry(self, request_body: dict[str, object]) -> httpx.Response: + with httpx.Client(timeout=self._timeout) as client: + for attempt in range(self._max_attempts): + try: + response = client.post( + f"{self._base_url}/embeddings", + headers={ + "Authorization": f"Bearer {self._api_key}", + "Content-Type": "application/json", + }, + json=request_body, + ) + response.raise_for_status() + return response + except httpx.HTTPStatusError as exc: + status_code = exc.response.status_code + if status_code != 429 and status_code < 500: + raise + last_error: httpx.HTTPError = exc + except httpx.TransportError as exc: + last_error = exc + + if attempt + 1 == self._max_attempts: + raise last_error + time.sleep(self._retry_backoff_seconds * (2**attempt)) + + raise RuntimeError("Embedding request retry loop exited unexpectedly.") + def cosine_similarity(left: list[float], right: list[float]) -> float: if len(left) != len(right) or not left: diff --git a/backend/app/services/governance_service.py b/backend/app/services/governance_service.py index 795831c..2dbc5df 100644 --- a/backend/app/services/governance_service.py +++ b/backend/app/services/governance_service.py @@ -77,6 +77,8 @@ class PolicyNotFoundError(Exception): class GovernanceRepository(Protocol): + # Governance is grouped behind one repository because policy, audit, conflict, + # forget requests, and timeline all read/write the same lifecycle tables. def create_policy(self, payload: PolicyCreateRequest) -> PolicyResponse: ... @@ -735,6 +737,8 @@ def list_audit_lifecycle( with self._database.connection() as conn: with conn.cursor() as cur: + # Lifecycle view is built at query time from audit, revisions, + # conflict records, and forget requests so no duplicate log table is needed. cur.execute( """ SELECT ts, kind, payload diff --git a/backend/app/services/graph_service.py b/backend/app/services/graph_service.py index 3dda72d..bb2ad8a 100644 --- a/backend/app/services/graph_service.py +++ b/backend/app/services/graph_service.py @@ -458,6 +458,8 @@ def __init__(self, settings: Settings) -> None: self._driver_instance = None def health(self) -> GraphHealthResponse: + # Neo4j is optional. Health reports configuration/availability instead of + # failing the whole app so PostgreSQL graph preview can remain usable. if not self.settings.neo4j_enabled: return GraphHealthResponse( enabled=False, @@ -499,6 +501,8 @@ def sync(self, graph: GraphResponse) -> GraphSyncResponse: edge_groups = _group_edges_by_relation_type(graph.edges) with driver.session(database=self.settings.neo4j_database) as session: + # Sync is snapshot-based for demo scale: replace one workspace graph, + # then batch-create nodes and grouped relationships with UNWIND. session.run( "MATCH (n:MemoryBaseNode {workspace_id: $workspace_id}) DETACH DELETE n", workspace_id=workspace_id, @@ -677,6 +681,8 @@ def _filter_graph( workspace_id: UUID, agent_id: UUID | None, ) -> GraphResponse: + # Graph visibility reuses the repository visibility policy: memory nodes + # outside the caller's view are removed before edges and isolated nodes are returned. visible_memory_ids = self.visibility_repository.list_visible_memory_ids( workspace_id, agent_id, diff --git a/backend/app/services/llm_analysis.py b/backend/app/services/llm_analysis.py new file mode 100644 index 0000000..6fc5814 --- /dev/null +++ b/backend/app/services/llm_analysis.py @@ -0,0 +1,408 @@ +"""Optional LLM-backed candidate extraction. + +This is the opt-in counterpart to the rule-based extraction path. It calls an +OpenAI-compatible chat-completions endpoint, asks the model to return strict +JSON, and turns that JSON into candidate-memory drafts. The output stays +compatible with the rule-based pipeline: every draft is meant to become a +``status='candidate'`` MemoryItem bound to exactly one source chunk and still +needs human approval before it enters the active lifecycle. The model never +writes facts directly, so all responses are validated and clamped here rather +than trusted as-is. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from uuid import UUID + +import httpx + +from ..models.memory import MemoryType +from ..models.memory_extraction import LlmAnalysisOptions + +# Memory types we accept back from the model. Any value outside this set is +# coerced to "fact" so a hallucinated label can never reach the database. +ALLOWED_MEMORY_TYPES: tuple[MemoryType, ...] = ( + "episodic", + "semantic", + "fact", + "profile", + "procedural", + "decision", + "preference", + "task", + "risk", + "constraint", + "policy", + "summary", +) + + +class LlmAnalysisError(Exception): + """Raised for any failure in the LLM analysis path (config, transport, JSON).""" + + +@dataclass(frozen=True, slots=True) +class LlmAnalysisDefaults: + """Fallback options sourced from settings/env (the ``LLM_ANALYSIS_*`` vars). + + ``api_key`` is empty by default: the feature is off until a key is supplied + either here (from env) or per-request. + """ + + api_key: str = "" + base_url: str = "https://api.openai.com/v1" + model: str = "gpt-4o-mini" + provider: str = "openai-compatible" + temperature: float = 0.1 + max_tokens: int = 1200 + + +@dataclass(frozen=True, slots=True) +class SourceChunkForAnalysis: + """A single source chunk handed to the model, with optional line range.""" + + chunk_id: UUID + chunk_no: int + text: str + start_line: int | None = None + end_line: int | None = None + + +@dataclass(frozen=True, slots=True) +class LlmCandidateDraft: + """One validated candidate memory parsed from the model response.""" + + chunk_id: UUID + canonical_text: str + memory_type: MemoryType + summary: str | None + confidence: float + importance: int + + +@dataclass(frozen=True, slots=True) +class ResolvedLlmAnalysisOptions: + """Effective options after merging per-request overrides with defaults.""" + + api_key: str + base_url: str + model: str + provider: str + temperature: float + max_tokens: int + + +class OpenAICompatibleAnalysisClient: + """Thin client over any OpenAI-compatible ``/chat/completions`` endpoint.""" + + def __init__(self, *, timeout: float = 60.0) -> None: + self._timeout = timeout + + def analyze( + self, + *, + chunks: list[SourceChunkForAnalysis], + max_candidates: int, + options: ResolvedLlmAnalysisOptions, + ) -> list[LlmCandidateDraft]: + """Send chunks to the model and return validated candidate drafts.""" + if not chunks: + return [] + content = self._complete_json( + system_prompt=_system_prompt(), + user_prompt=_user_prompt(chunks=chunks, max_candidates=max_candidates), + options=options, + ) + return parse_llm_candidates(content, chunks=chunks, max_candidates=max_candidates) + + def _complete_json( + self, + *, + system_prompt: str, + user_prompt: str, + options: ResolvedLlmAnalysisOptions, + ) -> str: + """POST one chat completion and return the raw message content string.""" + with httpx.Client(timeout=self._timeout) as client: + response = client.post( + f"{options.base_url.rstrip('/')}/chat/completions", + headers={ + "Authorization": f"Bearer {options.api_key}", + "Content-Type": "application/json", + }, + json={ + "model": options.model, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + "temperature": options.temperature, + "max_tokens": options.max_tokens, + # Ask the endpoint for a JSON object; we still parse defensively + # below since not every compatible provider honors this. + "response_format": {"type": "json_object"}, + }, + ) + try: + response.raise_for_status() + except httpx.HTTPStatusError as exc: + raise LlmAnalysisError(f"LLM analysis request failed: {exc}") from exc + data = response.json() + # Pull choices[0].message.content without assuming the shape is present. + content = data.get("choices", [{}])[0].get("message", {}).get("content") + if not isinstance(content, str) or not content.strip(): + raise LlmAnalysisError("LLM analysis response did not contain JSON content.") + return content + + +def resolve_llm_options( + requested: LlmAnalysisOptions | None, + defaults: LlmAnalysisDefaults, +) -> ResolvedLlmAnalysisOptions: + """Merge per-request options over env defaults and validate the result. + + Per-request values win when set, otherwise the env-backed defaults apply. + A missing API key / base URL / model raises ``LlmAnalysisError`` so callers + fail fast instead of sending an unauthenticated request. + """ + api_key = (requested.api_key if requested and requested.api_key else defaults.api_key).strip() + base_url = ( + requested.base_url if requested and requested.base_url else defaults.base_url + ).strip() + model = (requested.model if requested and requested.model else defaults.model).strip() + provider = ( + requested.provider if requested and requested.provider else defaults.provider + ).strip() + temperature = ( + requested.temperature + if requested and requested.temperature is not None + else defaults.temperature + ) + max_tokens = ( + requested.max_tokens + if requested and requested.max_tokens is not None + else defaults.max_tokens + ) + if not api_key: + raise LlmAnalysisError( + "LLM analysis requires an API key in request.llm.api_key or LLM_ANALYSIS_API_KEY." + ) + if not base_url: + raise LlmAnalysisError("LLM analysis requires a base URL.") + if not model: + raise LlmAnalysisError("LLM analysis requires a model name.") + return ResolvedLlmAnalysisOptions( + api_key=api_key, + base_url=base_url.rstrip("/"), + model=model, + provider=provider or "openai-compatible", + temperature=float(temperature), + max_tokens=int(max_tokens), + ) + + +def parse_llm_candidates( + content: str, + *, + chunks: list[SourceChunkForAnalysis], + max_candidates: int, +) -> list[LlmCandidateDraft]: + """Parse model JSON into validated drafts, dropping anything unusable. + + Every candidate is anchored to a real source chunk, deduplicated by + normalized text, length-filtered, and has its type/confidence/importance + coerced into valid ranges. Malformed entries are skipped rather than raising, + so one bad item never discards the whole response. + """ + try: + payload = json.loads(_strip_json_fence(content)) + except json.JSONDecodeError as exc: + raise LlmAnalysisError(f"LLM analysis returned invalid JSON: {exc}") from exc + + raw_candidates = payload.get("candidates") if isinstance(payload, dict) else None + if not isinstance(raw_candidates, list): + raise LlmAnalysisError("LLM analysis JSON must contain a candidates array.") + + chunk_ids = {str(chunk.chunk_id): chunk.chunk_id for chunk in chunks} + drafts: list[LlmCandidateDraft] = [] + seen_text: set[str] = set() + for raw in raw_candidates: + if not isinstance(raw, dict): + continue + # Drop candidates we cannot anchor back to one of the input chunks. + chunk_id = _resolve_chunk_id(raw.get("chunk_id"), raw.get("chunk_no"), chunks, chunk_ids) + if chunk_id is None: + continue + text = _coerce_text(raw.get("canonical_text") or raw.get("text")) + if len(text) < 12: # too short to be a meaningful memory + continue + # Deduplicate on case/whitespace-insensitive text. + normalized = " ".join(text.lower().split()) + if normalized in seen_text: + continue + seen_text.add(normalized) + drafts.append( + LlmCandidateDraft( + chunk_id=chunk_id, + canonical_text=text[:1000], + memory_type=_coerce_memory_type(raw.get("memory_type") or raw.get("type")), + summary=_optional_summary(raw.get("summary"), text), + confidence=_coerce_float(raw.get("confidence"), default=0.7, minimum=0, maximum=1), + importance=_coerce_int(raw.get("importance"), default=3, minimum=1, maximum=5), + ) + ) + if len(drafts) >= max_candidates: + break + return drafts + + +def _system_prompt() -> str: + # Pins the model to the rule-based pipeline's contract: JSON only, candidate + # semantics, allowed types, and the same confidence/importance conventions. + return ( + "You are MemoryBase's optional LLM analysis pipeline. Extract candidate memories " + "from source chunks for later human review. Keep behavior compatible with the " + "rule-based pipeline: every output item must become a status='candidate' " + "MemoryItem linked to exactly one source chunk. Return only JSON. Do not include " + "markdown or explanations. Use the original language of the source text. " + "Prefer concise, atomic memories that preserve decisions, constraints, policies, " + "risks, tasks, preferences, durable facts, procedures, and summaries. Avoid " + "duplicates and avoid unsupported inference. Memory type must be one of: " + f"{', '.join(ALLOWED_MEMORY_TYPES)}. Confidence is 0..1; use 0.65 for weak " + "rule-like extraction, 0.75-0.9 when the text directly supports the candidate. " + "Importance is 1..5; use 4 for decisions, constraints, policies, and risks; " + "use 3 for ordinary facts/tasks; use 5 only for central project commitments." + ) + + +def _user_prompt(*, chunks: list[SourceChunkForAnalysis], max_candidates: int) -> str: + # Serializes each chunk as one JSON line and spells out the exact response + # shape, so the model echoes back chunk_id values we can re-anchor against. + chunk_lines = [] + for chunk in chunks: + line_range = ( + f"lines {chunk.start_line}-{chunk.end_line}" + if chunk.start_line is not None and chunk.end_line is not None + else "lines unknown" + ) + chunk_lines.append( + json.dumps( + { + "chunk_id": str(chunk.chunk_id), + "chunk_no": chunk.chunk_no, + "line_range": line_range, + "text": chunk.text, + }, + ensure_ascii=False, + ) + ) + return ( + "Extract at most " + f"{max_candidates} candidate memories from these chunks.\n\n" + "Return this exact JSON shape:\n" + "{\n" + ' "candidates": [\n' + " {\n" + ' "chunk_id": "uuid copied from source chunk",\n' + ' "canonical_text": "atomic memory text supported by that chunk",\n' + ' "memory_type": ' + '"decision|constraint|policy|risk|task|preference|fact|summary|' + 'semantic|procedural|episodic|profile",\n' + ' "summary": "short label, max 80 chars",\n' + ' "confidence": 0.0,\n' + ' "importance": 3\n' + " }\n" + " ]\n" + "}\n\n" + "Source chunks:\n" + + "\n".join(chunk_lines) + ) + + +def _strip_json_fence(content: str) -> str: + # Some providers wrap JSON in a ```json ... ``` markdown fence; remove it. + stripped = content.strip() + if stripped.startswith("```"): + stripped = re.sub(r"^```(?:json)?\s*", "", stripped, flags=re.IGNORECASE) + stripped = re.sub(r"\s*```$", "", stripped) + return stripped.strip() + + +def _resolve_chunk_id( + raw_chunk_id: object, + raw_chunk_no: object, + chunks: list[SourceChunkForAnalysis], + chunk_ids: dict[str, UUID], +) -> UUID | None: + # Prefer an exact chunk_id match, then fall back to chunk_no. If neither is + # given but only one chunk was sent, attribute it to that chunk; otherwise + # we cannot anchor the candidate and return None so the caller drops it. + if isinstance(raw_chunk_id, str) and raw_chunk_id in chunk_ids: + return chunk_ids[raw_chunk_id] + chunk_no = _coerce_int_or_none(raw_chunk_no) + if chunk_no is not None: + for chunk in chunks: + if chunk.chunk_no == chunk_no: + return chunk.chunk_id + return chunks[0].chunk_id if len(chunks) == 1 else None + + +# The helpers below defensively coerce untrusted model output: collapse text +# whitespace, cap summary length, validate the memory type, and clamp numeric +# fields into their allowed ranges so a bad value never reaches the database. + + +def _coerce_text(value: object) -> str: + if not isinstance(value, str): + return "" + return " ".join(value.split()) + + +def _optional_summary(value: object, fallback_text: str) -> str: + summary = _coerce_text(value) + if not summary: + summary = fallback_text + if len(summary) <= 80: + return summary + return summary[:77].rstrip() + "..." + + +def _coerce_memory_type(value: object) -> MemoryType: + if isinstance(value, str): + normalized = value.strip().lower().replace("-", "_") + if normalized in ALLOWED_MEMORY_TYPES: + return normalized # type: ignore[return-value] + return "fact" + + +def _coerce_float(value: object, *, default: float, minimum: float, maximum: float) -> float: + try: + candidate = float(value) + except (TypeError, ValueError): + candidate = default + return min(max(candidate, minimum), maximum) + + +def _coerce_int(value: object, *, default: int, minimum: int, maximum: int) -> int: + candidate = _coerce_int_or_none(value) + if candidate is None: + candidate = default + return min(max(candidate, minimum), maximum) + + +def _coerce_int_or_none(value: object) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float) and value.is_integer(): + return int(value) + if isinstance(value, str): + try: + return int(value) + except ValueError: + return None + return None diff --git a/backend/app/services/llm_service.py b/backend/app/services/llm_service.py index 3fad12e..7f5cdf0 100644 --- a/backend/app/services/llm_service.py +++ b/backend/app/services/llm_service.py @@ -24,6 +24,9 @@ class ChatCompletionResponse: content: str provider: str model: str + prompt_tokens: int | None = None + completion_tokens: int | None = None + total_tokens: int | None = None class ChatProvider(Protocol): @@ -77,10 +80,14 @@ def complete(self, payload: ChatCompletionRequest) -> ChatCompletionResponse: content = data.get("choices", [{}])[0].get("message", {}).get("content") if not isinstance(content, str): raise ValueError(f"{self.provider_name} response did not contain message content.") + usage = data.get("usage") if isinstance(data.get("usage"), dict) else {} return ChatCompletionResponse( content=content.strip(), provider=self.provider_name, model=self.model, + prompt_tokens=_optional_int(usage.get("prompt_tokens")), + completion_tokens=_optional_int(usage.get("completion_tokens")), + total_tokens=_optional_int(usage.get("total_tokens")), ) @@ -124,6 +131,9 @@ def answer(self, payload: AnswerRequest) -> AnswerResponse: citation_map=context.citation_map, token_count=context.token_count, token_budget=context.token_budget, + prompt_tokens=completion.prompt_tokens, + completion_tokens=completion.completion_tokens, + total_tokens=completion.total_tokens, selected_memories=context.selected_memories, supporting_evidence=context.supporting_evidence, created_at=recall.created_at, @@ -134,10 +144,26 @@ def _system_prompt() -> str: return ( "You are MemoryBase's grounded answer generator. Answer in the user's language. " "Use only the supplied MemoryBase context as factual support. If the context does " - "not contain enough evidence, say you do not know. Prefer concise answers. " + "not contain enough evidence, say you do not know. When conflicting values have " + "explicit dates, versions, or sequence numbers, use the latest value and treat " + "older values as history; do not refuse only because an older value differs. " + "For labels formatted as [Memory sequence N], compare N for facts about the same " + "subject and relation; the highest N is authoritative even when its retrieval " + "score is lower or it appears later in the context pack. " + "Prefer concise answers. " "When using recalled memories or evidence, cite refs like [M1] or [E1]." ) def _user_prompt(*, query_text: str, context: str) -> str: return "Question:\n" f"{query_text}\n\n" "MemoryBase context:\n" f"{context}\n\n" "Answer:" + + +def _optional_int(value: object) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float) and value.is_integer(): + return int(value) + return None diff --git a/backend/app/services/memory_extraction_service.py b/backend/app/services/memory_extraction_service.py index 3c2a88b..91dedbd 100644 --- a/backend/app/services/memory_extraction_service.py +++ b/backend/app/services/memory_extraction_service.py @@ -2,7 +2,7 @@ import json import re -from dataclasses import dataclass +from dataclasses import dataclass, field from uuid import UUID, uuid4 from ..core.database import Database @@ -12,9 +12,19 @@ MemoryEvidenceInput, MemoryListResponse, MemorySummaryResponse, + MemoryType, MemoryUpdateRequest, ) from ..models.memory_extraction import MemoryExtractionFromChunksRequest +from .llm_analysis import ( + LlmAnalysisDefaults, + LlmAnalysisError, + LlmCandidateDraft, + OpenAICompatibleAnalysisClient, + ResolvedLlmAnalysisOptions, + SourceChunkForAnalysis, + resolve_llm_options, +) from .memory_service import MemoryService @@ -26,12 +36,16 @@ class MemoryExtractionValidationError(Exception): class MemoryExtractionService: database: Database memory_service: MemoryService + llm_client: OpenAICompatibleAnalysisClient | None = None + llm_defaults: LlmAnalysisDefaults = field(default_factory=LlmAnalysisDefaults) def extract_from_chunks( self, payload: MemoryExtractionFromChunksRequest, actor: ActorContext, ) -> list[MemorySummaryResponse]: + # Extraction writes candidate memories, not active facts. Human/admin review + # promotes or rejects them through the same memory lifecycle used elsewhere. chunk_rows = self._fetch_chunks(payload.workspace_id, payload.chunk_ids) if len(chunk_rows) != len(set(payload.chunk_ids)): raise MemoryExtractionValidationError( @@ -39,75 +53,166 @@ def extract_from_chunks( ) run_id = uuid4() + llm_options: ResolvedLlmAnalysisOptions | None = None + if payload.method == "llm": + try: + llm_options = resolve_llm_options(payload.llm, self.llm_defaults) + except LlmAnalysisError as exc: + raise MemoryExtractionValidationError(str(exc)) from exc + + # Run-level audit lets the demo answer "which chunks produced these + # candidate memories" without introducing extra analysis-run tables. + start_json: dict[str, object] = { + "run_id": str(run_id), + "method": _audit_method(payload.method), + "requested_chunk_ids": [str(chunk_id) for chunk_id in payload.chunk_ids], + "max_candidates": payload.max_candidates, + } + if llm_options is not None: + start_json["llm"] = { + "provider": llm_options.provider, + "base_url": llm_options.base_url, + "model": llm_options.model, + "temperature": llm_options.temperature, + "max_tokens": llm_options.max_tokens, + } self._insert_run_audit( workspace_id=payload.workspace_id, actor=actor, action_type="memory_extraction.run.start", + after_json=start_json, + ) + + try: + drafts = ( + self._llm_candidate_drafts( + chunk_rows, + max_candidates=payload.max_candidates, + options=llm_options, + ) + if payload.method == "llm" + else self._rule_based_candidate_drafts( + chunk_rows, + max_candidates=payload.max_candidates, + ) + ) + except LlmAnalysisError as exc: + raise MemoryExtractionValidationError(str(exc)) from exc + + row_by_chunk_id = {str(row["chunk_id"]): row for row in chunk_rows} + candidates = [ + self._create_candidate( + draft, + row_by_chunk_id=row_by_chunk_id, + workspace_id=payload.workspace_id, + method=payload.method, + actor=actor, + ) + for draft in drafts + ] + self._insert_run_audit( + workspace_id=payload.workspace_id, + actor=actor, + action_type="memory_extraction.run.complete", after_json={ "run_id": str(run_id), - "method": "rule-based", - "requested_chunk_ids": [str(chunk_id) for chunk_id in payload.chunk_ids], - "max_candidates": payload.max_candidates, + "candidate_count": len(candidates), + "candidate_memory_ids": [str(candidate.memory_id) for candidate in candidates], }, ) + return candidates - candidates: list[MemorySummaryResponse] = [] + def _rule_based_candidate_drafts( + self, + chunk_rows: list[dict[str, object]], + *, + max_candidates: int, + ) -> list[LlmCandidateDraft]: + drafts: list[LlmCandidateDraft] = [] seen_text: set[str] = set() for row in chunk_rows: for text in _candidate_texts(str(row["chunk_text"])): - normalized = text.lower() + normalized = " ".join(text.lower().split()) if normalized in seen_text: continue seen_text.add(normalized) - candidates.append( - self.memory_service.create_memory( - MemoryCreateRequest( - workspace_id=payload.workspace_id, - created_from_doc_id=row["doc_id"], - memory_type=_classify_memory_type(text), - canonical_text=text, - summary=_summary(text), - confidence=0.65, - importance=_importance(text), - status="candidate", - access_level="project", - evidence=[ - MemoryEvidenceInput( - chunk_id=row["chunk_id"], - evidence_role="source", - weight=1.0, - note="Generated by rule-based memory extraction.", - ) - ], - ), - actor, + drafts.append( + LlmCandidateDraft( + chunk_id=_uuid_value(row["chunk_id"]), + canonical_text=text, + memory_type=_classify_memory_type(text), + summary=_summary(text), + confidence=0.65, + importance=_importance(text), ) ) - if len(candidates) >= payload.max_candidates: - self._insert_run_audit( - workspace_id=payload.workspace_id, - actor=actor, - action_type="memory_extraction.run.complete", - after_json={ - "run_id": str(run_id), - "candidate_count": len(candidates), - "candidate_memory_ids": [ - str(candidate.memory_id) for candidate in candidates - ], - }, + if len(drafts) >= max_candidates: + return drafts + return drafts + + def _llm_candidate_drafts( + self, + chunk_rows: list[dict[str, object]], + *, + max_candidates: int, + options: ResolvedLlmAnalysisOptions | None, + ) -> list[LlmCandidateDraft]: + if options is None: + raise LlmAnalysisError("LLM analysis options were not resolved.") + client = self.llm_client or OpenAICompatibleAnalysisClient() + chunks = [ + SourceChunkForAnalysis( + chunk_id=_uuid_value(row["chunk_id"]), + chunk_no=int(row["chunk_no"]), + text=str(row["chunk_text"]), + start_line=_optional_int(row.get("start_line")), + end_line=_optional_int(row.get("end_line")), + ) + for row in chunk_rows + ] + return client.analyze( + chunks=chunks, + max_candidates=max_candidates, + options=options, + ) + + def _create_candidate( + self, + draft: LlmCandidateDraft, + *, + row_by_chunk_id: dict[str, dict[str, object]], + workspace_id: UUID, + method: str, + actor: ActorContext, + ) -> MemorySummaryResponse: + row = row_by_chunk_id[str(draft.chunk_id)] + note = ( + "Generated by LLM-backed memory extraction." + if method == "llm" + else "Generated by rule-based memory extraction." + ) + return self.memory_service.create_memory( + MemoryCreateRequest( + workspace_id=workspace_id, + created_from_doc_id=row["doc_id"], + memory_type=draft.memory_type, + canonical_text=draft.canonical_text, + summary=draft.summary, + confidence=draft.confidence, + importance=draft.importance, + status="candidate", + access_level="project", + evidence=[ + MemoryEvidenceInput( + chunk_id=draft.chunk_id, + evidence_role="source", + weight=1.0, + note=note, ) - return candidates - self._insert_run_audit( - workspace_id=payload.workspace_id, - actor=actor, - action_type="memory_extraction.run.complete", - after_json={ - "run_id": str(run_id), - "candidate_count": len(candidates), - "candidate_memory_ids": [str(candidate.memory_id) for candidate in candidates], - }, + ], + ), + actor, ) - return candidates def list_candidates( self, @@ -165,7 +270,13 @@ def _fetch_chunks( with conn.cursor() as cur: cur.execute( """ - SELECT sc.chunk_id, sc.doc_id, sc.chunk_text + SELECT + sc.chunk_id, + sc.doc_id, + sc.chunk_no, + sc.chunk_text, + sc.start_line, + sc.end_line FROM source_chunk sc JOIN source_document sd ON sd.doc_id = sc.doc_id WHERE sd.workspace_id = %(workspace_id)s @@ -233,7 +344,7 @@ def _candidate_texts(chunk_text: str) -> list[str]: return [part[:1000] for part in parts if len(part) >= 12] -def _classify_memory_type(text: str) -> str: +def _classify_memory_type(text: str) -> MemoryType: lowered = text.lower() if any(term in lowered for term in ("decided", "decision", "选择", "决定")): return "decision" @@ -262,3 +373,23 @@ def _summary(text: str) -> str: if len(compact) <= 80: return compact return compact[:77].rstrip() + "..." + + +def _audit_method(method: str) -> str: + return "rule-based" if method == "rule_based" else method + + +def _optional_int(value: object) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float) and value.is_integer(): + return int(value) + return None + + +def _uuid_value(value: object) -> UUID: + if isinstance(value, UUID): + return value + return UUID(str(value)) diff --git a/backend/app/services/memory_service.py b/backend/app/services/memory_service.py index dd63ab2..e47e9db 100644 --- a/backend/app/services/memory_service.py +++ b/backend/app/services/memory_service.py @@ -8,6 +8,8 @@ from ..core.database import Database from ..models.memory import ( ActorContext, + MemoryBatchCreateRequest, + MemoryBatchCreateResponse, MemoryCreateRequest, MemoryDeleteResponse, MemoryDetailResponse, @@ -19,6 +21,8 @@ from .chunking import _estimate_token_count from .tokenizer import build_search_text +# Lifecycle rules live in service code while database triggers record revisions +# and audit rows. This split keeps invalid transitions out before SQL executes. ALLOWED_STATUS_TRANSITIONS = { "candidate": {"active", "rejected"}, "active": {"superseded", "archived", "conflicted", "forgotten"}, @@ -42,6 +46,13 @@ def create_memory( ) -> MemorySummaryResponse: ... + def create_memories( + self, + payloads: list[MemoryCreateRequest], + actor: ActorContext | None = None, + ) -> list[MemorySummaryResponse]: + ... + def list_memories( self, *, @@ -80,9 +91,26 @@ class MemoryService: def create_memory( self, payload: MemoryCreateRequest, actor: ActorContext | None = None ) -> MemorySummaryResponse: - _validate_initial_status(payload.status) + _validate_create_payload(payload) return self.repository.create_memory(payload, actor) + def create_memories( + self, + payload: MemoryBatchCreateRequest, + actor: ActorContext | None = None, + ) -> MemoryBatchCreateResponse: + workspace_ids = {item.workspace_id for item in payload.items} + if len(workspace_ids) != 1: + raise MemoryValidationError("batch memory items must use one workspace_id") + for item in payload.items: + _validate_create_payload(item) + items = self.repository.create_memories(payload.items, actor) + return MemoryBatchCreateResponse( + workspace_id=next(iter(workspace_ids)), + count=len(items), + items=items, + ) + def list_memories( self, *, @@ -138,113 +166,143 @@ def __init__(self, database: Database) -> None: def create_memory( self, payload: MemoryCreateRequest, actor: ActorContext | None = None ) -> MemorySummaryResponse: + return self.create_memories([payload], actor)[0] + + def create_memories( + self, + payloads: list[MemoryCreateRequest], + actor: ActorContext | None = None, + ) -> list[MemorySummaryResponse]: actor = actor or ActorContext(actor_type="system", revision_reason="initial create") with self._database.connection() as conn: with conn.cursor() as cur: self._set_actor_context(cur, actor) - evidence_items = list(payload.evidence) - created_from_doc_id = payload.created_from_doc_id - if not evidence_items and actor.actor_type == "agent": - inline_evidence = self._create_inline_evidence_chunk(cur, payload, actor) - evidence_items = [ - MemoryEvidenceInput( - chunk_id=inline_evidence["chunk_id"], - evidence_role="source", - weight=1.0, - note="Inline agent note created by MemoryBase CLI.", - ) - ] - if created_from_doc_id is None: - created_from_doc_id = inline_evidence["doc_id"] + memory_ids = [self._insert_memory(cur, payload, actor) for payload in payloads] + rows = self._fetch_memory_summaries(cur, memory_ids) + conn.commit() + rows_by_id = {row["memory_id"]: row for row in rows} + if len(rows_by_id) != len(memory_ids): + raise RuntimeError("one or more created memories could not be loaded") + return [MemorySummaryResponse(**rows_by_id[memory_id]) for memory_id in memory_ids] - cur.execute( - """ - INSERT INTO memory_item ( - workspace_id, - created_from_doc_id, - memory_type, - canonical_text, - summary, - search_text_zh, - confidence, - importance, - status, - access_level, - owner_user_id, - owner_agent_id - ) - VALUES ( - %(workspace_id)s, - %(created_from_doc_id)s, - %(memory_type)s, - %(canonical_text)s, - %(summary)s, - %(search_text_zh)s, - %(confidence)s, - %(importance)s, - %(status)s, - %(access_level)s, - %(owner_user_id)s, - %(owner_agent_id)s - ) - RETURNING memory_id - """, - { - "workspace_id": payload.workspace_id, - "created_from_doc_id": created_from_doc_id, - "memory_type": payload.memory_type, - "canonical_text": payload.canonical_text, - "summary": payload.summary, - "search_text_zh": build_search_text( - payload.canonical_text, - payload.summary, - ), - "confidence": payload.confidence, - "importance": payload.importance, - "status": payload.status, - "access_level": payload.access_level, - "owner_user_id": payload.owner_user_id, - "owner_agent_id": payload.owner_agent_id, - }, + def _insert_memory( + self, + cur, + payload: MemoryCreateRequest, + actor: ActorContext, + ) -> UUID: + if payload.supersedes_memory_id is not None: + cur.execute( + """ + SELECT status + FROM memory_item + WHERE memory_id = %(memory_id)s + AND workspace_id = %(workspace_id)s + FOR UPDATE + """, + { + "memory_id": payload.supersedes_memory_id, + "workspace_id": payload.workspace_id, + }, + ) + superseded = cur.fetchone() + if superseded is None: + raise MemoryValidationError( + f"superseded memory {payload.supersedes_memory_id} does not belong " + f"to workspace {payload.workspace_id}" + ) + if superseded["status"] not in {"active", "conflicted"}: + raise MemoryValidationError( + f"memory {payload.supersedes_memory_id} with status " + f"{superseded['status']} cannot be superseded" ) - row = cur.fetchone() - if row is None: - raise RuntimeError("failed to create memory") - memory_id = row["memory_id"] - self._validate_evidence_chunks(cur, payload.workspace_id, evidence_items) - for evidence in evidence_items: - cur.execute( - """ - INSERT INTO memory_evidence ( - memory_id, - chunk_id, - evidence_role, - weight, - note - ) - VALUES ( - %(memory_id)s, - %(chunk_id)s, - %(evidence_role)s, - %(weight)s, - %(note)s - ) - """, - { - "memory_id": memory_id, - "chunk_id": evidence.chunk_id, - "evidence_role": evidence.evidence_role, - "weight": evidence.weight, - "note": evidence.note, - }, - ) + evidence_items = list(payload.evidence) + created_from_doc_id = payload.created_from_doc_id + if not evidence_items and actor.actor_type == "agent": + # Agent-created memories still receive provenance: an inline source + # document/chunk is created so memory_evidence is never empty. + inline_evidence = self._create_inline_evidence_chunk(cur, payload, actor) + evidence_items = [ + MemoryEvidenceInput( + chunk_id=inline_evidence["chunk_id"], + evidence_role="source", + weight=1.0, + note="Inline agent note created by MemoryBase CLI.", + ) + ] + if created_from_doc_id is None: + created_from_doc_id = inline_evidence["doc_id"] - conn.commit() - summary = self._get_memory_summary(memory_id) - if summary is None: - raise RuntimeError("created memory cannot be loaded") - return summary + cur.execute( + """ + INSERT INTO memory_item ( + workspace_id, created_from_doc_id, memory_type, canonical_text, + summary, search_text_zh, confidence, importance, status, + access_level, owner_user_id, owner_agent_id, valid_from + ) + VALUES ( + %(workspace_id)s, %(created_from_doc_id)s, %(memory_type)s, + %(canonical_text)s, %(summary)s, %(search_text_zh)s, + %(confidence)s, %(importance)s, %(status)s, %(access_level)s, + %(owner_user_id)s, %(owner_agent_id)s, + COALESCE(%(valid_from)s, now()) + ) + RETURNING memory_id, valid_from + """, + { + **payload.model_dump( + exclude={"evidence", "supersedes_memory_id"}, + ), + "created_from_doc_id": created_from_doc_id, + "search_text_zh": build_search_text( + payload.canonical_text, + payload.summary, + ), + }, + ) + row = cur.fetchone() + if row is None: + raise RuntimeError("failed to create memory") + memory_id = row["memory_id"] + if payload.supersedes_memory_id is not None: + cur.execute( + """ + UPDATE memory_item + SET status = 'superseded', + valid_to = %(valid_to)s, + superseded_by_memory_id = %(new_memory_id)s + WHERE memory_id = %(old_memory_id)s + AND workspace_id = %(workspace_id)s + """, + { + "valid_to": row["valid_from"], + "new_memory_id": memory_id, + "old_memory_id": payload.supersedes_memory_id, + "workspace_id": payload.workspace_id, + }, + ) + self._validate_evidence_chunks(cur, payload.workspace_id, evidence_items) + for evidence in evidence_items: + cur.execute( + """ + INSERT INTO memory_evidence ( + memory_id, chunk_id, evidence_role, weight, note + ) + VALUES ( + %(memory_id)s, %(chunk_id)s, %(evidence_role)s, + %(weight)s, %(note)s + ) + """, + { + "memory_id": memory_id, + "chunk_id": evidence.chunk_id, + "evidence_role": evidence.evidence_role, + "weight": evidence.weight, + "note": evidence.note, + }, + ) + return memory_id def _set_actor_context(self, cur, actor: ActorContext) -> None: cur.execute( @@ -639,14 +697,25 @@ def delete_memory( def _get_memory_summary( self, memory_id: UUID, workspace_id: UUID | None = None ) -> MemorySummaryResponse | None: + with self._database.connection() as conn: + with conn.cursor() as cur: + row = self._fetch_memory_summary(cur, memory_id, workspace_id) + if row is None: + return None + return MemorySummaryResponse(**row) + + def _fetch_memory_summary( + self, + cur, + memory_id: UUID, + workspace_id: UUID | None = None, + ) -> dict[str, object] | None: workspace_filter = "AND mi.workspace_id = %(workspace_id)s" if workspace_id else "" params: dict[str, object] = {"memory_id": memory_id} if workspace_id: params["workspace_id"] = workspace_id - with self._database.connection() as conn: - with conn.cursor() as cur: - cur.execute( - f""" + cur.execute( + f""" SELECT mi.memory_id, mi.workspace_id, @@ -680,13 +749,54 @@ def _get_memory_summary( mi.current_revision_no, mi.created_at, mi.updated_at - """, - params, - ) - row = cur.fetchone() - if row is None: - return None - return MemorySummaryResponse(**row) + """, + params, + ) + return cur.fetchone() + + def _fetch_memory_summaries( + self, + cur, + memory_ids: list[UUID], + ) -> list[dict[str, object]]: + cur.execute( + """ + SELECT + mi.memory_id, + mi.workspace_id, + mi.created_from_doc_id, + mi.memory_type, + mi.canonical_text, + mi.summary, + mi.confidence, + mi.importance, + mi.status, + mi.access_level, + mi.current_revision_no, + mi.created_at, + mi.updated_at, + COUNT(me.evidence_id) AS evidence_count + FROM memory_item mi + LEFT JOIN memory_evidence me ON me.memory_id = mi.memory_id + WHERE mi.memory_id = ANY(%(memory_ids)s::uuid[]) + GROUP BY + mi.memory_id, + mi.workspace_id, + mi.created_from_doc_id, + mi.memory_type, + mi.canonical_text, + mi.summary, + mi.confidence, + mi.importance, + mi.status, + mi.access_level, + mi.current_revision_no, + mi.created_at, + mi.updated_at + """, + {"memory_ids": memory_ids}, + ) + return cur.fetchall() def _get_memory_detail( self, memory_id: UUID, workspace_id: UUID | None = None @@ -901,3 +1011,9 @@ def _validate_initial_status(status: str) -> None: raise MemoryValidationError( f"illegal initial memory status: {status}. Use active or candidate." ) + + +def _validate_create_payload(payload: MemoryCreateRequest) -> None: + _validate_initial_status(payload.status) + if payload.supersedes_memory_id is not None and payload.status != "active": + raise MemoryValidationError("a superseding memory must start with active status") diff --git a/backend/app/services/recall_service.py b/backend/app/services/recall_service.py index 7539b57..525489a 100644 --- a/backend/app/services/recall_service.py +++ b/backend/app/services/recall_service.py @@ -16,6 +16,9 @@ QUERY_EXPANSION_FILE = ( Path(__file__).resolve().parents[3] / "data" / "recall" / "demo_query_expansions.json" ) + +# The fallback strings are part of the response contract: UI, CLI, and evaluation +# can explain why a requested vector/hybrid path effectively became keyword recall. HYBRID_FALLBACK_REASON = ( "No matching embedding records were available; " "hybrid recall fell back to keyword ranking." @@ -76,6 +79,8 @@ def __init__( self._embedding_dimension = embedding_dimension def execute_recall(self, payload: RecallRequest) -> RecallResponse: + # Recall is intentionally DB-first: apply workspace/status/type/access filters + # before ranking so private or out-of-window memories never enter the candidate set. search_text = _expand_query_text(payload.query_text) keyword_patterns = [f"%{term}%" for term in _keyword_terms(payload.query_text)] filters: list[str] = ["mi.workspace_id = %(workspace_id)s"] @@ -350,6 +355,8 @@ def _merge_vector_rows( keyword_rows: list[dict[str, object]], retrieval_info: RetrievalInfo, ) -> tuple[list[dict[str, object]], RetrievalInfo]: + # Vector recall is an optional second-stage merge over cached embeddings. + # If no usable vectors exist, retrieval_info records the downgrade explicitly. query_embedding = self._embedding_provider.embed( EmbeddingGenerateRequest( text=payload.query_text, diff --git a/backend/tests/test_chunking.py b/backend/tests/test_chunking.py new file mode 100644 index 0000000..d7d5ae6 --- /dev/null +++ b/backend/tests/test_chunking.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from app.services.chunking import build_chunks + + +def test_build_chunks_uses_short_lines_as_reviewable_chunks() -> None: + chunks = build_chunks( + "\n".join( + [ + "We decided to abandon the campus cafeteria system because it was too CRUD-heavy.", + "MemoryBase should use PostgreSQL as the source of truth.", + "Private budget notes must stay hidden from project-only retriever agents.", + ] + ) + ) + + assert [chunk.chunk_no for chunk in chunks] == [1, 2, 3] + assert [chunk.start_line for chunk in chunks] == [1, 2, 3] + assert [chunk.end_line for chunk in chunks] == [1, 2, 3] + assert chunks[0].chunk_text.startswith("We decided to abandon") + assert chunks[1].chunk_text == "MemoryBase should use PostgreSQL as the source of truth." + assert chunks[2].chunk_text.startswith("Private budget notes") + + +def test_build_chunks_falls_back_to_size_based_chunks_for_long_lines() -> None: + chunks = build_chunks("alpha beta gamma delta", max_chars=10, overlap_lines=0) + + assert len(chunks) == 1 + assert chunks[0].chunk_text == "alpha beta gamma delta" diff --git a/backend/tests/test_cli_skeleton.py b/backend/tests/test_cli_skeleton.py index ae789bc..64efd2e 100644 --- a/backend/tests/test_cli_skeleton.py +++ b/backend/tests/test_cli_skeleton.py @@ -131,6 +131,7 @@ def test_cli_help_includes_agent_facing_command_descriptions() -> None: assert result.exit_code == 0 assert "Configure CLI defaults" in result.stdout assert "Render agent context" in result.stdout + assert "Extract candidate memories from chunks" in result.stdout assert "Write conversation messages" in result.stdout assert "Recall governed memories" in result.stdout assert "Write a memory" in result.stdout @@ -245,3 +246,57 @@ def test_health_rejects_unknown_format() -> None: result = runner.invoke(app, ["health", "--format", "xml"]) assert result.exit_code == 2 + + +def test_extract_cli_sends_llm_options(monkeypatch) -> None: + chunk_id = str(uuid4()) + captured = {} + + class FakeClient: + def health_detail(self, *, workspace=None, agent=None): + return { + "workspace": { + "found": True, + "workspace_id": "00000000-0000-0000-0000-000000000201", + } + } + + def extract_candidates(self, payload, *, actor_type, actor_id): + captured["payload"] = payload + captured["actor_type"] = actor_type + captured["actor_id"] = actor_id + return { + "workspace_id": payload["workspace_id"], + "method": payload["method"], + "created_count": 0, + "candidates": [], + } + + monkeypatch.setattr( + "app.cli.commands.extract.build_client", + lambda *args, **kwargs: FakeClient(), + ) + runner = CliRunner() + + result = runner.invoke( + app, + [ + "extract", + "--workspace", + "demo", + "--chunk", + chunk_id, + "--method", + "llm", + "--llm-api-key", + "test-key", + "--llm-model", + "analysis-model", + ], + ) + + assert result.exit_code == 0 + assert captured["payload"]["method"] == "llm" + assert captured["payload"]["chunk_ids"] == [chunk_id] + assert captured["payload"]["llm"]["api_key"] == "test-key" + assert captured["payload"]["llm"]["model"] == "analysis-model" diff --git a/backend/tests/test_context_pack.py b/backend/tests/test_context_pack.py index abd807a..76a6c51 100644 --- a/backend/tests/test_context_pack.py +++ b/backend/tests/test_context_pack.py @@ -95,6 +95,7 @@ def test_format_context_pack_renders_agent_ready_markdown_with_citations() -> No assert "reason=ranked by score, importance, and confidence" in result.markdown assert result.token_budget == 800 assert result.selected_memories[0]["selection_reason"] + assert "campus cafeteria" in result.selected_memories[0]["canonical_text"] assert result.supporting_evidence[0]["memory_ref"] == "M1" assert result.citation_map["memories"]["M1"]["memory_type"] == "decision" assert result.citation_map["memories"]["M1"]["selection_reason"] diff --git a/backend/tests/test_embeddings.py b/backend/tests/test_embeddings.py index 6722195..08c17fa 100644 --- a/backend/tests/test_embeddings.py +++ b/backend/tests/test_embeddings.py @@ -1,5 +1,7 @@ from __future__ import annotations +import httpx +from app.api.deps import get_embedding_service from app.main import app from app.models.embedding import ( EmbeddingBackfillRequest, @@ -29,12 +31,17 @@ def test_local_hashing_embedding_is_deterministic_and_normalized() -> None: def test_embedding_api_generates_vector() -> None: - client = TestClient(app) - - response = client.post( - "/api/embeddings/generate", - json={"text": "Remember that PostgreSQL is the primary database.", "dimension": 32}, + app.dependency_overrides[get_embedding_service] = lambda: EmbeddingService( + provider=LocalHashingEmbeddingProvider(), ) + try: + client = TestClient(app) + response = client.post( + "/api/embeddings/generate", + json={"text": "Remember that PostgreSQL is the primary database.", "dimension": 32}, + ) + finally: + app.dependency_overrides.pop(get_embedding_service, None) assert response.status_code == 200 payload = response.json() @@ -123,3 +130,48 @@ def post(self, url, *, headers, json): assert result.model == "Qwen/Qwen3-Embedding-0.6B" assert result.dimension == 3 assert result.embedding == [0.1, 0.2, 0.3] + + +def test_siliconflow_embedding_provider_retries_transport_error(monkeypatch) -> None: + attempts = 0 + + class FakeResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return {"data": [{"embedding": [0.1, 0.2]}]} + + class FakeClient: + def __init__(self, *, timeout): + assert timeout == 30.0 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback) -> None: + return None + + def post(self, url, *, headers, json): + nonlocal attempts + attempts += 1 + if attempts == 1: + request = httpx.Request("POST", url) + raise httpx.ConnectError("temporary TLS failure", request=request) + return FakeResponse() + + monkeypatch.setattr("app.services.embedding_service.httpx.Client", FakeClient) + monkeypatch.setattr("app.services.embedding_service.time.sleep", lambda _: None) + provider = SiliconFlowEmbeddingProvider(api_key="test-key") + + result = provider.embed( + EmbeddingGenerateRequest( + text="demo", + provider="siliconflow", + model="Qwen/Qwen3-Embedding-0.6B", + dimension=1024, + ) + ) + + assert attempts == 2 + assert result.embedding == [0.1, 0.2] diff --git a/backend/tests/test_evaluation_adapters.py b/backend/tests/test_evaluation_adapters.py index 8896d35..3db98a9 100644 --- a/backend/tests/test_evaluation_adapters.py +++ b/backend/tests/test_evaluation_adapters.py @@ -3,6 +3,8 @@ import json from pathlib import Path +import pytest + from evaluation.adapters import locomo_adapter, longmemeval_adapter, memoryagentbench_adapter @@ -10,7 +12,7 @@ def test_longmemeval_adapter_converts_jsonl(tmp_path: Path) -> None: raw_dir = tmp_path / "raw" processed_dir = tmp_path / "processed" raw_dir.mkdir() - (raw_dir / "sample.jsonl").write_text( + (raw_dir / "longmemeval_oracle").write_text( json.dumps( { "question_id": "q1", @@ -18,15 +20,25 @@ def test_longmemeval_adapter_converts_jsonl(tmp_path: Path) -> None: "answer": "Beijing", "question_type": "knowledge_update", "haystack_sessions": [ - { - "session_id": "s1", - "messages": [{"role": "user", "content": "I live in Shanghai."}], - }, - { - "session_id": "s2", - "messages": [{"role": "user", "content": "I moved to Beijing."}], - }, + [ + { + "role": "user", + "content": "I live in Shanghai.", + "has_answer": False, + } + ], + [ + { + "role": "user", + "content": "I moved to Beijing.", + "has_answer": True, + } + ], ], + "haystack_session_ids": ["session-1", "session-2"], + "haystack_dates": ["2024-01-02", "2024-02-03"], + "answer_session_ids": ["session-2"], + "question_date": "2024-03-04", }, ensure_ascii=False, ) @@ -40,7 +52,67 @@ def test_longmemeval_adapter_converts_jsonl(tmp_path: Path) -> None: assert row["case_id"] == "longmemeval_q1" assert row["category"] == "temporal_update" assert row["expected_answer"] == "Beijing" + assert row["expected_behavior"] == "answer_latest" + assert row["query"].startswith("[Question date: 2024-03-04]") assert len(row["sessions"]) == 2 + assert row["sessions"][0]["session_id"] == "session-1" + assert row["sessions"][1]["turns"][0]["content"].startswith("[Session date: 2024-02-03]") + assert row["sessions"][1]["turns"][0]["metadata"]["has_answer"] is True + assert row["metadata"]["answer_session_ids"] == ["session-2"] + assert row["metadata"]["longmemeval_format"] == "official-v1" + + +def test_longmemeval_adapter_marks_abstention_and_scalar_answer(tmp_path: Path) -> None: + raw_dir = tmp_path / "raw" + processed_dir = tmp_path / "processed" + raw_dir.mkdir() + (raw_dir / "sample.json").write_text( + json.dumps( + [ + { + "question_id": "q2_abs", + "question": "How many siblings does the user have?", + "answer": 2, + "question_type": "multi-session", + "haystack_sessions": [[{"role": "user", "content": "No sibling facts."}]], + } + ] + ), + encoding="utf-8", + ) + + output = longmemeval_adapter.convert(raw_dir, processed_dir) + + row = json.loads(output.read_text(encoding="utf-8").strip()) + assert row["category"] == "abstention" + assert row["expected_behavior"] == "refuse_or_unknown" + assert row["expected_answer"] == "2" + + +def test_longmemeval_adapter_rejects_misaligned_session_metadata(tmp_path: Path) -> None: + raw_dir = tmp_path / "raw" + processed_dir = tmp_path / "processed" + raw_dir.mkdir() + (raw_dir / "sample.json").write_text( + json.dumps( + [ + { + "question_id": "q3", + "question": "Where does the user live?", + "answer": "Beijing", + "haystack_sessions": [ + [{"role": "user", "content": "I live in Beijing."}], + [{"role": "assistant", "content": "Understood."}], + ], + "haystack_dates": ["2024-01-01"], + } + ] + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="haystack_dates must contain 2 values"): + longmemeval_adapter.convert(raw_dir, processed_dir) def test_locomo_adapter_expands_qa_items(tmp_path: Path) -> None: @@ -69,6 +141,83 @@ def test_locomo_adapter_expands_qa_items(tmp_path: Path) -> None: assert all(row["source"] == "locomo" for row in rows) +def test_locomo_adapter_converts_official_sessions_and_adversarial_qa(tmp_path: Path) -> None: + raw_dir = tmp_path / "raw" + processed_dir = tmp_path / "processed" + raw_dir.mkdir() + (raw_dir / "locomo10.json").write_text( + json.dumps( + [ + { + "sample_id": "conv-1", + "conversation": { + "speaker_a": "Caroline", + "speaker_b": "Melanie", + "session_1_date_time": "1:56 pm on 8 May, 2023", + "session_1": [ + { + "speaker": "Caroline", + "dia_id": "D1:1", + "text": "I joined a support group yesterday.", + }, + { + "speaker": "Melanie", + "dia_id": "D1:2", + "text": "That sounds meaningful.", + "blip_caption": "a colorful mural", + }, + ], + }, + "qa": [ + { + "question": "When did Caroline join the support group?", + "answer": "7 May 2023", + "evidence": ["D1:1"], + "category": 2, + }, + { + "question": "What did Melanie buy?", + "evidence": ["D1:01 D:1:2"], + "category": 5, + "adversarial_answer": "a painting", + }, + { + "question": "Did Caroline buy the painting?", + "answer": "No", + "evidence": ["D1:1"], + "category": 5, + "adversarial_answer": "Yes", + }, + ], + } + ], + ensure_ascii=False, + ), + encoding="utf-8", + ) + + output = locomo_adapter.convert(raw_dir, processed_dir) + + rows = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] + assert rows[0]["case_id"] == "locomo_conv-1_qa001" + assert rows[0]["category"] == "multi_hop" + assert rows[0]["gold_memory_ids"] == ["D1:1"] + assert rows[0]["sessions"][0]["turns"][0]["role"] == "user" + assert rows[0]["sessions"][0]["turns"][1]["role"] == "assistant" + assert rows[0]["sessions"][0]["turns"][0]["content"].startswith( + "[Session date: 1:56 pm on 8 May, 2023] Caroline:" + ) + assert rows[0]["sessions"][0]["turns"][1]["metadata"]["blip_caption"] == ("a colorful mural") + assert rows[1]["category"] == "adversarial" + assert rows[1]["expected_behavior"] == "refuse_or_unknown" + assert rows[1]["expected_answer"] is None + assert rows[1]["forbidden_answers"] == ["a painting"] + assert rows[1]["gold_memory_ids"] == ["D1:1", "D1:2"] + assert rows[2]["expected_behavior"] == "answer" + assert rows[2]["expected_answer"] == "No" + assert rows[2]["forbidden_answers"] == ["Yes"] + + def test_memoryagentbench_adapter_marks_conflict_behavior(tmp_path: Path) -> None: raw_dir = tmp_path / "raw" processed_dir = tmp_path / "processed" @@ -98,3 +247,47 @@ def test_memoryagentbench_adapter_marks_conflict_behavior(tmp_path: Path) -> Non assert row["case_id"] == "memoryagentbench_m1" assert row["expected_behavior"] == "answer_latest" assert row["expected_answer"] == "Beijing" + + +def test_memoryagentbench_adapter_converts_official_conflict_parquet( + tmp_path: Path, +) -> None: + pyarrow = pytest.importorskip("pyarrow") + parquet = pytest.importorskip("pyarrow.parquet") + raw_dir = tmp_path / "raw" + processed_dir = tmp_path / "processed" + raw_dir.mkdir() + table = pyarrow.Table.from_pylist( + [ + { + "context": ( + "Here is a list of facts:\n" + "0. The chairperson is Alice.\n" + "1. The chairperson is Bob.\n" + ), + "questions": ["Who is the chairperson?", "Who holds the role now?"], + "answers": [["Bob"], ["Bob", "Robert"]], + "metadata": { + "source": "factconsolidation_sh_6k", + "qa_pair_ids": ["pair-1", "pair-2"], + }, + } + ] + ) + parquet.write_table(table, raw_dir / "Conflict_Resolution.parquet") + + output = memoryagentbench_adapter.convert(raw_dir, processed_dir) + + rows = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] + assert len(rows) == 2 + assert rows[0]["case_id"] == "memoryagentbench_pair-1" + assert rows[0]["category"] == "conflict_single_hop_6k" + assert rows[0]["expected_behavior"] == "answer_latest" + assert rows[0]["expected_answer"] == "Bob" + assert rows[0]["metadata"]["context_group_id"] == "factconsolidation_sh_6k" + assert rows[1]["metadata"]["accepted_answers"] == ["Bob", "Robert"] + assert rows[0]["sessions"][0]["turns"][0]["content"].startswith("[Memory sequence 0001]") + first_turn = rows[0]["sessions"][0]["turns"][0] + second_turn = rows[0]["sessions"][1]["turns"][0] + assert first_turn["metadata"]["valid_from"] < second_turn["metadata"]["valid_from"] + assert first_turn["metadata"]["supersession_key"] == second_turn["metadata"]["supersession_key"] diff --git a/backend/tests/test_evaluation_baselines.py b/backend/tests/test_evaluation_baselines.py index bf0543a..770b094 100644 --- a/backend/tests/test_evaluation_baselines.py +++ b/backend/tests/test_evaluation_baselines.py @@ -66,6 +66,89 @@ def test_summary_memory_clears_local_memory_after_forget_turn() -> None: assert result.retrieved_memory_texts == [] +def test_longmemeval_local_baseline_includes_assistant_turns() -> None: + case = EvaluationCase( + case_id="longmemeval_assistant_001", + source="longmemeval", + category="single_fact", + sessions=[ + EvaluationSession( + session_id="s1", + turns=[ + EvaluationTurn(role="user", content="What should I cook?"), + EvaluationTurn(role="assistant", content="Try mushroom risotto."), + ], + ) + ], + query="What dish did the assistant suggest?", + expected_answer="mushroom risotto", + ) + + result = build_baseline_with_config(mode="summary_memory", run_id="test").run_case(case) + + assert "Try mushroom risotto." in result.retrieved_memory_texts + + +def test_locomo_local_baseline_uses_dialog_ids_for_retrieval() -> None: + case = EvaluationCase( + case_id="locomo_conv-1_qa001", + source="locomo", + category="single_hop", + sessions=[ + EvaluationSession( + session_id="session_1", + turns=[ + EvaluationTurn( + role="assistant", + content="Melanie: The race supported mental health.", + metadata={"dia_id": "D1:2"}, + ) + ], + ) + ], + query="What did the race support?", + expected_answer="mental health", + gold_memory_ids=["D1:2"], + ) + + result = build_baseline_with_config(mode="summary_memory", run_id="test").run_case(case) + + assert result.retrieved_memory_ids == ["D1:2"] + assert "mental health" in result.generated_answer + + +def test_locomo_local_baseline_does_not_apply_deletion_forget_heuristic() -> None: + case = EvaluationCase( + case_id="locomo_conv-1_qa002", + source="locomo", + category="single_hop", + sessions=[ + EvaluationSession( + session_id="session_1", + turns=[ + EvaluationTurn( + role="user", + content="Caroline: Do not forget that the race supported mental health.", + metadata={"dia_id": "D1:1"}, + ), + EvaluationTurn( + role="assistant", + content="Melanie: I will remember that.", + metadata={"dia_id": "D1:2"}, + ), + ], + ) + ], + query="What did the race support?", + expected_answer="mental health", + gold_memory_ids=["D1:1"], + ) + + result = build_baseline_with_config(mode="summary_memory", run_id="test").run_case(case) + + assert result.retrieved_memory_ids[:2] == ["D1:1", "D1:2"] + + class FakeResponse: def __init__(self, payload: dict[str, Any], status_code: int = 200) -> None: self._payload = payload @@ -76,6 +159,311 @@ def json(self) -> dict[str, Any]: return self._payload +def test_longmemeval_live_baseline_writes_assistant_turns(monkeypatch) -> None: + memory_payloads: list[dict[str, Any]] = [] + + def fake_request(method: str, url: str, **kwargs: Any) -> FakeResponse: + if url.endswith("/api/health/detail"): + return FakeResponse( + { + "workspace": { + "found": True, + "workspace_id": "00000000-0000-0000-0000-000000000201", + } + } + ) + if url.endswith("/api/sessions"): + return FakeResponse({"session_id": "session-1"}) + if url.endswith("/api/observe"): + return FakeResponse({"message_id": "message-1"}) + if url.endswith("/api/memories/batch"): + memory_payloads.extend(kwargs["json"]["items"]) + return FakeResponse({"items": [{"memory_id": "memory-1"}]}) + if url.endswith("/api/recall"): + return FakeResponse({"memories": []}) + raise AssertionError(f"unexpected request {method} {url}") + + monkeypatch.setattr("evaluation.baselines.httpx.request", fake_request) + case = EvaluationCase( + case_id="longmemeval_assistant_002", + source="longmemeval", + category="single_fact", + sessions=[ + EvaluationSession( + session_id="s1", + turns=[EvaluationTurn(role="assistant", content="Try mushroom risotto.")], + ) + ], + query="What dish did the assistant suggest?", + expected_answer="mushroom risotto", + ) + + result = build_baseline_with_config( + mode="db_memory", + run_id="test", + api_base_url="http://testserver", + workspace="demo", + cleanup=False, + ).run_case(case) + + assert result.error == "" + assert [payload["canonical_text"] for payload in memory_payloads] == ["Try mushroom risotto."] + + +def test_live_baseline_maps_dates_and_supersession_into_batch_payloads(monkeypatch) -> None: + batches: list[list[dict[str, Any]]] = [] + next_memory_id = 1 + + def fake_request(method: str, url: str, **kwargs: Any) -> FakeResponse: + nonlocal next_memory_id + if url.endswith("/api/health/detail"): + return FakeResponse( + { + "workspace": { + "found": True, + "workspace_id": "00000000-0000-0000-0000-000000000201", + } + } + ) + if url.endswith("/api/sessions"): + return FakeResponse({"session_id": "session-1"}) + if url.endswith("/api/observe"): + return FakeResponse({"message_id": "message-1"}) + if url.endswith("/api/memories/batch"): + items = kwargs["json"]["items"] + batches.append(items) + response_items = [] + for _item in items: + response_items.append({"memory_id": f"memory-{next_memory_id}"}) + next_memory_id += 1 + return FakeResponse({"items": response_items}) + if url.endswith("/api/recall"): + return FakeResponse({"memories": []}) + raise AssertionError(f"unexpected request {method} {url}") + + monkeypatch.setattr("evaluation.baselines.httpx.request", fake_request) + case = EvaluationCase( + case_id="memoryagentbench_supersession", + source="memoryagentbench", + category="conflict_single_hop_6k", + sessions=[ + EvaluationSession( + session_id="s1", + turns=[ + EvaluationTurn( + role="user", + content="The chairperson is Alice.", + metadata={ + "valid_from": "2000-01-01T00:00:01+00:00", + "supersession_key": "chairperson", + }, + ) + ], + ), + EvaluationSession( + session_id="s2", + turns=[ + EvaluationTurn( + role="user", + content="The unrelated fact is stable.", + metadata={"session_date": "2023/04/10 (Mon) 17:50"}, + ) + ], + ), + EvaluationSession( + session_id="s3", + turns=[ + EvaluationTurn( + role="user", + content="The chairperson is Bob.", + metadata={ + "valid_from": "2000-01-01T00:00:03+00:00", + "supersession_key": "chairperson", + }, + ) + ], + ), + ], + query="Who is the chairperson?", + expected_answer="Bob", + expected_behavior="answer_latest", + ) + + result = build_baseline_with_config( + mode="db_memory", + run_id="test", + api_base_url="http://testserver", + workspace="demo", + cleanup=False, + ).run_case(case) + + assert result.error == "" + assert [len(batch) for batch in batches] == [2, 1] + assert batches[0][0]["valid_from"] == "2000-01-01T00:00:01+00:00" + assert batches[0][1]["valid_from"] == "2023-04-10T17:50:00+00:00" + assert batches[1][0]["supersedes_memory_id"] == "memory-1" + + +def test_locomo_live_baseline_maps_memory_uuid_to_dialog_id(monkeypatch) -> None: + def fake_request(method: str, url: str, **kwargs: Any) -> FakeResponse: + if url.endswith("/api/health/detail"): + return FakeResponse( + { + "workspace": { + "found": True, + "workspace_id": "00000000-0000-0000-0000-000000000201", + } + } + ) + if url.endswith("/api/sessions"): + return FakeResponse({"session_id": "session-1"}) + if url.endswith("/api/observe"): + return FakeResponse({"message_id": "message-1"}) + if url.endswith("/api/memories/batch"): + return FakeResponse({"items": [{"memory_id": "memory-uuid-1"}]}) + if url.endswith("/api/recall"): + return FakeResponse( + { + "memories": [ + { + "memory_id": "memory-uuid-1", + "canonical_text": "Melanie: The race supported mental health.", + "score": 0.9, + } + ] + } + ) + raise AssertionError(f"unexpected request {method} {url}") + + monkeypatch.setattr("evaluation.baselines.httpx.request", fake_request) + case = EvaluationCase( + case_id="locomo_conv-1_qa001", + source="locomo", + category="single_hop", + sessions=[ + EvaluationSession( + session_id="session_1", + turns=[ + EvaluationTurn( + role="assistant", + content="Melanie: The race supported mental health.", + metadata={"dia_id": "D1:2"}, + ) + ], + ) + ], + query="What did the race support?", + expected_answer="mental health", + gold_memory_ids=["D1:2"], + ) + + result = build_baseline_with_config( + mode="db_memory", + run_id="test", + api_base_url="http://testserver", + workspace="demo", + cleanup=False, + ).run_case(case) + + assert result.error == "" + assert result.retrieved_memory_ids == ["D1:2"] + assert result.metadata["memory_source_ids"] == {"memory-uuid-1": "D1:2"} + + +def test_grouped_live_baseline_injects_context_once_for_multiple_questions( + monkeypatch, +) -> None: + calls: list[tuple[str, str]] = [] + batch_sizes: list[int] = [] + + def fake_request(method: str, url: str, **kwargs: Any) -> FakeResponse: + calls.append((method, url)) + if url.endswith("/api/health/detail"): + return FakeResponse( + { + "workspace": { + "found": True, + "workspace_id": "00000000-0000-0000-0000-000000000201", + } + } + ) + if url.endswith("/api/sessions"): + return FakeResponse({"session_id": "session-1"}) + if url.endswith("/api/observe"): + return FakeResponse({"message_id": "message-1"}) + if url.endswith("/api/memories/batch"): + items = kwargs["json"]["items"] + batch_sizes.append(len(items)) + return FakeResponse( + { + "items": [ + {"memory_id": f"memory-{index}"} + for index, _item in enumerate(items, start=1) + ] + } + ) + if url.endswith("/api/qa/answer"): + return FakeResponse( + { + "answer": "Bob [M1].", + "provider": "deepseek", + "model": "deepseek-chat", + "total_tokens": 20, + "selected_memories": [ + { + "memory_id": "memory-1", + "canonical_text": "The chairperson is Bob.", + "score": 0.9, + } + ], + "citation_map": {"memories": {"M1": {"memory_id": "memory-1"}}}, + } + ) + raise AssertionError(f"unexpected request {method} {url}") + + monkeypatch.setattr("evaluation.baselines.httpx.request", fake_request) + sessions = [ + EvaluationSession( + session_id="context-1", + turns=[ + EvaluationTurn(role="user", content="The chairperson was Alice."), + EvaluationTurn(role="user", content="The chairperson is Bob."), + ], + ) + ] + cases = [ + EvaluationCase( + case_id=f"group-{index}", + source="memoryagentbench", + category="conflict_single_hop_6k", + sessions=sessions, + query=question, + expected_answer="Bob", + expected_behavior="answer_latest", + metadata={"context_group_id": "factconsolidation_sh_6k"}, + ) + for index, question in enumerate( + ["Who is the chairperson?", "Who holds the role now?"], + start=1, + ) + ] + baseline = build_baseline_with_config( + mode="db_qa", + run_id="test", + api_base_url="http://testserver", + workspace="demo", + cleanup=False, + ) + + results = baseline.run_group(cases) + + assert len(results) == 2 + assert batch_sizes == [2] + assert calls.count(("POST", "http://testserver/api/memories/batch")) == 1 + assert calls.count(("POST", "http://testserver/api/qa/answer")) == 2 + assert all(result.generated_answer == "Bob [M1]." for result in results) + + def test_naive_vector_rag_backfills_embeddings_before_recall(monkeypatch) -> None: calls: list[tuple[str, str]] = [] @@ -94,8 +482,8 @@ def fake_request(method: str, url: str, **kwargs: Any) -> FakeResponse: return FakeResponse({"session_id": "session-1"}) if url.endswith("/api/observe"): return FakeResponse({"message_id": "message-1"}) - if url.endswith("/api/memories"): - return FakeResponse({"memory_id": "memory-1"}) + if url.endswith("/api/memories/batch"): + return FakeResponse({"items": [{"memory_id": "memory-1"}]}) if url.endswith("/api/embeddings/backfill"): return FakeResponse({"memory_count": 1, "chunk_count": 1}) if url.endswith("/api/recall"): @@ -140,6 +528,7 @@ def fake_request(method: str, url: str, **kwargs: Any) -> FakeResponse: run_id="test", api_base_url="http://testserver", workspace="demo", + cleanup=False, ).run_case(case) assert result.error == "" @@ -218,6 +607,7 @@ def fake_request(method: str, url: str, **kwargs: Any) -> FakeResponse: run_id="test", api_base_url="http://testserver", workspace="demo", + cleanup=False, ).run_case(case) assert result.error == "" @@ -226,3 +616,81 @@ def fake_request(method: str, url: str, **kwargs: Any) -> FakeResponse: assert result.retrieved_memory_ids == ["memory-1"] assert ("POST", "http://testserver/api/memory-extraction/from-chunks") in calls assert ("POST", "http://testserver/api/memory-candidates/candidate-1/approve") in calls + + +def test_db_qa_calls_answer_endpoint_and_cleans_up(monkeypatch) -> None: + calls: list[tuple[str, str]] = [] + + def fake_request(method: str, url: str, **kwargs: Any) -> FakeResponse: + calls.append((method, url)) + if url.endswith("/api/health/detail"): + return FakeResponse( + { + "workspace": { + "found": True, + "workspace_id": "00000000-0000-0000-0000-000000000201", + } + } + ) + if url.endswith("/api/sessions"): + return FakeResponse({"session_id": "session-1"}) + if url.endswith("/api/observe"): + return FakeResponse({"message_id": "message-1"}) + if url.endswith("/api/memories/batch") and method == "POST": + return FakeResponse({"items": [{"memory_id": "memory-1"}]}) + if url.endswith("/api/qa/answer"): + return FakeResponse( + { + "answer": "Rust [M1].", + "provider": "deepseek", + "model": "deepseek-chat", + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + "token_count": 30, + "token_budget": 3000, + "citation_map": {"memories": {"M1": {"memory_id": "memory-1"}}}, + "supporting_evidence": [], + "selected_memories": [ + {"memory_id": "memory-1", "score": 0.9, "selection_reason": "match"} + ], + } + ) + if "/api/memories/memory-1" in url and method == "DELETE": + return FakeResponse({"memory_id": "memory-1", "status": "forgotten"}) + raise AssertionError(f"unexpected request {method} {url}") + + monkeypatch.setattr("evaluation.baselines.httpx.request", fake_request) + case = EvaluationCase( + case_id="single_fact_qa", + source="synthetic", + category="single_fact", + sessions=[ + EvaluationSession( + session_id="s1", + turns=[ + EvaluationTurn( + role="user", + content="Please remember that Rust is my favorite language.", + ) + ], + ) + ], + query="What is my favorite language?", + expected_answer="Rust", + ) + + result = build_baseline_with_config( + mode="db_qa", + run_id="test", + api_base_url="http://testserver", + workspace="demo", + ).run_case(case) + + assert result.error == "" + assert result.generated_answer == "Rust [M1]." + assert result.token_usage == 105 + assert result.metadata["provider"] == "deepseek" + assert result.retrieved_memory_ids == ["memory-1"] + assert ("POST", "http://testserver/api/qa/answer") in calls + assert ("DELETE", "http://testserver/api/memories/memory-1") in calls diff --git a/backend/tests/test_evaluation_judging.py b/backend/tests/test_evaluation_judging.py new file mode 100644 index 0000000..2277bcd --- /dev/null +++ b/backend/tests/test_evaluation_judging.py @@ -0,0 +1,172 @@ +import csv +import json +from pathlib import Path + +from evaluation.cases import EvaluationCase +from evaluation.judging import judge_answer, judge_results_csv + + +class FakeResponse: + status_code = 200 + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return { + "choices": [ + { + "message": { + "content": ( + '{"pass": false, "score": 0.1, ' + '"reason": "The answer is ultimately uncertain."}' + ) + } + } + ], + "usage": {"prompt_tokens": 100, "completion_tokens": 20}, + } + + +def test_judge_answer_rejects_uncertain_semantic_answer() -> None: + captured = {} + + def fake_request(method, url, **kwargs): + captured["method"] = method + captured["url"] = url + captured["json"] = kwargs["json"] + return FakeResponse() + + case = EvaluationCase( + case_id="case-1", + source="longmemeval", + category="temporal_reasoning", + sessions=[], + query="Which vehicle was first?", + expected_answer="bike", + ) + + result = judge_answer( + case=case, + generated_answer="The bike was mentioned, but I cannot determine which was first.", + api_key="test", + base_url="https://api.deepseek.com/v1", + model="deepseek-chat", + provider="deepseek", + request=fake_request, + ) + + assert result["judge_pass"] == "false" + assert result["judge_score"] == "0.100000" + assert result["judge_cost"] == "0.00014000" + assert captured["method"] == "POST" + + +def test_judge_results_loads_only_requested_cases(tmp_path: Path) -> None: + dataset = tmp_path / "cases.jsonl" + dataset.write_text( + "\n".join( + [ + json.dumps( + { + "case_id": "case-1", + "category": "qa", + "query": "Question?", + "expected_answer": "Answer", + } + ), + '{"case_id": "unused", "large_unsupported_field": true}', + ] + ), + encoding="utf-8", + ) + input_csv = tmp_path / "results.csv" + with input_csv.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=["case_id", "generated_answer"], + ) + writer.writeheader() + writer.writerow({"case_id": "case-1", "generated_answer": "Answer"}) + + output_csv = tmp_path / "judged.csv" + judge_results_csv( + input_csv=input_csv, + dataset=dataset, + output_csv=output_csv, + api_key="test", + base_url="https://api.deepseek.com/v1", + model="deepseek-chat", + request=lambda *args, **kwargs: FakeResponse(), + ) + + with output_csv.open("r", encoding="utf-8", newline="") as handle: + row = next(csv.DictReader(handle)) + assert row["case_id"] == "case-1" + assert row["judge_pass"] == "false" + + +def test_judge_results_retries_and_resumes_checkpoint(tmp_path: Path) -> None: + dataset = tmp_path / "cases.jsonl" + dataset.write_text( + "\n".join( + json.dumps( + { + "case_id": f"case-{index}", + "category": "qa", + "query": "Question?", + "expected_answer": "Answer", + } + ) + for index in range(1, 3) + ), + encoding="utf-8", + ) + input_csv = tmp_path / "results.csv" + with input_csv.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=["case_id", "generated_answer"]) + writer.writeheader() + writer.writerows( + [ + {"case_id": "case-1", "generated_answer": "Answer"}, + {"case_id": "case-2", "generated_answer": "Answer"}, + ] + ) + + calls = 0 + + class TruncatedResponse(FakeResponse): + def json(self) -> dict: + return { + "choices": [{"message": {"content": '{"pass": false, "reason": "cut'}}], + "usage": {}, + } + + def flaky_request(*args, **kwargs): + nonlocal calls + calls += 1 + return TruncatedResponse() if calls == 1 else FakeResponse() + + output_csv = tmp_path / "judged.csv" + judge_results_csv( + input_csv=input_csv, + dataset=dataset, + output_csv=output_csv, + api_key="test", + base_url="https://api.deepseek.com/v1", + model="deepseek-chat", + request=flaky_request, + ) + assert calls == 3 + + judge_results_csv( + input_csv=input_csv, + dataset=dataset, + output_csv=output_csv, + api_key="test", + base_url="https://api.deepseek.com/v1", + model="deepseek-chat", + request=lambda *args, **kwargs: (_ for _ in ()).throw( + AssertionError("completed rows should be skipped") + ), + ) diff --git a/backend/tests/test_evaluation_long_context.py b/backend/tests/test_evaluation_long_context.py new file mode 100644 index 0000000..b924a43 --- /dev/null +++ b/backend/tests/test_evaluation_long_context.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from pathlib import Path + +from evaluation.cases import load_cases +from evaluation.generators.long_context import generate_long_context_cases + + +def test_long_context_generator_measures_actual_tokens(tmp_path: Path) -> None: + output = generate_long_context_cases( + tmp_path / "long_context.jsonl", + token_lengths=[1000, 2000], + ) + + cases = load_cases(output) + + assert [case.metadata["actual_history_tokens"] for case in cases] == [1000, 2000] + assert all(case.expected_answer == "Aurora" for case in cases) + assert [case.metadata["memory_turn_count"] for case in cases] == [2, 3] + assert len(cases[0].sessions) == 3 + assert len(cases[1].sessions) == 4 + assert all( + "durable project code" not in session.turns[0].content + for case in cases + for session in case.sessions[1:-1] + ) + + +def test_long_context_generator_splits_100k_history_into_bounded_memories( + tmp_path: Path, +) -> None: + output = generate_long_context_cases( + tmp_path / "long_context_100k.jsonl", + token_lengths=[100000], + ) + + case = load_cases(output)[0] + + assert case.metadata["actual_history_tokens"] == 100000 + assert case.metadata["memory_turn_count"] == 101 + assert len(case.sessions) == 102 diff --git a/backend/tests/test_evaluation_metrics.py b/backend/tests/test_evaluation_metrics.py new file mode 100644 index 0000000..0cba36a --- /dev/null +++ b/backend/tests/test_evaluation_metrics.py @@ -0,0 +1,68 @@ +from evaluation.cases import EvaluationCase +from evaluation.metrics.qa_metrics import score_qa + + +def _case(**overrides: object) -> EvaluationCase: + values = { + "case_id": "temporal-1", + "source": "synthetic", + "category": "temporal_update", + "sessions": [], + "query": "Where do I live now?", + "expected_answer": "Beijing", + "forbidden_answers": ["Shanghai"], + "expected_behavior": "answer_latest", + } + values.update(overrides) + return EvaluationCase(**values) + + +def test_answer_latest_allows_historical_value_as_context() -> None: + metrics = score_qa(_case(), "I previously lived in Shanghai, but now live in Beijing.") + + assert metrics["pass"] is True + assert metrics["forbidden_answer_violation"] is True + assert metrics["historical_value_mention"] is True + assert metrics["stale_answer_error"] is False + + +def test_answer_latest_fails_when_latest_value_is_missing() -> None: + metrics = score_qa(_case(), "You live in Shanghai.") + + assert metrics["pass"] is False + assert metrics["stale_answer_error"] is True + + +def test_deletion_case_still_blocks_forbidden_content() -> None: + case = _case( + case_id="deletion-1", + category="deletion", + expected_answer=None, + forbidden_answers=["secret-code"], + expected_behavior="refuse_or_unknown", + ) + + metrics = score_qa(case, "The deleted value was secret-code.") + + assert metrics["pass"] is False + assert metrics["forbidden_answer_violation"] is True + assert metrics["historical_value_mention"] is False + + +def test_refusal_case_requires_explicit_unknown_answer() -> None: + case = _case( + case_id="adversarial-1", + category="adversarial", + expected_answer=None, + forbidden_answers=["a painting"], + expected_behavior="refuse_or_unknown", + ) + + hallucinated = score_qa(case, "She bought a sculpture.") + refused = score_qa(case, "There is not enough information to answer that.") + + assert hallucinated["pass"] is False + assert hallucinated["refusal_match"] is False + assert refused["pass"] is True + assert refused["refusal_match"] is True + assert refused["score"] == 1.0 diff --git a/backend/tests/test_evaluation_performance.py b/backend/tests/test_evaluation_performance.py new file mode 100644 index 0000000..05f7971 --- /dev/null +++ b/backend/tests/test_evaluation_performance.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from evaluation.performance import OperationSample, summarize_operations + + +def test_performance_summary_groups_operations_and_errors() -> None: + summary = summarize_operations( + [ + OperationSample("recall", 10.0, True, 200), + OperationSample("recall", 20.0, True, 200), + OperationSample("recall", 5.0, False, 500, "failed"), + OperationSample("write", 8.0, True, 201), + ] + ) + + assert summary["recall"]["samples"] == 3 + assert summary["recall"]["errors"] == 1 + assert summary["recall"]["p50_latency_ms"] == 15.0 + assert summary["recall"]["error_rate"] == 1 / 3 + assert summary["write"]["throughput_qps"] == 125.0 diff --git a/backend/tests/test_evaluation_pricing.py b/backend/tests/test_evaluation_pricing.py new file mode 100644 index 0000000..773843d --- /dev/null +++ b/backend/tests/test_evaluation_pricing.py @@ -0,0 +1,26 @@ +from evaluation.pricing import estimate_model_cost + + +def test_estimate_deepseek_chat_cost_uses_cache_miss_snapshot() -> None: + estimate = estimate_model_cost( + provider="deepseek", + model="deepseek-chat", + prompt_tokens=1_000_000, + completion_tokens=500_000, + ) + + assert estimate["estimated_cost"] == 2.0 + assert estimate["cost_currency"] == "CNY" + assert estimate["pricing_as_of"] == "2026-06-07" + assert estimate["pricing_assumption"] == "cache_miss" + + +def test_estimate_model_cost_returns_empty_for_unknown_model() -> None: + estimate = estimate_model_cost( + provider="unknown", + model="unknown", + prompt_tokens=100, + completion_tokens=20, + ) + + assert estimate["estimated_cost"] is None diff --git a/backend/tests/test_evaluation_report.py b/backend/tests/test_evaluation_report.py index 6364d08..bed76d8 100644 --- a/backend/tests/test_evaluation_report.py +++ b/backend/tests/test_evaluation_report.py @@ -136,3 +136,47 @@ def test_generate_report_reads_nested_mode_outputs(tmp_path: Path) -> None: report = report_path.read_text(encoding="utf-8") assert "`summary_memory/qa_results.csv`" in report assert "Run multiple baselines" in report + + +def test_generate_report_excludes_operation_sample_csv(tmp_path: Path) -> None: + output_dir = tmp_path / "outputs" + output_dir.mkdir() + with (output_dir / "api_results.csv").open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=["operation", "latency_ms", "success", "status_code", "error"], + ) + writer.writeheader() + writer.writerow( + { + "operation": "recall", + "latency_ms": "10", + "success": "true", + "status_code": "200", + "error": "", + } + ) + + report = generate_report(outputs_dir=output_dir).read_text(encoding="utf-8") + + assert "`api_results.csv`" not in report + assert "No result CSV files were found." in report + + +def test_generate_report_prefers_semantic_judge_result(tmp_path: Path) -> None: + output_dir = tmp_path / "outputs" + output_dir.mkdir() + (output_dir / "judged_results.csv").write_text( + ( + "case_id,category,pass,judge_pass,mode,run_id\n" + "case-1,qa,false,true,db_qa,run-1\n" + "case-2,qa,true,false,db_qa,run-1\n" + ), + encoding="utf-8", + ) + + report = generate_report(outputs_dir=output_dir).read_text(encoding="utf-8") + + assert "| `judged_results.csv` | 2 | 50.00% |" in report + assert "| case-2 | qa | db_qa |" in report + assert "| case-1 | qa | db_qa |" not in report diff --git a/backend/tests/test_evaluation_runner_checkpoint.py b/backend/tests/test_evaluation_runner_checkpoint.py new file mode 100644 index 0000000..14fdf3b --- /dev/null +++ b/backend/tests/test_evaluation_runner_checkpoint.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import csv +from pathlib import Path + +from evaluation.baselines import EvaluationResult +from evaluation.io import append_result_csv, completed_case_ids, write_results_csv + + +def _result(case_id: str) -> EvaluationResult: + return EvaluationResult( + case_id=case_id, + source="synthetic", + category="single_fact", + query="question", + expected_answer="answer", + generated_answer="answer", + mode="db_qa", + run_id="test", + ) + + +def test_append_result_csv_checkpoints_each_case(tmp_path: Path) -> None: + output = tmp_path / "results.csv" + + write_results_csv(output, []) + append_result_csv(output, _result("case-1")) + append_result_csv(output, _result("case-2")) + + with output.open("r", encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + assert [row["case_id"] for row in rows] == ["case-1", "case-2"] + assert completed_case_ids(output) == {"case-1", "case-2"} + + +def test_completed_case_ids_handles_missing_output(tmp_path: Path) -> None: + assert completed_case_ids(tmp_path / "missing.csv") == set() diff --git a/backend/tests/test_llm_analysis.py b/backend/tests/test_llm_analysis.py new file mode 100644 index 0000000..dba9c1b --- /dev/null +++ b/backend/tests/test_llm_analysis.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import json +from uuid import uuid4 + +import pytest +from app.services.llm_analysis import ( + LlmAnalysisDefaults, + LlmAnalysisError, + OpenAICompatibleAnalysisClient, + SourceChunkForAnalysis, + parse_llm_candidates, + resolve_llm_options, +) + + +def test_parse_llm_candidates_clamps_fields_and_preserves_chunk_id() -> None: + chunk_id = uuid4() + drafts = parse_llm_candidates( + json.dumps( + { + "candidates": [ + { + "chunk_id": str(chunk_id), + "canonical_text": "MemoryBase should keep PostgreSQL as source of truth.", + "memory_type": "constraint", + "summary": "PostgreSQL source of truth", + "confidence": 1.2, + "importance": 9, + } + ] + } + ), + chunks=[ + SourceChunkForAnalysis( + chunk_id=chunk_id, + chunk_no=1, + text="MemoryBase should keep PostgreSQL as source of truth.", + ) + ], + max_candidates=5, + ) + + assert len(drafts) == 1 + assert drafts[0].chunk_id == chunk_id + assert drafts[0].memory_type == "constraint" + assert drafts[0].confidence == 1 + assert drafts[0].importance == 5 + + +def test_resolve_llm_options_requires_api_key() -> None: + with pytest.raises(LlmAnalysisError, match="requires an API key"): + resolve_llm_options(None, LlmAnalysisDefaults(api_key="")) + + +def test_openai_compatible_analysis_client_maps_chat_completion(monkeypatch) -> None: + captured = {} + chunk_id = uuid4() + + class FakeResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return { + "choices": [ + { + "message": { + "content": json.dumps( + { + "candidates": [ + { + "chunk_id": str(chunk_id), + "canonical_text": "Private notes must stay hidden.", + "memory_type": "policy", + "summary": "Private notes visibility", + "confidence": 0.86, + "importance": 4, + } + ] + } + ) + } + } + ] + } + + class FakeClient: + def __init__(self, *, timeout): + captured["timeout"] = timeout + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback) -> None: + return None + + def post(self, url, *, headers, json): + captured["url"] = url + captured["headers"] = headers + captured["json"] = json + return FakeResponse() + + monkeypatch.setattr("app.services.llm_analysis.httpx.Client", FakeClient) + + client = OpenAICompatibleAnalysisClient() + drafts = client.analyze( + chunks=[ + SourceChunkForAnalysis( + chunk_id=chunk_id, + chunk_no=3, + text="Private notes must stay hidden.", + ) + ], + max_candidates=2, + options=resolve_llm_options( + None, + LlmAnalysisDefaults( + api_key="test-key", + base_url="https://api.example.com/v1/", + model="analysis-model", + ), + ), + ) + + assert captured["url"] == "https://api.example.com/v1/chat/completions" + assert captured["headers"]["Authorization"] == "Bearer test-key" + assert captured["json"]["model"] == "analysis-model" + assert captured["json"]["response_format"] == {"type": "json_object"} + assert drafts[0].memory_type == "policy" + assert drafts[0].confidence == 0.86 diff --git a/backend/tests/test_memories.py b/backend/tests/test_memories.py index 3e556f7..dff7111 100644 --- a/backend/tests/test_memories.py +++ b/backend/tests/test_memories.py @@ -9,6 +9,8 @@ from app.main import create_app from app.models.memory import ( ActorContext, + MemoryBatchCreateRequest, + MemoryBatchCreateResponse, MemoryCreateRequest, MemoryDeleteResponse, MemoryDetailResponse, @@ -19,6 +21,7 @@ from app.services.memory_service import ( MemoryNotFoundError, MemoryValidationError, + _validate_create_payload, _validate_initial_status, _validate_status_transition, ) @@ -118,6 +121,20 @@ def create_memory( ) return MemorySummaryResponse(**self.memory.model_dump(exclude={"evidence", "revisions"})) + def create_memories( + self, + payload: MemoryBatchCreateRequest, + actor: ActorContext | None = None, + ) -> MemoryBatchCreateResponse: + if len({item.workspace_id for item in payload.items}) != 1: + raise MemoryValidationError("batch memory items must use one workspace_id") + items = [self.create_memory(item, actor) for item in payload.items] + return MemoryBatchCreateResponse( + workspace_id=payload.items[0].workspace_id, + count=len(items), + items=items, + ) + def list_memories( self, *, @@ -225,6 +242,71 @@ def test_create_memory_accepts_candidate_status_and_new_memory_type() -> None: assert response.json()["status"] == "candidate" +def test_batch_create_memories_returns_all_items() -> None: + client, fake_service = build_client() + + response = client.post( + "/api/memories/batch", + headers={"X-Actor-Type": "agent"}, + json={ + "items": [ + { + "workspace_id": str(fake_service.workspace_id), + "memory_type": "fact", + "canonical_text": "First batch fact.", + }, + { + "workspace_id": str(fake_service.workspace_id), + "memory_type": "fact", + "canonical_text": "Second batch fact.", + }, + ] + }, + ) + + assert response.status_code == 201 + assert response.json()["count"] == 2 + assert len(response.json()["items"]) == 2 + + +def test_batch_create_rejects_mixed_workspaces() -> None: + client, fake_service = build_client() + + response = client.post( + "/api/memories/batch", + json={ + "items": [ + { + "workspace_id": str(fake_service.workspace_id), + "memory_type": "fact", + "canonical_text": "First workspace.", + }, + { + "workspace_id": str(uuid4()), + "memory_type": "fact", + "canonical_text": "Second workspace.", + }, + ] + }, + ) + + assert response.status_code == 400 + assert response.json()["detail"] == "batch memory items must use one workspace_id" + + +def test_superseding_memory_must_start_active() -> None: + with pytest.raises(MemoryValidationError, match="must start with active"): + _validate_create_payload( + MemoryCreateRequest( + workspace_id=uuid4(), + memory_type="fact", + canonical_text="Candidate replacement.", + status="candidate", + supersedes_memory_id=uuid4(), + ) + ) + + def test_list_memories_returns_collection() -> None: client, fake_service = build_client() diff --git a/backend/tests/test_memory_extraction_api.py b/backend/tests/test_memory_extraction_api.py new file mode 100644 index 0000000..3e01fae --- /dev/null +++ b/backend/tests/test_memory_extraction_api.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from app.api.deps import get_memory_extraction_service +from app.main import create_app +from app.models.memory import ActorContext, MemorySummaryResponse +from fastapi.testclient import TestClient + + +class FakeMemoryExtractionService: + def __init__(self) -> None: + self.workspace_id = uuid4() + self.chunk_id = uuid4() + self.payload = None + self.actor = None + + def extract_from_chunks(self, payload, actor: ActorContext): + self.payload = payload + self.actor = actor + now = datetime(2026, 6, 14, tzinfo=timezone.utc) + return [ + MemorySummaryResponse( + memory_id=uuid4(), + workspace_id=payload.workspace_id, + created_from_doc_id=uuid4(), + memory_type="policy", + canonical_text="Private notes must stay hidden from project-only agents.", + summary="Private notes visibility", + confidence=0.88, + importance=4, + status="candidate", + access_level="project", + current_revision_no=1, + created_at=now, + updated_at=now, + evidence_count=1, + ) + ] + + def list_candidates( + self, + *, + workspace_id: UUID | None, + memory_type: str | None, + keyword: str | None, + page: int, + page_size: int, + ): + raise AssertionError("not used") + + +def test_extract_from_chunks_accepts_llm_options() -> None: + app = create_app() + fake_service = FakeMemoryExtractionService() + app.dependency_overrides[get_memory_extraction_service] = lambda: fake_service + client = TestClient(app) + + response = client.post( + "/api/memory-extraction/from-chunks", + json={ + "workspace_id": str(fake_service.workspace_id), + "chunk_ids": [str(fake_service.chunk_id)], + "max_candidates": 3, + "method": "llm", + "llm": { + "api_key": "test-key", + "base_url": "https://api.example.com/v1", + "model": "analysis-model", + }, + }, + ) + + assert response.status_code == 201 + payload = response.json() + assert payload["method"] == "llm" + assert payload["created_count"] == 1 + assert payload["candidates"][0]["memory_type"] == "policy" + assert fake_service.payload.method == "llm" + assert fake_service.payload.llm.api_key == "test-key" + assert fake_service.actor.revision_reason == "llm memory extraction" diff --git a/backend/tests/test_memory_extraction_service.py b/backend/tests/test_memory_extraction_service.py new file mode 100644 index 0000000..3a220b4 --- /dev/null +++ b/backend/tests/test_memory_extraction_service.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from app.models.memory import ActorContext, MemorySummaryResponse +from app.models.memory_extraction import LlmAnalysisOptions, MemoryExtractionFromChunksRequest +from app.services.llm_analysis import LlmCandidateDraft +from app.services.memory_extraction_service import MemoryExtractionService + + +class FakeDatabase: + def __init__(self, rows): + self.rows = rows + self.audit_rows = [] + + def connection(self): + return FakeConnection(self) + + +class FakeConnection: + def __init__(self, database: FakeDatabase) -> None: + self.database = database + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback) -> None: + return None + + def cursor(self): + return FakeCursor(self.database) + + def commit(self) -> None: + return None + + +class FakeCursor: + def __init__(self, database: FakeDatabase) -> None: + self.database = database + self._rows = [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback) -> None: + return None + + def execute(self, query, params): + if "FROM source_chunk" in query: + self._rows = self.database.rows + return + if "INSERT INTO audit_log" in query: + self.database.audit_rows.append(json.loads(params["after_json"])) + + def fetchall(self): + return self._rows + + +class FakeMemoryService: + def __init__(self) -> None: + self.created = [] + + def create_memory(self, payload, actor): + self.created.append((payload, actor)) + now = datetime(2026, 6, 14, tzinfo=timezone.utc) + return MemorySummaryResponse( + memory_id=uuid4(), + workspace_id=payload.workspace_id, + created_from_doc_id=payload.created_from_doc_id, + memory_type=payload.memory_type, + canonical_text=payload.canonical_text, + summary=payload.summary, + confidence=payload.confidence, + importance=payload.importance, + status=payload.status, + access_level=payload.access_level, + current_revision_no=1, + created_at=now, + updated_at=now, + evidence_count=len(payload.evidence), + ) + + +class FakeLlmClient: + def __init__(self, chunk_id: UUID) -> None: + self.chunk_id = chunk_id + self.calls = [] + + def analyze(self, *, chunks, max_candidates, options): + self.calls.append( + {"chunks": chunks, "max_candidates": max_candidates, "options": options} + ) + return [ + LlmCandidateDraft( + chunk_id=self.chunk_id, + canonical_text="Private budget notes must stay hidden from project agents.", + memory_type="policy", + summary="Private budget visibility", + confidence=0.88, + importance=4, + ) + ] + + +def test_llm_extraction_writes_candidate_with_llm_fields() -> None: + workspace_id = uuid4() + doc_id = uuid4() + chunk_id = uuid4() + database = FakeDatabase( + [ + { + "chunk_id": chunk_id, + "doc_id": doc_id, + "chunk_no": 1, + "chunk_text": "Private budget notes must stay hidden from project agents.", + "start_line": 3, + "end_line": 3, + } + ] + ) + memory_service = FakeMemoryService() + llm_client = FakeLlmClient(chunk_id) + service = MemoryExtractionService( + database=database, + memory_service=memory_service, + llm_client=llm_client, + ) + + candidates = service.extract_from_chunks( + MemoryExtractionFromChunksRequest( + workspace_id=workspace_id, + chunk_ids=[chunk_id], + max_candidates=2, + method="llm", + llm=LlmAnalysisOptions(api_key="test-key", model="analysis-model"), + ), + ActorContext(actor_type="system", revision_reason="test extraction"), + ) + + assert len(candidates) == 1 + created_payload, _actor = memory_service.created[0] + assert created_payload.status == "candidate" + assert created_payload.memory_type == "policy" + assert created_payload.confidence == 0.88 + assert created_payload.importance == 4 + assert created_payload.evidence[0].chunk_id == chunk_id + assert created_payload.evidence[0].note == "Generated by LLM-backed memory extraction." + assert llm_client.calls[0]["options"].api_key == "test-key" + assert database.audit_rows[0]["method"] == "llm" + assert database.audit_rows[0]["llm"]["model"] == "analysis-model" diff --git a/backend/tests/test_qa.py b/backend/tests/test_qa.py index 587c26f..f108c99 100644 --- a/backend/tests/test_qa.py +++ b/backend/tests/test_qa.py @@ -18,7 +18,14 @@ def raise_for_status(self) -> None: return None def json(self) -> dict: - return {"choices": [{"message": {"content": "Answer with [M1]."}}]} + return { + "choices": [{"message": {"content": "Answer with [M1]."}}], + "usage": { + "prompt_tokens": 120, + "completion_tokens": 8, + "total_tokens": 128, + }, + } class FakeClient: def __init__(self, *, timeout): @@ -60,6 +67,9 @@ def post(self, url, *, headers, json): assert result.content == "Answer with [M1]." assert result.provider == "deepseek" assert result.model == "deepseek-chat" + assert result.prompt_tokens == 120 + assert result.completion_tokens == 8 + assert result.total_tokens == 128 def test_qa_answer_api_returns_generated_answer() -> None: diff --git a/database/00_init.sql b/database/00_init.sql index 7b37307..442caf2 100644 --- a/database/00_init.sql +++ b/database/00_init.sql @@ -1,2 +1,7 @@ +-- PostgreSQL extension bootstrap for the MemoryBase course demo. +-- pgcrypto supplies gen_random_uuid() used by all UUID primary keys. CREATE EXTENSION IF NOT EXISTS pgcrypto; + +-- pg_trgm supplies trigram similarity and GIN operator classes for fuzzy +-- source/chunk/memory search paths in database/04_indexes.sql. CREATE EXTENSION IF NOT EXISTS pg_trgm; diff --git a/database/01_schema_core.sql b/database/01_schema_core.sql index 99b07ee..c838796 100644 --- a/database/01_schema_core.sql +++ b/database/01_schema_core.sql @@ -1,3 +1,5 @@ +-- Human users who own workspaces, import sources, review governance actions, +-- and appear as actors in audit records. CREATE TABLE IF NOT EXISTS user_account ( user_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), username VARCHAR(80) NOT NULL UNIQUE, @@ -9,6 +11,8 @@ CREATE TABLE IF NOT EXISTS user_account ( updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); +-- A tenant boundary for all source, memory, governance, and wiki data. +-- Most query indexes start with workspace_id for course-scale multi-tenant isolation. CREATE TABLE IF NOT EXISTS workspace ( workspace_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), slug VARCHAR(64) NOT NULL UNIQUE, @@ -21,6 +25,8 @@ CREATE TABLE IF NOT EXISTS workspace ( updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); +-- Agent identities are stored separately from users so visibility policies can +-- target either a specific agent or an agent role/type. CREATE TABLE IF NOT EXISTS agent ( agent_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, @@ -34,6 +40,8 @@ CREATE TABLE IF NOT EXISTS agent ( UNIQUE(workspace_id, name) ); +-- Workspace membership accepts both users and agents. principal_id is generic +-- by design; application/service code validates the matching principal table. CREATE TABLE IF NOT EXISTS workspace_member ( workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, principal_type VARCHAR(20) NOT NULL CHECK (principal_type IN ('user', 'agent')), diff --git a/database/02_schema_memory.sql b/database/02_schema_memory.sql index 2afe8ec..8abc02f 100644 --- a/database/02_schema_memory.sql +++ b/database/02_schema_memory.sql @@ -1,3 +1,5 @@ +-- A logical work session for meetings, chats, imports, manual edits, or CLI runs. +-- Messages and imported sources can point back to the session that produced them. CREATE TABLE IF NOT EXISTS agent_session ( session_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, @@ -10,6 +12,8 @@ CREATE TABLE IF NOT EXISTS agent_session ( ended_at TIMESTAMPTZ ); +-- Conversation message log. It preserves Agent Runtime inputs/outputs without +-- forcing every message to become long-term memory. CREATE TABLE IF NOT EXISTS message ( message_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), session_id UUID NOT NULL REFERENCES agent_session(session_id) ON DELETE CASCADE, @@ -21,6 +25,8 @@ CREATE TABLE IF NOT EXISTS message ( reply_to_message_id UUID REFERENCES message(message_id) ON DELETE SET NULL ); +-- Raw imported document or inline agent note. Source rows are the provenance +-- root for chunks, memories, evidence, and wiki citations. CREATE TABLE IF NOT EXISTS source_document ( doc_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, @@ -39,6 +45,8 @@ CREATE TABLE IF NOT EXISTS source_document ( UNIQUE(workspace_id, checksum) ); +-- Searchable chunks derived from a source document. search_vector is generated +-- from pre-tokenized search_text_zh so PostgreSQL GIN can serve lexical recall. CREATE TABLE IF NOT EXISTS source_chunk ( chunk_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), doc_id UUID NOT NULL REFERENCES source_document(doc_id) ON DELETE CASCADE, @@ -54,6 +62,8 @@ CREATE TABLE IF NOT EXISTS source_chunk ( UNIQUE(doc_id, chunk_no) ); +-- Long-term memory unit. Candidate extraction, access control, validity windows, +-- conflict lifecycle, and revision pointers all converge on this table. CREATE TABLE IF NOT EXISTS memory_item ( memory_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, @@ -108,6 +118,8 @@ CREATE TABLE IF NOT EXISTS memory_item ( UNIQUE(memory_id, workspace_id) ); +-- Immutable version history for each memory. Triggers insert revision rows when +-- memory content or governance fields change. CREATE TABLE IF NOT EXISTS memory_revision ( memory_id UUID NOT NULL REFERENCES memory_item(memory_id) ON DELETE CASCADE, revision_no INT NOT NULL, @@ -121,6 +133,8 @@ CREATE TABLE IF NOT EXISTS memory_revision ( PRIMARY KEY (memory_id, revision_no) ); +-- Many-to-many evidence bridge between memories and source chunks. evidence_role +-- distinguishes supporting, refuting, contextual, and inline source evidence. CREATE TABLE IF NOT EXISTS memory_evidence ( evidence_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), memory_id UUID NOT NULL REFERENCES memory_item(memory_id) ON DELETE CASCADE, @@ -133,6 +147,8 @@ CREATE TABLE IF NOT EXISTS memory_evidence ( UNIQUE(memory_id, chunk_id, evidence_role) ); +-- Optional memory-level embedding cache. The project stores vectors in JSONB to +-- avoid a hard pgvector dependency while still supporting hybrid recall demos. CREATE TABLE IF NOT EXISTS memory_embedding ( embedding_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), memory_id UUID NOT NULL REFERENCES memory_item(memory_id) ON DELETE CASCADE, @@ -148,6 +164,8 @@ CREATE TABLE IF NOT EXISTS memory_embedding ( REFERENCES memory_item(memory_id, workspace_id) ON DELETE CASCADE ); +-- Optional source-chunk embedding cache. Chunk embeddings can contribute vector +-- candidates through memory_evidence during hybrid recall. CREATE TABLE IF NOT EXISTS source_chunk_embedding ( embedding_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), chunk_id UUID NOT NULL REFERENCES source_chunk(chunk_id) ON DELETE CASCADE, @@ -162,6 +180,8 @@ CREATE TABLE IF NOT EXISTS source_chunk_embedding ( UNIQUE(chunk_id, provider, model, embedding_text_hash) ); +-- Lightweight semantic entity inside a workspace, used for memory organization +-- and final-report/wiki inspection rather than a full knowledge graph. CREATE TABLE IF NOT EXISTS entity ( entity_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, @@ -178,6 +198,8 @@ CREATE TABLE IF NOT EXISTS entity ( UNIQUE(workspace_id, canonical_name) ); +-- Memory-to-entity bridge. workspace_id participates in composite FKs so the +-- database rejects cross-workspace associations. CREATE TABLE IF NOT EXISTS memory_entity ( memory_id UUID NOT NULL, entity_id UUID NOT NULL, @@ -192,6 +214,8 @@ CREATE TABLE IF NOT EXISTS memory_entity ( REFERENCES entity(entity_id, workspace_id) ON DELETE CASCADE ); +-- A named scene groups memories into a topic, decision, or narrative unit for +-- demo pages and wiki projection. CREATE TABLE IF NOT EXISTS memory_scene ( scene_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, @@ -204,6 +228,8 @@ CREATE TABLE IF NOT EXISTS memory_scene ( UNIQUE(workspace_id, scene_slug) ); +-- Scene membership bridge with ordering and narrative role metadata. +-- Composite FKs enforce that scene and memory belong to the same workspace. CREATE TABLE IF NOT EXISTS memory_scene_cell ( scene_id UUID NOT NULL, memory_id UUID NOT NULL, diff --git a/database/03_schema_governance.sql b/database/03_schema_governance.sql index ffae89c..a99e713 100644 --- a/database/03_schema_governance.sql +++ b/database/03_schema_governance.sql @@ -1,3 +1,5 @@ +-- Human-readable projection of memories or scenes. The page row stores current +-- state, while immutable page contents live in wiki_page_revision. CREATE TABLE IF NOT EXISTS wiki_page ( page_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, @@ -17,6 +19,8 @@ CREATE TABLE IF NOT EXISTS wiki_page ( UNIQUE(workspace_id, page_slug) ); +-- Immutable wiki version body. frontmatter_json keeps export metadata such as +-- memory_ids and source_doc_ids without changing the core page schema. CREATE TABLE IF NOT EXISTS wiki_page_revision ( page_id UUID NOT NULL REFERENCES wiki_page(page_id) ON DELETE CASCADE, revision_no INT NOT NULL, @@ -28,6 +32,7 @@ CREATE TABLE IF NOT EXISTS wiki_page_revision ( PRIMARY KEY (page_id, revision_no) ); +-- Project timeline event used by the demo and report to explain decisions over time. CREATE TABLE IF NOT EXISTS timeline_entry ( timeline_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, @@ -42,6 +47,8 @@ CREATE TABLE IF NOT EXISTS timeline_entry ( created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); +-- Recall invocation log. JSON fields preserve filters, ranked memory snapshots, +-- and generated context packs for later audit and evaluation. CREATE TABLE IF NOT EXISTS recall_log ( recall_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, @@ -55,6 +62,8 @@ CREATE TABLE IF NOT EXISTS recall_log ( created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); +-- Access policy rule for users, agents, or roles. principal_id may be NULL for +-- global role policies, which is why database/04_indexes.sql adds a partial unique index. CREATE TABLE IF NOT EXISTS access_policy ( policy_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, @@ -70,6 +79,8 @@ CREATE TABLE IF NOT EXISTS access_policy ( UNIQUE(workspace_id, principal_type, principal_id, resource_type, resource_scope, effect) ); +-- Soft-forget governance request. Targets are generic so one workflow can cover +-- memory_item, source_document, wiki_page, and entity. CREATE TABLE IF NOT EXISTS forget_request ( request_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, @@ -85,6 +96,8 @@ CREATE TABLE IF NOT EXISTS forget_request ( resolved_at TIMESTAMPTZ ); +-- Pairwise memory conflict record. CHECK(left_memory_id < right_memory_id) +-- canonicalizes unordered pairs and prevents A/B plus B/A duplicates. CREATE TABLE IF NOT EXISTS conflict_record ( conflict_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, @@ -115,6 +128,8 @@ CREATE TABLE IF NOT EXISTS conflict_record ( UNIQUE(left_memory_id, right_memory_id) ); +-- Append-only audit log for lifecycle events. before_json/after_json preserve +-- record snapshots even if the relational schema evolves later. CREATE TABLE IF NOT EXISTS audit_log ( audit_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workspace_id UUID NOT NULL REFERENCES workspace(workspace_id) ON DELETE CASCADE, diff --git a/database/05_views.sql b/database/05_views.sql index c3231fa..090994e 100644 --- a/database/05_views.sql +++ b/database/05_views.sql @@ -7,6 +7,8 @@ DROP VIEW IF EXISTS v_memory_recall_statistics; DROP VIEW IF EXISTS v_memory_statistics; DROP VIEW IF EXISTS v_active_memory; +-- Active, currently valid memories. This view is the safe default for recall, +-- wiki composition, and dashboard lists. CREATE OR REPLACE VIEW v_active_memory AS SELECT memory_id, @@ -32,6 +34,8 @@ WHERE status = 'active' AND valid_from <= now() AND (valid_to IS NULL OR valid_to > now()); +-- Provenance join from memory to evidence chunk and source document. It powers +-- Memory Inspector, SQL demos, and report screenshots. CREATE OR REPLACE VIEW v_memory_with_source AS SELECT mi.memory_id, @@ -67,6 +71,7 @@ JOIN source_chunk sc ON sc.chunk_id = me.chunk_id JOIN source_document sd ON sd.doc_id = sc.doc_id WHERE sd.status = 'active'; +-- Timeline entries enriched with optional memory/source context for project-history views. CREATE OR REPLACE VIEW v_project_timeline AS SELECT te.workspace_id, @@ -88,6 +93,8 @@ FROM timeline_entry te LEFT JOIN memory_item mi ON mi.memory_id = te.memory_id LEFT JOIN source_document sd ON sd.doc_id = te.doc_id; +-- Wiki provenance chain. It supports both direct generated_from_memory_id pages +-- and scene-generated pages through a LATERAL union. CREATE OR REPLACE VIEW v_wiki_page_sources AS SELECT wp.workspace_id, @@ -152,6 +159,7 @@ LEFT JOIN source_document sd ON sd.doc_id = sc.doc_id WHERE wp.status = 'active' AND (sd.doc_id IS NULL OR sd.status = 'active'); +-- Aggregated memory counts and averages for dashboard/report tables. CREATE OR REPLACE VIEW v_memory_statistics AS SELECT workspace_id, @@ -164,6 +172,8 @@ SELECT FROM memory_item GROUP BY workspace_id, memory_type, status, access_level; +-- Reverse lookup from recall_log snapshots to per-memory recall frequency. +-- The JSON array is intentionally a historical snapshot, not a live join table. CREATE OR REPLACE VIEW v_memory_recall_statistics AS SELECT rl.workspace_id, @@ -176,6 +186,8 @@ JOIN memory_item mi ON mi.memory_id = memory_ids.memory_id_text::uuid WHERE mi.workspace_id = rl.workspace_id GROUP BY rl.workspace_id, memory_ids.memory_id_text; +-- Per-agent visibility materialization. Callers must still filter by agent_id; +-- the view itself expands all active agent/memory pairs that pass AccessPolicy. CREATE OR REPLACE VIEW v_agent_visible_memory AS SELECT a.agent_id, @@ -229,6 +241,7 @@ WHERE mi.status = 'active' ) ); +-- Conflict records with both endpoint memories expanded for UI and SQL demos. CREATE OR REPLACE VIEW v_conflict_memory AS SELECT cr.conflict_id, diff --git a/database/06_triggers.sql b/database/06_triggers.sql index 64f5e44..127e3a2 100644 --- a/database/06_triggers.sql +++ b/database/06_triggers.sql @@ -15,6 +15,8 @@ DROP TRIGGER IF EXISTS trg_conflict_after_update ON conflict_record; DROP FUNCTION IF EXISTS fn_memory_revision_fields_changed(); +-- Actor context helpers read transaction-local settings set by services or SQL fixtures. +-- They let triggers attribute revisions and audit rows without extra trigger arguments. CREATE OR REPLACE FUNCTION fn_current_actor_type() RETURNS VARCHAR(20) AS $$ BEGIN @@ -22,6 +24,7 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- Returns NULL when no actor id is supplied, which is valid for system actions. CREATE OR REPLACE FUNCTION fn_current_actor_id() RETURNS UUID AS $$ BEGIN @@ -29,6 +32,7 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- Allows service code to override the revision reason for a transaction. CREATE OR REPLACE FUNCTION fn_revision_reason(default_reason TEXT) RETURNS TEXT AS $$ BEGIN @@ -36,6 +40,7 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- Shared updated_at maintainer for tables with mutable rows. CREATE OR REPLACE FUNCTION fn_touch_updated_at() RETURNS TRIGGER AS $$ BEGIN @@ -72,6 +77,8 @@ CREATE TRIGGER trg_conflict_touch BEFORE UPDATE ON conflict_record FOR EACH ROW EXECUTE FUNCTION fn_touch_updated_at(); +-- Revision-worthy memory changes. Pure updated_at changes should not create +-- a new revision number. CREATE OR REPLACE FUNCTION fn_memory_revision_fields_changed( old_memory memory_item, new_memory memory_item @@ -90,6 +97,7 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- Marks direct wiki projections as stale when their source memory changes. CREATE OR REPLACE FUNCTION fn_mark_wiki_rebuild_for_memory(target_memory_id UUID) RETURNS VOID AS $$ BEGIN @@ -100,6 +108,8 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- Before update: decide the next current_revision_no atomically before the row +-- is written, so the after trigger can insert the matching revision row. CREATE OR REPLACE FUNCTION fn_memory_before_update() RETURNS TRIGGER AS $$ BEGIN @@ -117,6 +127,7 @@ CREATE TRIGGER trg_memory_before_update BEFORE UPDATE ON memory_item FOR EACH ROW EXECUTE FUNCTION fn_memory_before_update(); +-- After insert: create the initial immutable revision and audit snapshot. CREATE OR REPLACE FUNCTION fn_memory_after_insert() RETURNS TRIGGER AS $$ BEGIN @@ -156,6 +167,8 @@ CREATE TRIGGER trg_memory_after_insert AFTER INSERT ON memory_item FOR EACH ROW EXECUTE FUNCTION fn_memory_after_insert(); +-- After update: append revision/audit rows and convert lifecycle status changes +-- into specific action_type values for governance reports. CREATE OR REPLACE FUNCTION fn_memory_after_update() RETURNS TRIGGER AS $$ DECLARE @@ -208,6 +221,8 @@ CREATE TRIGGER trg_memory_after_update AFTER UPDATE ON memory_item FOR EACH ROW EXECUTE FUNCTION fn_memory_after_update(); +-- Soft delete for direct memory deletes. Workspace cascade deletes are allowed +-- to hard-delete by checking whether the parent workspace still exists. CREATE OR REPLACE FUNCTION fn_memory_soft_delete() RETURNS TRIGGER AS $$ BEGIN @@ -233,6 +248,7 @@ CREATE TRIGGER trg_memory_soft_delete BEFORE DELETE ON memory_item FOR EACH ROW EXECUTE FUNCTION fn_memory_soft_delete(); +-- Helper used by conflict triggers to decide whether a memory can return to active. CREATE OR REPLACE FUNCTION fn_memory_has_open_conflict(target_memory_id UUID) RETURNS BOOLEAN AS $$ BEGIN @@ -248,6 +264,7 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- When an open conflict is created, active endpoint memories become conflicted. CREATE OR REPLACE FUNCTION fn_conflict_mark_memory_conflicted(target_memory_id UUID) RETURNS VOID AS $$ BEGIN @@ -258,6 +275,7 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- Restore a conflicted memory only after all open conflicts involving it are closed. CREATE OR REPLACE FUNCTION fn_conflict_restore_memory_if_clear(target_memory_id UUID) RETURNS VOID AS $$ BEGIN @@ -270,6 +288,7 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- New open conflicts immediately mark both endpoint memories as conflicted. CREATE OR REPLACE FUNCTION fn_conflict_after_insert() RETURNS TRIGGER AS $$ BEGIN @@ -286,6 +305,7 @@ CREATE TRIGGER trg_conflict_after_insert AFTER INSERT ON conflict_record FOR EACH ROW EXECUTE FUNCTION fn_conflict_after_insert(); +-- Conflict status transitions keep memory endpoint statuses synchronized. CREATE OR REPLACE FUNCTION fn_conflict_after_update() RETURNS TRIGGER AS $$ BEGIN @@ -305,6 +325,8 @@ CREATE TRIGGER trg_conflict_after_update AFTER UPDATE OF status ON conflict_record FOR EACH ROW EXECUTE FUNCTION fn_conflict_after_update(); +-- A wiki revision insert advances the page pointer, clears needs_rebuild, and +-- writes an audit row for the generated version. CREATE OR REPLACE FUNCTION fn_wiki_revision_after_insert() RETURNS TRIGGER AS $$ BEGIN diff --git a/database/07_seed.sql b/database/07_seed.sql index 4e29f49..d0b7699 100644 --- a/database/07_seed.sql +++ b/database/07_seed.sql @@ -28,6 +28,8 @@ TRUNCATE TABLE user_account CASCADE; +-- Users, workspace, agent, membership, and session create a closed demo tenant. +-- Fixed UUIDs make frontend constants, API examples, and SQL screenshots repeatable. INSERT INTO user_account(user_id, username, display_name, email, role_hint) VALUES ('00000000-0000-0000-0000-000000000101', 'alice', 'Alice Zhang', 'alice@example.com', 'admin'), @@ -63,6 +65,8 @@ VALUES ('00000000-0000-0000-0000-000000000201', 'user', '00000000-0000-0000-0000-000000000104', 'viewer'), ('00000000-0000-0000-0000-000000000201', 'agent', '00000000-0000-0000-0000-000000000301', 'agent'); +-- The session/message rows show that MemoryBase can preserve runtime dialogue +-- without forcing every message to become a long-term memory. INSERT INTO agent_session(session_id, workspace_id, agent_id, started_by_user_id, title, channel, started_at) VALUES ( '00000000-0000-0000-0000-000000000401', @@ -95,6 +99,8 @@ VALUES '2026-03-02 09:06:00+00' ); +-- Six source documents simulate a small project history. They are intentionally +-- compact so recall, evidence, wiki export, and SQL screenshots stay explainable. INSERT INTO source_document( doc_id, workspace_id, session_id, doc_type, title, source_path, raw_text, checksum, imported_by_user_id, imported_at @@ -173,6 +179,8 @@ VALUES '2026-03-25 10:00:00+00' ); +-- Source chunks preserve line ranges and become the evidence targets for memory +-- items. search_text_zh is backfilled after seed by backend/scripts/backfill_search_terms.py. INSERT INTO source_chunk(chunk_id, doc_id, chunk_no, chunk_text, start_line, end_line, token_count) VALUES ('00000000-0000-0000-0000-000000000601', '00000000-0000-0000-0000-000000000501', 1, 'The team first considered a campus cafeteria ordering system. The idea was familiar, but it did not show enough database depth for the course.', 6, 8, 24), @@ -200,6 +208,8 @@ SELECT set_config('app.actor_type', 'system', false); SELECT set_config('app.actor_id', '', false); SELECT set_config('app.revision_reason', 'seed import', false); +-- Memory inserts intentionally go through the normal trigger path. The triggers +-- create memory_revision and audit_log rows, proving the lifecycle logic in seed data. INSERT INTO memory_item( memory_id, workspace_id, created_from_doc_id, memory_type, canonical_text, summary, confidence, importance, status, access_level, owner_user_id, owner_agent_id, valid_from @@ -226,6 +236,8 @@ VALUES ('00000000-0000-0000-0000-000000000719', '00000000-0000-0000-0000-000000000201', '00000000-0000-0000-0000-000000000506', 'decision', 'The demo answer to the cafeteria question is that the cafeteria system was too CRUD-heavy and did not demonstrate enough database features.', 'Demo recall answer.', 0.970, 5, 'active', 'project', '00000000-0000-0000-0000-000000000102', NULL, '2026-03-25 10:15:00+00'), ('00000000-0000-0000-0000-000000000720', '00000000-0000-0000-0000-000000000201', '00000000-0000-0000-0000-000000000506', 'decision', 'LLM automatic extraction is useful later but is not required for the MVP demo.', 'LLM extraction is future work.', 0.880, 4, 'active', 'project', '00000000-0000-0000-0000-000000000103', '00000000-0000-0000-0000-000000000301', '2026-03-25 10:20:00+00'); +-- Evidence rows are the core provenance bridge: every seeded memory used in the +-- demo can be traced back to a concrete source chunk and evidence role. INSERT INTO memory_evidence(memory_id, chunk_id, evidence_role, weight, note) VALUES ('00000000-0000-0000-0000-000000000701', '00000000-0000-0000-0000-000000000602', 'supports', 1.000, 'Direct reason for abandoning cafeteria system.'), @@ -251,6 +263,8 @@ VALUES SELECT set_config('app.revision_reason', 'seed correction', false); +-- These updates are deliberate: they exercise revision/audit triggers so the demo +-- database contains non-trivial memory history, not only initial inserts. UPDATE memory_item SET summary = 'Reason cafeteria topic was rejected.' WHERE memory_id = '00000000-0000-0000-0000-000000000701'; @@ -267,10 +281,14 @@ UPDATE memory_item SET canonical_text = 'LLM automatic extraction is useful later but is not required for the deterministic MVP demo.' WHERE memory_id = '00000000-0000-0000-0000-000000000720'; +-- This team-level memory demonstrates that visibility is not only public/project; +-- recall without an agent and recall with a configured agent can differ. UPDATE memory_item SET access_level = 'team' WHERE memory_id = '00000000-0000-0000-0000-000000000717'; +-- Entity and scene rows provide lightweight semantic organization without turning +-- the project into a full knowledge-graph system. INSERT INTO entity(entity_id, workspace_id, canonical_name, entity_type, description) VALUES ( @@ -347,6 +365,8 @@ VALUES 'Captures the final answer expected in the demo.' ); +-- Access policies make the seeded retriever agent able to see project memory while +-- explicitly denying private memory. Query #4 in database/08_demo_queries.sql uses this. INSERT INTO access_policy(workspace_id, principal_type, principal_id, resource_type, resource_scope, effect) VALUES ('00000000-0000-0000-0000-000000000201', 'agent', '00000000-0000-0000-0000-000000000301', 'memory_item', 'project', 'allow'), @@ -366,6 +386,8 @@ VALUES ( '2026-03-25 11:00:00+00' ); +-- RecallLog is seeded as a historical snapshot. It is intentionally JSONB-heavy +-- because the context pack shape can evolve without changing old audit records. INSERT INTO recall_log( recall_id, workspace_id, agent_id, user_id, query_text, filter_json, result_count, top_memory_ids_json, context_pack_json, created_at @@ -383,6 +405,8 @@ VALUES ( '2026-03-25 11:10:00+00' ); +-- Timeline and Wiki rows give the frontend/report a human-readable projection of +-- the same memories that the database and Agent APIs use. INSERT INTO timeline_entry( timeline_id, workspace_id, memory_id, doc_id, event_type, title, description, event_time, importance ) diff --git a/database/08_demo_queries.sql b/database/08_demo_queries.sql index 901ebfd..6aa37a6 100644 --- a/database/08_demo_queries.sql +++ b/database/08_demo_queries.sql @@ -20,11 +20,11 @@ LIMIT 20; SELECT workspace_id, memory_type, status, access_level, memory_count, avg_confidence, avg_importance FROM v_memory_statistics; --- 5. Query agent-visible memory after the app sets app.agent_id -SELECT set_config('app.agent_id', '00000000-0000-0000-0000-000000000301', false); - +-- 5. Query memory visible to the demo retriever agent +-- v_agent_visible_memory materializes visibility for every agent; callers filter explicitly. SELECT memory_id, memory_type, canonical_text, access_level, confidence FROM v_agent_visible_memory +WHERE agent_id = '00000000-0000-0000-0000-000000000301' ORDER BY importance DESC, updated_at DESC LIMIT 20; diff --git a/database/09_graph_demo.sql b/database/09_graph_demo.sql index e67bcfa..41c18ea 100644 --- a/database/09_graph_demo.sql +++ b/database/09_graph_demo.sql @@ -1,6 +1,8 @@ -- Optional curated graph demo seed. -- Run after 07_seed.sql when you want a cleaner graph visualization example. +-- Remove the optional graph workspace before re-inserting fixed IDs. This keeps +-- the file re-runnable without touching the main cs3321-demo workspace. DELETE FROM workspace WHERE workspace_id = '00000000-0000-0000-0000-000000002201'; DELETE FROM user_account WHERE user_id IN ( @@ -8,6 +10,8 @@ WHERE user_id IN ( '00000000-0000-0000-0000-000000002102' ); +-- Graph demo uses a separate workspace so screenshots can show a small clean +-- provenance graph instead of the denser main course seed. INSERT INTO user_account(user_id, username, display_name, email, role_hint) VALUES ('00000000-0000-0000-0000-000000002101', 'graph-alice', 'Graph Demo Alice', 'graph-alice@example.com', 'admin'), @@ -39,6 +43,8 @@ VALUES ('00000000-0000-0000-0000-000000002201', 'user', '00000000-0000-0000-0000-000000002102', 'editor'), ('00000000-0000-0000-0000-000000002201', 'agent', '00000000-0000-0000-0000-000000002301', 'agent'); +-- Three source documents form the graph story: topic pivot, architecture split, +-- and governance. The downstream chunks/memories/entities mirror these themes. INSERT INTO source_document( doc_id, workspace_id, doc_type, title, source_path, raw_text, checksum, imported_by_user_id, imported_at ) @@ -77,6 +83,8 @@ VALUES '2026-04-05 09:00:00+00' ); +-- Chunks are deliberately short so Graph Explorer node tooltips and evidence +-- edges remain readable in screenshots. INSERT INTO source_chunk(chunk_id, doc_id, chunk_no, chunk_text, start_line, end_line, token_count) VALUES ('00000000-0000-0000-0000-000000002601', '00000000-0000-0000-0000-000000002501', 1, 'The team rejected the cafeteria ordering topic because it mostly demonstrated CRUD.', 4, 4, 12), @@ -92,6 +100,8 @@ SELECT set_config('app.actor_type', 'system', false); SELECT set_config('app.actor_id', '', false); SELECT set_config('app.revision_reason', 'graph demo seed', false); +-- Memory inserts go through the normal trigger path, creating revision/audit +-- records while also providing the Memory nodes used by the graph preview. INSERT INTO memory_item( memory_id, workspace_id, created_from_doc_id, memory_type, canonical_text, summary, confidence, importance, status, access_level, owner_user_id, valid_from @@ -105,6 +115,7 @@ VALUES ('00000000-0000-0000-0000-000000002706', '00000000-0000-0000-0000-000000002201', '00000000-0000-0000-0000-000000002503', 'semantic', 'Access policies decide which memories an agent can see.', 'Agent visibility policy', 0.910, 4, 'active', 'project', '00000000-0000-0000-0000-000000002102', '2026-04-05 09:10:00+00'), ('00000000-0000-0000-0000-000000002707', '00000000-0000-0000-0000-000000002201', '00000000-0000-0000-0000-000000002503', 'semantic', 'Wiki provenance lets readers trace generated pages back to memories and evidence.', 'Wiki provenance', 0.930, 5, 'active', 'project', '00000000-0000-0000-0000-000000002101', '2026-04-05 09:12:00+00'); +-- Evidence creates SUPPORTED_BY edges in graph_service.py. INSERT INTO memory_evidence(memory_id, chunk_id, evidence_role, weight, note) VALUES ('00000000-0000-0000-0000-000000002701', '00000000-0000-0000-0000-000000002601', 'supports', 1.000, 'Topic rejection reason.'), @@ -116,6 +127,8 @@ VALUES ('00000000-0000-0000-0000-000000002706', '00000000-0000-0000-0000-000000002607', 'supports', 1.000, 'Policy visibility note.'), ('00000000-0000-0000-0000-000000002707', '00000000-0000-0000-0000-000000002608', 'supports', 1.000, 'Traceability note.'); +-- Entities and memory_entity rows create MENTIONS edges, making the graph more +-- informative than a source->chunk->memory chain alone. INSERT INTO entity(entity_id, workspace_id, canonical_name, entity_type, description) VALUES ('00000000-0000-0000-0000-000000002801', '00000000-0000-0000-0000-000000002201', 'Cafeteria Ordering System', 'project', 'Rejected CRUD-heavy project idea.'), @@ -132,6 +145,8 @@ VALUES ('00000000-0000-0000-0000-000000002704', '00000000-0000-0000-0000-000000002804', '00000000-0000-0000-0000-000000002201', 'about'), ('00000000-0000-0000-0000-000000002707', '00000000-0000-0000-0000-000000002805', '00000000-0000-0000-0000-000000002201', 'about'); +-- Scenes group memories into story units. Graph Explorer uses them to show that +-- MemoryBase can organize memories into narrative views, not just flat search hits. INSERT INTO memory_scene(scene_id, workspace_id, scene_slug, title, summary) VALUES ('00000000-0000-0000-0000-000000002901', '00000000-0000-0000-0000-000000002201', 'topic-pivot', 'Topic Pivot', 'Why the project moved from cafeteria ordering to MemoryBase.'), @@ -148,6 +163,8 @@ VALUES ('00000000-0000-0000-0000-000000002903', '00000000-0000-0000-0000-000000002706', '00000000-0000-0000-0000-000000002201', 'support', 10, 'Agent visibility.'), ('00000000-0000-0000-0000-000000002903', '00000000-0000-0000-0000-000000002707', '00000000-0000-0000-0000-000000002201', 'outcome', 20, 'Trustworthy wiki output.'); +-- Wiki pages are derived from scenes/memories, producing DERIVED_FROM edges in +-- the workspace graph and demonstrating readable projections of database memory. INSERT INTO wiki_page(page_id, workspace_id, page_slug, page_type, title, generated_from_scene_id, generated_from_memory_id, needs_rebuild) VALUES ('00000000-0000-0000-0000-000000003301', '00000000-0000-0000-0000-000000002201', 'project-pivot-story', 'synthesis', 'Project Pivot Story', '00000000-0000-0000-0000-000000002901', '00000000-0000-0000-0000-000000002702', false), diff --git a/database/10_governance_demo_fixture.sql b/database/10_governance_demo_fixture.sql index d19043d..8719fb0 100644 --- a/database/10_governance_demo_fixture.sql +++ b/database/10_governance_demo_fixture.sql @@ -17,7 +17,7 @@ -- 复现： -- psql $DATABASE_URL -f database/10_governance_demo_fixture.sql -- --- 演示查询：见 docs/governance-demo-walkthrough.md（如有）或 database/08_demo_queries.sql +-- 演示查询：见 docs/governance-demo-walkthrough.md 或 database/08_demo_queries.sql -- ============================================================ \set ON_ERROR_STOP on diff --git a/docs/00-project-overview.md b/docs/00-project-overview.md index d2e550e..4deb182 100644 --- a/docs/00-project-overview.md +++ b/docs/00-project-overview.md @@ -2,11 +2,12 @@ ## 1. 项目名称 -MemoryBase：面向 AI Agent 协作研发的文件—数据库双态长期记忆系统。 +MemoryBase：面向组织与团队的 AI-native 可追溯长期记忆数据库系统。 ## 2. 一句话介绍 -把人类可读的 Markdown、会议纪要和项目文档，编译为 Agent 可检索、可追溯、可权限控制、可审计、可版本化的长期记忆数据库。 +把人类可读的 Markdown、会议纪要和项目文档，编译为人和 Agent 都能 +grep-style 访问、可追溯、可权限控制、可审计、可版本化的长期记忆数据库。 ## 3. 项目定位 @@ -36,9 +37,9 @@ SourceDocument | 表达层 | WikiPage、WikiPageRevision、TimelineEntry | 生成可读 Wiki 与时间线 | | 治理层 | AccessPolicy、RecallLog、ConflictRecord、ForgetRequest、AuditLog | 权限、审计、冲突、遗忘 | -## 6. P0 MVP +## 6. 已交付核心能力 -P0 不依赖 LLM，也不依赖向量数据库。保底功能包括： +核心能力不依赖外部 LLM，也不依赖向量数据库。保底功能包括： - 导入 Markdown / txt source - 自动切分 source chunk @@ -50,20 +51,23 @@ P0 不依赖 LLM，也不依赖向量数据库。保底功能包括： - Markdown Wiki 导出 - 基础前端页面 -## 7. P1 功能 +## 7. 已交付扩展能力 - Timeline 决策时间线 - Agent 可见视图 - AccessPolicy 权限过滤 - ConflictRecord 冲突治理 - Memory statistics 统计视图 +- rule-based + optional LLM candidate memory extraction +- local hashing embedding cache 与 hybrid recall fallback +- Graph Explorer（PostgreSQL preview + 可选 Neo4j sync） +- CLI / Agent Runtime sessions、observe、remember、search +- evaluation framework（LoCoMo / LongMemEval / MemoryAgentBench 等适配器） -## 8. P2 / Future +## 8. 未来扩展 -- ForgetRequest 轻量治理流程（已实现：提交、审批、审计、memory forgotten） -- Entity / MemoryScene 轻量语义组织（已实现：实体、场景与 M:N 关系） -- pgvector 语义检索 -- LLM 自动抽取 +- pgvector / ANN 大规模语义检索 +- LLM-backed analysis draft tables and richer review workflow - 复杂 temporal knowledge graph - 多 Agent 自动协作 - Obsidian 插件 diff --git a/docs/01-requirements.md b/docs/01-requirements.md index 3e81cf0..d2bce8c 100644 --- a/docs/01-requirements.md +++ b/docs/01-requirements.md @@ -13,15 +13,15 @@ MemoryBase 通过数据库管理长期记忆，使项目知识可以被结构化 | 普通用户 | 查看项目记忆、搜索结论、阅读 Wiki | 搜索、查看 source、查看 timeline、导出 Wiki | | 小组成员 | 维护课程项目记忆 | 导入 source、创建 memory、编辑 memory、处理 conflict | | Agent | 基于权限读取 context pack | recall、生成建议 memory、生成 wiki 草稿 | -| 管理员 | 管理用户、权限、审计 | 用户管理、Agent 管理、权限策略、归档、恢复 | +| 管理员 | 管理 Agent、权限、审计和治理状态 | Agent 注册、权限策略、审计、冲突、遗忘、归档 | | 访客 / 只读用户 | 查看公开 Wiki 或演示结果 | 浏览、搜索公开内容 | ## 3. 功能需求 | 模块 | 功能 | P0/P1/P2 | |---|---|---| -| Workspace 管理 | 创建工作区、成员管理 | P0 | -| 用户与 Agent 管理 | 管理用户、Agent 和角色 | P1 | +| Workspace 基础配置 | 使用 seed / 配置初始化工作区、维护 workspace 边界和成员表 | P0 | +| Agent 管理 | 注册 Agent、维护 Agent 可见范围和角色策略 | P1 | | Source 导入 | 导入 Markdown / txt，切分 chunk | P0 | | Message / Chunk 管理 | 保存会话消息和文档块 | P0 | | Memory 抽取与编辑 | 创建、编辑、删除 memory | P0 | @@ -33,8 +33,13 @@ MemoryBase 通过数据库管理长期记忆，使项目知识可以被结构化 | Timeline | 展示项目决策演进 | P1 | | AccessPolicy | Agent 权限过滤 | P1 | | ConflictRecord | 冲突记忆治理 | P1 | -| Entity / MemoryScene | 实体、场景和记忆聚合（轻量模型已实现） | P2 | -| ForgetRequest | 遗忘与归档申请（轻量审批流程已实现） | P2 | +| Entity / MemoryScene | 实体、场景和记忆聚合（轻量模型已实现） | P1 | +| ForgetRequest | 遗忘与归档申请（轻量审批流程已实现） | P1 | +| Graph Explorer | 展示 workspace 中 source / memory / evidence / wiki / governance 关系 | P1 | +| Embedding cache / Hybrid recall | 可选向量缓存、hybrid 检索与透明 fallback | P1 | +| Agent Runtime / CLI | session、observe、remember、search、recall 命令行入口 | P1 | +| Evaluation framework | 长期记忆评测适配器、指标与报告生成 | P1 | +| QA | 基于 recall context 的可选 LLM answer | P2 | | SkillMemory | 过程性经验归纳 | Future | ## 4. 非功能需求 @@ -71,3 +76,5 @@ MemoryBase 通过数据库管理长期记忆，使项目知识可以被结构化 - 版本审计 - 权限治理 - Wiki 投影 +- 可解释 hybrid recall 降级 +- 评测框架验证长期记忆效果 diff --git a/docs/02-data-flow.md b/docs/02-data-flow.md index 980521a..ae7dfad 100644 --- a/docs/02-data-flow.md +++ b/docs/02-data-flow.md @@ -51,7 +51,11 @@ Markdown / txt 文件 → 保存 SourceDocument → 按行数 / token 切分 → 保存 SourceChunk - → 写入 AuditLog + → 通过 imported_at / imported_by_user_id 保留导入来源 + +说明：当前实现中 source import 不直接写 `audit_log`；`audit_log` 主要覆盖 +memory、wiki、conflict、forget、extraction run 等治理事件。最终报告若要展示 +source import 审计，应先补代码或只展示 source 表内的导入元数据。 ``` ## 4. Recall 检索 2 层 DFD @@ -60,11 +64,12 @@ Markdown / txt 文件 用户 / Agent 输入 query → 解析 query 和过滤条件 → 应用 AccessPolicy - → 查询 SourceChunk FTS - → Join MemoryEvidence - → Join MemoryItem + → 查询 SourceChunk FTS / trigram + → 查询 MemoryItem 文本匹配 + → 可选查询 MemoryEmbedding / SourceChunkEmbedding + → Join MemoryEvidence / SourceChunk / SourceDocument → 返回 memory + evidence + source - → 写入 RecallLog + → 写入 RecallLog（含 retrieval_info / fallback reason） ``` ## 5. Wiki 导出 2 层 DFD diff --git a/docs/03-data-dictionary.md b/docs/03-data-dictionary.md index 4303ce3..762e179 100644 --- a/docs/03-data-dictionary.md +++ b/docs/03-data-dictionary.md @@ -21,10 +21,11 @@ | policy_id | 权限策略编号 | UUID | PK | pol001 | | audit_id | 审计日志编号 | UUID | PK | audit001 | | access_level | 访问范围 | VARCHAR | public/project/team/private | project | -| memory_type | 记忆类型 | VARCHAR | episodic/semantic/profile/procedural/decision/preference/task/risk | decision | +| memory_type | 记忆类型 | VARCHAR | episodic/semantic/fact/profile/procedural/decision/preference/task/risk/constraint/policy/summary | decision | | doc_type | SourceDocument 类型 | VARCHAR | markdown/txt/meeting/chat/note/report/inline_agent_note | inline_agent_note | | channel | AgentSession 来源通道 | VARCHAR | meeting/chat/import/manual/cli | cli | -| status | 数据状态 | VARCHAR | active/archived/forgotten/superseded/conflicted | active | +| status | MemoryItem 数据状态 | VARCHAR | candidate/active/archived/forgotten/superseded/rejected/conflicted | active | +| source_status | SourceDocument / WikiPage / Entity 状态 | VARCHAR | active/forgotten | active | | forgotten_at | 非 memory 目标被遗忘时间 | TIMESTAMPTZ | NULL 表示未被遗忘 | 2026-05-16T12:00:00Z | | confidence | 置信度 | NUMERIC | 0.00–1.00 | 0.85 | | importance | 重要性 | INT | 1–5 | 4 | @@ -34,6 +35,8 @@ | forget_status | 遗忘请求状态 | VARCHAR | pending/approved/rejected/done | approved | | search_text_zh | 中文/中英混排检索文本 | TEXT | 由 jieba 搜索模式分词后空格连接 | 校园 食堂 方向 | | search_vector | PostgreSQL 全文检索向量 | TSVECTOR | generated column, GIN index | '食堂':2 | +| embedding_json | 向量缓存 | JSONB | provider/model/dimension 下的整体向量值 | [0.1, -0.2] | +| embedding_text_hash | 向量输入文本 hash | VARCHAR(128) | 同一文本同一模型去重 | sha256... | ## 2. 数据结构字典 @@ -43,19 +46,21 @@ | Agent | agent_id、workspace_id、name、agent_type、status | 可参与检索和写入的 Agent | | AgentSession | session_id、workspace_id、agent_id、title、channel、started_at | Agent / CLI / 导入会话 | | Message | message_id、session_id、sender_type、role、content、created_at | 会话消息；observe API 的落库对象 | -| SourceDocument | doc_id、workspace_id、title、raw_text、checksum、status、forgotten_at | 原始文档；`inline_agent_note` 用于 Agent 写回时自动补证据链 | +| SourceDocument | doc_id、workspace_id、session_id、doc_type、title、raw_text、checksum、status、forgotten_at、imported_by_user_id、imported_at | 原始文档；`inline_agent_note` 用于 Agent 写回时自动补证据链 | | SourceChunk | chunk_id、doc_id、chunk_no、chunk_text、line range、search_text_zh、search_vector | 文档切块；search_vector 基于分词后的 search_text_zh 生成 | -| MemoryItem | memory_id、workspace_id、memory_type、canonical_text、summary、search_text_zh、search_vector、status | 长期记忆核心；支持 memory 级全文检索 | -| MemoryEvidence | evidence_id、memory_id、chunk_id、evidence_role | 记忆来源证据 | -| MemoryRevision | memory_id、revision_no、revision_text、editor | 记忆版本 | +| MemoryItem | memory_id、workspace_id、created_from_doc_id、memory_type、canonical_text、summary、search_text_zh、search_vector、confidence、importance、status、access_level、owner_user_id、owner_agent_id、valid_from、valid_to、superseded_by_memory_id、current_revision_no | 长期记忆核心；支持候选抽取、全文检索、权限、版本和生命周期 | +| MemoryEvidence | evidence_id、memory_id、chunk_id、evidence_role、weight、note、created_at | 记忆来源证据 | +| MemoryRevision | memory_id、revision_no、revision_text、revision_summary、revision_reason、editor_type、editor_id、created_at | 记忆版本 | +| MemoryEmbedding | embedding_id、memory_id、workspace_id、provider、model、dimension、embedding_json、embedding_text_hash、created_at | memory 级 embedding 缓存；用于 hybrid recall，不依赖 pgvector | +| SourceChunkEmbedding | embedding_id、chunk_id、doc_id、workspace_id、provider、model、dimension、embedding_json、embedding_text_hash、created_at | source chunk 级 embedding 缓存；用于 hybrid recall，不依赖 pgvector | | Entity | entity_id、workspace_id、canonical_name、entity_type、description、status、forgotten_at | 工作区内的项目对象、概念、文档或事件；支持软遗忘 | | MemoryEntity | memory_id、entity_id、workspace_id、relation_role | MemoryItem 与 Entity 的 M:N 关系 | | MemoryScene | scene_id、workspace_id、scene_slug、title、summary | 面向演示和 Wiki 的主题/决策场景 | | MemorySceneCell | scene_id、memory_id、workspace_id、cell_role、sort_order、note | MemoryScene 与 MemoryItem 的 M:N 聚合关系 | -| WikiPage | page_id、workspace_id、page_slug、title、status、forgotten_at | Wiki 页面索引；遗忘治理时保留版本历史并隐藏页面 | +| WikiPage | page_id、workspace_id、page_slug、page_type、title、current_revision_no、generated_from_scene_id、generated_from_memory_id、needs_rebuild、status、forgotten_at | Wiki 页面索引；遗忘治理时保留版本历史并隐藏页面 | | WikiPageRevision | page_id、revision_no、frontmatter_json、body_markdown | Wiki 页面版本 | -| ForgetRequest | request_id、target、requester、reviewer、reason、status、resolved_at | 遗忘/归档审批记录；审批 memory_item、source_document、wiki_page、entity 时执行对应软治理 | -| AuditLog | audit_id、actor、action、target、before/after、diff | 操作审计；支持按 target 生命周期、actor 时间线和聚合统计查询 | +| ForgetRequest | request_id、workspace_id、target_type、target_id、requester_user_id、reviewed_by_user_id、reason、status、requested_at、resolved_at | 遗忘/归档审批记录；审批 memory_item、source_document、wiki_page、entity 时执行对应软治理 | +| AuditLog | audit_id、workspace_id、actor_type、actor_id、action_type、target_type、target_id、before_json、after_json、created_at | 操作审计；支持按 target 生命周期、actor 时间线和聚合统计查询 | ## 3. 数据流字典 @@ -79,7 +84,7 @@ |---|---|---| | D1 用户与工作区库 | user_account、workspace、workspace_member、agent | 身份、成员、Agent | | D2 源文档库 | source_document、source_chunk、message、agent_session | 原始数据 | -| D3 记忆库 | memory_item、memory_revision、memory_evidence | 长期记忆 | +| D3 记忆库 | memory_item、memory_revision、memory_evidence、memory_embedding、source_chunk_embedding | 长期记忆、证据和可选 embedding 缓存 | | D4 语义结构库 | entity、memory_entity、memory_scene、memory_scene_cell | 实体、记忆实体关系、场景聚合 | | D5 表达层库 | wiki_page、wiki_page_revision、timeline_entry | Wiki 与时间线 | | D6 治理库 | access_policy、conflict_record、forget_request、audit_log、recall_log | 权限、冲突、遗忘、审计 | diff --git a/docs/04-er-design.md b/docs/04-er-design.md index c72066e..ed0941f 100644 --- a/docs/04-er-design.md +++ b/docs/04-er-design.md @@ -5,7 +5,7 @@ | 层次 | 实体 | |---|---| | 源文档层 | SourceDocument、SourceChunk、Session、Message | -| 记忆层 | MemoryItem、MemoryRevision、MemoryScene | +| 记忆层 | MemoryItem、MemoryRevision、MemoryScene、MemoryEmbedding、SourceChunkEmbedding | | 证据层 | MemoryEvidence、Entity、MemoryEntity | | 表达层 | WikiPage、WikiPageRevision、TimelineEntry | | 治理层 | AccessPolicy、RecallLog、ConflictRecord、ForgetRequest、AuditLog | @@ -17,9 +17,12 @@ | UserAccount | username、display_name、role_hint | user_id | | Agent | name、agent_type、status | agent_id | | Workspace | name、description、owner_user_id | workspace_id | +| WorkspaceMember | principal_type、principal_id、member_role | workspace_id + principal_type + principal_id | | SourceDocument | title、doc_type、source_path、raw_text、checksum | doc_id | | SourceChunk | chunk_no、chunk_text、line range | chunk_id | | MemoryItem | memory_type、canonical_text、summary、status、confidence | memory_id | +| MemoryEmbedding | provider、model、dimension、embedding_text_hash、embedding_json | embedding_id | +| SourceChunkEmbedding | provider、model、dimension、embedding_text_hash、embedding_json | embedding_id | | MemoryRevision | revision_no、revision_text、reason、editor | memory_id + revision_no | | MemoryEvidence | evidence_role、weight、note | evidence_id | | Entity | canonical_name、entity_type、description | entity_id | @@ -40,10 +43,13 @@ |---|---|---| | UserAccount — Workspace | 1:N | workspace.owner_user_id | | Workspace — Agent | 1:N | agent.workspace_id | +| Workspace — UserAccount / Agent | M:N | workspace_member | | Workspace — SourceDocument | 1:N | source_document.workspace_id | | SourceDocument — SourceChunk | 1:N | source_chunk.doc_id | | Session — Message | 1:N | message.session_id | | SourceChunk — MemoryItem | M:N | memory_evidence | +| MemoryItem — MemoryEmbedding | 1:N | memory_embedding | +| SourceChunk — SourceChunkEmbedding | 1:N | source_chunk_embedding | | MemoryItem — MemoryRevision | 1:N | memory_revision.memory_id | | MemoryScene — MemoryItem | M:N | memory_scene_cell | | MemoryItem — Entity | M:N | memory_entity | @@ -57,10 +63,13 @@ erDiagram USER_ACCOUNT ||--o{ WORKSPACE : owns WORKSPACE ||--o{ AGENT : contains + WORKSPACE ||--o{ WORKSPACE_MEMBER : has_members WORKSPACE ||--o{ SOURCE_DOCUMENT : imports SOURCE_DOCUMENT ||--o{ SOURCE_CHUNK : splits_into SOURCE_CHUNK ||--o{ MEMORY_EVIDENCE : supports + SOURCE_CHUNK ||--o{ SOURCE_CHUNK_EMBEDDING : caches_embedding MEMORY_ITEM ||--o{ MEMORY_EVIDENCE : has + MEMORY_ITEM ||--o{ MEMORY_EMBEDDING : caches_embedding MEMORY_ITEM ||--o{ MEMORY_REVISION : has_versions WORKSPACE ||--o{ ENTITY : defines MEMORY_ITEM ||--o{ MEMORY_ENTITY : tags @@ -74,10 +83,15 @@ erDiagram WORKSPACE ||--o{ RECALL_LOG : records WORKSPACE ||--o{ ACCESS_POLICY : controls WORKSPACE ||--o{ FORGET_REQUEST : reviews - MEMORY_ITEM ||--o{ CONFLICT_RECORD : conflicts + MEMORY_ITEM ||--o{ CONFLICT_RECORD : left_conflict + MEMORY_ITEM ||--o{ CONFLICT_RECORD : right_conflict WORKSPACE ||--o{ AUDIT_LOG : audits ``` +该 Mermaid ER 图是 repo 内的可审源文件。最终提交时应从本段导出 +SVG/PNG 到 `docs/final-assets/`；渲染工具可用 Mermaid CLI、HTML/SVG +脚本或 draw.io，以最终图片清晰度为准。 + ## 5. 设计说明 这个 E-R 设计体现了 MemoryBase 的核心生命周期： diff --git a/docs/05-logical-design.md b/docs/05-logical-design.md index bf3c535..c062619 100644 --- a/docs/05-logical-design.md +++ b/docs/05-logical-design.md @@ -39,6 +39,10 @@ MemoryRevision(memory_id FK, revision_no, revision_text, revision_summary, revis MemoryEvidence(evidence_id PK, memory_id FK, chunk_id FK, evidence_role, weight, note, created_at, UNIQUE(memory_id, chunk_id, evidence_role)) +MemoryEmbedding(embedding_id PK, memory_id FK, workspace_id FK, provider, model, dimension, embedding_json, embedding_text_hash, created_at, UNIQUE(memory_id, provider, model, embedding_text_hash), FK(memory_id, workspace_id)) + +SourceChunkEmbedding(embedding_id PK, chunk_id FK, doc_id FK, workspace_id FK, provider, model, dimension, embedding_json, embedding_text_hash, created_at, UNIQUE(chunk_id, provider, model, embedding_text_hash)) + Entity(entity_id PK, workspace_id FK, canonical_name, entity_type, description, status, forgotten_at, created_at, updated_at, UNIQUE(workspace_id, canonical_name), UNIQUE(entity_id, workspace_id)) MemoryEntity(memory_id FK, entity_id FK, workspace_id FK, relation_role, created_at, PK(memory_id, entity_id, relation_role), FK(memory_id, workspace_id), FK(entity_id, workspace_id)) @@ -61,7 +65,7 @@ RecallLog(recall_id PK, workspace_id FK, agent_id FK, user_id FK, query_text, fi AccessPolicy(policy_id PK, workspace_id FK, principal_type, principal_id, resource_type, resource_scope, effect, predicate_json, created_at) -ConflictRecord(conflict_id PK, workspace_id FK, left_memory_id FK, right_memory_id FK, conflict_type, status, resolution_note, created_at, resolved_at, CHECK(left_memory_id < right_memory_id), UNIQUE(left_memory_id, right_memory_id)) +ConflictRecord(conflict_id PK, workspace_id FK, left_memory_id FK, right_memory_id FK, conflict_type, status, resolution_note, resolved_by_actor_type, resolved_by_actor_id, resolved_at, created_at, updated_at, CHECK(left_memory_id < right_memory_id), UNIQUE(left_memory_id, right_memory_id)) ForgetRequest(request_id PK, workspace_id FK, target_type, target_id, requester_user_id FK, reviewed_by_user_id FK, reason, status, requested_at, resolved_at) @@ -93,6 +97,7 @@ AuditLog(audit_id PK, workspace_id FK, actor_type, actor_id, action_type, target - 审计记录独立为 AuditLog。 - Entity、MemoryScene 与 MemoryItem 的 M:N 关系通过 MemoryEntity 和 MemorySceneCell 拆分，关系属性 relation_role、cell_role、sort_order 只依赖各自复合主键。 - SourceDocument、WikiPage、Entity 的 `status` / `forgotten_at` 是治理状态，不承载业务内容依赖；默认视图和 API 过滤 `active`，遗忘审批只做软治理。 +- MemoryEmbedding、SourceChunkEmbedding 作为 embedding 缓存表保存 provider/model/dimension/hash 与 JSONB 向量值；它们服务 hybrid recall，不把向量维度拆成关系列，也不依赖 pgvector。 例如 MemoryItem 不直接保存来源文本，而通过 MemoryEvidence 关联 SourceChunk，避免将证据来源冗余存储在主表中。 @@ -105,6 +110,7 @@ AuditLog(audit_id PK, workspace_id FK, actor_type, actor_id, action_type, target | current_revision_no | wiki_page | 快速读取当前 Wiki | | token_count | source_chunk | 避免重复计算 | | search_text_zh / search_vector | source_chunk / memory_item | 预计算 jieba 搜索文本和 PostgreSQL FTS 向量，避免查询时重复分词和建向量 | +| embedding_json | memory_embedding / source_chunk_embedding | 缓存可选 embedding provider 的向量结果，避免重复调用模型 | | top_memory_ids_json | recall_log | 保留召回快照 | | workspace_id | memory_entity / memory_scene_cell | 支撑 workspace 过滤，并通过复合 FK 保证 M:N 两端属于同一 workspace | | status / forgotten_at | source_document / wiki_page / entity | 支撑 ForgetRequest 软治理和审计回放 | diff --git a/docs/06-physical-design.md b/docs/06-physical-design.md index 7ea7272..7eaf74e 100644 --- a/docs/06-physical-design.md +++ b/docs/06-physical-design.md @@ -2,11 +2,12 @@ ## 1. 数据库选择 -主方案采用 PostgreSQL，保底方案保留 SQLite。 +主方案采用 PostgreSQL。早期设计中曾考虑 SQLite + FTS5 作为本地保底方案； +当前仓库的脚本、测试和演示均以 PostgreSQL 为准，SQLite 不作为已交付能力。 | 维度 | PostgreSQL | SQLite | |---|---|---| -| 部署难度 | 中等，需要服务或 Docker | 极低 | +| 部署难度 | 中等，需要服务或 Docker | 极低（设计备选，当前未实现） | | 展示数据库能力 | 强，支持复杂视图、JSONB、GIN、触发器 | 中等 | | 全文检索 | tsvector + GIN | FTS5 | | 多人协作 | 强 | 弱 | @@ -16,7 +17,7 @@ ```text 主方案：PostgreSQL -保底方案：SQLite + FTS5 +历史备选：SQLite + FTS5（当前未进入交付脚本） ``` ## 2. 数据库名称 @@ -82,7 +83,7 @@ project-root/ |---|---| | v_active_memory | 查询 active 且仍在有效期内的 memory，使用显式列名避免 schema 漂移 | | v_memory_with_source | 串联 memory、evidence、source chunk 和 source document，支持来源追溯与行号展示 | -| v_agent_visible_memory | 基于 `app.agent_id` 和 AccessPolicy 过滤 Agent 可见 memory，未设置 agent 时默认不返回数据 | +| v_agent_visible_memory | 展开每个 Agent 可见的 active memory；调用方必须显式 `WHERE agent_id = ...`，服务层再按 AccessPolicy 过滤 | | v_project_timeline | 串联 timeline、memory 和 source，支持项目决策演进展示 | | v_conflict_memory | 展开 conflict_record 两端 memory，支持冲突页面和 SQL 演示 | | v_wiki_page_sources | 追溯 WikiPage 由 memory 或 scene 到 evidence/source chunk 的来源链路 | @@ -119,7 +120,7 @@ project-root/ 1. `memory_item` 枚举扩展 - `memory_type` 新增：`fact`、`constraint`、`policy`、`summary` - - `status` 新增：`candidate`、`rejected` + - `status` 新增：`candidate`、`rejected`、`conflicted` 作用：把自动抽取出来、尚未人工确认的候选记忆直接纳入主表生命周期，而不是另起一套孤立草稿表。这样治理、审计、召回过滤都可以沿用一套主线逻辑。 diff --git a/docs/07-system-architecture.md b/docs/07-system-architecture.md index 563604a..b383bfe 100644 --- a/docs/07-system-architecture.md +++ b/docs/07-system-architecture.md @@ -1,36 +1,40 @@ # 系统总体架构 -## 1. 架构图 +本文档描述 HEAD `2dd0d64` 的实际系统结构。MemoryBase 是一个以 +PostgreSQL 为核心的组织级 / 团队级可追溯记忆数据库，前端、CLI、 +Agent 调用和评测框架都围绕同一套数据库事实源工作。 + +## 1. 总体架构图 ```text -User / Agent - ↓ -React Frontend - ↓ -FastAPI Backend - ↓ -Service Layer - ↓ -PostgreSQL - ↓ -Markdown Wiki Export +Human User / Admin / Agent / Evaluation Runner + | + v +React Frontend MemoryBase CLI (`mb` / `memorybase`) + \ / + v v + FastAPI Backend + | + v + API layer (`backend/app/api`) + | + v + Service + Repository layer + | + v + PostgreSQL core database <---- optional ----> Neo4j graph cache + | + v + Markdown Wiki files / evaluation reports / screenshots ``` -## 2. 模块划分 +设计重点是 **DB-first**：所有长期记忆、证据、权限、版本、审计、 +召回日志和 Wiki 投影都先进入 PostgreSQL。前端和 CLI 只是不同入口； +evaluation runner 用同一套 API / service contract 度量系统表现。 -| 模块 | 职责 | -|---|---| -| Frontend | 页面展示、表单、搜索、Wiki 预览 | -| Backend API | 对外提供接口 | -| Source Service | 导入文档、切分 chunk | -| Memory Service | 创建、编辑、删除 memory | -| Recall Service | 检索与召回 | -| Policy Service | 权限过滤 | -| Audit Service | 审计日志 | -| Wiki Service | Markdown Wiki 导出 | -| Database | 保存 source、memory、evidence、revision、audit | - -## 3. 后端目录 +## 2. 后端模块 + +当前后端目录以 FastAPI `api/` router + service/repository 分层为主： ```text backend/app/ @@ -38,30 +42,189 @@ backend/app/ core/ config.py database.py - services/ - ingest_service.py - memory_service.py - recall_service.py - policy_service.py - audit_service.py - wiki_service.py - routers/ + api/ + health.py sources.py memories.py + memory_extraction.py recall.py + search.py wiki.py - audit.py + governance.py + agents.py + sessions.py + observe.py + semantic.py + stats.py + graph.py + embeddings.py + qa.py + services/ + source_service.py + memory_service.py + memory_extraction_service.py + recall_service.py + search_service.py + context_pack_service.py + wiki_service.py + governance_service.py + agent_service.py + conversation_service.py + semantic_service.py + stats_service.py + graph_service.py + embedding_service.py + llm_service.py + models/ + *.py + cli/ + main.py + commands/ ``` +### 2.1 API layer + +| API file | 主要端点 | 作用 | +|---|---|---| +| `sources.py` | `/api/sources` | 导入和查看 source document / chunks | +| `memories.py` | `/api/memories` | 创建、查看、编辑、软删除 memory | +| `memory_extraction.py` | `/api/memory-extraction/*`, `/api/memory-candidates/*` | rule-based 候选记忆抽取、approve / reject | +| `recall.py` | `/api/recall`, `/api/recall/context-pack` | keyword / vector / hybrid recall 与 context pack | +| `search.py` | `/api/search` | 面向 Agent/CLI 的 zero-config lexical search | +| `wiki.py` | `/api/wiki/*` | Wiki 页面列表、详情、版本和导出 | +| `governance.py` | `/api/policies`, `/api/audit`, `/api/conflicts`, `/api/forget-requests`, `/api/timeline` | 权限、审计、冲突、遗忘和时间线 | +| `agents.py` | `/api/agents/*` | Agent 注册与可见 memory 查询 | +| `sessions.py` / `observe.py` | `/api/sessions`, `/api/observe` | Agent Runtime 会话与消息落库 | +| `semantic.py` | `/api/entities`, `/api/scenes` | 实体和场景查询 | +| `stats.py` | `/api/stats/overview` | Dashboard 统计 | +| `graph.py` | `/api/graph/*` | PostgreSQL graph preview 与可选 Neo4j sync/load | +| `embeddings.py` | `/api/embeddings/*` | local hashing / provider embedding 生成与回填 | +| `qa.py` | `/api/qa/answer` | 基于 recall context 的可选 LLM 问答 | + +### 2.2 Service layer + +Service 层封装业务规则和 SQL repository： + +- `source_service.py`：导入文档、计算 checksum、切分 chunk、写入搜索文本。 +- `memory_service.py`：memory lifecycle 校验、evidence 绑定、inline agent note。 +- `memory_extraction_service.py`：rule-based extraction，创建 `candidate` + memory，并写入 run-level `audit_log`。 +- `recall_service.py`：keyword / vector / hybrid recall、权限过滤、 + retrieval fallback metadata、`recall_log`。 +- `search_service.py`：chunk / memory / source 多路 lexical search 与 RRF 排序。 +- `governance_service.py`：policy、audit、conflict、forget request、 + timeline、agent-visible memory。 +- `graph_service.py`：PostgreSQL workspace graph 构建、可见性过滤、 + Neo4j 同步和载入。 +- `embedding_service.py`：local hashing provider、SiliconFlow provider、 + memory/chunk embedding cache。 +- `llm_service.py`：可选 OpenAI-compatible / SiliconFlow / DeepSeek QA。 + +## 3. 数据库层 + +PostgreSQL 是唯一必需数据库。核心 SQL 文件： + +| 文件 | 内容 | +|---|---| +| `database/00_init.sql` | `pgcrypto`、`pg_trgm` 扩展 | +| `database/01_schema_core.sql` | 用户、工作区、Agent、成员 | +| `database/02_schema_memory.sql` | 会话、source、chunk、memory、evidence、embedding、entity、scene | +| `database/03_schema_governance.sql` | Wiki、timeline、recall log、policy、forget、conflict、audit | +| `database/04_indexes.sql` | B+ tree / GIN / BRIN / covering / partial indexes | +| `database/05_views.sql` | active memory、provenance、statistics、agent visibility 等视图 | +| `database/06_triggers.sql` | revision、audit、soft delete、wiki dirty bit、conflict lifecycle | +| `database/07_seed.sql` | 课程演示 workspace seed | +| `database/08_demo_queries.sql` | 课程演示 SQL 查询 | +| `database/10_governance_demo_fixture.sql` | forgotten / archived / superseded / resolved conflict fixture | + +可选 Neo4j 只作为 graph explorer 的缓存 / 可视化后端；PostgreSQL preview +路径仍可在没有 Neo4j 时返回 workspace graph。 + ## 4. 前端页面 +当前前端按工作流分组： + +| 分组 | 页面 | +|---|---| +| Dashboard | `Dashboard.jsx` | +| Sources | `sources/SourceList.jsx`, `sources/SourceDetail.jsx` | +| Memories | `memories/MemoryList.jsx`, `MemoryDetail.jsx`, `MemoryCreate.jsx`, `MemoryEdit.jsx` | +| Recall | `recall/Recall.jsx` | +| Wiki | `wiki/WikiExport.jsx` | +| Governance | `governance/Policies.jsx`, `Audit.jsx`, `Conflicts.jsx`, `ForgetRequests.jsx`, `Timeline.jsx` | +| Runtime | `runtime/Sessions.jsx`, `Messages.jsx`, `HybridSearch.jsx` | +| Graph | `graph/GraphExplorer.jsx`, `GraphSvg.jsx`, `graphLayout.js` | + +前端默认演示路径偏向人类用户：Recall 页面默认 keyword 模式，避免在未回填 +embedding 的 demo DB 中每次都显示 hybrid fallback。后端 API 默认仍为 +`hybrid`，并通过 `retrieval_info` 明确返回 requested/effective mode、 +候选数量和 fallback reason。 + +## 5. CLI 与 Agent 入口 + +`pyproject.toml` 暴露两个命令： + ```text -Dashboard -Sources -Memories -Recall -Wiki -Timeline -Audit -Conflicts +memorybase +mb ``` + +CLI 命令覆盖 configure、health、context、recall、search、observe、 +remember、sessions 和 eval。它面向 Agent 的 "grep-style database access"： +Agent 可以用 search / recall 找证据，用 remember 写入带 inline evidence 的 +memory，用 sessions / observe 把上下文落库。 + +## 6. Evaluation 扩展 + +`evaluation/` 不是数据库主线的替代品，而是验证 MemoryBase 在长期记忆任务上的 +扩展能力： + +- adapters：BEAM、BEIR、LoCoMo、LongMemEval、MemoryAgentBench。 +- metrics：retrieval、QA、forgetting、system。 +- runners：conflict、deletion、forgetting、locomo、longmemeval、 + memoryagentbench、performance、preference、QA、retrieval。 +- reports：生成评测报告。 + +最终报告中应把 evaluation 放在"创新与验证"章节，而不是取代数据库设计章节。 + +## 7. 关键运行链路 + +### 7.1 Source 到 Memory + +```text +POST /api/sources + -> source_service.import_source() + -> source_document + source_chunk + -> optional memory_extraction.from_chunks() + -> memory_item(status='candidate') + memory_evidence + -> approve/reject candidate +``` + +### 7.2 Recall 到 Context Pack + +```text +POST /api/recall + -> recall_service + -> permission filter (public/project or v_agent_visible_memory) + -> keyword FTS / trigram + optional embedding candidates + -> recall_log with retrieval_info + -> context_pack_service formats Markdown for agents/UI +``` + +### 7.3 Governance Lifecycle + +```text +memory update/delete/conflict/forget/wiki revision + -> database trigger or governance_service + -> memory_revision / audit_log / conflict_record / forget_request + -> frontend governance pages and demo SQL queries +``` + +## 8. 设计取舍 + +- 数据库层是事实源，Markdown/Wiki 是可读投影。 +- P0 不依赖 LLM；rule-based extraction 和 local hashing embedding 可在无外网时演示。 +- Hybrid recall 是增强路径；无 embedding 时系统显式降级并报告原因。 +- `v_agent_visible_memory` 是 per-agent visibility view，调用方必须按 + `agent_id` 过滤。 +- Evaluation 是扩展验证能力，数据库建模、索引、视图、触发器和治理链路仍是课程主线。 diff --git a/docs/08-api-design.md b/docs/08-api-design.md index b05247d..b06e58f 100644 --- a/docs/08-api-design.md +++ b/docs/08-api-design.md @@ -1,6 +1,6 @@ # API 设计文档 -本文档是课程报告用的 API 摘要。后端实现、前端 mock 与集成验收以 `docs/15-api-contract-plan.md` 的执行契约为准。 +本文档是课程报告用的 API 摘要。后端实现、前端 mock 与集成验收以 `docs/process/15-api-contract-plan.md` 的执行契约为准。 ## 1. Health @@ -57,6 +57,8 @@ "confidence": 0.9, "importance": 5, "access_level": "project", + "valid_from": "2026-06-08T12:00:00Z", + "supersedes_memory_id": "optional-older-memory-uuid", "evidence": [ { "chunk_id": "uuid", @@ -74,9 +76,39 @@ Lifecycle notes: - `memory_type` supports `episodic`, `semantic`, `fact`, `profile`, `procedural`, `decision`, `preference`, `task`, `risk`, `constraint`, `policy`, and `summary`. - New memory may start as `active` or `candidate`; automatic extraction should use `candidate`. +- An active memory may atomically supersede an older active/conflicted memory in + the same workspace by setting `supersedes_memory_id`. The backend closes the + older validity interval and records `superseded_by_memory_id`. - Allowed status transitions are `candidate -> active/rejected`, `active -> superseded/archived/conflicted/forgotten`, `conflicted -> active/superseded/forgotten`, and `archived -> forgotten`. - Status changes are rejected if they skip the lifecycle state machine. Database triggers still write memory revision and audit rows for accepted state changes. +### POST /api/memories/batch + +Creates 1 to 500 memories in one database transaction: + +```json +{ + "items": [ + { + "workspace_id": "uuid", + "memory_type": "fact", + "canonical_text": "First fact.", + "evidence": [] + }, + { + "workspace_id": "uuid", + "memory_type": "fact", + "canonical_text": "Second fact.", + "evidence": [] + } + ] +} +``` + +All items must use the same `workspace_id`. The operation is atomic: validation +or evidence failure for any item rolls back the complete batch. Existing memory +revision, audit, and inline agent-evidence behavior applies to every item. + ### GET /api/memories 查询 memory 列表。默认只返回 `status = active` 的 memory；如需审计或管理视角读取归档/遗忘记录，需要显式传入 `status`，其中 `status=all` 表示不做状态过滤。 @@ -109,16 +141,41 @@ page_size ### POST /api/memory-extraction/from-chunks -使用 rule-based extractor 从已有 source chunks 生成 `candidate` memory，并自动绑定 source evidence。候选记忆不会进入默认 recall，必须 approve 后才会转为 `active`。 +从已有 source chunks 生成 `candidate` memory，并自动绑定 source evidence。默认走 `rule_based` extractor；也可传 `method = llm` 调用 OpenAI-compatible LLM analysis path。候选记忆不会进入默认 recall，必须 approve 后才会转为 `active`。 ```json { "workspace_id": "uuid", "chunk_ids": ["uuid"], - "max_candidates": 10 + "max_candidates": 10, + "method": "rule_based" } ``` +LLM mode example: + +```json +{ + "workspace_id": "uuid", + "chunk_ids": ["uuid"], + "max_candidates": 10, + "method": "llm", + "llm": { + "api_key": "sk-...", + "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "model": "qwen-plus", + "provider": "dashscope" + } +} +``` + +说明： + +- `method` 支持 `rule_based` 和 `llm`，默认 `rule_based`。 +- `llm` 字段仅在 `method = llm` 时需要。 +- `provider` 当前主要作为 audit/debug 元数据标签；真正的调用由 `api_key`、`base_url`、`model` 决定。 +- 响应会额外返回 `method`，表示本次候选抽取实际使用的路径。 + ### GET /api/memory-candidates 查询 `status = candidate` 的 memory，支持 `workspace_id`、`memory_type`、`keyword`、`page`、`page_size`。 @@ -507,3 +564,35 @@ Runs a forgetting verification report after approval. The report checks target s ### POST /api/observe/batch 批量写入 messages。单次最多 100 条，后端在一个 transaction 中写入；任一 message 校验失败时整批回滚。 + +## 12. Extension APIs + +以下端点是当前代码已交付的扩展能力。最终报告可按篇幅选择摘要，不必把每个 +request/response 全量展开。 + +### Graph API + +| Endpoint | 作用 | +|---|---| +| `GET /api/graph/health` | 查看 Neo4j / graph store 状态 | +| `GET /api/graph/workspace` | 读取 workspace graph；可选 `agent_id` 做可见性过滤，Neo4j 不可用时可 fallback | +| `GET /api/graph/workspace/preview` | 直接从 PostgreSQL 构建 graph preview | +| `POST /api/graph/workspace/sync` | 将 PostgreSQL workspace snapshot 同步到 Neo4j，并写入 audit attribution | + +### Embedding API + +| Endpoint | 作用 | +|---|---| +| `POST /api/embeddings/generate` | 用 local hashing 或配置的 provider 生成 embedding | +| `POST /api/embeddings/memories/{memory_id}` | 为单条 memory 回填 embedding cache | +| `POST /api/embeddings/chunks/{chunk_id}` | 为单个 source chunk 回填 embedding cache | +| `POST /api/embeddings/backfill` | 按 workspace/provider/model 批量回填 memory/chunk embedding | + +### QA / Stats / Semantic API + +| Endpoint | 作用 | +|---|---| +| `POST /api/qa/answer` | 基于 recall context 调用可选 LLM 生成回答 | +| `GET /api/stats/overview` | Dashboard 统计汇总 | +| `GET /api/entities` | 查询 workspace entities | +| `GET /api/scenes` | 查询 memory scenes | diff --git a/docs/09-module-ipo.md b/docs/09-module-ipo.md index 27554af..5c25d54 100644 --- a/docs/09-module-ipo.md +++ b/docs/09-module-ipo.md @@ -1,12 +1,22 @@ # 模块 IPO 表 -| 模块 | Input | Process | Output | 涉及表 | +| 模块 | Input | Process | Output | 涉及表 / 文件 | |---|---|---|---|---| -| Source / Ingest | Markdown / txt | 校验、checksum、切 chunk | SourceDocument、SourceChunk | source_document、source_chunk | -| Memory / Evidence | chunk、人工表单 | 创建 memory，绑定 evidence | MemoryItem、MemoryEvidence | memory_item、memory_evidence | -| Revision / Audit | memory 修改 | 触发器生成版本和审计 | MemoryRevision、AuditLog | memory_revision、audit_log | -| Recall | query、filters、agent_id | FTS、join evidence、权限过滤 | Context Pack | recall_log、memory_item、source_chunk | -| Policy | principal、resource、scope | 创建 allow / deny 策略 | AccessPolicy | access_policy | -| Wiki | memory / scene | 渲染 Markdown 和 frontmatter | WikiPage、WikiPageRevision | wiki_page、wiki_page_revision | -| Timeline | memory、doc、event_time | 排序聚合项目事件 | 时间线 | timeline_entry | -| Conflict | 两条 memory | 标记冲突、处理状态 | ConflictRecord | conflict_record | +| Source / Ingest | Markdown / txt / meeting text | 校验、checksum、切 chunk、生成 search_text_zh | SourceDocument、SourceChunk | `source_document`、`source_chunk` | +| Memory / Evidence | chunk、人工表单、Agent 写回 | 创建 memory、绑定 evidence、必要时创建 inline_agent_note | MemoryItem、MemoryEvidence | `memory_item`、`memory_evidence` | +| Memory Extraction | workspace_id、chunk_ids、max_candidates、method、llm options | rule-based 或 optional OpenAI-compatible LLM 抽取、分类、创建 candidate、记录 run audit | Candidate Memory | `memory_item(status='candidate')`、`memory_evidence`、`audit_log` | +| Revision / Audit | memory / wiki / governance 修改 | 触发器或 service 写 revision 与 before/after JSON | MemoryRevision、AuditLog | `memory_revision`、`wiki_page_revision`、`audit_log` | +| Recall | query、filters、agent_id、retrieval_mode | 权限过滤、FTS/trigram、可选 embedding scoring、fallback metadata | RecallResponse、Context Pack | `recall_log`、`memory_item`、`source_chunk`、embedding tables | +| Search | query、scope、agent_id | chunk/memory/source 多路 lexical search、RRF 融合 | Search results | `source_chunk`、`memory_item`、`source_document` | +| Policy / Agent Visibility | principal、resource、scope、effect | allow/deny 策略、per-agent visibility view | AccessPolicy、visible memory list | `access_policy`、`v_agent_visible_memory` | +| Wiki | memory / scene / workspace | 渲染 Markdown frontmatter/body、写版本、可选写文件 | WikiPage、WikiPageRevision、Markdown 文件 | `wiki_page`、`wiki_page_revision`、`data/markdown_wiki/` | +| Timeline | memory、doc、event_time | 排序聚合项目事件 | Timeline entries | `timeline_entry` | +| Conflict Governance | 两条 memory、conflict_type | 创建/更新 conflict，trigger 标记 conflicted 或恢复 active | ConflictRecord、AuditLog | `conflict_record`、`memory_item`、`audit_log` | +| Forget Governance | target、reason、reviewer | 审批、软遗忘、验证、审计 | ForgetRequest、target status、verification report | `forget_request`、`audit_log` | +| Sessions / Observe | session metadata、messages | Agent Runtime 会话与消息落库 | AgentSession、Message | `agent_session`、`message` | +| Semantic Organization | workspace_id、keyword | 查询 entity / scene 聚合 | Entity / Scene response | `entity`、`memory_entity`、`memory_scene`、`memory_scene_cell` | +| Graph Explorer | workspace_id、agent_id、limit | 从 PostgreSQL 构图、可选同步/读取 Neo4j、前端 SVG 布局 | Graph nodes/edges | `graph_service.py`、`frontend/src/pages/graph/` | +| Embedding Cache | memory/chunk text、provider、model | local hashing 或 provider embedding、JSONB 缓存、backfill | MemoryEmbedding、ChunkEmbedding | `memory_embedding`、`source_chunk_embedding` | +| QA | question、workspace_id、agent_id | recall context + optional LLM provider | AnswerResponse | `qa.py`、`llm_service.py`、`recall_log` | +| Stats Dashboard | workspace_id | 汇总 source/memory/wiki/audit/forget/conflict/recall 计数 | StatsOverviewResponse | `stats_service.py`、统计 views | +| Evaluation | dataset adapter、runner config | 加载 memory benchmark case、运行 metrics、生成 report | Evaluation report | `evaluation/` | diff --git a/docs/10-test-plan.md b/docs/10-test-plan.md index 8f983be..98ae4d6 100644 --- a/docs/10-test-plan.md +++ b/docs/10-test-plan.md @@ -1,40 +1,98 @@ # 测试计划 -## 1. Source 导入测试 +本文档按当前 `backend/tests/test_*.py` 与前端构建脚本整理测试覆盖。最终报告中 +应把这里作为测试方案来源，并在提交前补充最新一次命令输出截图。 -| 用例 | 输入 | 预期结果 | +## 1. 后端 API 与服务测试 + +| 测试文件 | 覆盖目标 | 关键断言 | |---|---|---| -| 导入 Markdown | discussion_01.md | 生成 SourceDocument | -| 自动切分 chunk | 100 行文本 | 生成多个 SourceChunk | -| 重复导入 | 相同 checksum | 阻止重复或提示 | +| `test_health.py` | health API | 服务与数据库状态返回正确 | +| `test_sources.py` | Source API | 导入、列表、详情、forgotten source 过滤 | +| `test_memories.py` | Memory API | 创建、详情、更新、软删除、evidence/revision | +| `test_postgres_integration.py` / `test_evaluation_baselines.py` | Candidate extraction | candidate 创建、approve/reject 生命周期、run-level audit | +| `test_recall_query.py` | Recall query | keyword/hybrid recall、权限过滤、retrieval_info | +| `test_context_pack.py` | Context pack | token budget、citation、excluded/risk/conflict metadata | +| `test_recall_wiki.py` | Recall + Wiki | recall 结果与 wiki provenance 结合 | +| `test_search_api.py` | Lexical search | chunk/memory/source scope、tokenized_query、权限过滤 | +| `test_governance.py` | Governance | policy、audit、conflict、forget request、verification | +| `test_agent_registration.py` | Agent API | 注册 Agent、可见 memory | +| `test_semantic.py` | Entity / Scene API | entity 和 scene 查询 | +| `test_stats.py` | Stats overview | dashboard count 与 breakdown | +| `test_graph_api.py` | Graph API | health、preview、load、sync contract | +| `test_graph_service.py` | Graph service | PostgreSQL graph build、Neo4j batch sync、visibility filter | +| `test_embeddings.py` | Embedding service/API | local hashing、memory/chunk embedding、backfill | +| `test_embedding_schema.py` | Embedding schema | cache 表结构与约束 | +| `test_qa.py` | QA API | recall context + optional LLM answer path | +| `test_workspace_slug_schema.py` | Workspace schema | slug 唯一和 workspace contract | +| `test_postgres_integration.py` | PostgreSQL integration | schema / seed / trigger 基础集成 | -## 2. Memory 测试 +## 2. CLI / Agent Runtime 测试 -| 用例 | 输入 | 预期结果 | +| 测试文件 | 覆盖目标 | 关键断言 | |---|---|---| -| 创建 memory | canonical_text + chunk_id | 生成 MemoryItem 和 MemoryEvidence | -| 修改 memory | 新文本 | 生成新 MemoryRevision | -| 删除 memory | memory_id | status 改为 archived | +| `test_cli_skeleton.py` | CLI 命令骨架 | `mb` / `memorybase` 命令可发现 | +| `test_cli_context.py` | Context render | CLI context 输出结构和 token 控制 | +| `test_cli_recall.py` | CLI recall | 参数映射到 recall API | +| `test_cli_sessions.py` | CLI sessions | 创建/列出 session | +| `test_cli_writeback.py` | CLI remember / observe | Agent 写回 memory 和 inline evidence | +| `test_repo_context.py` | Repo context | 工作区上下文发现 | -## 3. Recall 测试 +## 3. Evaluation Framework 测试 -| 用例 | 输入 | 预期结果 | +| 测试文件 | 覆盖目标 | 关键断言 | |---|---|---| -| 关键词检索 | “校园食堂系统” | 返回相关 memory | -| evidence 追溯 | memory_id | 返回 source chunk | -| 权限过滤 | project-only agent | 不返回 private memory | +| `test_evaluation_adapters.py` | LoCoMo / LongMemEval / MemoryAgentBench adapters | 外部 benchmark case 能转换为统一格式；MemoryAgentBench parquet 用例在缺少 `pyarrow` 时跳过 | +| `test_evaluation_baselines.py` | baseline runner | local/live baseline 输出结构稳定，覆盖 QA、vector、extraction、batch 写入路径 | +| `test_evaluation_judging.py` | semantic judge | judge 输出、token/cost 字段、retry/resume 行为稳定 | +| `test_evaluation_metrics.py` | evaluation metrics | QA、retrieval、forgetting、system 指标可计算 | +| `test_evaluation_runner_checkpoint.py` | checkpoint I/O | 每个 case 可增量写入，已完成 case 可跳过 | +| `test_evaluation_long_context.py` | long-context generator | 长上下文 retention case 生成结构稳定 | +| `test_evaluation_performance.py` | performance sampler | 操作级延迟采样输出结构稳定 | +| `test_evaluation_pricing.py` | pricing helper | provider token cost 估算可复现 | +| `test_evaluation_report.py` | report generation | 指标可聚合成报告 | -## 4. Wiki 测试 +## 4. 数据库演示测试 -| 用例 | 输入 | 预期结果 | -|---|---|---| -| 导出 Wiki | workspace_id | 生成 markdown 文件 | -| 版本记录 | 重复导出 | 生成 WikiPageRevision | +| 命令 | 预期 | +|---|---| +| `npm run db:setup` | drop/recreate public schema，加载 00-06、07 seed、search backfill、10 governance fixture、08 demo queries | +| `npm run db:check` | 输出 workspace、memory、conflict、timeline 基础状态 | +| `psql $DATABASE_URL -f database/08_demo_queries.sql` | 13 个课程演示查询可执行 | -## 5. Audit 测试 +重点 SQL 验证： -| 用例 | 操作 | 预期结果 | -|---|---|---| -| 创建 memory | insert | audit_log 有记录 | -| 修改 memory | update | audit_log 有 before/after | -| 删除 memory | delete | audit_log 有 soft_delete | +- memory delete 走 `trg_memory_soft_delete`，状态变为 `archived`。 +- memory insert/update 触发 `memory_revision` 与 `audit_log`。 +- open conflict 触发相关 active memory 变为 `conflicted`。 +- forget request 审批后 target 被软治理，verification 写入 `audit_log`。 +- `v_agent_visible_memory` 查询必须显式按 `agent_id` 过滤。 + +## 5. 前端验证 + +| 命令 | 预期 | +|---|---| +| `cd frontend && npm run lint` | React 代码 lint 通过 | +| `cd frontend && npm run build` | Vite production build 通过 | + +最终截图前还需要人工/浏览器验证这些页面： + +- Dashboard +- Source list/detail +- Memory list/detail/edit +- Recall + retrieval_info fallback badge +- Wiki export/list/detail +- Governance: audit / conflicts / forget requests / policies / timeline +- Runtime: sessions / messages / hybrid search +- Graph Explorer + +## 6. 最终提交前检查命令 + +```bash +uv run ruff check backend/app backend/tests evaluation +uv run --with pytest python -m pytest backend/tests -q +cd frontend && npm run lint && npm run build +git diff --check +``` + +这些命令通过后，再把输出截图或摘录放入最终报告的"测试结果"章节。 diff --git a/docs/11-demo-script.md b/docs/11-demo-script.md index 3adc137..9adfdac 100644 --- a/docs/11-demo-script.md +++ b/docs/11-demo-script.md @@ -15,8 +15,8 @@ npm run db:setup ## 演示流程 1. 打开 Dashboard，展示 workspace 统计。 -2. 进入 Sources 页面，导入 6 份讨论记录。 -3. 展示 SourceDocument 和 SourceChunk。 +2. 进入 Sources 页面，展示 `db:setup` 已导入的 6 份讨论记录。 +3. 展示 SourceDocument 和 SourceChunk；如需要演示写入流程，可现场额外导入一份短 note，而不是重复导入 seed 中已有的 6 份记录。 4. 从 chunk 创建 MemoryItem。 5. 展示 MemoryEvidence 来源追溯。 6. 进入 Recall 页面，搜索“为什么放弃校园食堂系统”。 @@ -36,6 +36,10 @@ npm run db:setup - 至少 20 条 evidence - 至少 5 条 revision - 至少 10 条 audit -- 至少 3 个 Wiki 页面 +- 至少 3 个 Wiki 页面（`07_seed.sql` 已包含 `why-memorybase`、`database-design`、`demo-playbook`） - 至少 1 个 conflict - 至少 1 个 private memory + +## 可选治理 walkthrough + +治理演示的详细 SQL 和页面检查见 `docs/governance-demo-walkthrough.md`。 diff --git a/docs/13-final-report-outline.md b/docs/13-final-report-outline.md deleted file mode 100644 index f714465..0000000 --- a/docs/13-final-report-outline.md +++ /dev/null @@ -1,40 +0,0 @@ -# 最终报告大纲 - -报告标题： - -《MemoryBase：面向 AI Agent 协作研发的文件—数据库双态长期记忆系统设计与实现》 - -## 目录 - -1. 摘要 -2. 项目背景与研究现状 -3. 需求分析 -4. 数据流图 -5. 数据字典 -6. 概念结构设计 -7. E-R 图设计说明 -8. 逻辑结构设计 -9. E-R 到关系模型转换 -10. 系统总体架构 -11. 物理结构设计 -12. 存储安排与路径设计 -13. 索引、视图、触发器设计 -14. 模块设计与 IPO 表 -15. 系统实现 -16. 系统演示与截图清单 -17. 测试方案与结果 -18. 创新点总结 -19. 小组分工与个人完成情况 -20. 总结与展望 -21. 附录 A SQL 源程序 -22. 附录 B 高级语言源程序说明 -23. 附录 C 演示数据 - -## 创新点写法 - -- 文件—数据库双态架构 -- Source → Memory → Evidence → Revision → Audit → Wiki 治理链路 -- Agent 权限可见视图 -- 可追溯 memory evidence -- 版本化 Wiki 投影 -- P0 不依赖 LLM，系统可稳定演示 diff --git a/docs/25-evaluation-remaining-work.md b/docs/25-evaluation-remaining-work.md deleted file mode 100644 index bab04b4..0000000 --- a/docs/25-evaluation-remaining-work.md +++ /dev/null @@ -1,123 +0,0 @@ -# Evaluation Remaining Work - -## Purpose - -This document records evaluation-route items that are intentionally not complete yet. They require backend features, external datasets, a running API, or model/judge configuration. - -## Backend-Dependent Items - -```text -- embedding similarity scoring beyond backend recall scores -- keyword-only vs vector vs hybrid baseline comparison report -- real write/query/update/delete P50/P95/P99/QPS performance suite -- real context leakage and answer leakage split -- long history 1K / 10K / 50K / 100K / 500K retention curves -``` - -Required backend support: - -```text -- source chunk vector retrieval path -- context package leakage inspection -- agent answer or answer-generation endpoint -``` - -## External Dataset Items - -```text -- official LongMemEval full-format validation -- official LoCoMo full-format validation -- official MemoryAgentBench full-format validation -- BEIR / MS MARCO retriever-only benchmark conversion -- BEAM long-context stress benchmark conversion -``` - -Current state: - -```text -- LongMemEval / LoCoMo / MemoryAgentBench support common JSON/JSONL shapes. -- Raw official datasets are not stored in this repository. -- Download URLs, licenses, and exact file names still need to be documented before use. -``` - -## Model-Dependent Items - -```text -- LLM-as-judge -- groundedness -- hallucination rate -- embedding similarity judge -- token usage -- token cost per answer -``` - -Required configuration: - -```text -- model provider -- judge prompt -- API key through environment/config only -- deterministic evaluation settings -- cost accounting policy -``` - -## Live API Items - -```text -- db_memory synthetic benchmark evidence against a live API -- real deletion retrieval leakage -- real conflict stale memory error rate -- real context-pack leakage -- real wiki/export leakage -``` - -Current `db_memory` behavior: - -```text -/api/health/detail -/api/sessions -/api/observe -/api/memories -/api/memory-extraction/from-chunks -/api/memory-candidates -/api/recall -DELETE /api/memories/{memory_id} for injected deletion cases -``` - -Limitation: - -```text -It uses direct memory API injection. It does not yet test automatic memory extraction or full agent answer generation. -``` - -Current extraction support: - -```text -- rule-based chunk -> candidate memory extraction exists -- candidate approve/reject workflow exists -- evaluation db_extraction mode can exercise source import -> extraction -> approve -> recall -- extraction from full documents and sessions is not implemented yet -``` - -## Current Backend Unlocks - -```text -- memory and source chunk embedding storage exists -- local hashing embedding provider exists -- /api/embeddings/backfill exists -- recall returns keyword/vector/recency/evidence score fields when embeddings are present -- recall supports explicit keyword, vector, and hybrid retrieval modes -- recall vector mode can use both memory embeddings and source chunk embeddings through evidence -- evaluation naive_vector_rag can now backfill embeddings and call live recall -``` - -## Next Backend Route - -Continue backend B1 by expanding vector coverage and reporting comparisons: - -```text -1. Add benchmark report comparison for keyword-only / vector-only / hybrid. -2. Add embedding similarity scoring beyond backend recall scores. -3. Add real context leakage and answer leakage inspection. -4. Keep naive_vector_rag as the live API vector-backed baseline. -``` diff --git a/docs/26-optional-llm-memory-extraction.md b/docs/26-optional-llm-memory-extraction.md new file mode 100644 index 0000000..e2ae2f1 --- /dev/null +++ b/docs/26-optional-llm-memory-extraction.md @@ -0,0 +1,170 @@ +# Optional LLM Memory Extraction + +## 1. Background + +MemoryBase now supports two candidate extraction paths from selected `source_chunk` +records: + +- `rule_based`: deterministic local heuristics, no external model required. +- `llm`: optional OpenAI-compatible analysis path, still writing reviewed + candidates into the existing memory lifecycle. + +This is an extension of the existing candidate workflow, not a replacement for +it. The product default remains `rule_based`, so the demo can still run without +network access or API keys. + +## 2. Current lifecycle + +The current end-to-end flow is: + +```text +SourceDocument + -> SourceChunk + -> /api/memory-extraction/from-chunks + -> MemoryItem(status='candidate') + -> MemoryEvidence + -> approve / reject + -> active memory enters Memories / Recall / Graph / Wiki +``` + +Important points: + +- Both extraction methods create `status='candidate'` memories first. +- New candidates do not enter default recall until a human approves them. +- Evidence still points back to exactly one selected chunk. +- The run is still recorded in `audit_log`. + +## 3. Frontend usage + +On `Sources -> Source Detail`: + +1. Select one or more chunks. +2. Keep the default `Use LLM` unchecked to run `rule_based`. +3. Check `Use LLM` to reveal `API Key`, `Base URL`, `Model`, and `Provider`. +4. Click `Extract Candidates`. +5. Review generated candidates in place. +6. Click `Approve` or `Reject`. + +The candidate cards still show the same review-oriented fields: + +- `canonical_text` +- `memory_type` +- `importance` +- `confidence` +- source chunk title and line range + +## 4. API contract + +`POST /api/memory-extraction/from-chunks` now accepts an optional method switch: + +```json +{ + "workspace_id": "uuid", + "chunk_ids": ["uuid"], + "max_candidates": 10, + "method": "llm", + "llm": { + "api_key": "sk-...", + "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "model": "qwen-plus", + "provider": "dashscope" + } +} +``` + +Notes: + +- `method` defaults to `rule_based`. +- `llm` is only needed when `method = "llm"`. +- `provider` is currently a metadata label for audit/debug visibility; the real + request routing depends on `api_key`, `base_url`, and `model`. +- The backend expects an OpenAI-compatible `POST {base_url}/chat/completions` + endpoint and requests JSON output. + +Response shape: + +```json +{ + "workspace_id": "uuid", + "method": "llm", + "created_count": 3, + "candidates": [] +} +``` + +## 5. CLI usage + +The CLI now exposes `mb extract`: + +```bash +mb extract \ + --workspace 00000000-0000-0000-0000-000000000201 \ + --chunk 11111111-1111-1111-1111-111111111111 \ + --chunk 22222222-2222-2222-2222-222222222222 \ + --max-candidates 10 \ + --method llm \ + --llm-api-key "$LLM_ANALYSIS_API_KEY" \ + --llm-base-url https://dashscope.aliyuncs.com/compatible-mode/v1 \ + --llm-model qwen-plus \ + --llm-provider dashscope +``` + +If you omit `--method`, the CLI falls back to `rule_based`. + +## 6. LLM behavior + +The new LLM path lives in an independent backend module: + +- `backend/app/services/llm_analysis.py` + +Its job is intentionally narrow: + +- receive selected chunks +- ask an OpenAI-compatible model for JSON candidate drafts +- normalize `memory_type`, `confidence`, `importance`, and summary +- clamp outputs into the existing MemoryBase schema +- hand the results back to the existing extraction service + +This keeps the main lifecycle unchanged: + +- no new approval rules +- no direct write into `active` +- no schema expansion required for the first LLM pass + +## 7. Qwen example + +For Qwen through DashScope compatible mode, the frontend fields can be filled as: + +- `Base URL`: `https://dashscope.aliyuncs.com/compatible-mode/v1` +- `Model`: `qwen-plus` +- `Provider`: `dashscope` + +If using the international endpoint: + +- `Base URL`: `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` + +Use a non-thinking model for this flow, because the backend expects JSON output +compatible with `response_format = {"type": "json_object"}`. + +## 8. What is still not implemented + +This change does **not** yet implement the richer draft-table workflow discussed +in design notes: + +- no `analysis_run` +- no `analysis_memory_draft` +- no `analysis_draft_evidence` + +So the current architecture is: + +- optional LLM-assisted candidate extraction: implemented +- analysis staging tables and richer review workspace: future work + +## 9. Verification + +The implementation was validated with: + +- backend tests for LLM parsing, API, service, and CLI paths +- frontend `npm run lint` +- frontend `npm run build` + diff --git a/docs/27-longmemeval-full-evaluation-20260609.md b/docs/27-longmemeval-full-evaluation-20260609.md new file mode 100644 index 0000000..208fb6b --- /dev/null +++ b/docs/27-longmemeval-full-evaluation-20260609.md @@ -0,0 +1,80 @@ +# LongMemEval Full Evaluation - 2026-06-09 + +## Scope + +This run evaluated all 500 official LongMemEval oracle cases with the live +MemoryBase `db_qa` path and DeepSeek `deepseek-chat`. + +The evaluator included: + +- session dates mapped to memory `valid_from` +- batched memory injection +- isolated evaluation workspaces +- per-case CSV checkpoints and resume support +- semantic LLM judgement with retry and resume support + +## Results + +| Metric | Result | +| --- | ---: | +| Cases | 500 | +| API errors | 0 | +| Deterministic pass | 176 / 500 (35.2%) | +| Semantic judge pass | 292 / 500 (58.4%) | +| Semantic judge fail | 208 / 500 (41.6%) | +| Deterministic failures upgraded by judge | 139 | +| Deterministic passes rejected by judge | 23 | +| Judge prompt tokens | 124,430 | +| Judge completion tokens | 27,327 | +| Estimated judge cost | 0.1791 CNY | + +## Semantic Results By Category + +| Category | Cases | Pass | Pass Rate | +| --- | ---: | ---: | ---: | +| abstention | 30 | 28 | 93.3% | +| temporal_update | 72 | 58 | 80.6% | +| single_fact | 120 | 95 | 79.2% | +| temporal_reasoning | 127 | 67 | 52.8% | +| preference_following | 30 | 11 | 36.7% | +| multi_session | 121 | 33 | 27.3% | + +## Interpretation + +The deterministic 35.2% score substantially underestimates semantic answer +quality because it relies on strict answer matching. The semantic judge accepts +paraphrases and equivalent calculations while rejecting answers that mention +the expected value but ultimately refuse or reach the wrong conclusion. + +The strongest current capabilities are abstention, temporal updates, and +single-fact recall. Multi-session reasoning and preference following remain the +main quality bottlenecks. + +The answer model and judge both used `deepseek-chat`. The 58.4% result is useful +engineering evidence, but publication-grade evaluation should use an +independent judge model or the official evaluator. + +## Artifacts + +The reviewable per-case score artifact is stored at: + +```text +evaluation/results/longmemeval_full_20260609.csv +``` + +It contains case ID, category, deterministic pass, semantic pass, and judge +score for all 500 cases. + +The complete generated-answer and judge-reason CSV remains a local generated +artifact under `evaluation/outputs/`. It exceeds the repository's large-file +limit and is not required to reproduce the aggregate results. + +## Validation + +```text +python -m pytest backend/tests/test_evaluation_judging.py -q +python -m pytest backend/tests/test_evaluation_baselines.py backend/tests/test_evaluation_adapters.py -q +python -m pytest backend/tests/test_memories.py -q +python -m pytest backend/tests/test_postgres_integration.py -k "batch_memory_create or atomically_supersede" -q +ruff check evaluation/judging.py backend/tests/test_evaluation_judging.py +``` diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..c7e9d04 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,32 @@ +# 文档与材料导览 + +MemoryBase 的设计文档、演示材料、源程序说明与参考资料统一组织在 `docs/` 目录下。其中 [`final-report.md`](final-report.md)（及其导出的 [`final-report.pdf`](final-report.pdf)）是整合后的主报告，已覆盖需求分析、研究现状、方案设计、系统实现、演示与分工；本目录下的其余文档是各章节的详细展开与证据，可按需查阅。 + +## 主要交付物 + +- **大作业报告**：[`final-report.md`](final-report.md) / [`final-report.pdf`](final-report.pdf) +- **答辩 PPT**：[`final-assets/slides/final-defense.pdf`](final-assets/slides/final-defense.pdf) +- **流程图 / ER / 时序图**：[`final-assets/diagrams/`](final-assets/diagrams/) +- **系统演示截图**：[`final-assets/screenshots/`](final-assets/screenshots/) +- **源程序**：`database/*.sql`（SQL）、`backend/`、`frontend/`、`evaluation/`（高级语言），清单见 [`source-sql-appendix.md`](source-sql-appendix.md) + +## 按主题分文档 + +| 主题 | 文档 | +|---|---| +| 项目总览 | [`00-project-overview.md`](00-project-overview.md) | +| 需求分析（含数据流图、数据字典） | [`01-requirements.md`](01-requirements.md)、[`02-data-flow.md`](02-data-flow.md)、[`03-data-dictionary.md`](03-data-dictionary.md) | +| 概念设计（E-R） | [`04-er-design.md`](04-er-design.md) + [`final-assets/diagrams/`](final-assets/diagrams/) | +| 逻辑设计与范式 | [`05-logical-design.md`](05-logical-design.md)、[`normalization.md`](normalization.md) | +| 物理设计、索引与 EXPLAIN | [`06-physical-design.md`](06-physical-design.md)、[`index-rationale.md`](index-rationale.md)、[`explain-analyze.md`](explain-analyze.md) | +| 系统架构 / API / 模块 IPO | [`07-system-architecture.md`](07-system-architecture.md)、[`08-api-design.md`](08-api-design.md)、[`09-module-ipo.md`](09-module-ipo.md) | +| 测试与演示 | [`10-test-plan.md`](10-test-plan.md)、[`11-demo-script.md`](11-demo-script.md)、[`governance-demo-walkthrough.md`](governance-demo-walkthrough.md) | +| 研究现状分析 | [`research-landscape.md`](research-landscape.md) | +| 创新点 | [`innovation-analysis.md`](innovation-analysis.md) | +| 可选 LLM 抽取与评测证据 | [`26-optional-llm-memory-extraction.md`](26-optional-llm-memory-extraction.md)、[`27-longmemeval-full-evaluation-20260609.md`](27-longmemeval-full-evaluation-20260609.md) | +| 小组分工与个人贡献 | [`contribution-ledger.md`](contribution-ledger.md) + `contribution-audit-*.tsv` | +| 带注释源程序附录 | [`source-sql-appendix.md`](source-sql-appendix.md) | + +## 开发过程记录 + +[`process/`](process/) 保存项目推进中的规划、roadmap、issue 记录与内部审计，供追溯开发脉络，不属于最终交付主体。 diff --git a/docs/contribution-audit-cofstars.tsv b/docs/contribution-audit-cofstars.tsv new file mode 100644 index 0000000..a493f2a --- /dev/null +++ b/docs/contribution-audit-cofstars.tsv @@ -0,0 +1,6 @@ +short_sha date kind author committer branches file_count top_level subject full_sha +fa6ecd3 2026-06-14 non-merge Cofstars <2089039907@qq.com> Cofstars <2089039907@qq.com> dev,origin/dev 5 backend,docs Fix chunking lint case and refresh final PDFs fa6ecd3360f6a3c9a5943469f782a1047cadbf18 +899530e 2026-06-14 non-merge Cofstars <2089039907@qq.com> Cofstars <2089039907@qq.com> dev,origin/dev 5 docs,scripts Refresh final report artifacts and sanitize PDF exports 899530ea9445003e99e851423d5ae3cd04ee8ca9 +89ca761 2026-06-14 non-merge Cofstars <2089039907@qq.com> Cofstars <2089039907@qq.com> dev,origin/dev 19 docs,scripts Update final report, slides, and audit artifacts 89ca7618e4bffabc056ec9f9b6787480921e2c55 +ee623c7 2026-06-14 non-merge Cofstars <2089039907@qq.com> Cofstars <2089039907@qq.com> dev,origin/dev 15 .env.example,backend,frontend Add optional LLM memory extraction pipeline ee623c78432a8b0544e1093a2a7ca4938da303e9 +c2b8cd9 2026-06-14 non-merge Cofstars <2089039907@qq.com> Cofstars <2089039907@qq.com> dev,origin/dev 12 backend,frontend Add source extraction candidate workflow c2b8cd96d0c95ebbccbdc628292d8fe6d3c24f81 diff --git a/docs/contribution-audit-dzx0902.tsv b/docs/contribution-audit-dzx0902.tsv new file mode 100644 index 0000000..84a0994 --- /dev/null +++ b/docs/contribution-audit-dzx0902.tsv @@ -0,0 +1,51 @@ +short_sha date kind author committer branches file_count top_level subject full_sha +8edbc00 2026-06-09 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub dev,origin/dev,origin/jflin 0 Merge pull request #72 from dzx0902/feat/evaluation-benchmark-framework 8edbc00cb51891dc20709a8d0ac3c9767c0f7899 +525c4d8 2026-06-09 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin 1 backend fix(test): avoid unused datetime imports in PR build 525c4d84f409330ad3f7e61da10be2103dd62377 +a7e5aae 2026-06-09 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub dev,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin 0 Merge branch 'dev' into feat/evaluation-benchmark-framework a7e5aae3002d1b77f21da8e05bd6cf7f2a29d333 +965ea4a 2026-06-09 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin 12 backend,docs,evaluation feat(evaluation): add resumable benchmark judging 965ea4aa40e74b07b3168cb1440173fcc9608aae +231474c 2026-06-08 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin 11 backend,docs,evaluation feat(memory): add batch writes and explicit supersession 231474c5fe28f085a9e8a21955b4f23a2ce1bef2 +767eb29 2026-06-08 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin 32 .gitignore,backend,docs,evaluation feat(evaluation): add official benchmarks and semantic judging 767eb297fe164c7423816b0a152c71982cbd228b +cc37c58 2026-06-07 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin 9 backend,evaluation fix(evaluation): harden live benchmark scoring and embedding retries cc37c588e7e09d3cc48c1a91763900d4e7cc646a +50609db 2026-06-06 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin 30 .env.example,backend,docs,evaluation feat(evaluation): complete end-to-end benchmark pipeline 50609db7d001a683f93f92a3ef2e1567be30f06c +9625ab2 2026-06-04 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub origin,origin/main 0 Merge pull request #71 from dzx0902/dev 9625ab238f107fdec978f11db942a9583e718e65 +28c2cc6 2026-06-04 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 0 Merge pull request #70 from dzx0902/jflin 28c2cc61fbb63d82f7425a01b7eea024500f2cc3 +27abaa9 2026-06-03 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 12 backend,frontend feat: add provider-backed QA generation 27abaa9471b2cf363fa411b6f0bfb2f50e714ced +79e1d96 2026-06-02 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub origin,origin/main 0 Merge pull request #69 from dzx0902/dev 79e1d96ce71c2db2bb31e78c07c5a00792177083 +98d13ef 2026-05-26 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 6 backend,docs Add structured context pack metadata 98d13ef9c72cb7aa85cac52cdd3eee634bad169a +8ff5609 2026-05-26 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 5 backend,docs Add forgetting verification report 8ff5609f63c8d57c31c9083f8bd3ade5b1f69950 +45dbdef 2026-05-26 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 5 backend,docs Add memory conflict detection endpoint 45dbdef6469a3bd61603788c1ca61740186da0c4 +11cc9ae 2026-05-26 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 7 backend,docs,evaluation Exercise extraction in evaluation baseline 11cc9ae6a825ef156810db3103d243ab4648e97a +bddf464 2026-05-26 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 8 backend,docs Add rule based memory extraction candidates bddf46446972f0a4bd1c9d9dbb9fdc05f17dafb4 +5b4c7b1 2026-05-26 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 8 backend,docs,evaluation Compare evaluation baselines and chunk vectors 5b4c7b11caf0adcd2bc8aeec4c5f043075394baf +c618588 2026-05-26 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 11 backend,docs,evaluation Add vector recall evaluation baseline c618588e8fa38bdafe18fd019d30a10b1b2fcd5e +58218a6 2026-05-26 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 9 backend,database,docs Implement memory lifecycle status validation 58218a6d1a302389236f1b2e2ed1e5e1f712a5d6 +ce1c77c 2026-05-25 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 2 backend feat(backend): add hybrid memory recall ce1c77c9ad97bb94aadcd9791e2e9b15ce19d7fb +e8b5c25 2026-05-25 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 5 backend feat(backend): persist generated embeddings e8b5c25f4ce56779cc8ed1fb6757ea8c00c4f3f1 +16c83f9 2026-05-25 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 10 backend,database,docs feat(backend): add embedding foundation 16c83f93dfa34ee74276a1439d9e5b8a2ae8b2e4 +1fe96fe 2026-05-25 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 5 backend,docs,evaluation feat(evaluation): report governance metrics 1fe96febb151f509330e27fdf37055befd4e8885 +d4ff9ea 2026-05-25 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 6 backend,evaluation feat(evaluation): add local memory baselines d4ff9ea32b208dd747faf240e81f98dee797c32c +17723f1 2026-05-25 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 6 backend,evaluation feat(evaluation): convert external benchmark samples 17723f11396e26e989f201969d7009e1131f3f05 +d8072ab 2026-05-25 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 54 .gitignore,backend,docs,evaluation feat(evaluation): add benchmark framework d8072abcc593e25f2ee45616900b2ffee3b667fd +69b78a6 2026-05-24 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub main,origin,origin/main 0 Merge pull request #66 from dzx0902/dev 69b78a6c47585933c5b79976071b0a4649f57227 +fb2647d 2026-05-24 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 5 frontend fix(frontend): address runtime governance review issues fb2647dd1d663be0f5cf04a916e04cd0c67d819b +d6666dc 2026-05-24 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 0 Merge pull request #63 from dzx0902/jflin-backend-completions d6666dc58613fade1ed0fb4b4855241d5845f54f +3953a93 2026-05-23 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 6 backend feat(governance): add policy lifecycle APIs 3953a9341fd51bc1949d2cad1518e6c5a961a7ef +68a7799 2026-05-23 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 7 backend feat(semantic): add entity and scene list APIs 68a77995d92a98150d886d8581585ce37458329a +7c579e8 2026-05-23 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 7 backend feat(stats): add overview API 7c579e88578d2dfc54f0cfa61540e1916b6741c8 +41060dc 2026-05-23 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 5 backend feat(wiki): add read APIs 41060dc7f7e31041d2e140603b44b80c0bdddb75 +38264a5 2026-05-23 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 7 .gitignore,backend,database,package.json,README.md,scripts chore(db): add env-aware sql runner 38264a5a7877f9c77b3e068211bd795249bb08d3 +87c7fd0 2026-05-23 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 7 backend,database,docs feat(wiki): support batch export contract 87c7fd0f42608c1dd3409dcb0333c75af3051c1a +2bc5276 2026-05-22 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub main,origin,origin/frontend-api-integration,origin/main 0 Merge pull request #53 from dzx0902/dev 2bc5276d98a17754d27740e18dace5c7a58a3de1 +eb34e59 2026-05-19 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> origin/fix/db-cli-windows-encoding 2 scripts,tests Fix database CLI encoding on Windows eb34e5921b08ab0cc9348752363341b25b7cdb09 +7e20cc3 2026-05-19 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #50 from dzx0902/fix/backend-api-contract-demo 7e20cc333a4c1651a07a4f11def50b519befd3f2 +3ec62f3 2026-05-17 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #47 from dzx0902/backend-p0 3ec62f302d2c78a11812d1694e4d398cef45e371 +4efac9a 2026-05-17 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 2 .github fix: add postgres service for backend ci health check 4efac9abee1060de6e3aa34641fa3e9f8076b834 +2af1bbf 2026-05-17 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 16 backend,database fix: align backend package imports and SQL/test consistency 2af1bbf3c37cbc9bcde8fd9b6b673729c396aa2f +5caecc7 2026-05-17 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge branch 'main' into backend-p0 5caecc73d76066bd12801d01d0691abb6a905dd7 +67845b0 2026-05-17 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 37 backend,database,docs,package.json,README.md,scripts p0阶段backend实现包括测试部分 67845b062666440fd3293f2b5b1623733a6aafe4 +598a29e 2026-05-16 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #46 from dzx0902/dev 598a29e224c6aa892736fbe1c4cd3382dedf4dfb +03c4cb8 2026-05-16 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #45 from dzx0902/feat/CI-fix 03c4cb82f3a88bbfcd681dd2f681d4fc1a040e67 +6b19695 2026-05-16 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 10 .github,docs,frontend,README.md chore: 完善项目工程化体系与文档规范 6b19695877af464a4c427c5b83836cd523d88ef7 +f7ac477 2026-05-16 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #44 from dzx0902/jflin f7ac4771dc61ec46f7c1b5a59c56a3275ed1ab09 +2bad431 2026-05-15 merge dzx0902 <145189098+dzx0902@users.noreply.github.com> GitHub dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #1 from dzx0902/dev 2bad43169fabea7a289682eaf80006f701f3f9f2 +dc0e327 2026-05-15 non-merge dzx0902 <3575895791@qq.com> dzx0902 <3575895791@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 55 .editorconfig,.env.example,.github,.gitignore,.pre-commit-config.yaml,backend,CONTRIBUTING.md,database,docker-compose.yml,docs,frontend,pyproject.toml,README.md,scripts first commit dc0e3276dad0d7f81d37c4c115ebbec3df88d73d diff --git a/docs/contribution-audit-hopecommon.tsv b/docs/contribution-audit-hopecommon.tsv new file mode 100644 index 0000000..754577c --- /dev/null +++ b/docs/contribution-audit-hopecommon.tsv @@ -0,0 +1,82 @@ +short_sha date kind author committer branches file_count top_level subject full_sha +1e9419e 2026-06-10 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,origin/dev 0 Merge pull request #73 from dzx0902/jflin 1e9419eb7d9bbb9a6cb76aa1f343862bb84eda4e +3a60334 2026-06-10 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 1 .gitignore chore: ignore .claude/ local config and skills 3a60334c8dbeeb2bae910db78add36d268f52108 +251013e 2026-06-10 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 2 docs docs(gap7): add 16-page final defense slides 251013e858e4c03ca91dded7780d545f6acac6d1 +3e5c51f 2026-06-10 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 2 docs docs(report): integrate final report body 3e5c51fd6e5c1dd4db7e794e9c81edede76e3eff +dfa7adf 2026-06-10 merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 0 merge: integrate latest benchmark evaluation work from dev dfa7adfa5a898121ecdacd9f4b36db3247e85370 +6620abf 2026-06-09 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 4 docs docs(gap6): contribution ledger with per-member commit audits 6620abf5460bb337ff1b6d257e2f8f858b19fcfe +1302a5e 2026-06-09 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 7 .gitignore,docs docs(gap4): include raw command logs as PNG renderings' source of truth 1302a5e7fddc48d58e74e8844bf7122462a569f3 +02e51e8 2026-06-09 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 33 docs docs(gap4): add screenshot and evidence inventory 02e51e89ad64f9c6a0b7a05d03ffadd70d4bcb5f +072c376 2026-06-08 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 14 backend,database,docs,frontend docs(gap8): map annotated source and SQL appendix 072c376d699004372f886e747d2c504452693aaf +b70f789 2026-06-08 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 1 docs docs(gap3): expand innovation analysis for final report b70f7898698eb38788ff4c684e45611b55f646ec +9c2594d 2026-06-08 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 1 docs docs(gap2): add research landscape analysis 9c2594de4f5fd1af212fffb9c26b7468dd96632f +055f89c 2026-06-08 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 11 docs docs(gap5): codex review pass — correct API paths, triggers, ER edges 055f89c36e0f59f5a38f283ff7788e4c218feb37 +8883161 2026-06-08 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 2 docs docs(gap5): shorten 02-er-full labels to reduce overlap at MEMORY_ITEM 88831611d2f555a3a2f31ca7ceb1299eec6408d5 +ddf1d93 2026-06-08 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 13 docs docs(gap5): add ER and sequence diagrams (mermaid + SVG) ddf1d933c4e6dbd8cd095ba5cf3ce56b6e8fe328 +db7c101 2026-06-08 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin/dev,origin/jflin 25 database,docs docs: gap 9 re-audit pass — align deliverables with HEAD 2dd0d64 db7c1014baa2e670afdf132fdabca3ef9bdf1fb9 +2dd0d64 2026-06-03 merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 0 merge: integrate evaluation framework with hybrid retrieval and extraction 2dd0d64bc08f395542c3b58c2ae676e82eecb023 +e952bf9 2026-06-03 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 1 uv.lock chore: commit uv.lock for reproducible Python deps e952bf9da0e2363612cb4885530d9d5c07e43494 +7754d50 2026-06-03 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 7 database,docs,scripts docs(db): add course-alignment artifacts and governance demo fixture 7754d509358b638cf8e2e6f1bdab1adbd0884929 +6119d36 2026-06-02 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 12 backend,frontend,README.md feat(graph): agent-aware visibility, driver singleton, batch UNWIND, audit attribution 6119d360b0857d61e85296aa8905a1825016f736 +b6f2cfb 2026-05-26 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 0 Merge pull request #68 from dzx0902/jflin b6f2cfb69c84d89813b68382d96cd826c187935b +bb182b4 2026-05-25 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 7 frontend style(frontend): adopt blueprint-and-parchment theme with layered warm surfaces bb182b4fef3a4b1eeff508a3ecdbe60e2079c236 +95b5da8 2026-05-25 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 2 frontend style(frontend): switch to parchment + navy theme inspired by tw93/kami 95b5da81cd54d985acd7f5b844d3749671bccb29 +c9d13f5 2026-05-24 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 0 Merge pull request #67 from dzx0902/jflin c9d13f5c3dd0bca289f90ec89de4a56855799cd2 +914e85d 2026-05-24 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 4 frontend fix(frontend): align wiki status filter and add UX polish 914e85d21496f04436374185f924a3c9b9883f0e +3bf4692 2026-05-23 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #62 from dzx0902/jflin 3bf4692c7cd0e649968f0b34cd618a0e3ca4bf5f +c18a8ee 2026-05-23 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 2 docs,README.md docs: record course alignment and AI-grep thesis c18a8ee3a870e52f17e3b963b90c496edd4c1fb9 +0417f75 2026-05-23 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #61 from dzx0902/jflin 0417f754edce958bbbf2b7e1a952f7468b851065 +ac3f76f 2026-05-23 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 7 backend,docs feat(context): add repo and session aware context packs ac3f76f3b94b182274987418bd48503570b4c800 +25d1a1b 2026-05-23 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 8 backend feat(sessions): expose messages and active session config 25d1a1b4c3a1f703d6db5bc12d47d2f10944a575 +b382b8b 2026-05-23 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 13 backend fix(agent-runtime): align CLI dogfood contracts b382b8bf77d0b92079c42b17722956f4d482d5af +db3a548 2026-05-22 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #60 from dzx0902/jflin db3a548abe71c2198375c9b63037bb87ddc76265 +05bd2fa 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 4 AGENTS.md,backend chore(agent-runtime): improve CLI dogfood ergonomics 05bd2fa5ea9fdfc54ee897d6417df973e9eba5bc +a400161 2026-05-22 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #59 from dzx0902/jflin a400161e9085594f4b0173dd6ceaaf91aaa89b8f +a49fe14 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 17 backend,data,docs feat(search): add lexical search API and CLI a49fe14f8be165ca5a1e8015e042029b56c78b57 +a951bae 2026-05-22 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #58 from dzx0902/jflin a951bae059fba21a823f4fecd3ce8993e05f07fb +e51a500 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 19 backend,database,docs,pyproject.toml,scripts feat(search): add lexical search infrastructure e51a500a497722f4971a7aac3a03df0aff83eb5c +e6122e3 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 2 docs docs(pr4-design): split into PR4a and PR4b e6122e385f72785dc71f6c22fd13a5e48ea1338f +19336e1 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 1 docs docs: add PR4 lexical search design 19336e1d3fd519d287e73292eab59173412c27f1 +3b6358f 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 1 AGENTS.md chore(agents): require review before landing actions 3b6358fc9f23608455a98805073a9a30879bd3e3 +18888f9 2026-05-22 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 0 feat(agent-runtime): PR3 - observe API and CLI writeback 18888f94454ab0eb447592533b5de63edd16b3b6 +ec5dba6 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 5 docs docs(agent-runtime): document observe and writeback workflow ec5dba67b2fb8a617de79053bfb6b9789ac6f451 +610d871 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 6 backend feat(cli): add sessions observe and remember commands 610d87128185d593b99274f10ce2bc4c637ae384 +93eac83 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 12 backend,database feat(api): add observe sessions and inline evidence writeback 93eac83e951f47b6769682183202b0e85086724f +edb92d7 2026-05-22 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 0 feat(agent-runtime): PR2 - context pack formatter and CLI edb92d704b0ede6198d5a751735f8f635cf1f824 +af49021 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 2 data,docs chore(recall): extend demo query expansions and runtime docs af4902178007bb846918db14e3ddc44ed0c7aa3e +c20afcc 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 8 backend,data feat(cli): add recall context and eval commands c20afcc5e6d430f3939b2b9bec4172d4679d6471 +f86b0f7 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 9 backend,pyproject.toml feat(api): add recall context pack formatter f86b0f7fe47c1ec7f1fc8348ab84b4428d37e20d +3254604 2026-05-22 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #55 from dzx0902/jflin 3254604f1e2a0cc7d95eb5bc4a4bbc2b9f911167 +59f2470 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 12 backend,docs,pyproject.toml feat(cli): add memorybase console scripts 59f24702e121eed9eaa7275b4953374f181ef72d +ff59638 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 10 backend feat(api): add health detail and agent registration ff5963818ce0cccf65b7ad6ba345e4a0aabb9d36 +10182e8 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 6 backend,database,docs feat(schema): add workspace slug for agent lookup 10182e89b2b75bcb9d928191f269e1b04cebd912 +72d1325 2026-05-22 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #54 from dzx0902/jflin 72d1325c1e0f9cc0712cfa96a688e15a7976f10f +fedfc69 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 6 docs docs(governance): align P3 governance contracts fedfc6943eaf33b52b62bbb7d55767815dbb1b91 +a707094 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 4 backend test(governance): cover P3 governance edge cases a7070944739bc1d0d44ad9f09a36e30412cc805d +a90152c 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 10 backend feat(api): complete P3 governance workflows a90152c7fd3b05071661dcde3403da59303f1712 +5958ad0 2026-05-22 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/jflin-backend-completions,origin/main 5 database feat(governance): harden P3 database foundations 5958ad0c82285dc952060bc2d82474b0ea7021fe +62f5c03 2026-05-22 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #52 from dzx0902/feat/entity-memory-scene 62f5c03c96156357773afe0641195f9afbf76e2c +30852d6 2026-05-21 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 20 backend,database,docs feat(semantic): add entity and memory scene model 30852d67b6eefec2b1e492df3a4e56b5cbcb9823 +c6a2d77 2026-05-21 merge evanlin257 <143060834+hopecommon@users.noreply.github.com> GitHub dev,main,origin,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 0 Merge pull request #51 from dzx0902/jflin c6a2d7736693b89db49c9d58b2deb807d60ed7d2 +91a6b35 2026-05-21 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 7 backend,docs feat(governance): implement forget request workflow (I040) 91a6b358be65abdfd185725c1781995caaaa6c77 +5d428c0 2026-05-19 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 20 backend,data,database fix(backend): align API contracts and demo flow 5d428c0111e9b3b85a51d0b16d68b876af1aff3a +ada660f 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 1 docs docs: record deferred governance API contracts ada660fd7d277170c37ca68aa81eaf4e88f5a7e7 +9bbd803 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 1 AGENTS.md docs: require owner approval for git and GitHub actions 9bbd803ae09a0ceb8d58dfeefa7c678445fd30b8 +4a9fe55 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 3 docs,README.md docs: add backend API contract plan 4a9fe55c9fc5fa4008406823ede6dbd0ee17b6d8 +91c7776 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 2 database,docs fix(sql): track forget request reviewer 91c7776cbe294364ae12d324e68c54f1cc4e3025 +fc1ccd5 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 3 database,docs fix(demo): use session config for seed queries fc1ccd536898f00dcf45c193497c706cfb4cd993 +872587d 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 8 data,database,scripts feat(demo): add reproducible seed data (I017) 872587daa42acdf540d063721e1f392873a82a69 +6fa7974 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 1 database fix(sql): allow workspace cascade deletes for memory trigger (I016) 6fa79749e0915cb38b962e64f1c87bfee596eb05 +3787a7f 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 1 database feat(sql): add memory audit and revision triggers (I016) 3787a7fffd734a5bda092c688fcaf036f745d44f +df6cdf3 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 3 database,docs feat(sql): add governance and provenance views (I015) df6cdf3e3c258b5c3fecd467bab85f8f5350bea8 +1f964d3 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 2 database,docs feat(sql): align indexes for recall and governance queries (I014) 1f964d3ce3f29cc437028cd98467a69a5e1e9173 +50ea7ab 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 1 database fix(sql): canonicalize memory conflict pairs (I013) 50ea7ab49008b79c5033f095a24e388347065c36 +fb2f48e 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 1 database feat(sql): add conflict and forget governance schema (I013) fb2f48eab4202ce485b06e77c28be354662a15d2 +18eaef9 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 1 database fix(sql): align wiki timeline and recall references (I012) 18eaef9e317081f6f3a437f104b9f6cd47f32854 +a9aa40f 2026-05-16 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 1 database fix(sql): preserve memory records on nullable reference deletion (I011) a9aa40fb2c4407131f5f2c20a75a15cf54b4b55c +1980ee7 2026-05-15 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 1 database fix(sql): preserve memories when source documents are deleted (I010) 1980ee7f2fe2450fb212a55c8b4187d0ce3b5a06 +f1de003 2026-05-15 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 1 AGENTS.md docs: allow phase-level issue batching f1de0032b028de68f0280a67235f479833e40bec +1dfd643 2026-05-15 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 1 database fix(sql): scope source checksums by workspace (I010) 1dfd64347c298cb0f159c98da54fdfc6d3cf75dc +209c694 2026-05-15 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 1 database fix(sql): preserve owned resources on user delete (I009) 209c694b40f8996b07c35172585a4fc77c9a9d7e +9b8c358 2026-05-15 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 1 AGENTS.md docs: add agent workflow guide 9b8c3584a1ca9cca508f239331fd512c9f059f89 +fb8faba 2026-05-15 non-merge hopecommon <1841778349@qq.com> hopecommon <1841778349@qq.com> dev,main,origin,origin/backend-p0,origin/dev,origin/feat/entity-memory-scene,origin/feat/evaluation-benchmark-framework,origin/fix/backend-api-contract-demo,origin/fix/db-cli-windows-encoding,origin/frontend-api-integration,origin/jflin,origin/jflin-backend-completions,origin/main 3 .github,.gitignore,docs docs: add initial GitHub issue breakdown for MemoryBase fb8faba5dd926af9787b2e0d92f3aa149c9538f5 diff --git a/docs/contribution-audit-lywzc0419.tsv b/docs/contribution-audit-lywzc0419.tsv new file mode 100644 index 0000000..123f2d7 --- /dev/null +++ b/docs/contribution-audit-lywzc0419.tsv @@ -0,0 +1,9 @@ +short_sha date kind author committer branches file_count top_level subject full_sha +0288584 2026-06-02 non-merge lywzc0419 lywzc0419 dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 2 backend Fix Ruff import sorting 0288584e6a75e6dbd25a58e2f4070fd695369667 +ff49314 2026-06-02 non-merge lywzc0419 lywzc0419 dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 5 backend Fix graph integration lint issues ff493144c9ce7337fee7bd8af5ef772ab7668b87 +09c9336 2026-05-29 non-merge lywzc0419 lywzc0419 dev,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 15 backend,database,docker-compose.yml,frontend,README.md Add Neo4j graph explorer integration 09c933624419bec8cba1ae33378dc239042b412c +40d9376 2026-05-24 non-merge lywzc0419 lywzc0419 dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 12 frontend Enhance frontend runtime and governance UI 40d937646505915cd7f97fd2642c8c813e13c484 +7e01bc8 2026-05-24 non-merge lywzc0419 lywzc0419 origin/frontend-api-integration 2 frontend Declare frontend router dependency 7e01bc8c3572b0f1440f1bf6af7a60da9025b3fa +72155bc 2026-05-24 non-merge lywzc0419 lywzc0419 dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 2 frontend Declare frontend router dependency 72155bc234246e4515bb05d22a11bca05e300172 +360c42a 2026-05-24 non-merge lywzc0419 lywzc0419 origin/frontend-api-integration 28 frontend,package.json,package-lock.json Implement frontend API integration 360c42acb32b63f1a59069c2bbe0e4419245d78f +be6dd14 2026-05-24 non-merge lywzc0419 lywzc0419 dev,main,origin,origin/dev,origin/feat/evaluation-benchmark-framework,origin/jflin,origin/main 28 frontend,package.json,package-lock.json Implement frontend API integration be6dd143da1beabb0ad7c9ae421186fcd51a3406 diff --git a/docs/contribution-ledger.md b/docs/contribution-ledger.md new file mode 100644 index 0000000..12b375f --- /dev/null +++ b/docs/contribution-ledger.md @@ -0,0 +1,267 @@ +# Contribution Ledger + +> Scope: this ledger audits commits authored by `hopecommon` / `jflin` identities +> across local and remote branches. It is evidence for the final report section +> "小组分工与个人完成情况". As of the 2026-06-14 refresh, the ledger also +> records Git-backed evidence for the other three observed identities. + +## 1. Audit Baseline + +Audit date: 2026-06-15 (refreshed from the 2026-06-14 snapshot) + +Repository state at audit time: + +- Branch: `dev` +- HEAD: `b038e1b docs: organize final submission package` +- Remote refresh command: `git fetch --all --prune` +- Branch status at refresh time: local `dev` == `origin/dev` == `b038e1b` (in sync). All five `Cofstars <2089039907@qq.com>` non-merge commits (`c2b8cd9`, `ee623c7`, `89ca761`, `899530e`, `fa6ecd3`) are now on `origin/dev`. + +Refs considered: + +- Local: `dev`, `main` +- Remote: `origin/dev`, `origin/main`, `origin/jflin`, `origin/feat/evaluation-benchmark-framework`, `origin/feat/entity-memory-scene`, `origin/frontend-api-integration`, `origin/backend-p0`, `origin/fix/backend-api-contract-demo`, `origin/fix/db-cli-windows-encoding` + +Author identities included: + +| Identity | Treatment | +|---|---| +| `hopecommon <1841778349@qq.com>` | Primary direct author identity; 65 non-merge commits in the refreshed all-ref audit | +| `evanlin257 <143060834+hopecommon@users.noreply.github.com>` | GitHub noreply identity tied to hopecommon; 14 merge / PR-integration commits plus 2 authored merge-style delivery commits in the refreshed all-ref audit | + +Important counting rule: + +- Non-merge commits are treated as direct implementation / documentation contributions. +- Merge commits and PR integration commits are tracked separately as integration evidence. +- The large evaluation branch merge is counted as integration/curation unless individual files are authored by hopecommon commits. This avoids overstating original implementation ownership of teammate work. +- Raw commit audit is stored in `docs/contribution-audit-hopecommon.tsv`. + +## 2. Reproducible Commands + +```bash +git fetch --all --prune + +git log --all \ + --regexp-ignore-case \ + --author='hopecommon\|1841778349' \ + --format='%H%x09%h%x09%aI%x09%aN <%aE>%x09%s' + +git log --all \ + --regexp-ignore-case \ + --author='hopecommon\|1841778349' \ + --no-merges \ + --shortstat \ + --format='@@@%H%x09%h%x09%aI%x09%aN <%aE>%x09%s' + +git log --all \ + --regexp-ignore-case \ + --author='hopecommon\|1841778349' \ + --merges \ + --format='%h %aI %aN <%aE> %s' +``` + +Branch-contained direct commit counts: + +```text +dev 65 +origin/dev 65 +origin/jflin 65 +main 46 +origin/main 52 +origin/feat/evaluation-benchmark-framework 52 +origin/feat/entity-memory-scene 23 +origin/frontend-api-integration 23 +origin/backend-p0 20 +``` + +Aggregate result across all refs: + +| Metric | Count / value | +|---|---:| +| Related commits across all refs, de-duplicated | 81 | +| Direct non-merge commits | 65 | +| Merge / PR integration commits | 16 | +| Approx. changed files touched by related commits | 301 | +| Approx. aggregate diff across related commits | +59,100 / -3,000 | + +The aggregate diff is a rough audit signal, not a grading claim: merges, generated +assets, docs, and reworked lines can inflate totals. The module-level evidence +below is the safer source for the final report. + +Refresh note: compared with the 2026-06-10 ledger snapshot, this refresh now +includes the final report body (`3e5c51f`), final defense slides (`251013e`), +the latest benchmark-integration merge (`dfa7adf`), and the five `Cofstars` +non-merge commits now merged to `origin/dev` (source/LLM extraction plus final +report, slides, and audit-artifact refreshes). + +## 3. Module-Level Contribution Summary + +| Area | Evidence count | Representative files | Contribution summary | +|---|---:|---|---| +| Database schema, SQL, views, triggers, seed/demo data | ~26 commits | `database/01_schema_core.sql`, `database/02_schema_memory.sql`, `database/03_schema_governance.sql`, `database/04_indexes.sql`, `database/05_views.sql`, `database/06_triggers.sql`, `database/07_seed.sql`, `database/08_demo_queries.sql`, `database/10_governance_demo_fixture.sql` | Built and hardened the relational foundation: workspace/user/agent ownership, memory/evidence/revision/audit, conflict/forget governance, semantic entities/scenes, views, indexes, triggers, reproducible seed data, and SQL demo queries. | +| Backend API, services, CLI, tests | ~20 commits | `backend/app/services/*`, `backend/app/api/*`, `backend/app/cli/*`, `backend/tests/test_postgres_integration.py`, `backend/tests/test_graph_service.py`, `backend/tests/test_recall_query.py` | Implemented or hardened source/memory/recall/governance/search/runtime/graph service paths, CLI dogfood flows, context pack generation, agent session writeback, and focused backend tests. | +| Frontend UI and demo polish | ~5 commits | `frontend/src/pages/recall/Recall.jsx`, `frontend/src/pages/wiki/WikiExport.jsx`, `frontend/src/pages/graph/*`, `frontend/src/index.css`, `frontend/src/api/client.js` | Delivered theme polish and targeted UX fixes; integrated graph and recall surfaces needed for final demo evidence. The low count is intentional: core multi-page frontend work is mainly credited to other teammates, while this ledger only claims theme/demo polish and final-report screenshot support. | +| Agent runtime and agent-facing database substrate | ~12 commits across backend/database/docs | `agent_session`, `message`, `backend/app/cli/commands/*`, `backend/app/services/context_pack_service.py`, `docs/process/16-agent-runtime-gap-analysis.md`, `docs/process/17-agent-runtime-plan.md` | Added agent registration, sessions/messages, observe/remember writeback, context pack export, and CLI entry points so MemoryBase can be used by both humans and agents. | +| Governance, provenance, and access-control workflows | ~12 commits across SQL/backend/tests/docs | `memory_evidence`, `memory_revision`, `audit_log`, `access_policy`, `forget_request`, `conflict_record`, `v_agent_visible_memory`, governance API/tests/docs | Implemented the project’s main differentiator: evidence-backed memories, revision/audit trail, conflict/forget governance, and agent-aware visibility. | +| Graph explorer / graph sync hardening | ~1 large feature commit | `backend/app/services/graph_service.py`, `backend/app/api/graph.py`, `frontend/src/pages/graph/*`, `backend/tests/test_graph_service.py` | Added agent-aware graph visibility, Neo4j driver singleton lifecycle, batch `UNWIND` sync, sync audit attribution, frontend graph module split, and graph tests. | +| Search, recall, and context-pack retrieval | ~8 commits | `backend/app/services/recall_service.py`, `backend/app/services/search_service.py`, `backend/app/cli/commands/eval.py`, `data/eval/*`, `frontend/src/pages/recall/Recall.jsx` | Added lexical search infrastructure, recall API/CLI, context pack formatter, retrieval/evaluation seed data, and UI evidence for recall and QA. | +| Evaluation / benchmark integration | Integration plus limited earlier eval CLI work | `backend/app/cli/commands/eval.py`, `data/eval/*`, merged `evaluation/` assets | Teammate-owned area. Hopecommon's contribution is limited to early eval CLI/gold-data support plus final review, merge, conflict resolution, and alignment into the main demo/report story. | +| Final-report materials and evidence assets | ~36 commits | `docs/research-landscape.md`, `docs/innovation-analysis.md`, `docs/final-assets/diagrams/*`, `docs/final-assets/screenshots/*`, `docs/source-sql-appendix.md`, `docs/process/audit-findings.md` | Produced final report support materials and evidence packages: research landscape, innovation analysis, ER/sequence diagrams, screenshot inventory, source/SQL appendix map, and deliverable re-audit. | +| Tooling and reproducibility | 6 commits | `scripts/db_cli.py`, `scripts/import_demo_sources.py`, `pyproject.toml`, `uv.lock`, `AGENTS.md` | Improved reproducible database setup/demo execution, CLI packaging, dependency lockfile, and project workflow guardrails. | + +## 4. Representative Commit Evidence + +### 4.1 Database and SQL Foundations + +| Commit | Date | Evidence | +|---|---|---| +| `fb8faba` | 2026-05-15 | Initial GitHub issue breakdown and project work structure. | +| `209c694` | 2026-05-15 | Preserved owned resources on user delete; early referential integrity hardening. | +| `1dfd643` / `1980ee7` / `a9aa40f` / `18eaef9` | 2026-05-15 to 2026-05-16 | Fixed workspace-scoped source checksums, nullable references, source deletion behavior, wiki/timeline/recall references. | +| `fb2f48e` / `50ea7ab` | 2026-05-16 | Added conflict and forget governance schema, then canonicalized conflict pairs. | +| `1f964d3` | 2026-05-16 | Added recall/governance index alignment. | +| `df6cdf3` | 2026-05-16 | Added governance and provenance views. | +| `3787a7f` / `6fa7974` | 2026-05-16 | Added memory audit/revision triggers and cascade-delete compatibility. | +| `872587d` | 2026-05-16 | Added reproducible seed data. | +| `5958ad0` | 2026-05-22 | Hardened P3 governance database foundations. | +| `30852d6` | 2026-05-21 | Added entity and memory scene model. | +| `7754d50` | 2026-06-03 | Added course-alignment SQL artifacts, BRIN/covering index rationale, governance demo fixture. | + +### 4.2 Backend, API, CLI, and Tests + +| Commit | Date | Evidence | +|---|---|---| +| `5d428c0` | 2026-05-19 | Aligned backend API contracts and demo flow across backend/data/database. | +| `91a6b35` | 2026-05-21 | Implemented forget request workflow. | +| `a90152c` / `a707094` | 2026-05-22 | Completed P3 governance API workflows and edge-case tests. | +| `ff59638` / `59f2470` | 2026-05-22 | Added API health detail, agent registration, and `memorybase` console scripts. | +| `f86b0f7` / `c20afcc` | 2026-05-22 | Added recall context pack formatter and CLI recall/context/eval commands. | +| `93eac83` / `610d871` | 2026-05-22 | Added observe sessions and inline evidence writeback API/CLI. | +| `e51a500` / `a49fe14` | 2026-05-22 | Added lexical search infrastructure, API, and CLI. | +| `b382b8b` / `25d1a1b` / `ac3f76f` | 2026-05-23 | Hardened agent runtime contracts, session config, repo/session-aware context packs. | +| `6119d36` | 2026-06-02 | Hardened graph service: visibility, driver singleton, batch sync, audit attribution, tests. | + +### 4.3 Frontend and Demo Surfaces + +| Commit | Date | Evidence | +|---|---|---| +| `914e85d` | 2026-05-24 | Aligned wiki status filter and added UX polish. | +| `95b5da8` / `bb182b4` | 2026-05-25 | Switched the app to the parchment/navy theme and then refined layered warm surfaces. | +| `6119d36` | 2026-06-02 | Added/refined Graph Explorer frontend module and graph routing. | +| `072c376` | 2026-06-08 | Audited frontend source for annotated source appendix and final report evidence. | + +### 4.4 Final Deliverables, Report Evidence, and Review Passes + +| Commit | Date | Evidence | +|---|---|---| +| `3e5c51f` / `251013e` | 2026-06-10 | Integrated the final report body and added the 16-page final defense slide deck. | +| `db7c101` | 2026-06-08 | Deliverable re-audit pass aligned deliverables with merged HEAD. | +| `ddf1d93` / `8883161` / `055f89c` | 2026-06-08 | Added and reviewed final ER/sequence diagrams, including SVG renderings and schema/API corrections. | +| `9c2594d` | 2026-06-08 | Added research landscape analysis. | +| `b70f789` | 2026-06-08 | Expanded innovation analysis. | +| `072c376` | 2026-06-08 | Mapped annotated source and SQL appendix. | +| `02e51e8` / `1302a5e` | 2026-06-09 | Added screenshot/evidence inventory and complete command-log renderings for the final evidence inventory. | + +### 4.5 Evaluation / Benchmark Integration + +| Commit | Date | Evidence | +|---|---|---| +| `c20afcc` / `a49fe14` | 2026-05-22 | Added earlier recall/search evaluation CLI entry points and gold data. | +| `2dd0d64` | 2026-06-03 | Integrated the teammate evaluation branch with hybrid retrieval and memory extraction into the main `jflin`/`dev` line; count this as integration, review, and alignment evidence rather than original authorship of the whole framework. | +| `dfa7adf` / `1e9419e` | 2026-06-10 | Performed the later benchmark merge/update pass and merged refreshed `jflin` work back into `origin/dev`; count as integration/review evidence, not sole authorship of the benchmark stack. | +| `origin/dev` `50609db` / `767eb29` / `965ea4a` | 2026-06-06 to 2026-06-09 | Teammate branch added end-to-end benchmark pipeline, official adapters, semantic judging, and resumable LongMemEval evaluation. Hopecommon's role in this ledger is merge review, conflict resolution, and final-report positioning, not original benchmark implementation. | + +## 5. Report-Ready Personal Contribution Statement + +Suggested final-report wording for member `hopecommon / jflin`: + +| Member | Main responsibility | Delivered artifacts | Course requirement coverage | +|---|---|---|---| +| hopecommon / jflin | Project lead; database schema and governance/provenance design; lexical search / context-pack formatter / CLI dogfood; graph hardening; review/merge/integration; final evidence/report materials | Core SQL schema, views, triggers, indexes, seed/demo data; governance/provenance workflows; lexical search infrastructure; recall context-pack formatter; CLI recall/context/eval/sessions/observe/remember commands; graph visibility/sync/audit hardening over the initial Neo4j integration; ER/sequence diagrams; screenshot/test evidence package; research/innovation/source appendix docs | E-R design, relational schema, integrity constraints, SQL views, triggers, indexes, EXPLAIN validation, governance/provenance implementation, backend/CLI implementation, demo screenshots, member contribution evidence | + +Expanded paragraph: + +> hopecommon / jflin 主要负责 MemoryBase 的数据库主线、治理/溯源模型、agent-facing backend/CLI 能力和最终集成收尾：设计并迭代 `source_document → source_chunk → memory_item → memory_evidence → memory_revision → audit_log` 的 provenance 链路，补齐 conflict / forget / access policy 等 governance 表、视图、触发器和演示数据；同时完整实现 lexical search infrastructure、recall context-pack formatter、CLI recall/context/eval/sessions/observe/remember 命令链和 agent runtime dogfood 配合，并在 lywzc0419 初版 Neo4j Graph Explorer 集成基础上完成 graph visibility/sync/audit 的工程化补强。recall 路径上的 hybrid memory recall、embedding、QA 和 context-pack metadata 由 dzx0902 主导，本人主要协作对齐 API、CLI 和 demo。对同学主责的前端、评测、QA 等模块，主要承担审查、合并、接口对齐、演示证据和报告材料整理，确保各模块收敛成一个可用 SQL 验证、可审计、可权限裁剪、可演示的数据库系统。 + +If the report needs a shorter one-line version: + +> hopecommon / jflin：负责数据库 schema、治理与溯源链路、lexical search / context-pack formatter / CLI dogfood、Graph hardening、审查合并/集成补强，以及最终报告证据资产。 + +## 6. Cautions for Final Report + +Do not overstate these points: + +1. The evaluation framework was integrated into `jflin`/`dev`, but its original implementation should be credited primarily to `dzx0902`. In personal contribution text, count hopecommon's role as integration, review, and alignment unless citing specific hopecommon-authored commits. +2. GitHub merge commits by `evanlin257 <143060834+hopecommon@users.noreply.github.com>` often have no direct file diff relative to the merged branch. They are useful process evidence, not implementation-line evidence. +3. The five `Cofstars` non-merge commits (`c2b8cd9`, `ee623c7`, `89ca761`, `899530e`, `fa6ecd3`) are now merged to `origin/dev`, so they are remote-backed evidence and can be cited directly. +4. This refresh now includes raw Git-backed rows for the other observed members, but their responsibility wording is still lighter-weight than the hopecommon deep audit and should not be over-claimed. +5. GitHub issue state was not re-pulled for this refresh; code/docs artifacts and commit history remain the primary evidence. + +## 7. Other Member Audit Notes + +The same audit method can be reused for other members by changing the author +pattern and storing one TSV per person: + +```bash +git log --all --regexp-ignore-case --author='' ... +``` + +Raw audit files created so far: + +| Member identity | Raw audit file | Direct commits | Merge / integration commits | Notes | +|---|---|---:|---:|---| +| `hopecommon <1841778349@qq.com>` / `evanlin257 <143060834+hopecommon@users.noreply.github.com>` | `docs/contribution-audit-hopecommon.tsv` | 65 non-merge | 16 merge / PR integration | Project lead, database/governance/provenance, lexical search/context-pack formatter/CLI dogfood, graph hardening, review/merge/integration, final evidence. | +| `lywzc0419 ` | `docs/contribution-audit-lywzc0419.tsv` | 8 non-merge | 0 direct merge | Mainly frontend API integration, runtime/governance UI, Neo4j Graph Explorer integration, graph lint/import fixes. | +| `dzx0902 <3575895791@qq.com>` / `dzx0902 <145189098+dzx0902@users.noreply.github.com>` | `docs/contribution-audit-dzx0902.tsv` | 35 non-merge | 15 merge / PR integration | Project bootstrap, P0 backend/tests, evaluation framework, official benchmark adapters/runners, LongMemEval full-run evidence, embedding/hybrid recall, QA, memory extraction, CI/tooling, and integration merges. | +| `Cofstars <2089039907@qq.com>` | `docs/contribution-audit-cofstars.tsv` | 5 non-merge | 0 direct merge | Added source extraction candidate workflow and optional LLM-backed memory extraction pipeline on top of the existing source/memory path, then refreshed final report, slides, and audit artifacts. | + +Recommended responsibility split for the final report: + +| Member | Primary part to emphasize | Secondary / support wording | +|---|---|---| +| hopecommon / jflin | Database schema, governance/provenance model, SQL evidence, lexical search + recall context-pack formatter + CLI dogfood, agent-runtime CLI/sessions/observe/remember, graph hardening over lywzc0419's Neo4j integration, final report assets | Reviewed and integrated teammate frontend/evaluation/QA work; strengthened cross-module consistency and demo readiness. | +| dzx0902 / dzx | P0 backend, tests, evaluation benchmark framework including LoCoMo / LongMemEval / MemoryAgentBench adapters and LongMemEval full-run evidence, embedding/hybrid recall, memory extraction, QA, CI/tooling | Also contributed project bootstrap, batch memory writes / supersession support, and integration merges. | +| huiyijian / lywzc0419 / lzc | Frontend API integration, runtime/governance UI, multi-page UI wiring, Neo4j Graph Explorer integration | Also fixed graph lint/Ruff import issues and supporting dependencies/config. | +| Cofstars | Source extraction workflow, chunking/test additions, source UI creation/detail flow, optional LLM-backed candidate extraction and related CLI/API/tests, plus final report / slides / audit-artifact refreshes | Git evidence is five non-merge commits on `origin/dev` (2026-06-14); keep the statement proportional to that visible scope. | + +Boundary note: the recall/search path is collaborative. Hopecommon mainly owns +lexical search, the recall context-pack formatter, and CLI/dogfood flows; dzx0902 +mainly owns hybrid recall, embedding storage/retrieval, QA, and structured +context-pack metadata. + +Report-ready draft for `dzx0902` / `dzx`: + +> dzx0902 / dzx 主要负责项目初始脚手架与工程化配置、P0 FastAPI 后端和测试体系，并在后续补充 wiki / stats / semantic / governance lifecycle、embedding、hybrid recall、memory lifecycle validation、conflict detection、forgetting verification、context pack metadata、provider-backed QA、batch memory writes 和 explicit supersession 等后端能力；同时实现 evaluation benchmark 框架，包括 LoCoMo / LongMemEval / MemoryAgentBench adapter、datasets、baselines、metrics、runners、semantic judging、checkpoint/resume 和 report generation，并完成 LongMemEval oracle 500-case 工程评测记录。该评测结果主要作为系统能力和限制分析证据，不作为高分榜单卖点。少量前端贡献集中在 runtime/governance review 修正和 Recall/QA client 联调。 + +Report-ready draft for `huiyijian` / `lywzc0419` / `lzc`: + +> huiyijian / lywzc0419 / lzc 主要负责前端 API 集成与运行时/治理页面建设，包括前端路由依赖、API client、布局、Dashboard、governance、memories、recall、sources、wiki、runtime sessions/messages/hybrid search 等页面联调；同时完成 Neo4j Graph Explorer 的前后端集成相关工作，覆盖 graph API/service/model、Neo4j demo SQL、Docker/依赖配置和 README 说明，并修复 graph integration lint / Ruff import 问题。 + +Additional caution for `dzx0902`: merge / PR integration commits prove account-level +integration activity, not authorship of every merged line. `origin/feat/evaluation- +benchmark-framework` is a real evaluation contribution, but it also contains backend, +docs, and frontend support changes; do not describe all benchmark branch commits as pure +`evaluation/` code. `eb34e592` is visible only on +`origin/fix/db-cli-windows-encoding`, so treat it as branch-level contribution unless +it is later merged. + +Identity note: the teammate previously referred to as `huiyijian` maps to the +Git identity `lywzc0419 ` in this repository. As of the +refresh, the previously blank fourth-member slot now has Git-backed evidence +under `Cofstars <2089039907@qq.com>`, covering five non-merge commits now on +`origin/dev`. + +## 8. Suggested Appendix Snippet + +```text +Evidence command: +git log --all --regexp-ignore-case --author='hopecommon\|1841778349' --no-merges --oneline + +Audit result: +65 direct non-merge commits, primarily across database/governance/provenance, +agent-facing backend/CLI, graph hardening, review/integration, and final-report +evidence materials. +16 merge / PR integration commits tracked separately. +``` + +For the final report, use the module table in §3 and the paragraph in §5 rather +than pasting the entire commit list. diff --git a/docs/explain-analyze.md b/docs/explain-analyze.md index 7c6c159..ef90b97 100644 --- a/docs/explain-analyze.md +++ b/docs/explain-analyze.md @@ -2,7 +2,7 @@ 本文件收集 4 个有代表性的查询，配套 `database/04_indexes.sql` 中的索引设计，演示 PostgreSQL 查询执行的实际计划。配套阅读：`docs/index-rationale.md`。 -所有计划均取自实际数据库执行（PostgreSQL 16.13，本课程 demo workspace `00000000-0000-0000-0000-000000000201`，截至本文件撰写时表规模：`memory_item` 20 行 / `audit_log` 30 行 / `source_chunk` 20 行 / `wiki_page` 3 行）。 +所有计划均取自实际数据库执行（PostgreSQL 16.13，本课程 demo workspace `00000000-0000-0000-0000-000000000201`，截至本文件撰写时表规模：`memory_item` 25 行 / `audit_log` 39 行 / `source_chunk` 20 行 / `wiki_page` 3 行）。 > **数据规模说明**：demo 数据量较小，部分场景下 PostgreSQL planner 会**主动选 Seq Scan**（这是正确决定，因为对几十行数据走索引反而更慢——B+ tree 要先读根节点、再读叶子、再回表）。为了**演示索引路径的形态**，几个案例使用 `SET LOCAL enable_seqscan = off` 临时强制 planner 走索引，对比"小数据 + Seq Scan"与"假设大数据时 + Index 路径"的差异。**生产数据量（1 万行以上）下，planner 会自动选择索引路径**，无需强制。 @@ -53,13 +53,13 @@ SELECT memory_id, memory_type, confidence, access_level 强制 `enable_seqscan = off` 让 planner 选索引路径（在大数据量下会自动选）： ``` - Limit (cost=0.14..4.76 rows=10 width=48) (actual time=0.181..0.189 rows=10 loops=1) + Limit (cost=0.14..4.31 rows=10 width=48) (actual time=0.006..0.007 rows=10 loops=1) Buffers: shared hit=2 -> Index Only Scan using idx_memory_active_ranking on memory_item Index Cond: (workspace_id = '00000000-0000-0000-0000-000000000201'::uuid) Heap Fetches: 0 Buffers: shared hit=2 - Execution Time: 0.232 ms + Execution Time: 0.016 ms ``` **关键观察**： @@ -72,25 +72,24 @@ SELECT memory_id, memory_type, confidence, access_level 在事务里 DROP 同一索引，看 fallback 行为： ``` - Limit (cost=11.97..11.99 rows=10 width=48) (actual time=0.109..0.111 rows=10 loops=1) - Buffers: shared hit=10 + Limit (cost=13.07..13.10 rows=10 width=48) (actual time=0.026..0.027 rows=10 loops=1) + Buffers: shared hit=4 -> Sort Sort Key: importance DESC, updated_at DESC Sort Method: quicksort Memory: 27kB -> Bitmap Heap Scan on memory_item - Recheck Cond: (workspace_id = '00000000-0000-0000-0000-000000000201'::uuid) - Filter: ((status)::text = 'active'::text) - Rows Removed by Filter: 2 + Recheck Cond: ((workspace_id = '00000000-0000-0000-0000-000000000201'::uuid) + AND ((status)::text = 'active'::text)) Heap Blocks: exact=3 - -> Bitmap Index Scan on idx_memory_created_at - Execution Time: 0.148 ms + -> Bitmap Index Scan on idx_memory_workspace_status_validity + Execution Time: 0.037 ms ``` **关键退化**： -- 没有 covering 时，planner 退到 `idx_memory_created_at`（B+ tree 复合索引）走 Bitmap Index Scan，**再做 Bitmap Heap Scan 回表**取 `memory_type / confidence / access_level`。 +- 没有 covering 时，planner 退到 `idx_memory_workspace_status_validity`（B+ tree 复合索引）走 Bitmap Index Scan，**再做 Bitmap Heap Scan 回表**取 `memory_type / confidence / access_level`。 - 因为索引顺序不是 `importance DESC`，多了一个 **Sort 节点**。 -- 没有 partial（`WHERE status='active'`），还要在 Recheck 后做一次 **Filter `status = 'active'`**，多扫了 2 行（`Rows Removed by Filter: 2`）。 -- **Buffers 命中 10 个**（vs 2 个），IO 多 5 倍。 +- status 条件仍能进入复合索引，但该索引不覆盖 SELECT 列，也不能消除 `ORDER BY importance DESC, updated_at DESC` 的排序。 +- **Buffers 命中 4 个**（vs 2 个），小表规模下只多 2 页；生产规模下差异主要来自 Sort 和 heap fetch。 > **教学要点**：单一索引同时实现了 partial（缩小规模）+ composite DESC（消除 Sort）+ INCLUDE（消除 heap fetch）三种现代特性。配合 visibility map 即可达到"读 2 个 buffer 页完成 LIMIT 10"的极致。 @@ -127,23 +126,24 @@ SELECT mi.memory_id, mi.memory_type, mi.canonical_text, ### 2.1 完整 plan（默认） ``` - Limit (cost=8.28..8.28 rows=2 width=146) - -> Sort Sort Key: score DESC - -> GroupAggregate Group Key: mi.memory_id - -> Sort Sort Key: mi.memory_id - -> Hash Join Hash Cond: (sc.doc_id = sd.doc_id) - -> Nested Loop - -> Hash Join Hash Cond: (me.chunk_id = sc.chunk_id) - -> Seq Scan on memory_evidence me (20 rows) - -> Seq Scan on source_chunk sc (Filter: FTS OR trigram) - -> Index Scan using memory_item_pkey on memory_item mi - Index Cond: (memory_id = me.memory_id AND workspace_id = ...) - -> Seq Scan on source_document sd - Execution Time: 0.946 ms + Limit (cost=8.72..8.72 rows=2 width=138) (actual time=0.065..0.066 rows=2 loops=1) + Buffers: shared hit=12 + -> Sort Sort Key: score DESC + -> GroupAggregate Group Key: mi.memory_id + -> Sort Sort Key: mi.memory_id + -> Hash Join Hash Cond: (sc.doc_id = sd.doc_id) + -> Nested Loop + -> Hash Join Hash Cond: (me.chunk_id = sc.chunk_id) + -> Seq Scan on memory_evidence me (20 rows) + -> Seq Scan on source_chunk sc (Filter: FTS OR trigram) + -> Index Scan using memory_item_memory_id_workspace_id_key on memory_item mi + Index Cond: (memory_id = me.memory_id AND workspace_id = ...) + -> Seq Scan on source_document sd + Execution Time: 0.106 ms ``` **关键观察**： -- **Nested Loop + Index Scan on `memory_item`** — 内层循环对每个匹配 chunk 通过 (memory_id, workspace_id) 复合 PK index 定位 memory，是典型的"小驱动表 + 索引主键查找"模式。 +- **Nested Loop + Index Scan on `memory_item`** — 内层循环对每个匹配 chunk 通过 `(memory_id, workspace_id)` 复合唯一索引定位 memory，是典型的"小驱动表 + 索引查找"模式。 - `source_chunk` 的 FTS + ILIKE 在小数据量下走 Seq Scan + Filter；大数据量下会切换到 BitmapOr(BitmapIndexScan(`idx_source_chunk_fts`), BitmapIndexScan(`idx_source_chunk_text_trgm`))。 - `GroupAggregate` 在 Sort 之后，按 `memory_id` 聚合多个 evidence。 @@ -165,7 +165,7 @@ SELECT sc.chunk_id, sc.chunk_no, sc.chunk_text 实际 plan： ``` - Limit (cost=48.10..51.23 rows=2 width=167) (actual time=0.029..0.031 rows=2 loops=1) + Limit (cost=48.10..51.23 rows=2 width=167) (actual time=0.016..0.017 rows=2 loops=1) Buffers: shared hit=21 -> Bitmap Heap Scan on source_chunk sc Recheck Cond: ((search_vector @@ '''memorybase'''::tsquery) @@ -178,7 +178,7 @@ SELECT sc.chunk_id, sc.chunk_no, sc.chunk_text -> Bitmap Index Scan on idx_source_chunk_text_trgm Index Cond: (chunk_text ~~* '%memorybase%'::text) Buffers: shared hit=17 - Execution Time: 0.082 ms + Execution Time: 0.022 ms ``` **关键观察**： @@ -223,28 +223,33 @@ SELECT date_trunc('day', created_at) AS day, action_type, count(*) AS cnt ``` Sort Sort Key: day DESC, cnt DESC + Buffers: shared hit=12 -> HashAggregate Group Key: date_trunc(...), action_type + Buffers: shared hit=6 -> Seq Scan on audit_log Filter: created_at >= now() - 30 days - Buffers: shared hit=5 - Execution Time: 0.674 ms + Rows Removed by Filter: 1 + Buffers: shared hit=6 + Execution Time: 0.148 ms ``` -30 行 + 时间过滤几乎全部命中 → Seq Scan 是最优解。 +39 行 + 时间过滤几乎全部命中 → Seq Scan 是最优解。 -### 3.2 假设大数据：强制走索引（B-tree 路径） +### 3.2 假设大数据：强制走索引（B-tree 路径，已复验） `SET enable_seqscan = off`，未单独限制其他索引： ``` Bitmap Heap Scan on audit_log Recheck Cond: (created_at >= now() - 30 days) - Heap Blocks: exact=5 + Heap Blocks: exact=6 + Buffers: shared hit=7 -> Bitmap Index Scan on idx_audit_target_time Index Cond: (created_at >= now() - 30 days) - Buffers: shared hit=6 + Buffers: shared hit=1 + Execution Time: 0.070 ms ``` -注意 planner 选了 `idx_audit_target_time = (workspace_id, target_type, target_id, created_at DESC)` 的复合 B-tree——它的最右列正好是 `created_at`，对 trailing 列范围查询仍然能给出 Bitmap Index Scan 候选页。 +注意：该计划已在 fresh demo DB 上复验。B+ tree 复合索引最擅长使用先导列；这里查询只约束 trailing column `created_at`，planner 仍选择 `idx_audit_target_time`，是因为 demo 数据量很小且我们显式关闭了 Seq Scan。生产报告中应把它解释为"B-tree 竞争路径"，不是推荐的全库时间窗索引。 ### 3.3 BRIN 路径（drop 竞争索引后） @@ -261,16 +266,17 @@ ROLLBACK; ``` Bitmap Heap Scan on audit_log Recheck Cond: (created_at >= now() - 30 days) - Heap Blocks: lossy=5 + Heap Blocks: lossy=6 -> Bitmap Index Scan on idx_audit_brin_time Index Cond: (created_at >= now() - 30 days) Buffers: shared hit=5 - Buffers: shared hit=10 + Buffers: shared hit=11 + Execution Time: 0.195 ms ``` **关键观察**： -- BRIN 报 **`Heap Blocks: lossy=5`** —— 这是 BRIN 的特征：page range 内可能有不满足条件的行，所以是"lossy"，需要 Recheck Cond 在 heap 上验证。B-tree 报的是 `exact=5`，索引精确定位到行。 -- Buffers 略多（10 vs 6），因为 30 行规模下 BRIN 的 page range 元数据反而引入额外读。 +- BRIN 报 **`Heap Blocks: lossy=6`** —— 这是 BRIN 的特征：page range 内可能有不满足条件的行，所以是"lossy"，需要 Recheck Cond 在 heap 上验证。B-tree 报的是 `exact=6`，索引精确定位到行。 +- Buffers 略多（11 vs 7），因为 39 行规模下 BRIN 的 page range 元数据反而引入额外读。 ### 3.4 索引大小对比 @@ -292,7 +298,7 @@ SELECT indexname, | `idx_audit_target_time` | 16 kB | btree | | `idx_audit_workspace_time` | 16 kB | btree | -> **诚实披露**：在 30 行规模下 BRIN 实际**比 B-tree 还大**——因为 BRIN 有元页 + range descriptor 的固定开销，B-tree 30 行也只有 1 个叶子页（最小 16 kB）。 +> **诚实披露**：在 39 行规模下 BRIN 实际**比 B-tree 还大**——因为 BRIN 有元页 + range descriptor 的固定开销，B-tree 39 行也只有 1 个叶子页（最小 16 kB）。 > > **BRIN 的优势在哪儿出现**：当 `audit_log` 行数到 100 万级别时： > - B-tree 索引大小 ≈ 几十 MB（与行数线性相关） @@ -322,32 +328,34 @@ SELECT page_id, page_slug, memory_id, chunk_id, doc_id, source_title, start_line ### 4.1 完整 plan ``` - Limit (cost=10.32..15.60 rows=15 width=115) - -> Hash Left Join Hash Cond: (sc.doc_id = sd.doc_id) - Filter: ((sd.doc_id IS NULL) OR (sd.status = 'active')) - -> Hash Left Join Hash Cond: (me.chunk_id = sc.chunk_id) - -> Nested Loop Left Join (3 loops, 6 rows total) - -> Seq Scan on wiki_page wp Filter: status='active' AND workspace=... - -> Hash Right Join Hash Cond: (me.memory_id = mi.memory_id) - -> Seq Scan on memory_evidence me - -> Hash -> Hash Right Join Hash Cond: (mi.memory_id = page_memory.memory_id) - -> Seq Scan on memory_item mi - -> Hash -> Subquery Scan on page_memory - -> HashAggregate Group Key: memory_id, cell_role, sort_order - -> Append - -> Result One-Time Filter: ... - InitPlan 1 - -> Seq Scan on memory_scene_cell direct_scene_cell - -> Seq Scan on memory_scene_cell msc - -> Hash -> Seq Scan on source_chunk sc - -> Hash -> Seq Scan on source_document sd - Execution Time: 0.652 ms + Limit (cost=18.39..40.01 rows=1 width=1068) (actual time=0.160..0.286 rows=6 loops=1) + Buffers: shared hit=55 + -> Nested Loop Left Join + Filter: ((sd.doc_id IS NULL) OR ((sd.status)::text = 'active'::text)) + -> Nested Loop Left Join + -> Index Scan using idx_wiki_page_workspace_status on wiki_page wp + Index Cond: (workspace_id = ... AND status = 'active') + -> Hash Right Join Hash Cond: (mi.memory_id = page_memory.memory_id) + -> Seq Scan on memory_item mi + -> Hash -> Subquery Scan on page_memory + -> HashAggregate + -> Append + -> Result One-Time Filter: ... + InitPlan 1 + -> Index Only Scan using memory_scene_cell_pkey + -> Bitmap Heap Scan on memory_scene_cell msc + -> Bitmap Index Scan on idx_memory_scene_cell_order + -> Nested Loop Left Join + -> Index Only Scan using memory_evidence_memory_id_chunk_id_evidence_role_key on memory_evidence me + -> Index Scan using source_chunk_pkey on source_chunk sc + -> Index Scan using source_document_pkey on source_document sd + Execution Time: 0.492 ms ``` **关键观察**： - 整棵树有 **6 个 join 节点**（5 个 LEFT JOIN + 1 个 LATERAL），是项目最复杂的查询之一。 -- LATERAL 子查询里用 `Append + Result + Seq Scan` 把"直接关联 memory（generated_from_memory_id）"与"通过 scene_cell 间接关联 memory"两种来源合并去重。 -- 小数据下到处是 Seq Scan，但每个 Seq Scan 后的 Filter 都对应一个有效索引（如 `idx_wiki_page_workspace_status`、`idx_memory_scene_cell_memory`、`idx_memory_scene_cell_order`、`idx_memory_evidence_chunk`）。大数据下计划会切换为对应的 Bitmap Index Scan + Hash Join，整体复杂度 $O(N \log N)$。 +- LATERAL 子查询里用 `Append + Result + Bitmap Heap Scan` 把"直接关联 memory（generated_from_memory_id）"与"通过 scene_cell 间接关联 memory"两种来源合并去重。 +- 当前 demo DB 已经点亮 `idx_wiki_page_workspace_status`、`memory_scene_cell_pkey`、`idx_memory_scene_cell_order`、`memory_evidence_memory_id_chunk_id_evidence_role_key`、`source_chunk_pkey` 和 `source_document_pkey`。这说明视图虽然封装了复杂 join，planner 仍能把过滤和连接下推到具体索引。 > **教学要点**：这个查询展示了**视图作为复杂 join 抽象**的价值——业务代码只需要 `SELECT ... FROM v_wiki_page_sources WHERE workspace_id = ?`，视图把 6 个表的 join + 去重逻辑全部封装。对应课程上"视图作为派生关系 / 关系代数表达"的概念。同时也是 §5.1 提到的 `v_provenance_lineage` 递归 CTE 视图（Tier 2 #11）的语义前身——后者要在此基础上把"固定 5 跳"扩展为"任意深度递归"。 @@ -360,15 +368,20 @@ SELECT page_id, page_slug, memory_id, chunk_id, doc_id, source_title, start_line | 索引 | 案例 | 类别 | |---|---|---| | `idx_memory_active_ranking` | 案例 1（covering） | Partial + Composite DESC + Covering (INCLUDE) | -| `memory_item_pkey` | 案例 2（Nested Loop 内层） | B+ tree PK | -| `memory_item_memory_id_workspace_id_key` | 案例 1B（fallback） | B+ tree 唯一复合 | -| `idx_memory_created_at` | 案例 1B（fallback） | B+ tree 复合 | +| `memory_item_memory_id_workspace_id_key` | 案例 2（Nested Loop 内层） | B+ tree 唯一复合 | +| `idx_memory_workspace_status_validity` | 案例 1B（fallback） | B+ tree 复合 | | `idx_source_chunk_fts` | 案例 2（大数据切换） | GIN tsvector | | `idx_source_chunk_text_trgm` | 案例 2（OR 分支大数据切换） | GIN trigram | | `idx_audit_target_time` | 案例 3.2 | B+ tree 复合（trailing time） | | `idx_audit_brin_time` | 案例 3.3 | **BRIN** | - -> 计 7 个不同索引、覆盖 **7 种索引类别**（PK / 唯一复合 / 复合带 DESC / GIN tsvector / GIN trigram / BRIN / Covering）。完整 8 类索引清单见 `docs/index-rationale.md` §1。 +| `idx_wiki_page_workspace_status` | 案例 4 | B+ tree 复合 | +| `memory_scene_cell_pkey` | 案例 4 | B+ tree PK | +| `idx_memory_scene_cell_order` | 案例 4 | B+ tree 复合排序 | +| `memory_evidence_memory_id_chunk_id_evidence_role_key` | 案例 4 | B+ tree 唯一复合 | +| `source_chunk_pkey` | 案例 4 | B+ tree PK | +| `source_document_pkey` | 案例 4 | B+ tree PK | + +> 计 13 个不同索引、覆盖 **7 种索引类别**（PK / 唯一复合 / 复合带 DESC / GIN tsvector / GIN trigram / BRIN / Covering）。完整 9 类索引清单见 `docs/index-rationale.md` §1。 ## 复现命令汇总 diff --git a/docs/final-assets/diagrams/01-er-core.mmd b/docs/final-assets/diagrams/01-er-core.mmd new file mode 100644 index 0000000..a3289d9 --- /dev/null +++ b/docs/final-assets/diagrams/01-er-core.mmd @@ -0,0 +1,82 @@ +%% MemoryBase — Core ER (8 lifecycle entities, attribute boxes shown) +%% Use this diagram in the report introduction (§3 概念结构设计). +%% Full schema (25 entities) is in 02-er-full. + +erDiagram + WORKSPACE ||--o{ SOURCE_DOCUMENT : "owns" + SOURCE_DOCUMENT ||--o{ SOURCE_CHUNK : "splits_into" + SOURCE_CHUNK ||--o{ MEMORY_EVIDENCE : "supports" + MEMORY_ITEM ||--o{ MEMORY_EVIDENCE : "has" + MEMORY_ITEM ||--o{ MEMORY_REVISION : "has_versions" + WORKSPACE ||--o{ MEMORY_ITEM : "owns" + WORKSPACE ||--o{ WIKI_PAGE : "publishes" + MEMORY_ITEM ||--o{ WIKI_PAGE : "projects_to" + WORKSPACE ||--o{ AUDIT_LOG : "audits" + + WORKSPACE { + uuid workspace_id PK + varchar slug UK + varchar name + text description + } + SOURCE_DOCUMENT { + uuid doc_id PK + uuid workspace_id FK + varchar title + varchar doc_type + varchar status + timestamptz imported_at + } + SOURCE_CHUNK { + uuid chunk_id PK + uuid doc_id FK + int chunk_no + text chunk_text + text search_text_zh + tsvector search_vector + } + MEMORY_ITEM { + uuid memory_id PK + uuid workspace_id FK + varchar memory_type + text canonical_text + text summary + numeric confidence + int importance + varchar status + varchar access_level + int current_revision_no + } + MEMORY_EVIDENCE { + uuid evidence_id PK + uuid memory_id FK + uuid chunk_id FK + varchar evidence_role + numeric weight + } + MEMORY_REVISION { + uuid memory_id FK + int revision_no PK + text revision_text + text revision_summary + timestamptz created_at + } + WIKI_PAGE { + uuid page_id PK + uuid workspace_id FK + varchar page_slug + varchar page_type + varchar status + boolean needs_rebuild + int current_revision_no + } + AUDIT_LOG { + uuid audit_id PK + uuid workspace_id FK + varchar action_type + varchar target_type + uuid target_id + jsonb before_json + jsonb after_json + timestamptz created_at + } diff --git a/docs/final-assets/diagrams/01-er-core.svg b/docs/final-assets/diagrams/01-er-core.svg new file mode 100644 index 0000000..74873b1 --- /dev/null +++ b/docs/final-assets/diagrams/01-er-core.svg @@ -0,0 +1 @@ +

owns

splits_into

supports

has

has_versions

owns

publishes

projects_to

audits

WORKSPACE

uuid

workspace_id

PK

varchar

slug

UK

varchar

name

text

description

SOURCE_DOCUMENT

uuid

doc_id

PK

uuid

workspace_id

FK

varchar

title

varchar

doc_type

varchar

status

timestamptz

imported_at

SOURCE_CHUNK

uuid

chunk_id

PK

uuid

doc_id

FK

int

chunk_no

text

chunk_text

text

search_text_zh

tsvector

search_vector

MEMORY_EVIDENCE

uuid

evidence_id

PK

uuid

memory_id

FK

uuid

chunk_id

FK

varchar

evidence_role

numeric

weight

MEMORY_ITEM

uuid

memory_id

PK

uuid

workspace_id

FK

varchar

memory_type

text

canonical_text

text

summary

numeric

confidence

int

importance

varchar

status

varchar

access_level

int

current_revision_no

MEMORY_REVISION

uuid

memory_id

FK

int

revision_no

PK

text

revision_text

text

revision_summary

timestamptz

created_at

WIKI_PAGE

uuid

page_id

PK

uuid

workspace_id

FK

varchar

page_slug

varchar

page_type

varchar

status

boolean

needs_rebuild

int

current_revision_no

AUDIT_LOG

uuid

audit_id

PK

uuid

workspace_id

FK

varchar

action_type

varchar

target_type

uuid

target_id

jsonb

before_json

jsonb

after_json

timestamptz

created_at

 \ No newline at end of file diff --git a/docs/final-assets/diagrams/02-er-full.mmd b/docs/final-assets/diagrams/02-er-full.mmd new file mode 100644 index 0000000..a3a81b8 --- /dev/null +++ b/docs/final-assets/diagrams/02-er-full.mmd @@ -0,0 +1,59 @@ +%% MemoryBase — Full ER (25 entities, main persisted/logical relationships) +%% Use this diagram as the E-R appendix. Generic principal and polymorphic targets are explained in text. +%% For core lifecycle with attribute boxes, see 01-er-core. + +erDiagram + USER_ACCOUNT ||--o{ WORKSPACE : "owns" + WORKSPACE ||--o{ AGENT : "registers" + USER_ACCOUNT ||--o{ AGENT : "owns" + WORKSPACE ||--o{ WORKSPACE_MEMBER : "has_members" + USER_ACCOUNT ||--o{ WORKSPACE_MEMBER : "joins_as" + AGENT ||--o{ WORKSPACE_MEMBER : "joins_as" + + WORKSPACE ||--o{ AGENT_SESSION : "scopes" + AGENT ||--o{ AGENT_SESSION : "drives" + USER_ACCOUNT ||--o{ AGENT_SESSION : "starts" + AGENT_SESSION ||--o{ MESSAGE : "records" + MESSAGE ||--o{ MESSAGE : "replies" + + WORKSPACE ||--o{ SOURCE_DOCUMENT : "imports" + AGENT_SESSION ||--o{ SOURCE_DOCUMENT : "captures" + USER_ACCOUNT ||--o{ SOURCE_DOCUMENT : "imports" + SOURCE_DOCUMENT ||--o{ SOURCE_CHUNK : "splits_into" + SOURCE_CHUNK ||--o{ MEMORY_EVIDENCE : "supports" + SOURCE_CHUNK ||--o{ SOURCE_CHUNK_EMBEDDING : "caches_embedding" + + WORKSPACE ||--o{ MEMORY_ITEM : "owns" + SOURCE_DOCUMENT ||--o{ MEMORY_ITEM : "creates" + USER_ACCOUNT ||--o{ MEMORY_ITEM : "owns" + AGENT ||--o{ MEMORY_ITEM : "owns" + MEMORY_ITEM ||--o{ MEMORY_EVIDENCE : "has" + MEMORY_ITEM ||--o{ MEMORY_REVISION : "has_versions" + MEMORY_ITEM ||--o{ MEMORY_EMBEDDING : "caches_embedding" + MEMORY_ITEM ||--o{ MEMORY_ITEM : "supersedes" + + WORKSPACE ||--o{ ENTITY : "defines" + MEMORY_ITEM ||--o{ MEMORY_ENTITY : "tags" + ENTITY ||--o{ MEMORY_ENTITY : "appears_in" + WORKSPACE ||--o{ MEMORY_SCENE : "defines" + MEMORY_SCENE ||--o{ MEMORY_SCENE_CELL : "contains" + MEMORY_ITEM ||--o{ MEMORY_SCENE_CELL : "grouped_by" + + WORKSPACE ||--o{ WIKI_PAGE : "publishes" + WIKI_PAGE ||--o{ WIKI_PAGE_REVISION : "has_versions" + MEMORY_SCENE ||--o{ WIKI_PAGE : "scene_page" + MEMORY_ITEM ||--o{ WIKI_PAGE : "memory_page" + + WORKSPACE ||--o{ TIMELINE_ENTRY : "tracks" + MEMORY_ITEM ||--o{ TIMELINE_ENTRY : "anchors" + SOURCE_DOCUMENT ||--o{ TIMELINE_ENTRY : "anchors" + WORKSPACE ||--o{ RECALL_LOG : "logs" + AGENT ||--o{ RECALL_LOG : "queries" + USER_ACCOUNT ||--o{ RECALL_LOG : "queries" + WORKSPACE ||--o{ ACCESS_POLICY : "controls" + WORKSPACE ||--o{ FORGET_REQUEST : "reviews" + USER_ACCOUNT ||--o{ FORGET_REQUEST : "requests" + USER_ACCOUNT ||--o{ FORGET_REQUEST : "reviews" + MEMORY_ITEM ||--o{ CONFLICT_RECORD : "left" + MEMORY_ITEM ||--o{ CONFLICT_RECORD : "right" + WORKSPACE ||--o{ AUDIT_LOG : "audits" diff --git a/docs/final-assets/diagrams/02-er-full.svg b/docs/final-assets/diagrams/02-er-full.svg new file mode 100644 index 0000000..c78ffe5 --- /dev/null +++ b/docs/final-assets/diagrams/02-er-full.svg @@ -0,0 +1 @@ +

owns

registers

owns

has_members

joins_as

joins_as

scopes

drives

starts

records

replies

imports

captures

imports

splits_into

supports

caches_embedding

owns

creates

owns

owns

has

has_versions

caches_embedding

supersedes

defines

tags

appears_in

defines

contains

grouped_by

publishes

has_versions

scene_page

memory_page

tracks

anchors

anchors

logs

queries

queries

controls

reviews

requests

reviews

left

right

audits

USER_ACCOUNT

WORKSPACE

AGENT

WORKSPACE_MEMBER

AGENT_SESSION

MESSAGE

SOURCE_DOCUMENT

SOURCE_CHUNK

MEMORY_EVIDENCE

SOURCE_CHUNK_EMBEDDING

MEMORY_ITEM

MEMORY_REVISION

MEMORY_EMBEDDING

ENTITY

MEMORY_ENTITY

MEMORY_SCENE

MEMORY_SCENE_CELL

WIKI_PAGE

WIKI_PAGE_REVISION

TIMELINE_ENTRY

RECALL_LOG

ACCESS_POLICY

FORGET_REQUEST

CONFLICT_RECORD

AUDIT_LOG

 \ No newline at end of file diff --git a/docs/final-assets/diagrams/03-source-to-wiki.mmd b/docs/final-assets/diagrams/03-source-to-wiki.mmd new file mode 100644 index 0000000..26b0daf --- /dev/null +++ b/docs/final-assets/diagrams/03-source-to-wiki.mmd @@ -0,0 +1,68 @@ +%% MemoryBase — Source-to-Wiki end-to-end business sequence +%% Covers: import → chunk → extract candidate → approve → evidence → revision → wiki export +%% Use in report §4 数据流 / §6 系统实现. + +sequenceDiagram + autonumber + actor User as User / Agent + participant FE as Frontend / CLI + participant API as FastAPI (backend/app/api) + participant Service as Service Layer + participant DB as PostgreSQL + participant Trig as Triggers + participant Wiki as data/markdown_wiki/ + + rect rgb(245, 247, 255) + Note over User,DB: 1) Source ingestion + User->>FE: upload markdown file + FE->>API: POST /api/sources + API->>Service: SourceService.import_source(payload) + Service->>DB: INSERT source_document (status='active', imported_at) + Service->>DB: INSERT source_chunk rows (chunk_no, chunk_text, search_text_zh) + DB->>DB: GENERATED ALWAYS computes search_vector (tsvector) + Service-->>API: SourceImportResponse {doc_id, chunk_count} + API-->>FE: 200 OK + end + + rect rgb(245, 255, 247) + Note over User,DB: 2) Rule-based memory extraction (candidate) + User->>FE: click "Extract candidates" + FE->>API: POST /api/memory-extraction/from-chunks + API->>Service: MemoryExtractionService.extract_from_chunks(payload) + Service->>DB: INSERT audit_log (memory_extraction.run.start) + loop for each chunk match + Service->>DB: INSERT memory_item (status='candidate') + DB-->>Trig: trg_memory_after_insert INSERT memory_revision + audit_log + Service->>DB: INSERT memory_evidence (evidence_role='source') + end + Service->>DB: INSERT audit_log (memory_extraction.run.complete, candidate_count) + Service-->>API: MemoryExtractionResponse + API-->>FE: candidates list + end + + rect rgb(255, 251, 240) + Note over User,DB: 3) Candidate approval and revision + User->>FE: approve a candidate + FE->>API: POST /api/memory-candidates/{id}/approve?workspace_id=... + API->>Service: MemoryExtractionService.approve_candidate(...) + Service->>DB: UPDATE memory_item + DB-->>Trig: trg_memory_before_update increments current_revision_no + DB-->>Trig: trg_memory_after_update INSERT memory_revision + DB-->>Trig: trg_memory_after_update INSERT audit_log + Service-->>API: MemoryCandidateDecisionResponse {memory} + API-->>FE: 200 OK + end + + rect rgb(248, 245, 255) + Note over User,Wiki: 4) Wiki projection + User->>FE: click "Export wiki" + FE->>API: POST /api/wiki/export + API->>Service: WikiService.export_page/export_pages(payload) + Service->>DB: INSERT wiki_page (page_slug, generated_from_memory_id) + Service->>DB: INSERT wiki_page_revision (body_markdown, frontmatter_json) + DB-->>Trig: trg_wiki_revision_after_insert updates current_revision_no + DB-->>Trig: trg_wiki_revision_after_insert INSERT audit_log (wiki.revision.insert) + Service->>Wiki: write markdown file (provenance citations) + Service-->>API: WikiExportResponse / WikiBatchExportResponse + API-->>FE: 200 OK + end diff --git a/docs/final-assets/diagrams/03-source-to-wiki.svg b/docs/final-assets/diagrams/03-source-to-wiki.svg new file mode 100644 index 0000000..3d77bb6 --- /dev/null +++ b/docs/final-assets/diagrams/03-source-to-wiki.svg @@ -0,0 +1 @@ +data/markdown_wiki/TriggersPostgreSQLService LayerFastAPI (backend/app/api)Frontend / CLIdata/markdown_wiki/TriggersPostgreSQLService LayerFastAPI (backend/app/api)Frontend / CLI1) Source ingestion2) Rule-based memory extraction (candidate)loop[for each chunk match]3) Candidate approval and revision4) Wiki projectionUser / Agentupload markdown file1POST /api/sources2SourceService.import_source(payload)3INSERT source_document (status='active', imported_at)4INSERT source_chunk rows (chunk_no, chunk_text, search_text_zh)5GENERATED ALWAYS computes search_vector (tsvector)6SourceImportResponse {doc_id, chunk_count}7200 OK8click "Extract candidates"9POST /api/memory-extraction/from-chunks10MemoryExtractionService.extract_from_chunks(payload)11INSERT audit_log (memory_extraction.run.start)12INSERT memory_item (status='candidate')13trg_memory_after_insert INSERT memory_revision + audit_log14INSERT memory_evidence (evidence_role='source')15INSERT audit_log (memory_extraction.run.complete, candidate_count)16MemoryExtractionResponse17candidates list18approve a candidate19POST /api/memory-candidates/{id}/approve?workspace_id=...20MemoryExtractionService.approve_candidate(...)21UPDATE memory_item22trg_memory_before_update increments current_revision_no23trg_memory_after_update INSERT memory_revision24trg_memory_after_update INSERT audit_log25MemoryCandidateDecisionResponse {memory}26200 OK27click "Export wiki"28POST /api/wiki/export29WikiService.export_page/export_pages(payload)30INSERT wiki_page (page_slug, generated_from_memory_id)31INSERT wiki_page_revision (body_markdown, frontmatter_json)32trg_wiki_revision_after_insert updates current_revision_no33trg_wiki_revision_after_insert INSERT audit_log (wiki.revision.insert)34write markdown file (provenance citations)35WikiExportResponse / WikiBatchExportResponse36200 OK37User / Agent \ No newline at end of file diff --git a/docs/final-assets/diagrams/04-recall.mmd b/docs/final-assets/diagrams/04-recall.mmd new file mode 100644 index 0000000..eb8f242 --- /dev/null +++ b/docs/final-assets/diagrams/04-recall.mmd @@ -0,0 +1,53 @@ +%% MemoryBase — Recall sequence (keyword / hybrid path with transparent fallback) +%% Covers: query -> permission filter -> lexical/hybrid retrieval -> context pack -> recall_log +%% Use in report Recall module section. + +sequenceDiagram + autonumber + actor Caller as Agent / User / CLI + participant API as FastAPI /api/recall + participant Recall as RecallService + participant Embed as EmbeddingService + participant DB as PostgreSQL + participant Vis as v_agent_visible_memory + participant GIN as GIN tsvector / trigram + participant Cache as memory_embedding + source_chunk_embedding + + Caller->>API: POST /api/recall {query, agent_id, workspace_id, mode} + + Note over API,Vis: Permission filter + API->>Recall: recall(query, agent_id, mode=hybrid) + alt agent_id provided + Recall->>Vis: SELECT memory_id WHERE agent_id = $1 + Vis-->>Recall: visible_memory_ids + else no agent_id + Recall->>DB: filter access_level IN ('public', 'project') + end + + Note over Recall,GIN: Lexical retrieval (keyword/hybrid path) + Recall->>GIN: chunk FTS via idx_source_chunk_fts (+ trigram fallback) + GIN-->>Recall: chunk_id ranked list + Recall->>GIN: memory FTS via idx_memory_fts on memory_item.search_vector + GIN-->>Recall: memory_id keyword candidates + + alt mode is hybrid AND embeddings backfilled + Note over Recall,Cache: Vector retrieval (optional) + Recall->>Embed: encode(query) -> query_vector + Embed-->>Recall: query_vector (JSONB) + Recall->>Cache: cosine match memory + chunk embeddings + Cache-->>Recall: embedding candidates + Recall->>Recall: merge keyword + vector, effective_mode = hybrid + else embeddings missing + Recall->>Recall: effective_mode = keyword, fallback_reason = no_embeddings + end + + Note over Recall,DB: Provenance assembly + Recall->>DB: JOIN memory_evidence + source_chunk + source_document + DB-->>Recall: candidates with chunks, doc title, line range + Recall->>Recall: score + truncate to token budget, build context_pack_json + + Note over Recall,DB: Audit and observability + Recall->>DB: INSERT recall_log (query, count, context_pack_json with retrieval_info) + + Recall-->>API: RecallResponse {memories, citations, retrieval_info} + API-->>Caller: 200 OK diff --git a/docs/final-assets/diagrams/04-recall.svg b/docs/final-assets/diagrams/04-recall.svg new file mode 100644 index 0000000..63f9678 --- /dev/null +++ b/docs/final-assets/diagrams/04-recall.svg @@ -0,0 +1 @@ +memory_embedding + source_chunk_embeddingGIN tsvector / trigramv_agent_visible_memoryPostgreSQLEmbeddingServiceRecallServiceFastAPI /api/recallmemory_embedding + source_chunk_embeddingGIN tsvector / trigramv_agent_visible_memoryPostgreSQLEmbeddingServiceRecallServiceFastAPI /api/recallPermission filteralt[agent_id provided][no agent_id]Lexical retrieval (keyword/hybrid path)Vector retrieval (optional)alt[mode is hybrid AND embeddings backfilled][embeddings missing]Provenance assemblyAudit and observabilityAgent / User / CLIPOST /api/recall {query, agent_id, workspace_id, mode}1recall(query, agent_id, mode=hybrid)2SELECT memory_id WHERE agent_id = $13visible_memory_ids4filter access_level IN ('public', 'project')5chunk FTS via idx_source_chunk_fts (+ trigram fallback)6chunk_id ranked list7memory FTS via idx_memory_fts on memory_item.search_vector8memory_id keyword candidates9encode(query) -> query_vector10query_vector (JSONB)11cosine match memory + chunk embeddings12embedding candidates13merge keyword + vector, effective_mode = hybrid14effective_mode = keyword, fallback_reason = no_embeddings15JOIN memory_evidence + source_chunk + source_document16candidates with chunks, doc title, line range17score + truncate to token budget, build context_pack_json18INSERT recall_log (query, count, context_pack_json with retrieval_info)19RecallResponse {memories, citations, retrieval_info}20200 OK21Agent / User / CLI \ No newline at end of file diff --git a/docs/final-assets/diagrams/05-governance.mmd b/docs/final-assets/diagrams/05-governance.mmd new file mode 100644 index 0000000..9009cf3 --- /dev/null +++ b/docs/final-assets/diagrams/05-governance.mmd @@ -0,0 +1,68 @@ +%% MemoryBase — Governance lifecycle sequences +%% Covers: (A) conflict create/resolve, (B) forget request approval, (C) revision audit +%% Use in report §7 创新点 / 治理章节. + +sequenceDiagram + autonumber + actor Editor as Editor / Admin + participant API as FastAPI governance router + participant Service as Service Layer + participant DB as PostgreSQL + participant Trig as Triggers + participant Audit as audit_log + + rect rgb(255, 245, 245) + Note over Editor,Audit: (A) Conflict — create, trigger flag, resolve/ignore + Editor->>API: POST /api/conflicts {left_memory_id, right_memory_id, conflict_type} + API->>Service: GovernanceService.create_conflict(payload) + Service->>DB: INSERT conflict_record (status='open', left/right_memory_id) + DB-->>Trig: trg_conflict_after_insert marks active endpoints conflicted + DB-->>Trig: trg_memory_after_update writes memory_revision + audit_log + Service->>Audit: INSERT audit_log (conflict.create) + Service-->>API: ConflictRecord (open) + + Editor->>API: PATCH /api/conflicts/{id}?workspace_id=... {status, resolution_note} + API->>Service: GovernanceService.update_conflict(conflict_id, payload) + Service->>DB: UPDATE conflict_record SET status='resolved' or 'ignored' + DB-->>Trig: trg_conflict_after_update checks remaining open conflicts + alt no remaining open conflicts + DB-->>Trig: restore conflicted endpoints to active + DB-->>Trig: trg_memory_after_update writes memory_revision + audit_log + else still conflicted elsewhere + DB-->>Trig: keep endpoint status unchanged + end + Service->>Audit: INSERT audit_log (conflict.update) + end + + rect rgb(245, 255, 247) + Note over Editor,Audit: (B) Forget — request, review, soft-delete + Editor->>API: POST /api/forget-requests {target_type, target_id, reason} + API->>Service: GovernanceService.create_forget_request(payload) + Service->>DB: INSERT forget_request (status='pending') + Service->>Audit: INSERT audit_log (forget_request.create) + + Editor->>API: PATCH /api/forget-requests/{id}?workspace_id=... {status='approved'|'done'} + API->>Service: GovernanceService.update_forget_request(request_id, payload) + alt target_type = 'memory_item' + Service->>DB: UPDATE memory_item SET status='forgotten' WHERE id = target_id + DB-->>Trig: trg_memory_after_update writes memory_revision + audit_log + Service->>DB: UPDATE generated wiki_page SET status='forgotten' + else target_type = 'source_document' or 'wiki_page' or 'entity' + Service->>DB: UPDATE SET status='forgotten', forgotten_at=now() + Service->>Audit: INSERT audit_log (.forget) + end + Service->>DB: UPDATE forget_request SET requested status, reviewed_by_user_id, resolved_at + Service->>Audit: INSERT audit_log (forget_request.update) + Service-->>API: ForgetRequest + end + + rect rgb(245, 247, 255) + Note over Editor,Audit: (C) Revision — edit memory, immutable history + Editor->>API: PATCH /api/memories/{id} (canonical_text or governance fields) + API->>Service: MemoryService.update_memory(memory_id, patch) + Service->>DB: UPDATE memory_item SET ... + DB-->>Trig: trg_memory_before_update increments current_revision_no + DB-->>Trig: trg_memory_after_update INSERT memory_revision (immutable) + DB-->>Trig: trg_memory_after_update INSERT audit_log (before_json, after_json) + Service-->>API: MemoryItem (new current_revision_no) + end diff --git a/docs/final-assets/diagrams/05-governance.svg b/docs/final-assets/diagrams/05-governance.svg new file mode 100644 index 0000000..c3f5b84 --- /dev/null +++ b/docs/final-assets/diagrams/05-governance.svg @@ -0,0 +1 @@ +audit_logTriggersPostgreSQLService LayerFastAPI governance routeraudit_logTriggersPostgreSQLService LayerFastAPI governance router(A) Conflict — create, trigger flag, resolve/ignorealt[no remaining open conflicts][still conflicted elsewhere](B) Forget — request, review, soft-deletealt[target_type = 'memory_item'][target_type = 'source_document' or 'wiki_page' or 'entity'](C) Revision — edit memory, immutable historyEditor / AdminPOST /api/conflicts {left_memory_id, right_memory_id, conflict_type}1GovernanceService.create_conflict(payload)2INSERT conflict_record (status='open', left/right_memory_id)3trg_conflict_after_insert marks active endpoints conflicted4trg_memory_after_update writes memory_revision + audit_log5INSERT audit_log (conflict.create)6ConflictRecord (open)7PATCH /api/conflicts/{id}?workspace_id=... {status, resolution_note}8GovernanceService.update_conflict(conflict_id, payload)9UPDATE conflict_record SET status='resolved' or 'ignored'10trg_conflict_after_update checks remaining open conflicts11restore conflicted endpoints to active12trg_memory_after_update writes memory_revision + audit_log13keep endpoint status unchanged14INSERT audit_log (conflict.update)15POST /api/forget-requests {target_type, target_id, reason}16GovernanceService.create_forget_request(payload)17INSERT forget_request (status='pending')18INSERT audit_log (forget_request.create)19PATCH /api/forget-requests/{id}?workspace_id=... {status='approved'|'done'}20GovernanceService.update_forget_request(request_id, payload)21UPDATE memory_item SET status='forgotten' WHERE id = target_id22trg_memory_after_update writes memory_revision + audit_log23UPDATE generated wiki_page SET status='forgotten'24UPDATE <target_table> SET status='forgotten', forgotten_at=now()25INSERT audit_log (<target_type>.forget)26UPDATE forget_request SET requested status, reviewed_by_user_id, resolved_at27INSERT audit_log (forget_request.update)28ForgetRequest29PATCH /api/memories/{id} (canonical_text or governance fields)30MemoryService.update_memory(memory_id, patch)31UPDATE memory_item SET ...32trg_memory_before_update increments current_revision_no33trg_memory_after_update INSERT memory_revision (immutable)34trg_memory_after_update INSERT audit_log (before_json, after_json)35MemoryItem (new current_revision_no)36Editor / Admin \ No newline at end of file diff --git a/docs/final-assets/diagrams/06-memory-status.mmd b/docs/final-assets/diagrams/06-memory-status.mmd new file mode 100644 index 0000000..e2fe384 --- /dev/null +++ b/docs/final-assets/diagrams/06-memory-status.mmd @@ -0,0 +1,28 @@ +%% MemoryBase — memory_item.status state machine +%% Optional appendix diagram showing the 7-state lifecycle enforced by +%% the CHECK constraint in database/02_schema_memory.sql and trigger-driven +%% transitions in database/06_triggers.sql. +%% Labels stay short; service paths are explained in the report text. + +stateDiagram-v2 + direction LR + + [*] --> candidate : rule-based extraction + [*] --> active : manual creation + + candidate --> active : approve + candidate --> rejected : reject + + active --> conflicted : conflict detected + conflicted --> active : resolved (both_valid) + conflicted --> superseded : manual merge + + active --> superseded : newer memory replaces + active --> archived : soft delete + active --> forgotten : forget approved + + archived --> forgotten : escalated forget + + rejected --> [*] + forgotten --> [*] + superseded --> [*] diff --git a/docs/final-assets/diagrams/06-memory-status.svg b/docs/final-assets/diagrams/06-memory-status.svg new file mode 100644 index 0000000..56f10db --- /dev/null +++ b/docs/final-assets/diagrams/06-memory-status.svg @@ -0,0 +1 @@ +

rule-based extraction

manual creation

approve

reject

conflict detected

resolved (both_valid)

manual merge

newer memory replaces

soft delete

forget approved

escalated forget

candidate

active

rejected

conflicted

superseded

archived

forgotten

 \ No newline at end of file diff --git a/docs/final-assets/diagrams/README.md b/docs/final-assets/diagrams/README.md new file mode 100644 index 0000000..323fc86 --- /dev/null +++ b/docs/final-assets/diagrams/README.md @@ -0,0 +1,36 @@ +# MemoryBase Diagrams + +Mermaid sources (`.mmd`) and rendered SVG outputs for the final report and PPT. + +## How to re-render + +```bash +npm install -g @mermaid-js/mermaid-cli +cd docs/final-assets/diagrams +for f in *.mmd; do + mmdc -i "$f" -o "${f%.mmd}.svg" -t neutral -b white +done +``` + +Use SVG outputs for PDF/PPT embedding (vector quality). Keep `.mmd` sources +as the source of truth; regenerate SVGs after edits. + +## Diagram index + +| File | Type | Purpose | Where to use in the report | +|---|---|---|---| +| `01-er-core.{mmd,svg}` | erDiagram | Simplified 8-entity ER with attribute boxes (Workspace / SourceDocument / SourceChunk / MemoryItem / MemoryEvidence / MemoryRevision / WikiPage / AuditLog) | §3 概念结构设计 — opening figure | +| `02-er-full.{mmd,svg}` | erDiagram | Full 25-entity ER with the main persisted and logical relationships covering core + governance + semantic + runtime. Generic principal and polymorphic audit/forget targets are described in text rather than drawn exhaustively. | E-R 完整图 / 附录 | +| `03-source-to-wiki.{mmd,svg}` | sequenceDiagram | End-to-end business sequence: import → chunk → extract candidate → approve → evidence → revision → wiki export. Includes trigger-driven memory revision and wiki revision audit writes. | §4 数据流图 / §6 系统实现章节 | +| `04-recall.{mmd,svg}` | sequenceDiagram | Recall path: caller → permission filter (`v_agent_visible_memory` when `agent_id` is provided, otherwise public/project fallback) → lexical (GIN) + optional vector (JSONB embedding cache) → context pack + provenance → `recall_log` with `retrieval_info` (mode/fallback_reason). | §6 Recall 模块 / §7 创新点 hybrid retrieval | +| `05-governance.{mmd,svg}` | sequenceDiagram | Three governance flows in one diagram: (A) conflict create → trigger flag → resolve/ignore, (B) forget request → approval/done → soft-forget, (C) memory revision via triggers (`trg_memory_before_update`, `trg_memory_after_update`). | §7 创新点 / governance 章节 | +| `06-memory-status.{mmd,svg}` | stateDiagram-v2 | `memory_item.status` lifecycle (7-state CHECK constraint): candidate → active → archived/forgotten/superseded/rejected/conflicted, with service-validated and trigger-driven transitions. | 附录 / status 状态机 | + +## Canonical sources + +- ER entities and relationships: `database/01_schema_core.sql`, `02_schema_memory.sql`, `03_schema_governance.sql`. +- Recall path: `backend/app/services/recall_service.py` + `database/05_views.sql` (v_agent_visible_memory). +- Governance triggers: `database/06_triggers.sql`. +- Status enum: `database/02_schema_memory.sql` CHECK constraint on `memory_item.status`. + +When the schema changes, re-verify diagrams against the SQL canonical source. diff --git a/docs/final-assets/screenshots/README.md b/docs/final-assets/screenshots/README.md new file mode 100644 index 0000000..a0ef409 --- /dev/null +++ b/docs/final-assets/screenshots/README.md @@ -0,0 +1,86 @@ +# Screenshot and Evidence Inventory + +This directory contains report/PPT-ready screenshots captured from the final demo +workspace on 2026-06-09. + +## Runtime Used + +- Branch: `jflin` +- Capture baseline commit: `072c376` +- Database setup: + - `docker compose up -d postgres` + - `npm run db:setup` + - `npm run db:run -- database/09_graph_demo.sql` +- Backend: `../.venv/bin/python -m uvicorn app.main:app --host 127.0.0.1 --port 8000` from `backend/` +- Frontend: `npm run dev -- --host 127.0.0.1 --port 5173` from `frontend/` +- Core demo workspace: `00000000-0000-0000-0000-000000000201` +- Graph demo workspace: `00000000-0000-0000-0000-000000002201` + +## UI Screenshots + +| File | URL / Action | What It Proves | Suggested Use | +|---|---|---|---| +| `ui/01-dashboard.png` | `/` | Demo data is loaded: sources, memories, wiki, recall, policies, conflicts, audit, forget requests. | Report §16 overview; PPT opening demo slide | +| `ui/02-sources-list.png` | `/sources` | Seeded discussion documents are visible as SourceDocument rows. | Source import / data input | +| `ui/03-source-detail-project-pivot.png` | `/sources/00000000-0000-0000-0000-000000000501?workspace_id=...0201` | Source detail shows raw document metadata and chunk line ranges. | Provenance chain | +| `ui/04-memories-list.png` | `/memories` | MemoryItem list exposes type, status, importance, workspace, and updated time. | Memory management | +| `ui/05-memory-detail-evidence-revisions.png` | `/memories/00000000-0000-0000-0000-000000000701?workspace_id=...0201` | A memory can be traced to evidence and revision history. | MemoryEvidence / MemoryRevision demo | +| `ui/06-recall-search-results.png` | `/recall`, query `为什么放弃校园食堂系统？`, Search | Recall returns the expected cafeteria-topic answer with retrieval info and recall id. | Main recall demo | +| `ui/07-recall-context-pack.png` | Same query, Context Pack | Recall results are converted into agent-ready Markdown context. | Agent integration / context pack | +| `ui/08-recall-qa-answer-or-config-state.png` | Same query, Ask with `LLM_PROVIDER=deepseek` | Optional QA path works when an OpenAI-compatible LLM provider is configured; the screenshot shows `deepseek`, `deepseek-v4-flash`, 4 supporting memories, and cited answer text. | Optional QA / LLM integration | +| `ui/09-governance-timeline.png` | `/governance/timeline` | TimelineEntry projects project history into a human-readable view. | Timeline / project evolution | +| `ui/10-governance-audit.png` | `/governance/audit` | AuditLog records memory lifecycle events and actor attribution. | Audit/governance | +| `ui/11-governance-policies.png` | `/governance/policies` | AccessPolicy rows define agent-visible resource scopes. | Agent-aware visibility | +| `ui/12-governance-conflicts.png` | `/governance/conflicts` | ConflictRecord shows open and resolved memory conflicts. | Conflict governance | +| `ui/13-governance-forget-requests.png` | `/governance/forget-requests` | ForgetRequest workflow keeps forgotten targets auditable. | Forget governance | +| `ui/14-wiki-export.png` | `/wiki` | Wiki pages/export settings expose Markdown projection from database memory. | Wiki projection | +| `ui/15-graph-explorer-demo-workspace.png` | `/graph`, click `Use Graph Demo` | PostgreSQL graph preview renders the curated graph workspace; Neo4j is transparently optional/disabled. | Graph Explorer | +| `ui/16-runtime-sessions.png` | `/runtime/sessions` | Agent runtime sessions and messages are persisted. | Agent runtime | +| `ui/17-runtime-messages.png` | `/runtime/messages` | Runtime message recording UI is available. | Agent runtime write path | +| `ui/18-runtime-hybrid-search-results.png` | `/runtime/search`, query `why abandoned cafeteria project`, Run Hybrid Search | Search returns chunk/memory results with RRF-style strategies such as `memory_fts`, `chunk_fts`, `trigram_fuzzy`. | Hybrid search / retrieval | +| `ui/19-llm-not-used.png` | `/sources/{id}`, select 3 chunks, keep `Use LLM` unchecked, Extract Candidates | Default rule-based extraction classifies the demo sentences into `decision` / `task` / `constraint` with lower confidence. | Report AI comparison | +| `ui/19-llm-used.png` | Same page, check `Use LLM`, fill Qwen-compatible provider fields, Extract Candidates | Optional LLM extraction classifies the same sentences into `decision` / `decision` / `policy` with higher confidence while keeping the same candidate-review workflow. | Report AI comparison | + +Additional raw pre-interaction captures are kept as `ui/15-graph-explorer.png` and +`ui/18-runtime-hybrid-search.png`; prefer the `*-demo-workspace` and `*-results` +versions for final materials. + +## SQL / EXPLAIN Evidence + +| File | Source Log | What It Proves | Suggested Use | +|---|---|---|---| +| `sql/01-db-check.png` | `logs/db-check.txt` | `npm run db:check` succeeds; 25 core tables plus demo/graph workspace data are present. | Test/result appendix | +| `sql/02-demo-queries.png` | `logs/demo-queries.txt` | `database/08_demo_queries.sql` runs and returns broad demo query results. | SQL demo appendix | +| `sql/03-explain-analyze.png` | `logs/explain-cases.txt` | EXPLAIN shows `Index Only Scan`, `Heap Fetches: 0`, `BitmapOr`, GIN FTS/trigram, and BRIN paths. | Physical design / index chapter | +| `sql/04-focused-sql-evidence.png` | `logs/focused-sql.txt` | Concise SQL proof for provenance, visibility, governance states, conflict records, and recall logs. | Main report §16 / PPT SQL slide | + +The matching `.svg` files are also included for sharper scaling in slides/PDFs. +PNG command-output screenshots are rendered directly from the full files in +`logs/`, with long lines wrapped for readability, so they preserve the complete +command output instead of relying on clipped thumbnail previews. + +## Test / Build Evidence + +| File | Source Log | What It Proves | Suggested Use | +|---|---|---|---| +| `tests/01-pytest-core.png` | `logs/pytest-core.txt` | Backend focused subset passes: `test_health.py`, `test_recall_query.py`, `test_graph_service.py` (`16 passed`). | Test plan/results chapter | +| `tests/02-frontend-build.png` | `logs/frontend-build.txt` | React/Vite production build succeeds. | Frontend validation | + +## Notes and Caveats + +- `ui/08-recall-qa-answer-or-config-state.png` was captured with optional DeepSeek + QA configuration (`LLM_PROVIDER=deepseek`, + `DEEPSEEK_CHAT_MODEL=deepseek-v4-flash`). + The API key was supplied out-of-band for the capture and is not stored in this + repository. Without an LLM provider, the stable demo path remains Recall + + Context Pack. +- `ui/19-llm-used.png` was captured with an optional Qwen-compatible analysis + provider (`base_url=https://dashscope.aliyuncs.com/compatible-mode/v1`, + `model=qwen-plus`, `provider=qwen`). + The API key was also supplied out-of-band. Without an external provider, + `ui/19-llm-not-used.png` remains the default stable extraction path. +- Neo4j is disabled in the local capture. This is expected: Graph Explorer falls back + to PostgreSQL graph preview and displays the disabled state explicitly. +- The `logs/` directory contains the full command outputs. SQL/test PNG files are + complete direct renderings of those logs; long lines are wrapped instead of + clipped. diff --git a/docs/final-assets/screenshots/logs/db-check.txt b/docs/final-assets/screenshots/logs/db-check.txt new file mode 100644 index 0000000..b477650 --- /dev/null +++ b/docs/final-assets/screenshots/logs/db-check.txt @@ -0,0 +1,102 @@ +$ npm run db:check +cwd: /Users/jflin/Workspace/Code/CS3321-MemoryBase +exit_code: 0 + +> memorybase-workspace@0.1.0 db:check +> python scripts/db_cli.py check + + List of relations + Schema | Name | Type | Owner +--------+------------------------+-------+------------ + public | access_policy | table | memorybase + public | agent | table | memorybase + public | agent_session | table | memorybase + public | audit_log | table | memorybase + public | conflict_record | table | memorybase + public | entity | table | memorybase + public | forget_request | table | memorybase + public | memory_embedding | table | memorybase + public | memory_entity | table | memorybase + public | memory_evidence | table | memorybase + public | memory_item | table | memorybase + public | memory_revision | table | memorybase + public | memory_scene | table | memorybase + public | memory_scene_cell | table | memorybase + public | message | table | memorybase + public | recall_log | table | memorybase + public | source_chunk | table | memorybase + public | source_chunk_embedding | table | memorybase + public | source_document | table | memorybase + public | timeline_entry | table | memorybase + public | user_account | table | memorybase + public | wiki_page | table | memorybase + public | wiki_page_revision | table | memorybase + public | workspace | table | memorybase + public | workspace_member | table | memorybase +(25 rows) + + workspace_id | name | scope_type +--------------------------------------+-----------------------------------+------------ + 00000000-0000-0000-0000-000000000201 | MemoryBase Course Demo | project + 00000000-0000-0000-0000-000000002201 | Graph Demo: Project Knowledge Map | project +(2 rows) + + memory_id | summary | access_level | status +--------------------------------------+--------------------------------------------------------+--------------+------------ + 00000000-0000-0000-0000-000000000701 | Reason cafeteria topic was rejected. | project | active + 00000000-0000-0000-0000-000000000702 | Reason for choosing MemoryBase. | project | active + 00000000-0000-0000-0000-000000000703 | Required demo question. | project | active + 00000000-0000-0000-0000-000000000704 | File and database dual state. | project | active + 00000000-0000-0000-0000-000000000705 | Core data flow. | project | active + 00000000-0000-0000-0000-000000000706 | Backend API scope. | project | active + 00000000-0000-0000-0000-000000000707 | Database course requirements. | project | active + 00000000-0000-0000-0000-000000000708 | Source and chunk responsibilities. | project | active + 00000000-0000-0000-0000-000000000709 | Evidence relationship. | project | active + 00000000-0000-0000-0000-000000000710 | Audit and revision responsibility. | project | active + 00000000-0000-0000-0000-000000000711 | Recall output shape. | project | active + 00000000-0000-0000-0000-000000000712 | Recall implementation baseline. | project | conflicted + 00000000-0000-0000-0000-000000000713 | Private permission rule. | private | active + 00000000-0000-0000-0000-000000000714 | RecallLog content. | project | active + 00000000-0000-0000-0000-000000000715 | Wiki export purpose. | project | active + 00000000-0000-0000-0000-000000000716 | Wiki provenance chain. | project | active + 00000000-0000-0000-0000-000000000717 | Timeline value. | team | active + 00000000-0000-0000-0000-000000000718 | Demo checklist. | project | active + 00000000-0000-0000-0000-000000000719 | Demo recall answer. | project | active + 00000000-0000-0000-0000-000000000720 | LLM extraction is future work. | project | conflicted + 00000000-0000-0000-0000-000000000731 | Early (wrong) decision: prefer Hash index. | project | forgotten + 00000000-0000-0000-0000-000000000733 | Final recall design: GIN FTS primary + ILIKE fallback. | project | active + 00000000-0000-0000-0000-000000000732 | Early decision: ILIKE only. | project | archived + 00000000-0000-0000-0000-000000000741 | Standup cadence: weekly Monday. | project | active + 00000000-0000-0000-0000-000000000742 | Standup cadence: biweekly Mon/Thu (rejected). | project | superseded + 00000000-0000-0000-0000-000000002701 | Rejected CRUD-heavy topic | project | active + 00000000-0000-0000-0000-000000002702 | Selected MemoryBase | project | active + 00000000-0000-0000-0000-000000002703 | PostgreSQL as source of truth | project | active + 00000000-0000-0000-0000-000000002704 | Neo4j as graph index | project | active + 00000000-0000-0000-0000-000000002705 | Provenance chain | project | active + 00000000-0000-0000-0000-000000002706 | Agent visibility policy | project | active + 00000000-0000-0000-0000-000000002707 | Wiki provenance | project | active +(32 rows) + + conflict_id | status | conflict_type +--------------------------------------+----------+--------------- + 00000000-0000-0000-0000-000000001001 | open | uncertain + 00000000-0000-0000-0000-000000001002 | resolved | contradiction +(2 rows) + + title | event_type | event_time +----------------------------+------------+------------------------ + Abandoned cafeteria system | decision | 2026-03-02 18:30:00+08 + Selected MemoryBase | decision | 2026-03-02 18:40:00+08 + Locked schema requirements | proposal | 2026-03-10 18:30:00+08 + Defined recall shape | proposal | 2026-03-15 18:30:00+08 + Added wiki provenance | decision | 2026-03-20 18:30:00+08 +(5 rows) + +Using .env.example because .env was not found. +Using DATABASE_URL=postgresql://memorybase:memorybase@localhost:5432/memorybase_db +==> Checking core tables +==> Checking workspace +==> Checking memory +==> Checking conflict +==> Checking timeline +Database check completed. diff --git a/docs/final-assets/screenshots/logs/demo-queries.txt b/docs/final-assets/screenshots/logs/demo-queries.txt new file mode 100644 index 0000000..8ebb01f --- /dev/null +++ b/docs/final-assets/screenshots/logs/demo-queries.txt @@ -0,0 +1,188 @@ +$ npm run db:run -- database/08_demo_queries.sql +cwd: /Users/jflin/Workspace/Code/CS3321-MemoryBase +exit_code: 0 + +> memorybase-workspace@0.1.0 db:run +> python scripts/db_cli.py run database/08_demo_queries.sql + + memory_id | memory_type | canonical_text | access_level | confidence | importance +--------------------------------------+-------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+------------+------------ + 00000000-0000-0000-0000-000000002702 | decision | MemoryBase was selected because it demonstrates provenance, governance, and graph exploration. | project | 0.970 | 5 + 00000000-0000-0000-0000-000000002701 | decision | The cafeteria ordering topic was rejected because it mostly demonstrated CRUD. | project | 0.960 | 5 + 00000000-0000-0000-0000-000000002707 | semantic | Wiki provenance lets readers trace generated pages back to memories and evidence. | project | 0.930 | 5 + 00000000-0000-0000-0000-000000002705 | procedural | Memory provenance flows from wiki page to scene, memory, evidence chunk, and source document. | project | 0.930 | 5 + 00000000-0000-0000-0000-000000002704 | semantic | Neo4j is used as a relationship index for graph exploration, not as the source of truth. | project | 0.940 | 5 + 00000000-0000-0000-0000-000000002703 | semantic | PostgreSQL remains the authoritative source of business data. | project | 0.950 | 5 + 00000000-0000-0000-0000-000000000733 | decision | Recall combines GIN tsvector full-text search as the primary path with ILIKE pattern matching as fallback for short queries. | project | 0.900 | 5 + 00000000-0000-0000-0000-000000000716 | semantic | Wiki statements should trace back to MemoryItem, MemoryEvidence, SourceChunk, and SourceDocument. | project | 0.930 | 5 + 00000000-0000-0000-0000-000000000702 | decision | MemoryBase was selected because it demonstrates source chunks, evidence, revisions, audit logs, policies, recall logs, conflicts, and wiki projection. | project | 0.980 | 5 + 00000000-0000-0000-0000-000000000701 | decision | The team abandoned the campus cafeteria system because it was too CRUD-heavy and did not demonstrate enough database depth. | project | 0.950 | 5 + 00000000-0000-0000-0000-000000000718 | task | The five to eight minute demo should show source import, chunk line numbers, memory evidence, recall, revision, audit, policy filtering, conflict handling, wiki export, and SQL queries. | project | 0.920 | 5 + 00000000-0000-0000-0000-000000000719 | decision | The demo answer to the cafeteria question is that the cafeteria system was too CRUD-heavy and did not demonstrate enough database features. | project | 0.970 | 5 + 00000000-0000-0000-0000-000000000709 | semantic | MemoryEvidence is the many-to-many bridge between MemoryItem and SourceChunk. | project | 0.940 | 5 + 00000000-0000-0000-0000-000000000707 | semantic | The schema must demonstrate primary keys, foreign keys, unique constraints, check constraints, indexes, views, and triggers. | project | 0.930 | 5 + 00000000-0000-0000-0000-000000000705 | procedural | The core data flow is SourceDocument to SourceChunk to MemoryItem to MemoryEvidence. | project | 0.920 | 5 + 00000000-0000-0000-0000-000000002706 | semantic | Access policies decide which memories an agent can see. | project | 0.910 | 4 + 00000000-0000-0000-0000-000000000703 | semantic | The recall demo must answer why the campus cafeteria system was abandoned. | project | 0.900 | 4 + 00000000-0000-0000-0000-000000000713 | risk | Private budget or personal coordination notes must stay hidden from a project-only retriever agent unless explicitly allowed. | private | 0.900 | 4 + 00000000-0000-0000-0000-000000000715 | semantic | Wiki export makes database memory readable as Markdown and WikiPageRevision audits repeated exports. | project | 0.900 | 4 + 00000000-0000-0000-0000-000000000710 | semantic | AuditLog stores before and after JSON, while MemoryRevision stores immutable memory versions. | project | 0.910 | 4 +(20 rows) + + memory_id | canonical_text | evidence_role | source_title | chunk_no | start_line | end_line +--------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------+----------------------------------+----------+------------+---------- + 00000000-0000-0000-0000-000000000701 | The team abandoned the campus cafeteria system because it was too CRUD-heavy and did not demonstrate enough database depth. | supports | Discussion 01: Project Pivot | 2 | 10 | 10 + 00000000-0000-0000-0000-000000000702 | MemoryBase was selected because it demonstrates source chunks, evidence, revisions, audit logs, policies, recall logs, conflicts, and wiki projection. | supports | Discussion 01: Project Pivot | 3 | 12 | 12 + 00000000-0000-0000-0000-000000000703 | The recall demo must answer why the campus cafeteria system was abandoned. | supports | Discussion 01: Project Pivot | 4 | 14 | 14 + 00000000-0000-0000-0000-000000000704 | MemoryBase keeps both human-readable Markdown and database records as a file-database dual state system. | supports | Discussion 02: Architecture | 1 | 6 | 6 + 00000000-0000-0000-0000-000000000705 | The core data flow is SourceDocument to SourceChunk to MemoryItem to MemoryEvidence. | supports | Discussion 02: Architecture | 2 | 8 | 8 + 00000000-0000-0000-0000-000000000706 | The backend should expose import, source, memory, recall, policy, audit, and wiki export APIs. | supports | Discussion 02: Architecture | 3 | 10 | 10 + 00000000-0000-0000-0000-000000000707 | The schema must demonstrate primary keys, foreign keys, unique constraints, check constraints, indexes, views, and triggers. | supports | Discussion 03: Schema | 1 | 6 | 6 + 00000000-0000-0000-0000-000000000708 | SourceDocument stores imported files and checksums; SourceChunk stores chunk text, line ranges, token counts, and search vectors. | supports | Discussion 03: Schema | 2 | 8 | 8 + 00000000-0000-0000-0000-000000000709 | MemoryEvidence is the many-to-many bridge between MemoryItem and SourceChunk. | supports | Discussion 03: Schema | 3 | 10 | 10 + 00000000-0000-0000-0000-000000000710 | AuditLog stores before and after JSON, while MemoryRevision stores immutable memory versions. | supports | Discussion 03: Schema | 4 | 12 | 12 + 00000000-0000-0000-0000-000000000711 | Recall should return memory items together with supporting evidence and source chunks. | supports | Discussion 04: Policy and Recall | 1 | 6 | 6 + 00000000-0000-0000-0000-000000000712 | The first recall implementation can use keyword search and PostgreSQL full text search. | context | Discussion 04: Policy and Recall | 1 | 6 | 6 + 00000000-0000-0000-0000-000000000713 | Private budget or personal coordination notes must stay hidden from a project-only retriever agent unless explicitly allowed. | supports | Discussion 04: Policy and Recall | 2 | 8 | 8 + 00000000-0000-0000-0000-000000000714 | RecallLog records query text, filters, result count, top memory IDs, and context pack JSON. | supports | Discussion 04: Policy and Recall | 3 | 12 | 12 + 00000000-0000-0000-0000-000000000715 | Wiki export makes database memory readable as Markdown and WikiPageRevision audits repeated exports. | supports | Discussion 05: Wiki and Timeline | 1 | 6 | 8 + 00000000-0000-0000-0000-000000000716 | Wiki statements should trace back to MemoryItem, MemoryEvidence, SourceChunk, and SourceDocument. | supports | Discussion 05: Wiki and Timeline | 2 | 10 | 10 + 00000000-0000-0000-0000-000000000717 | TimelineEntry shows how the project evolved from topic selection to schema design, recall, governance, and final demo. | supports | Discussion 05: Wiki and Timeline | 3 | 12 | 12 + 00000000-0000-0000-0000-000000000718 | The five to eight minute demo should show source import, chunk line numbers, memory evidence, recall, revision, audit, policy filtering, conflict handling, wiki export, and SQL queries. | supports | Discussion 06: Demo Plan | 1 | 6 | 6 + 00000000-0000-0000-0000-000000000719 | The demo answer to the cafeteria question is that the cafeteria system was too CRUD-heavy and did not demonstrate enough database features. | supports | Discussion 06: Demo Plan | 2 | 8 | 8 + 00000000-0000-0000-0000-000000000720 | LLM automatic extraction is useful later but is not required for the deterministic MVP demo. | supports | Discussion 06: Demo Plan | 3 | 14 | 14 +(20 rows) + + timeline_id | event_time | event_type | title | memory_text | source_title +--------------------------------------+------------------------+------------+----------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------- + 00000000-0000-0000-0000-000000001205 | 2026-03-20 18:30:00+08 | decision | Added wiki provenance | Wiki statements should trace back to MemoryItem, MemoryEvidence, SourceChunk, and SourceDocument. | Discussion 05: Wiki and Timeline + 00000000-0000-0000-0000-000000001204 | 2026-03-15 18:30:00+08 | proposal | Defined recall shape | Recall should return memory items together with supporting evidence and source chunks. | Discussion 04: Policy and Recall + 00000000-0000-0000-0000-000000001203 | 2026-03-10 18:30:00+08 | proposal | Locked schema requirements | The schema must demonstrate primary keys, foreign keys, unique constraints, check constraints, indexes, views, and triggers. | Discussion 03: Schema + 00000000-0000-0000-0000-000000001202 | 2026-03-02 18:40:00+08 | decision | Selected MemoryBase | MemoryBase was selected because it demonstrates source chunks, evidence, revisions, audit logs, policies, recall logs, conflicts, and wiki projection. | Discussion 01: Project Pivot + 00000000-0000-0000-0000-000000001201 | 2026-03-02 18:30:00+08 | decision | Abandoned cafeteria system | The team abandoned the campus cafeteria system because it was too CRUD-heavy and did not demonstrate enough database depth. | Discussion 01: Project Pivot +(5 rows) + + workspace_id | memory_type | status | access_level | memory_count | avg_confidence | avg_importance +--------------------------------------+-------------+------------+--------------+--------------+------------------------+-------------------- + 00000000-0000-0000-0000-000000000201 | semantic | conflicted | project | 1 | 0.84000000000000000000 | 3.0000000000000000 + 00000000-0000-0000-0000-000000000201 | task | active | project | 2 | 0.88000000000000000000 | 4.0000000000000000 + 00000000-0000-0000-0000-000000000201 | decision | forgotten | project | 1 | 0.50000000000000000000 | 2.0000000000000000 + 00000000-0000-0000-0000-000000000201 | decision | active | project | 5 | 0.93000000000000000000 | 4.6000000000000000 + 00000000-0000-0000-0000-000000000201 | decision | archived | project | 1 | 0.50000000000000000000 | 2.0000000000000000 + 00000000-0000-0000-0000-000000000201 | risk | active | private | 1 | 0.90000000000000000000 | 4.0000000000000000 + 00000000-0000-0000-0000-000000000201 | procedural | active | project | 2 | 0.90500000000000000000 | 4.5000000000000000 + 00000000-0000-0000-0000-000000000201 | semantic | active | team | 1 | 0.85000000000000000000 | 3.0000000000000000 + 00000000-0000-0000-0000-000000000201 | decision | conflicted | project | 1 | 0.88000000000000000000 | 4.0000000000000000 + 00000000-0000-0000-0000-000000002201 | procedural | active | project | 1 | 0.93000000000000000000 | 5.0000000000000000 + 00000000-0000-0000-0000-000000002201 | decision | active | project | 2 | 0.96500000000000000000 | 5.0000000000000000 + 00000000-0000-0000-0000-000000000201 | semantic | active | project | 9 | 0.90666666666666666667 | 4.2222222222222222 + 00000000-0000-0000-0000-000000000201 | decision | superseded | project | 1 | 0.60000000000000000000 | 2.0000000000000000 + 00000000-0000-0000-0000-000000002201 | semantic | active | project | 4 | 0.93250000000000000000 | 4.7500000000000000 +(14 rows) + + memory_id | memory_type | canonical_text | access_level | confidence +--------------------------------------+-------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------+------------ + 00000000-0000-0000-0000-000000000733 | decision | Recall combines GIN tsvector full-text search as the primary path with ILIKE pattern matching as fallback for short queries. | project | 0.900 + 00000000-0000-0000-0000-000000000716 | semantic | Wiki statements should trace back to MemoryItem, MemoryEvidence, SourceChunk, and SourceDocument. | project | 0.930 + 00000000-0000-0000-0000-000000000702 | decision | MemoryBase was selected because it demonstrates source chunks, evidence, revisions, audit logs, policies, recall logs, conflicts, and wiki projection. | project | 0.980 + 00000000-0000-0000-0000-000000000701 | decision | The team abandoned the campus cafeteria system because it was too CRUD-heavy and did not demonstrate enough database depth. | project | 0.950 + 00000000-0000-0000-0000-000000000719 | decision | The demo answer to the cafeteria question is that the cafeteria system was too CRUD-heavy and did not demonstrate enough database features. | project | 0.970 + 00000000-0000-0000-0000-000000000718 | task | The five to eight minute demo should show source import, chunk line numbers, memory evidence, recall, revision, audit, policy filtering, conflict handling, wiki export, and SQL queries. | project | 0.920 + 00000000-0000-0000-0000-000000000707 | semantic | The schema must demonstrate primary keys, foreign keys, unique constraints, check constraints, indexes, views, and triggers. | project | 0.930 + 00000000-0000-0000-0000-000000000705 | procedural | The core data flow is SourceDocument to SourceChunk to MemoryItem to MemoryEvidence. | project | 0.920 + 00000000-0000-0000-0000-000000000709 | semantic | MemoryEvidence is the many-to-many bridge between MemoryItem and SourceChunk. | project | 0.940 + 00000000-0000-0000-0000-000000000710 | semantic | AuditLog stores before and after JSON, while MemoryRevision stores immutable memory versions. | project | 0.910 + 00000000-0000-0000-0000-000000000703 | semantic | The recall demo must answer why the campus cafeteria system was abandoned. | project | 0.900 + 00000000-0000-0000-0000-000000000704 | semantic | MemoryBase keeps both human-readable Markdown and database records as a file-database dual state system. | project | 0.880 + 00000000-0000-0000-0000-000000000708 | semantic | SourceDocument stores imported files and checksums; SourceChunk stores chunk text, line ranges, token counts, and search vectors. | project | 0.900 + 00000000-0000-0000-0000-000000000711 | procedural | Recall should return memory items together with supporting evidence and source chunks. | project | 0.890 + 00000000-0000-0000-0000-000000000715 | semantic | Wiki export makes database memory readable as Markdown and WikiPageRevision audits repeated exports. | project | 0.900 + 00000000-0000-0000-0000-000000000741 | decision | Team standup is held every Monday morning at 10am. | project | 0.850 + 00000000-0000-0000-0000-000000000717 | semantic | TimelineEntry shows how the project evolved from topic selection to schema design, recall, governance, and final demo. | team | 0.850 + 00000000-0000-0000-0000-000000000714 | semantic | RecallLog records query text, filters, result count, top memory IDs, and context pack JSON. | project | 0.870 + 00000000-0000-0000-0000-000000000706 | task | The backend should expose import, source, memory, recall, policy, audit, and wiki export APIs. | project | 0.840 +(19 rows) + + conflict_id | conflict_type | conflict_status | left_memory_text | right_memory_text +--------------------------------------+---------------+-----------------+-----------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------- + 00000000-0000-0000-0000-000000001002 | contradiction | resolved | Team standup is held every Monday morning at 10am. | Team standup is held twice a week, on Monday and Thursday mornings. + 00000000-0000-0000-0000-000000001001 | uncertain | open | The first recall implementation can use keyword search and PostgreSQL full text search. | LLM automatic extraction is useful later but is not required for the deterministic MVP demo. +(2 rows) + + page_slug | page_title | memory_id | source_title | chunk_no | start_line | end_line +---------------------+---------------------+--------------------------------------+-----------------------------------+----------+------------+---------- + architecture-map | Architecture Map | 00000000-0000-0000-0000-000000002703 | Graph Demo 02: Architecture Chain | 1 | 4 | 4 + architecture-map | Architecture Map | 00000000-0000-0000-0000-000000002704 | Graph Demo 02: Architecture Chain | 2 | 6 | 6 + architecture-map | Architecture Map | 00000000-0000-0000-0000-000000002705 | Graph Demo 02: Architecture Chain | 3 | 8 | 8 + architecture-map | Architecture Map | 00000000-0000-0000-0000-000000002705 | Graph Demo 02: Architecture Chain | 4 | 10 | 10 + database-design | Database Design | 00000000-0000-0000-0000-000000000707 | Discussion 03: Schema | 1 | 6 | 6 + demo-playbook | Demo Playbook | 00000000-0000-0000-0000-000000000718 | Discussion 06: Demo Plan | 1 | 6 | 6 + governance-demo | Governance Demo | 00000000-0000-0000-0000-000000002706 | Graph Demo 03: Governance Story | 1 | 4 | 4 + governance-demo | Governance Demo | 00000000-0000-0000-0000-000000002707 | Graph Demo 03: Governance Story | 2 | 8 | 8 + project-pivot-story | Project Pivot Story | 00000000-0000-0000-0000-000000002701 | Graph Demo 01: Topic Pivot | 1 | 4 | 4 + project-pivot-story | Project Pivot Story | 00000000-0000-0000-0000-000000002702 | Graph Demo 01: Topic Pivot | 2 | 6 | 6 + why-memorybase | Why MemoryBase | 00000000-0000-0000-0000-000000000701 | Discussion 01: Project Pivot | 2 | 10 | 10 + why-memorybase | Why MemoryBase | 00000000-0000-0000-0000-000000000719 | Discussion 06: Demo Plan | 2 | 8 | 8 + why-memorybase | Why MemoryBase | 00000000-0000-0000-0000-000000000702 | Discussion 01: Project Pivot | 3 | 12 | 12 + why-memorybase | Why MemoryBase | 00000000-0000-0000-0000-000000000703 | Discussion 01: Project Pivot | 4 | 14 | 14 +(14 rows) + + doc_id | chunk_no | chunk_text +--------------------------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 00000000-0000-0000-0000-000000000501 | 3 | The MemoryBase direction gives us stronger database requirements: source documents, chunks, memory evidence, revisions, audit logs, access policy, recall logs, conflicts, and wiki projection. + 00000000-0000-0000-0000-000000000502 | 1 | MemoryBase uses a file-database dual state model. Human readable Markdown remains useful for review, while PostgreSQL provides query, constraints, views, triggers, and auditability. +(2 rows) + + scene_title | cell_role | sort_order | memory_type | canonical_text | note +----------------+------------+------------+-------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------- + Topic Decision | background | 10 | decision | The team abandoned the campus cafeteria system because it was too CRUD-heavy and did not demonstrate enough database depth. | Explains why the original cafeteria idea was rejected. + Topic Decision | decision | 20 | decision | MemoryBase was selected because it demonstrates source chunks, evidence, revisions, audit logs, policies, recall logs, conflicts, and wiki projection. | Records the positive decision to choose MemoryBase. + Topic Decision | context | 30 | semantic | The recall demo must answer why the campus cafeteria system was abandoned. | Connects the topic decision to the required recall demo question. + Topic Decision | outcome | 40 | decision | The demo answer to the cafeteria question is that the cafeteria system was too CRUD-heavy and did not demonstrate enough database features. | Captures the final answer expected in the demo. +(4 rows) + + memory_id | memory_type | search_text_zh +--------------------------------------+-------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 00000000-0000-0000-0000-000000000733 | decision | Recall combines GIN tsvector full-text search as primary path with ILIKE pattern matching as fallback for short queries. + 00000000-0000-0000-0000-000000000716 | semantic | wiki statements should trace back to memoryitem memoryevidence sourcechunk and sourcedocument wiki provenance chain + 00000000-0000-0000-0000-000000000702 | decision | memorybase was selected because it demonstrates source chunks evidence revisions audit logs policies recall logs conflicts and wiki projection reason for choosing memorybase + 00000000-0000-0000-0000-000000000701 | decision | the team abandoned the campus cafeteria system because it was too crud heavy and did not demonstrate enough database depth reason cafeteria topic was rejected + 00000000-0000-0000-0000-000000000707 | semantic | the schema must demonstrate primary keys foreign keys unique constraints check constraints indexes views and triggers database course requirements + 00000000-0000-0000-0000-000000000718 | task | the five to eight minute demo should show source import chunk line numbers memory evidence recall revision audit policy filtering conflict handling wiki export and sql queries demo checklist + 00000000-0000-0000-0000-000000000709 | semantic | memoryevidence is the many to many bridge between memoryitem and sourcechunk evidence relationship + 00000000-0000-0000-0000-000000000719 | decision | the demo answer to the cafeteria question is that the cafeteria system was too crud heavy and did not demonstrate enough database features demo recall answer + 00000000-0000-0000-0000-000000000705 | procedural | the core data flow is sourcedocument to sourcechunk to memoryitem to memoryevidence core data flow + 00000000-0000-0000-0000-000000000720 | decision | llm automatic extraction is useful later but is not required for the deterministic mvp demo llm extraction is future work +(10 rows) + + recall_id | query_text | result_count | top_memory_ids_json | filters | created_at +--------------------------------------+------------------------------------------------+--------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------+------------------------------- + d8540368-de1d-477f-b4f6-ea98c0b2fc39 | 为什么放弃校园食堂系统？ | 4 | ["00000000-0000-0000-0000-000000000719", "00000000-0000-0000-0000-000000000701", "00000000-0000-0000-0000-000000000703", "00000000-0000-0000-0000-000000000704"] | {"as_of": null, "status": "active", "agent_id": null, "memory_type": null, "access_level": null, "retrieval_mode": "keyword"} | 2026-06-09 10:11:14.601449+08 + 2c6c3599-6e6d-4426-b4b0-02100048c0ac | 为什么放弃校园食堂系统？ | 4 | ["00000000-0000-0000-0000-000000000719", "00000000-0000-0000-0000-000000000701", "00000000-0000-0000-0000-000000000703", "00000000-0000-0000-0000-000000000704"] | {"as_of": null, "status": "active", "agent_id": null, "memory_type": null, "access_level": null, "retrieval_mode": "keyword"} | 2026-06-09 10:10:59.788748+08 + 00000000-0000-0000-0000-000000001101 | why did we abandon the campus cafeteria system | 3 | ["00000000-0000-0000-0000-000000000701", "00000000-0000-0000-0000-000000000719", "00000000-0000-0000-0000-000000000702"] | | 2026-03-25 19:10:00+08 +(3 rows) + + audit_id | action_type | target_type | target_id | before_json | after_json | created_at +--------------------------------------+--------------------+-------------+--------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------- + 84d312b4-5a4b-4009-9bbd-dddfa8425443 | memory.insert | memory_item | 00000000-0000-0000-0000-000000000731 | | {"status": "active", "summary": "Early (wrong) decision: prefer Hash index.", "valid_to": null, "memory_id": "00000000-0000-0000-0000-000000000731", "confidence": 0.500, "created_at": "2026-06-09T10:06:28.235074+08:00", "importance": 2, "updated_at": "2026-06-09T10:06:28.235074+08:00", "valid_from": "2026-05-20T10:06:28.235074+08:00", "memory_type": "decision", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000101", "search_vector": "'for':5 'hash':3 'index':4 'item':7 'key':9 'lookup':10 'memory':6 'postgresql':2 'primary':8 'use':1", "canonical_text": "Use PostgreSQL Hash index for memory_item primary key lookup.", "owner_agent_id": null, "search_text_zh": "Use PostgreSQL Hash index for memory_item primary key lookup.", "created_from_doc_id": null, "current_revision_no": 1, "superseded_by_memory_id": null} | 2026-06-09 10:06:28.235074+08 + 590bd9c6-cf8b-413a-af23-46735d9fa5ed | memory.forget | memory_item | 00000000-0000-0000-0000-000000000731 | {"status": "active", "summary": "Early (wrong) decision: prefer Hash index.", "valid_to": null, "memory_id": "00000000-0000-0000-0000-000000000731", "confidence": 0.500, "created_at": "2026-06-09T10:06:28.235074+08:00", "importance": 2, "updated_at": "2026-06-09T10:06:28.235074+08:00", "valid_from": "2026-05-20T10:06:28.235074+08:00", "memory_type": "decision", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000101", "search_vector": "'for':5 'hash':3 'index':4 'item':7 'key':9 'lookup':10 'memory':6 'postgresql':2 'primary':8 'use':1", "canonical_text": "Use PostgreSQL Hash index for memory_item primary key lookup.", "owner_agent_id": null, "search_text_zh": "Use PostgreSQL Hash index for memory_item primary key lookup.", "created_from_doc_id": null, "current_revision_no": 1, "superseded_by_memory_id": null} | {"status": "forgotten", "summary": "Early (wrong) decision: prefer Hash index.", "valid_to": null, "memory_id": "00000000-0000-0000-0000-000000000731", "confidence": 0.500, "created_at": "2026-06-09T10:06:28.235074+08:00", "importance": 2, "updated_at": "2026-06-09T10:06:28.24913+08:00", "valid_from": "2026-05-20T10:06:28.235074+08:00", "memory_type": "decision", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000101", "search_vector": "'for':5 'hash':3 'index':4 'item':7 'key':9 'lookup':10 'memory':6 'postgresql':2 'primary':8 'use':1", "canonical_text": "Use PostgreSQL Hash index for memory_item primary key lookup.", "owner_agent_id": null, "search_text_zh": "Use PostgreSQL Hash index for memory_item primary key lookup.", "created_from_doc_id": null, "current_revision_no": 2, "superseded_by_memory_id": null} | 2026-06-09 10:06:28.235074+08 + 986d16ce-9087-4c56-8bd4-ea0f3d79bb65 | memory.insert | memory_item | 00000000-0000-0000-0000-000000000733 | | {"status": "active", "summary": "Final recall design: GIN FTS primary + ILIKE fallback.", "valid_to": null, "memory_id": "00000000-0000-0000-0000-000000000733", "confidence": 0.900, "created_at": "2026-06-09T10:06:28.235074+08:00", "importance": 5, "updated_at": "2026-06-09T10:06:28.235074+08:00", "valid_from": "2026-05-30T10:06:28.235074+08:00", "memory_type": "decision", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000101", "search_vector": "'as':9,16 'combines':2 'fallback':17 'for':18 'full':6 'full-text':5 'gin':3 'ilike':13 'matching':15 'path':11 'pattern':14 'primary':10 'queries':20 'recall':1 'search':8 'short':19 'text':7 'tsvector':4 'with':12", "canonical_text": "Recall combines GIN tsvector full-text search as the primary path with ILIKE pattern matching as fallback for short queries.", "owner_agent_id": null, "search_text_zh": "Recall combines GIN tsvector full-text search as primary path with ILIKE pattern matching as fallback for short queries.", "created_from_doc_id": "00000000-0000-0000-0000-000000000504", "current_revision_no": 1, "superseded_by_memory_id": null} | 2026-06-09 10:06:28.235074+08 + a52e76f1-ced5-4c1a-b9e7-3a2d7896fe7b | memory.insert | memory_item | 00000000-0000-0000-0000-000000000732 | | {"status": "active", "summary": "Early decision: ILIKE only.", "valid_to": "2026-05-30T10:06:28.235074+08:00", "memory_id": "00000000-0000-0000-0000-000000000732", "confidence": 0.500, "created_at": "2026-06-09T10:06:28.235074+08:00", "importance": 2, "updated_at": "2026-06-09T10:06:28.235074+08:00", "valid_from": "2026-05-22T10:06:28.235074+08:00", "memory_type": "decision", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000101", "search_vector": "'ilike':5 'implementation':2 'matching':7 'only':8 'pattern':6 'recall':1 'use':4 'will':3", "canonical_text": "Recall implementation will use ILIKE pattern matching only.", "owner_agent_id": null, "search_text_zh": "Recall implementation will use ILIKE pattern matching only.", "created_from_doc_id": "00000000-0000-0000-0000-000000000504", "current_revision_no": 1, "superseded_by_memory_id": "00000000-0000-0000-0000-000000000733"} | 2026-06-09 10:06:28.235074+08 + 1a18b147-883d-4157-8977-7e4704e1b0ef | memory.soft_delete | memory_item | 00000000-0000-0000-0000-000000000732 | {"status": "active", "summary": "Early decision: ILIKE only.", "valid_to": "2026-05-30T10:06:28.235074+08:00", "memory_id": "00000000-0000-0000-0000-000000000732", "confidence": 0.500, "created_at": "2026-06-09T10:06:28.235074+08:00", "importance": 2, "updated_at": "2026-06-09T10:06:28.235074+08:00", "valid_from": "2026-05-22T10:06:28.235074+08:00", "memory_type": "decision", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000101", "search_vector": "'ilike':5 'implementation':2 'matching':7 'only':8 'pattern':6 'recall':1 'use':4 'will':3", "canonical_text": "Recall implementation will use ILIKE pattern matching only.", "owner_agent_id": null, "search_text_zh": "Recall implementation will use ILIKE pattern matching only.", "created_from_doc_id": "00000000-0000-0000-0000-000000000504", "current_revision_no": 1, "superseded_by_memory_id": "00000000-0000-0000-0000-000000000733"} | {"status": "archived", "summary": "Early decision: ILIKE only.", "valid_to": "2026-05-30T10:06:28.235074+08:00", "memory_id": "00000000-0000-0000-0000-000000000732", "confidence": 0.500, "created_at": "2026-06-09T10:06:28.235074+08:00", "importance": 2, "updated_at": "2026-06-09T10:06:28.253349+08:00", "valid_from": "2026-05-22T10:06:28.235074+08:00", "memory_type": "decision", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000101", "search_vector": "'ilike':5 'implementation':2 'matching':7 'only':8 'pattern':6 'recall':1 'use':4 'will':3", "canonical_text": "Recall implementation will use ILIKE pattern matching only.", "owner_agent_id": null, "search_text_zh": "Recall implementation will use ILIKE pattern matching only.", "created_from_doc_id": "00000000-0000-0000-0000-000000000504", "current_revision_no": 2, "superseded_by_memory_id": "00000000-0000-0000-0000-000000000733"} | 2026-06-09 10:06:28.235074+08 + 8e3d6ecc-a6a0-4f13-ba70-0848305ef0df | memory.insert | memory_item | 00000000-0000-0000-0000-000000000741 | | {"status": "active", "summary": "Standup cadence: weekly Monday.", "valid_to": null, "memory_id": "00000000-0000-0000-0000-000000000741", "confidence": 0.850, "created_at": "2026-06-09T10:06:28.235074+08:00", "importance": 3, "updated_at": "2026-06-09T10:06:28.235074+08:00", "valid_from": "2026-04-30T10:06:28.235074+08:00", "memory_type": "decision", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000101", "search_vector": "'10am':9 'at':8 'every':5 'held':4 'is':3 'monday':6 'morning':7 'standup':2 'team':1", "canonical_text": "Team standup is held every Monday morning at 10am.", "owner_agent_id": null, "search_text_zh": "Team standup is held every Monday morning at 10am.", "created_from_doc_id": "00000000-0000-0000-0000-000000000501", "current_revision_no": 1, "superseded_by_memory_id": null} | 2026-06-09 10:06:28.235074+08 + f65f09b2-8ed8-4803-b3d7-c51229d6d5c6 | memory.insert | memory_item | 00000000-0000-0000-0000-000000000742 | | {"status": "active", "summary": "Standup cadence: biweekly Mon/Thu (rejected).", "valid_to": null, "memory_id": "00000000-0000-0000-0000-000000000742", "confidence": 0.600, "created_at": "2026-06-09T10:06:28.235074+08:00", "importance": 2, "updated_at": "2026-06-09T10:06:28.235074+08:00", "valid_from": "2026-05-05T10:06:28.235074+08:00", "memory_type": "decision", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000101", "search_vector": "'a':6 'and':10 'held':4 'is':3 'monday':9 'mornings':12 'on':8 'standup':2 'team':1 'thursday':11 'twice':5 'week':7", "canonical_text": "Team standup is held twice a week, on Monday and Thursday mornings.", "owner_agent_id": null, "search_text_zh": "Team standup is held twice a week on Monday and Thursday mornings.", "created_from_doc_id": "00000000-0000-0000-0000-000000000501", "current_revision_no": 1, "superseded_by_memory_id": null} | 2026-06-09 10:06:28.235074+08 + f00e2eb1-9583-467b-a945-8304b8f9576e | memory.update | memory_item | 00000000-0000-0000-0000-000000000742 | {"status": "active", "summary": "Standup cadence: biweekly Mon/Thu (rejected).", "valid_to": null, "memory_id": "00000000-0000-0000-0000-000000000742", "confidence": 0.600, "created_at": "2026-06-09T10:06:28.235074+08:00", "importance": 2, "updated_at": "2026-06-09T10:06:28.235074+08:00", "valid_from": "2026-05-05T10:06:28.235074+08:00", "memory_type": "decision", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000101", "search_vector": "'a':6 'and':10 'held':4 'is':3 'monday':9 'mornings':12 'on':8 'standup':2 'team':1 'thursday':11 'twice':5 'week':7", "canonical_text": "Team standup is held twice a week, on Monday and Thursday mornings.", "owner_agent_id": null, "search_text_zh": "Team standup is held twice a week on Monday and Thursday mornings.", "created_from_doc_id": "00000000-0000-0000-0000-000000000501", "current_revision_no": 1, "superseded_by_memory_id": null} | {"status": "superseded", "summary": "Standup cadence: biweekly Mon/Thu (rejected).", "valid_to": "2026-05-08T10:06:28.235074+08:00", "memory_id": "00000000-0000-0000-0000-000000000742", "confidence": 0.600, "created_at": "2026-06-09T10:06:28.235074+08:00", "importance": 2, "updated_at": "2026-06-09T10:06:28.255857+08:00", "valid_from": "2026-05-05T10:06:28.235074+08:00", "memory_type": "decision", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000101", "search_vector": "'a':6 'and':10 'held':4 'is':3 'monday':9 'mornings':12 'on':8 'standup':2 'team':1 'thursday':11 'twice':5 'week':7", "canonical_text": "Team standup is held twice a week, on Monday and Thursday mornings.", "owner_agent_id": null, "search_text_zh": "Team standup is held twice a week on Monday and Thursday mornings.", "created_from_doc_id": "00000000-0000-0000-0000-000000000501", "current_revision_no": 2, "superseded_by_memory_id": "00000000-0000-0000-0000-000000000741"} | 2026-06-09 10:06:28.235074+08 + 6ffe7694-70c6-4d04-8a81-9cab832c06b5 | memory.update | memory_item | 00000000-0000-0000-0000-000000000712 | {"status": "active", "summary": "Recall implementation baseline.", "valid_to": null, "memory_id": "00000000-0000-0000-0000-000000000712", "confidence": 0.840, "created_at": "2026-06-09T10:06:27.602701+08:00", "importance": 3, "updated_at": "2026-06-09T10:06:27.602701+08:00", "valid_from": "2026-03-15T18:15:00+08:00", "memory_type": "semantic", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000102", "search_vector": "", "canonical_text": "The first recall implementation can use keyword search and PostgreSQL full text search.", "owner_agent_id": "00000000-0000-0000-0000-000000000301", "search_text_zh": null, "created_from_doc_id": "00000000-0000-0000-0000-000000000504", "current_revision_no": 1, "superseded_by_memory_id": null} | {"status": "conflicted", "summary": "Recall implementation baseline.", "valid_to": null, "memory_id": "00000000-0000-0000-0000-000000000712", "confidence": 0.840, "created_at": "2026-06-09T10:06:27.602701+08:00", "importance": 3, "updated_at": "2026-06-09T10:06:27.617544+08:00", "valid_from": "2026-03-15T18:15:00+08:00", "memory_type": "semantic", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000102", "search_vector": "", "canonical_text": "The first recall implementation can use keyword search and PostgreSQL full text search.", "owner_agent_id": "00000000-0000-0000-0000-000000000301", "search_text_zh": null, "created_from_doc_id": "00000000-0000-0000-0000-000000000504", "current_revision_no": 2, "superseded_by_memory_id": null} | 2026-06-09 10:06:27.617014+08 + 31736dcf-67e8-4926-92d7-484fd31a0135 | memory.update | memory_item | 00000000-0000-0000-0000-000000000720 | {"status": "active", "summary": "LLM extraction is future work.", "valid_to": null, "memory_id": "00000000-0000-0000-0000-000000000720", "confidence": 0.880, "created_at": "2026-06-09T10:06:27.602701+08:00", "importance": 4, "updated_at": "2026-06-09T10:06:27.611646+08:00", "valid_from": "2026-03-25T18:20:00+08:00", "memory_type": "decision", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000103", "search_vector": "", "canonical_text": "LLM automatic extraction is useful later but is not required for the deterministic MVP demo.", "owner_agent_id": "00000000-0000-0000-0000-000000000301", "search_text_zh": null, "created_from_doc_id": "00000000-0000-0000-0000-000000000506", "current_revision_no": 2, "superseded_by_memory_id": null} | {"status": "conflicted", "summary": "LLM extraction is future work.", "valid_to": null, "memory_id": "00000000-0000-0000-0000-000000000720", "confidence": 0.880, "created_at": "2026-06-09T10:06:27.602701+08:00", "importance": 4, "updated_at": "2026-06-09T10:06:27.61776+08:00", "valid_from": "2026-03-25T18:20:00+08:00", "memory_type": "decision", "access_level": "project", "workspace_id": "00000000-0000-0000-0000-000000000201", "owner_user_id": "00000000-0000-0000-0000-000000000103", "search_vector": "", "canonical_text": "LLM automatic extraction is useful later but is not required for the deterministic MVP demo.", "owner_agent_id": "00000000-0000-0000-0000-000000000301", "search_text_zh": null, "created_from_doc_id": "00000000-0000-0000-0000-000000000506", "current_revision_no": 3, "superseded_by_memory_id": null} | 2026-06-09 10:06:27.617014+08 +(10 rows) + + request_id | target_type | target_id | request_status | memory_status | requested_at | resolved_at +--------------------------------------+-----------------+--------------------------------------+----------------+---------------+-------------------------------+------------------------------- + 00000000-0000-0000-0000-000000001101 | memory_item | 00000000-0000-0000-0000-000000000731 | done | forgotten | 2026-06-04 10:06:28.235074+08 | 2026-06-05 10:06:28.235074+08 + 00000000-0000-0000-0000-000000001102 | source_document | 00000000-0000-0000-0000-000000000507 | done | | 2026-04-24 10:06:28.235074+08 | 2026-04-25 10:06:28.235074+08 +(2 rows) + +Using .env.example because .env was not found. +Using DATABASE_URL=postgresql://memorybase:memorybase@localhost:5432/memorybase_db +==> Running /Users/jflin/Workspace/Code/CS3321-MemoryBase/database/08_demo_queries.sql +SQL file completed: /Users/jflin/Workspace/Code/CS3321-MemoryBase/database/08_demo_queries.sql diff --git a/docs/final-assets/screenshots/logs/explain-cases.txt b/docs/final-assets/screenshots/logs/explain-cases.txt new file mode 100644 index 0000000..177933f --- /dev/null +++ b/docs/final-assets/screenshots/logs/explain-cases.txt @@ -0,0 +1,99 @@ +exit_code: 0 + +> memorybase-workspace@0.1.0 db:run +> python scripts/db_cli.py run /tmp/memorybase_gap4_explain.sql + +psql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_authid", skipping it +psql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_subscription", skipping it +psql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_database", skipping it +psql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_db_role_setting", skipping it +psql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_tablespace", skipping it +psql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_auth_members", skipping it +psql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_shdepend", skipping it +psql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_shdescription", skipping it +psql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_replication_origin", skipping it +psql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_shseclabel", skipping it +psql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_parameter_acl", skipping it +VACUUM +CASE 1: covering partial composite index for active memory ranking +BEGIN +SET + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=0.14..4.12 rows=10 width=48) (actual time=0.003..0.005 rows=10 loops=1) + Buffers: shared hit=2 + -> Index Only Scan using idx_memory_active_ranking on memory_item (cost=0.14..8.51 rows=21 width=48) (actual time=0.002..0.003 rows=10 loops=1) + Index Cond: (workspace_id = '00000000-0000-0000-0000-000000000201'::uuid) + Heap Fetches: 0 + Buffers: shared hit=2 + Planning: + Buffers: shared hit=74 + Planning Time: 0.123 ms + Execution Time: 0.015 ms +(10 rows) + +ROLLBACK + +CASE 2: GIN FTS plus trigram bitmap OR path +BEGIN +SET +SET + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (cost=48.10..51.25 rows=3 width=147) (actual time=0.049..0.054 rows=3 loops=1) + Buffers: shared hit=22 + -> Bitmap Heap Scan on source_chunk sc (cost=48.10..51.25 rows=3 width=147) (actual time=0.049..0.053 rows=3 loops=1) + Recheck Cond: ((search_vector @@ '''memorybase'''::tsquery) OR (chunk_text ~~* '%memorybase%'::text)) + Heap Blocks: exact=3 + Buffers: shared hit=22 + -> BitmapOr (cost=48.10..48.10 rows=3 width=0) (actual time=0.042..0.042 rows=0 loops=1) + Buffers: shared hit=19 + -> Bitmap Index Scan on idx_source_chunk_fts (cost=0.00..8.55 rows=2 width=0) (actual time=0.006..0.006 rows=2 loops=1) + Index Cond: (search_vector @@ '''memorybase'''::tsquery) + Buffers: shared hit=2 + -> Bitmap Index Scan on idx_source_chunk_text_trgm (cost=0.00..39.55 rows=1 width=0) (actual time=0.036..0.036 rows=3 loops=1) + Index Cond: (chunk_text ~~* '%memorybase%'::text) + Buffers: shared hit=17 + Planning: + Buffers: shared hit=48 + Planning Time: 1.695 ms + Execution Time: 0.069 ms +(18 rows) + +ROLLBACK + +CASE 3: BRIN audit time-window path after dropping competing btree indexes in transaction +BEGIN +DROP INDEX +DROP INDEX +DROP INDEX +SET + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------- + Sort (cost=20.74..20.77 rows=12 width=30) (actual time=0.151..0.151 rows=5 loops=1) + Sort Key: (date_trunc('day'::text, created_at)) DESC, (count(*)) DESC + Sort Method: quicksort Memory: 25kB + Buffers: shared hit=15 + -> HashAggregate (cost=20.37..20.52 rows=12 width=30) (actual time=0.141..0.142 rows=5 loops=1) + Group Key: date_trunc('day'::text, created_at), action_type + Batches: 1 Memory Usage: 24kB + Buffers: shared hit=12 + -> Bitmap Heap Scan on audit_log (cost=12.03..20.01 rows=49 width=22) (actual time=0.109..0.131 rows=48 loops=1) + Recheck Cond: (created_at >= (now() - '30 days'::interval)) + Rows Removed by Index Recheck: 1 + Heap Blocks: lossy=7 + Buffers: shared hit=12 + -> Bitmap Index Scan on idx_audit_brin_time (cost=0.00..12.01 rows=49 width=0) (actual time=0.011..0.012 rows=70 loops=1) + Index Cond: (created_at >= (now() - '30 days'::interval)) + Buffers: shared hit=5 + Planning: + Buffers: shared hit=45 + Planning Time: 0.515 ms + Execution Time: 0.186 ms +(20 rows) + +ROLLBACK +Using .env.example because .env was not found. +Using DATABASE_URL=postgresql://memorybase:memorybase@localhost:5432/memorybase_db +==> Running /tmp/memorybase_gap4_explain.sql +SQL file completed: /tmp/memorybase_gap4_explain.sql diff --git a/docs/final-assets/screenshots/logs/focused-sql.txt b/docs/final-assets/screenshots/logs/focused-sql.txt new file mode 100644 index 0000000..001ddd7 --- /dev/null +++ b/docs/final-assets/screenshots/logs/focused-sql.txt @@ -0,0 +1,61 @@ +$ npm run db:run -- /tmp/memorybase_gap4_focused.sql +cwd: /Users/jflin/Workspace/Code/CS3321-MemoryBase +exit_code: 0 + + +> memorybase-workspace@0.1.0 db:run +> python scripts/db_cli.py run /tmp/memorybase_gap4_focused.sql + +FOCUSED SQL EVIDENCE: provenance, visibility, governance, recall log + +1) Memory provenance: MemoryItem -> SourceChunk -> SourceDocument + memory_id | memory | source_title | chunk_no | start_line | end_line +--------------------------------------+------------------------------------------------------------------------+------------------------------+----------+------------+---------- + 00000000-0000-0000-0000-000000000701 | The team abandoned the campus cafeteria system because it was too CRUD | Discussion 01: Project Pivot | 2 | 10 | 10 + 00000000-0000-0000-0000-000000000719 | The demo answer to the cafeteria question is that the cafeteria system | Discussion 06: Demo Plan | 2 | 8 | 8 +(2 rows) + + +2) Agent-visible memories exclude private items via v_agent_visible_memory + memory_id | memory_type | access_level | memory +--------------------------------------+-------------+--------------+------------------------------------------------------------------------ + 00000000-0000-0000-0000-000000000733 | decision | project | Recall combines GIN tsvector full-text search as the primary path with + 00000000-0000-0000-0000-000000000716 | semantic | project | Wiki statements should trace back to MemoryItem, MemoryEvidence, Sourc + 00000000-0000-0000-0000-000000000702 | decision | project | MemoryBase was selected because it demonstrates source chunks, evidenc + 00000000-0000-0000-0000-000000000701 | decision | project | The team abandoned the campus cafeteria system because it was too CRUD + 00000000-0000-0000-0000-000000000718 | task | project | The five to eight minute demo should show source import, chunk line nu + 00000000-0000-0000-0000-000000000707 | semantic | project | The schema must demonstrate primary keys, foreign keys, unique constra + 00000000-0000-0000-0000-000000000705 | procedural | project | The core data flow is SourceDocument to SourceChunk to MemoryItem to M + 00000000-0000-0000-0000-000000000709 | semantic | project | MemoryEvidence is the many-to-many bridge between MemoryItem and Sourc +(8 rows) + + +3) Governance lifecycle: forgotten / archived / superseded memories + memory_id | status | summary | superseded_by_memory_id +--------------------------------------+------------+-----------------------------------------------+-------------------------------------- + 00000000-0000-0000-0000-000000000732 | archived | Early decision: ILIKE only. | 00000000-0000-0000-0000-000000000733 + 00000000-0000-0000-0000-000000000731 | forgotten | Early (wrong) decision: prefer Hash index. | + 00000000-0000-0000-0000-000000000742 | superseded | Standup cadence: biweekly Mon/Thu (rejected). | 00000000-0000-0000-0000-000000000741 +(3 rows) + + +4) Conflict records: open + resolved + conflict_id | conflict_type | status | resolution_note +--------------------------------------+---------------+----------+---------------------------------------------------------------------------------- + 00000000-0000-0000-0000-000000001001 | uncertain | open | Clarify whether LLM extraction belongs to MVP or future scope. + 00000000-0000-0000-0000-000000001002 | contradiction | resolved | Resolution by carol: weekly Monday cadence wins per team vote 4-1. Memory 0742 k +(2 rows) + + +5) Recall log records query, filters, and top memory IDs + query_text | result_count | filters | top_memory_ids_json +------------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 为什么放弃校园食堂系统？ | 4 | {"as_of": null, "status": "active", "agent_id": null, "memory_type": null, "access_level": null, "retrieval_mode": "keyword"} | ["00000000-0000-0000-0000-000000000719", "00000000-0000-0000-0000-000000000701", "00000000-0000-0000-0000-000000000703", "00000000-0000-0000-0000-000000000704"] + 为什么放弃校园食堂系统？ | 4 | {"as_of": null, "status": "active", "agent_id": null, "memory_type": null, "access_level": null, "retrieval_mode": "keyword"} | ["00000000-0000-0000-0000-000000000719", "00000000-0000-0000-0000-000000000701", "00000000-0000-0000-0000-000000000703", "00000000-0000-0000-0000-000000000704"] + why did we abandon the campus cafeteria system | 3 | | ["00000000-0000-0000-0000-000000000701", "00000000-0000-0000-0000-000000000719", "00000000-0000-0000-0000-000000000702"] +(3 rows) + +Using .env.example because .env was not found. +Using DATABASE_URL=postgresql://memorybase:memorybase@localhost:5432/memorybase_db +==> Running /tmp/memorybase_gap4_focused.sql +SQL file completed: /tmp/memorybase_gap4_focused.sql diff --git a/docs/final-assets/screenshots/logs/frontend-build.txt b/docs/final-assets/screenshots/logs/frontend-build.txt new file mode 100644 index 0000000..a80a102 --- /dev/null +++ b/docs/final-assets/screenshots/logs/frontend-build.txt @@ -0,0 +1,16 @@ +$ npm run build +cwd: /Users/jflin/Workspace/Code/CS3321-MemoryBase/frontend +exit_code: 0 + +> memorybase-frontend@0.1.0 build +> vite build + +vite v5.4.21 building for production... +transforming... +✓ 64 modules transformed. +rendering chunks... +computing gzip size... +dist/index.html 0.72 kB │ gzip: 0.40 kB +dist/assets/index-CcdpLdfc.css 9.54 kB │ gzip: 2.89 kB +dist/assets/index-CfwQVtRD.js 292.93 kB │ gzip: 80.69 kB +✓ built in 630ms diff --git a/docs/final-assets/screenshots/logs/pytest-core.txt b/docs/final-assets/screenshots/logs/pytest-core.txt new file mode 100644 index 0000000..f016a0a --- /dev/null +++ b/docs/final-assets/screenshots/logs/pytest-core.txt @@ -0,0 +1,12 @@ +$ .venv/bin/python -m pytest backend/tests/test_health.py backend/tests/test_recall_query.py backend/tests/test_graph_service.py -q +cwd: /Users/jflin/Workspace/Code/CS3321-MemoryBase +exit_code: 0 + +................ [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /Users/jflin/Workspace/Code/CS3321-MemoryBase/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +16 passed, 1 warning in 0.12s diff --git a/docs/final-assets/screenshots/sql/01-db-check.png b/docs/final-assets/screenshots/sql/01-db-check.png new file mode 100644 index 0000000..01ea39f Binary files /dev/null and b/docs/final-assets/screenshots/sql/01-db-check.png differ diff --git a/docs/final-assets/screenshots/sql/01-db-check.svg b/docs/final-assets/screenshots/sql/01-db-check.svg new file mode 100644 index 0000000..e68134b --- /dev/null +++ b/docs/final-assets/screenshots/sql/01-db-check.svg @@ -0,0 +1,12 @@ + + + + Database Check Result + MemoryBase evidence · generated from logs/db-check.txt + + $ npm run db:checkcwd: /Users/jflin/Workspace/Code/CS3321-MemoryBaseexit_code: 0> memorybase-workspace@0.1.0 db:check> python scripts/db_cli.py check List of relations Schema | Name | Type | Owner --------+------------------------+-------+------------ public | access_policy | table | memorybase public | agent | table | memorybase public | agent_session | table | memorybase public | audit_log | table | memorybase public | conflict_record | table | memorybase public | entity | table | memorybase public | forget_request | table | memorybase public | memory_embedding | table | memorybase public | memory_entity | table | memorybase public | memory_evidence | table | memorybase public | memory_item | table | memorybase public | memory_revision | table | memorybase public | memory_scene | table | memorybase public | memory_scene_cell | table | memorybase public | message | table | memorybase public | recall_log | table | memorybase public | source_chunk | table | memorybase public | source_chunk_embedding | table | memorybase public | source_document | table | memorybase public | timeline_entry | table | memorybase public | user_account | table | memorybase public | wiki_page | table | memorybase public | wiki_page_revision | table | memorybase public | workspace | table | memorybase public | workspace_member | table | memorybase(25 rows) workspace_id | name | scope_type --------------------------------------+-----------------------------------+------------ 00000000-0000-0000-0000-000000000201 | MemoryBase Course Demo | project 00000000-0000-0000-0000-000000002201 | Graph Demo: Project Knowledge Map | project(2 rows) memory_id | summary | access_level | status --------------------------------------+--------------------------------------------------------+--------------+------------ 00000000-0000-0000-0000-000000000701 | Reason cafeteria topic was rejected. | project | active 00000000-0000-0000-0000-000000000702 | Reason for choosing MemoryBase. | project | active 00000000-0000-0000-0000-000000000703 | Required demo question. | project | active 00000000-0000-0000-0000-000000000704 | File and database dual state. | project | active 00000000-0000-0000-0000-000000000705 | Core data flow. | project | active 00000000-0000-0000-0000-000000000706 | Backend API scope. | project | active 00000000-0000-0000-0000-000000000707 | Database course requirements. | project | active 00000000-0000-0000-0000-000000000708 | Source and chunk responsibilities. | project | active 00000000-0000-0000-0000-000000000709 | Evidence relationship. | project | active 00000000-0000-0000-0000-000000000710 | Audit and revision responsibility. | project | active 00000000-0000-0000-0000-000000000711 | Recall output shape. | project | active 00000000-0000-0000-0000-000000000712 | Recall implementation baseline. | project | conflicted 00000000-0000-0000-0000-000000000713 | Private permission rule. | private | active 00000000-0000-0000-0000-000000000714 | RecallLog content. | project | active 00000000-0000-0000-0000-000000000715 | Wiki export purpose. | project | active 00000000-0000-0000-0000-000000000716 | Wiki provenance chain. | project | active 00000000-0000-0000-0000-000000000717 | Timeline value. | team | active 00000000-0000-0000-0000-000000000718 | Demo checklist. | project | active 00000000-0000-0000-0000-000000000719 | Demo recall answer. | project | active 00000000-0000-0000-0000-000000000720 | LLM extraction is future work. | project | conflicted 00000000-0000-0000-0000-000000000731 | Early (wrong) decision: prefer Hash index. | project | forgotten 00000000-0000-0000-0000-000000000733 | Final recall design: GIN FTS primary + ILIKE fallback. | project | active 00000000-0000-0000-0000-000000000732 | Early decision: ILIKE only. | project | archived 00000000-0000-0000-0000-000000000741 | Standup cadence: weekly Monday. | project | active 00000000-0000-0000-0000-000000000742 | Standup cadence: biweekly Mon/Thu (rejected). | project | superseded 00000000-0000-0000-0000-000000002701 | Rejected CRUD-heavy topic | project | active 00000000-0000-0000-0000-000000002702 | Selected MemoryBase | project | active 00000000-0000-0000-0000-000000002703 | PostgreSQL as source of truth | project | active 00000000-0000-0000-0000-000000002704 | Neo4j as graph index | project | active 00000000-0000-0000-0000-000000002705 | Provenance chain | project | active 00000000-0000-0000-0000-000000002706 | Agent visibility policy | project | active 00000000-0000-0000-0000-000000002707 | Wiki provenance | project | active(32 rows) conflict_id | status | conflict_type --------------------------------------+----------+--------------- 00000000-0000-0000-0000-000000001001 | open | uncertain 00000000-0000-0000-0000-000000001002 | resolved | contradiction(2 rows) title | event_type | event_time ----------------------------+------------+------------------------ Abandoned cafeteria system | decision | 2026-03-02 18:30:00+08 Selected MemoryBase | decision | 2026-03-02 18:40:00+08 Locked schema requirements | proposal | 2026-03-10 18:30:00+08 Defined recall shape | proposal | 2026-03-15 18:30:00+08 Added wiki provenance | decision | 2026-03-20 18:30:00+08(5 rows)Using .env.example because .env was not found.... clipped 7 lines; see logs/db-check.txt for full output ... + diff --git a/docs/final-assets/screenshots/sql/02-demo-queries.png b/docs/final-assets/screenshots/sql/02-demo-queries.png new file mode 100644 index 0000000..fa0b36a Binary files /dev/null and b/docs/final-assets/screenshots/sql/02-demo-queries.png differ diff --git a/docs/final-assets/screenshots/sql/02-demo-queries.svg b/docs/final-assets/screenshots/sql/02-demo-queries.svg new file mode 100644 index 0000000..0013b52 --- /dev/null +++ b/docs/final-assets/screenshots/sql/02-demo-queries.svg @@ -0,0 +1,12 @@ + + + + Demo SQL Query Results + MemoryBase evidence · generated from logs/demo-queries.txt + + $ npm run db:run -- database/08_demo_queries.sqlcwd: /Users/jflin/Workspace/Code/CS3321-MemoryBaseexit_code: 0> memorybase-workspace@0.1.0 db:run> python scripts/db_cli.py run database/08_demo_queries.sql memory_id | memory_type | canonical_text --------------------------------------+-------------+------------------------------------------------------------------------------------------------------------------------------- 00000000-0000-0000-0000-000000002702 | decision | MemoryBase was selected because it demonstrates provenance, governance, and graph exploration. 00000000-0000-0000-0000-000000002701 | decision | The cafeteria ordering topic was rejected because it mostly demonstrated CRUD. 00000000-0000-0000-0000-000000002707 | semantic | Wiki provenance lets readers trace generated pages back to memories and evidence. 00000000-0000-0000-0000-000000002705 | procedural | Memory provenance flows from wiki page to scene, memory, evidence chunk, and source document. 00000000-0000-0000-0000-000000002704 | semantic | Neo4j is used as a relationship index for graph exploration, not as the source of truth. 00000000-0000-0000-0000-000000002703 | semantic | PostgreSQL remains the authoritative source of business data. 00000000-0000-0000-0000-000000000733 | decision | Recall combines GIN tsvector full-text search as the primary path with ILIKE pattern matching as fallback for short queries. 00000000-0000-0000-0000-000000000716 | semantic | Wiki statements should trace back to MemoryItem, MemoryEvidence, SourceChunk, and SourceDocument. 00000000-0000-0000-0000-000000000702 | decision | MemoryBase was selected because it demonstrates source chunks, evidence, revisions, audit logs, policies, recall logs, conflic 00000000-0000-0000-0000-000000000701 | decision | The team abandoned the campus cafeteria system because it was too CRUD-heavy and did not demonstrate enough database depth. 00000000-0000-0000-0000-000000000718 | task | The five to eight minute demo should show source import, chunk line numbers, memory evidence, recall, revision, audit, policy 00000000-0000-0000-0000-000000000719 | decision | The demo answer to the cafeteria question is that the cafeteria system was too CRUD-heavy and did not demonstrate enough datab 00000000-0000-0000-0000-000000000709 | semantic | MemoryEvidence is the many-to-many bridge between MemoryItem and SourceChunk. 00000000-0000-0000-0000-000000000707 | semantic | The schema must demonstrate primary keys, foreign keys, unique constraints, check constraints, indexes, views, and triggers. 00000000-0000-0000-0000-000000000705 | procedural | The core data flow is SourceDocument to SourceChunk to MemoryItem to MemoryEvidence. 00000000-0000-0000-0000-000000002706 | semantic | Access policies decide which memories an agent can see. 00000000-0000-0000-0000-000000000703 | semantic | The recall demo must answer why the campus cafeteria system was abandoned. 00000000-0000-0000-0000-000000000713 | risk | Private budget or personal coordination notes must stay hidden from a project-only retriever agent unless explicitly allowed. 00000000-0000-0000-0000-000000000715 | semantic | Wiki export makes database memory readable as Markdown and WikiPageRevision audits repeated exports. 00000000-0000-0000-0000-000000000710 | semantic | AuditLog stores before and after JSON, while MemoryRevision stores immutable memory versions. (20 rows) memory_id | canonical_text --------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------- 00000000-0000-0000-0000-000000000701 | The team abandoned the campus cafeteria system because it was too CRUD-heavy and did not demonstrate enough database depth. 00000000-0000-0000-0000-000000000702 | MemoryBase was selected because it demonstrates source chunks, evidence, revisions, audit logs, policies, recall logs, conflicts, and wiki p 00000000-0000-0000-0000-000000000703 | The recall demo must answer why the campus cafeteria system was abandoned. 00000000-0000-0000-0000-000000000704 | MemoryBase keeps both human-readable Markdown and database records as a file-database dual state system. 00000000-0000-0000-0000-000000000705 | The core data flow is SourceDocument to SourceChunk to MemoryItem to MemoryEvidence. 00000000-0000-0000-0000-000000000706 | The backend should expose import, source, memory, recall, policy, audit, and wiki export APIs. 00000000-0000-0000-0000-000000000707 | The schema must demonstrate primary keys, foreign keys, unique constraints, check constraints, indexes, views, and triggers. 00000000-0000-0000-0000-000000000708 | SourceDocument stores imported files and checksums; SourceChunk stores chunk text, line ranges, token counts, and search vectors. 00000000-0000-0000-0000-000000000709 | MemoryEvidence is the many-to-many bridge between MemoryItem and SourceChunk. 00000000-0000-0000-0000-000000000710 | AuditLog stores before and after JSON, while MemoryRevision stores immutable memory versions. 00000000-0000-0000-0000-000000000711 | Recall should return memory items together with supporting evidence and source chunks. 00000000-0000-0000-0000-000000000712 | The first recall implementation can use keyword search and PostgreSQL full text search. 00000000-0000-0000-0000-000000000713 | Private budget or personal coordination notes must stay hidden from a project-only retriever agent unless explicitly allowed. 00000000-0000-0000-0000-000000000714 | RecallLog records query text, filters, result count, top memory IDs, and context pack JSON. 00000000-0000-0000-0000-000000000715 | Wiki export makes database memory readable as Markdown and WikiPageRevision audits repeated exports. 00000000-0000-0000-0000-000000000716 | Wiki statements should trace back to MemoryItem, MemoryEvidence, SourceChunk, and SourceDocument. 00000000-0000-0000-0000-000000000717 | TimelineEntry shows how the project evolved from topic selection to schema design, recall, governance, and final demo. 00000000-0000-0000-0000-000000000718 | The five to eight minute demo should show source import, chunk line numbers, memory evidence, recall, revision, audit, policy filtering, con 00000000-0000-0000-0000-000000000719 | The demo answer to the cafeteria question is that the cafeteria system was too CRUD-heavy and did not demonstrate enough database features. 00000000-0000-0000-0000-000000000720 | LLM automatic extraction is useful later but is not required for the deterministic MVP demo. (20 rows) timeline_id | event_time | event_type | title | memo--------------------------------------+------------------------+------------+----------------------------+-------------------------------------------------------------------------- 00000000-0000-0000-0000-000000001205 | 2026-03-20 18:30:00+08 | decision | Added wiki provenance | Wiki statements should trace back to MemoryItem, MemoryEvidence, SourceCh 00000000-0000-0000-0000-000000001204 | 2026-03-15 18:30:00+08 | proposal | Defined recall shape | Recall should return memory items together with supporting evidence and s 00000000-0000-0000-0000-000000001203 | 2026-03-10 18:30:00+08 | proposal | Locked schema requirements | The schema must demonstrate primary keys, foreign keys, unique constraint 00000000-0000-0000-0000-000000001202 | 2026-03-02 18:40:00+08 | decision | Selected MemoryBase | MemoryBase was selected because it demonstrates source chunks, evidence, 00000000-0000-0000-0000-000000001201 | 2026-03-02 18:30:00+08 | decision | Abandoned cafeteria system | The team abandoned the campus cafeteria system because it was too CRUD-he(5 rows) workspace_id | memory_type | status | access_level | memory_count | avg_confidence | avg_importance --------------------------------------+-------------+------------+--------------+--------------+------------------------+-------------------- 00000000-0000-0000-0000-000000000201 | semantic | conflicted | project | 1 | 0.84000000000000000000 | 3.0000000000000000 00000000-0000-0000-0000-000000000201 | task | active | project | 2 | 0.88000000000000000000 | 4.0000000000000000 00000000-0000-0000-0000-000000000201 | decision | forgotten | project | 1 | 0.50000000000000000000 | 2.0000000000000000 00000000-0000-0000-0000-000000000201 | decision | active | project | 5 | 0.93000000000000000000 | 4.6000000000000000 00000000-0000-0000-0000-000000000201 | decision | archived | project | 1 | 0.50000000000000000000 | 2.0000000000000000 00000000-0000-0000-0000-000000000201 | risk | active | private | 1 | 0.90000000000000000000 | 4.0000000000000000 00000000-0000-0000-0000-000000000201 | procedural | active | project | 2 | 0.90500000000000000000 | 4.5000000000000000 00000000-0000-0000-0000-000000000201 | semantic | active | team | 1 | 0.85000000000000000000 | 3.0000000000000000 00000000-0000-0000-0000-000000000201 | decision | conflicted | project | 1 | 0.88000000000000000000 | 4.0000000000000000 00000000-0000-0000-0000-000000002201 | procedural | active | project | 1 | 0.93000000000000000000 | 5.0000000000000000 00000000-0000-0000-0000-000000002201 | decision | active | project | 2 | 0.96500000000000000000 | 5.0000000000000000 00000000-0000-0000-0000-000000000201 | semantic | active | project | 9 | 0.90666666666666666667 | 4.2222222222222222 00000000-0000-0000-0000-000000000201 | decision | superseded | project | 1 | 0.60000000000000000000 | 2.0000000000000000 00000000-0000-0000-0000-000000002201 | semantic | active | project | 4 | 0.93250000000000000000 | 4.7500000000000000(14 rows) memory_id | memory_type | canonical_text --------------------------------------+-------------+------------------------------------------------------------------------------------------------------------------------------- 00000000-0000-0000-0000-000000000733 | decision | Recall combines GIN tsvector full-text search as the primary path with ILIKE pattern matching as fallback for short queries. 00000000-0000-0000-0000-000000000716 | semantic | Wiki statements should trace back to MemoryItem, MemoryEvidence, SourceChunk, and SourceDocument. 00000000-0000-0000-0000-000000000702 | decision | MemoryBase was selected because it demonstrates source chunks, evidence, revisions, audit logs, policies, recall logs, conflic 00000000-0000-0000-0000-000000000701 | decision | The team abandoned the campus cafeteria system because it was too CRUD-heavy and did not demonstrate enough database depth. 00000000-0000-0000-0000-000000000719 | decision | The demo answer to the cafeteria question is that the cafeteria system was too CRUD-heavy and did not demonstrate enough datab 00000000-0000-0000-0000-000000000718 | task | The five to eight minute demo should show source import, chunk line numbers, memory evidence, recall, revision, audit, policy 00000000-0000-0000-0000-000000000707 | semantic | The schema must demonstrate primary keys, foreign keys, unique constraints, check constraints, indexes, views, and triggers. 00000000-0000-0000-0000-000000000705 | procedural | The core data flow is SourceDocument to SourceChunk to MemoryItem to MemoryEvidence. 00000000-0000-0000-0000-000000000709 | semantic | MemoryEvidence is the many-to-many bridge between MemoryItem and SourceChunk. 00000000-0000-0000-0000-000000000710 | semantic | AuditLog stores before and after JSON, while MemoryRevision stores immutable memory versions. 00000000-0000-0000-0000-000000000703 | semantic | The recall demo must answer why the campus cafeteria system was abandoned. 00000000-0000-0000-0000-000000000704 | semantic | MemoryBase keeps both human-readable Markdown and database records as a file-database dual state system. 00000000-0000-0000-0000-000000000708 | semantic | SourceDocument stores imported files and checksums; SourceChunk stores chunk text, line ranges, token counts, and search vecto 00000000-0000-0000-0000-000000000711 | procedural | Recall should return memory items together with supporting evidence and source chunks. 00000000-0000-0000-0000-000000000715 | semantic | Wiki export makes database memory readable as Markdown and WikiPageRevision audits repeated exports. 00000000-0000-0000-0000-000000000741 | decision | Team standup is held every Monday morning at 10am. 00000000-0000-0000-0000-000000000717 | semantic | TimelineEntry shows how the project evolved from topic selection to schema design, recall, governance, and final demo. 00000000-0000-0000-0000-000000000714 | semantic | RecallLog records query text, filters, result count, top memory IDs, and context pack JSON. 00000000-0000-0000-0000-000000000706 | task | The backend should expose import, source, memory, recall, policy, audit, and wiki export APIs. (19 rows) conflict_id | conflict_type | conflict_status | left_memory_text | --------------------------------------+---------------+-----------------+-----------------------------------------------------------------------------------------+----------------- 00000000-0000-0000-0000-000000001002 | contradiction | resolved | Team standup is held every Monday morning at 10am. | Team standup is 00000000-0000-0000-0000-000000001001 | uncertain | open | The first recall implementation can use keyword search and PostgreSQL full text search. | LLM automatic ex(2 rows) page_slug | page_title | memory_id | source_title | chunk_no | start_line | end_line ---------------------+---------------------+--------------------------------------+-----------------------------------+----------+------------+---------- architecture-map | Architecture Map | 00000000-0000-0000-0000-000000002703 | Graph Demo 02: Architecture Chain | 1 | 4 | 4 architecture-map | Architecture Map | 00000000-0000-0000-0000-000000002704 | Graph Demo 02: Architecture Chain | 2 | 6 | 6 architecture-map | Architecture Map | 00000000-0000-0000-0000-000000002705 | Graph Demo 02: Architecture Chain | 3 | 8 | 8 architecture-map | Architecture Map | 00000000-0000-0000-0000-000000002705 | Graph Demo 02: Architecture Chain | 4 | 10 | 10 database-design | Database Design | 00000000-0000-0000-0000-000000000707 | Discussion 03: Schema | 1 | 6 | 6 demo-playbook | Demo Playbook | 00000000-0000-0000-0000-000000000718 | Discussion 06: Demo Plan | 1 | 6 | 6 governance-demo | Governance Demo | 00000000-0000-0000-0000-000000002706 | Graph Demo 03: Governance Story | 1 | 4 | 4 governance-demo | Governance Demo | 00000000-0000-0000-0000-000000002707 | Graph Demo 03: Governance Story | 2 | 8 | 8 project-pivot-story | Project Pivot Story | 00000000-0000-0000-0000-000000002701 | Graph Demo 01: Topic Pivot | 1 | 4 | 4 project-pivot-story | Project Pivot Story | 00000000-0000-0000-0000-000000002702 | Graph Demo 01: Topic Pivot | 2 | 6 | 6 why-memorybase | Why MemoryBase | 00000000-0000-0000-0000-000000000701 | Discussion 01: Project Pivot | 2 | 10 | 10 why-memorybase | Why MemoryBase | 00000000-0000-0000-0000-000000000719 | Discussion 06: Demo Plan | 2 | 8 | 8 why-memorybase | Why MemoryBase | 00000000-0000-0000-0000-000000000702 | Discussion 01: Project Pivot | 3 | 12 | 12 why-memorybase | Why MemoryBase | 00000000-0000-0000-0000-000000000703 | Discussion 01: Project Pivot | 4 | 14 | 14(14 rows) doc_id | chunk_no | chunk_text --------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------- 00000000-0000-0000-0000-000000000501 | 3 | The MemoryBase direction gives us stronger database requirements: source documents, chunks, memory evidence, revisions, audit log 00000000-0000-0000-0000-000000000502 | 1 | MemoryBase uses a file-database dual state model. Human readable Markdown remains useful for review, while PostgreSQL provides qu(2 rows) scene_title | cell_role | sort_order | memory_type | canonical_text ----------------+------------+------------+-------------+--------------------------------------------------------------------------------------------------------------------------- Topic Decision | background | 10 | decision | The team abandoned the campus cafeteria system because it was too CRUD-heavy and did not demonstrate enough database depth Topic Decision | decision | 20 | decision | MemoryBase was selected because it demonstrates source chunks, evidence, revisions, audit logs, policies, recall logs, con Topic Decision | context | 30 | semantic | The recall demo must answer why the campus cafeteria system was abandoned. Topic Decision | outcome | 40 | decision | The demo answer to the cafeteria question is that the cafeteria system was too CRUD-heavy and did not demonstrate enough d(4 rows) memory_id | memory_type | search_text_zh --------------------------------------+-------------+------------------------------------------------------------------------------------------------------------------------------- 00000000-0000-0000-0000-000000000733 | decision | Recall combines GIN tsvector full-text search as primary path with ILIKE pattern matching as fallback for short queries. 00000000-0000-0000-0000-000000000716 | semantic | wiki statements should trace back to memoryitem memoryevidence sourcechunk and sourcedocument wiki provenance chain 00000000-0000-0000-0000-000000000702 | decision | memorybase was selected because it demonstrates source chunks evidence revisions audit logs policies recall logs conflicts and 00000000-0000-0000-0000-000000000701 | decision | the team abandoned the campus cafeteria system because it was too crud heavy and did not demonstrate enough database depth rea 00000000-0000-0000-0000-000000000707 | semantic | the schema must demonstrate primary keys foreign keys unique constraints check constraints indexes views and triggers database... clipped 38 lines; see logs/demo-queries.txt for full output ... + diff --git a/docs/final-assets/screenshots/sql/03-explain-analyze.png b/docs/final-assets/screenshots/sql/03-explain-analyze.png new file mode 100644 index 0000000..4e147af Binary files /dev/null and b/docs/final-assets/screenshots/sql/03-explain-analyze.png differ diff --git a/docs/final-assets/screenshots/sql/03-explain-analyze.svg b/docs/final-assets/screenshots/sql/03-explain-analyze.svg new file mode 100644 index 0000000..69516dc --- /dev/null +++ b/docs/final-assets/screenshots/sql/03-explain-analyze.svg @@ -0,0 +1,12 @@ + + + + EXPLAIN ANALYZE Evidence + MemoryBase evidence · generated from logs/explain-cases.txt + + exit_code: 0> memorybase-workspace@0.1.0 db:run> python scripts/db_cli.py run /tmp/memorybase_gap4_explain.sqlpsql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_authid", skipping itpsql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_subscription", skipping itpsql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_database", skipping itpsql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_db_role_setting", skipping itpsql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_tablespace", skipping itpsql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_auth_members", skipping itpsql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_shdepend", skipping itpsql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_shdescription", skipping itpsql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_replication_origin", skipping itpsql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_shseclabel", skipping itpsql:/tmp/memorybase_gap4_explain.sql:1: WARNING: permission denied to vacuum "pg_parameter_acl", skipping itVACUUMCASE 1: covering partial composite index for active memory rankingBEGINSET QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------- Limit (cost=0.14..4.12 rows=10 width=48) (actual time=0.003..0.005 rows=10 loops=1) Buffers: shared hit=2 -> Index Only Scan using idx_memory_active_ranking on memory_item (cost=0.14..8.51 rows=21 width=48) (actual time=0.002..0.003 rows=10 loops=1) Index Cond: (workspace_id = '00000000-0000-0000-0000-000000000201'::uuid) Heap Fetches: 0 Buffers: shared hit=2 Planning: Buffers: shared hit=74 Planning Time: 0.123 ms Execution Time: 0.015 ms(10 rows)ROLLBACKCASE 2: GIN FTS plus trigram bitmap OR pathBEGINSETSET QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------ Limit (cost=48.10..51.25 rows=3 width=147) (actual time=0.049..0.054 rows=3 loops=1) Buffers: shared hit=22 -> Bitmap Heap Scan on source_chunk sc (cost=48.10..51.25 rows=3 width=147) (actual time=0.049..0.053 rows=3 loops=1) Recheck Cond: ((search_vector @@ '''memorybase'''::tsquery) OR (chunk_text ~~* '%memorybase%'::text)) Heap Blocks: exact=3 Buffers: shared hit=22 -> BitmapOr (cost=48.10..48.10 rows=3 width=0) (actual time=0.042..0.042 rows=0 loops=1) Buffers: shared hit=19 -> Bitmap Index Scan on idx_source_chunk_fts (cost=0.00..8.55 rows=2 width=0) (actual time=0.006..0.006 rows=2 loops=1) Index Cond: (search_vector @@ '''memorybase'''::tsquery) Buffers: shared hit=2 -> Bitmap Index Scan on idx_source_chunk_text_trgm (cost=0.00..39.55 rows=1 width=0) (actual time=0.036..0.036 rows=3 loops=1) Index Cond: (chunk_text ~~* '%memorybase%'::text) Buffers: shared hit=17 Planning: Buffers: shared hit=48 Planning Time: 1.695 ms Execution Time: 0.069 ms(18 rows)ROLLBACKCASE 3: BRIN audit time-window path after dropping competing btree indexes in transactionBEGINDROP INDEXDROP INDEXDROP INDEXSET QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------- Sort (cost=20.74..20.77 rows=12 width=30) (actual time=0.151..0.151 rows=5 loops=1) Sort Key: (date_trunc('day'::text, created_at)) DESC, (count(*)) DESC Sort Method: quicksort Memory: 25kB Buffers: shared hit=15 -> HashAggregate (cost=20.37..20.52 rows=12 width=30) (actual time=0.141..0.142 rows=5 loops=1) Group Key: date_trunc('day'::text, created_at), action_type Batches: 1 Memory Usage: 24kB Buffers: shared hit=12 -> Bitmap Heap Scan on audit_log (cost=12.03..20.01 rows=49 width=22) (actual time=0.109..0.131 rows=48 loops=1) Recheck Cond: (created_at >= (now() - '30 days'::interval)) Rows Removed by Index Recheck: 1 Heap Blocks: lossy=7 Buffers: shared hit=12 -> Bitmap Index Scan on idx_audit_brin_time (cost=0.00..12.01 rows=49 width=0) (actual time=0.011..0.012 rows=70 loops=1) Index Cond: (created_at >= (now() - '30 days'::interval)) Buffers: shared hit=5 Planning: Buffers: shared hit=45 Planning Time: 0.515 ms Execution Time: 0.186 ms(20 rows)ROLLBACKUsing .env.example because .env was not found.Using DATABASE_URL=postgresql://memorybase:memorybase@localhost:5432/memorybase_db==> Running /tmp/memorybase_gap4_explain.sqlSQL file completed: /tmp/memorybase_gap4_explain.sql + diff --git a/docs/final-assets/screenshots/sql/04-focused-sql-evidence.png b/docs/final-assets/screenshots/sql/04-focused-sql-evidence.png new file mode 100644 index 0000000..6165759 Binary files /dev/null and b/docs/final-assets/screenshots/sql/04-focused-sql-evidence.png differ diff --git a/docs/final-assets/screenshots/sql/04-focused-sql-evidence.svg b/docs/final-assets/screenshots/sql/04-focused-sql-evidence.svg new file mode 100644 index 0000000..653b253 --- /dev/null +++ b/docs/final-assets/screenshots/sql/04-focused-sql-evidence.svg @@ -0,0 +1,12 @@ + + + + Focused SQL Evidence + MemoryBase evidence · generated from logs/focused-sql.txt + + $ npm run db:run -- /tmp/memorybase_gap4_focused.sqlcwd: /Users/jflin/Workspace/Code/CS3321-MemoryBaseexit_code: 0> memorybase-workspace@0.1.0 db:run> python scripts/db_cli.py run /tmp/memorybase_gap4_focused.sqlFOCUSED SQL EVIDENCE: provenance, visibility, governance, recall log1) Memory provenance: MemoryItem -> SourceChunk -> SourceDocument memory_id | memory | source_title | chunk_no | start_line | end_line --------------------------------------+------------------------------------------------------------------------+------------------------------+----------+------------+---------- 00000000-0000-0000-0000-000000000701 | The team abandoned the campus cafeteria system because it was too CRUD | Discussion 01: Project Pivot | 2 | 10 | 10 00000000-0000-0000-0000-000000000719 | The demo answer to the cafeteria question is that the cafeteria system | Discussion 06: Demo Plan | 2 | 8 | 8(2 rows)2) Agent-visible memories exclude private items via v_agent_visible_memory memory_id | memory_type | access_level | memory --------------------------------------+-------------+--------------+------------------------------------------------------------------------ 00000000-0000-0000-0000-000000000733 | decision | project | Recall combines GIN tsvector full-text search as the primary path with 00000000-0000-0000-0000-000000000716 | semantic | project | Wiki statements should trace back to MemoryItem, MemoryEvidence, Sourc 00000000-0000-0000-0000-000000000702 | decision | project | MemoryBase was selected because it demonstrates source chunks, evidenc 00000000-0000-0000-0000-000000000701 | decision | project | The team abandoned the campus cafeteria system because it was too CRUD 00000000-0000-0000-0000-000000000718 | task | project | The five to eight minute demo should show source import, chunk line nu 00000000-0000-0000-0000-000000000707 | semantic | project | The schema must demonstrate primary keys, foreign keys, unique constra 00000000-0000-0000-0000-000000000705 | procedural | project | The core data flow is SourceDocument to SourceChunk to MemoryItem to M 00000000-0000-0000-0000-000000000709 | semantic | project | MemoryEvidence is the many-to-many bridge between MemoryItem and Sourc(8 rows)3) Governance lifecycle: forgotten / archived / superseded memories memory_id | status | summary | superseded_by_memory_id --------------------------------------+------------+-----------------------------------------------+-------------------------------------- 00000000-0000-0000-0000-000000000732 | archived | Early decision: ILIKE only. | 00000000-0000-0000-0000-000000000733 00000000-0000-0000-0000-000000000731 | forgotten | Early (wrong) decision: prefer Hash index. | 00000000-0000-0000-0000-000000000742 | superseded | Standup cadence: biweekly Mon/Thu (rejected). | 00000000-0000-0000-0000-000000000741(3 rows)4) Conflict records: open + resolved conflict_id | conflict_type | status | resolution_note --------------------------------------+---------------+----------+---------------------------------------------------------------------------------- 00000000-0000-0000-0000-000000001001 | uncertain | open | Clarify whether LLM extraction belongs to MVP or future scope. 00000000-0000-0000-0000-000000001002 | contradiction | resolved | Resolution by carol: weekly Monday cadence wins per team vote 4-1. Memory 0742 k(2 rows)5) Recall log records query, filters, and top memory IDs query_text | result_count | filters ------------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------- 为什么放弃校园食堂系统？ | 4 | {"as_of": null, "status": "active", "agent_id": null, "memory_type": null, "access_level": null, "retrieval_mode": "keyword"} | 为什么放弃校园食堂系统？ | 4 | {"as_of": null, "status": "active", "agent_id": null, "memory_type": null, "access_level": null, "retrieval_mode": "keyword"} | why did we abandon the campus cafeteria system | 3 | (3 rows)Using .env.example because .env was not found.Using DATABASE_URL=postgresql://memorybase:memorybase@localhost:5432/memorybase_db==> Running /tmp/memorybase_gap4_focused.sqlSQL file completed: /tmp/memorybase_gap4_focused.sql + diff --git a/docs/final-assets/screenshots/tests/01-pytest-core.png b/docs/final-assets/screenshots/tests/01-pytest-core.png new file mode 100644 index 0000000..71378a3 Binary files /dev/null and b/docs/final-assets/screenshots/tests/01-pytest-core.png differ diff --git a/docs/final-assets/screenshots/tests/01-pytest-core.svg b/docs/final-assets/screenshots/tests/01-pytest-core.svg new file mode 100644 index 0000000..36a269c --- /dev/null +++ b/docs/final-assets/screenshots/tests/01-pytest-core.svg @@ -0,0 +1,12 @@ + + + + Backend Test Subset Result + MemoryBase evidence · generated from logs/pytest-core.txt + + $ .venv/bin/python -m pytest backend/tests/test_health.py backend/tests/test_recall_query.py backend/tests/test_graph_service.py -qcwd: /Users/jflin/Workspace/Code/CS3321-MemoryBaseexit_code: 0................ [100%]=============================== warnings summary ===============================.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 /Users/jflin/Workspace/Code/CS3321-MemoryBase/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` i from starlette.testclient import TestClient as TestClient # noqa-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html16 passed, 1 warning in 0.12s + diff --git a/docs/final-assets/screenshots/tests/02-frontend-build.png b/docs/final-assets/screenshots/tests/02-frontend-build.png new file mode 100644 index 0000000..5f54d02 Binary files /dev/null and b/docs/final-assets/screenshots/tests/02-frontend-build.png differ diff --git a/docs/final-assets/screenshots/tests/02-frontend-build.svg b/docs/final-assets/screenshots/tests/02-frontend-build.svg new file mode 100644 index 0000000..819fbbd --- /dev/null +++ b/docs/final-assets/screenshots/tests/02-frontend-build.svg @@ -0,0 +1,12 @@ + + + + Frontend Build Result + MemoryBase evidence · generated from logs/frontend-build.txt + + $ npm run buildcwd: /Users/jflin/Workspace/Code/CS3321-MemoryBase/frontendexit_code: 0> memorybase-frontend@0.1.0 build> vite buildvite v5.4.21 building for production...transforming...✓ 64 modules transformed.rendering chunks...computing gzip size...dist/index.html 0.72 kB │ gzip: 0.40 kBdist/assets/index-CcdpLdfc.css 9.54 kB │ gzip: 2.89 kBdist/assets/index-CfwQVtRD.js 292.93 kB │ gzip: 80.69 kB✓ built in 630ms + diff --git a/docs/final-assets/screenshots/ui/01-dashboard.png b/docs/final-assets/screenshots/ui/01-dashboard.png new file mode 100644 index 0000000..3c38401 Binary files /dev/null and b/docs/final-assets/screenshots/ui/01-dashboard.png differ diff --git a/docs/final-assets/screenshots/ui/02-sources-list.png b/docs/final-assets/screenshots/ui/02-sources-list.png new file mode 100644 index 0000000..8c6da00 Binary files /dev/null and b/docs/final-assets/screenshots/ui/02-sources-list.png differ diff --git a/docs/final-assets/screenshots/ui/03-source-detail-project-pivot.png b/docs/final-assets/screenshots/ui/03-source-detail-project-pivot.png new file mode 100644 index 0000000..4ebe450 Binary files /dev/null and b/docs/final-assets/screenshots/ui/03-source-detail-project-pivot.png differ diff --git a/docs/final-assets/screenshots/ui/04-memories-list.png b/docs/final-assets/screenshots/ui/04-memories-list.png new file mode 100644 index 0000000..affaadf Binary files /dev/null and b/docs/final-assets/screenshots/ui/04-memories-list.png differ diff --git a/docs/final-assets/screenshots/ui/05-memory-detail-evidence-revisions.png b/docs/final-assets/screenshots/ui/05-memory-detail-evidence-revisions.png new file mode 100644 index 0000000..6e66aa5 Binary files /dev/null and b/docs/final-assets/screenshots/ui/05-memory-detail-evidence-revisions.png differ diff --git a/docs/final-assets/screenshots/ui/06-recall-search-results.png b/docs/final-assets/screenshots/ui/06-recall-search-results.png new file mode 100644 index 0000000..58ff15d Binary files /dev/null and b/docs/final-assets/screenshots/ui/06-recall-search-results.png differ diff --git a/docs/final-assets/screenshots/ui/07-recall-context-pack.png b/docs/final-assets/screenshots/ui/07-recall-context-pack.png new file mode 100644 index 0000000..77e30f2 Binary files /dev/null and b/docs/final-assets/screenshots/ui/07-recall-context-pack.png differ diff --git a/docs/final-assets/screenshots/ui/08-recall-qa-answer-or-config-state.png b/docs/final-assets/screenshots/ui/08-recall-qa-answer-or-config-state.png new file mode 100644 index 0000000..892b3d0 Binary files /dev/null and b/docs/final-assets/screenshots/ui/08-recall-qa-answer-or-config-state.png differ diff --git a/docs/final-assets/screenshots/ui/09-governance-timeline.png b/docs/final-assets/screenshots/ui/09-governance-timeline.png new file mode 100644 index 0000000..6522f2d Binary files /dev/null and b/docs/final-assets/screenshots/ui/09-governance-timeline.png differ diff --git a/docs/final-assets/screenshots/ui/10-governance-audit.png b/docs/final-assets/screenshots/ui/10-governance-audit.png new file mode 100644 index 0000000..5ef093a Binary files /dev/null and b/docs/final-assets/screenshots/ui/10-governance-audit.png differ diff --git a/docs/final-assets/screenshots/ui/11-governance-policies.png b/docs/final-assets/screenshots/ui/11-governance-policies.png new file mode 100644 index 0000000..bee6b81 Binary files /dev/null and b/docs/final-assets/screenshots/ui/11-governance-policies.png differ diff --git a/docs/final-assets/screenshots/ui/12-governance-conflicts.png b/docs/final-assets/screenshots/ui/12-governance-conflicts.png new file mode 100644 index 0000000..e7fe271 Binary files /dev/null and b/docs/final-assets/screenshots/ui/12-governance-conflicts.png differ diff --git a/docs/final-assets/screenshots/ui/13-governance-forget-requests.png b/docs/final-assets/screenshots/ui/13-governance-forget-requests.png new file mode 100644 index 0000000..31aa542 Binary files /dev/null and b/docs/final-assets/screenshots/ui/13-governance-forget-requests.png differ diff --git a/docs/final-assets/screenshots/ui/14-wiki-export.png b/docs/final-assets/screenshots/ui/14-wiki-export.png new file mode 100644 index 0000000..3b329a1 Binary files /dev/null and b/docs/final-assets/screenshots/ui/14-wiki-export.png differ diff --git a/docs/final-assets/screenshots/ui/15-graph-explorer-demo-workspace.png b/docs/final-assets/screenshots/ui/15-graph-explorer-demo-workspace.png new file mode 100644 index 0000000..69fd23e Binary files /dev/null and b/docs/final-assets/screenshots/ui/15-graph-explorer-demo-workspace.png differ diff --git a/docs/final-assets/screenshots/ui/15-graph-explorer.png b/docs/final-assets/screenshots/ui/15-graph-explorer.png new file mode 100644 index 0000000..5d8321d Binary files /dev/null and b/docs/final-assets/screenshots/ui/15-graph-explorer.png differ diff --git a/docs/final-assets/screenshots/ui/16-runtime-sessions.png b/docs/final-assets/screenshots/ui/16-runtime-sessions.png new file mode 100644 index 0000000..4a90ac0 Binary files /dev/null and b/docs/final-assets/screenshots/ui/16-runtime-sessions.png differ diff --git a/docs/final-assets/screenshots/ui/17-runtime-messages.png b/docs/final-assets/screenshots/ui/17-runtime-messages.png new file mode 100644 index 0000000..bab3fb6 Binary files /dev/null and b/docs/final-assets/screenshots/ui/17-runtime-messages.png differ diff --git a/docs/final-assets/screenshots/ui/18-runtime-hybrid-search-results.png b/docs/final-assets/screenshots/ui/18-runtime-hybrid-search-results.png new file mode 100644 index 0000000..f2725ff Binary files /dev/null and b/docs/final-assets/screenshots/ui/18-runtime-hybrid-search-results.png differ diff --git a/docs/final-assets/screenshots/ui/18-runtime-hybrid-search.png b/docs/final-assets/screenshots/ui/18-runtime-hybrid-search.png new file mode 100644 index 0000000..c90ea0a Binary files /dev/null and b/docs/final-assets/screenshots/ui/18-runtime-hybrid-search.png differ diff --git a/docs/final-assets/screenshots/ui/19-llm-not-used.png b/docs/final-assets/screenshots/ui/19-llm-not-used.png new file mode 100644 index 0000000..9d86740 Binary files /dev/null and b/docs/final-assets/screenshots/ui/19-llm-not-used.png differ diff --git a/docs/final-assets/screenshots/ui/19-llm-used.png b/docs/final-assets/screenshots/ui/19-llm-used.png new file mode 100644 index 0000000..9d303e2 Binary files /dev/null and b/docs/final-assets/screenshots/ui/19-llm-used.png differ diff --git a/docs/final-assets/slides/final-defense.html b/docs/final-assets/slides/final-defense.html new file mode 100644 index 0000000..6df8bf9 --- /dev/null +++ b/docs/final-assets/slides/final-defense.html @@ -0,0 +1,784 @@ + + + + +MemoryBase · 课程答辩 PPT + + + + + + + + + +
+

MemoryBase

+
面向组织与团队的 AI-native 可追溯长期记忆数据库系统
设计与实现
+
CS3321 数据库系统课程小组 · 2026 年 6 月
林纪帆 · 杜卓轩 · 李昭成 · 王星睿
+
+ + +
+ 01 · 项目概述 +

我们解决什么问题

+

AI Agent 与团队协作系统持续产生会议纪要、讨论、决策、偏好与变更，传统做法把它们留在 Markdown、聊天历史或文件树里。

+ +
+
+

普通文件 / 聊天的局限

+

+ · 信息散落，难统一查询
+ · 修改无法回放、无 actor、无审计
+ · Agent 缺可靠权限边界，只能靠 prompt 自律
+ · 遗忘 / 冲突缺少审批链与可验证状态
+ · 难以回答“这条结论来自哪一段原文” +

+
+
+

MemoryBase 的一句话定位

+

+ 把人类可读的 Markdown、会议纪要与项目文档，编译为人和 Agent 都能访问、可追溯、可权限控制、可审计、可版本化的长期记忆数据库。 +

+

+ 同时, 我们完整呈现了其概念结构、ER、范式、物理设计、索引、视图、触发器与 SQL 证据。 +

+
+
+ +
02
+
+ + +
+ 02 · 研究现状 +

对照四类已有路线

+

我们吸收它们的强项；差异主要发生在数据库这一层。

+ + + + + + + + + + + + + + + + + + + + + + +
传统 RAG / 向量检索LangChain、LlamaIndex、Pinecone、ChromaDB语义召回强，但把 chunk / memory / evidence / revision / audit / policy 当业务对象建模的较少
Agent 长期记忆系统MemGPT / Letta、Mem0、MemoryBank、LongMem关心“模型记得并用上”，较少展示 schema、外键、范式、触发器、审计链与 SQL 可验证性
产品化记忆 / 企业知识库ChatGPT Memory、Confluence AI / Rovo、Notion AI、Obsidian / Logseq体验成熟，但不开放底层关系 schema、触发器、SQL EXPLAIN 与多租户权限策略
长期记忆评测 + GraphRAGLoCoMo、LongMemEval、MemoryAgentBench、GraphRAG提供基准与图增强思路；MemoryBase 把 evaluation 与 graph 作为验证与可视化层，不替代 DB 主线
+ +
差异点：长期记忆按数据库应用建模；provenance、governance、agent visibility、audit、SQL lifecycle 全部写在 schema 里。
+ +
03
+
+ + +
+ 03 · 系统定位 +

DB-first ：以 PostgreSQL 为事实源

+

长期记忆不是孤立文本，而是带来源、版本、权限、审计与表达投影的数据库对象。

+ +
+
+

核心数据链路

+ 
+SourceDocument
+  └── SourceChunk
+        └── MemoryItem
+              ├── MemoryEvidence  (chunk + 行号)
+              ├── MemoryRevision  (版本)
+              ├── AuditLog        (操作回放)
+              ├── AccessPolicy    (权限过滤)
+              ├── RecallLog       (检索证据)
+              └── WikiPage
+                    └── WikiPageRevision
+
+
+
+

关系骨架

+

+ · 25 张核心关系表（主线 11 张节选）：source_document、source_chunk、memory_item、memory_evidence、memory_revision、audit_log、access_policy、wiki_page、conflict_record、forget_request、recall_log
+ · 8 个核心视图：v_active_memory / v_memory_with_source / v_agent_visible_memory / v_project_timeline / v_conflict_memory / v_wiki_page_sources / v_memory_statistics / v_memory_recall_statistics
+ · 7 个业务触发器（另有 7 个 _touch 维护 updated_at）：memory revision / audit / soft delete + conflict lifecycle + wiki revision
+ · 8 种索引：B+ tree / GIN tsvector / GIN trigram / partial / partial unique / composite / BRIN / covering +

+
+
+ +
04
+
+ + +
+ 04 · 概念结构设计 +

核心 E-R 图

+ +
+ MemoryBase 核心 ER 图 +
+ +
13 个核心实体 · 60+ 关系 · 主键、外键、CHECK、UNIQUE、GENERATED 全约束 · provenance 与 governance 内嵌 schema
+ +
05
+
+ + +
+ 05 · 逻辑结构设计 +

范式分析与受控反规范化

+

核心业务表以 3NF / BCNF 为主；少量派生字段做受控反规范化，显式标注并用 GENERATED 或 trigger 保持一致。

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
核心业务表（多数）3NF / BCNFsource_document、memory_evidence、memory_revision、audit_log、access_policy、conflict_record、forget_request 等
memory_item.search_vector⚠ 反规范化tsvector，GENERATED ALWAYS AS 自动维护，供 GIN FTS
memory_item.current_revision_no⚠ 反规范化触发器自动维护，供短列表 Index Only Scan
source_chunk.search_text_zh⚠ 反规范化dirty-bit cache，避免每次召回再拼接原文
wiki_page.markdown_body⚠ 反规范化维度投影 / 人类可读输出，由 service 同步写
audit_log（追加写）BCNFappend-only，BRIN 索引专属场景
+ +
06
+
+ + +
+ 06 · 物理结构设计 +

八种索引策略 + EXPLAIN 证据

+ +
+
+

索引类型 (覆盖经典 / 现代 / Postgres 专属)

+

+ · B+ tree (B-link 变种) — 30+ composite，叶子兄弟指针
+ · GIN tsvector — idx_memory_fts、idx_source_chunk_fts
+ · GIN trigram — 任意位置子串模糊匹配
+ · Partial — WHERE status='active' 过滤
+ · Partial unique — idx_policy_global_unique
+ · Composite leading-key — workspace_id 优先 (多租户标准)
+ · BRIN — audit_log(created_at) append-only 教科书场景
+ · Covering (INCLUDE) — idx_memory_active_ranking 走 Index Only Scan +

+
+
+

EXPLAIN ANALYZE 证据

+ EXPLAIN ANALYZE 执行计划 +
+
+ +
EXPLAIN 已验证：Index Only Scan · Heap Fetches = 0 · BitmapOr · BRIN range scan · GIN FTS / trigram 命中
+ +
07
+
+ + +
+ 07 · 物理结构设计 +

视图、触发器与完整性约束

+ +
+
+

8 个核心视图

+

+ · v_active_memory — 排除 forgotten / archived
+ · v_memory_with_source — provenance 拼接
+ · v_agent_visible_memory — 权限过滤
+ · v_project_timeline — 时间线投影
+ · v_conflict_memory — 冲突状态
+ · v_wiki_page_sources — wiki 引用追溯
+ · v_memory_statistics — 课程指标
+ · v_memory_recall_statistics — 召回统计 +

+
+
+

7 个触发器

+

+ · trg_memory_before_update — revision 累加
+ · trg_memory_after_insert — 写 audit_log
+ · trg_memory_after_update — 写 audit_log
+ · trg_memory_soft_delete — 状态变更
+ · trg_conflict_after_insert — 冲突初始化
+ · trg_conflict_after_update — 冲突 lifecycle
+ · trg_wiki_revision_after_insert — wiki 版本追加 +

+

+ 完整性约束：PK / FK / UNIQUE / CHECK 枚举 (status × 7、evidence_role × 4、principal_type × 3、effect × 2) / NOT NULL / GENERATED ALWAYS。 +

+
+
+ +
08
+
+ + +
+ 08 · 系统架构 +

DB-first 多入口架构

+

前端、CLI、Agent Runtime 与 evaluation runner 是围绕同一事实源的不同入口。

+ +
+
+

分层与入口

+ 
+Human / Admin / Agent / Evaluation Runner
+                |
+                v
+   React Frontend     mb / memorybase CLI
+                \    /
+                 v  v
+            FastAPI Backend
+       (API · Service · Repository)
+                |
+                v
+   PostgreSQL  <--optional-->  Neo4j
+                |
+                v
+   Markdown Wiki / Context Pack / Eval Report
+
+
+
+

后端模块分层

+

+ · API layer — backend/app/api/*.py：HTTP endpoints + Pydantic contract
+ · Service layer — backend/app/services/*.py：source / memory / recall / governance / graph / embedding / QA
+ · Core — backend/app/core/*.py：配置、DB 连接
+ · CLI — backend/app/cli/：mb / memorybase 命令，覆盖 health / context / recall / search / observe / remember / sessions / eval
+ · Tests — backend/tests/：API、service、evaluation、PostgreSQL integration +

+
+
+ +
09
+
+ + +
+ 09 · 系统演示 +

演示一 · Source → Memory → Evidence 端到端 provenance

+

任意一条 memory 都可追到 SourceChunk 行号、改写历史与操作者。

+ +
+ Memory 详情 · evidence + revision +
+ +
一条 memory · 多条 evidence · N 次 revision · 全部带 actor + 时间戳 · 由 memory_evidence + 触发器自动维护
+ +
10
+
+ + +
+ 10 · 系统演示 +

演示二 · Rule-based vs Qwen LLM 候选抽取

+

同一组三条 chunk，默认 rule_based 与 optional Qwen-compatible LLM 给出了不同的类型判断与 confidence。

+ +
+
+

默认 rule-based

+ Rule-based candidate extraction +
+
+

Qwen-compatible optional LLM

+ Qwen LLM candidate extraction +
+
+ +
+ `abandoned cafeteria`: `decision`, confidence `0.65` → `0.90`；`PostgreSQL source of truth`: `task` → `decision`；`private budget notes hidden`: `constraint` → `policy`。 +
+ +
默认仍是离线可演示的 rule_based；LLM 只提升 candidate 语义质量，结果仍写入 status='candidate' 并需要人工审批
+ +
11
+
+ + +
+ 11 · 系统演示 +

演示三 · Recall + Context Pack 与透明 fallback

+

keyword / vector / hybrid 三种模式；无 embedding 时透明 fallback 到 keyword，并写入 retrieval_info。

+ +
+ Recall 搜索结果 + retrieval_info +
+ +
retrieval_info 写明 effective_mode、fallback_reason、vector_used;Context Pack 投影为 Agent-ready Markdown
+ +
12
+
+ + +
+ 12 · 系统演示 +

演示四 · Governance lifecycle + Graph 可视化

+ +
+
+

冲突治理 (Governance)

+ Conflict governance UI +
+
+

provenance 图谱 (Graph)

+ Graph Explorer demo workspace +
+
+ +
conflict · policy · forget · timeline · audit · graph — 全部从 schema 投影；Graph 是可视化层，PostgreSQL 仍是权威事实源
+ +
13
+
+ + +
+ 13 · 测试与评估 +

自动化测试覆盖

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
API / servicetest_sources.py、test_memories.py、test_recall_query.pysource 导入、memory CRUD、召回
Governancetest_governance.py、test_graph_service.pypolicy / audit / conflict / forget / graph visibility
CLI / Agent Runtimetest_cli_context.py、test_cli_recall.py、test_cli_sessions.py、test_cli_writeback.pycontext / recall / observe / remember 全链路
Evaluationtest_evaluation_*adapters / metrics / judging / checkpoint / report
PostgreSQL integrationtest_postgres_integration.pyschema / seed / trigger / batch create / supersession
Frontendnpm run lint + npm run buildReact 生产构建
+ +
最近一次合并后：pytest 197 passed · 1 skipped · ruff 通过 · frontend lint + build 通过 · PostgreSQL integration 2/2 passed
+ +
14
+
+ + +
+ 14 · 测试与评估 +

LongMemEval 500-case 工程评测

+

评测在我们这里只用来验证工程能力、暴露弱项，不是榜单卖点。

+ +
+
+

整体指标 (oracle / db_qa)

+ + + + + + +
Cases500
API errors0
Deterministic pass176 / 500 (35.2%)
Semantic judge pass292 / 500 (58.4%)
Semantic judge fail208 / 500 (41.6%)
+
+
+

分类结果 (semantic judge)

+ + + + + + + +
Abstention (强项)93.3%
Temporal update (强项)80.6%
Single fact (强项)79.2%
Temporal reasoning52.8%
Preference following (弱项)36.7%
Multi-session reasoning (弱项)27.3%
+
+
+ +
主价值在 provenance · governance · agent visibility · audit · SQL-verifiable lifecycle；弱项 (multi-session、preference) 已写入 future work
+ +
15
+
+ + +
+ 15 · 创新点 +

八大创新点

+ +
+
+
+
ADB-first AI substrate
+

把 AI 长期记忆从黑盒能力变成 PostgreSQL 里可验证的关系模型。LLM、embedding、Graph、eval 都是围绕 DB 事实源的增强层。

+
+
+
BProvenance-first 证据链
+

memory_evidence / v_memory_with_source / v_wiki_page_sources / recall_log 可回答 “来自哪里、谁用过、改成什么”。

+
+
+
CGovernance-by-default lifecycle
+

revision / audit / conflict / forget / policy 是核心表 + 触发器,状态变更全可用 SQL 验证。

+
+
+
DAgent-aware visibility
+

Agent 是独立 principal,通过 access_policy + v_agent_visible_memory 把权限过滤放进数据库查询路径。

+
+
+
+
+
ETransparent hybrid retrieval
+

支持 keyword / vector / hybrid。无 embedding 时透明 fallback 并在 retrieval_info 中记录 effective_mode + fallback_reason。

+
+
+
FAI-assisted candidate extraction
+

默认是离线可演示的 rule_based 抽取；可选 LLM 只负责生成 status='candidate' 草稿,仍要绑定 evidence、记录 run-level audit,并经人工审批后才进入 active。

+
+
+
GGraph as provenance visualization
+

Graph Explorer 用 PostgreSQL 构图,可选同步 Neo4j。图层只做可视化,权威仍在 DB 里。

+
+
+
HEvaluation-backed validation
+

接入 LoCoMo / LongMemEval / MemoryAgentBench adapters。系统因此能跑长期记忆基准,把真实边界一起暴露出来。

+
+
+
+ +
16
+
+ + +
+ 16 · 分工与展望 +

分工、贡献与未来工作

+ +
+
+

小组分工 (基于 Git commit 审计)

+

+ · 林纪帆 (hopecommon / jflin) — DB schema、治理与溯源模型、lexical search、context-pack formatter、CLI dogfood、Graph hardening、最终报告与答辩材料
+ · 杜卓轩 (dzx0902 / dzx) — P0 backend、evaluation framework、LoCoMo / LongMemEval / MemoryAgentBench adapters、embedding / hybrid recall、QA、CI / tooling
+ · 李昭成 (huiyijian / lywzc0419) — 前端 API 集成、Memory / Recall / Governance 等核心页面交互实现，Runtime / Governance UI、Neo4j Graph Explorer 初版集成，将 PostgreSQL 中的 memory、source、evidence 等关系投影为 provenance 图谱，用于记忆来源追踪与图可视化展示
+ · 王星睿 (Cofstars) — source extraction workflow、optional LLM candidate extraction、相关 source UI , CLI , API , tests / final report 和 ppt +

+
+
+

当前边界 → 未来工作

+

+ · 当前仅实现 optional LLM candidate extraction → 后续引入 analysis_run / analysis_memory_draft / analysis_draft_evidence 草稿表
+ · JSONB embedding cache → 引入 pgvector / HNSW / IVFFlat 支撑大规模向量
+ · LongMemEval 弱项 → 增强 multi-session reasoning、preference following
+ · memory-level visibility → 扩展 source / wiki 资源 policy
+ · 完善 LoCoMo / MemoryAgentBench 对照实验 + 独立 judge
+ · Obsidian / Git 双向同步与多 Agent 协作 +

+
+
+ +
MemoryBase 把 AI memory 从产品黑盒搬进 PostgreSQL — 每条都带来源、可审计、可被 Agent 调用
+ +
17
+
+ + + diff --git a/docs/final-assets/slides/final-defense.pdf b/docs/final-assets/slides/final-defense.pdf new file mode 100644 index 0000000..ae0144f Binary files /dev/null and b/docs/final-assets/slides/final-defense.pdf differ diff --git a/docs/final-report.md b/docs/final-report.md new file mode 100644 index 0000000..88a0b82 --- /dev/null +++ b/docs/final-report.md @@ -0,0 +1,910 @@ +# MemoryBase：面向组织与团队的 AI-native 可追溯长期记忆数据库系统设计与实现 + +**课程**：CS3321 数据库技术 + +**小组成员**：林纪帆、杜卓轩、李昭成、王星睿 + +## 摘要 + +AI Agent 和团队协作系统会持续产生会议纪要、讨论记录、项目文档、决策、偏好等多种多样的日志文档。这些长期知识通常只保存在聊天历史或 Markdown 文件中，这导致后续很难进行结构化查询、来源追溯、权限过滤、版本审计以及选择性遗忘。面对以上困境，MemoryBase 尝试以 PostgreSQL 为事实源，把长期记忆建模为一套可查询、可追溯、可治理、可被人和 Agent 共同使用的数据库系统。 + +具体来说，我们的系统采用“文件—数据库双态”架构：文件侧保留 Markdown / txt source 和可导出的 Markdown Wiki，以服务人类阅读、迁移和审阅；数据库侧则使用 `source_document`、`source_chunk`、`memory_item`、`memory_evidence`、`memory_revision`、`audit_log`、`access_policy`、`wiki_page` 等关系表管理记忆生命周期，以针对具体功能。系统已实现 source 导入、chunk 切分、memory 创建与候选抽取、evidence 追溯、revision/audit 自动记录、全文检索、agent-aware visibility、冲突/遗忘治理、Wiki 投影、Graph Explorer、CLI/Agent Runtime、hybrid recall fallback 和 evaluation framework等多种功能。 + +我们的项目展示了完整的概念结构设计、E-R 图、关系模式转换、范式分析、物理结构设计、索引、视图、触发器、完整性约束、SQL 查询、EXPLAIN 证据，并提供带注释源程序。LongMemEval 500-case 工程评测显示当前系统在 semantic judge 下通过率为 58.4%，说明我们的系统具备长期记忆任务验证能力；当然，我们的系统也仍有值得提升之处： multi-session reasoning 和 preference following 仍是后续优化重点。 + +**关键词**：长期记忆数据库；PostgreSQL；Provenance；Governance；Agent Visibility；Audit Log；Hybrid Recall；Graph Explorer；Evaluation + +## 1. 项目概述 + +### 1.1 项目定位 + +MemoryBase 是一个面向组织与团队的 AI-native 可追溯长期记忆数据库系统。它不是从零实现 DBMS，也不是普通 RAG 问答应用，而是数据库应用系统：以成熟关系数据库作为底座，专注于设计和实现长期记忆的关系模型、完整性约束、检索路径、权限治理、审计和人机协作入口。 + +系统的一句话介绍是： + +> 把人类可读的 Markdown、会议纪要和项目文档，编译为人和 Agent 都能访问、可追溯、可权限控制、可审计、可版本化的长期记忆数据库。 + +MemoryBase 解决的核心问题包括： + +| 问题 | 普通文件 / 聊天记录的局限 | MemoryBase 的处理方式 | +|---|---|---| +| 信息分散 | 文档、会议纪要、聊天记录散落各处 | `source_document` / `source_chunk` 统一导入和切分 | +| 结论难追溯 | 记忆不知道来自哪段原文 | `memory_evidence` 绑定 source chunk 和行号 | +| 修改不可审计 | 旧版本和操作者不可回放 | `memory_revision` + `audit_log` | +| 权限不可靠 | 只靠 prompt 或 UI 约定 | `access_policy` + `v_agent_visible_memory` | +| 冲突和遗忘难治理 | 删除或覆盖后缺少审批链 | `conflict_record` + `forget_request` | +| Agent 难使用 | 文件适合人读，不适合结构化上下文 | Recall / Search / Context Pack / CLI | +| 人类难审阅 | 数据库记录不适合直接阅读 | `wiki_page` / `wiki_page_revision` 导出 Markdown | + +### 1.2 核心链路 + +系统核心数据链路为： + +```text +SourceDocument + -> SourceChunk + -> MemoryItem + -> MemoryEvidence + -> MemoryRevision / AuditLog + -> RecallLog / AccessPolicy + -> WikiPage / WikiPageRevision +``` + +这条链路体现了项目的主线：长期记忆不是孤立文本，而是带来源、版本、权限、审计和表达投影的数据库对象。 + +### 1.3 已实现功能 + +核心能力（不依赖外部 LLM，也不依赖向量数据库）： + +- 导入 Markdown / txt source； +- 自动切分 source chunk； +- 创建、编辑、归档 memory； +- 绑定 memory evidence； +- 记录 memory revision 和 audit log； +- 关键词 / 全文检索； +- Markdown Wiki 导出； +- 前端基础页面和 SQL 演示数据。 + +扩展能力包括： + +- agent-aware visibility 和 policy 过滤； +- ConflictRecord 冲突治理； +- ForgetRequest 遗忘/归档审批； +- rule-based + optional LLM candidate memory extraction； +- local hashing embedding cache 与 hybrid recall fallback； +- Graph Explorer（PostgreSQL preview + 可选 Neo4j sync）； +- CLI / Agent Runtime sessions、observe、remember、search、recall； +- evaluation framework（LoCoMo / LongMemEval / MemoryAgentBench 等 adapter）。 + +### 1.4 报告材料入口 + +如果想对本项目有更全面了解，可以参考以下报告： + +| 材料 | 用途 | +|---|---| +| `docs/00-project-overview.md` | 项目总览和核心链路 | +| `docs/research-landscape.md` | 研究现状分析 | +| `docs/01-requirements.md` | 需求分析 | +| `docs/02-data-flow.md` | 数据流图 | +| `docs/03-data-dictionary.md` | 数据字典 | +| `docs/04-er-design.md` / `docs/final-assets/diagrams/` | ER 和流程图 | +| `docs/05-logical-design.md` / `docs/normalization.md` | 逻辑结构和范式分析 | +| `docs/06-physical-design.md` / `docs/index-rationale.md` / `docs/explain-analyze.md` | 物理设计、索引和 EXPLAIN | +| `docs/07-system-architecture.md` / `docs/08-api-design.md` / `docs/09-module-ipo.md` | 系统架构、API 和模块 IPO | +| `docs/10-test-plan.md` / `docs/final-assets/screenshots/` | 测试与截图证据 | +| `docs/innovation-analysis.md` | 创新点展开 | +| `docs/contribution-ledger.md` | 小组分工和个人贡献 | +| `docs/source-sql-appendix.md` | 带注释 SQL / 高级语言源程序附录 | + +## 2. 研究现状分析 + +### 2.1 传统 RAG 与向量检索 + +Retrieval-Augmented Generation（RAG）通过“文档切分、embedding、向量召回、上下文拼接”增强 LLM 对外部知识的访问能力。它的优势是实现快、语义召回强、生态成熟，LangChain、LlamaIndex 等框架也降低了工程门槛。 + +但普通 RAG 更关注 chunk 检索和回答效果，较少把 chunk、memory、evidence、revision、audit、policy、wiki projection 建模为有完整生命周期的业务对象。它能返回相关片段，却不一定能回答：这条 memory 来自哪份 source？谁修改过？Agent 是否有权限看？source 被遗忘后是否还会被召回？一次 hybrid recall 为什么降级到 keyword？ + +MemoryBase 保留 RAG 的检索思想，但以 PostgreSQL 关系模型为核心：source、chunk、memory、evidence、recall log 和 audit 都是可查询对象；而embedding cache 只是可选增强，不是唯一事实源。 + +### 2.2 Agent 长期记忆系统 + +MemGPT / Letta、Mem0、MemoryBank、LongMem、Generative Agents 等系统说明长期记忆已经成为 Agent 基础设施。它们强调 core memory、archival memory、动态抽取、合并、检索和跨会话状态维护。 + +这些工作证明 Agent 确实需要长期记忆，但从数据库课程和组织治理角度看，它们通常更关注“模型是否记得并用上”，而不是 schema、外键、范式、触发器、审计链、权限视图和 SQL 可验证性。MemoryBase 的差异化是把 Agent 记忆问题落到数据库系统：Agent 可以用 CLI/API 做 recall、search、observe、remember，但每次写入和召回都落入可约束、可审计、可复现的数据库结构。 + +### 2.3 产品化记忆和企业知识库 + +ChatGPT Memory 代表消费者产品中的记忆能力，用户可以管理 saved memories 和 reference chat history；Confluence AI / Atlassian Rovo 与 Notion AI Enterprise Search 代表企业知识管理中的 AI search、chat、agents 和 workspace automation；Obsidian / Logseq 则代表本地文件型知识库。 + +这些产品既说明长期记忆和知识管理是实际需求，也显示一大问题：这些产品大多不开放底层关系 schema、触发器、SQL EXPLAIN、审计表和多租户权限策略等内容。MemoryBase 吸收了它们在人类可读、wiki、搜索和引用方面的优点，同时也满足数据库课程所要求的可建模、可约束、可审计和可复现。 + +### 2.4 长期记忆评测与 GraphRAG + +LoCoMo、LongMemEval、MemoryAgentBench 等 benchmark 从多 session、长期对话、时间推理、信息更新、选择性遗忘等角度评估 memory agent 能力。GraphRAG 则强调在 chunk / vector 之外构建实体关系图和社区摘要，提升跨文档关系理解。 + +MemoryBase 已接入 evaluation framework，并实现 Graph Explorer。graph 用于 provenance 和 governance 可视化，PostgreSQL 是权威事实源。当前 LongMemEval 结果显示系统已有长期记忆工程验证路径。 + +### 2.5 现有方案的共同不足 + +综合来看，现有路线存在四类不足： + +1. 可追溯性不足：能召回结果，但不能稳定说明 memory 与 source、chunk、revision、approval 的关系。 +2. 关系建模不足：向量库擅长相似度，但不擅长表达 evidence、policy、audit、forget request 等关系。 +3. 治理能力不足：组织场景需要权限、冲突、遗忘、归档、审计和多角色访问。 +4. 课程可验证性不足：黑盒 AI 工具难以展示 E-R 图、关系模式、范式、索引、视图、触发器和 EXPLAIN。 + +MemoryBase 的切入点正是把长期记忆作为数据库应用系统来建模。 + +## 3. 需求分析 + +### 3.1 用户角色 + +| 角色 | 使用目标 | 主要操作 | 权限边界 | +|---|---|---|---| +| 普通用户 | 搜索项目记忆、查看 source、阅读 Wiki | 搜索、查看、导出 | 默认只看 public/project 范围 | +| 小组成员 | 维护课程项目记忆 | 导入 source、创建/编辑 memory、处理 conflict | 不能绕过 workspace 和 policy | +| Agent | 基于权限读取 context pack，写入观察和记忆 | recall、search、observe、remember | 必须经过 agent visibility 过滤 | +| 管理员 | 管理权限、审计、冲突和遗忘 | policy、audit、conflict、forget request | 操作写入 audit | +| 访客 / 只读用户 | 查看公开 Wiki 或演示结果 | 浏览和搜索公开内容 | 不能编辑或查看审计详情 | + +### 3.2 功能需求 + +| 模块 | 功能 | 优先级 | +|---|---|---| +| Workspace 基础配置 | 初始化 workspace、维护成员和边界 | P0 | +| Source 导入 | 导入 Markdown / txt，切分 chunk | P0 | +| Memory 管理 | 创建、编辑、删除 memory | P0 | +| Evidence 追溯 | memory 绑定 source chunk | P0 | +| Revision / Audit | 修改自动记录版本和审计 | P0 | +| Recall 检索 | keyword / FTS / hybrid recall | P0/P1 | +| Markdown Wiki | 导出 WikiPage 和版本 | P0 | +| AccessPolicy | Agent 权限过滤 | P1 | +| ConflictRecord | 冲突记忆治理 | P1 | +| ForgetRequest | 遗忘与归档审批 | P1 | +| Graph Explorer | 展示 source/memory/evidence/wiki/governance 关系 | P1 | +| Agent Runtime / CLI | sessions、observe、remember、search、recall | P1 | +| Evaluation | 长期记忆评测适配器、指标和报告 | P1 | +| QA | 基于 recall context 的可选 LLM answer | P2 | + +### 3.3 非功能需求 + +| 类别 | 要求 | 实现方式 | +|---|---|---| +| 数据完整性 | 不允许孤立 evidence、revision | 主键、外键、UNIQUE、CHECK | +| 可追溯性 | 每条 memory 可追到 source chunk | `memory_evidence`、`v_memory_with_source` | +| 权限控制 | Agent 只能看授权范围 | `access_policy`、`v_agent_visible_memory` | +| 可审计性 | 关键操作可回放 | `audit_log`、trigger、before/after JSON | +| 可降级性 | 没有 LLM / embedding 也能演示 | rule-based extraction、keyword fallback | +| 可演示性 | 5-8 分钟讲清核心闭环 | seed + governance fixture + screenshot assets | +| 性能需求 | 课程规模下秒级查询 | B+ tree、GIN、BRIN、covering index | + +## 4. 数据流设计 + +### 4.1 0 层数据流图 + +外部实体包括普通用户/小组成员、Agent、管理员、Markdown/会议纪要文件和 Markdown Wiki 文件系统。核心内容是 MemoryBase 长期记忆数据库系统。主要数据存储包括 Source Store、Memory Store、Governance Store、Wiki Store 和 Audit Store。 + +```text +User / Agent / Admin / Source files + -> MemoryBase + -> Source Store / Memory Store / Governance Store / Wiki Store / Audit Store + -> Markdown Wiki / Context Pack / UI / SQL evidence +``` + +### 4.2 1 层数据流 + +```text +Markdown / txt 文件 + -> Source Ingest + -> SourceDocument / SourceChunk + -> Memory Extract / Manual Edit + -> MemoryItem / MemoryEvidence + -> Semantic Organization + -> Entity / MemoryScene + -> Recall Search + -> Context Pack + -> Wiki Export + -> WikiPage / WikiPageRevision + +关键操作 + -> Governance + -> AuditLog / MemoryRevision / RecallLog / AccessPolicy +``` + +### 4.3 关键 2 层流程 + +Source 导入流程： + +```text +用户上传文件 + -> 校验文件类型 + -> 计算 checksum + -> 保存 SourceDocument + -> 按行数 / token 切分 + -> 保存 SourceChunk + -> 通过 imported_at / imported_by_user_id 保留导入元数据 +``` + +Recall 检索流程： + +```text +用户 / Agent 输入 query + -> 解析 query 和过滤条件 + -> 应用 public/project 或 v_agent_visible_memory 过滤 + -> 查询 SourceChunk FTS / trigram + -> 查询 MemoryItem 文本匹配 + -> 可选查询 MemoryEmbedding / SourceChunkEmbedding + -> Join MemoryEvidence / SourceChunk / SourceDocument + -> 返回 memory + evidence + source + -> 写入 RecallLog +``` + +Wiki 导出流程： + +```text +选择 scene / memory / workspace + -> 查询 MemoryItem + -> 查询 MemoryEvidence + -> 查询 SourceChunk + -> 生成 Markdown frontmatter + -> 写入 WikiPage + -> 写入 WikiPageRevision + -> 导出 data/markdown_wiki/ +``` + +## 5. 数据字典 + +### 5.1 核心数据项 + +| 数据项 | 含义 | 类型 | 约束示例 | +|---|---|---|---| +| `user_id` | 用户编号 | UUID | PK | +| `agent_id` | Agent 编号 | UUID | PK | +| `workspace_id` | 工作区编号 | UUID | PK / FK | +| `doc_id` | 源文档编号 | UUID | PK | +| `chunk_id` | 文档块编号 | UUID | PK | +| `memory_id` | 记忆项编号 | UUID | PK | +| `evidence_id` | 证据编号 | UUID | PK | +| `revision_no` | 版本号 | INT | 与 memory/page 组成复合 PK | +| `audit_id` | 审计日志编号 | UUID | PK | +| `access_level` | 访问范围 | VARCHAR | public/project/team/private | +| `memory_type` | 记忆类型 | VARCHAR | episodic/semantic/fact/profile/procedural/decision/preference/task/risk/constraint/policy/summary | +| `status` | 记忆状态 | VARCHAR | candidate/active/archived/forgotten/superseded/rejected/conflicted | + +### 5.2 主要数据结构 + +| 数据结构 | 组成 | 说明 | +|---|---|---| +| `SourceDocument` | doc_id、workspace_id、doc_type、title、raw_text、checksum、status、imported_at | 原始文档 | +| `SourceChunk` | chunk_id、doc_id、chunk_no、chunk_text、line range、search_vector | 文档切块 | +| `MemoryItem` | memory_id、workspace_id、memory_type、canonical_text、status、access_level、valid_from/valid_to | 长期记忆核心 | +| `MemoryEvidence` | evidence_id、memory_id、chunk_id、role、weight、note | 记忆来源证据 | +| `MemoryRevision` | memory_id、revision_no、text、summary、reason、editor | 记忆版本历史 | +| `AuditLog` | actor、action、target、before_json、after_json、created_at | 操作审计 | +| `AccessPolicy` | principal、resource、effect、scope | 权限策略 | +| `WikiPage` / `WikiPageRevision` | page metadata + markdown body | 人类可读投影 | +| `MemoryEmbedding` / `SourceChunkEmbedding` | provider、model、dimension、embedding_json、hash | 可选向量缓存 | + +完整数据字典见 `docs/03-data-dictionary.md`。 + +## 6. 概念结构设计与 E-R 图 + +### 6.1 概念分层 + +| 层次 | 主要实体 | 作用 | +|---|---|---| +| 源文档层 | SourceDocument、SourceChunk、AgentSession、Message | 保存原始输入 | +| 记忆层 | MemoryItem、MemoryRevision、MemoryScene、MemoryEmbedding | 保存长期记忆、版本和可选向量 | +| 证据层 | MemoryEvidence、Entity、MemoryEntity | 绑定记忆与来源，支持语义组织 | +| 表达层 | WikiPage、WikiPageRevision、TimelineEntry | 生成 Wiki 和时间线 | +| 治理层 | AccessPolicy、RecallLog、ConflictRecord、ForgetRequest、AuditLog | 权限、审计、冲突、遗忘 | + +### 6.2 核心 E-R 图 + +![MemoryBase core ER](final-assets/diagrams/01-er-core.svg) + +完整 25 实体 ER 图见 `docs/final-assets/diagrams/02-er-full.svg`。核心关系包括： + +| 联系 | 类型 | 转换方式 | +|---|---|---| +| Workspace - SourceDocument | 1:N | `source_document.workspace_id` | +| SourceDocument - SourceChunk | 1:N | `source_chunk.doc_id` | +| SourceChunk - MemoryItem | M:N | `memory_evidence` | +| MemoryItem - MemoryRevision | 1:N | `memory_revision.memory_id` | +| MemoryItem - Entity | M:N | `memory_entity` | +| MemoryScene - MemoryItem | M:N | `memory_scene_cell` | +| WikiPage - WikiPageRevision | 1:N | `wiki_page_revision` | +| Workspace - AccessPolicy / AuditLog / RecallLog | 1:N | workspace-scoped governance tables | + +### 6.3 设计说明 + +E-R 设计围绕以下生命周期组织： + +```text +ingest -> extract -> evidence -> revise -> retrieve -> govern -> project -> forget +``` + +其中 `MemoryEvidence` 是最关键的 M:N 关联：一条 memory 可以由多个 source chunk 支撑，一个 source chunk 也可以支撑多个 memory。它让系统从“文本检索”升级为“证据链建模”。 + +## 7. 逻辑结构设计与范式分析 + +### 7.1 关系模式 + +主要关系模式包括： + +```text +UserAccount(user_id PK, username UNIQUE, display_name, email UNIQUE, role_hint, created_at, updated_at) + +Workspace(workspace_id PK, slug UNIQUE, name, description, scope_type, owner_user_id FK, created_at, updated_at) + +Agent(agent_id PK, workspace_id FK, name, agent_type, status, owner_user_id FK, created_at) + +SourceDocument(doc_id PK, workspace_id FK, session_id FK, doc_type, title, source_path, raw_text, checksum, status, forgotten_at, imported_by_user_id FK, imported_at) + +SourceChunk(chunk_id PK, doc_id FK, chunk_no, chunk_text, start_line, end_line, token_count, search_text_zh, search_vector, UNIQUE(doc_id, chunk_no)) + +MemoryItem(memory_id PK, workspace_id FK, created_from_doc_id FK, memory_type, canonical_text, summary, search_text_zh, search_vector, confidence, importance, status, access_level, owner_user_id FK, owner_agent_id FK, valid_from, valid_to, superseded_by_memory_id FK, current_revision_no, created_at, updated_at) + +MemoryEvidence(evidence_id PK, memory_id FK, chunk_id FK, evidence_role, weight, note, created_at, UNIQUE(memory_id, chunk_id, evidence_role)) + +MemoryRevision(memory_id FK, revision_no, revision_text, revision_summary, revision_reason, editor_type, editor_id, created_at, PK(memory_id, revision_no)) + +WikiPage(page_id PK, workspace_id FK, page_slug, page_type, title, current_revision_no, generated_from_scene_id FK, generated_from_memory_id FK, needs_rebuild, status, forgotten_at, created_at, updated_at) + +WikiPageRevision(page_id FK, revision_no, frontmatter_json, body_markdown, generated_by, created_at, PK(page_id, revision_no)) + +AccessPolicy(policy_id PK, workspace_id FK, principal_type, principal_id, resource_type, resource_scope, effect, predicate_json, created_at) + +ConflictRecord(conflict_id PK, workspace_id FK, left_memory_id FK, right_memory_id FK, conflict_type, status, resolution_note, resolved_by_actor_type, resolved_by_actor_id, resolved_at, created_at, updated_at) + +ForgetRequest(request_id PK, workspace_id FK, target_type, target_id, requester_user_id FK, reviewed_by_user_id FK, reason, status, requested_at, resolved_at) + +AuditLog(audit_id PK, workspace_id FK, actor_type, actor_id, action_type, target_type, target_id, before_json, after_json, created_at) +``` + +完整关系模式见 `docs/05-logical-design.md`。 + +### 7.2 E-R 到关系模型转换 + +1:N 关系通过在 N 端保存外键实现，例如 `source_document.workspace_id`、`source_chunk.doc_id`、`memory_revision.memory_id`。M:N 关系通过中间表实现，例如 `memory_evidence`、`memory_entity`、`memory_scene_cell`。版本化实体采用复合主键，例如 `memory_revision(memory_id, revision_no)` 和 `wiki_page_revision(page_id, revision_no)`。 + +### 7.3 范式分析 + +核心业务表以 3NF / BCNF 为主： + +- 每张表有明确主键或复合主键； +- 非主属性直接依赖候选键； +- M:N 关系拆分为关联表； +- 版本内容从主表拆到 revision 表； +- 权限、审计、冲突、遗忘独立成治理表。 + +少数受控反规范化是有意设计： + +| 字段 | 所在表 | 原因 | 一致性维护 | +|---|---|---|---| +| `search_vector` | `memory_item` / `source_chunk` | 支撑 GIN 全文检索 | GENERATED column | +| `current_revision_no` | `memory_item` / `wiki_page` | 快速读取当前版本 | trigger | +| `needs_rebuild` | `wiki_page` | Wiki dirty bit | trigger | +| `workspace_id` | `memory_entity` / `memory_scene_cell` | 支撑复合 FK 保证同租户 | FK constraint | +| `embedding_json` | embedding tables | 缓存模型输出向量 | provider/model/hash 去重 | + +完整函数依赖和范式判定见 `docs/normalization.md`。 + +## 8. 物理结构设计 + +### 8.1 数据库选择 + +主数据库采用 PostgreSQL。早期曾考虑 SQLite + FTS5 作为本地备选，但当前仓库脚本、测试和演示均以 PostgreSQL 为准，SQLite 不作为已交付能力。 + +选择 PostgreSQL 的原因： + +- 支持复杂外键、CHECK、UNIQUE 和复合 FK； +- 支持 JSONB、GIN、BRIN、covering index、partial index； +- 支持 PL/pgSQL trigger； +- 适合展示数据库课程中的视图、索引、触发器和 EXPLAIN； +- 能承载多 workspace、多 user、多 agent 的组织级数据模型。 + +### 8.2 存储路径 + +```text +project-root/ + database/ SQL schema, indexes, views, triggers, seed + backend/app/ FastAPI backend, service/repository, CLI + frontend/src/ React frontend + evaluation/ benchmark adapters, metrics, runners, reports + data/ + raw_sources/ + uploads/ + markdown_wiki/ + exports/ + backups/ + logs/ + docs/final-assets/ diagrams, screenshots, command logs +``` + +### 8.3 索引设计 + +项目使用多类 PostgreSQL 索引： + +| 类别 | 用途 | 示例 | +|---|---|---| +| B+ tree / B-link tree | workspace 过滤、排序、主键/外键查找 | `idx_memory_workspace_status` | +| 复合索引 | 租户隔离 + 状态/类型筛选 | `idx_memory_workspace_type_status` | +| DESC 排序索引 | 时间线、审计、recall 最近记录 | `idx_audit_workspace_time` | +| GIN FTS | `tsvector @@ tsquery` 全文检索 | `idx_memory_fts`、`idx_source_chunk_fts` | +| GIN trigram | `ILIKE '%keyword%'` 模糊匹配 | `idx_memory_canonical_text_trgm` | +| Partial index | active hot path / NULL 策略去重 | `idx_policy_global_unique` | +| BRIN | append-only audit 时间窗分析 | `idx_audit_brin_time` | +| Covering index | index-only scan | `idx_memory_active_ranking` | +| Embedding metadata index | provider/model 缓存定位 | `idx_memory_embedding_workspace_model` | + +课堂上讲的 B+ tree 与 PostgreSQL 文档中的 B-tree 名称需要解释：PostgreSQL 实际实现是 Lehman-Yao B-link tree，是 B+ tree 的并发变种，叶子层保存 row pointer 并支持范围扫描。因此报告中可把 PostgreSQL 默认 btree 作为 B+ tree 变种说明。 + +### 8.4 EXPLAIN 证据 + +`docs/explain-analyze.md` 收集了 4 个实际 EXPLAIN ANALYZE 案例： + +1. `idx_memory_active_ranking` 支撑 active memory 排序，出现 `Index Only Scan` 和 `Heap Fetches: 0`； +2. Recall 主查询使用 GIN tsvector + GIN trigram，并在强制索引路径下出现 `BitmapOr`； +3. `audit_log` 时间窗聚合展示 B-tree 竞争路径与 BRIN 路径； +4. Wiki provenance join 展示多跳来源追溯。 + +这些案例用于证明索引不是“写在 SQL 文件里”，而是能通过 PostgreSQL planner 的真实执行计划验证。 + +### 8.5 视图设计 + +| 视图 | 作用 | +|---|---| +| `v_active_memory` | 查询 active 且仍在有效期内的 memory | +| `v_memory_with_source` | 串联 memory、evidence、chunk、source，支撑 provenance | +| `v_agent_visible_memory` | 展开每个 Agent 可见的 active memory | +| `v_project_timeline` | 串联 timeline、memory 和 source | +| `v_conflict_memory` | 展开 conflict 两端 memory | +| `v_wiki_page_sources` | 追溯 WikiPage 到 source chunk | +| `v_memory_statistics` | 按 workspace、类型、状态、访问级别统计 | +| `v_memory_recall_statistics` | 从 recall log 反查 recall_count 和 last_recalled_at | + +### 8.6 触发器设计 + +| 触发器 | 用途 | +|---|---| +| `trg_memory_before_update` | 同一事务内递增 revision、维护状态变化前置逻辑 | +| `trg_memory_after_insert` | memory 创建后写初始 revision 和 audit | +| `trg_memory_after_update` | memory 修改后写 revision、audit，并标记 Wiki rebuild | +| `trg_memory_soft_delete` | DELETE memory 转换为 archived | +| `trg_wiki_revision_after_insert` | Wiki revision 插入后同步 current revision | +| `trg_conflict_after_insert` / `trg_conflict_after_update` | open conflict 自动标记 memory 为 conflicted，解决后恢复 | + +这些触发器让 revision、audit、soft delete 和 conflict lifecycle 不依赖应用层自觉执行。 + +## 9. 系统总体架构 + +### 9.1 总体架构 + +```text +Human User / Admin / Agent / Evaluation Runner + | + v +React Frontend MemoryBase CLI (`mb` / `memorybase`) + \ / + v v + FastAPI Backend + | + v + API layer (`backend/app/api`) + | + v + Service + Repository layer + | + v + PostgreSQL core database <---- optional ----> Neo4j graph cache + | + v + Markdown Wiki files / evaluation reports / screenshots +``` + +设计重点是 DB-first：所有长期记忆、证据、权限、版本、审计、召回日志和 Wiki 投影都先进入 PostgreSQL。前端、CLI、Agent、Graph Explorer 和 evaluation runner 是围绕同一事实源的不同入口。 + +### 9.2 后端模块 + +后端采用 FastAPI router + service/repository 分层： + +| 层 | 主要文件 | 作用 | +|---|---|---| +| API layer | `backend/app/api/*.py` | 定义 HTTP endpoints 和 Pydantic contract | +| Service layer | `backend/app/services/*.py` | 封装 source、memory、recall、governance、graph、embedding、QA 等业务规则 | +| Core | `backend/app/core/*.py` | 配置、数据库连接 | +| CLI | `backend/app/cli/` | `mb` / `memorybase` command line | +| Tests | `backend/tests/` | API、service、evaluation、PostgreSQL integration 测试 | + +### 9.3 前端页面 + +前端按工作流分组： + +| 分组 | 页面 | +|---|---| +| Dashboard | `Dashboard.jsx` | +| Sources | Source list/detail | +| Memories | Memory list/detail/create/edit | +| Recall | Recall / Context Pack / QA | +| Wiki | Wiki export | +| Governance | Policies、Audit、Conflicts、ForgetRequests、Timeline | +| Runtime | Sessions、Messages、HybridSearch | +| Graph | GraphExplorer、GraphSvg | + +### 9.4 CLI 与 Agent Runtime + +`pyproject.toml` 暴露 `memorybase` 和 `mb` 命令。CLI 覆盖 configure、health、context、recall、search、observe、remember、sessions 和 eval。Agent 可以通过 search/recall 获取上下文，通过 observe/remember 写入消息和 memory；Agent 写回 memory 时即使没有显式 evidence，后端也会创建 `inline_agent_note` source chunk，保证 provenance 不断裂。 + +## 10. API 与模块 IPO + +### 10.1 API 摘要 + +| 模块 | 端点示例 | 作用 | +|---|---|---| +| Health | `GET /api/health` | 服务和数据库状态 | +| Source | `POST /api/sources`、`GET /api/sources/{id}` | source 导入、列表、详情 | +| Memory | `POST /api/memories`、`PATCH /api/memories/{id}`、`POST /api/memories/batch` | memory CRUD、批量创建、supersession | +| Memory Extraction | `POST /api/memory-extraction/from-chunks`、`/api/memory-candidates/{memory_id}/approve` | rule-based / optional LLM candidate extraction 和审批 | +| Recall | `POST /api/recall`、`POST /api/recall/context-pack` | keyword/vector/hybrid recall 和 context pack | +| Search | `POST /api/search` | lexical search | +| Wiki | `POST /api/wiki/export` | Markdown Wiki 投影 | +| Governance | `/api/audit`、`/api/policies`、`/api/conflicts`、`/api/forget-requests`、`/api/timeline` | 审计、权限、冲突、遗忘、时间线 | +| Runtime | `/api/sessions`、`/api/observe` | Agent Runtime 会话与消息 | +| Graph | `/api/graph/*` | PostgreSQL graph preview、Neo4j sync/load | +| Embeddings / QA | `/api/embeddings/*`、`/api/qa/answer` | 可选 embedding 和 LLM answer | + +完整 API 设计见 `docs/08-api-design.md`。 + +### 10.2 模块 IPO 表 + +| 模块 | Input | Process | Output | +|---|---|---|---| +| Source / Ingest | Markdown / txt / meeting text | 校验、checksum、切 chunk、生成 search text | SourceDocument、SourceChunk | +| Memory / Evidence | chunk、表单、Agent 写回 | 创建 memory、绑定 evidence、必要时创建 inline evidence | MemoryItem、MemoryEvidence | +| Memory Extraction | workspace_id、chunk_ids、max_candidates、method、llm options | rule-based 或 optional LLM 抽取、分类、创建 candidate、记录 run audit | Candidate Memory | +| Recall | query、filters、agent_id、retrieval_mode | visibility filter、FTS/trigram、可选 embedding、fallback metadata | RecallResponse、Context Pack | +| Policy / Visibility | principal、resource、scope、effect | allow/deny 策略和 per-agent visibility view | AccessPolicy、visible memory | +| Wiki | memory / scene / workspace | 渲染 Markdown frontmatter/body、写 revision | WikiPage、Markdown 文件 | +| Conflict Governance | 两条 memory | 创建/更新 conflict，trigger 标记状态 | ConflictRecord、AuditLog | +| Forget Governance | target、reason、reviewer | 审批、软遗忘、验证、审计 | ForgetRequest、target status | +| Graph Explorer | workspace_id、agent_id | PostgreSQL 构图、可选 Neo4j sync | Graph nodes/edges | +| Evaluation | dataset adapter、runner config | 加载 benchmark case、运行 metrics、生成 report | Evaluation report | + +## 11. 系统实现 + +### 11.1 Source 到 Wiki 端到端实现 + +![Source to Wiki sequence](final-assets/diagrams/03-source-to-wiki.svg) + +端到端链路包括： + +1. `POST /api/sources` 导入原始文档； +2. service 计算 checksum 并切分 source chunk； +3. 手动创建 memory 或从 chunk 通过 rule-based / optional LLM 生成 candidate； +4. approve candidate 后进入 active lifecycle； +5. `memory_evidence` 保证 memory 可追溯； +6. trigger 写 revision / audit； +7. Wiki service 将 memory/scene 投影为 Markdown。 + +### 11.2 Recall 与 Context Pack + +![Recall sequence](final-assets/diagrams/04-recall.svg) + +Recall 支持 keyword、vector、hybrid 三种模式。未配置 embedding 或没有 embedding records 时，hybrid 会透明降级到 keyword，并在 `retrieval_info` 中返回 requested mode、effective mode、candidate count 和 fallback reason。前端 Recall 页面默认使用 keyword 以保证 demo 稳定，后端 API 默认保留 hybrid 合约。 + +### 11.3 当前已实现的 AI 功能与边界 + +当前项目已经实现的 AI 相关能力，主要集中在“可选增强层”而不是“事实源主链路”： + +1. `Memory Extraction` 支持两条候选抽取路径：默认 `rule_based`，以及 `method='llm'` 的 optional OpenAI-compatible analysis path。 +2. 两条路径都会先写入 `memory_item(status='candidate')`，并绑定 `memory_evidence`、记录 `memory_extraction.run.start/complete` 审计事件；新候选不会直接进入默认 recall，仍需人工 approve / reject。 +3. `Recall` 支持 keyword / vector / hybrid 三种模式；embedding 默认是本地 hashing cache，未配置外部 embedding provider 或没有匹配 embedding records 时，会透明 fallback 到 keyword。 +4. `QA` 支持基于 recall context 的 optional LLM answer path；只有配置兼容 provider 后才启用，不影响离线 demo 主路径。 +5. 前端 `Sources -> Source Detail` 已支持勾选 `Use LLM` 触发候选抽取；CLI 也提供 `mb extract --method llm` 入口。 + +这部分能力的设计边界也需要明确说明： + +- 默认演示链路仍是本地可运行的 `rule_based extraction + keyword recall`，不依赖外部 API key。 +- LLM 当前只参与“候选记忆草拟”和“可选 QA 回答”，不直接写入 `active` facts，也不绕过 evidence / approval / audit。 +- 当前尚未实现更完整的 `analysis_run`、`analysis_memory_draft`、`analysis_draft_evidence` 草稿表工作流；optional LLM extraction 是在现有 candidate lifecycle 上的增强，而不是替代现有 schema。 + +### 11.4 Governance Lifecycle + +![Governance sequence](final-assets/diagrams/05-governance.svg) + +治理链路覆盖 memory update/delete、conflict create/resolve、forget request approve/verify、wiki revision 等事件。数据库触发器负责 revision、audit、soft delete 和 conflict status，同步到前端 governance 页面和 SQL demo 查询。 + +### 11.5 Memory 状态机 + +![Memory status lifecycle](final-assets/diagrams/06-memory-status.svg) + +`memory_item.status` 当前支持 7 个状态： + +```text +candidate -> active / rejected +active -> archived / forgotten / superseded / conflicted +conflicted -> active / superseded / forgotten +archived -> forgotten +``` + +状态转换由 service 校验，数据库 trigger 负责记录 revision 和 audit。 + +## 12. 系统演示 + +### 12.1 演示准备 + +```bash +npm run db:setup +``` + +该命令会重建 public schema，加载 `database/00_init.sql` 到 `database/07_seed.sql`，执行 search backfill，加载 `database/10_governance_demo_fixture.sql`，并运行 `database/08_demo_queries.sql`。演示数据包含 source、memory、evidence、revision、audit、wiki、policy、conflict、forget request 等对象。 + +### 12.2 演示路径 + +1. 打开 Dashboard，展示 workspace 统计。 +2. 进入 Sources，展示 seeded discussion documents。 +3. 打开 Source detail，查看 chunk 行号并选择待抽取的 chunk。 +4. 先运行默认 rule-based candidate extraction，再勾选 `Use LLM` 运行 optional LLM extraction，对比候选类型与置信度。 +5. 进入 Memories，展示 memory type/status/importance。 +6. 打开 Memory detail，展示 evidence 和 revision。 +7. 在 Recall 搜索“为什么放弃校园食堂系统”。 +8. 生成 Context Pack，展示 Agent-ready Markdown。 +9. 如配置 LLM provider，点击 Ask，展示 optional QA answer。 +10. 展示 Governance：audit、policies、conflicts、forget requests、timeline。 +11. 展示 Graph Explorer。 +12. 展示 SQL 查询结果和 EXPLAIN 证据。 + +### 12.3 具体演示示例 + + + +#### 12.3.1 Source 详情、chunk 行号与候选抽取入口 + +![Source detail with chunk selection and LLM toggle](final-assets/screenshots/ui/03-source-detail-project-pivot.png) + +*图 12-1 Source detail 页面同时展示原始文本、chunk 行号，以及 `Use LLM` 候选抽取入口。* + +这张图说明 source 导入后并不是黑盒切分；用户可以明确看到文档内容、chunk 的边界与行号，并在同一页面选择哪些 chunk 进入 candidate extraction 流程。 + +#### 12.3.2 Memory detail 的 evidence 与 revision + +![Memory detail with evidence and revisions](final-assets/screenshots/ui/05-memory-detail-evidence-revisions.png) + +*图 12-2 Memory detail 页面展示 memory 内容、evidence、revision、entity 和 scene。* + +这张图对应本系统“provenance-first”的核心设计：每条 memory 都能向下追溯到 source chunk，并能向后查看 revision 历史，而不是只保留一段不可解释的摘要文本。 + +#### 12.3.3 Recall 结果与 retrieval metadata + +![Recall search results](final-assets/screenshots/ui/06-recall-search-results.png) + +*图 12-3 Recall 页面返回与“为什么放弃校园食堂系统”相关的记忆结果，并显示 requested / effective retrieval mode。* + +该图展示了数据库事实源上的 recall 能力：不仅返回答案相关 memory，还把 `retrieval_info` 暴露给前端，说明实际使用了什么检索模式，以及是否发生了 fallback。 + +#### 12.3.4 Context Pack 投影 + +![Context pack view](final-assets/screenshots/ui/07-recall-context-pack.png) + +*图 12-4 Recall 结果可进一步投影为 Agent-ready Markdown Context Pack。* + +这张图说明 Recall 并不止于“查到结果”，而是能够把 memory、evidence、citation 和 retrieval metadata 组织成可供 Agent 继续使用的上下文包。 + +#### 12.3.5 Optional LLM candidate extraction 对比 + +为验证 optional LLM extraction 的价值，我们对同一组三条 chunk 分别运行默认 rule-based 路径和 Qwen-compatible analysis path。 + +![Rule-based candidate extraction](final-assets/screenshots/ui/19-llm-not-used.png) + +*图 12-5 默认 rule-based candidate extraction 结果。* + +![Qwen LLM candidate extraction](final-assets/screenshots/ui/19-llm-used.png) + +*图 12-6 启用 Qwen-compatible optional LLM extraction 后的结果。* + +| Chunk 内容（缩写） | Rule-based 结果 | Qwen LLM 结果 | +|---|---|---| +| `abandoned cafeteria` | `decision`，confidence `0.65`，importance `4` | `decision`，confidence `0.90`，importance `4` | +| `PostgreSQL source of truth` | `task`，confidence `0.65`，importance `3` | `decision`，confidence `0.90`，importance `5` | +| `private budget notes hidden` | `constraint`，confidence `0.65`，importance `4` | `policy`，confidence `0.90`，importance `4` | + +这组对比很好地验证了 rule-based 路径的局限与 optional LLM 路径的优势。第一句两条路径都能识别为 `decision`，但 LLM 给出了更高的 confidence；第二句没有再被 `should` 误判成 `task`，而是更准确地识别为架构 / 数据库方向的 `decision`；第三句也从较宽泛的 `constraint` 收敛为更贴近权限治理语义的 `policy`。同时，两条路径都只把结果写入 `status='candidate'`，仍然需要人工 approve / reject，因此语义增强并没有破坏 provenance、audit 和 human-in-the-loop 的主链路。 + +#### 12.3.6 Optional QA answer + +![Optional QA answer](final-assets/screenshots/ui/08-recall-qa-answer-or-config-state.png) + +*图 12-7 配置 optional LLM provider 后，Recall 页面可直接生成带引用的回答。* + +这里展示的是基于 recall context 的 optional QA path。它不是演示主链路所必需的能力，但在配置兼容 provider 后，可以把已召回的 memory 组织成最终回答，并显示使用的模型和 supporting memories 数量。 + +#### 12.3.7 Governance 冲突治理 + +![Governance conflicts](final-assets/screenshots/ui/12-governance-conflicts.png) + +*图 12-8 Governance 页面展示 open / resolved conflict 的生命周期。* + +这张图说明 conflict governance 并不是文档中的概念，而是落在了可浏览、可 resolve / ignore / reopen 的具体工作流中，并与后端 trigger / audit 逻辑对应。 + +#### 12.3.8 Graph Explorer + +![Graph Explorer demo workspace](final-assets/screenshots/ui/15-graph-explorer-demo-workspace.png) + +*图 12-9 Graph Explorer 基于 PostgreSQL 事实源渲染 workspace knowledge graph。* + +该图一方面展示了 source、chunk、entity、scene、wiki 等节点的可视化关系，另一方面也清楚显示 Neo4j 是 optional 的；即使 Neo4j disabled，PostgreSQL preview 仍可独立完成图谱展示。 + +完整截图索引见 `docs/final-assets/screenshots/README.md`。 + +## 13. 测试与评估 + +### 13.1 自动化测试 + +当前测试覆盖： + +| 类别 | 测试文件示例 | 覆盖目标 | +|---|---|---| +| API / service | `test_sources.py`、`test_memories.py`、`test_recall_query.py` | source、memory、recall | +| Governance | `test_governance.py`、`test_graph_service.py` | policy、audit、conflict、forget、graph visibility | +| CLI / Agent Runtime | `test_cli_context.py`、`test_cli_recall.py`、`test_cli_sessions.py`、`test_cli_writeback.py` | context、recall、observe、remember | +| Evaluation | `test_evaluation_*` | adapters、metrics、judging、checkpoint、report | +| PostgreSQL integration | `test_postgres_integration.py` | schema、seed、trigger、batch create、supersession | +| Frontend | `npm run lint`、`npm run build` | React lint 和 production build | + +最终提交前推荐命令： + +```bash +uv run ruff check backend/app backend/tests evaluation +uv run --with pytest python -m pytest backend/tests -q +cd frontend && npm run lint && npm run build +git diff --check +``` + +最近一次合并后的验证记录包括： + +- `uv run ruff check backend/app backend/tests evaluation` 通过； +- `.venv/bin/python -m pytest backend/tests -q`：197 passed, 1 skipped； +- PostgreSQL batch/supersession focused integration tests：2 passed, 41 deselected。 + +### 13.2 SQL 演示测试 + +`database/08_demo_queries.sql` 提供课程演示 SQL，覆盖 provenance、agent visibility、audit/revision、conflict、forget、statistics、wiki sources 等路径。`docs/final-assets/screenshots/logs/` 保存完整命令输出，PNG/SVG 是从完整日志渲染得到，避免终端截图截断。 + +### 13.3 Evaluation 结果与解释 + +Evaluation framework 覆盖 datasets、adapters、metrics、runners 和 reports。当前已完成一次 LongMemEval oracle 500-case live `db_qa` 运行： + +| Metric | Result | +|---|---:| +| Cases | 500 | +| API errors | 0 | +| Deterministic pass | 176 / 500 (35.2%) | +| Semantic judge pass | 292 / 500 (58.4%) | +| Semantic judge fail | 208 / 500 (41.6%) | + +分类结果显示系统在 abstention、temporal_update、single_fact 上表现较好，在 multi_session reasoning 和 preference_following 上较弱。这一结果应作为工程验证和限制分析，而不是榜单宣传。MemoryBase 的主价值是：用数据库事实源、provenance、governance、agent visibility 和 audit 支撑长期记忆任务，并用 evaluation 暴露真实边界。 + +## 14. 创新点总结 + +### 14.1 DB-first AI substrate + +MemoryBase 把 AI 长期记忆从模型或向量库的黑盒能力转化为 PostgreSQL 中可验证的关系模型。LLM、embedding、Graph 和 evaluation 都是围绕数据库事实源的增强层。 + +### 14.2 Provenance-first 证据链 + +`memory_evidence`、`v_memory_with_source`、`v_wiki_page_sources`、`recall_log.context_pack_json` 让系统能回答“这条记忆来自哪里”“这次 recall 用了哪些 memory”“Wiki 页面引用了哪些 source”。 + +### 14.3 Governance-by-default 生命周期 + +revision、audit、conflict、forget、policy 被设计为核心表和触发器逻辑，不是 UI 装饰。状态变更、软删除和冲突 lifecycle 都可被 SQL 验证。 + +### 14.4 Agent-aware visibility + +Agent 不是普通用户别名，而是独立 principal。系统通过 `access_policy` 和 `v_agent_visible_memory` 把权限过滤放入数据库查询路径，而不是依赖 prompt 中的自律。 + +### 14.5 Transparent hybrid retrieval + +系统支持 keyword/vector/hybrid recall，但不假装向量永远可用。无 embedding 时明确 fallback 到 keyword，并记录在 `retrieval_info` 和 `recall_log` 中。 + +### 14.6 AI-assisted candidate extraction + +MemoryBase 先用 rule-based 路径交付了稳定的 candidate extraction，随后又增加了 optional LLM analysis path。两条路径都会把候选记忆写入 `memory_item(status='candidate')`，并绑定 evidence、记录 run-level audit。这样既避免了“LLM 直接写事实”的风险，也避免了“大量文档完全靠人工处理”的低效。 + +### 14.7 Graph as provenance visualization + +Graph Explorer 用 PostgreSQL 构建 workspace graph，并可选同步 Neo4j。图层服务关系展示和 provenance 可视化，不替代 PostgreSQL 事实源。 + +### 14.8 Evaluation-backed validation + +Evaluation framework 让系统不只做 UI demo，还能用 LoCoMo / LongMemEval / MemoryAgentBench 等基准检查长期记忆行为。当前低分项也帮助明确 future work。 + +## 15. 小组分工与个人完成情况 + +> 本节基于 `docs/contribution-ledger.md` 在 2026-06-15 的 Git commit 审计刷新。 + +| 成员 | 主要负责 | 交付成果 | +|---|---|---| +| 林纪帆 (hopecommon / jflin) | 数据库 schema、治理与溯源模型、lexical search / context-pack formatter / CLI dogfood、Graph hardening、审查合并/集成补强、最终报告证据资产 | SQL schema、views、triggers、indexes、seed/demo data；governance/provenance workflows；CLI recall/context/eval/sessions/observe/remember；Graph visibility/sync/audit hardening；ER/sequence diagrams；截图、final report 和 final defense slides | +| 杜卓轩 (dzx0902 / dzx) | P0 backend/tests、evaluation benchmark framework、LoCoMo / LongMemEval / MemoryAgentBench adapters、LongMemEval full-run evidence、embedding/hybrid recall、memory extraction、QA、CI/tooling | FastAPI 后端基础、测试体系、evaluation datasets/adapters/metrics/runners/reports、semantic judging、checkpoint/resume、embedding/cache/hybrid recall、batch memory write、supersession | +| 李昭成 (huiyijian / lywzc0419 / lzc) | 前端 API 集成、Memory / Recall / Governance 等核心页面交互实现，负责冲突治理、权限策略、审计日志等 Runtime / Governance UI；集成 Neo4j Graph Explorer，将 PostgreSQL 中的 memory、source、evidence 等关系投影为 provenance 图谱，用于记忆来源追踪与图可视化展示。 | Dashboard、sources、memories、recall、governance、wiki、runtime、graph 等页面联调；graph API/service/model、Neo4j demo SQL、依赖配置和 lint/import 修复 | +| 王星睿 (Cofstars) | source extraction workflow、chunking/test additions、source UI creation/detail flow、optional LLM-backed candidate extraction、相关 CLI/API/tests、final report 与 PPT | source extraction candidate workflow；chunking 逻辑与测试；`SourceCreate.jsx` / `SourceDetail.jsx` 抽取交互；`backend/app/services/llm_analysis.py`；`mb extract` CLI；LLM memory extraction API/service/tests；final report 与 final defense slides | + +分工边界说明：recall/search/context pack 是协作模块。hopecommon 主要负责 lexical search、context-pack formatter 和 CLI/dogfood；dzx0902 主要负责 hybrid recall、embedding、QA 和 structured context metadata；Graph 初版由 lywzc0419 集成，后续 visibility/sync/audit hardening 由 hopecommon 完成；Cofstars 工作集中在 source extraction、optional LLM extraction、PPT 和报告。 + + +## 16. 带注释源程序与附录 +### 16.1 SQL 源程序 + +| 文件 | 内容 | +|---|---| +| `database/00_init.sql` | PostgreSQL extension 初始化 | +| `database/01_schema_core.sql` | 用户、workspace、agent、member | +| `database/02_schema_memory.sql` | session、source、chunk、memory、evidence、embedding、entity、scene | +| `database/03_schema_governance.sql` | wiki、timeline、recall log、policy、forget、conflict、audit | +| `database/04_indexes.sql` | B+ tree、GIN、BRIN、covering、partial index | +| `database/05_views.sql` | provenance、visibility、statistics、conflict、wiki source views | +| `database/06_triggers.sql` | revision、audit、soft delete、conflict lifecycle | +| `database/07_seed.sql` | demo seed | +| `database/08_demo_queries.sql` | 演示查询 | +| `database/09_graph_demo.sql` | optional graph demo data | +| `database/10_governance_demo_fixture.sql` | governance fixture | + +### 16.2 高级语言源程序 + +| 目录 / 文件 | 内容 | +|---|---| +| `backend/app/main.py` | FastAPI app 和 router 挂载 | +| `backend/app/api/` | HTTP API | +| `backend/app/services/` | 核心业务逻辑 | +| `backend/app/cli/` | `mb` / `memorybase` CLI | +| `frontend/src/` | React 前端 | +| `evaluation/` | benchmark adapters、metrics、runners、reports | +| `backend/tests/` | 测试 | + +详细源码清单和建议摘录见 `docs/source-sql-appendix.md`。 + +## 17. 总结与展望 + +MemoryBase 完成了一个工程上可运行、演示闭环清晰的组织级长期记忆系统。它把 AI memory 从“聊天产品里的黑盒功能”转化为 PostgreSQL 中可建模、可约束、可审计、可追溯、可被 Agent 调用的数据库应用。 + +已完成的核心价值： + +- 完整的 source -> memory -> evidence -> revision -> audit -> wiki 链路； +- 关系模型、E-R、范式、物理设计和 SQL 源程序齐全； +- B+ tree、GIN、BRIN、covering index 等索引策略有 EXPLAIN 证据； +- governance、provenance、agent visibility 是 schema 级能力； +- 前端、后端、CLI、evaluation 和截图证据形成可演示系统。 + +当前边界： + +- 自动抽取已支持 rule-based 和 optional LLM 两条路径，但还不是完整的 LLM analysis draft-table workflow； +- JSONB embedding cache 适合课程规模，不替代大规模 pgvector / ANN； +- LongMemEval 结果适合作为工程验证，不适合作为高分榜单 claim； +- memory-level visibility 已实现，source/wiki 的更细粒度策略仍可扩展； +- Graph Explorer 是 provenance 可视化，不是默认 GraphRAG 检索主路径。 + +未来的可能拓展方向： + +1. 引入 pgvector / HNSW / IVFFlat 支撑大规模向量检索； +2. 引入 LLM-backed `analysis_run` / `analysis_memory_draft` / `analysis_draft_evidence` 草稿表； +3. 扩展 source/wiki resource policy，形成更完整的多租户治理； +4. 增强 temporal reasoning、multi-session reasoning 和 preference following； +5. 完善 LoCoMo / MemoryAgentBench 对照实验和独立 judge； +6. 扩展 Obsidian/Git 双向同步和多 Agent 自动协作。 + +## 参考资料 + +- Lewis et al., Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks, 2020. +- Park et al., Generative Agents: Interactive Simulacra of Human Behavior, 2023. +- Zhong et al., MemoryBank: Enhancing Large Language Models with Long-Term Memory, 2023. +- Packer et al., MemGPT: Towards LLMs as Operating Systems, 2023. +- Maharana et al., LoCoMo / Evaluating Very Long-Term Conversational Memory of LLM Agents, 2024. +- Wu et al., LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory, 2024. +- Chhikara et al., Mem0: Building Production-Ready AI Agents with Scalable Long-Term Memory, 2025. +- Hu et al., MemoryAgentBench, 2025. +- Edge et al., From Local to Global: A Graph RAG Approach to Query-Focused Summarization, 2024. +- PostgreSQL Documentation, B-Tree, GIN, BRIN, Generated Columns, PL/pgSQL Triggers. +- Atlassian Rovo / Confluence AI documentation. +- OpenAI Help Center, Memory FAQ. +- Letta, LangChain, LlamaIndex, Notion, Obsidian documentation. diff --git a/docs/final-report.pdf b/docs/final-report.pdf new file mode 100644 index 0000000..19a280f Binary files /dev/null and b/docs/final-report.pdf differ diff --git a/docs/governance-demo-walkthrough.md b/docs/governance-demo-walkthrough.md new file mode 100644 index 0000000..3297f09 --- /dev/null +++ b/docs/governance-demo-walkthrough.md @@ -0,0 +1,96 @@ +# Governance Demo Walkthrough + +This walkthrough pairs with `database/10_governance_demo_fixture.sql` and is intended +for final-report screenshots and the live demo. + +## 1. Prepare the demo database + +```bash +npm run db:setup +``` + +The setup command loads: + +1. Core schema files `00_init.sql` through `06_triggers.sql`. +2. Seed workspace data from `07_seed.sql`. +3. Search backfill. +4. Governance fixture data from `10_governance_demo_fixture.sql`. +5. Demo SQL queries from `08_demo_queries.sql`. + +## 2. Verify governance counts + +```sql +SELECT status, count(*) +FROM memory_item +WHERE workspace_id = '00000000-0000-0000-0000-000000000201' +GROUP BY status +ORDER BY status; + +SELECT status, count(*) +FROM conflict_record +WHERE workspace_id = '00000000-0000-0000-0000-000000000201' +GROUP BY status +ORDER BY status; + +SELECT target_type, status, count(*) +FROM forget_request +WHERE workspace_id = '00000000-0000-0000-0000-000000000201' +GROUP BY target_type, status +ORDER BY target_type, status; +``` + +Expected demo signals: + +- at least one `forgotten` memory; +- at least one `archived` memory; +- at least one `superseded` memory; +- at least one `resolved` conflict; +- at least one forgotten source document; +- at least two completed forget requests. + +## 3. UI screenshot checklist + +Capture these pages after backend and frontend are running: + +1. Governance / Forget Requests: show completed requests for memory and source targets. +2. Governance / Conflicts: show one open conflict and one resolved conflict. +3. Governance / Audit: filter for `memory.forget`, `memory.soft_delete`, or + `source.forget`. If the live demo also runs candidate extraction, additionally + filter for `memory_extraction.run.complete`. +4. Memory Detail: show evidence and revision history for a governed memory. +5. Source Detail: show the forgotten source only from a governance/admin view if exposed. + +## 4. SQL screenshot checklist + +Use `database/08_demo_queries.sql` for broad demo results, then add these focused +queries if the report needs governance-specific evidence: + +```sql +SELECT memory_id, status, canonical_text, superseded_by_memory_id +FROM memory_item +WHERE workspace_id = '00000000-0000-0000-0000-000000000201' + AND status IN ('forgotten', 'archived', 'superseded') +ORDER BY status, memory_id; + +SELECT conflict_id, conflict_type, status, resolution_note +FROM conflict_record +WHERE workspace_id = '00000000-0000-0000-0000-000000000201' +ORDER BY status, created_at DESC; + +SELECT action_type, target_type, target_id, created_at +FROM audit_log +WHERE workspace_id = '00000000-0000-0000-0000-000000000201' + AND action_type IN ('memory.forget', 'memory.soft_delete', 'source.forget') +ORDER BY created_at DESC; +``` + +## 5. Interpretation for the report + +The fixture is intentionally small. Its goal is not to simulate a production +governance dataset, but to make the lifecycle visible: + +- memory records are not physically deleted; +- old decisions can be archived or superseded; +- forgotten targets remain auditable; +- conflict records preserve both sides and the resolution; +- audit rows provide traceable operation history. diff --git a/docs/index-rationale.md b/docs/index-rationale.md index 0bf6273..3873a35 100644 --- a/docs/index-rationale.md +++ b/docs/index-rationale.md @@ -21,7 +21,7 @@ PostgreSQL 文档统一使用 "B-tree" 这个历史名称，但**真实实现是 ## 1. 本项目使用的索引类别一览 -`database/04_indexes.sql` 共维护以下 **8 类**索引，覆盖经典关系数据库 + 现代 PostgreSQL 专属能力： +`database/04_indexes.sql` 共维护以下 **9 类**索引，覆盖经典关系数据库 + 现代 PostgreSQL 专属能力： | 类别 | PostgreSQL 实现 | 教学意义 | 本项目案例 | |---|---|---|---| diff --git a/docs/innovation-analysis.md b/docs/innovation-analysis.md new file mode 100644 index 0000000..02dbb16 --- /dev/null +++ b/docs/innovation-analysis.md @@ -0,0 +1,225 @@ +# 创新点展开 + +> 用途：可直接纳入最终报告“创新点总结”章节，也可作为答辩时解释项目差异化的材料。本文与 `docs/research-landscape.md` 对应：研究现状说明现有方案缺什么，本文说明 MemoryBase 如何用已实现的数据库、后端、CLI、前端和评测框架补上这些空缺。 + +## 3.1 总体创新定位 + +MemoryBase 的创新点不在于“再做一个聊天机器人”，也不在于“把文档扔进向量库再做 RAG”。项目的核心 thesis 是： + +> 面向组织与团队的长期记忆，应该首先被建模为一个可追溯、可治理、可审计、可被 Agent 调用的数据库系统；LLM、embedding、Graph 和 evaluation 都是围绕这个数据库事实源的增强层。 + +这个定位和现有研究/产品有明显差异： + +- 传统 RAG 关注 chunk 检索和回答效果，但通常弱化 memory 生命周期、证据关系、权限和审计。 +- MemGPT / Letta / Mem0 等 Agent memory 系统关注 Agent 是否能“记得并用上”，但数据库 schema、触发器、SQL 可验证性、课程级 E-R / 范式 / 索引展示不是主目标。 +- ChatGPT Memory、Confluence AI / Rovo、Notion AI 等产品能提供很好的使用体验，但底层 schema、审计链路、权限视图和 EXPLAIN 计划通常不可见。 +- GraphRAG 关注图结构辅助检索和全局 sensemaking，但如果把图作为主路径，容易偏离课程项目的关系数据库主线。 + +MemoryBase 的差异化是 **DB-first AI substrate**：PostgreSQL 是长期记忆的事实源；前端、CLI、Agent、Graph Explorer、Markdown Wiki 和 evaluation 都围绕同一套关系模型工作。 + +## 3.2 创新点一：文件—数据库双态长期记忆架构 + +传统文件知识库保留了 Markdown 的可读性，但缺少数据库约束和治理；传统数据库系统结构严谨，但对非结构化文本、Agent context 和 Wiki 导出不友好。MemoryBase 把两者结合成“文件—数据库双态”： + +```text +SourceDocument + -> SourceChunk + -> MemoryItem + -> MemoryEvidence + -> MemoryRevision / AuditLog + -> RecallLog / AccessPolicy + -> WikiPage / WikiPageRevision +``` + +代码和 SQL 证据： + +- `source_document` / `source_chunk` 保存原始文档和切片。 +- `memory_item` 保存可召回的长期记忆。 +- `memory_evidence` 把 memory 绑定回 source chunk。 +- `wiki_page` / `wiki_page_revision` 把数据库内容投影成 Markdown Wiki。 +- `backend/app/services/source_service.py`、`memory_service.py`、`wiki_service.py` 分别实现导入、记忆维护和 Wiki 导出。 + +相比只用文件系统，MemoryBase 可以用外键、CHECK、UNIQUE、视图和触发器保证数据一致性；相比只用数据库，它又保留了 Markdown export 和可读 source，使人类用户、答辩演示和 Agent 都能访问同一套知识。 + +## 3.3 创新点二：Provenance-first 的证据链建模 + +长期记忆系统最容易被问到的问题是：“这条结论从哪里来？”普通 RAG 往往只能给出 top-k chunk，而 memory 本身、证据、版本和回答之间不是稳定关系。MemoryBase 把 provenance 设计成一等公民。 + +核心设计： + +- `memory_evidence` 是 memory 与 source chunk 的多对多证据桥，支持 `supports`、`refutes`、`context`、`source` 等角色。 +- `v_memory_with_source` 把 memory、evidence、chunk、source 串成可查询视图。 +- `v_wiki_page_sources` 把 WikiPage 反向追溯到 memory / scene / evidence / source。 +- `recall_log.context_pack_json` 记录一次 recall 的过滤参数、top memory 和 context pack 快照。 +- `/api/recall/context-pack` 返回 Markdown context，同时带 citation map、supporting evidence、conflict warnings、risk notes 和 excluded memories。 + +这种建模让系统能回答四类问题： + +| 问题 | MemoryBase 的回答路径 | +|---|---| +| 一条 memory 来自哪份 source？ | `memory_evidence -> source_chunk -> source_document` | +| 这次 context pack 选了哪些 memory？ | `recall_log.top_memory_ids_json` / `context_pack_json` | +| 一个 Wiki 页面引用了哪些 source？ | `v_wiki_page_sources` / `wiki_page_revision.frontmatter_json` | +| 证据是支持、反驳还是上下文？ | `memory_evidence.evidence_role` | + +这补齐了传统 RAG 和黑盒记忆产品的关键不足：它们能返回结果，但不一定能用 SQL 复现“结果为何出现”。 + +## 3.4 创新点三：Governance-by-default 的记忆生命周期 + +组织级长期记忆不能只考虑“写入”和“召回”。真实团队还需要归档、遗忘、冲突处理、权限变更、版本追踪和审计。MemoryBase 把治理能力直接放进 schema 和触发器，而不是作为 UI 装饰。 + +核心表： + +- `memory_revision`：不可变 memory 版本历史。 +- `audit_log`：append-only 操作日志，保存 before / after JSON 快照。 +- `forget_request`：遗忘申请和审批状态。 +- `conflict_record`：冲突记忆对，约束 `left_memory_id < right_memory_id` 防止重复 A/B 与 B/A。 +- `access_policy`：用户、Agent 或 role 级策略。 +- `timeline_entry`：项目决策演进记录。 + +核心触发器： + +- `trg_memory_after_insert`：memory 创建后写入初始 revision 和 audit。 +- `trg_memory_before_update` / `trg_memory_after_update`：memory 修改时递增 revision、写 audit、标记 Wiki 需要 rebuild。 +- `trg_memory_soft_delete`：直接 DELETE memory 时转成 archived，避免物理丢失。 +- `trg_conflict_after_insert` / `trg_conflict_after_update`：open conflict 自动把相关 memory 标记为 `conflicted`，冲突关闭后在无其他 open conflict 时恢复。 +- `trg_wiki_revision_after_insert`：Wiki revision 插入后同步页面当前版本并写 audit。 + +这使 MemoryBase 的生命周期不是“应用层记得做就做”，而是被数据库约束和触发器强制执行。对数据库课程而言，这也是项目最能体现关系数据库能力的部分：状态机、外键、触发器、审计日志和 SQL 查询共同承担业务规则。 + +## 3.5 创新点四：Agent-aware visibility，而不是 prompt 里的“请不要看” + +很多 Agent 系统把权限问题交给 prompt 或上层工具约定：告诉 Agent 不要读取某些内容。但在组织场景中，权限必须成为数据库查询的一部分。MemoryBase 用 `agent`、`workspace_member`、`access_policy` 和 `v_agent_visible_memory` 实现 Agent 级可见性。 + +实现方式： + +- `agent` 表独立于 `user_account`，允许每个 Agent 有类型、状态和 owner。 +- `access_policy` 支持 `principal_type = user / agent / role`，也支持 allow / deny。 +- `v_agent_visible_memory` 展开每个 Agent 可见的 active memory，并处理 deny 优先和 role policy。 +- `recall_service.py` 和 `search_service.py` 在传入 `agent_id` 时使用 `v_agent_visible_memory` 过滤 memory。 +- recall 路径未传 `agent_id` 时默认只允许 `public` / `project` 范围；search 路径采用同类的无 Agent 可见性分支，避免人类或普通前端路径直接看到 private/team memory。 +- `graph_service.py` 加载图后也按 Agent 可见 memory 做过滤，避免图视角绕过 memory-level visibility。 + +这比普通 RAG 的 metadata filter 更强：权限不是临时参数，而是关系模型中的可审计对象。它也比黑盒企业工具更适合课程展示，因为我们能直接展示视图 SQL、策略表、API 行为和测试用例。 + +当前边界也需要讲清楚：已实现的主线是 **memory-level visibility**。source/chunk 的独立访问控制仍以 workspace 和 active status 为主，若投入生产，需要进一步把 `source_document` / `wiki_page` 的 resource policy 纳入所有读取路径。 + +## 3.6 创新点五：可解释的 hybrid retrieval 与透明降级 + +MemoryBase 没有把“向量召回”当成唯一答案。当前检索系统同时保留数据库可解释性和 AI-native 扩展性： + +- `source_chunk.search_vector` / `memory_item.search_vector` 使用 PostgreSQL generated `tsvector` + GIN 支撑全文检索。 +- `search_service.py` 合并 chunk FTS、memory FTS、trigram fuzzy、title boost，并用 RRF 融合排序。 +- `recall_service.py` 支持 `keyword`、`vector`、`hybrid` 三种模式。 +- `memory_embedding` / `source_chunk_embedding` 使用 JSONB 缓存 embedding，避免默认依赖 `pgvector`。 +- `embedding_service.py` 提供 local hashing provider，可无外网演示；也保留 SiliconFlow provider 扩展路径。 +- `retrieval_info` 返回 requested mode、effective mode、embedding provider/model、vector candidate count、fallback reason。 +- `recall_log` 记录每次召回的 filter、top memory 和 context pack 快照。 + +这形成一个重要创新口径：MemoryBase 不假装“向量永远可用”。如果没有 embedding 记录，hybrid recall 会明确降级到 keyword，并把 fallback reason 返回给 UI / API。相比普通 RAG demo 只展示结果，这种透明降级更符合组织系统的可观测性要求。 + +与 `pgvector` 的关系也要讲清楚：本项目选择 JSONB embedding cache 是课程演示和部署稳定性的取舍，不是说 JSONB 比 pgvector 更适合大规模向量检索。未来数据量增大时，pgvector / HNSW / IVFFlat 是自然演进方向；当前版本重点证明“可降级、可审计、可接入 Agent”的检索闭环。 + +## 3.7 创新点六：AI-assisted candidate extraction，但不让 LLM 成为 P0 依赖 + +很多长期记忆产品依赖 LLM 自动抽取，这在效果上有优势，但课程演示和本地复现会被 API key、模型质量、网络环境影响。MemoryBase 采用分层策略： + +- P0 能力不依赖外部 LLM：人工创建 memory、规则抽取、keyword recall、Wiki 导出、审计和治理都能本地运行。 +- `memory_extraction_service.py` 实现 rule-based extraction，从 chunk 生成 `status = candidate` 的 memory。 +- candidate memory 自动绑定 `memory_evidence`，不会成为无来源草稿。 +- `/api/memory-candidates/{memory_id}/approve` 和 `/reject` 把候选记忆纳入生命周期状态机。 +- extraction run 写入 `audit_log` 的 `memory_extraction.run.start` 和 `memory_extraction.run.complete`，保留 run_id、chunk_ids、候选数量和 candidate memory ids。 + +这个设计避免了两个极端：一端是完全不做自动抽取，所有文档都要人工处理；另一端是把 LLM 输出直接写成事实，缺少审批和证据链。MemoryBase v1 选择“候选 -> 审批 -> active”的中间路线，既能演示 AI-assisted import，又保留治理闭环。 + +未来可扩展方向是引入 `analysis_run` / `analysis_memory_draft` / `analysis_draft_evidence` 这类更完整的 LLM 分析草稿表，但当前版本已经可以用主表生命周期 + audit_log run trace 回答“这次抽取产生了哪些候选”。 + +## 3.8 创新点七：Graph Explorer 作为 provenance 可视化层 + +GraphRAG 和知识图谱系统强调实体关系和图检索。MemoryBase 吸收了图的可解释性，但没有让图替代 PostgreSQL。 + +当前实现： + +- `graph_service.py` 可以从 PostgreSQL 构建 workspace graph，节点包括 workspace、source、chunk、memory、entity、scene、wiki。 +- 边包括 `CONTAINS`、`HAS_CHUNK`、`CREATED_FROM`、`SUPPORTED_BY`、`MENTIONS`、`CONTAINS_MEMORY`、`DERIVED_FROM`。 +- `/api/graph/workspace/preview` 在没有 Neo4j 时也能返回 PostgreSQL preview。 +- `/api/graph/workspace/sync` 可选同步到 Neo4j，并写入 `audit_log` 的 `graph.sync`。 +- `GraphExplorer.jsx` / `GraphSvg.jsx` 提供前端关系图展示。 + +创新点在于定位：图层是 provenance 与关系展示的增强层，不是事实源。PostgreSQL 仍负责约束、版本、权限、审计和查询；Neo4j 只是可选缓存 / 可视化后端。这样既能展示图结构，又不会把数据库课程主线变成知识图谱项目。 + +## 3.9 创新点八：Evaluation 作为系统验证层，而不是产品主线替代品 + +MemoryBase 已经包含 evaluation framework，覆盖 synthetic datasets、local baselines、live API modes、adapter、metrics、runner 和 report generation。它的作用不是把项目包装成 benchmark-only 产品，而是回答：“这个数据库设计是否真的能支持长期记忆任务？” + +已实现能力： + +- `evaluation/datasets/` 包含 conflict、deletion、performance、preference、synthetic memory cases。 +- `evaluation/metrics/` 覆盖 retrieval、QA、forgetting、system metrics。 +- `evaluation/runners/` 覆盖 retrieval、QA、forgetting、conflict、deletion、preference、performance 以及 LoCoMo / LongMemEval / MemoryAgentBench 等外部 benchmark runner。 +- `evaluation/reports/generate_report.py` 生成评测报告。 +- `backend/app/cli/commands/eval.py` 提供 CLI gold-set evaluation 入口。 +- 后端测试中有 `test_evaluation_baselines.py`、`test_evaluation_adapters.py`、`test_evaluation_judging.py`、`test_evaluation_metrics.py`、`test_evaluation_runner_checkpoint.py` 等验证 evaluation 基础行为。 +- 当前已完成一次 LongMemEval oracle 500-case live `db_qa` 运行，结果记录在 `docs/27-longmemeval-full-evaluation-20260609.md` 和 `evaluation/results/longmemeval_full_20260609.csv`：deterministic pass 为 35.2%，DeepSeek semantic judge pass 为 58.4%。 + +这使项目比普通课程 CRUD 系统多一层“可量化验证”。LongMemEval 的当前结果不高，尤其是 multi-session reasoning 和 preference following 仍明显弱于专门优化长期记忆问答的系统；因此报告不应把 benchmark 分数作为主卖点。MemoryBase 的独立价值在于：它用 evaluation 暴露数据库记忆层在 retrieval、forgetting、conflict、system behavior 和 answer generation 上的真实边界，同时保留 provenance、governance、multi-tenant visibility、audit trail 和 SQL-verifiable lifecycle 这些专门记忆产品通常不突出展示的数据库系统能力。 + +需要在报告中保持诚实：当前 LongMemEval 数字适合作为工程验证和限制分析，不适合作为对外榜单 claim。LoCoMo 全量结果、MemoryAgentBench 非 conflict 任务、更独立的 judge model、groundedness / hallucination 评估和更强 baseline 对照可以作为 future work；当前主线仍是 database-backed evaluation harness 和可运行的本地 / API 评测路径。 + +## 3.10 创新点九：人和 Agent 共用同一套数据库入口 + +MemoryBase 不是只给前端用户看的系统，也不是只给 Agent 的隐藏工具。它同时提供： + +- React 前端：Dashboard、Sources、Memories、Recall、Wiki、Governance、Runtime、Graph 页面。 +- FastAPI：sources、memories、recall、search、wiki、governance、agents、sessions、observe、semantic、stats、graph、embeddings、qa 等端点。 +- CLI：`memorybase` / `mb`，支持 configure、health、context、recall、search、observe、remember、sessions、eval。 +- Agent runtime 入口：`agent_session` / `message` 表 + observe / remember / context pack API。 + +这个多入口设计的重点不是“页面多”，而是 **同一套 source / memory / evidence / audit 数据结构同时服务人类和 Agent**。人类可以浏览、编辑、审批和导出；Agent 可以 search / recall / context / remember；管理员可以看 audit、policy、conflict 和 forget request。 + +这补齐了现有系统常见的割裂： + +- 文件知识库适合人读，但 Agent 使用不稳定。 +- Agent memory store 适合 Agent，但人类审计和治理弱。 +- 企业 AI 工具适合协作，但数据库内部不可展示。 + +MemoryBase 把这三者统一到数据库应用系统中。 + +## 3.11 与研究现状的差异化总结 + +| 研究/产品路线的不足 | MemoryBase 的对应创新 | 主要实现证据 | +|---|---|---| +| RAG 只给 top-k chunk，证据链弱 | Provenance-first evidence chain | `memory_evidence`、`v_memory_with_source`、context pack | +| Agent memory 关注效果，治理弱 | Governance-by-default lifecycle | `memory_revision`、`audit_log`、`forget_request`、`conflict_record`、触发器 | +| 黑盒产品不可展示 schema / SQL | DB-first 可验证 substrate | `database/*.sql`、EXPLAIN、索引、视图、触发器 | +| prompt 权限不可靠 | Agent-aware visibility | `access_policy`、`v_agent_visible_memory`、recall/search filters | +| 向量检索依赖外部服务 | Hybrid retrieval with transparent fallback | JSONB embedding cache、local hashing、`retrieval_info` | +| LLM 抽取不可控 | Candidate extraction + approval | `memory_extraction_service.py`、candidate/approve/reject、run audit | +| 图系统容易替代数据库主线 | Graph as optional provenance visualization | PostgreSQL preview、optional Neo4j sync、`graph.sync` audit | +| benchmark 与产品割裂 | Evaluation as validation layer | `evaluation/` runners/metrics/reports、LongMemEval 500-case result、CLI eval | + +## 3.12 答辩推荐口径 + +如果只用一句话概括创新点： + +> MemoryBase 把 AI 长期记忆从“模型或向量库里的黑盒能力”转化为一个以 PostgreSQL 为事实源的组织级数据库系统，核心创新是 provenance、governance、agent-aware visibility、transparent retrieval fallback 和 evaluation-backed validation。 + +如果分点讲，建议用五点： + +1. **DB-first AI substrate**：长期记忆先进入关系模型，LLM / embedding / Graph 都是增强层。 +2. **Provenance-first**：memory、source chunk、evidence、Wiki、recall log 可以被 SQL 串起来。 +3. **Governance-by-default**：revision、audit、forget、conflict、policy 被 schema 和 trigger 固化。 +4. **Agent-aware visibility**：Agent 不是普通用户别名，而是有独立身份和可见视图的 principal。 +5. **Evaluation-backed**：评测框架验证 retrieval、forgetting、conflict 和长期记忆行为，不只做 UI demo。 + +## 3.13 边界与未来增强 + +为了避免答辩中过度承诺，需要明确当前版本边界： + +- 自动抽取是 rule-based v1，不是完整 LLM analysis pipeline；LLM 高质量抽取属于后续增强。 +- JSONB embedding cache 适合课程规模和可部署性，不替代大规模 `pgvector` / ANN。 +- Agent visibility 当前主线覆盖 memory-level recall/search/graph 过滤；source/wiki 的细粒度策略可继续扩展。 +- Graph Explorer 是 provenance 可视化和可选关系缓存，不是默认 GraphRAG 检索主路径。 +- Evaluation framework 已具备本地和 API 模式，并有 LongMemEval 500-case 工程验证结果；但完整 LoCoMo / MemoryAgentBench 对照、正式榜单数字和独立 judge 仍需后续补齐。 + +这些边界不削弱项目创新，反而说明系统设计有清晰主线：先把数据库层做成可复现、可治理、可演示的事实源，再逐步接入更强的模型、向量索引和评测数据。 diff --git a/docs/normalization.md b/docs/normalization.md index d5ae25a..cd82229 100644 --- a/docs/normalization.md +++ b/docs/normalization.md @@ -122,7 +122,10 @@ status: **工程理由**：消除每次读取 memory 时对 `memory_revision` 的 MAX 聚合。该值在 trigger `trg_memory_after_update` 中维护。 - **补偿措施**：trigger 自动维护，应用层只读不写。代价是 trigger 必须正确，**已在 `tests/test_governance.py` 覆盖**。 + **补偿措施**：`trg_memory_before_update` 先在同一事务内递增 + `current_revision_no`，`trg_memory_after_update` 再写入 `memory_revision` + 和 `audit_log`。应用层只读不直接维护该字段。代价是 trigger 必须正确， + **已在 `tests/test_governance.py` 覆盖**。 ### 1.5 `memory_revision`（记忆历史版本） @@ -383,7 +386,7 @@ SQL:2016 标准引入 JSON 数据类型，将 JSONB 视为**一个原子的"半 | 派生字段 | 维护机制 | 一致性保证 | |---|---|---| | `memory_item.search_vector` / `source_chunk.search_vector` | `GENERATED ALWAYS AS (...) STORED` | DBMS 强制，应用层无法直接写 | -| `memory_item.current_revision_no` | trigger `trg_memory_after_update` | trigger 单事务原子更新 | +| `memory_item.current_revision_no` | trigger `trg_memory_before_update` 递增，`trg_memory_after_update` 写 revision/audit | trigger 单事务原子更新 | | `wiki_page.current_revision_no` | trigger `trg_wiki_revision_after_insert` | 同上 | | `wiki_page.needs_rebuild` | trigger `trg_memory_after_update` | dirty bit 设置；rebuild 时清零 | | `memory_entity.workspace_id` / `memory_scene_cell.workspace_id` | 应用层显式写入 + 复合 FK 校验 | DBMS 复合 FK 拒绝跨租户引用 | diff --git a/docs/12-github-workflow.md b/docs/process/12-github-workflow.md similarity index 100% rename from docs/12-github-workflow.md rename to docs/process/12-github-workflow.md diff --git a/docs/process/13-final-report-outline.md b/docs/process/13-final-report-outline.md new file mode 100644 index 0000000..a2a1aa0 --- /dev/null +++ b/docs/process/13-final-report-outline.md @@ -0,0 +1,123 @@ +# 最终报告整合入口 + +> Gap 1 输出说明：`docs/final-report.md` 是当前可直接排版的最终报告主体草稿；本文负责说明正文、图表、截图、SQL/源码附录和成员信息的装配关系。 + +## 1. 报告标题 + +《MemoryBase：面向组织与团队的 AI-native 可追溯长期记忆数据库系统设计与实现》 + +旧标题“面向 AI Agent 协作研发的文件—数据库双态长期记忆系统”可作为副标题或答辩口径，但最终主叙事建议使用“组织与团队”，避免场景过窄。 + +## 2. 主体正文 + +主体文件： + +```text +docs/final-report.md +``` + +当前正文结构： + +| 章节 | 内容 | 主要素材来源 | +|---|---|---| +| 摘要 | 项目定位、数据库主线、评测边界 | `docs/00-project-overview.md`、`docs/innovation-analysis.md` | +| 1 项目概述 | 项目定位、核心链路、已交付能力 | `docs/00-project-overview.md` | +| 2 研究现状分析 | RAG、Agent memory、企业知识库、GraphRAG、benchmark | `docs/research-landscape.md` | +| 3 需求分析 | 角色、功能需求、非功能需求 | `docs/01-requirements.md` | +| 4 数据流设计 | 0 层、1 层、Source/Recall/Wiki 2 层流程 | `docs/02-data-flow.md` | +| 5 数据字典 | 核心数据项、数据结构 | `docs/03-data-dictionary.md` | +| 6 概念结构设计与 E-R 图 | 概念分层、核心 ER、关系和基数 | `docs/04-er-design.md`、`docs/final-assets/diagrams/` | +| 7 逻辑结构设计与范式分析 | 关系模式、E-R 转换、3NF/BCNF | `docs/05-logical-design.md`、`docs/normalization.md` | +| 8 物理结构设计 | PostgreSQL、路径、索引、EXPLAIN、视图、触发器 | `docs/06-physical-design.md`、`docs/index-rationale.md`、`docs/explain-analyze.md` | +| 9 系统总体架构 | 前后端、CLI、Agent、evaluation、Graph | `docs/07-system-architecture.md` | +| 10 API 与模块 IPO | API 摘要、模块 IPO | `docs/08-api-design.md`、`docs/09-module-ipo.md` | +| 11 系统实现 | Source→Wiki、Recall、Governance、状态机 | `docs/final-assets/diagrams/` | +| 12 系统演示 | demo setup、操作流程、截图索引 | `docs/11-demo-script.md`、`docs/final-assets/screenshots/README.md` | +| 13 测试与评估 | 自动化测试、SQL 测试、LongMemEval 解释 | `docs/10-test-plan.md`、`docs/27-longmemeval-full-evaluation-20260609.md` | +| 14 创新点总结 | DB-first、provenance、governance、visibility、fallback、evaluation | `docs/innovation-analysis.md` | +| 15 小组分工 | 成员贡献表和边界说明 | `docs/contribution-ledger.md` | +| 16 带注释源程序 | SQL 和高级语言源程序清单 | `docs/source-sql-appendix.md` | +| 17 总结与展望 | 已完成、边界、future work | `docs/innovation-analysis.md`、`docs/process/22-backend-memory-roadmap.md` | + +## 3. 图表插入清单 + +图源和 SVG 位于： + +```text +docs/final-assets/diagrams/ +``` + +| 建议位置 | 图片 | +|---|---| +| 第 6 章概念结构设计开头 | `01-er-core.svg` | +| 第 6 章附录或正文后半 | `02-er-full.svg` | +| 第 11.1 节 Source 到 Wiki | `03-source-to-wiki.svg` | +| 第 11.2 节 Recall | `04-recall.svg` | +| 第 11.3 节 Governance | `05-governance.svg` | +| 第 11.4 节 Memory 状态机 | `06-memory-status.svg` | + +如需重新渲染，按 `docs/final-assets/diagrams/README.md` 使用 Mermaid CLI。 + +## 4. 截图插入清单 + +截图和完整命令日志位于： + +```text +docs/final-assets/screenshots/ +``` + +推荐正文截图： + +| 报告位置 | 截图 | +|---|---| +| 系统演示总览 | `ui/01-dashboard.png` | +| Source 导入 / chunk | `ui/02-sources-list.png`、`ui/03-source-detail-project-pivot.png` | +| Memory evidence / revision | `ui/05-memory-detail-evidence-revisions.png` | +| Recall / context pack | `ui/06-recall-search-results.png`、`ui/07-recall-context-pack.png` | +| Optional LLM QA | `ui/08-recall-qa-answer-or-config-state.png` | +| Governance audit / conflict / forget | `ui/10-governance-audit.png`、`ui/12-governance-conflicts.png`、`ui/13-governance-forget-requests.png` | +| Graph Explorer | `ui/15-graph-explorer-demo-workspace.png` | +| SQL 证据 | `sql/04-focused-sql-evidence.png` | +| 测试结果 | `tests/01-pytest-core.png`、`tests/02-frontend-build.png` | + +注意：SQL/test PNG 是从完整日志渲染，避免终端预览截断。若报告需要更清晰缩放，优先使用同名 `.svg`。 + +## 5. 附录安排 + +| 附录 | 内容 | 入口 | +|---|---|---| +| 附录 A SQL 源程序 | DDL、indexes、views、triggers、seed、demo queries | `database/*.sql`、`docs/source-sql-appendix.md` | +| 附录 B 高级语言源程序说明 | FastAPI、React、CLI、evaluation、tests | `docs/source-sql-appendix.md` | +| 附录 C 演示数据 | seed workspace、governance fixture、graph demo data | `database/07_seed.sql`、`database/09_graph_demo.sql`、`database/10_governance_demo_fixture.sql` | +| 附录 D 测试与命令日志 | backend tests、frontend build、SQL outputs、EXPLAIN | `docs/final-assets/screenshots/logs/` | +| 附录 E 评测结果 | LongMemEval summary 和 CSV | `docs/27-longmemeval-full-evaluation-20260609.md`、`evaluation/results/longmemeval_full_20260609.csv` | + +## 6. 正式排版前待补项 + +这些是内容层面之外的最终排版信息，需由团队确认： + +1. 封面：课程名、教师、学院/班级、组号、四位成员姓名和学号。 +2. 第 15 章成员分工：第四位成员的实际贡献证据仍为空，需要补 Git 身份或非 Git 证据。 +3. 图表编号：Word/PDF 中按“图 6-1”“表 8-1”等重新编号。 +4. 截图压缩：正文只放关键截图，其余放附录或 PPT。 +5. 引用格式：参考资料可按老师要求改成 GB/T 7714、APA 或脚注格式。 + +## 7. 最终提交前检查 + +建议在最终导出报告前重新执行： + +```bash +uv run ruff check backend/app backend/tests evaluation +uv run --with pytest python -m pytest backend/tests -q +cd frontend && npm run lint && npm run build +git diff --check +``` + +如果数据库或截图更新，再执行： + +```bash +npm run db:setup +psql $DATABASE_URL -f database/08_demo_queries.sql +``` + +并同步更新 `docs/final-assets/screenshots/README.md` 中的截图基线。 diff --git a/docs/14-initial-issues.md b/docs/process/14-initial-issues.md similarity index 100% rename from docs/14-initial-issues.md rename to docs/process/14-initial-issues.md diff --git a/docs/15-api-contract-plan.md b/docs/process/15-api-contract-plan.md similarity index 100% rename from docs/15-api-contract-plan.md rename to docs/process/15-api-contract-plan.md diff --git a/docs/16-agent-runtime-gap-analysis.md b/docs/process/16-agent-runtime-gap-analysis.md similarity index 100% rename from docs/16-agent-runtime-gap-analysis.md rename to docs/process/16-agent-runtime-gap-analysis.md diff --git a/docs/17-agent-runtime-plan.md b/docs/process/17-agent-runtime-plan.md similarity index 100% rename from docs/17-agent-runtime-plan.md rename to docs/process/17-agent-runtime-plan.md diff --git a/docs/18-pr4-lexical-search-design.md b/docs/process/18-pr4-lexical-search-design.md similarity index 100% rename from docs/18-pr4-lexical-search-design.md rename to docs/process/18-pr4-lexical-search-design.md diff --git a/docs/19-repo-session-aware-context-design.md b/docs/process/19-repo-session-aware-context-design.md similarity index 100% rename from docs/19-repo-session-aware-context-design.md rename to docs/process/19-repo-session-aware-context-design.md diff --git a/docs/20-course-alignment-risk-and-recovery-plan.md b/docs/process/20-course-alignment-risk-and-recovery-plan.md similarity index 100% rename from docs/20-course-alignment-risk-and-recovery-plan.md rename to docs/process/20-course-alignment-risk-and-recovery-plan.md diff --git a/docs/21-evaluation-memory-benchmark-prompt.md b/docs/process/21-evaluation-memory-benchmark-prompt.md similarity index 100% rename from docs/21-evaluation-memory-benchmark-prompt.md rename to docs/process/21-evaluation-memory-benchmark-prompt.md diff --git a/docs/22-backend-memory-roadmap.md b/docs/process/22-backend-memory-roadmap.md similarity index 100% rename from docs/22-backend-memory-roadmap.md rename to docs/process/22-backend-memory-roadmap.md diff --git a/docs/23-evaluation-benchmark-roadmap.md b/docs/process/23-evaluation-benchmark-roadmap.md similarity index 100% rename from docs/23-evaluation-benchmark-roadmap.md rename to docs/process/23-evaluation-benchmark-roadmap.md diff --git a/docs/24-evaluation-test-guide.md b/docs/process/24-evaluation-test-guide.md similarity index 65% rename from docs/24-evaluation-test-guide.md rename to docs/process/24-evaluation-test-guide.md index 0e365f9..9b3db18 100644 --- a/docs/24-evaluation-test-guide.md +++ b/docs/process/24-evaluation-test-guide.md @@ -170,10 +170,14 @@ MEMORYBASE_AGENT /api/health/detail /api/sessions /api/observe -/api/memories +/api/memories/batch /api/recall ``` +Direct-memory modes reuse one HTTP client and batch up to 500 memory-bearing +turns in one atomic database transaction. Forget operations flush the pending +batch before changing memory status so event order remains deterministic. + For deletion cases it soft-deletes memories it injected: ```text @@ -200,24 +204,49 @@ Processed `EvaluationCase` JSONL is written to: evaluation/external//processed/ ``` -Convert LongMemEval-like JSON/JSONL: +Download one official LongMemEval variant into +`evaluation/external/longmemeval/raw/`, then convert it: ```bash python evaluation/runners/run_external_eval.py --benchmark longmemeval ``` -Convert LoCoMo-like JSON/JSONL: +The official Hugging Face files are extensionless. The adapter supports that +layout directly and validates the parallel session ID/date arrays. See +`evaluation/external/longmemeval/README.md` for current file names, sizes, and +the dataset-license boundary. + +Download the official `locomo10.json`, then convert it: ```bash python evaluation/runners/run_external_eval.py --benchmark locomo ``` -Convert MemoryAgentBench-like JSON/JSONL: +The official adapter preserves session timestamps, both speakers, image +captions, dialog evidence IDs, numeric QA categories, and adversarial answers. +See `evaluation/external/locomo/README.md` for the CC BY-NC 4.0 restriction, +processed-file size, and recommended smoke/live commands. + +Download the official MemoryAgentBench Conflict Resolution parquet shard, then +convert it: ```bash python evaluation/runners/run_external_eval.py --benchmark memoryagentbench ``` +Run multiple questions against one shared official context: + +```bash +python evaluation/runners/run_grouped_benchmark_eval.py \ + --dataset evaluation/external/memoryagentbench/processed/memoryagentbench_cases.jsonl \ + --group factconsolidation_sh_6k \ + --limit 3 \ + --output evaluation/outputs/memoryagentbench/db_qa_results.csv +``` + +See `evaluation/external/memoryagentbench/README.md` for source, license, +grouping, and conflict-sequence details. + Direct wrappers: ```bash @@ -226,7 +255,9 @@ python evaluation/runners/run_locomo_eval.py python evaluation/runners/run_memoryagentbench_eval.py ``` -Current adapter support is partial. It handles common JSON/JSONL shapes with fields such as: +LongMemEval, LoCoMo, and MemoryAgentBench Conflict Resolution support their +official record shapes. Other adapters still handle common JSON/JSONL shapes +with fields such as: ```text question / query @@ -236,7 +267,22 @@ qa / qas / questions task_type / category ``` -Official dataset variants must be verified before using results as benchmark evidence. +LongMemEval oracle conversion has been validated against 500 official records. +LoCoMo conversion has been validated across all 1,986 official QA items. +Open-ended pass rates still require the official or an equivalent independent +LLM judge before they should be treated as final benchmark evidence. + +For paid smoke runs, append semantic judgement fields to an existing result: + +```bash +python evaluation/runners/run_semantic_judge.py \ + --input evaluation/outputs/run/benchmark/db_qa_results.csv \ + --dataset evaluation/external/benchmark/processed/benchmark_cases.jsonl \ + --output evaluation/outputs/run/benchmark/db_qa_results.csv +``` + +When `judge_pass` is present, report generation uses it instead of the +deterministic substring result. Keep the deterministic columns for diagnosis. ## Step 8: Report Generation @@ -328,27 +374,64 @@ python evaluation/runners/run_all.py \ --limit 6 python evaluation/runners/run_all.py \ --dataset evaluation/datasets/synthetic_memory_cases.jsonl \ - --modes summary_memory,db_memory,naive_vector_rag \ + --modes summary_memory,db_qa,vector_qa,db_extraction_qa \ --api-base http://localhost:8000 \ --workspace \ --agent \ --limit 6 ``` +Run measured long-context retention: + +```bash +python -m evaluation.runners.run_long_context_eval \ + --token-lengths 1000,10000,50000,100000 \ + --mode db_qa \ + --workspace \ + --agent +``` + +Run operation-level performance sampling: + +```bash +python -m evaluation.runners.run_api_performance \ + --workspace \ + --agent \ + --iterations 20 +``` + +Add `--include-qa` only when model latency and provider cost should be measured. + +Convert and execute operator-supplied official benchmark data: + +```bash +python -m evaluation.runners.run_benchmark_eval \ + --benchmark longmemeval \ + --mode db_qa \ + --workspace \ + --agent +``` + +Live modes create an isolated workspace per case by default and cascade-delete it after +scoring. Use `--preserve-eval-data` for failure inspection. Use `--shared-workspace` only +when intentional; shared runs can soft-delete memories but cannot remove sessions or +imported sources through the current public API. + ## Remaining Work -Still not fully implemented: +Still operator- or model-dependent: ```text - embedding similarity scoring -- official LongMemEval / LoCoMo / MemoryAgentBench full-format validation -- LLM-as-judge -- groundedness +- official LongMemEval S/M full benchmark execution +- LoCoMo event-summarization task integration +- MemoryAgentBench non-conflict task validation +- independent LLM-as-judge +- semantic groundedness beyond citation validation - hallucination rate -- token usage and token cost -- real write/query/update/delete database performance suite -- real context leakage and answer leakage split -- long history 1K / 10K / 50K / 100K / 500K retention curves +- embedding-provider cost accounting +- concurrent saturation and load testing +- hard cleanup of evaluation sessions and imported sources ``` These require either a running API with seed data, external benchmark files, or an LLM judge configuration. diff --git a/docs/process/25-evaluation-remaining-work.md b/docs/process/25-evaluation-remaining-work.md new file mode 100644 index 0000000..9ab107a --- /dev/null +++ b/docs/process/25-evaluation-remaining-work.md @@ -0,0 +1,174 @@ +# Evaluation Remaining Work + +## Purpose + +This document records evaluation-route items that are intentionally not complete yet. They require backend features, external datasets, a running API, or model/judge configuration. + +## Backend-Dependent Items + +```text +- embedding similarity scoring beyond backend recall scores +- keyword-only vs vector vs hybrid baseline comparison report +- concurrent saturation and load testing +- hard cleanup APIs for evaluation sessions and source documents +``` + +Required backend support: + +```text +- source chunk vector retrieval path +- independent judge model configuration +``` + +## External Dataset Items + +```text +- official LongMemEval S/M full benchmark execution +- official LoCoMo event-summarization integration +- official MemoryAgentBench non-conflict task validation +- BEIR / MS MARCO retriever-only benchmark conversion +- BEAM long-context stress benchmark conversion +``` + +Current state: + +```text +- LongMemEval official oracle format is validated across all 500 records. +- LongMemEval S/M files still need full paid benchmark execution. +- LoCoMo official QA format is validated across all 1,986 QA items. +- LoCoMo event summarization is not integrated. +- MemoryAgentBench official Conflict Resolution parquet conversion is validated + across 800 QA cases. +- Raw official datasets are not stored in this repository. +- LongMemEval download URLs, current file names, sizes, and license boundary are documented. +- LongMemEval and LoCoMo source/license notes are documented. +- MemoryAgentBench source, license, grouping, and sequence semantics are documented. +``` + +## Model-Dependent Items + +```text +- independent LLM-as-judge +- semantic groundedness beyond citation validation +- hallucination rate +- embedding similarity judge +- embedding cost per answer +``` + +Required configuration: + +```text +- model provider +- judge prompt +- API key through environment/config only +- deterministic evaluation settings +- embedding usage and cost accounting policy +``` + +## Live API Items + +```text +- db_memory synthetic benchmark evidence against a live API +- real deletion retrieval leakage +- real conflict stale memory error rate +- real context-pack leakage +- real wiki/export leakage +``` + +Current `db_memory` behavior: + +```text +/api/health/detail +/api/sessions +/api/observe +/api/memories/batch +/api/memory-extraction/from-chunks +/api/memory-candidates +/api/recall +DELETE /api/memories/{memory_id} for injected deletion cases +``` + +Limitation: + +```text +Retrieval-only modes use direct memory API injection. `db_qa`, `vector_qa`, and +`db_extraction_qa` call the real answer endpoint. The extraction QA mode exercises source +import, candidate extraction, approval, recall, context packaging, and answer generation. +``` + +Current extraction support: + +```text +- rule-based chunk -> candidate memory extraction exists +- candidate approve/reject workflow exists +- evaluation db_extraction mode can exercise source import -> extraction -> approve -> recall +- extraction from full documents and sessions is not implemented yet +``` + +## Current Backend Unlocks + +```text +- memory and source chunk embedding storage exists +- local hashing embedding provider exists +- /api/embeddings/backfill exists +- recall returns keyword/vector/recency/evidence score fields when embeddings are present +- recall supports explicit keyword, vector, and hybrid retrieval modes +- recall vector mode can use both memory embeddings and source chunk embeddings through evidence +- evaluation naive_vector_rag can now backfill embeddings and call live recall +``` + +## Next Backend Route + +Continue backend B1 by expanding vector coverage and reporting comparisons: + +```text +1. Add benchmark report comparison for keyword-only / vector-only / hybrid. +2. Add embedding similarity scoring beyond backend recall scores. +3. Add real context leakage and answer leakage inspection. +4. Keep naive_vector_rag as the live API vector-backed baseline. +``` + +## Newly Completed + +```text +- provider-backed QA modes with prompt/completion/total token usage +- deterministic citation validity and citation-groundedness +- retrieval leakage and answer leakage split +- generated measured long-context retention curves +- sequential write/recall/context/QA/update/delete performance sampling +- external benchmark convert-and-run entry point +- default soft cleanup for evaluation-created memories +- official LongMemEval oracle conversion across 500 cases +- official LoCoMo QA conversion across 1,986 cases +- official MemoryAgentBench Conflict Resolution conversion across 800 cases +- inject-once/query-many grouped benchmark execution +- optional semantic LLM judge with token and cost accounting +- report pass rates that prefer semantic judgements over string matching +- MemoryAgentBench context blocks below the context-pack compaction boundary +- explicit MemoryAgentBench conflict sequence semantics +- atomic batch memory creation with per-item revision and audit triggers +- live evaluation HTTP connection reuse and batched direct-memory injection +- atomic explicit supersession using existing validity and lifecycle columns +``` + +## Latest Paid Smoke Evidence + +The clean final report is: + +```text +evaluation/outputs/external-live-final-20260608/benchmark_report.md +``` + +Strict semantic results: + +```text +LongMemEval: 1 / 3 +LoCoMo: 2 / 3 +MemoryAgentBench: 2 / 3 +Overall: 5 / 9 +API errors: 0 +``` + +The semantic judge currently uses the same DeepSeek model family as answer +generation. This is useful smoke evidence, but publication-grade evaluation +still requires an independent judge model or official benchmark evaluator. diff --git a/docs/process/audit-findings.md b/docs/process/audit-findings.md new file mode 100644 index 0000000..eb00419 --- /dev/null +++ b/docs/process/audit-findings.md @@ -0,0 +1,245 @@ +# Gap 9 — Re-audit Findings (HEAD 2dd0d64) + +## Summary + +- Files audited: 26 +- High-severity issues: 11 +- Medium-severity issues: 27 +- Low-severity issues: 30 +- Files marked "needs rewrite": `docs/07-system-architecture.md`, `docs/13-final-report-outline.md` +- Schema drift summary: The canonical source is the current `database/` SQL at HEAD `2dd0d64`. The largest drift is that early course docs still describe the pre-merge core: no embedding cache, no rule-based candidate extraction, no graph/QA/evaluation endpoints, and older memory enum values. The current schema adds `memory_embedding`, `source_chunk_embedding`, four memory types (`fact`, `constraint`, `policy`, `summary`), and seven memory statuses (`candidate`, `active`, `archived`, `forgotten`, `superseded`, `rejected`, `conflicted`). Some final-report-facing docs have been updated, but older overview/architecture/test/demo docs still need alignment before they can be reused directly. + +## Per-file findings + +### docs/00-project-overview.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | §8 still lists "LLM automatic extraction" and semantic retrieval as future, but current HEAD has rule-based extraction endpoints (`/api/memory-extraction/from-chunks`) and embedding cache tables/services for hybrid recall. | medium | Rephrase as: rule-based candidate extraction and JSONB embedding cache are delivered; high-quality LLM extraction and ANN/pgvector retrieval remain future work. | +| report-fit | Title narrows the product to "AI Agent collaboration R&D"; final narrative should be broader organization/team memory database. | medium | Change report-facing title to "AI-native, traceable organizational/team memory database system"; keep agent collaboration as a primary use case. | +| polish | P1/P2 labels say some items are "已实现", while P0/P1/P2 taxonomy is no longer clean after PR #70. | low | Replace stage labels with "delivered core", "delivered extension", and "future extension" for final report reuse. | + +### docs/01-requirements.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Functional table omits delivered graph explorer, embedding backfill, QA, stats overview, semantic entity/scene APIs, CLI, and evaluation framework. | medium | Add an "extension capabilities" subsection so final report does not understate completed work. | +| report-fit | Requirements still mark Entity/MemoryScene and ForgetRequest as P2 even though they are implemented in schema/API/frontend and demo fixture. | medium | Move them to delivered governance/semantic requirements; reserve Future for pgvector, production LLM extraction, plugins, and sync ecosystem. | +| polish | The "管理员" role claims user management, but there is no user CRUD API; seed users exist, and agent registration exists. | medium | Rewrite as "manage agents, policies, audit, forgetting, and governance state"; do not claim full user management unless implemented. | + +### docs/02-data-flow.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Source import 2-level DFD says "写入 AuditLog", but `PostgresSourceRepository.create_source()` only inserts `source_document` and `source_chunk`; no source import audit trigger exists. | high | Either add source import audit in code later, or change DFD to "memory/wiki/governance operations write AuditLog; source import is visible via source_document/imported_at". For final report, avoid claiming source import audit unless implemented. | +| code-consistency | Recall DFD only shows SourceChunk FTS; current recall also supports memory text keyword fallback and optional vector scoring from `memory_embedding`/`source_chunk_embedding`. | medium | Update Recall DFD to keyword/hybrid path: permission filter -> chunk FTS/trigram + memory text match + optional embedding candidates -> context pack -> recall_log. | +| report-fit | Wiki export DFD says export to `data/markdown_wiki/`, which is true, but final report should show DB write (`wiki_page`, `wiki_page_revision`) before file output. | low | Expand one line to make the DB-first design clearer. | + +### docs/03-data-dictionary.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | `memory_type` enum omits `fact`, `constraint`, `policy`, and `summary`, all present in `database/02_schema_memory.sql` and `backend/app/models/memory.py`. | high | Align enum with current canonical list: `episodic`, `semantic`, `fact`, `profile`, `procedural`, `decision`, `preference`, `task`, `risk`, `constraint`, `policy`, `summary`. | +| code-consistency | `status` enum omits `candidate` and `rejected`; these are required by candidate extraction. | high | Align with current `memory_item.status`: `candidate`, `active`, `archived`, `forgotten`, `superseded`, `rejected`, `conflicted`. | +| code-consistency | Data structure dictionary omits `memory_embedding` and `source_chunk_embedding`. | medium | Add embedding cache structures and note JSONB vector cache, not pgvector/ANN. | +| polish | `AuditLog` row says `diff`, but table has `before_json` and `after_json`, not a `diff` column. | low | Rename to `before_json / after_json`. | + +### docs/04-er-design.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | ER entities omit `WorkspaceMember`, `memory_embedding`, and `source_chunk_embedding`, all current schema tables. | medium | Add them or explicitly state the diagram is a simplified core ER view. | +| code-consistency | Mermaid relation `MEMORY_ITEM ||--o{ CONFLICT_RECORD : conflicts` only covers one side, while SQL has `left_memory_id` and `right_memory_id`. | low | Show two relationships or label as left/right conflict endpoints. | +| report-fit | Mermaid ER source is useful but not enough for teacher-required visual flow/ER submission. | medium | Export a PNG/SVG into final assets and keep Mermaid as source. | + +### docs/05-logical-design.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Relationship schemas omit `memory_embedding` and `source_chunk_embedding`. | medium | Add a "Embedding cache" subsection matching `database/02_schema_memory.sql`. | +| code-consistency | `ConflictRecord` omits `resolved_by_actor_type`, `resolved_by_actor_id`, and `updated_at`, which exist in `database/03_schema_governance.sql`. | medium | Add the missing fields to keep logical design aligned. | +| report-fit | 3NF section says "大部分表满足 3NF" but does not include the newer nuance already captured in `docs/normalization.md`. | low | In final report, import the stronger wording from `docs/normalization.md`: core tables mainly 3NF/BCNF with controlled denormalization. | + +### docs/06-physical-design.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | View table says `v_agent_visible_memory` is based on `app.agent_id` and returns nothing if unset. Actual view joins `agent` and returns rows for all agents; callers filter by `agent_id`. | high | Rewrite as: `v_agent_visible_memory` materializes per-agent visibility; service/API must add `WHERE agent_id = ...`. Also fix `database/08_demo_queries.sql` query #5. | +| code-consistency | §9 says `status`新增 only `candidate` and `rejected`, but current status also includes `conflicted`. | medium | Include `conflicted` in the schema-extension note. | +| code-consistency | "保底方案 SQLite + FTS5" is not implemented in repo scripts/tests. | medium | Mark SQLite as earlier design/fallback idea, not delivered capability, or remove from final report unless a working SQLite path is added. | +| report-fit | Physical design is strong and mostly final-report ready. | low | Minor edit only after fixing the view description and SQLite wording. | + +### docs/07-system-architecture.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Backend directory tree is stale: it lists `routers/`, `ingest_service.py`, `policy_service.py`, and `audit_service.py`. Current code uses `backend/app/api/`, `source_service.py`, `governance_service.py`, `graph_service.py`, `embedding_service.py`, `llm_service.py`, and many more. | high | Rewrite this file against current `backend/app` layout. | +| code-consistency | Architecture diagram omits CLI, evaluation framework, graph/Neo4j optional store, embedding provider/cache, and QA path. | medium | Add an updated component diagram for Backend API + CLI + PostgreSQL + optional Neo4j + evaluation runners. | +| report-fit | Needs rewrite before final report. | high | Use this file only as a historical draft until rewritten. | + +### docs/08-api-design.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | API summary omits current delivered endpoints for `/api/graph/*`, `/api/embeddings/*`, `/api/qa/answer`, `/api/stats/overview`, `/api/entities`, and `/api/scenes`. | medium | Add a concise "extension APIs" section, or explicitly mark this as a selected core API summary. | +| code-consistency | Source import response example omits `workspace_id`/etc. This is acceptable because actual `SourceImportResponse` only returns `doc_id` and `chunk_count`. | low | No functional fix needed. | +| report-fit | Good core API summary and mostly matches current backend for source/memory/extraction/recall/search/wiki/governance/runtime. | low | Reuse in final report after adding omitted endpoint families. | + +### docs/09-module-ipo.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | IPO table omits delivered modules: Memory Extraction, Search, Sessions/Observe CLI runtime, Graph, Embeddings, QA, Stats, Semantic entities/scenes, and Evaluation. | medium | Expand table or add a second "extension module IPO" table. | +| report-fit | Too short for final report as-is. | medium | Use as a seed, but rewrite into a fuller module-design section. | + +### docs/10-test-plan.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Test plan omits most current test suites: graph, embeddings, evaluation, QA, semantic, stats, CLI context/recall/sessions/writeback, search API, workspace slug, context pack. | medium | Add a coverage matrix keyed to actual `backend/tests/test_*.py`. | +| report-fit | Understates real test coverage and will make the project look weaker. | medium | Rewrite/extend before final report; include latest validation evidence. | +| polish | Expected "delete memory -> archived" is correct for soft delete, but should mention trigger path and audit. | low | Add trigger/audit verification rows. | + +### docs/11-demo-script.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | `npm run db:setup` claim is correct: `scripts/db_cli.py reset` loads 00-06, 07 seed, search backfill, 10 fixture, then 08 queries. | low | No issue found for setup command. | +| cross-doc consistency | Demo flow says "进入 Sources 页面，导入 6 份讨论记录", but `db:setup` already seeds the 6 discussion records. | medium | Decide final demo mode: either show pre-seeded Sources, or import a seventh/new source live. Update script accordingly. | +| code-consistency | Required demo data says at least 3 Wiki pages, but this audit did not verify that seed + setup creates 3 wiki pages before app actions. | medium | Verify with `npm run db:setup`/SQL before final screenshots, or change to "create/export 3 wiki pages during demo". | + +### docs/13-final-report-outline.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| report-fit | Only a 40-line outline; it is not an integrated final report. | high | Rewrite into `docs/final-report.md` with full chapters, screenshots, diagrams, citations, source/SQL appendix, and member contribution section. | +| report-fit | Title is still narrow ("AI Agent 协作研发"). | medium | Use the broader organization/team memory database title agreed in the roundtable. | +| code-consistency | Innovation list omits evaluation framework, hybrid retrieval with transparent fallback, rule-based candidate extraction, graph explorer, and DB course depth (BRIN/covering/EXPLAIN). | medium | Expand innovation list using current code and supporting docs. | + +### docs/normalization.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Mostly aligned with current schema: includes expanded memory enums and embedding tables. | low | No blocking issue found. | +| polish | §1.4 says `current_revision_no` is maintained by `trg_memory_after_update`; actual implementation increments in `trg_memory_before_update`, with after trigger inserting revision. | medium | Adjust wording to `trg_memory_before_update` increments, `trg_memory_after_update` writes revision/audit. | +| report-fit | Strong final-report source; may need shortening. | low | Use as a condensed subsection, with full doc as appendix/reference. | + +### docs/index-rationale.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Mostly aligned with `database/04_indexes.sql`; includes embedding indexes, BRIN, covering, B+ tree/B-link explanation. | low | No blocking issue found. | +| polish | §1 says "8 类" but table lists 9 categories after BRIN/Covering were added. | low | Change "8 类" to "9 类". | +| report-fit | Strong final-report source; keep B+ tree/B-link explanation because it directly answers the course concern. | low | Use condensed version in report. | + +### docs/explain-analyze.md + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Case 3.2 claims planner can use `idx_audit_target_time = (workspace_id, target_type, target_id, created_at DESC)` for a predicate only on trailing `created_at`. That is suspicious for PostgreSQL B-tree because leading columns are unconstrained; the pasted plan may be from a different index state or needs re-verification. | high | Re-run Case 3 on current DB and either paste the real plan or rewrite as "competing B-tree path only when leading columns are constrained"; keep BRIN demonstration honest. | +| code-consistency | Index coverage summary references `memory_item_memory_id_workspace_id_key`; this unique constraint exists via `UNIQUE(memory_id, workspace_id)`, but `database/04_indexes.sql` does not name it. | low | Clarify it is an auto-created unique index from `database/02_schema_memory.sql`, not a manual index. | +| report-fit | Good but should be treated as evidence only after re-running plans on final demo DB. | medium | Re-validate before screenshot/PDF inclusion. | + +### database/00_init.sql + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Defines `pgcrypto` and `pg_trgm`, matching UUID generation and trigram indexes. | low | No issues found. | + +### database/01_schema_core.sql + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Core user/workspace/agent/member schema matches backend models and seed. | low | No issues found. | + +### database/02_schema_memory.sql + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Current canonical memory schema includes expanded enums and embedding tables; docs that omit these must be updated. | low | No SQL issue found; use this file as canonical source for doc fixes. | +| polish | `source_chunk_embedding` has `chunk_id`, `doc_id`, and `workspace_id`, but no composite FK ensuring chunk/doc/workspace consistency like `memory_embedding` has for memory/workspace. | medium | Consider adding composite uniqueness/FKs for `source_chunk(chunk_id, doc_id)` and `source_document(doc_id, workspace_id)` if you want DB-level tenant consistency for chunk embeddings. This is not required for final report if documented as current design. | + +### database/03_schema_governance.sql + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Governance schema matches current API and fixture. | low | No blocking issue found. | +| polish | `wiki_page.status` remains `active/forgotten`; ensure frontend/docs do not use stale/archived values. | low | Already fixed in frontend per prior review; keep docs aligned. | + +### database/04_indexes.sql + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Manual indexes align with `docs/index-rationale.md` and current schema. | low | No blocking issue found. | +| report-fit | Good source for DB-course depth; include BRIN, GIN, covering, partial unique, and B+ tree/B-link in final report. | low | No SQL change needed. | + +### database/05_views.sql + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| cross-doc consistency | `v_agent_visible_memory` semantics are misdescribed in `docs/06` and misused in `database/08_demo_queries.sql`; the view itself is per-agent materialized visibility, not session-setting dependent. | high | Keep view, but fix docs/demo query to filter `WHERE agent_id = ...`. | +| code-consistency | Views otherwise align with current schema. | low | No view SQL change required for gap 9. | + +### database/06_triggers.sql + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Trigger set aligns with memory/wiki/conflict governance lifecycle. | low | No blocking SQL issue found. | +| cross-doc consistency | `docs/02` implies source import writes audit log, but triggers do not cover source inserts. | high | Fix doc or add source audit later. | +| polish | `fn_memory_soft_delete` says workspace cascade allows hard delete by checking workspace existence; this is a deliberate design, but final report should explain it to avoid "delete trigger blocks cascade" confusion. | low | Add a note in final report if discussing soft delete. | + +### database/07_seed.sql + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Seed uses current memory enum values and sets up the course demo workspace. | low | No blocking issue found. | +| report-fit | Seed says LLM extraction is not required for MVP, but current system now has rule-based extraction. | low | In final report, distinguish "LLM not required" from "candidate extraction delivered". | + +### database/08_demo_queries.sql + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Query #5 sets `app.agent_id` and then selects from `v_agent_visible_memory` without `WHERE agent_id = ...`; the view does not read `app.agent_id`. This can show all agents' visible memories in multi-agent data. | high | Replace with `WHERE agent_id = '00000000-0000-0000-0000-000000000301'` and remove or explain the unused `set_config`. | +| report-fit | Demo queries do not include the newer embedding/hybrid retrieval, graph, or extraction audit paths. | medium | Add optional final-report/demo queries for `memory_embedding`, `recall_log.context_pack_json->'retrieval_info'`, and `audit_log` extraction run events. | + +### database/10_governance_demo_fixture.sql + +| Dimension | Issue | Severity | Suggested fix | +|---|---|---|---| +| code-consistency | Fixture is transaction-wrapped, idempotent, and matches current governance schema. | low | No blocking issue found. | +| cross-doc consistency | Comment references `docs/governance-demo-walkthrough.md (如有)`, but that file does not exist. | low | Either add the walkthrough later or remove the reference before final submission. | +| polish | Fixture inserts a resolved conflict directly to avoid marking endpoints conflicted; that is valid but should be described as demo fixture design, not the normal conflict lifecycle. | low | Mention in final report/demo notes if using it as evidence. | + +## Cross-doc inconsistencies + +- `docs/03-data-dictionary.md` lists `memory_type` as `episodic/semantic/profile/procedural/decision/preference/task/risk`, while `docs/08-api-design.md`, `docs/normalization.md`, and `database/02_schema_memory.sql` include `fact/constraint/policy/summary`. Resolution: `database/02_schema_memory.sql` wins; update `docs/03`. +- `docs/03-data-dictionary.md` lists memory `status` as `active/archived/forgotten/superseded/conflicted`, while `docs/08-api-design.md`, `docs/normalization.md`, and `database/02_schema_memory.sql` include `candidate` and `rejected`. Resolution: current schema wins; update `docs/03`. +- `docs/06-physical-design.md` and `database/08_demo_queries.sql` treat `v_agent_visible_memory` as `app.agent_id`-driven, while `database/05_views.sql`, `recall_service.py`, `search_service.py`, `graph_service.py`, and `governance_service.py` all explicitly filter `agent_id`. Resolution: services/current view semantics win; update doc and demo query. +- `docs/02-data-flow.md` says Source import writes `AuditLog`, while `backend/app/services/source_service.py` and `database/06_triggers.sql` do not implement source import audit. Resolution: either implement source audit in a later gap or remove that claim from DFD/final report. +- `docs/00-project-overview.md` and `docs/01-requirements.md` frame LLM/semantic extraction as future-only; `docs/08-api-design.md` and `memory_extraction_service.py` now deliver rule-based candidate extraction. Resolution: final report should say rule-based v1 is delivered; high-quality LLM extraction/v2 analysis tables remain planned. +- `docs/07-system-architecture.md` lists obsolete backend paths (`routers/`, `ingest_service.py`, `policy_service.py`, `audit_service.py`), while current code lives under `backend/app/api/` and consolidated services such as `source_service.py`, `governance_service.py`, `graph_service.py`, `embedding_service.py`, and `llm_service.py`. Resolution: rewrite `docs/07`. +- `docs/11-demo-script.md` says the demo imports 6 discussion records after `npm run db:setup`, while `database/07_seed.sql` already seeds those records. Resolution: final demo should choose pre-seeded display or a live extra import. +- `docs/10-test-plan.md` covers only early Source/Memory/Recall/Wiki/Audit tests, while `backend/tests/` now covers graph, embeddings, evaluation, QA, semantic, stats, CLI, search, and context pack. Resolution: expand test plan from actual test files. + +## Schema drift + +Canonical source wins in this order for final report claims: `database/*.sql` for schema/index/view/trigger facts, `backend/app/models` and `backend/app/api` for API contract facts, `backend/app/services` for behavioral facts, and `backend/tests` for test coverage facts. + +Drift to fix before final report: + +- Memory enums: use the current 12 memory types and 7 memory statuses from `database/02_schema_memory.sql`. +- Embedding cache: include `memory_embedding` and `source_chunk_embedding` in logical/physical/data-dictionary docs. +- Agent visibility: describe `v_agent_visible_memory` as a per-agent view requiring caller-side `agent_id` filtering, not as a view reading `app.agent_id`. +- Candidate extraction: document `memory_item(status='candidate')` + `memory_evidence` + run-level `audit_log` as delivered v1; keep three-table LLM analysis design as future only. +- Architecture/API scope: add graph, embeddings, QA, stats, semantic, CLI, and evaluation as delivered extensions or clearly mark API docs as selected core summary. +- Source audit: either add source import audit later or remove current DFD claim. + +## Files marked "needs rewrite" + +### docs/07-system-architecture.md + +This file is too stale to enter the final report. It points to non-existent `routers/` and service filenames and omits major delivered modules. Proposed outline: (1) current component architecture, (2) backend API/service/repository layout, (3) frontend pages, (4) CLI runtime, (5) PostgreSQL + optional Neo4j graph sync, (6) embedding/QA/evaluation extension paths, (7) data/control flow across these layers. + +### docs/13-final-report-outline.md + +This file is only an outline, not a deliverable report. It also under-represents the latest merged work. Proposed rewrite outline: full report with database-first mixed narrative, teacher checklist, requirements, related work, schema/ER/logical/physical design, normalization, index/EXPLAIN, API/module design, implementation, governance/provenance/evaluation innovations, screenshots, tests, member contributions, and source/SQL appendices. diff --git a/docs/process/audit-review.md b/docs/process/audit-review.md new file mode 100644 index 0000000..167d101 --- /dev/null +++ b/docs/process/audit-review.md @@ -0,0 +1,152 @@ +# Gap 9 — Audit Review (claude pass on codex findings) + +**Reviewer**: claude +**Auditee**: codex (`docs/process/audit-findings.md`) +**HEAD**: `2dd0d64` +**Verdict**: **APPROVED — accurate findings, ready for fix execution after addressing supplements below.** + +## 1. Spot-check verification (7 high-severity claims) + +| Finding | Verified against | Status | +|---|---|---| +| `v_agent_visible_memory` joins `agent`, requires caller-side `WHERE agent_id` | `database/05_views.sql:179-230` | ✓ confirmed | +| `memory_type` has 12 values incl. fact/constraint/policy/summary | `database/02_schema_memory.sql:61-77` | ✓ confirmed | +| `memory_item.status` has 7 values incl. candidate/rejected/conflicted | `database/02_schema_memory.sql:86-97` | ✓ confirmed | +| docs/07 references obsolete `routers/` + ingest_/policy_/audit_service | `ls backend/app/` shows `api/` + `source_service/governance_service/...` | ✓ confirmed | +| `source_service.py` does NOT write audit_log | `grep audit_log backend/app/services/source_service.py` = empty | ✓ confirmed | +| `trg_memory_before_update` (not after) maintains `current_revision_no` | `database/06_triggers.sql:107,116` | ✓ confirmed | +| `database/08_demo_queries.sql` query #5 misuses view | docs/08 + view definition | ✓ confirmed | + +All 7 hold. + +## 2. Push-back / nuance + +### 2.1 `docs/explain-analyze.md` Case 3.2 — re-classify high → medium + +Codex flags Case 3.2 (`idx_audit_target_time` trailing-column scan) as **high**. The doc itself is honest about the composite index structure and acknowledges the planner choice at small scale (lines 247, 295-301). The real risk is just that the pasted EXPLAIN may have come from a different DB state. **Action**: re-run Case 3 on fresh `npm run db:setup` before final-report inclusion — not a "rewrite" item. Re-classify as **medium with "verify before report inclusion" tag**. + +### 2.2 `docs/03-data-dictionary.md` — extend sweep beyond two enum rows + +Codex flags `memory_type` and `status` enum rows. Spot-check confirms drift, but codex's row-level fix doesn't mandate a full doc sweep. The doc is 80+ lines; other rows (e.g., data structure summary, AuditLog column names already flagged) likely have similar drift. **Action**: gap 9 fix pass on docs/03 must sweep entire doc, not just enum rows. + +## 3. Supplements — what codex did not call out + +### S1. SQL inline comment density is materially deficient + +Teacher slide explicitly requires **"具有注释的源程序，包括高级语言、SQL 语言等"**. Codex marked all SQL files "no issues found" on code-consistency, but did not assess comment density. I counted leading-whitespace `--` comment lines per file: + +| File | lines | comment lines | density | +|---|---|---|---| +| 01_schema_core.sql | ? | **0** | ⚠ | +| 02_schema_memory.sql | 221 | **0** | ⚠ | +| 03_schema_governance.sql | ? | **0** | ⚠ | +| 04_indexes.sql | ? | 18 | OK | +| 05_views.sql | ? | **0** | ⚠ | +| 06_triggers.sql | 337 | **0** | ⚠ | +| 07_seed.sql | ? | 3 | thin | +| 08_demo_queries.sql | ? | 13 | OK | +| 10_governance_demo_fixture.sql | ? | 66 | OK | + +**Five core SQL files (schema_core / schema_memory / schema_governance / views / triggers) have zero line comments.** For the SQL-program-with-comments appendix requirement, these must be annotated before submission. **Add to gap 8 task scope: SQL annotation pass on the five files above.** Bare minimum: `--` block above each CREATE TABLE / INDEX / VIEW / TRIGGER explaining its purpose, business meaning, and any non-obvious design choice. + +### S2. Report-readiness disposition map (no such overview in findings) + +Codex's per-file findings (26 files × 2-4 rows) are dense. Stanley needs an at-a-glance view of "what to do with each file before report assembly": + +| File | Disposition | Pre-report action | +|---|---|---| +| docs/00 | reusable + edit | retitle to broader org/team memory; relabel P0/P1/P2 | +| docs/01 | reusable + edit | add extension capabilities; reclassify Entity/Scene/Forget; trim admin role | +| docs/02 | reusable + edit | fix source-import audit claim; update Recall DFD with hybrid path | +| docs/03 | partial rewrite | full-doc sweep for enum/type drift; add embedding cache | +| docs/04 | source OK, render needed | Mermaid is good; claude renders to PNG/SVG in gap 5 | +| docs/05 | reusable + edit | add embedding cache section; fill ConflictRecord missing fields | +| docs/06 | reusable + edit | fix v_agent_visible_memory description; mark SQLite as design-only | +| docs/07 | **REWRITE** | full re-author against current `backend/app/`; add CLI/graph/embed/QA | +| docs/08 | reusable + edit | add extension API families (graph/embeddings/QA/stats/semantic) | +| docs/09 | partial rewrite | add Extraction/Search/CLI/Graph/Embedding/QA/Stats/Semantic/Eval modules | +| docs/10 | partial rewrite | rebuild coverage matrix against actual `backend/tests/test_*.py` | +| docs/11 | reusable + edit | reconcile seed-vs-import for 6 records; verify 3 wiki pages exist post-setup | +| docs/13 | **REWRITE → final-report.md** | this IS gap 1 | +| docs/normalization.md | reusable + edit | fix `trg_memory_before/after` wording (§1.4) | +| docs/index-rationale.md | reusable + edit | "8 类" → "9 类" | +| docs/explain-analyze.md | re-verify | re-run Case 3 + 4 on current DB; refresh plans | +| db/00-07 + 10 | no SQL change for gap 9 | use as canonical for doc fixes; **gap 8 will add comments** | +| db/08_demo_queries.sql | small fix | query #5 `WHERE agent_id = ...` | + +**Two REWRITE items (docs/07, docs/13) are the highest-leverage edits.** docs/13 is the report itself (gap 1); docs/07 must be rewritten BEFORE report assembly because gap 1 will reference it. + +### S3. ER rendering toolchain decision needed + +Codex says "export PNG/SVG into final assets" but doesn't specify the toolchain. **Decision needed before gap 5**: + +- (a) **`mermaid-cli` (`mmdc`)** — keeps `.mmd` source in repo, deterministic re-render, no manual web steps. Requires Node + Chrome/Puppeteer. +- (b) dbdiagram.io — manual web export, no toolchain +- (c) drawio offline — manual editing +- (d) Mermaid embedded in PDF via Pandoc + mermaid-filter + +**Recommend (a)** for source-of-truth in repo + reproducibility. Confirm in gap 5 kickoff. + +### S4. `docs/governance-demo-walkthrough.md` — referenced but absent + +`database/10_governance_demo_fixture.sql` references `docs/governance-demo-walkthrough.md (如有)` but the file doesn't exist. Codex flags as low; I think it's medium because it interacts with gap 4 screenshots: the walkthrough would list the exact UI steps to capture. **Recommend**: write the walkthrough as part of gap 1's demo section (or as a sibling doc) BEFORE gap 4 screenshots. + +### S5. Gap ordering: gap 9 fixes MUST land before gap 4 screenshots + +Implication codex didn't make explicit: if we screenshot the demo with docs/02 claim "source import writes AuditLog" still in the report but the system doesn't, the screenshots will visibly contradict the report. **Hard constraint**: complete gap 9 doc-fix pass BEFORE gap 4 starts. The current task dependency graph (`#21 blocks #30 blocks #22/23/24/26/29`) is correct. + +### S6. docs/07 rewrite must verify frontend page list + +Codex flags docs/07 stale paths and obsolete service names. Cross-checking, `frontend/src/pages/` contains: `Dashboard.jsx, governance/, graph/, memories/, recall/, runtime/, sources/, wiki/`. docs/07's frontend list (Dashboard / Sources / Memories / Recall / Wiki / Timeline / Audit / Conflicts) **omits the new graph/runtime pages and inaccurately splits Timeline/Audit/Conflicts as top-level pages** (they actually live under governance/runtime). When rewriting docs/07, explicitly verify against `ls frontend/src/pages/`. + +## 4. Priority ordering for fix execution + +Recommend three tiers; execute Tier A first since they block gap 1 + gap 4. + +**Tier A (blocks gap 1 report assembly + gap 4 screenshots):** + +1. docs/07 full rewrite (against current `backend/app/` + `frontend/src/pages/`) +2. docs/03 enum + embedding full-doc sweep +3. docs/02 source-import audit claim fix +4. docs/06 v_agent_visible_memory description + SQLite wording +5. database/08 query #5 `WHERE agent_id` fix +6. docs/normalization.md `trg_memory_before/after` wording + +**Tier B (improves report quality, gap 1 can start in parallel):** + +7. docs/00 title + label cleanup +8. docs/01 extension capabilities + Entity/Scene/Forget reclassification + admin role rewrite +9. docs/05 embedding cache + ConflictRecord missing fields +10. docs/08 extension API families +11. docs/09 module table expansion +12. docs/10 test coverage matrix rebuild +13. docs/11 demo seed-vs-import reconciliation + 3-wiki verification + +**Tier C (polish, can land anytime before submission):** + +14. docs/index-rationale "8 类" → "9 类" +15. docs/explain-analyze re-run Case 3 + Case 4 on fresh DB +16. docs/04 add WorkspaceMember + embeddings or note "simplified core ER" +17. database/10 dangling `docs/governance-demo-walkthrough.md` reference (or write the walkthrough) + +**docs/13 → docs/final-report.md is NOT in this list — that IS gap 1 (Task #22), not a gap 9 fix.** + +## 5. Execution mode recommendation + +Stanley to choose between two modes for Tier A/B: + +- **(α) Batch fix → single review**: codex executes all Tier A in one pass, claude reviews entire result, stanley approves. **Recommend for Tier A** — these are mechanical drift fixes with clear canonical sources (`database/*.sql` and `backend/app/*` are the truth). Per-file review on mechanical fixes is overkill. +- **(β) Per-file fix → per-file review**: slower but more careful. **Recommend for docs/07 rewrite specifically** — this is structural rework where claude should review the new outline before codex commits the rewrite. + +**Suggested split**: Tier A items 2/3/4/5/6 batch via (α); docs/07 rewrite (item 1) via (β) with structural outline review first. + +## 6. Open questions for stanley + +1. **mermaid-cli OK for ER rendering?** (vs dbdiagram.io / drawio / pandoc-mermaid) +2. **Write `docs/governance-demo-walkthrough.md`?** (mid-S4) +3. **Execution mode**: accept the (α)+(β) split in §5, or different? +4. **gap 8 scope expansion**: confirm SQL annotation pass on 5 schema files (S1) is added to Task #29 gap 8. + +## 7. Approval gate + +Gap 9 audit + this review are **complete**. Approval to proceed = stanley answers §6, then codex starts Tier A fixes. Task #21 + #30 can close once stanley approves. diff --git a/docs/research-landscape.md b/docs/research-landscape.md new file mode 100644 index 0000000..3a6b285 --- /dev/null +++ b/docs/research-landscape.md @@ -0,0 +1,114 @@ +# 本领域研究现状分析 + +> 用途：可直接纳入最终报告第 2 章“项目背景与研究现状”。本节以数据库课程报告为目标，不把 MemoryBase 写成普通聊天机器人，而是定位为“面向组织与团队的 AI-native 可追溯长期记忆数据库系统”。 + +## 2.1 研究背景 + +随着大语言模型和 Agent 应用的发展，长期记忆已经从“把历史聊天塞进 prompt”演变为一个独立系统问题。团队协作、软件研发、课程项目、企业知识库等场景中，信息来源通常分散在 Markdown 文档、会议纪要、聊天记录、Issue、Wiki、数据库条目和自动化工具日志中。人类用户希望快速找到“当时为什么这样决策”，Agent 则需要在执行任务前读取可信上下文、写入新观察、更新旧结论，并在必要时忘记、归档或审计敏感内容。 + +传统数据库课程中的信息系统通常关注结构化业务数据，例如用户、订单、库存、审批记录等；而 AI 时代的“记忆系统”面对的是半结构化或非结构化文本。单纯文件系统便于人阅读和版本管理，但缺少关系约束、权限过滤和审计；单纯向量库便于语义检索，但难以表达“这条记忆来自哪段原文、何时被谁修改、哪些 Agent 可见、是否已经被遗忘”；单纯聊天产品的记忆功能面向个人体验，通常不暴露完整 schema、触发器、审计链路和可复现查询。 + +因此，本项目关注的问题不是“如何再做一个聊天助手”，而是：如何把组织长期知识编译成一个可查询、可追溯、可治理、可被人和 Agent 同时使用的数据库系统。 + +## 2.2 主要技术路线 + +### 2.2.1 传统 RAG 与向量检索 + +Retrieval-Augmented Generation（RAG）是当前知识增强 LLM 应用的基础范式。Lewis 等人在 RAG 论文中指出，参数化模型难以精确访问和更新知识，也难以提供决策 provenance，因此引入外部非参数记忆，并用检索结果辅助生成。工程上，LangChain、LlamaIndex 等框架进一步普及了“切分文档 → 建 embedding → 向量召回 → 拼接上下文 → 交给 LLM”的流程。 + +这一路线的优点是实现快、语义召回强、容易接入已有文档库。但它的数据库建模通常较弱：chunk、向量、metadata 往往被当作检索材料，而不是有完整生命周期的业务对象。实际系统还需要回答“哪条 memory 支撑了这次回答”“source 被删除后是否还能被召回”“谁有权限看 private memory”“一次 recall 为什么 fallback 到 keyword”，这些问题很难只靠向量相似度解决。 + +MemoryBase 保留 RAG 的检索思想，但没有把向量库作为唯一核心。系统以 PostgreSQL 关系模型为主，使用 `source_document`、`source_chunk`、`memory_item`、`memory_evidence`、`recall_log` 等表保存来源、证据、召回结果和可观测信息；embedding cache 和 hybrid recall 被设计为可选增强，并提供 keyword fallback，保证没有外部模型或向量服务时仍能完成核心演示。 + +在向量存储实现上，本项目也刻意没有把 `pgvector` 作为默认依赖。当前 schema 使用 `memory_embedding` 和 `source_chunk_embedding` 两张表，以 JSONB 存储 `embedding_json`，再由服务层做 cosine similarity。这一选择牺牲了大规模 ANN 检索性能，但换来部署简单、课程演示稳定、无需额外数据库扩展、fallback 行为可观测。若后续扩展到更大数据量，`pgvector` / IVFFlat / HNSW 仍然是自然演进方向。 + +### 2.2.2 Agent 长期记忆系统 + +近年来出现了多种面向 Agent 的长期记忆系统。MemGPT / Letta 把 LLM 上下文窗口类比为操作系统中的层级内存，用 core memory、archival memory 和工具调用来管理跨会话状态。Mem0 则强调生产环境中的长期记忆层，通过动态抽取、合并、检索对话中的显著信息，并进一步探索 graph memory 表达关系。MemoryBank、LongMem、Generative Agents 等研究也从不同角度证明了“观察、反思、长期存储、动态召回”对 Agent 行为一致性的重要性。 + +这些工作共同说明：长期记忆不是可有可无的 prompt 技巧，而是 Agent 系统的核心基础设施。它们的优势在于 agent-friendly，能自动抽取、压缩和召回信息，面向连续对话或长期交互有明显价值。但从数据库课程和组织治理角度看，它们通常更关注“如何让模型记得并用上”，较少把关系完整性、证据表、版本表、审计表、权限策略、遗忘审批和 SQL 可验证性作为一等公民。 + +MemoryBase 的设计选择是把 Agent 记忆问题落到关系数据库上。系统允许 Agent 通过 CLI/API 做 recall、search、observe、remember，但每次写入都落入可约束的表结构；每条 memory 可以追溯到 source chunk；修改会触发 revision 和 audit；Agent 可见范围通过 `v_agent_visible_memory` 和 policy 约束，而不是只依赖 prompt 自觉。 + +### 2.2.3 产品化记忆：ChatGPT Memory、Confluence AI、Notion AI 与个人知识库 + +ChatGPT Memory 代表了面向终端用户的产品化记忆能力。官方帮助文档区分 saved memories 与 reference chat history，用户可以查看、删除、关闭或管理记忆；启用后，系统会在后续对话中参考这些信息以提供更个性化的回答。这类功能降低了普通用户使用长期记忆的门槛，但它本质上是黑盒产品能力：用户通常不能直接看到底层 schema、完整日志、检索 SQL、触发器或多租户权限策略。 + +Confluence AI / Atlassian Rovo 是企业 wiki 和团队协作场景中更直接的对照对象。Atlassian 把 Rovo 定位为连接 Confluence、Jira、Slack 和企业应用的 AI 方案，包含 search、chat、agents、studio 等能力；Rovo agents 也可以在 Confluence / Jira 编辑与自动化流程中协作，帮助生成、整理或修改团队内容。这说明企业知识管理正在从“静态文档库”走向“带 Agent 的工作流知识系统”。但从本课程项目角度看，Rovo 的核心价值在 Atlassian 生态集成与协作体验，而不是开放一个可由学生展示的关系 schema、触发器、SQL EXPLAIN、审计表和可复现实验数据库。 + +Notion AI Enterprise Search、Obsidian、Logseq 等工具则代表知识管理路线。Notion AI 可以在 workspace、连接器和 web 中搜索，并给出来源引用；Obsidian 以本地 Markdown、内部链接、反向链接和文件系统可控性为核心，适合构建个人或团队知识库。这一路线在人类可读性、文档组织和手工维护方面很强，但对 Agent 来说，文件链接和全文搜索仍不足以表达严谨的数据生命周期：记忆是否 active、archived、forgotten、candidate？一次编辑是否留下不可变 revision？一个 forget request 是否被审批？这些治理问题通常不在普通笔记软件的核心模型里。 + +MemoryBase 吸收文件型知识库的优点：source 保留原文，wiki 可导出 Markdown，CLI 支持 grep-style 查询；同时用数据库补齐关系、状态、约束、审计和权限。也就是说，文件系统负责人类可读与可迁移，数据库负责治理与可验证。 + +### 2.2.4 长期记忆评测基准 + +长期记忆系统需要评测，而不仅是演示。LoCoMo 提供多 session、长对话、事件图和问答/总结任务，用于评估模型对长期对话的理解；LongMemEval 关注聊天助手在信息抽取、多 session 推理、时间推理、知识更新和拒答等能力上的表现；MemoryAgentBench 进一步把 memory agent 的能力拆成 accurate retrieval、test-time learning、long-range understanding 和 selective forgetting。 + +这些 benchmark 的共同趋势是：从静态长上下文问答转向动态、多轮、可更新、可遗忘的长期记忆能力。它们也暴露出一个现实问题：单纯长上下文或普通 RAG 在时间关系、冲突更新、选择性遗忘、跨 session 证据整合上仍有不足。 + +MemoryBase 在代码中已经接入 evaluation framework，包含 LoCoMo、LongMemEval、MemoryAgentBench 等 adapter，以及 retrieval、QA、forgetting、system 等指标。对本课程项目而言，评测不是主线替代品，而是证明数据库设计能服务长期记忆任务的扩展证据。 + +如果在 LoCoMo / LongMemEval 等原始 recall accuracy 上，MemoryBase 不一定超过 Mem0、Letta 这类专门优化长期记忆效果的产品，这并不构成本项目失败。MemoryBase 的评价重点是数据库系统能力：provenance、governance、multi-tenant visibility、audit trail、SQL-verifiable lifecycle 和可降级检索。评测结果应被解释为“长期记忆任务上的验证信号”，而不是唯一产品目标。 + +### 2.2.5 GraphRAG 与知识图谱记忆 + +GraphRAG 代表了另一条重要路线：在普通 RAG 的 chunk / vector 之外，先从私有语料中抽取实体、关系和社区结构，再结合局部图搜索与全局摘要回答问题。Microsoft GraphRAG 的核心动机是让系统能回答跨文档、跨主题的 global sensemaking 问题，而不只是做 top-k chunk 拼接。Mem0 等长期记忆系统也在探索 graph memory，用图结构表达人物、事件、偏好和事实之间的关系。 + +MemoryBase 已经实现 Graph Explorer、PostgreSQL workspace graph preview 和可选 Neo4j sync，但项目定位与 GraphRAG 有意区分：图不是默认检索主路径，也不是替代 PostgreSQL 的事实源。当前图层主要服务 provenance 与 governance 可视化：展示 source、chunk、memory、evidence、wiki、conflict、forget request 等对象之间的关系；Neo4j 只是可选关系索引和展示后端，PostgreSQL 仍是权威数据源。这样既吸收图结构的可解释性，也避免把课程主线变成复杂知识图谱构建项目。 + +## 2.3 研究现状对比 + +| 路线 / 代表 | 主要能力 | 优点 | 局限 | 对 MemoryBase 的启发 | +|---|---|---|---|---| +| 传统 RAG / LangChain / LlamaIndex | 文档切分、向量索引、语义召回、上下文拼接 | 上手快，适合知识问答，生态成熟 | chunk 关系弱，生命周期和审计弱，权限与遗忘通常需额外实现 | 保留检索能力，但把 source、memory、evidence、recall 关系化 | +| MemGPT / Letta | core memory、archival memory、stateful agent、工具管理上下文 | 面向 Agent，能跨会话管理上下文 | 更像 Agent runtime，数据库层 provenance 和治理不是主目标 | Agent 可用 API/CLI，但记忆写入必须落入可审计关系模型 | +| Mem0 / graph memory | 动态抽取、合并、检索长期记忆，探索图结构记忆 | 生产化记忆层，强调低延迟和 token 成本 | 更重 memory 效果，较少强调课程数据库层面的约束、触发器和审计 | graph / hybrid retrieval 可作为增强，核心仍是可验证 DB schema | +| ChatGPT Memory | saved memories、reference chat history、用户管理记忆 | 普通用户体验好，自动化程度高 | 产品目标是消费者体验，没有公开 schema / SQL / 数据库治理接口 | 说明用户确实需要长期记忆，但 MemoryBase 要做可解释、可治理版本 | +| Confluence AI / Rovo / Notion AI / Obsidian | enterprise search、workspace AI、Markdown 知识库、链接和引用 | 人类可读，适合组织文档和知识管理 | 对课程要求的关系 schema、触发器、EXPLAIN、遗忘审批链路展示不足 | 保留 Markdown/Wiki 友好性，同时加入关系数据库治理 | +| LoCoMo / LongMemEval / MemoryAgentBench | 长期记忆能力评测 | 让系统效果可量化 | benchmark 不等同产品，且不能替代数据库设计 | evaluation 作为验证层，主线仍是数据库建模与治理 | +| GraphRAG / graph memory | 实体关系图、社区摘要、局部/全局图检索 | 适合跨文档关系理解和复杂语料组织 | 图构建成本高，若作为主路径会偏离课程数据库主线 | Graph Explorer / Neo4j 作为 provenance 可视化增强，PostgreSQL 仍是事实源 | + +## 2.4 现有方案的共同不足 + +综合以上路线，可以看到当前长期记忆系统存在四类空缺： + +1. **可追溯性不足**：许多系统能“记住并召回”，但不能稳定说明记忆来自哪个 source、哪个 chunk、哪次编辑、哪次审批。 +2. **关系建模不足**：向量库擅长相似度，但对 memory、source、evidence、revision、audit、policy、wiki projection 之间的关系表达不够自然。 +3. **治理能力不足**：组织场景需要权限、冲突、遗忘、归档、审计和多角色访问。个人记忆产品通常无法直接满足这一层要求。 +4. **课程可验证性不足**：很多 AI memory 系统是黑盒或框架调用，难以展示数据库课程要求的 E-R 图、关系模式、范式分析、索引、视图、触发器、SQL 查询和 EXPLAIN。 + +这也是 MemoryBase 的切入点：不是追求最强 LLM 自动抽取，也不是替代 Notion 或向量数据库，而是把长期记忆作为一个数据库应用系统来建模。 + +## 2.5 本项目定位 + +MemoryBase 的核心 thesis 是：组织级长期记忆需要同时服务人和 Agent。人需要可读 Wiki、来源引用、审计记录和演示界面；Agent 需要可查询 API、context pack、权限可见视图和可持续写入的记忆层；数据库课程则要求结构化设计、完整约束和可执行 SQL 证据。 + +因此，本项目选择“文件—数据库双态”架构： + +- 文件侧：保存 Markdown / txt source，导出 Markdown Wiki，保持人类可读、可迁移、可 grep。 +- 数据库侧：用关系模型管理 source、chunk、memory、evidence、revision、audit、policy、conflict、forget request、recall log。 +- AI 侧：提供 recall/search/context-pack/CLI/Agent runtime/evaluation，让 Agent 可以使用数据库记忆，但不把 LLM 作为 P0 依赖。 + +与现有方案相比，MemoryBase 的差异化不在于“又做一个向量检索”或“又做一个聊天记忆”，而在于把 provenance、governance、multi-tenant visibility 和 SQL-verifiable lifecycle 放在系统中心。这一点也更符合数据库课程大作业的评价重点：概念结构、逻辑结构、物理结构、索引、视图、触发器、完整性约束和可复现查询都能在项目中找到对应实现。 + +## 参考资料 + +- Lewis et al., [Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks](https://arxiv.org/abs/2005.11401), 2020. +- Park et al., [Generative Agents: Interactive Simulacra of Human Behavior](https://arxiv.org/abs/2304.03442), 2023. +- Zhong et al., [MemoryBank: Enhancing Large Language Models with Long-Term Memory](https://arxiv.org/abs/2305.10250), 2023. +- Packer et al., [MemGPT: Towards LLMs as Operating Systems](https://arxiv.org/abs/2310.08560), 2023. +- Maharana et al., [Evaluating Very Long-Term Conversational Memory of LLM Agents / LoCoMo](https://arxiv.org/abs/2402.17753), 2024. +- Wu et al., [LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory](https://arxiv.org/abs/2410.10813), 2024. +- Chhikara et al., [Mem0: Building Production-Ready AI Agents with Scalable Long-Term Memory](https://arxiv.org/abs/2504.19413), 2025. +- Hu et al., [Evaluating Memory in LLM Agents via Incremental Multi-Turn Interactions / MemoryAgentBench](https://arxiv.org/abs/2507.05257), 2025. +- OpenAI Help Center, [Memory FAQ](https://help.openai.com/en/articles/8590148-memory-in-chatgpt). +- Letta Docs, [Agent memory and architecture](https://docs.letta.com/guides/agents/architectures/memgpt), accessed 2026-06. +- LangChain Docs, [Retrieval](https://docs.langchain.com/oss/python/langchain/retrieval). +- LlamaIndex Docs, [Vector Stores](https://docs.llamaindex.ai/en/v0.10.23/module_guides/storing/vector_stores/). +- Atlassian Support, [What is Rovo?](https://support.atlassian.com/rovo/docs/what-is-rovo/). +- Atlassian Support, [Rovo Agents](https://support.atlassian.com/rovo/docs/agents/). +- Atlassian, [Rovo in Confluence: AI features](https://www.atlassian.com/software/confluence/ai). +- Edge et al., [From Local to Global: A Graph RAG Approach to Query-Focused Summarization](https://arxiv.org/abs/2404.16130), 2024. +- Microsoft, [GraphRAG GitHub repository](https://github.com/microsoft/graphrag). +- Notion Help Center, [Enterprise Search](https://www.notion.com/en-gb/help/enterprise-search). +- Obsidian Help, [About Obsidian](https://obsidian.md/help/obsidian). diff --git a/docs/source-sql-appendix.md b/docs/source-sql-appendix.md new file mode 100644 index 0000000..ff8b8d2 --- /dev/null +++ b/docs/source-sql-appendix.md @@ -0,0 +1,127 @@ +# 带注释源程序 / SQL 附录 + +> 用途：回应老师“具有注释的源程序，包括高级语言、SQL 语言等”的要求。最终报告可将本文拆成“附录 A SQL 源程序”和“附录 B 高级语言源程序说明”，源码全文以仓库文件为准，报告正文引用代表性片段即可。 + +## 8.1 交付口径 + +MemoryBase 的源程序分为两类： + +1. **SQL 源程序**：位于 `database/`，包括扩展初始化、表结构、索引、视图、触发器、seed、演示查询和治理 fixture。 +2. **高级语言源程序**：位于 `backend/app/`、`frontend/src/`、`backend/scripts/`、`evaluation/`，包括 FastAPI 后端、React 前端、CLI、数据处理脚本和评测框架。 + +本项目不建议在最终报告中全文粘贴所有代码。更稳妥的做法是： + +- 报告附录列出源程序文件、功能说明和注释位置； +- 正文只摘取最能体现数据库课程要求的 SQL / Python / JSX 片段； +- 完整源码通过仓库文件提交，老师可以按本文清单逐项核对。 + +## 8.2 SQL 源程序清单 + +| 文件 | 类型 | 作用 | 注释覆盖 | 报告引用建议 | +|---|---|---|---|---| +| `database/00_init.sql` | SQL 初始化 | 启用 `pgcrypto` 和 `pg_trgm` 扩展 | 已补每个扩展用途注释 | 附录 A；物理设计章节引用 | +| `database/01_schema_core.sql` | DDL | 用户、workspace、Agent、成员表 | 每个核心表前有业务说明 | 附录 A；E-R / 逻辑设计引用 | +| `database/02_schema_memory.sql` | DDL | session、message、source、chunk、memory、evidence、embedding、entity、scene | 每个表前有业务说明，重点解释 evidence / embedding / composite FK | 附录 A；逻辑设计和创新点引用 | +| `database/03_schema_governance.sql` | DDL | Wiki、timeline、recall log、policy、forget、conflict、audit | 每个治理表前有业务说明 | 附录 A；governance 章节引用 | +| `database/04_indexes.sql` | SQL 索引 | B+ tree、GIN、BRIN、covering、partial indexes | 分组注释说明索引用途 | 索引设计章节重点摘录 | +| `database/05_views.sql` | SQL 视图 | active memory、provenance、agent visibility、statistics、conflict view | 每个视图前有用途注释 | 视图设计章节重点摘录 | +| `database/06_triggers.sql` | PL/pgSQL | revision、audit、soft delete、conflict lifecycle、wiki dirty bit | 函数 / 触发器均有用途注释 | 触发器设计章节重点摘录 | +| `database/07_seed.sql` | DML seed | 固定 UUID demo workspace、source、memory、evidence、policy、wiki | 已补阶段性注释，说明每段 seed 的演示意义 | 演示数据附录 | +| `database/08_demo_queries.sql` | SQL 查询 | 课程演示 SQL：provenance、visibility、audit、forget、stats | 查询前有说明注释 | 系统演示 / SQL 查询结果引用 | +| `database/09_graph_demo.sql` | DML seed | 可选 Graph Explorer 清爽图数据 | 已补阶段性注释，说明 graph 节点/边来源 | Graph demo 截图附录 | +| `database/10_governance_demo_fixture.sql` | DML fixture | forgotten / archived / superseded / resolved conflict fixture | 注释密度高，含幂等和治理说明 | Governance demo 附录 | + +## 8.3 关键 SQL 片段索引 + +| 课程要求 | 推荐展示文件 / 片段 | 说明 | +|---|---|---| +| 主键、外键、唯一约束、CHECK | `database/01_schema_core.sql`、`02_schema_memory.sql`、`03_schema_governance.sql` | 表级约束最集中 | +| M:N 关系转换 | `memory_evidence`、`memory_entity`、`memory_scene_cell` | 体现 E-R 到关系模型转换 | +| 范式与受控反规范化 | `memory_item.search_vector`、`current_revision_no`、`wiki_page.needs_rebuild` | 与 `docs/normalization.md` 对应 | +| 全文检索与模糊检索 | `database/04_indexes.sql` 的 GIN / trigram 索引 | 与 search / recall 服务对应 | +| 现代索引 | `idx_audit_brin_time`、`idx_memory_active_ranking` | BRIN + covering index 加分点 | +| 权限视图 | `v_agent_visible_memory` | Agent-aware visibility 关键 SQL | +| 来源追溯视图 | `v_memory_with_source`、`v_wiki_page_sources` | provenance-first 关键 SQL | +| 版本与审计触发器 | `trg_memory_after_insert`、`trg_memory_before_update`、`trg_memory_after_update` | revision + audit 自动化 | +| 软删除 | `trg_memory_soft_delete` | DELETE 转 archived | +| 冲突生命周期 | `trg_conflict_after_insert`、`trg_conflict_after_update` | open conflict 自动改变 memory 状态 | + +## 8.4 高级语言源程序清单 + +### 8.4.1 FastAPI 后端 + +| 文件 / 目录 | 作用 | 说明依据 | 报告引用建议 | +|---|---|---|---| +| `backend/app/main.py` | FastAPI app 创建、router 挂载、lifespan 清理 | 已补 app lifespan 和 `/api` router 注释 | 系统架构 / 后端入口 | +| `backend/app/api/deps.py` | 依赖注入、数据库单例、service/repository 组装、embedding/LLM provider 选择 | 已补依赖构造和 provider 选择注释 | 模块设计 / 系统实现 | +| `backend/app/api/*.py` | API router 层，定义 sources、memories、recall、search、wiki、governance、graph 等端点 | 代码结构清晰，主要通过路径和 Pydantic model 表达契约 | API 设计章节引用路径表 | +| `backend/app/models/*.py` | Pydantic request/response model | 类型注解即接口说明，适合 API contract 附录 | API request/response 附录 | +| `backend/app/core/database.py` | PostgreSQL 连接封装 | 简洁边界代码 | 系统实现可简述 | +| `backend/app/core/config.py` | 环境变量配置 | 类型化配置 | 部署说明引用 | + +### 8.4.2 后端 service / repository + +| 文件 | 作用 | 说明依据 | 报告引用建议 | +|---|---|---|---| +| `backend/app/services/source_service.py` | source 导入、checksum、chunk 写入、搜索文本处理 | 依靠现有函数边界，配合 SQL 注释说明 source/chunk | Source 导入模块 | +| `backend/app/services/memory_service.py` | memory 创建、更新、删除、inline agent evidence、生命周期校验 | 已补 lifecycle 和 agent inline evidence 注释 | Memory lifecycle 重点摘录 | +| `backend/app/services/recall_service.py` | keyword/vector/hybrid recall、visibility 过滤、fallback、recall_log | 已补 DB-first recall 和 vector fallback 注释 | Recall / context pack 重点摘录 | +| `backend/app/services/search_service.py` | FTS + trigram + title boost + RRF 多路 search | 依靠现有阈值调参注释 | Search API / RRF 说明 | +| `backend/app/services/context_pack_service.py` | 将 recall 结果格式化成 agent-ready Markdown | 依靠现有函数边界展示 context pack 构造 | Agent 使用章节 | +| `backend/app/services/governance_service.py` | policy、audit、conflict、forget、timeline、agent-visible memory | 已补 governance repository 和 lifecycle query 注释 | Governance 重点摘录 | +| `backend/app/services/memory_extraction_service.py` | rule-based candidate extraction、approve/reject、run audit | 已补 candidate 和 run-level audit 注释 | AI-assisted import 章节 | +| `backend/app/services/embedding_service.py` | local hashing / SiliconFlow embedding、JSONB cache、backfill | 依靠 provider 类名和方法结构 | Hybrid retrieval 附录 | +| `backend/app/services/graph_service.py` | PostgreSQL graph preview、Neo4j sync/load、graph visibility | 已补 optional Neo4j、batch sync、visibility 注释 | Graph Explorer 章节 | +| `backend/app/services/wiki_service.py` | Wiki list/detail/revision/export/batch export | 依靠现有函数边界，并与 `wiki_page` / `wiki_page_revision` SQL 对应 | Wiki projection 章节 | +| `backend/app/services/llm_service.py` | 可选 OpenAI-compatible QA | 依靠现有函数边界；P2 功能，不作为主线 | Future / optional QA | + +### 8.4.3 CLI / scripts / evaluation + +| 文件 / 目录 | 作用 | 说明依据 | 报告引用建议 | +|---|---|---|---| +| `backend/app/cli/main.py` | `memorybase` / `mb` CLI 入口 | 依靠 Typer command 结构和现有 docstring | Agent 工具入口 | +| `backend/app/cli/commands/*.py` | configure、health、context、recall、search、observe、remember、sessions、eval | 依靠命令函数名、参数说明和 Typer help | CLI 附录 | +| `backend/app/cli/client.py` | CLI HTTP client | 依靠现有函数边界，与 `frontend/src/api/client.js` 类似 | CLI 实现说明 | +| `backend/scripts/backfill_search_terms.py` | seed 后回填 `search_text_zh` | 依靠脚本入口和现有用途说明 | 搜索/分词附录 | +| `evaluation/` | benchmark adapter、metrics、runner、report | 依靠 README 和测试覆盖说明框架边界 | Evaluation 章节 | + +### 8.4.4 React 前端 + +| 文件 / 目录 | 作用 | 说明依据 | 报告引用建议 | +|---|---|---|---| +| `frontend/src/App.jsx` | 页面路由入口 | 依靠路由结构 | 前端页面总览 | +| `frontend/src/api/client.js` | 前端 API client、错误处理、query string 构造、各 API namespace | 已补错误处理、proxy、空参数注释 | API 对账和前端实现 | +| `frontend/src/components/Layout.jsx` | 导航与页面框架 | 依靠组件结构 | UI 架构 | +| `frontend/src/components/Toast.jsx` | 全局提示 | 依靠组件结构 | 前端辅助组件 | +| `frontend/src/pages/sources/*` | source 列表和详情 | 依靠页面名和 API 调用 | Source demo 截图 | +| `frontend/src/pages/memories/*` | memory CRUD、evidence、revision | 依靠页面名和 API 调用 | Memory inspector 截图 | +| `frontend/src/pages/recall/Recall.jsx` | recall、context pack、QA 同页演示 | 已补共享表单注释 | Recall demo 截图 | +| `frontend/src/pages/governance/*` | audit、policies、conflicts、forget requests、timeline | 依靠页面名和 API 调用 | Governance demo 截图 | +| `frontend/src/pages/runtime/*` | sessions、messages、hybrid search | 依靠页面名和 API 调用 | Agent runtime demo | +| `frontend/src/pages/graph/*` | Graph Explorer、SVG 渲染、布局 | 依靠组件结构，并与 `graph_service.py` 对应 | Graph demo 截图 | +| `frontend/src/pages/wiki/WikiExport.jsx` | Wiki export UI | 依靠组件结构，并与 `wiki_service.py` 对应 | Wiki demo 截图 | + +## 8.5 最终报告建议摘录 + +最终报告附录不需要贴全部源码，可选择以下代表片段： + +1. `database/02_schema_memory.sql`：`memory_item`、`memory_evidence`、`memory_embedding`。 +2. `database/05_views.sql`：`v_memory_with_source`、`v_agent_visible_memory`。 +3. `database/06_triggers.sql`：`fn_memory_before_update`、`fn_memory_after_insert`、`fn_memory_after_update`、`fn_memory_soft_delete`、`fn_conflict_after_update`。 +4. `backend/app/services/recall_service.py`：权限过滤 + hybrid fallback + `recall_log` 写入。 +5. `backend/app/services/memory_extraction_service.py`：candidate extraction + run-level audit。 +6. `backend/app/services/graph_service.py`：PostgreSQL graph preview + Neo4j sync + visibility filter。 +7. `frontend/src/api/client.js`：统一 API client。 +8. `frontend/src/pages/recall/Recall.jsx`：Recall / Context Pack / QA 页面闭环。 + +这 8 组片段覆盖 SQL DDL、SQL view、PL/pgSQL trigger、Python service、React UI，足够回应“高级语言 + SQL 源程序均具有注释”的要求。 + +## 8.6 对账结论 + +当前仓库已经具备： + +- SQL 源程序：11 个 `database/*.sql` 文件，核心 DDL / index / view / trigger / seed / query 均有注释。 +- 高级语言源程序：FastAPI、React、CLI、script、evaluation 均有清晰目录结构；代表性核心模块已补充解释性注释。 +- 报告路径：`docs/process/13-final-report-outline.md` 的“附录 A SQL 源程序”“附录 B 高级语言源程序说明”可直接引用本文作为清单来源。 + +剩余注意事项：正式报告排版时不要粘贴过长源文件全文，应以“代表片段 + 文件清单 + 仓库路径”方式呈现，避免报告主体被源码淹没。 diff --git a/evaluation/README.md b/evaluation/README.md index 7ed8ef8..0e91900 100644 --- a/evaluation/README.md +++ b/evaluation/README.md @@ -6,8 +6,9 @@ This framework measures MemoryBase as a database-backed agent memory system, not generic RAG demo. It is designed to evaluate retrieval, long-term QA, conflict handling, forgetting, preference following, performance, and external benchmark compatibility. -Phase E1 is intentionally minimal. It validates data formats, runs local baselines, writes -CSV files, and generates a markdown report. It does not call the real MemoryBase backend yet. +The framework supports local baselines and live end-to-end MemoryBase runs. Live QA modes +write session turns and memories, recall context, call the configured LLM provider, record +token usage and citations, then soft-delete memories created by the case. ## Data Format @@ -71,7 +72,7 @@ Run several baselines into one combined report: ```bash python evaluation/runners/run_all.py \ - --modes summary_memory,db_memory,db_extraction,naive_vector_rag \ + --modes summary_memory,db_qa,vector_qa,db_extraction_qa \ --api-base http://localhost:8000 \ --workspace cs3321-demo \ --agent codex \ @@ -81,15 +82,26 @@ python evaluation/runners/run_all.py \ When `--modes` is used, results are written under `evaluation/outputs//` and `benchmark_report.md` summarizes all modes together. -Phase E2 uses the current memory API directly: +Live retrieval modes use the current memory API directly: ```text case sessions -> /api/sessions + /api/observe -memory-bearing user turns -> /api/memories +memory-bearing turns -> /api/memories/batch query -> /api/recall ``` -This is not full automatic memory extraction yet. It is marked as `injection_mode=memory_api`. +Live QA modes call `/api/qa/answer` after writing the case: + +```text +db_qa direct memory write + hybrid recall + LLM answer +vector_qa embedding backfill + vector recall + LLM answer +db_extraction_qa source extraction + approval + hybrid recall + LLM answer +``` + +Live CLI runs create an isolated workspace per case by default and delete it with all +dependent rows after scoring. Use `--shared-workspace` to reuse the requested workspace, or +`--preserve-eval-data` to keep isolated failure data. Shared-workspace cleanup can only +soft-delete created memories because session and source delete APIs do not exist yet. Run individual suites: @@ -122,20 +134,24 @@ Implemented in Phase E1: - MRR - nDCG@10 - p50 / p95 / p99 latency helpers +- provider/model and prompt/completion/total token usage +- citation validity and deterministic citation-groundedness +- retrieval leakage and answer leakage +- retention/forgetting by measured history token length ``` -Reserved for later backend-connected phases: +Still model- or dataset-dependent: ```text -- groundedness - hallucination rate - LLM-as-judge score -- retention rate by real history length -- stale memory error rate from real conflict resolution -- privacy leakage rate from real forgetting execution -- token cost per answer +- embedding cost per answer ``` +LLM token cost per answer is estimated from a dated model-price snapshot when +the provider/model is known. The current DeepSeek snapshot assumes cache-miss +input pricing; update `evaluation/pricing.py` when provider prices change. + ## Baselines The framework reserves these modes: @@ -147,6 +163,9 @@ naive_vector_rag summary_memory db_memory db_extraction +db_qa +vector_qa +db_extraction_qa ``` `no_memory`, `recency_only`, `summary_memory`, `--dry-run`, `db_memory`, `db_extraction`, and `naive_vector_rag` execute. @@ -154,10 +173,33 @@ db_extraction backend. `db_memory` requires a running API, workspace, and optionally an agent. `db_extraction` imports each memory-bearing turn as a source, extracts candidate memories from chunks, approves them, and then recalls active memory. -`naive_vector_rag` also requires a running API. It writes evaluation memories through -the memory API, calls `/api/embeddings/backfill` with the local hashing provider, and -then calls `/api/recall` with `retrieval_mode=vector` so the result records backend -vector scoring fields. +`naive_vector_rag` also requires a running API. Direct-memory live modes batch up +to 500 evaluation memories per request, calls `/api/embeddings/backfill` with the +local hashing provider, and then calls `/api/recall` with +`retrieval_mode=vector` so the result records backend vector scoring fields. + +## Long-Context Retention + +```bash +python -m evaluation.runners.run_long_context_eval \ + --token-lengths 1000,10000,50000,100000 \ + --mode db_qa \ + --workspace cs3321-demo \ + --agent demo-retriever +``` + +The generator materializes filler history and records actual `cl100k_base` token counts. + +## API Performance + +```bash +python -m evaluation.runners.run_api_performance \ + --workspace cs3321-demo \ + --agent demo-retriever \ + --iterations 20 +``` + +Add `--include-qa` only when real model latency and provider cost are intended. ## Adding Cases @@ -187,20 +229,35 @@ evaluation/external//processed/ Adapters convert raw benchmark data into `EvaluationCase` JSONL. +Convert and execute supplied benchmark data in one command: + +```bash +python -m evaluation.runners.run_benchmark_eval \ + --benchmark longmemeval \ + --mode db_qa \ + --workspace cs3321-demo \ + --agent demo-retriever +``` + | Benchmark | Purpose | Current Status | | --- | --- | --- | -| LongMemEval | Long-term dialogue memory, temporal reasoning, abstention | TODO adapter skeleton | -| LoCoMo | Multi-session dialogue memory and event QA | TODO adapter skeleton | -| MemoryAgentBench | Memory-agent retrieval, learning, conflict tasks | TODO adapter skeleton | +| LongMemEval | Long-term dialogue memory, temporal reasoning, abstention | Official JSON format supported; independent judge pending | +| LoCoMo | Multi-session dialogue memory and event QA | Official QA format supported; event summarization pending | +| MemoryAgentBench | Memory-agent retrieval, learning, conflict tasks | Official Conflict Resolution parquet supported; other tasks pending | | BEIR / MS MARCO | Retriever/RAG only, not primary memory evidence | TODO adapter skeleton | | BEAM | Ultra-long context retention stress test | TODO adapter skeleton | -Manual download instructions and license notes should be added in -`evaluation/external/README.md` before storing raw benchmark data. +LongMemEval, LoCoMo, and MemoryAgentBench source, license, and execution +instructions are in their respective directories under `evaluation/external/`. +Equivalent notes are still required before using other official benchmark files. ## Current Limitations - `recency_only` and `summary_memory` are local baselines and do not call the backend. -- `db_memory` and `naive_vector_rag` use explicit memory API injection; they do not yet test automatic memory extraction or full agent answer generation. -- External adapters support common JSON/JSONL shapes but still need official dataset download and license documentation. +- Retrieval-only modes do not generate an LLM answer; use the QA modes for that path. +- External adapters cover the documented official LongMemEval, LoCoMo, and + MemoryAgentBench Conflict Resolution shapes; other benchmark variants still + need download, license, and format verification before being used as evidence. +- Citation-groundedness validates references against the returned citation map; it is not an LLM judge. +- Shared-workspace runs cannot delete evaluation sessions or imported source documents. - No API keys, database URLs, or real user data should be stored in benchmark files. diff --git a/evaluation/adapters/common.py b/evaluation/adapters/common.py index f27362c..ffc5c8e 100644 --- a/evaluation/adapters/common.py +++ b/evaluation/adapters/common.py @@ -17,8 +17,12 @@ def load_raw_records(raw_dir: Path) -> list[dict[str, Any]]: records.extend(_load_jsonl(path)) elif path.suffix.lower() == ".json": records.extend(_load_json(path)) + elif path.suffix.lower() == ".parquet": + records.extend(_load_parquet(path)) + elif not path.suffix: + records.extend(_load_json(path)) if not records: - raise FileNotFoundError(f"no .json or .jsonl benchmark files found in {raw_dir}") + raise FileNotFoundError(f"no JSON benchmark files found in {raw_dir}") return records @@ -76,6 +80,8 @@ def normalize_expected_answer(raw: dict[str, Any]) -> str | None: return value if isinstance(value, list) and value: return ", ".join(str(item) for item in value) + if isinstance(value, (int, float, bool)): + return str(value) return None @@ -152,6 +158,14 @@ def _load_jsonl(path: Path) -> list[dict[str, Any]]: return records +def _load_parquet(path: Path) -> list[dict[str, Any]]: + try: + import pyarrow.parquet as parquet + except ImportError as exc: + raise RuntimeError("pyarrow is required to convert Parquet benchmark files") from exc + return [item for item in parquet.read_table(path).to_pylist() if isinstance(item, dict)] + + def _normalize_session(raw: dict[str, Any], *, fallback_id: str) -> dict[str, Any]: session_id = str(raw.get("session_id") or raw.get("id") or fallback_id) turns_raw = raw.get("turns") or raw.get("messages") or [] diff --git a/evaluation/adapters/locomo_adapter.py b/evaluation/adapters/locomo_adapter.py index ec4fd02..776cceb 100644 --- a/evaluation/adapters/locomo_adapter.py +++ b/evaluation/adapters/locomo_adapter.py @@ -1,10 +1,10 @@ from __future__ import annotations +import re from pathlib import Path from .common import ( case_id, - category, load_raw_records, metadata_without_heavy_fields, normalize_expected_answer, @@ -18,6 +18,9 @@ def convert(raw_dir: Path, processed_dir: Path) -> Path: records = load_raw_records(raw_dir) cases: list[dict[str, object]] = [] for index, record in enumerate(records, start=1): + if _is_official_locomo_record(record): + cases.extend(_official_cases(record, index)) + continue qa_items = record.get("qa") or record.get("qas") or record.get("questions") conversation = ( record.get("conversation") @@ -48,7 +51,7 @@ def _case_from_record( return { "case_id": base_id, "source": "locomo", - "category": category(record), + "category": _category(record), "sessions": normalize_sessions(conversation), "query": normalize_query(record), "expected_answer": normalize_expected_answer(record), @@ -59,3 +62,166 @@ def _case_from_record( "expected_behavior": "answer", "metadata": metadata_without_heavy_fields(record), } + + +def _is_official_locomo_record(record: dict[str, object]) -> bool: + conversation = record.get("conversation") + return ( + isinstance(conversation, dict) + and isinstance(record.get("qa"), list) + and any(re.fullmatch(r"session_\d+", key) for key in conversation) + ) + + +def _official_cases(record: dict[str, object], index: int) -> list[dict[str, object]]: + sessions, dialog_index = _official_sessions(record) + sample_id = str(record.get("sample_id") or f"{index:06d}") + cases: list[dict[str, object]] = [] + qa_items = record.get("qa") + if not isinstance(qa_items, list): + return cases + for qa_index, qa in enumerate(qa_items, start=1): + if not isinstance(qa, dict): + continue + evidence_ids = _evidence_ids(qa.get("evidence")) + cases.append( + { + "case_id": f"locomo_{sample_id}_qa{qa_index:03d}", + "source": "locomo", + "category": _category(qa), + "sessions": sessions, + "query": normalize_query(qa), + "expected_answer": normalize_expected_answer(qa), + "expected_answer_contains": [], + "forbidden_answers": _forbidden_answers(qa), + "forbidden_patterns": [], + "gold_memory_ids": evidence_ids, + "expected_behavior": _expected_behavior(qa), + "metadata": { + **metadata_without_heavy_fields(qa), + "sample_id": sample_id, + "qa_index": qa_index, + "evidence": evidence_ids, + "missing_evidence": [ + evidence_id + for evidence_id in evidence_ids + if evidence_id not in dialog_index + ], + "locomo_format": "official-v1", + }, + } + ) + return cases + + +def _official_sessions( + record: dict[str, object], +) -> tuple[list[dict[str, object]], dict[str, dict[str, object]]]: + conversation = record["conversation"] + if not isinstance(conversation, dict): + return [], {} + speaker_a = str(conversation.get("speaker_a") or "speaker_a") + speaker_b = str(conversation.get("speaker_b") or "speaker_b") + dialog_index: dict[str, dict[str, object]] = {} + sessions: list[dict[str, object]] = [] + for session_key in sorted( + (key for key in conversation if re.fullmatch(r"session_\d+", key)), + key=lambda value: int(value.split("_")[1]), + ): + raw_turns = conversation.get(session_key) + if not isinstance(raw_turns, list): + continue + session_date = str(conversation.get(f"{session_key}_date_time") or "") + turns: list[dict[str, object]] = [] + for raw_turn in raw_turns: + if not isinstance(raw_turn, dict): + continue + speaker = str(raw_turn.get("speaker") or "") + dia_id = str(raw_turn.get("dia_id") or "") + metadata = metadata_without_heavy_fields(raw_turn) + metadata.update( + { + "dia_id": dia_id, + "speaker": speaker, + "session_date": session_date, + "speaker_a": speaker_a, + "speaker_b": speaker_b, + } + ) + if raw_turn.get("img_url"): + metadata["img_url"] = raw_turn.get("img_url") + if raw_turn.get("blip_caption"): + metadata["blip_caption"] = raw_turn.get("blip_caption") + content = _turn_content(raw_turn, session_date=session_date) + turn = { + "role": _role_for_speaker(speaker, speaker_a=speaker_a), + "content": content, + "metadata": metadata, + } + turns.append(turn) + if dia_id: + dialog_index[dia_id] = turn + sessions.append({"session_id": session_key, "turns": turns}) + return sessions, dialog_index + + +def _turn_content(raw_turn: dict[str, object], *, session_date: str) -> str: + parts: list[str] = [] + if session_date: + parts.append(f"[Session date: {session_date}]") + speaker = str(raw_turn.get("speaker") or "").strip() + text = str(raw_turn.get("text") or "").strip() + if speaker: + text = f"{speaker}: {text}" + parts.append(text) + caption = str(raw_turn.get("blip_caption") or "").strip() + if caption: + parts.append(f"[Image caption: {caption}]") + return " ".join(part for part in parts if part) + + +def _role_for_speaker(speaker: str, *, speaker_a: str) -> str: + return "user" if speaker == speaker_a else "assistant" + + +def _category(record: dict[str, object]) -> str: + raw = record.get("category") or record.get("question_type") or record.get("type") + if isinstance(raw, int): + return { + 1: "single_hop", + 2: "multi_hop", + 3: "temporal_reasoning", + 4: "open_domain", + 5: "adversarial", + }.get(raw, f"category_{raw}") + if isinstance(raw, str) and raw.strip(): + normalized = raw.strip().lower().replace("-", "_").replace(" ", "_") + return { + "single_hop": "single_hop", + "temporal": "temporal_reasoning", + "temporal_reasoning": "temporal_reasoning", + "multi_hop": "multi_hop", + "adversarial": "adversarial", + }.get(normalized, normalized) + return "multi_session" + + +def _expected_behavior(record: dict[str, object]) -> str: + if _category(record) == "adversarial" and normalize_expected_answer(record) is None: + return "refuse_or_unknown" + return "answer" + + +def _forbidden_answers(record: dict[str, object]) -> list[str]: + value = record.get("adversarial_answer") + return [str(value)] if value is not None else [] + + +def _evidence_ids(value: object) -> list[str]: + if not isinstance(value, list): + return [] + evidence_ids: list[str] = [] + for item in value: + for session_no, turn_no in re.findall(r"D:?(\d+):(\d+)", str(item)): + evidence_ids.append(f"D{int(session_no)}:{int(turn_no)}") + return evidence_ids diff --git a/evaluation/adapters/longmemeval_adapter.py b/evaluation/adapters/longmemeval_adapter.py index c9c2ccc..8f92699 100644 --- a/evaluation/adapters/longmemeval_adapter.py +++ b/evaluation/adapters/longmemeval_adapter.py @@ -1,10 +1,11 @@ from __future__ import annotations +from collections.abc import Callable from pathlib import Path +from typing import Any from .common import ( case_id, - category, load_raw_records, metadata_without_heavy_fields, normalize_expected_answer, @@ -17,27 +18,152 @@ def convert(raw_dir: Path, processed_dir: Path) -> Path: records = load_raw_records(raw_dir) cases: list[dict[str, object]] = [] + seen_case_ids: set[str] = set() for index, record in enumerate(records, start=1): - sessions = normalize_sessions( - record.get("haystack_sessions") - or record.get("sessions") - or record.get("conversation") - or record.get("messages") + resolved_case_id = case_id(record, source="longmemeval", index=index) + if resolved_case_id in seen_case_ids: + raise ValueError( + f"duplicate LongMemEval case id {resolved_case_id}; " + "convert one dataset variant at a time" + ) + seen_case_ids.add(resolved_case_id) + sessions = _normalize_longmemeval_sessions(record) + question = normalize_query(record) + question_date = _optional_text(record.get("question_date")) + answer_session_ids = _string_list(record.get("answer_session_ids")) + metadata = metadata_without_heavy_fields(record) + metadata.update( + { + "answer_session_ids": answer_session_ids, + "question_date": question_date, + "haystack_session_count": len(sessions), + "longmemeval_format": "official-v1", + } ) cases.append( { - "case_id": case_id(record, source="longmemeval", index=index), + "case_id": resolved_case_id, "source": "longmemeval", - "category": category(record), + "category": _category(record), "sessions": sessions, - "query": normalize_query(record), + "query": _with_date(question, question_date, label="Question date"), "expected_answer": normalize_expected_answer(record), "expected_answer_contains": [], "forbidden_answers": [], "forbidden_patterns": [], "gold_memory_ids": [], - "expected_behavior": "answer", - "metadata": metadata_without_heavy_fields(record), + "expected_behavior": _expected_behavior(record), + "metadata": metadata, } ) return write_cases(processed_dir, "longmemeval_cases.jsonl", cases) + + +def _normalize_longmemeval_sessions(record: dict[str, Any]) -> list[dict[str, Any]]: + raw_sessions = record.get("haystack_sessions") + if not _is_official_session_list(raw_sessions): + return normalize_sessions( + raw_sessions + or record.get("sessions") + or record.get("conversation") + or record.get("messages") + ) + + session_count = len(raw_sessions) + session_ids = _parallel_values( + record, + "haystack_session_ids", + expected_length=session_count, + fallback=lambda index: f"s{index + 1}", + ) + session_dates = _parallel_values( + record, + "haystack_dates", + expected_length=session_count, + fallback=lambda _index: "", + ) + sessions: list[dict[str, Any]] = [] + for index, turns_raw in enumerate(raw_sessions): + normalized = normalize_sessions(turns_raw) + turns = normalized[0]["turns"] if normalized else [] + session_date = session_dates[index] + for turn in turns: + metadata = turn.setdefault("metadata", {}) + metadata["longmemeval_session_id"] = session_ids[index] + if session_date: + metadata["session_date"] = session_date + turn["content"] = _with_date( + str(turn["content"]), + session_date, + label="Session date", + ) + sessions.append({"session_id": session_ids[index], "turns": turns}) + return sessions + + +def _is_official_session_list(raw: object) -> bool: + return isinstance(raw, list) and all(isinstance(session, list) for session in raw) + + +def _parallel_values( + record: dict[str, Any], + key: str, + *, + expected_length: int, + fallback: Callable[[int], object], +) -> list[str]: + raw = record.get(key) + if raw is None: + return [str(fallback(index)) for index in range(expected_length)] + if not isinstance(raw, list) or len(raw) != expected_length: + raise ValueError( + f"LongMemEval {key} must contain {expected_length} values " "to match haystack_sessions" + ) + return [str(value) for value in raw] + + +def _category(record: dict[str, Any]) -> str: + question_id = str(record.get("question_id") or record.get("id") or "").lower() + if question_id.endswith("_abs"): + return "abstention" + question_type = _normalized_question_type(record) + mapping = { + "single_session_user": "single_fact", + "single_session_assistant": "single_fact", + "single_session_preference": "preference_following", + "knowledge_update": "temporal_update", + } + return mapping.get(question_type, question_type or "multi_session") + + +def _expected_behavior(record: dict[str, Any]) -> str: + question_id = str(record.get("question_id") or record.get("id") or "").lower() + if question_id.endswith("_abs"): + return "refuse_or_unknown" + question_type = _normalized_question_type(record) + if question_type == "knowledge_update": + return "answer_latest" + if question_type == "single_session_preference": + return "follow_preference" + return "answer" + + +def _normalized_question_type(record: dict[str, Any]) -> str: + value = str(record.get("question_type") or record.get("category") or "") + return value.strip().lower().replace("-", "_").replace(" ", "_") + + +def _string_list(value: object) -> list[str]: + if not isinstance(value, list): + return [] + return [str(item) for item in value] + + +def _optional_text(value: object) -> str: + return str(value) if value is not None else "" + + +def _with_date(text: str, date: str, *, label: str) -> str: + if not date: + return text + return f"[{label}: {date}] {text}" diff --git a/evaluation/adapters/memoryagentbench_adapter.py b/evaluation/adapters/memoryagentbench_adapter.py index 273d3de..2a85e3a 100644 --- a/evaluation/adapters/memoryagentbench_adapter.py +++ b/evaluation/adapters/memoryagentbench_adapter.py @@ -1,5 +1,7 @@ from __future__ import annotations +import re +from datetime import UTC, datetime, timedelta from pathlib import Path from .common import ( @@ -13,11 +15,16 @@ write_cases, ) +MEMORY_SEQUENCE_EPOCH = datetime(2000, 1, 1, tzinfo=UTC) + def convert(raw_dir: Path, processed_dir: Path) -> Path: records = load_raw_records(raw_dir) cases: list[dict[str, object]] = [] for index, record in enumerate(records, start=1): + if _is_conflict_resolution_record(record): + cases.extend(_conflict_resolution_cases(record, index)) + continue cases.append( { "case_id": case_id(record, source="memoryagentbench", index=index), @@ -42,6 +49,157 @@ def convert(raw_dir: Path, processed_dir: Path) -> Path: return write_cases(processed_dir, "memoryagentbench_cases.jsonl", cases) +def _is_conflict_resolution_record(record: dict[str, object]) -> bool: + metadata = record.get("metadata") + return ( + isinstance(record.get("context"), str) + and isinstance(record.get("questions"), list) + and isinstance(record.get("answers"), list) + and isinstance(metadata, dict) + and str(metadata.get("source") or "").startswith("factconsolidation_") + ) + + +def _conflict_resolution_cases( + record: dict[str, object], + row_index: int, +) -> list[dict[str, object]]: + context = str(record["context"]) + questions = record["questions"] + answers = record["answers"] + metadata = record["metadata"] + if not isinstance(questions, list) or not isinstance(answers, list): + return [] + if len(questions) != len(answers): + raise ValueError("MemoryAgentBench Conflict Resolution questions and answers must align") + if not isinstance(metadata, dict): + raise ValueError("MemoryAgentBench Conflict Resolution metadata must be an object") + source = str(metadata.get("source") or f"conflict_row_{row_index}") + qa_pair_ids = metadata.get("qa_pair_ids") + if not isinstance(qa_pair_ids, list) or len(qa_pair_ids) != len(questions): + raise ValueError( + "MemoryAgentBench Conflict Resolution qa_pair_ids must align with questions" + ) + sessions = _context_sessions(context) + cases: list[dict[str, object]] = [] + for qa_index, (question, raw_answers, qa_pair_id) in enumerate( + zip(questions, answers, qa_pair_ids, strict=True), + start=1, + ): + accepted_answers = _accepted_answers(raw_answers) + cases.append( + { + "case_id": f"memoryagentbench_{qa_pair_id}", + "source": "memoryagentbench", + "category": _conflict_category(source), + "sessions": sessions, + "query": str(question), + "expected_answer": accepted_answers[0] if accepted_answers else None, + "expected_answer_contains": [], + "forbidden_answers": [], + "forbidden_patterns": [], + "gold_memory_ids": [], + "expected_behavior": "answer_latest", + "metadata": { + "context_group_id": source, + "context_chars": len(context), + "context_chunk_count": len(sessions), + "qa_index": qa_index, + "qa_pair_id": str(qa_pair_id), + "accepted_answers": accepted_answers, + "memoryagentbench_competency": "conflict_resolution", + "memoryagentbench_format": "official-v1", + "source_subset": source, + }, + } + ) + return cases + + +def _context_sessions(context: str) -> list[dict[str, object]]: + facts = [ + match.group("fact").strip() + for line in context.splitlines() + if (match := re.match(r"^\s*(?P\d+)\.\s+(?P.+?)\s*$", line)) + ] + return [ + { + "session_id": f"context_{index:04d}", + "turns": [ + { + "role": "user", + "content": ( + f"[Memory sequence {index:04d}] Higher sequence numbers are " + "later and supersede earlier conflicting facts.\n" + f"{fact}" + ), + "metadata": { + "context_chunk": index, + "memory_sequence": index, + "valid_from": ( + MEMORY_SEQUENCE_EPOCH + timedelta(seconds=index) + ).isoformat(), + "supersession_key": _fact_supersession_key(fact), + }, + } + ], + } + for index, fact in enumerate(facts, start=1) + ] + + +_FACT_RELATIONS = ( + r"(?P.+?) was born in the city of ", + r"The chairperson of (?P.+?) is ", + r"(?P.+?) died in the city of ", + r"(?P.+?) plays the position of ", + r"(?P.+?) is located in the continent of ", + r"(?P.+?) worked in the city of ", + r"The director of (?P.+?) is ", + r"(?P.+?) is married to ", + r"The headquarters of (?P.+?) is located in the city of ", + r"The author of (?P.+?) is ", + r"The univeristy where (?P.+?) was educated is ", + r"(?P.+?) was founded by ", + r"(?P.+?) was founded in the city of ", + r"(?P.+?) is associated with the sport of ", + r"The capital of (?P.+?) is ", + r"(?P.+?) is a citizen of ", + r"(?P.+?) was performed by ", + r"(?P.+?) is employed by ", + r"(?P.+?) speaks the language of ", + r"(?P.+?) is famous for ", + r"(?P.+?) was created by ", + r"(?P.+?) was created in the country of ", + r"(?P.+?)'s child is ", + r"(?PThe .+?) is ", +) + + +def _fact_supersession_key(fact: str) -> str | None: + for relation_index, pattern in enumerate(_FACT_RELATIONS): + match = re.match(pattern, fact, flags=re.IGNORECASE) + if match: + subject = " ".join(match.group("subject").lower().split()) + return f"{relation_index}:{subject}" + return None + + +def _accepted_answers(raw_answers: object) -> list[str]: + if isinstance(raw_answers, list): + return [str(answer) for answer in raw_answers if str(answer)] + if raw_answers is None: + return [] + return [str(raw_answers)] + + +def _conflict_category(source: str) -> str: + reasoning = "multi_hop" if "_mh_" in source else "single_hop" + match = re.search(r"_(6k|32k|64k|262k)$", source) + length = match.group(1) if match else "unknown" + return f"conflict_{reasoning}_{length}" + + def _expected_behavior(record: dict[str, object]) -> str: task_type = str(record.get("task_type") or record.get("category") or "").lower() if "conflict" in task_type: diff --git a/evaluation/baselines.py b/evaluation/baselines.py index 00f5691..cd49d12 100644 --- a/evaluation/baselines.py +++ b/evaluation/baselines.py @@ -3,19 +3,30 @@ import os import time from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path from typing import Any, Protocol +from urllib.parse import urlparse import httpx +import psycopg +from dotenv import load_dotenv -from evaluation.cases import EvaluationCase +from evaluation.cases import EvaluationCase, EvaluationTurn + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +load_dotenv(PROJECT_ROOT / ".env") SUPPORTED_MODES = { + "db_extraction_qa", "no_memory", "recency_only", "naive_vector_rag", "summary_memory", "db_memory", "db_extraction", + "db_qa", + "vector_qa", } @@ -111,6 +122,9 @@ def __init__( agent: str | None, backfill_embeddings: bool, use_extraction: bool, + generate_answer: bool, + cleanup: bool, + isolate: bool, ) -> None: self.mode = mode self.run_id = run_id @@ -119,6 +133,9 @@ def __init__( self.agent = agent or os.getenv("MEMORYBASE_AGENT") self.backfill_embeddings = backfill_embeddings self.use_extraction = use_extraction + self.generate_answer = generate_answer + self.cleanup = cleanup + self.isolate = isolate class LocalMemoryBaseline: @@ -128,11 +145,14 @@ def __init__(self, *, mode: str, run_id: str) -> None: def run_case(self, case: EvaluationCase) -> EvaluationResult: started = time.perf_counter() - memories = _local_case_memories(case) + memory_entries = _local_case_memory_entries(case) if self._mode == "recency_only": - selected = list(reversed(memories))[:3] + selected_entries = list(reversed(memory_entries))[:3] else: - selected = [_summarize_memory(memory) for memory in memories[:8]] + selected_entries = [ + (memory_id, _summarize_memory(memory)) for memory_id, memory in memory_entries[:8] + ] + selected = [memory for _memory_id, memory in selected_entries] generated_answer = "\n".join(selected) return EvaluationResult( case_id=case.case_id, @@ -141,10 +161,7 @@ def run_case(self, case: EvaluationCase) -> EvaluationResult: query=case.query, expected_answer=case.expected_answer, generated_answer=generated_answer, - retrieved_memory_ids=[ - f"{self._mode}:{case.case_id}:{index}" - for index, _memory in enumerate(selected, start=1) - ], + retrieved_memory_ids=[memory_id for memory_id, _memory in selected_entries], retrieved_memory_texts=selected, retrieved_scores=[1 / index for index, _memory in enumerate(selected, start=1)], latency_ms=(time.perf_counter() - started) * 1000, @@ -171,6 +188,8 @@ def build_baseline_with_config( api_base_url: str | None = None, workspace: str | None = None, agent: str | None = None, + cleanup: bool = True, + isolate: bool = False, ) -> BaselineRunner: if mode not in SUPPORTED_MODES: raise ValueError(f"unsupported mode {mode!r}; expected one of {sorted(SUPPORTED_MODES)}") @@ -178,7 +197,14 @@ def build_baseline_with_config( return NoMemoryBaseline(mode=mode, run_id=run_id, dry_run=dry_run) if mode in {"recency_only", "summary_memory"}: return LocalMemoryBaseline(mode=mode, run_id=run_id) - if mode in {"db_memory", "naive_vector_rag", "db_extraction"}: + if mode in { + "db_memory", + "naive_vector_rag", + "db_extraction", + "db_qa", + "vector_qa", + "db_extraction_qa", + }: return LiveMemoryBaseline( config=LiveMemoryBaselineConfig( mode=mode, @@ -186,8 +212,11 @@ def build_baseline_with_config( api_base_url=api_base_url or "http://localhost:8000", workspace=workspace, agent=agent, - backfill_embeddings=mode == "naive_vector_rag", - use_extraction=mode == "db_extraction", + backfill_embeddings=mode in {"naive_vector_rag", "vector_qa"}, + use_extraction=mode in {"db_extraction", "db_extraction_qa"}, + generate_answer=mode in {"db_qa", "vector_qa", "db_extraction_qa"}, + cleanup=cleanup, + isolate=isolate, ) ) return UnsupportedBaseline(mode=mode, run_id=run_id) @@ -196,13 +225,26 @@ def build_baseline_with_config( class LiveMemoryBaseline: def __init__(self, *, config: LiveMemoryBaselineConfig) -> None: self._config = config + self._client = ( + httpx.Client(timeout=120.0, trust_env=False) + if _is_loopback_url(config.api_base_url) + else None + ) def run_case(self, case: EvaluationCase) -> EvaluationResult: started = time.perf_counter() created_memory_ids: list[str] = [] + memory_source_ids: dict[str, str] = {} embedding_backfill: dict[str, Any] | None = None + workspace_id: str | None = None + isolated_workspace = False + pending_memories: list[tuple[str, EvaluationTurn]] = [] try: - workspace_id, agent_id = self._resolve_workspace_and_agent() + if self._config.isolate: + workspace_id, agent_id = self._create_isolated_workspace(case) + isolated_workspace = True + else: + workspace_id, agent_id = self._resolve_workspace_and_agent() for session in case.sessions: session_id = self._create_session( workspace_id=workspace_id, @@ -216,11 +258,23 @@ def run_case(self, case: EvaluationCase) -> EvaluationResult: content=turn.content, agent_id=agent_id, ) - if turn.role != "user": + if not _is_memory_turn(case, turn): continue if _is_query_turn(turn.content, case.query): continue - if _is_forget_turn(turn.content): + if _should_apply_forget_turn(case, turn): + created = self._create_memories( + workspace_id=workspace_id, + agent_id=agent_id, + entries=pending_memories, + case=case, + ) + for memory_id, pending_turn in created: + source_id = _turn_source_id(pending_turn) + if source_id: + memory_source_ids[memory_id] = source_id + created_memory_ids.extend(memory_id for memory_id, _turn in created) + pending_memories.clear() self._delete_created_memories(workspace_id, created_memory_ids) created_memory_ids.clear() continue @@ -233,24 +287,56 @@ def run_case(self, case: EvaluationCase) -> EvaluationResult: ) ) else: - created_memory_ids.append( - self._create_memory( - workspace_id=workspace_id, - agent_id=agent_id, - content=turn.content, - case=case, - ) - ) + pending_memories.append((turn.content, turn)) - if self._config.backfill_embeddings: - embedding_backfill = self._backfill_embeddings(workspace_id) - recall = self._recall( + created = self._create_memories( workspace_id=workspace_id, agent_id=agent_id, - query=case.query, + entries=pending_memories, + case=case, ) - memories = [item for item in recall.get("memories", []) if isinstance(item, dict)] - generated_answer = "\n".join(str(item.get("canonical_text", "")) for item in memories) + for memory_id, pending_turn in created: + created_memory_ids.append(memory_id) + source_id = _turn_source_id(pending_turn) + if source_id: + memory_source_ids[memory_id] = source_id + + if self._config.backfill_embeddings: + embedding_backfill = self._backfill_embeddings(workspace_id) + if self._config.generate_answer: + answer = self._answer( + workspace_id=workspace_id, + agent_id=agent_id, + query=case.query, + ) + memories = [ + item for item in answer.get("selected_memories", []) if isinstance(item, dict) + ] + generated_answer = str(answer.get("answer", "")) + token_usage = _optional_int(answer.get("total_tokens")) + response_metadata = { + "provider": answer.get("provider"), + "model": answer.get("model"), + "prompt_tokens": answer.get("prompt_tokens"), + "completion_tokens": answer.get("completion_tokens"), + "total_tokens": answer.get("total_tokens"), + "context_tokens": answer.get("token_count"), + "token_budget": answer.get("token_budget"), + "citation_map": answer.get("citation_map", {}), + "supporting_evidence": answer.get("supporting_evidence", []), + } + else: + recall = self._recall( + workspace_id=workspace_id, + agent_id=agent_id, + query=case.query, + ) + memories = [item for item in recall.get("memories", []) if isinstance(item, dict)] + generated_answer = "\n".join( + str(item.get("canonical_text", "")) for item in memories + ) + token_usage = None + response_metadata = {} return EvaluationResult( case_id=case.case_id, source=case.source, @@ -258,10 +344,17 @@ def run_case(self, case: EvaluationCase) -> EvaluationResult: query=case.query, expected_answer=case.expected_answer, generated_answer=generated_answer, - retrieved_memory_ids=[str(item.get("memory_id", "")) for item in memories], + retrieved_memory_ids=[ + memory_source_ids.get( + str(item.get("memory_id", "")), + str(item.get("memory_id", "")), + ) + for item in memories + ], retrieved_memory_texts=[str(item.get("canonical_text", "")) for item in memories], retrieved_scores=[_float(item.get("score")) for item in memories], latency_ms=(time.perf_counter() - started) * 1000, + token_usage=token_usage, mode=self._config.mode, run_id=self._config.run_id, metadata={ @@ -277,12 +370,17 @@ def run_case(self, case: EvaluationCase) -> EvaluationResult: "workspace_id": workspace_id, "agent_id": agent_id, "created_memory_ids": created_memory_ids, + "memory_source_ids": memory_source_ids, "embedding_backfill": embedding_backfill, "rank_reasons": [str(item.get("rank_reason", "")) for item in memories], "vector_scores": [_float(item.get("vector_score")) for item in memories], + "answer_generation": self._config.generate_answer, + "cleanup_requested": self._config.cleanup, + "isolated_workspace": isolated_workspace, + **response_metadata, "limitation": ( - "Uses existing live APIs. Full agent answer generation is " - "not implemented yet." + "Evaluation-created sessions and extraction source documents cannot " + "be deleted through the current public API." ), }, ) @@ -300,6 +398,143 @@ def run_case(self, case: EvaluationCase) -> EvaluationResult: error=str(exc), metadata={"injection_mode": "memory_api"}, ) + finally: + if isolated_workspace and workspace_id and self._config.cleanup: + try: + self._delete_isolated_workspace(workspace_id) + except Exception: + pass + elif self._config.cleanup and workspace_id and created_memory_ids: + try: + self._delete_created_memories(workspace_id, created_memory_ids) + except Exception: + # Cleanup must not replace the benchmark result or original failure. + pass + + def run_group(self, cases: list[EvaluationCase]) -> list[EvaluationResult]: + if not cases: + return [] + if not self._config.generate_answer or self._config.use_extraction: + raise ValueError("grouped live evaluation currently requires direct-memory QA mode") + if any(case.sessions != cases[0].sessions for case in cases[1:]): + raise ValueError("grouped evaluation cases must share identical sessions") + + workspace_id: str | None = None + created_memory_ids: list[str] = [] + results: list[EvaluationResult] = [] + pending_memories: list[tuple[str, EvaluationTurn]] = [] + try: + if self._config.isolate: + workspace_id, agent_id = self._create_isolated_workspace(cases[0]) + else: + workspace_id, agent_id = self._resolve_workspace_and_agent() + for session in cases[0].sessions: + session_id = self._create_session( + workspace_id=workspace_id, + agent_id=agent_id, + title=f"{self._config.run_id}:{cases[0].metadata.get('context_group_id')}", + ) + for turn in session.turns: + self._observe_message( + session_id=session_id, + role=turn.role, + content=turn.content, + agent_id=agent_id, + ) + if not _is_memory_turn(cases[0], turn): + continue + pending_memories.append((turn.content, turn)) + created_memory_ids.extend( + memory_id + for memory_id, _turn in self._create_memories( + workspace_id=workspace_id, + agent_id=agent_id, + entries=pending_memories, + case=cases[0], + ) + ) + + for case in cases: + started = time.perf_counter() + try: + answer = self._answer( + workspace_id=workspace_id, + agent_id=agent_id, + query=case.query, + ) + memories = [ + item + for item in answer.get("selected_memories", []) + if isinstance(item, dict) + ] + results.append( + EvaluationResult( + case_id=case.case_id, + source=case.source, + category=case.category, + query=case.query, + expected_answer=case.expected_answer, + generated_answer=str(answer.get("answer", "")), + retrieved_memory_ids=[ + str(item.get("memory_id", "")) for item in memories + ], + retrieved_memory_texts=[ + str(item.get("canonical_text", "")) for item in memories + ], + retrieved_scores=[_float(item.get("score")) for item in memories], + latency_ms=(time.perf_counter() - started) * 1000, + token_usage=_optional_int(answer.get("total_tokens")), + mode=self._config.mode, + run_id=self._config.run_id, + metadata={ + "injection_mode": "grouped_memory_api", + "write_mode": "direct_memory_create", + "retrieval_mode": "backend_hybrid_recall", + "workspace_id": workspace_id, + "agent_id": agent_id, + "created_memory_ids": created_memory_ids, + "answer_generation": True, + "provider": answer.get("provider"), + "model": answer.get("model"), + "prompt_tokens": answer.get("prompt_tokens"), + "completion_tokens": answer.get("completion_tokens"), + "total_tokens": answer.get("total_tokens"), + "context_tokens": answer.get("token_count"), + "token_budget": answer.get("token_budget"), + "citation_map": answer.get("citation_map", {}), + "supporting_evidence": answer.get( + "supporting_evidence", + [], + ), + "context_group_id": case.metadata.get("context_group_id"), + }, + ) + ) + except Exception as exc: + results.append( + EvaluationResult( + case_id=case.case_id, + source=case.source, + category=case.category, + query=case.query, + expected_answer=case.expected_answer, + generated_answer="", + latency_ms=(time.perf_counter() - started) * 1000, + mode=self._config.mode, + run_id=self._config.run_id, + error=str(exc), + metadata={ + "injection_mode": "grouped_memory_api", + "context_group_id": case.metadata.get("context_group_id"), + }, + ) + ) + return results + finally: + if self._config.isolate and workspace_id and self._config.cleanup: + self._delete_isolated_workspace(workspace_id) + elif self._config.cleanup and workspace_id and created_memory_ids: + self._delete_created_memories(workspace_id, created_memory_ids) def _resolve_workspace_and_agent(self) -> tuple[str, str | None]: if not self._config.workspace: @@ -314,6 +549,47 @@ def _resolve_workspace_and_agent(self) -> tuple[str, str | None]: agent = _checked_target(payload, "agent") if self._config.agent else None return str(workspace["workspace_id"]), str(agent["agent_id"]) if agent else None + def _create_isolated_workspace(self, case: EvaluationCase) -> tuple[str, str]: + database_url = os.getenv("DATABASE_URL") + if not database_url: + raise ValueError("isolated live evaluation requires DATABASE_URL") + slug = _evaluation_slug(self._config.run_id, case.case_id) + with psycopg.connect(database_url) as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO workspace (slug, name, description, scope_type) + VALUES (%s, %s, %s, 'project') + RETURNING workspace_id + """, + ( + slug, + f"Evaluation {case.case_id}"[:120], + f"Isolated evaluation workspace for {self._config.run_id}", + ), + ) + workspace_id = str(cur.fetchone()[0]) + cur.execute( + """ + INSERT INTO agent (workspace_id, name, agent_type, status) + VALUES (%s, %s, 'retriever', 'active') + RETURNING agent_id + """, + (workspace_id, "evaluation-agent"), + ) + agent_id = str(cur.fetchone()[0]) + conn.commit() + return workspace_id, agent_id + + def _delete_isolated_workspace(self, workspace_id: str) -> None: + database_url = os.getenv("DATABASE_URL") + if not database_url: + return + with psycopg.connect(database_url) as conn: + with conn.cursor() as cur: + cur.execute("DELETE FROM workspace WHERE workspace_id = %s", (workspace_id,)) + conn.commit() + def _create_session( self, *, @@ -348,33 +624,91 @@ def _observe_message( } self._request("POST", "/api/observe", json=payload) - def _create_memory( + def _create_memories( self, *, workspace_id: str, agent_id: str | None, - content: str, + entries: list[tuple[str, EvaluationTurn]], case: EvaluationCase, - ) -> str: - payload = { - "workspace_id": workspace_id, - "memory_type": _memory_type_for_case(case, content), - "canonical_text": content, - "summary": f"Evaluation case {case.case_id}", - "confidence": 0.7, - "importance": 3, - "access_level": "project", - "owner_agent_id": agent_id, - "evidence": [], - } - headers = { - "X-Actor-Type": "agent", - "X-Revision-Reason": f"evaluation injection {self._config.run_id}", - } - if agent_id: - headers["X-Actor-Id"] = agent_id - response = self._request("POST", "/api/memories", json=payload, headers=headers) - return str(response["memory_id"]) + ) -> list[tuple[str, EvaluationTurn]]: + created: list[tuple[str, EvaluationTurn]] = [] + supersession_ids: dict[str, str] = {} + pending: list[tuple[str, EvaluationTurn, str | None, str | None]] = [] + pending_keys: set[str] = set() + + def flush() -> None: + if not pending: + return + payload = { + "items": [ + { + "workspace_id": workspace_id, + "memory_type": _memory_type_for_case(case, content), + "canonical_text": content, + "summary": f"Evaluation case {case.case_id}", + "confidence": 0.7, + "importance": 3, + "access_level": "project", + "owner_agent_id": agent_id, + **({"valid_from": valid_from} if valid_from else {}), + **( + {"supersedes_memory_id": supersession_ids[supersession_key]} + if supersession_key and supersession_key in supersession_ids + else {} + ), + "evidence": [], + } + for content, _turn, valid_from, supersession_key in pending + ] + } + headers = { + "X-Actor-Type": "agent", + "X-Revision-Reason": f"evaluation batch injection {self._config.run_id}", + } + if agent_id: + headers["X-Actor-Id"] = agent_id + response = self._request( + "POST", + "/api/memories/batch", + json=payload, + headers=headers, + ) + items = response.get("items", []) + if not isinstance(items, list) or len(items) != len(pending): + raise RuntimeError("MemoryBase batch memory response size did not match request") + for item, (_content, turn, _valid_from, supersession_key) in zip( + items, + pending, + strict=True, + ): + if not isinstance(item, dict) or not item.get("memory_id"): + continue + memory_id = str(item["memory_id"]) + created.append((memory_id, turn)) + if supersession_key: + supersession_ids[supersession_key] = memory_id + pending.clear() + pending_keys.clear() + + for content, turn in entries: + supersession_key = _turn_supersession_key(turn) + if supersession_key and supersession_key in pending_keys: + flush() + pending.append( + ( + content, + turn, + _turn_valid_from(turn), + supersession_key, + ) + ) + if supersession_key: + pending_keys.add(supersession_key) + if len(pending) == 500: + flush() + flush() + return created def _extract_and_approve_memory( self, @@ -465,14 +799,32 @@ def _recall(self, *, workspace_id: str, agent_id: str | None, query: str) -> dic } return self._request("POST", "/api/recall", json=payload) + def _answer(self, *, workspace_id: str, agent_id: str | None, query: str) -> dict[str, Any]: + payload = { + "workspace_id": workspace_id, + "agent_id": agent_id, + "query_text": query, + "status": "active", + "retrieval_mode": "vector" if self._config.backfill_embeddings else "hybrid", + "limit": 10, + } + return self._request("POST", "/api/qa/answer", json=payload) + def _request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]: try: - response = httpx.request( - method, - f"{self._config.api_base_url}{path}", - timeout=10.0, - **kwargs, - ) + if self._client is not None: + response = self._client.request( + method, + f"{self._config.api_base_url}{path}", + **kwargs, + ) + else: + response = httpx.request( + method, + f"{self._config.api_base_url}{path}", + timeout=120.0, + **kwargs, + ) except httpx.RequestError as exc: raise RuntimeError(f"MemoryBase API request failed: {exc}") from exc if response.status_code >= 400: @@ -519,21 +871,90 @@ def _float(value: Any) -> float: return 0.0 +def _optional_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float) and value.is_integer(): + return int(value) + return None + + +def _is_loopback_url(value: str) -> bool: + return (urlparse(value).hostname or "").lower() in {"127.0.0.1", "localhost", "::1"} + + +def _evaluation_slug(run_id: str, case_id: str) -> str: + raw = f"{run_id}-{case_id}".lower() + normalized = "".join(char if char.isalnum() else "-" for char in raw) + return normalized.strip("-")[:55] + "-" + str(time.time_ns())[-8:] + + def _local_case_memories(case: EvaluationCase) -> list[str]: - memories: list[str] = [] + return [memory for _memory_id, memory in _local_case_memory_entries(case)] + + +def _local_case_memory_entries(case: EvaluationCase) -> list[tuple[str, str]]: + memories: list[tuple[str, str]] = [] + fallback_index = 0 for session in case.sessions: for turn in session.turns: - if turn.role != "user": + if not _is_memory_turn(case, turn): continue if _is_query_turn(turn.content, case.query): continue - if _is_forget_turn(turn.content): + if _should_apply_forget_turn(case, turn): memories.clear() continue - memories.append(turn.content) + fallback_index += 1 + memory_id = _turn_source_id(turn) or f"local:{case.case_id}:{fallback_index}" + memories.append((memory_id, turn.content)) return memories +def _is_memory_turn(case: EvaluationCase, turn: EvaluationTurn) -> bool: + if turn.role == "user": + return True + return case.source in {"locomo", "longmemeval"} and turn.role == "assistant" + + +def _should_apply_forget_turn(case: EvaluationCase, turn: EvaluationTurn) -> bool: + return case.category == "deletion" and _is_forget_turn(turn.content) + + +def _turn_source_id(turn: EvaluationTurn) -> str: + for key in ("dia_id", "longmemeval_session_id"): + value = turn.metadata.get(key) + if value is not None and str(value): + return str(value) + return "" + + +def _turn_supersession_key(turn: EvaluationTurn) -> str | None: + value = turn.metadata.get("supersession_key") + return str(value) if value else None + + +def _turn_valid_from(turn: EvaluationTurn) -> str | None: + explicit = turn.metadata.get("valid_from") + if explicit: + return str(explicit) + session_date = turn.metadata.get("session_date") + if not session_date: + return None + value = str(session_date).strip() + for date_format in ( + "%Y/%m/%d (%a) %H:%M", + "%I:%M %p on %d %B, %Y", + ): + try: + return datetime.strptime(value, date_format).replace(tzinfo=UTC).isoformat() + except ValueError: + continue + return None + + def _summarize_memory(memory: str) -> str: compact = " ".join(memory.strip().split()) if len(compact) <= 120: diff --git a/evaluation/external/README.md b/evaluation/external/README.md index 3e41fef..224873a 100644 --- a/evaluation/external/README.md +++ b/evaluation/external/README.md @@ -19,10 +19,12 @@ responsible for conversion. | Benchmark | Status | Notes | | --- | --- | --- | -| LongMemEval | Partially supported | Converts common JSON/JSONL records with question, answer, and session/message fields. | -| LoCoMo | Partially supported | Converts common conversation + qa records into one case per QA item. | -| MemoryAgentBench | Partially supported | Converts common interaction/message records into EvaluationCase JSONL. | +| LongMemEval | Official format supported | Handles extensionless official files, nested sessions, dates, answer-session IDs, abstention, preferences, and knowledge updates. | +| LoCoMo | Official format supported | Handles session dictionaries, timestamps, speakers, image captions, QA categories, evidence IDs, and adversarial questions. | +| MemoryAgentBench | Official Conflict Resolution supported | Converts the official Conflict Resolution parquet shard and common interaction/message records into EvaluationCase JSONL. Other task families remain pending. | | BEIR | TODO | Retriever-only supplemental benchmark. | | BEAM | TODO | Optional ultra-long context retention benchmark. | -Add download URLs, license notes, and exact raw file names before integrating each dataset. +LongMemEval, LoCoMo, and MemoryAgentBench download, license, and execution +details are documented in their benchmark directories. Add equivalent +documentation before promoting another adapter from partial support. diff --git a/evaluation/external/locomo/README.md b/evaluation/external/locomo/README.md new file mode 100644 index 0000000..403b881 --- /dev/null +++ b/evaluation/external/locomo/README.md @@ -0,0 +1,108 @@ +# LoCoMo Integration + +MemoryBase supports the official `data/locomo10.json` release from the +`snap-research/locomo` repository. + +## Official Sources + +- Repository: https://github.com/snap-research/locomo +- Dataset file: https://github.com/snap-research/locomo/blob/main/data/locomo10.json +- Paper: https://arxiv.org/abs/2402.17753 +- License: CC BY-NC 4.0 + +LoCoMo is licensed for attribution and non-commercial use. Do not use the raw +dataset or derived benchmark results for commercial purposes without separate +permission. + +## Supported Official Fields + +```text +sample_id +conversation.speaker_a / conversation.speaker_b +conversation.session_N / conversation.session_N_date_time +turn.speaker / turn.dia_id / turn.text +turn.img_url / turn.blip_caption +qa.question / qa.answer / qa.evidence / qa.category +qa.adversarial_answer +``` + +Both speakers are persisted as benchmark memories. Dialog IDs such as `D1:3` +are retained as source IDs so Recall@K, MRR, and nDCG can be calculated against +the official evidence annotations. + +The paper defines the QA categories as: + +| Value | Category | +| ---: | --- | +| 1 | single-hop | +| 2 | multi-hop | +| 3 | temporal reasoning | +| 4 | open-domain knowledge | +| 5 | adversarial | + +Adversarial examples without a gold `answer` require an explicit unknown or +insufficient-information response. The supplied `adversarial_answer` is treated +as forbidden output. Two official examples include both a correct answer and an +adversarial answer; those are scored as normal answer cases with the misleading +answer forbidden. + +## Download + +```powershell +$url = "https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json" +Invoke-WebRequest -Uri $url -OutFile evaluation/external/locomo/raw/locomo10.json +``` + +Raw and processed files are ignored by Git. + +## Convert + +```bash +python evaluation/runners/run_external_eval.py --benchmark locomo +``` + +Output: + +```text +evaluation/external/locomo/processed/locomo_cases.jsonl +``` + +The current `EvaluationCase` format stores the full conversation in every QA +case. The 2.8 MB official source therefore expands to roughly 654 MB for 1,986 +QA cases. Convert once and reuse the processed file. + +## Local Smoke Test + +```bash +python -m evaluation.runners.run_qa_eval \ + --dataset evaluation/external/locomo/processed/locomo_cases.jsonl \ + --mode summary_memory \ + --limit 10 \ + --output evaluation/outputs/external/locomo/summary_memory_results.csv +``` + +This verifies conversion and deterministic retrieval scoring without paid API +calls. It is not expected to produce a competitive answer score. + +## Live MemoryBase Test + +Start with a small paid subset: + +```bash +python -m evaluation.runners.run_qa_eval \ + --dataset evaluation/external/locomo/processed/locomo_cases.jsonl \ + --mode db_qa \ + --limit 10 \ + --api-base http://127.0.0.1:8000 \ + --workspace cs3321-demo \ + --agent demo-retriever \ + --output evaluation/outputs/external/locomo/db_qa_results.csv +``` + +The default live runner creates and removes one isolated workspace per case. + +## Known Dataset Anomalies + +The adapter normalizes combined and zero-padded evidence IDs. Two official QA +items still reference dialog IDs that do not exist in their conversations. +Those IDs remain visible in `metadata.missing_evidence` and are not guessed. diff --git a/evaluation/external/longmemeval/README.md b/evaluation/external/longmemeval/README.md new file mode 100644 index 0000000..ac61107 --- /dev/null +++ b/evaluation/external/longmemeval/README.md @@ -0,0 +1,100 @@ +# LongMemEval Integration + +MemoryBase supports the official LongMemEval record shape published by the +LongMemEval authors: + +```text +question_id +question / question_date / question_type +answer / answer_session_ids +haystack_sessions / haystack_session_ids / haystack_dates +``` + +The adapter preserves session IDs, answer-session IDs, dates, and per-turn +`has_answer` metadata. Session dates are also prefixed to turn text because the +current MemoryBase write APIs do not expose a separate event-time field. +LongMemEval assistant turns are persisted as benchmark memories as well as user +turns, which is required for its assistant-memory question type. Other datasets +retain the default user-turn-only injection behavior. + +## Official Sources + +- Repository: https://github.com/xiaowu0162/LongMemEval +- Dataset: https://huggingface.co/datasets/xiaowu0162/LongMemEval +- Paper: https://arxiv.org/abs/2410.10813 + +As of June 7, 2026, the Hugging Face repository exposes extensionless files: + +| File | Approximate Size | Intended Use | +| --- | ---: | --- | +| `longmemeval_oracle` | 15 MB | Adapter validation and oracle-session experiments | +| `longmemeval_s` | 278 MB | Small-context benchmark | +| `longmemeval_m` | 2.7 GB | Medium-context benchmark | + +The official code repository is MIT licensed. The dataset repository does not +currently declare a dataset license in its card metadata. Do not redistribute +the raw dataset until its data license and usage terms have been confirmed. + +## Download + +Download one variant at a time. Keeping a single variant in `raw/` avoids +duplicate question IDs across variants. + +PowerShell example: + +```powershell +$url = "https://huggingface.co/datasets/xiaowu0162/LongMemEval/resolve/main/longmemeval_oracle" +Invoke-WebRequest -Uri $url -OutFile evaluation/external/longmemeval/raw/longmemeval_oracle +``` + +The raw and processed files are ignored by Git. Only `.gitkeep` and this guide +should be committed. + +## Convert + +```bash +python evaluation/runners/run_external_eval.py --benchmark longmemeval +``` + +Output: + +```text +evaluation/external/longmemeval/processed/longmemeval_cases.jsonl +``` + +## Local Smoke Test + +This path does not call a paid provider: + +```bash +python -m evaluation.runners.run_benchmark_eval \ + --benchmark longmemeval \ + --mode summary_memory \ + --limit 10 \ + --output evaluation/outputs/external/longmemeval/summary_memory_results.csv +``` + +## Live MemoryBase Test + +Start the API and then run a small paid subset first: + +```bash +python -m evaluation.runners.run_benchmark_eval \ + --benchmark longmemeval \ + --mode db_qa \ + --limit 10 \ + --api-base http://127.0.0.1:8000 \ + --workspace cs3321-demo \ + --agent demo-retriever \ + --output evaluation/outputs/external/longmemeval/db_qa_results.csv +``` + +The default live runner creates one isolated workspace per case and removes it +afterward. Use `--shared-workspace` only for deliberate persistence tests. + +## Current Scoring Boundary + +The adapter is format-complete for the official JSON records. Exact/contains +answer scoring is still weaker than the official LongMemEval LLM judge for +open-ended answers. Treat deterministic pass rates as smoke-test evidence until +an independent judge is configured. diff --git a/evaluation/external/memoryagentbench/README.md b/evaluation/external/memoryagentbench/README.md new file mode 100644 index 0000000..6df1e5a --- /dev/null +++ b/evaluation/external/memoryagentbench/README.md @@ -0,0 +1,36 @@ +# MemoryAgentBench Dataset Notes + +## Official Source + +- Repository: `https://github.com/HUST-AI-HYZ/MemoryAgentBench` +- Dataset: `https://huggingface.co/datasets/ai-hyz/MemoryAgentBench` +- Repository license: MIT + +The official dataset contains parquet shards for Accurate Retrieval, Conflict +Resolution, Long Range Understanding, and Test-Time Learning. + +## Supported Format + +The adapter supports the official Conflict Resolution parquet shard. Each row +contains one shared context plus parallel question, answer, and QA identifier +arrays. Conversion expands every question into an `EvaluationCase` while +retaining a shared `context_group_id`. + +Context is split into memory blocks below the 600-character context-pack +compaction boundary. Every block carries an explicit sequence label because the +official Conflict Resolution answer is determined by the later conflicting +fact. + +## Commands + +```bash +python evaluation/runners/run_external_eval.py --benchmark memoryagentbench +python evaluation/runners/run_grouped_benchmark_eval.py \ + --dataset evaluation/external/memoryagentbench/processed/memoryagentbench_cases.jsonl \ + --group factconsolidation_sh_6k \ + --limit 3 \ + --output evaluation/outputs/memoryagentbench/db_qa_results.csv +``` + +Grouped execution injects the shared context once and then evaluates multiple +questions in the same isolated workspace. diff --git a/evaluation/generators/__init__.py b/evaluation/generators/__init__.py new file mode 100644 index 0000000..4fbd1cf --- /dev/null +++ b/evaluation/generators/__init__.py @@ -0,0 +1 @@ +"""Synthetic evaluation dataset generators.""" diff --git a/evaluation/generators/long_context.py b/evaluation/generators/long_context.py new file mode 100644 index 0000000..6b650e1 --- /dev/null +++ b/evaluation/generators/long_context.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import tiktoken + +FACT = "Remember this durable project code: Aurora." +QUERY = "What is the durable project code?" +FILLER = ( + "The team reviewed routine implementation details, unrelated modules, meeting notes, " + "test fixtures, documentation edits, and ordinary progress updates. " +) +DEFAULT_FILLER_CHUNK_TOKENS = 1000 + + +def generate_long_context_cases( + output: Path, + *, + token_lengths: list[int], + encoding_name: str = "cl100k_base", +) -> Path: + encoding = tiktoken.get_encoding(encoding_name) + output.parent.mkdir(parents=True, exist_ok=True) + rows = [ + _build_case(target_tokens, encoding=encoding, encoding_name=encoding_name) + for target_tokens in token_lengths + ] + with output.open("w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row, ensure_ascii=False) + "\n") + return output + + +def _build_case(target_tokens: int, *, encoding, encoding_name: str) -> dict[str, object]: + if target_tokens < 128: + raise ValueError("long-context token lengths must be at least 128") + fact_tokens = encoding.encode(FACT) + remaining = max(target_tokens - len(fact_tokens), 0) + filler_chunks: list[str] = [] + while remaining: + chunk_tokens = min(remaining, DEFAULT_FILLER_CHUNK_TOKENS) + filler_chunks.append( + _filler_with_exact_tokens( + chunk_tokens, + encoding=encoding, + encoding_name=encoding_name, + ) + ) + remaining -= chunk_tokens + actual_tokens = len(fact_tokens) + sum(len(encoding.encode(chunk)) for chunk in filler_chunks) + sessions = [ + { + "session_id": "s0000", + "turns": [{"role": "user", "content": FACT}], + } + ] + sessions.extend( + { + "session_id": f"s{index:04d}", + "turns": [{"role": "user", "content": chunk}], + } + for index, chunk in enumerate(filler_chunks, start=1) + ) + sessions.append( + { + "session_id": f"s{len(filler_chunks) + 1:04d}", + "turns": [{"role": "user", "content": QUERY}], + } + ) + return { + "case_id": f"long_context_{target_tokens}", + "source": "synthetic_long_context", + "category": "long_context_retention", + "sessions": sessions, + "query": QUERY, + "expected_answer": "Aurora", + "expected_answer_contains": ["Aurora"], + "forbidden_answers": [], + "forbidden_patterns": [], + "gold_memory_ids": [], + "expected_behavior": "answer", + "metadata": { + "history_length_tokens": target_tokens, + "actual_history_tokens": actual_tokens, + "memory_turn_count": len(filler_chunks) + 1, + "filler_chunk_tokens": DEFAULT_FILLER_CHUNK_TOKENS, + "tokenizer": encoding_name, + }, + "notes": "Generated long-context retention case with measured filler tokens.", + } + + +def _filler_with_exact_tokens( + target_tokens: int, + *, + encoding, + encoding_name: str, +) -> str: + filler_tokens = encoding.encode(FILLER) + filler = FILLER * (target_tokens // len(filler_tokens)) + padding = " x" + padding_tokens = encoding.encode(padding) + if len(padding_tokens) != 1: + raise ValueError(f"tokenizer {encoding_name!r} does not encode padding as one token") + while len(encoding.encode(filler)) < target_tokens: + filler += padding + while len(encoding.encode(filler)) > target_tokens: + filler = filler[: -len(padding)] + return filler diff --git a/evaluation/io.py b/evaluation/io.py index 1393541..74b483f 100644 --- a/evaluation/io.py +++ b/evaluation/io.py @@ -5,6 +5,7 @@ from typing import Iterable from evaluation.baselines import EvaluationResult +from evaluation.pricing import estimate_model_cost RESULT_FIELDS = [ "case_id", @@ -18,12 +19,25 @@ "retrieved_scores", "latency_ms", "token_usage", + "prompt_tokens", + "completion_tokens", + "context_tokens", + "provider", + "model", + "estimated_cost", + "cost_currency", + "pricing_as_of", + "pricing_assumption", + "history_length_tokens", "score", "pass", "exact_match", "contains_match", + "refusal_match", "simple_f1", "forbidden_answer_violation", + "historical_value_mention", + "stale_answer_error", "recall_at_1", "recall_at_3", "recall_at_5", @@ -32,6 +46,10 @@ "ndcg_at_10", "deletion_success", "privacy_leakage", + "retrieval_leakage", + "answer_leakage", + "citation_valid", + "groundedness", "stale_memory_error", "preference_following", "error", @@ -49,8 +67,31 @@ def write_results_csv(path: Path, results: Iterable[EvaluationResult]) -> None: writer.writerow(result_to_row(result)) +def append_result_csv(path: Path, result: EvaluationResult) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + write_header = not path.exists() or path.stat().st_size == 0 + with path.open("a", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=RESULT_FIELDS) + if write_header: + writer.writeheader() + writer.writerow(result_to_row(result)) + + +def completed_case_ids(path: Path) -> set[str]: + if not path.exists() or path.stat().st_size == 0: + return set() + with path.open("r", encoding="utf-8", newline="") as handle: + return {row["case_id"] for row in csv.DictReader(handle) if row.get("case_id")} + + def result_to_row(result: EvaluationResult) -> dict[str, object]: metrics = result.metadata.get("metrics", {}) + cost = estimate_model_cost( + provider=result.metadata.get("provider"), + model=result.metadata.get("model"), + prompt_tokens=result.metadata.get("prompt_tokens"), + completion_tokens=result.metadata.get("completion_tokens"), + ) return { "case_id": result.case_id, "source": result.source, @@ -63,12 +104,25 @@ def result_to_row(result: EvaluationResult) -> dict[str, object]: "retrieved_scores": "|".join(f"{score:.6f}" for score in result.retrieved_scores), "latency_ms": f"{result.latency_ms:.3f}", "token_usage": result.token_usage if result.token_usage is not None else "", + "prompt_tokens": _metadata(result, "prompt_tokens"), + "completion_tokens": _metadata(result, "completion_tokens"), + "context_tokens": _metadata(result, "context_tokens"), + "provider": _metadata(result, "provider"), + "model": _metadata(result, "model"), + "estimated_cost": _cost_value(cost["estimated_cost"]), + "cost_currency": cost["cost_currency"] or "", + "pricing_as_of": cost["pricing_as_of"] or "", + "pricing_assumption": cost["pricing_assumption"] or "", + "history_length_tokens": _metadata(result, "history_length_tokens"), "score": f"{result.score:.6f}", "pass": "true" if result.passed else "false", "exact_match": _metric(metrics, "exact_match"), "contains_match": _metric(metrics, "contains_match"), + "refusal_match": _metric(metrics, "refusal_match"), "simple_f1": _metric(metrics, "simple_f1"), "forbidden_answer_violation": _metric(metrics, "forbidden_answer_violation"), + "historical_value_mention": _metric(metrics, "historical_value_mention"), + "stale_answer_error": _metric(metrics, "stale_answer_error"), "recall_at_1": _metric(metrics, "recall_at_1"), "recall_at_3": _metric(metrics, "recall_at_3"), "recall_at_5": _metric(metrics, "recall_at_5"), @@ -77,6 +131,10 @@ def result_to_row(result: EvaluationResult) -> dict[str, object]: "ndcg_at_10": _metric(metrics, "ndcg_at_10"), "deletion_success": _metric(metrics, "deletion_success"), "privacy_leakage": _metric(metrics, "privacy_leakage"), + "retrieval_leakage": _metric(metrics, "retrieval_leakage"), + "answer_leakage": _metric(metrics, "answer_leakage"), + "citation_valid": _metric(metrics, "citation_valid"), + "groundedness": _metric(metrics, "groundedness"), "stale_memory_error": _metric(metrics, "stale_memory_error"), "preference_following": _metric(metrics, "preference_following"), "error": result.error, @@ -94,3 +152,14 @@ def _metric(metrics: dict[str, object], key: str) -> str: if isinstance(value, float): return f"{value:.6f}" return str(value) + + +def _metadata(result: EvaluationResult, key: str) -> object: + value = result.metadata.get(key) + return "" if value is None else value + + +def _cost_value(value: object) -> str: + if not isinstance(value, float): + return "" + return f"{value:.8f}" diff --git a/evaluation/judging.py b/evaluation/judging.py new file mode 100644 index 0000000..2c459e2 --- /dev/null +++ b/evaluation/judging.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +import csv +import json +import re +import time +from pathlib import Path +from typing import Callable + +import httpx + +from evaluation.cases import EvaluationCase, parse_case +from evaluation.pricing import estimate_model_cost + +JUDGE_FIELDS = [ + "judge_pass", + "judge_score", + "judge_reason", + "judge_provider", + "judge_model", + "judge_prompt_tokens", + "judge_completion_tokens", + "judge_cost", + "judge_same_model", +] + + +def judge_results_csv( + *, + input_csv: Path, + dataset: Path, + output_csv: Path, + api_key: str, + base_url: str, + model: str, + provider: str = "deepseek", + request: Callable[..., httpx.Response] | None = None, +) -> Path: + source_csv = output_csv if output_csv.exists() else input_csv + with source_csv.open("r", encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + rows = list(reader) + fields = list(reader.fieldnames or []) + output_fields = fields + [field for field in JUDGE_FIELDS if field not in fields] + case_ids = {row.get("case_id", "") for row in rows} + cases = _load_selected_cases(dataset, case_ids) + requester = request or httpx.request + output_csv.parent.mkdir(parents=True, exist_ok=True) + _write_judge_rows(output_csv, rows, output_fields) + for index, row in enumerate(rows, start=1): + if row.get("judge_pass") in {"true", "false"}: + continue + case = cases.get(row.get("case_id", "")) + if case is None: + raise ValueError(f"case {row.get('case_id')!r} was not found in {dataset}") + judgement = _judge_answer_with_retry( + case=case, + generated_answer=row.get("generated_answer", ""), + api_key=api_key, + base_url=base_url, + model=model, + provider=provider, + request=requester, + ) + row.update(judgement) + _write_judge_rows(output_csv, rows, output_fields) + if index % 10 == 0 or index == len(rows): + print(f"judged {index}/{len(rows)}", flush=True) + return output_csv + + +def _write_judge_rows( + output_csv: Path, + rows: list[dict[str, str]], + fields: list[str], +) -> None: + with output_csv.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fields) + writer.writeheader() + writer.writerows(rows) + + +def _judge_answer_with_retry( + *, + case: EvaluationCase, + generated_answer: str, + api_key: str, + base_url: str, + model: str, + provider: str, + request: Callable[..., httpx.Response], + max_attempts: int = 3, +) -> dict[str, str]: + for attempt in range(max_attempts): + try: + return judge_answer( + case=case, + generated_answer=generated_answer, + api_key=api_key, + base_url=base_url, + model=model, + provider=provider, + request=request, + ) + except (httpx.HTTPError, ValueError): + if attempt + 1 == max_attempts: + raise + time.sleep(0.5 * (2**attempt)) + raise RuntimeError("judge retry loop exited unexpectedly") + + +def _load_selected_cases( + dataset: Path, + case_ids: set[str], +) -> dict[str, EvaluationCase]: + cases: dict[str, EvaluationCase] = {} + with dataset.open("r", encoding="utf-8") as handle: + for line_no, line in enumerate(handle, start=1): + raw = json.loads(line) + if raw.get("case_id") not in case_ids: + continue + case = parse_case(raw, path=dataset, line_no=line_no) + cases[case.case_id] = case + if len(cases) == len(case_ids): + break + return cases + + +def judge_answer( + *, + case: EvaluationCase, + generated_answer: str, + api_key: str, + base_url: str, + model: str, + provider: str, + request: Callable[..., httpx.Response], +) -> dict[str, str]: + response = request( + "POST", + f"{base_url.rstrip('/')}/chat/completions", + timeout=120.0, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + json={ + "model": model, + "temperature": 0, + "max_tokens": 200, + "response_format": {"type": "json_object"}, + "messages": [ + { + "role": "system", + "content": ( + "You are a strict benchmark answer judge. Evaluate the overall " + "meaning, not substring overlap. A response that mentions the gold " + "answer but ultimately denies it or says it cannot determine the " + "answer is incorrect. For refuse_or_unknown cases, require an " + "explicit unanswerable or insufficient-information conclusion. " + "Return JSON with pass (boolean), score (0 to 1), and reason." + ), + }, + { + "role": "user", + "content": json.dumps( + { + "question": case.query, + "gold_answer": case.expected_answer, + "accepted_answers": case.metadata.get("accepted_answers", []), + "expected_behavior": case.expected_behavior, + "forbidden_answers": case.forbidden_answers, + "candidate_answer": generated_answer, + }, + ensure_ascii=False, + ), + }, + ], + }, + ) + response.raise_for_status() + payload = response.json() + content = str(payload.get("choices", [{}])[0].get("message", {}).get("content", "")) + parsed = _parse_judgement(content) + usage = payload.get("usage") if isinstance(payload.get("usage"), dict) else {} + cost = estimate_model_cost( + provider=provider, + model=model, + prompt_tokens=usage.get("prompt_tokens"), + completion_tokens=usage.get("completion_tokens"), + ) + return { + "judge_pass": str(bool(parsed["pass"])).lower(), + "judge_score": f"{float(parsed['score']):.6f}", + "judge_reason": str(parsed["reason"]), + "judge_provider": provider, + "judge_model": model, + "judge_prompt_tokens": str(usage.get("prompt_tokens") or ""), + "judge_completion_tokens": str(usage.get("completion_tokens") or ""), + "judge_cost": ( + f"{float(cost['estimated_cost']):.8f}" + if isinstance(cost["estimated_cost"], float) + else "" + ), + "judge_same_model": "true", + } + + +def _parse_judgement(content: str) -> dict[str, object]: + try: + raw = json.loads(content) + except json.JSONDecodeError: + match = re.search(r"\{.*\}", content, re.DOTALL) + if match is None: + raise ValueError(f"judge response was not JSON: {content[:200]}") + raw = json.loads(match.group(0)) + if not isinstance(raw, dict) or not isinstance(raw.get("pass"), bool): + raise ValueError(f"judge response has invalid schema: {raw!r}") + score = raw.get("score", 1.0 if raw["pass"] else 0.0) + return { + "pass": raw["pass"], + "score": min(max(float(score), 0.0), 1.0), + "reason": str(raw.get("reason") or ""), + } diff --git a/evaluation/metrics/qa_metrics.py b/evaluation/metrics/qa_metrics.py index 7a441ca..68e4a75 100644 --- a/evaluation/metrics/qa_metrics.py +++ b/evaluation/metrics/qa_metrics.py @@ -8,19 +8,39 @@ def score_qa(case: EvaluationCase, generated_answer: str) -> dict[str, float | bool]: exact = _normalize(generated_answer) == _normalize(case.expected_answer or "") contains = _contains_expected(case, generated_answer) - forbidden = _has_forbidden_answer(case, generated_answer) or _has_forbidden_pattern( + refusal = _is_refusal(generated_answer) + forbidden_mention = _has_forbidden_answer(case, generated_answer) or _has_forbidden_pattern( case, generated_answer ) + answer_latest = case.expected_behavior == "answer_latest" + blocking_forbidden = forbidden_mention and not answer_latest + stale_answer_error = answer_latest and not contains + expected_refusal = case.expected_behavior == "refuse_or_unknown" expected_text = case.expected_answer or " ".join(case.expected_answer_contains) f1 = simple_f1(expected_text, generated_answer) - passed = _passes_case(case, exact=exact, contains=contains, forbidden=forbidden) + passed = _passes_case( + case, + exact=exact, + contains=contains, + forbidden=blocking_forbidden, + refusal=refusal, + ) return { "exact_match": exact, "contains_match": contains, - "forbidden_answer_violation": forbidden, + "refusal_match": refusal, + "forbidden_answer_violation": forbidden_mention, + "historical_value_mention": answer_latest and forbidden_mention, + "stale_answer_error": stale_answer_error, "simple_f1": f1, "pass": passed, - "score": _score(exact=exact, contains=contains, forbidden=forbidden, f1=f1), + "score": _score( + exact=exact, + contains=contains, + forbidden=blocking_forbidden, + refusal=expected_refusal and refusal, + f1=f1, + ), } @@ -53,11 +73,12 @@ def _passes_case( exact: bool, contains: bool, forbidden: bool, + refusal: bool, ) -> bool: if forbidden: return False if case.expected_behavior == "refuse_or_unknown" and case.expected_answer is None: - return True + return refusal if case.expected_answer_contains: return contains if case.expected_answer is not None: @@ -88,9 +109,18 @@ def _has_forbidden_pattern(case: EvaluationCase, generated_answer: str) -> bool: return False -def _score(*, exact: bool, contains: bool, forbidden: bool, f1: float) -> float: +def _score( + *, + exact: bool, + contains: bool, + forbidden: bool, + refusal: bool, + f1: float, +) -> float: if forbidden: return 0.0 + if refusal: + return 1.0 if exact: return 1.0 if contains: @@ -98,6 +128,25 @@ def _score(*, exact: bool, contains: bool, forbidden: bool, f1: float) -> float: return f1 +def _is_refusal(value: str) -> bool: + normalized = _normalize(value) + patterns = ( + r"\bi do not know\b", + r"\bi don't know\b", + r"\bnot enough (?:information|context)\b", + r"\binsufficient (?:information|context)\b", + r"\bcannot (?:determine|answer|tell)\b", + r"\bcan't (?:determine|answer|tell)\b", + r"\bunknown\b", + r"不知道", + r"不清楚", + r"信息不足", + r"无法(?:确定|判断|回答)", + r"没有足够(?:的信息|上下文)", + ) + return any(re.search(pattern, normalized) for pattern in patterns) + + def _normalize(value: str) -> str: return " ".join(value.strip().lower().split()) diff --git a/evaluation/metrics/system_metrics.py b/evaluation/metrics/system_metrics.py index 3754e22..d4ae9c9 100644 --- a/evaluation/metrics/system_metrics.py +++ b/evaluation/metrics/system_metrics.py @@ -20,4 +20,7 @@ def latency_summary(latencies_ms: list[float], error_count: int = 0) -> dict[str "p99_latency_ms": percentile(latencies_ms, 0.99), "average_latency_ms": sum(latencies_ms) / total if total else 0.0, "error_rate": error_count / (total + error_count) if total + error_count else 0.0, + "throughput_qps": ( + total / (sum(latencies_ms) / 1000) if total and sum(latencies_ms) > 0 else 0.0 + ), } diff --git a/evaluation/performance.py b/evaluation/performance.py new file mode 100644 index 0000000..29db1b7 --- /dev/null +++ b/evaluation/performance.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +import csv +import json +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable +from urllib.parse import urlparse +from uuid import uuid4 + +import httpx + +from evaluation.metrics.system_metrics import latency_summary + + +@dataclass(slots=True) +class OperationSample: + operation: str + latency_ms: float + success: bool + status_code: int | None = None + error: str = "" + + +def run_api_performance_suite( + *, + api_base_url: str, + workspace: str, + agent: str | None, + iterations: int, + include_qa: bool, + request: Callable[..., httpx.Response] | None = None, +) -> list[OperationSample]: + if iterations < 1: + raise ValueError("iterations must be at least 1") + base_url = api_base_url.rstrip("/") + if request is None: + client = httpx.Client(trust_env=not _is_loopback_url(base_url)) + request = client.request + else: + client = None + try: + workspace_id, agent_id = _resolve_targets( + request=request, + base_url=base_url, + workspace=workspace, + agent=agent, + ) + samples: list[OperationSample] = [] + for index in range(iterations): + text = f"Performance probe {uuid4()} iteration {index}." + created = _sample_request( + samples, + "write", + lambda: request( + "POST", + f"{base_url}/api/memories", + timeout=60.0, + headers={ + "X-Actor-Type": "system", + "X-Revision-Reason": "evaluation performance", + }, + json={ + "workspace_id": workspace_id, + "memory_type": "semantic", + "canonical_text": text, + "summary": "Evaluation performance probe", + "owner_agent_id": agent_id, + "evidence": [], + }, + ), + ) + memory_id = _json_object(created).get("memory_id") if created is not None else None + _sample_request( + samples, + "recall", + lambda: request( + "POST", + f"{base_url}/api/recall", + timeout=60.0, + json={ + "workspace_id": workspace_id, + "agent_id": agent_id, + "query_text": text, + "retrieval_mode": "hybrid", + "limit": 5, + }, + ), + ) + _sample_request( + samples, + "context", + lambda: request( + "POST", + f"{base_url}/api/recall/context-pack", + timeout=60.0, + json={ + "workspace_id": workspace_id, + "agent_id": agent_id, + "query_text": text, + "retrieval_mode": "hybrid", + "limit": 5, + "max_tokens": 1000, + }, + ), + ) + if include_qa: + _sample_request( + samples, + "qa", + lambda: request( + "POST", + f"{base_url}/api/qa/answer", + timeout=120.0, + json={ + "workspace_id": workspace_id, + "agent_id": agent_id, + "query_text": text, + "retrieval_mode": "hybrid", + "limit": 5, + "max_answer_tokens": 100, + }, + ), + ) + if memory_id: + _sample_request( + samples, + "update", + lambda: request( + "PATCH", + f"{base_url}/api/memories/{memory_id}", + timeout=60.0, + params={"workspace_id": workspace_id}, + headers={ + "X-Actor-Type": "system", + "X-Revision-Reason": "evaluation performance update", + }, + json={"summary": "Updated evaluation performance probe"}, + ), + ) + _sample_request( + samples, + "delete", + lambda: request( + "DELETE", + f"{base_url}/api/memories/{memory_id}", + timeout=60.0, + params={"workspace_id": workspace_id}, + headers={ + "X-Actor-Type": "system", + "X-Revision-Reason": "evaluation performance cleanup", + }, + ), + ) + return samples + finally: + if client is not None: + client.close() + + +def write_performance_outputs( + *, + output_csv: Path, + samples: list[OperationSample], +) -> tuple[Path, Path]: + output_csv.parent.mkdir(parents=True, exist_ok=True) + with output_csv.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=["operation", "latency_ms", "success", "status_code", "error"], + ) + writer.writeheader() + for sample in samples: + writer.writerow( + { + "operation": sample.operation, + "latency_ms": f"{sample.latency_ms:.3f}", + "success": str(sample.success).lower(), + "status_code": sample.status_code or "", + "error": sample.error, + } + ) + summary_path = output_csv.with_suffix(".summary.json") + summary_path.write_text( + json.dumps(summarize_operations(samples), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + return output_csv, summary_path + + +def summarize_operations(samples: list[OperationSample]) -> dict[str, dict[str, float | int]]: + operations = sorted({sample.operation for sample in samples}) + result: dict[str, dict[str, float | int]] = {} + for operation in operations: + rows = [sample for sample in samples if sample.operation == operation] + successful = [sample.latency_ms for sample in rows if sample.success] + errors = sum(1 for sample in rows if not sample.success) + result[operation] = { + "samples": len(rows), + "successes": len(successful), + "errors": errors, + **latency_summary(successful, error_count=errors), + } + return result + + +def _sample_request( + samples: list[OperationSample], + operation: str, + call: Callable[[], httpx.Response], +) -> httpx.Response | None: + started = time.perf_counter() + try: + response = call() + latency_ms = (time.perf_counter() - started) * 1000 + success = response.status_code < 400 + samples.append( + OperationSample( + operation=operation, + latency_ms=latency_ms, + success=success, + status_code=response.status_code, + error="" if success else response.text[:500], + ) + ) + return response if success else None + except Exception as exc: + samples.append( + OperationSample( + operation=operation, + latency_ms=(time.perf_counter() - started) * 1000, + success=False, + error=str(exc), + ) + ) + return None + + +def _resolve_targets( + *, + request: Callable[..., httpx.Response], + base_url: str, + workspace: str, + agent: str | None, +) -> tuple[str, str | None]: + params = {"workspace": workspace} + if agent: + params["agent"] = agent + response = request("GET", f"{base_url}/api/health/detail", params=params, timeout=30.0) + response.raise_for_status() + payload = _json_object(response) + workspace_data = payload.get("workspace") + if not isinstance(workspace_data, dict) or not workspace_data.get("found"): + raise ValueError(f"workspace {workspace!r} was not found") + agent_data = payload.get("agent") + agent_id = ( + str(agent_data["agent_id"]) + if agent and isinstance(agent_data, dict) and agent_data.get("found") + else None + ) + if agent and agent_id is None: + raise ValueError(f"agent {agent!r} was not found") + return str(workspace_data["workspace_id"]), agent_id + + +def _json_object(response: httpx.Response) -> dict[str, Any]: + payload = response.json() + return payload if isinstance(payload, dict) else {} + + +def _is_loopback_url(value: str) -> bool: + return (urlparse(value).hostname or "").lower() in {"127.0.0.1", "localhost", "::1"} diff --git a/evaluation/pricing.py b/evaluation/pricing.py new file mode 100644 index 0000000..e3259c8 --- /dev/null +++ b/evaluation/pricing.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class ModelPrice: + currency: str + input_per_million: float + output_per_million: float + pricing_as_of: str + source: str + assumption: str + + +DEEPSEEK_PRICING_SOURCE = "https://api-docs.deepseek.com/zh-cn/quick_start/pricing" + +MODEL_PRICES: dict[tuple[str, str], ModelPrice] = { + ("deepseek", "deepseek-chat"): ModelPrice( + currency="CNY", + input_per_million=1.0, + output_per_million=2.0, + pricing_as_of="2026-06-07", + source=DEEPSEEK_PRICING_SOURCE, + assumption="cache_miss", + ), + ("deepseek", "deepseek-v4-flash"): ModelPrice( + currency="CNY", + input_per_million=1.0, + output_per_million=2.0, + pricing_as_of="2026-06-07", + source=DEEPSEEK_PRICING_SOURCE, + assumption="cache_miss", + ), +} + + +def estimate_model_cost( + *, + provider: object, + model: object, + prompt_tokens: object, + completion_tokens: object, +) -> dict[str, object]: + price = MODEL_PRICES.get((str(provider or "").lower(), str(model or "").lower())) + input_tokens = _optional_int(prompt_tokens) + output_tokens = _optional_int(completion_tokens) + if price is None or input_tokens is None or output_tokens is None: + return { + "estimated_cost": None, + "cost_currency": None, + "pricing_as_of": None, + "pricing_source": None, + "pricing_assumption": None, + } + estimated_cost = ( + input_tokens * price.input_per_million + output_tokens * price.output_per_million + ) / 1_000_000 + return { + "estimated_cost": estimated_cost, + "cost_currency": price.currency, + "pricing_as_of": price.pricing_as_of, + "pricing_source": price.source, + "pricing_assumption": price.assumption, + } + + +def _optional_int(value: object) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float) and value.is_integer(): + return int(value) + if isinstance(value, str): + try: + return int(value) + except ValueError: + return None + return None diff --git a/evaluation/reports/generate_report.py b/evaluation/reports/generate_report.py index 17d02ea..66a9116 100644 --- a/evaluation/reports/generate_report.py +++ b/evaluation/reports/generate_report.py @@ -2,6 +2,7 @@ import argparse import csv +import json import sys from collections import Counter, defaultdict from pathlib import Path @@ -9,6 +10,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from evaluation.metrics.system_metrics import latency_summary # noqa: E402 +from evaluation.pricing import estimate_model_cost # noqa: E402 PROJECT_ROOT = Path(__file__).resolve().parents[2] DEFAULT_OUTPUTS = PROJECT_ROOT / "evaluation" / "outputs" @@ -17,7 +19,11 @@ def generate_report(*, outputs_dir: Path = DEFAULT_OUTPUTS) -> Path: outputs_dir.mkdir(parents=True, exist_ok=True) report_path = outputs_dir / "benchmark_report.md" - csv_paths = sorted(path for path in outputs_dir.rglob("*_results.csv") if path.is_file()) + csv_paths = sorted( + path + for path in outputs_dir.rglob("*_results.csv") + if path.is_file() and _is_evaluation_result_csv(path) + ) lines = [ "# MemoryBase Benchmark Report", "", @@ -26,8 +32,8 @@ def generate_report(*, outputs_dir: Path = DEFAULT_OUTPUTS) -> Path: "| Field | Value |", "| --- | --- |", "| framework | local evaluation runner |", - "| model | not configured in Phase E1 |", - "| database | not called by no_memory/dry-run baseline |", + f"| providers/models | {_provider_summary(csv_paths)} |", + "| database | live modes call MemoryBase APIs; local modes do not |", "| outputs_dir | `%s` |" % outputs_dir.as_posix(), "", "## Result Files", @@ -44,7 +50,7 @@ def generate_report(*, outputs_dir: Path = DEFAULT_OUTPUTS) -> Path: ) for path in csv_paths: rows = _read_rows(path) - pass_count = sum(1 for row in rows if row.get("pass") == "true") + pass_count = sum(1 for row in rows if _effective_pass(row) == "true") error_count = sum(1 for row in rows if row.get("error")) latencies = [_float(row.get("latency_ms")) for row in rows] summary = latency_summary(latencies, error_count=error_count) @@ -71,9 +77,9 @@ def generate_report(*, outputs_dir: Path = DEFAULT_OUTPUTS) -> Path: if baseline_rows: lines.extend( [ - "| Baseline | Cases | Pass Rate | Avg F1 | Privacy Leakage | " - "Stale Error | P95 Latency (ms) |", - "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + "| Baseline | Cases | Pass Rate | Avg F1 | Groundedness | Avg Tokens | " + "Avg Cost | Privacy Leakage | Stale Error | P95 Latency (ms) |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", ] ) for mode, rows in sorted(baseline_rows.items()): @@ -95,13 +101,53 @@ def generate_report(*, outputs_dir: Path = DEFAULT_OUTPUTS) -> Path: else: lines.append("No detailed category metrics are available yet.") + lines.extend(["", "## Long-Context Retention", ""]) + retention_rows = _retention_rows(csv_paths) + if retention_rows: + lines.extend( + [ + "| Baseline | History Tokens | Cases | Accuracy | Retention | Forgetting |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ] + ) + for row in retention_rows: + lines.append( + f"| {row['mode']} | {row['tokens']} | {row['cases']} | " + f"{row['accuracy']:.2%} | {row['retention']:.2%} | " + f"{row['forgetting']:.2%} |" + ) + else: + lines.append("No measured long-context token-length results are available.") + + lines.extend(["", "## API Operation Performance", ""]) + performance_rows = _operation_performance_rows(outputs_dir) + if performance_rows: + lines.extend( + [ + "| Operation | Samples | Errors | P50 (ms) | P95 (ms) | P99 (ms) | QPS |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + ] + ) + for operation, values in performance_rows: + lines.append( + f"| {operation} | {int(values.get('samples', 0))} | " + f"{int(values.get('errors', 0))} | " + f"{float(values.get('p50_latency_ms', 0)):.3f} | " + f"{float(values.get('p95_latency_ms', 0)):.3f} | " + f"{float(values.get('p99_latency_ms', 0)):.3f} | " + f"{float(values.get('throughput_qps', 0)):.3f} |" + ) + else: + lines.append("No API operation performance summary is available.") + lines.extend( [ "", "## Baseline Comparison", "", "`no_memory`, `recency_only`, `summary_memory`, `db_memory`, " - "`db_extraction`, and `naive_vector_rag` execute. `db_extraction` " + "`db_extraction`, `naive_vector_rag`, `db_qa`, `vector_qa`, and " + "`db_extraction_qa` execute. `db_extraction` " "uses rule-based candidate extraction before recall. `naive_vector_rag` " "runs local embedding backfill and then calls vector-mode recall.", "", @@ -133,10 +179,13 @@ def generate_report(*, outputs_dir: Path = DEFAULT_OUTPUTS) -> Path: "- `db_memory` uses memory create and recall APIs.", "- `db_extraction` uses source import, chunk extraction, candidate approve, " "and recall.", - "- `naive_vector_rag` depends on backend embedding tables and " - "the local hashing provider.", + "- Vector modes depend on the configured backend embedding provider.", "- External benchmark adapters support common JSON/JSONL shapes only.", - "- LLM-as-judge, groundedness, and token cost are not implemented yet.", + "- Official external benchmark files and licenses remain operator-supplied.", + "- Semantic judge results override deterministic pass when present.", + "- The current smoke judge may use the same model family as answer generation.", + "- Groundedness is citation validation, not a separate LLM judge.", + "- Session and extraction source cleanup awaits public delete APIs.", "", ] ) @@ -157,13 +206,19 @@ def _read_rows(path: Path) -> list[dict[str, str]]: return list(csv.DictReader(handle)) +def _is_evaluation_result_csv(path: Path) -> bool: + with path.open("r", encoding="utf-8", newline="") as handle: + fields = set(csv.DictReader(handle).fieldnames or []) + return {"case_id", "category", "pass", "mode", "run_id"}.issubset(fields) + + def _category_rows(csv_paths: list[Path]) -> dict[str, Counter]: categories: dict[str, Counter] = defaultdict(Counter) for path in csv_paths: for row in _read_rows(path): category = row.get("category") or "uncategorized" categories[category]["total"] += 1 - if row.get("pass") == "true": + if _effective_pass(row) == "true": categories[category]["passed"] += 1 return categories @@ -172,7 +227,7 @@ def _failed_rows(csv_paths: list[Path]) -> list[dict[str, str]]: rows: list[dict[str, str]] = [] for path in csv_paths: for row in _read_rows(path): - if row.get("pass") != "true" or row.get("error"): + if _effective_pass(row) != "true" or row.get("error"): rows.append(row) return rows @@ -190,14 +245,41 @@ def _metric_summary_row(label: str, rows: list[dict[str, str]]) -> str: avg_f1 = _average(rows, "simple_f1") privacy = _bool_rate(rows, "privacy_leakage") stale = _bool_rate(rows, "stale_memory_error") + groundedness = _average(rows, "groundedness") + average_tokens = _average(rows, "token_usage") + average_cost, currency = _average_cost(rows) latencies = [_float(row.get("latency_ms")) for row in rows] p95 = latency_summary(latencies)["p95_latency_ms"] return ( f"| {label} | {len(rows)} | {pass_rate:.2%} | {avg_f1:.3f} | " - f"{privacy:.2%} | {stale:.2%} | {p95:.3f} |" + f"{groundedness:.3f} | {average_tokens:.1f} | " + f"{average_cost:.6f} {currency} | {privacy:.2%} | " + f"{stale:.2%} | {p95:.3f} |" ) +def _average_cost(rows: list[dict[str, str]]) -> tuple[float, str]: + costs: list[float] = [] + currency = "" + for row in rows: + raw_cost = row.get("estimated_cost") + if raw_cost: + costs.append(_float(raw_cost)) + currency = row.get("cost_currency") or currency + continue + estimate = estimate_model_cost( + provider=row.get("provider"), + model=row.get("model"), + prompt_tokens=row.get("prompt_tokens"), + completion_tokens=row.get("completion_tokens"), + ) + estimated_cost = estimate["estimated_cost"] + if isinstance(estimated_cost, float): + costs.append(estimated_cost) + currency = str(estimate["cost_currency"] or currency) + return (sum(costs) / len(costs) if costs else 0.0, currency or "-") + + def _category_metric_summary_row(label: str, rows: list[dict[str, str]]) -> str: return ( f"| {label} | {len(rows)} | {_bool_rate(rows, 'pass'):.2%} | " @@ -208,12 +290,23 @@ def _category_metric_summary_row(label: str, rows: list[dict[str, str]]) -> str: def _bool_rate(rows: list[dict[str, str]], key: str) -> float: - values = [row.get(key) for row in rows if row.get(key) in {"true", "false"}] + values = [ + _effective_pass(row) if key == "pass" else row.get(key) + for row in rows + if (_effective_pass(row) if key == "pass" else row.get(key)) in {"true", "false"} + ] if not values: return 0.0 return sum(1 for value in values if value == "true") / len(values) +def _effective_pass(row: dict[str, str]) -> str | None: + judge_pass = row.get("judge_pass") + if judge_pass in {"true", "false"}: + return judge_pass + return row.get("pass") + + def _average(rows: list[dict[str, str]], key: str) -> float: values = [_float(row.get(key)) for row in rows if row.get(key)] return sum(values) / len(values) if values else 0.0 @@ -226,5 +319,64 @@ def _float(value: str | None) -> float: return 0.0 +def _provider_summary(csv_paths: list[Path]) -> str: + values: set[str] = set() + for path in csv_paths: + for row in _read_rows(path): + provider = row.get("provider") + model = row.get("model") + if provider or model: + values.add("/".join(item for item in (provider, model) if item)) + return ", ".join(sorted(values)) if values else "not recorded" + + +def _retention_rows(csv_paths: list[Path]) -> list[dict[str, object]]: + grouped: dict[tuple[str, int], list[dict[str, str]]] = defaultdict(list) + for path in csv_paths: + if "long_context" not in path.name and "forgetting" not in path.name: + continue + for row in _read_rows(path): + raw_tokens = row.get("history_length_tokens") + if not raw_tokens: + continue + grouped[(row.get("mode") or "unknown", int(float(raw_tokens)))].append(row) + result: list[dict[str, object]] = [] + by_mode: dict[str, list[tuple[int, float, int]]] = defaultdict(list) + for (mode, tokens), rows in grouped.items(): + accuracy = _bool_rate(rows, "pass") + by_mode[mode].append((tokens, accuracy, len(rows))) + for mode, values in sorted(by_mode.items()): + values.sort() + short_accuracy = values[0][1] + for tokens, accuracy, count in values: + retention = accuracy / short_accuracy if short_accuracy > 0 else 0.0 + result.append( + { + "mode": mode, + "tokens": tokens, + "cases": count, + "accuracy": accuracy, + "retention": retention, + "forgetting": max(0.0, 1.0 - retention), + } + ) + return result + + +def _operation_performance_rows(outputs_dir: Path) -> list[tuple[str, dict[str, object]]]: + rows: list[tuple[str, dict[str, object]]] = [] + for path in sorted(outputs_dir.rglob("*.summary.json")): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + if not isinstance(payload, dict): + continue + for operation, values in payload.items(): + if isinstance(values, dict): + rows.append((str(operation), values)) + return rows + + if __name__ == "__main__": main() diff --git a/evaluation/results/longmemeval_full_20260609.csv b/evaluation/results/longmemeval_full_20260609.csv new file mode 100644 index 0000000..d372567 --- /dev/null +++ b/evaluation/results/longmemeval_full_20260609.csv @@ -0,0 +1,501 @@ +"case_id","category","deterministic_pass","semantic_pass","judge_score" +"longmemeval_gpt4_2655b836","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_2487a7cb","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_76048e76","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_2312f94c","temporal_reasoning","true","true","1.000000" +"longmemeval_0bb5a684","temporal_reasoning","false","false","0.000000" +"longmemeval_08f4fc43","temporal_reasoning","false","true","1.000000" +"longmemeval_2c63a862","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_385a5000","temporal_reasoning","false","false","0.000000" +"longmemeval_2a1811e2","temporal_reasoning","false","false","0.000000" +"longmemeval_bbf86515","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_5dcc0aab","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_0b2f1d21","temporal_reasoning","false","false","0.000000" +"longmemeval_f0853d11","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_6ed717ea","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_70e84552","temporal_reasoning","false","true","1.000000" +"longmemeval_a3838d2b","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_93159ced","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_2d58bcd6","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_65aabe59","temporal_reasoning","false","true","1.000000" +"longmemeval_982b5123","temporal_reasoning","false","false","0.000000" +"longmemeval_b9cfe692","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_4edbafa2","temporal_reasoning","false","true","1.000000" +"longmemeval_c8090214","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_483dd43c","temporal_reasoning","false","true","1.000000" +"longmemeval_e4e14d04","temporal_reasoning","false","false","0.000000" +"longmemeval_c9f37c46","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_2c50253f","temporal_reasoning","false","false","0.000000" +"longmemeval_dcfa8644","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_b4a80587","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_9a159967","temporal_reasoning","true","false","0.000000" +"longmemeval_cc6d1ec1","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_8c8961ae","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_d9af6064","temporal_reasoning","true","false","0.000000" +"longmemeval_gpt4_7de946e7","temporal_reasoning","false","true","1.000000" +"longmemeval_d01c6aa8","temporal_reasoning","false","false","0.000000" +"longmemeval_993da5e2","temporal_reasoning","false","true","1.000000" +"longmemeval_a3045048","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_d31cdae3","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_cd90e484","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_88806d6e","temporal_reasoning","true","false","0.000000" +"longmemeval_gpt4_4cd9eba1","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_93f6379c","temporal_reasoning","true","true","1.000000" +"longmemeval_b29f3365","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_2f56ae70","temporal_reasoning","false","false","0.000000" +"longmemeval_6613b389","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_78cf46a3","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_0a05b494","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_1a1dc16d","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_2f584639","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_213fd887","temporal_reasoning","true","false","0.000000" +"longmemeval_gpt4_5438fa52","temporal_reasoning","true","false","0.000000" +"longmemeval_gpt4_c27434e8","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_fe651585","temporal_reasoning","true","false","0.000000" +"longmemeval_8c18457d","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_70e84552_abs","abstention","false","false","0.000000" +"longmemeval_gpt4_93159ced_abs","abstention","false","true","1.000000" +"longmemeval_982b5123_abs","abstention","false","true","1.000000" +"longmemeval_c8090214_abs","abstention","false","true","1.000000" +"longmemeval_gpt4_c27434e8_abs","abstention","false","true","1.000000" +"longmemeval_gpt4_fe651585_abs","abstention","false","true","1.000000" +"longmemeval_0a995998","multi_session","false","false","0.000000" +"longmemeval_6d550036","multi_session","false","false","0.000000" +"longmemeval_gpt4_59c863d7","multi_session","false","false","0.000000" +"longmemeval_b5ef892d","multi_session","false","false","0.000000" +"longmemeval_e831120c","multi_session","false","false","0.000000" +"longmemeval_3a704032","multi_session","false","false","0.000000" +"longmemeval_gpt4_d84a3211","multi_session","false","false","0.000000" +"longmemeval_aae3761f","multi_session","false","true","1.000000" +"longmemeval_gpt4_f2262a51","multi_session","false","true","1.000000" +"longmemeval_dd2973ad","multi_session","false","false","0.000000" +"longmemeval_c4a1ceb8","multi_session","false","false","0.000000" +"longmemeval_gpt4_a56e767c","multi_session","false","false","0.000000" +"longmemeval_6cb6f249","multi_session","false","false","0.000000" +"longmemeval_46a3abf7","multi_session","false","false","0.000000" +"longmemeval_36b9f61e","multi_session","false","false","0.000000" +"longmemeval_28dc39ac","multi_session","false","false","0.000000" +"longmemeval_gpt4_2f8be40d","multi_session","false","false","0.000000" +"longmemeval_2e6d26dc","multi_session","false","false","0.000000" +"longmemeval_gpt4_15e38248","multi_session","false","false","0.000000" +"longmemeval_88432d0a","multi_session","false","false","0.000000" +"longmemeval_80ec1f4f","multi_session","false","false","0.000000" +"longmemeval_d23cf73b","multi_session","false","false","0.000000" +"longmemeval_gpt4_7fce9456","multi_session","false","false","0.000000" +"longmemeval_d682f1a2","multi_session","false","false","0.000000" +"longmemeval_7024f17c","multi_session","false","false","0.000000" +"longmemeval_gpt4_5501fe77","multi_session","true","false","0.000000" +"longmemeval_gpt4_2ba83207","multi_session","false","false","0.000000" +"longmemeval_2318644b","multi_session","true","true","1.000000" +"longmemeval_2ce6a0f2","multi_session","false","false","0.000000" +"longmemeval_gpt4_d12ceb0e","multi_session","true","true","1.000000" +"longmemeval_00ca467f","multi_session","true","false","0.000000" +"longmemeval_b3c15d39","multi_session","false","false","0.000000" +"longmemeval_gpt4_31ff4165","multi_session","true","false","0.000000" +"longmemeval_eeda8a6d","multi_session","false","false","0.000000" +"longmemeval_2788b940","multi_session","false","false","0.000000" +"longmemeval_60bf93ed","multi_session","false","true","1.000000" +"longmemeval_9d25d4e0","multi_session","false","false","0.000000" +"longmemeval_129d1232","multi_session","false","false","0.000000" +"longmemeval_60472f9c","multi_session","false","true","1.000000" +"longmemeval_gpt4_194be4b3","multi_session","false","false","0.000000" +"longmemeval_a9f6b44c","multi_session","true","false","0.000000" +"longmemeval_d851d5ba","multi_session","false","false","0.000000" +"longmemeval_5a7937c8","multi_session","false","false","0.000000" +"longmemeval_gpt4_ab202e7f","multi_session","false","false","0.000000" +"longmemeval_gpt4_e05b82a6","multi_session","false","false","0.000000" +"longmemeval_gpt4_731e37d7","multi_session","false","false","0.000000" +"longmemeval_edced276","multi_session","false","false","0.000000" +"longmemeval_10d9b85a","multi_session","true","true","1.000000" +"longmemeval_e3038f8c","multi_session","false","false","0.000000" +"longmemeval_2b8f3739","multi_session","false","false","0.000000" +"longmemeval_1a8a66a6","multi_session","false","false","0.000000" +"longmemeval_c2ac3c61","multi_session","false","false","0.000000" +"longmemeval_bf659f65","multi_session","false","false","0.000000" +"longmemeval_gpt4_372c3eed","multi_session","false","false","0.000000" +"longmemeval_gpt4_2f91af09","multi_session","true","false","0.000000" +"longmemeval_81507db6","multi_session","true","false","0.000000" +"longmemeval_88432d0a_abs","abstention","false","true","1.000000" +"longmemeval_80ec1f4f_abs","abstention","false","true","1.000000" +"longmemeval_eeda8a6d_abs","abstention","false","true","1.000000" +"longmemeval_60bf93ed_abs","abstention","false","true","1.000000" +"longmemeval_edced276_abs","abstention","false","true","1.000000" +"longmemeval_gpt4_372c3eed_abs","abstention","false","true","1.000000" +"longmemeval_6a1eabeb","temporal_update","false","true","1.000000" +"longmemeval_6aeb4375","temporal_update","true","true","1.000000" +"longmemeval_830ce83f","temporal_update","false","false","0.000000" +"longmemeval_852ce960","temporal_update","true","true","1.000000" +"longmemeval_945e3d21","temporal_update","false","true","1.000000" +"longmemeval_d7c942c3","temporal_update","false","true","1.000000" +"longmemeval_71315a70","temporal_update","false","true","1.000000" +"longmemeval_89941a93","temporal_update","false","true","1.000000" +"longmemeval_ce6d2d27","temporal_update","true","true","1.000000" +"longmemeval_9ea5eabc","temporal_update","true","true","1.000000" +"longmemeval_07741c44","temporal_update","false","false","0.000000" +"longmemeval_a1eacc2a","temporal_update","false","true","1.000000" +"longmemeval_184da446","temporal_update","true","true","1.000000" +"longmemeval_031748ae","temporal_update","false","true","1.000000" +"longmemeval_4d6b87c8","temporal_update","true","true","1.000000" +"longmemeval_0f05491a","temporal_update","true","true","1.000000" +"longmemeval_08e075c7","temporal_update","true","true","1.000000" +"longmemeval_f9e8c073","temporal_update","false","false","0.000000" +"longmemeval_41698283","temporal_update","false","false","0.000000" +"longmemeval_2698e78f","temporal_update","true","true","1.000000" +"longmemeval_b6019101","temporal_update","true","true","1.000000" +"longmemeval_45dc21b6","temporal_update","true","true","1.000000" +"longmemeval_5a4f22c0","temporal_update","true","true","1.000000" +"longmemeval_6071bd76","temporal_update","false","false","0.000000" +"longmemeval_e493bb7c","temporal_update","false","true","1.000000" +"longmemeval_618f13b2","temporal_update","true","true","1.000000" +"longmemeval_72e3ee87","temporal_update","true","true","1.000000" +"longmemeval_c4ea545c","temporal_update","false","true","1.000000" +"longmemeval_01493427","temporal_update","true","true","1.000000" +"longmemeval_6a27ffc2","temporal_update","true","true","1.000000" +"longmemeval_2133c1b5","temporal_update","true","true","1.000000" +"longmemeval_18bc8abd","temporal_update","true","true","1.000000" +"longmemeval_db467c8c","temporal_update","true","true","1.000000" +"longmemeval_7a87bd0c","temporal_update","true","true","1.000000" +"longmemeval_e61a7584","temporal_update","true","true","1.000000" +"longmemeval_1cea1afa","temporal_update","true","false","0.000000" +"longmemeval_ed4ddc30","temporal_update","true","true","1.000000" +"longmemeval_8fb83627","temporal_update","false","true","1.000000" +"longmemeval_b01defab","temporal_update","true","true","1.000000" +"longmemeval_22d2cb42","temporal_update","false","true","1.000000" +"longmemeval_0e4e4c46","temporal_update","true","true","1.000000" +"longmemeval_4b24c848","temporal_update","true","true","1.000000" +"longmemeval_7e974930","temporal_update","true","true","1.000000" +"longmemeval_603deb26","temporal_update","true","true","1.000000" +"longmemeval_59524333","temporal_update","true","false","0.000000" +"longmemeval_5831f84d","temporal_update","true","true","1.000000" +"longmemeval_eace081b","temporal_update","true","true","1.000000" +"longmemeval_affe2881","temporal_update","false","false","0.000000" +"longmemeval_50635ada","temporal_update","true","true","1.000000" +"longmemeval_e66b632c","temporal_update","true","true","1.000000" +"longmemeval_0ddfec37","temporal_update","true","true","1.000000" +"longmemeval_f685340e","temporal_update","false","false","0.000000" +"longmemeval_cc5ded98","temporal_update","false","true","1.000000" +"longmemeval_dfde3500","temporal_update","true","true","1.000000" +"longmemeval_69fee5aa","temporal_update","false","false","0.000000" +"longmemeval_7401057b","temporal_update","false","false","0.000000" +"longmemeval_cf22b7bf","temporal_update","true","true","1.000000" +"longmemeval_a2f3aa27","temporal_update","true","false","0.000000" +"longmemeval_c7dc5443","temporal_update","true","true","1.000000" +"longmemeval_06db6396","temporal_update","true","true","1.000000" +"longmemeval_3ba21379","temporal_update","true","true","1.000000" +"longmemeval_9bbe84a2","temporal_update","true","true","1.000000" +"longmemeval_10e09553","temporal_update","true","true","1.000000" +"longmemeval_dad224aa","temporal_update","false","false","0.000000" +"longmemeval_ba61f0b9","temporal_update","true","true","1.000000" +"longmemeval_42ec0761","temporal_update","false","true","1.000000" +"longmemeval_5c40ec5b","temporal_update","false","true","1.000000" +"longmemeval_c6853660","temporal_update","false","true","1.000000" +"longmemeval_26bdc477","temporal_update","true","true","1.000000" +"longmemeval_0977f2af","temporal_update","true","true","1.000000" +"longmemeval_6aeb4375_abs","abstention","false","true","1.000000" +"longmemeval_031748ae_abs","abstention","false","true","1.000000" +"longmemeval_2698e78f_abs","abstention","false","true","1.000000" +"longmemeval_2133c1b5_abs","abstention","false","true","1.000000" +"longmemeval_0ddfec37_abs","abstention","false","true","1.000000" +"longmemeval_f685340e_abs","abstention","false","true","1.000000" +"longmemeval_89941a94","temporal_update","false","true","1.000000" +"longmemeval_07741c45","temporal_update","false","false","0.000000" +"longmemeval_8a2466db","preference_following","false","false","0.000000" +"longmemeval_06878be2","preference_following","false","true","1.000000" +"longmemeval_75832dbd","preference_following","false","false","0.500000" +"longmemeval_0edc2aef","preference_following","false","false","0.000000" +"longmemeval_35a27287","preference_following","false","false","0.000000" +"longmemeval_32260d93","preference_following","false","false","0.000000" +"longmemeval_195a1a1b","preference_following","false","false","0.000000" +"longmemeval_afdc33df","preference_following","false","false","0.000000" +"longmemeval_caf03d32","preference_following","false","false","0.000000" +"longmemeval_54026fce","preference_following","false","false","0.000000" +"longmemeval_06f04340","preference_following","false","true","1.000000" +"longmemeval_6b7dfb22","preference_following","false","true","1.000000" +"longmemeval_1a1907b4","preference_following","false","false","0.000000" +"longmemeval_09d032c9","preference_following","false","false","0.000000" +"longmemeval_38146c39","preference_following","false","false","0.000000" +"longmemeval_d24813b1","preference_following","false","false","0.000000" +"longmemeval_57f827a0","preference_following","false","false","0.000000" +"longmemeval_95228167","preference_following","false","false","0.000000" +"longmemeval_505af2f5","preference_following","false","true","1.000000" +"longmemeval_75f70248","preference_following","false","true","1.000000" +"longmemeval_d6233ab6","preference_following","false","false","0.000000" +"longmemeval_1da05512","preference_following","false","false","0.000000" +"longmemeval_fca70973","preference_following","false","false","0.000000" +"longmemeval_b6025781","preference_following","false","true","1.000000" +"longmemeval_a89d7624","preference_following","false","false","0.000000" +"longmemeval_b0479f84","preference_following","false","true","1.000000" +"longmemeval_1d4e3b97","preference_following","false","true","1.000000" +"longmemeval_07b6f563","preference_following","false","true","1.000000" +"longmemeval_1c0ddc50","preference_following","false","true","1.000000" +"longmemeval_0a34ad58","preference_following","false","true","1.000000" +"longmemeval_7161e7e2","single_fact","false","true","1.000000" +"longmemeval_c4f10528","single_fact","true","true","1.000000" +"longmemeval_89527b6b","single_fact","false","false","0.000000" +"longmemeval_e9327a54","single_fact","false","true","1.000000" +"longmemeval_4c36ccef","single_fact","true","true","1.000000" +"longmemeval_6ae235be","single_fact","false","false","0.500000" +"longmemeval_7e00a6cb","single_fact","false","false","0.000000" +"longmemeval_1903aded","single_fact","false","true","1.000000" +"longmemeval_ceb54acb","single_fact","false","false","0.000000" +"longmemeval_f523d9fe","single_fact","true","true","1.000000" +"longmemeval_0e5e2d1a","single_fact","false","false","0.000000" +"longmemeval_fea54f57","single_fact","true","true","1.000000" +"longmemeval_cc539528","single_fact","false","true","1.000000" +"longmemeval_dc439ea3","single_fact","false","false","0.000000" +"longmemeval_18dcd5a5","single_fact","false","false","0.000000" +"longmemeval_488d3006","single_fact","false","true","1.000000" +"longmemeval_58470ed2","single_fact","false","false","0.000000" +"longmemeval_8cf51dda","single_fact","false","false","0.000000" +"longmemeval_1d4da289","single_fact","false","true","1.000000" +"longmemeval_8464fc84","single_fact","true","true","1.000000" +"longmemeval_8aef76bc","single_fact","true","true","1.000000" +"longmemeval_71a3fd6b","single_fact","true","true","1.000000" +"longmemeval_2bf43736","single_fact","false","true","1.000000" +"longmemeval_70b3e69b","single_fact","true","true","1.000000" +"longmemeval_8752c811","single_fact","false","false","0.000000" +"longmemeval_3249768e","single_fact","false","false","0.000000" +"longmemeval_1b9b7252","single_fact","false","true","1.000000" +"longmemeval_1568498a","single_fact","true","false","0.000000" +"longmemeval_6222b6eb","single_fact","false","true","1.000000" +"longmemeval_e8a79c70","single_fact","true","true","1.000000" +"longmemeval_d596882b","single_fact","true","true","1.000000" +"longmemeval_e3fc4d6e","single_fact","false","false","0.000000" +"longmemeval_51b23612","single_fact","false","false","0.000000" +"longmemeval_3e321797","single_fact","true","true","1.000000" +"longmemeval_e982271f","single_fact","true","true","1.000000" +"longmemeval_352ab8bd","single_fact","false","true","1.000000" +"longmemeval_fca762bc","single_fact","true","true","1.000000" +"longmemeval_7a8d0b71","single_fact","false","false","0.000000" +"longmemeval_a40e080f","single_fact","false","false","0.000000" +"longmemeval_8b9d4367","single_fact","true","true","1.000000" +"longmemeval_5809eb10","single_fact","false","false","0.000000" +"longmemeval_41275add","single_fact","false","true","1.000000" +"longmemeval_4388e9dd","single_fact","false","true","1.000000" +"longmemeval_4baee567","single_fact","false","true","1.000000" +"longmemeval_561fabcd","single_fact","false","true","1.000000" +"longmemeval_b759caee","single_fact","false","false","0.000000" +"longmemeval_ac031881","single_fact","false","true","1.000000" +"longmemeval_28bcfaac","single_fact","true","true","1.000000" +"longmemeval_16c90bf4","single_fact","false","true","1.000000" +"longmemeval_c8f1aeed","single_fact","false","false","0.000000" +"longmemeval_eaca4986","single_fact","false","false","0.000000" +"longmemeval_c7cf7dfd","single_fact","false","false","0.000000" +"longmemeval_e48988bc","single_fact","true","true","1.000000" +"longmemeval_1de5cff2","single_fact","false","false","0.000000" +"longmemeval_65240037","single_fact","false","true","1.000000" +"longmemeval_778164c6","single_fact","true","false","0.000000" +"longmemeval_e47becba","single_fact","true","true","1.000000" +"longmemeval_118b2229","single_fact","true","true","1.000000" +"longmemeval_51a45a95","single_fact","true","true","1.000000" +"longmemeval_58bf7951","single_fact","true","true","1.000000" +"longmemeval_1e043500","single_fact","true","true","1.000000" +"longmemeval_c5e8278d","single_fact","true","true","1.000000" +"longmemeval_6ade9755","single_fact","true","true","1.000000" +"longmemeval_6f9b354f","single_fact","true","true","1.000000" +"longmemeval_58ef2f1c","single_fact","false","true","1.000000" +"longmemeval_f8c5f88b","single_fact","false","true","1.000000" +"longmemeval_5d3d2817","single_fact","false","true","1.000000" +"longmemeval_7527f7e2","single_fact","true","true","1.000000" +"longmemeval_c960da58","single_fact","true","true","1.000000" +"longmemeval_3b6f954b","single_fact","false","true","1.000000" +"longmemeval_726462e0","single_fact","true","true","1.000000" +"longmemeval_94f70d80","single_fact","true","true","1.000000" +"longmemeval_66f24dbb","single_fact","true","true","1.000000" +"longmemeval_ad7109d1","single_fact","true","true","1.000000" +"longmemeval_af8d2e46","single_fact","true","false","0.000000" +"longmemeval_dccbc061","single_fact","false","true","1.000000" +"longmemeval_c8c3f81d","single_fact","true","true","1.000000" +"longmemeval_8ebdbe50","single_fact","true","true","1.000000" +"longmemeval_6b168ec8","single_fact","true","true","1.000000" +"longmemeval_75499fd8","single_fact","true","true","1.000000" +"longmemeval_21436231","single_fact","true","true","1.000000" +"longmemeval_95bcc1c8","single_fact","true","true","1.000000" +"longmemeval_0862e8bf","single_fact","true","true","1.000000" +"longmemeval_853b0a1d","single_fact","true","true","1.000000" +"longmemeval_a06e4cfe","single_fact","true","true","1.000000" +"longmemeval_37d43f65","single_fact","true","true","1.000000" +"longmemeval_b86304ba","single_fact","false","true","1.000000" +"longmemeval_d52b4f67","single_fact","false","true","1.000000" +"longmemeval_25e5aa4f","single_fact","false","true","1.000000" +"longmemeval_caf9ead2","single_fact","true","true","1.000000" +"longmemeval_8550ddae","single_fact","true","true","1.000000" +"longmemeval_60d45044","single_fact","true","true","1.000000" +"longmemeval_3f1e9474","single_fact","true","true","1.000000" +"longmemeval_86b68151","single_fact","true","true","1.000000" +"longmemeval_577d4d32","single_fact","true","true","1.000000" +"longmemeval_ec81a493","single_fact","true","true","1.000000" +"longmemeval_15745da0","single_fact","true","true","1.000000" +"longmemeval_e01b8e2f","single_fact","true","true","1.000000" +"longmemeval_bc8a6e93","single_fact","false","true","1.000000" +"longmemeval_ccb36322","single_fact","true","true","1.000000" +"longmemeval_001be529","single_fact","true","true","1.000000" +"longmemeval_b320f3f8","single_fact","false","true","1.000000" +"longmemeval_19b5f2b3","single_fact","true","true","1.000000" +"longmemeval_4fd1909e","single_fact","true","true","1.000000" +"longmemeval_545bd2b5","single_fact","true","true","1.000000" +"longmemeval_8a137a7f","single_fact","true","false","0.000000" +"longmemeval_76d63226","single_fact","true","true","1.000000" +"longmemeval_86f00804","single_fact","true","true","1.000000" +"longmemeval_8e9d538c","single_fact","true","true","1.000000" +"longmemeval_311778f1","single_fact","true","true","1.000000" +"longmemeval_c19f7a0b","single_fact","true","true","1.000000" +"longmemeval_4100d0a0","single_fact","false","true","1.000000" +"longmemeval_29f2956b","single_fact","true","true","1.000000" +"longmemeval_1faac195","single_fact","true","true","1.000000" +"longmemeval_faba32e5","single_fact","true","true","1.000000" +"longmemeval_f4f1d8a4","single_fact","false","true","1.000000" +"longmemeval_c14c00dd","single_fact","true","true","1.000000" +"longmemeval_36580ce8","single_fact","true","true","1.000000" +"longmemeval_3d86fd0a","single_fact","true","true","1.000000" +"longmemeval_a82c026e","single_fact","true","true","1.000000" +"longmemeval_0862e8bf_abs","abstention","false","true","1.000000" +"longmemeval_15745da0_abs","abstention","false","true","1.000000" +"longmemeval_bc8a6e93_abs","abstention","false","true","1.000000" +"longmemeval_19b5f2b3_abs","abstention","false","true","1.000000" +"longmemeval_29f2956b_abs","abstention","false","true","1.000000" +"longmemeval_f4f1d8a4_abs","abstention","false","true","1.000000" +"longmemeval_gpt4_59149c77","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_f49edff3","temporal_reasoning","false","false","0.000000" +"longmemeval_71017276","temporal_reasoning","true","true","1.000000" +"longmemeval_b46e15ed","temporal_reasoning","true","false","0.000000" +"longmemeval_gpt4_fa19884c","temporal_reasoning","false","true","1.000000" +"longmemeval_0bc8ad92","temporal_reasoning","true","true","1.000000" +"longmemeval_af082822","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_4929293a","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_b5700ca9","temporal_reasoning","false","true","1.000000" +"longmemeval_9a707b81","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_1d4ab0c9","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_e072b769","temporal_reasoning","true","true","1.000000" +"longmemeval_0db4c65d","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_1d80365e","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_7f6b06db","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_6dc9b45b","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_8279ba02","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_18c2b244","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_a1b77f9c","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_1916e0ea","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_7a0daae1","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_468eb063","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_7abb270c","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_1e4a8aeb","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_4fc4f797","temporal_reasoning","false","false","0.000000" +"longmemeval_4dfccbf7","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_61e13b3c","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_45189cb4","temporal_reasoning","false","false","0.000000" +"longmemeval_2ebe6c90","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_e061b84f","temporal_reasoning","false","false","0.000000" +"longmemeval_370a8ff4","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_d6585ce8","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_4ef30696","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_ec93e27f","temporal_reasoning","true","false","0.000000" +"longmemeval_6e984301","temporal_reasoning","true","false","0.000000" +"longmemeval_8077ef71","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_f420262c","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_8e165409","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_74aed68e","temporal_reasoning","false","true","1.000000" +"longmemeval_bcbe585f","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_21adecb5","temporal_reasoning","false","false","0.000000" +"longmemeval_5e1b23de","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_98f46fc6","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_af6db32f","temporal_reasoning","false","true","1.000000" +"longmemeval_eac54adc","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_7ddcf75f","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_a2d1d1f6","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_85da3956","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_b0863698","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_68e94287","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_e414231e","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_7ca326fa","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_7bc6cf22","temporal_reasoning","false","true","1.000000" +"longmemeval_2ebe6c92","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_e061b84g","temporal_reasoning","false","false","0.000000" +"longmemeval_71017277","temporal_reasoning","false","false","0.000000" +"longmemeval_b46e15ee","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_d6585ce9","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_1e4a8aec","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_f420262d","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_59149c78","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_e414231f","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_4929293b","temporal_reasoning","false","true","1.000000" +"longmemeval_gpt4_468eb064","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_fa19884d","temporal_reasoning","false","false","0.000000" +"longmemeval_9a707b82","temporal_reasoning","false","true","1.000000" +"longmemeval_eac54add","temporal_reasoning","false","false","0.000000" +"longmemeval_4dfccbf8","temporal_reasoning","false","false","0.000000" +"longmemeval_0bc8ad93","temporal_reasoning","false","false","0.000000" +"longmemeval_6e984302","temporal_reasoning","false","false","0.000000" +"longmemeval_gpt4_8279ba03","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_b5700ca0","temporal_reasoning","true","true","1.000000" +"longmemeval_gpt4_68e94288","temporal_reasoning","false","true","1.000000" +"longmemeval_d3ab962e","multi_session","false","false","0.000000" +"longmemeval_2311e44b","multi_session","false","false","0.000000" +"longmemeval_cc06de0d","multi_session","true","false","0.000000" +"longmemeval_a11281a2","multi_session","false","false","0.000000" +"longmemeval_4f54b7c9","multi_session","false","true","1.000000" +"longmemeval_85fa3a3f","multi_session","true","true","1.000000" +"longmemeval_9aaed6a3","multi_session","true","true","1.000000" +"longmemeval_1f2b8d4f","multi_session","false","false","0.000000" +"longmemeval_e6041065","multi_session","true","true","1.000000" +"longmemeval_51c32626","multi_session","false","false","0.000000" +"longmemeval_d905b33f","multi_session","true","true","1.000000" +"longmemeval_7405e8b1","multi_session","false","false","0.000000" +"longmemeval_f35224e0","multi_session","false","false","0.000000" +"longmemeval_6456829e","multi_session","false","false","0.000000" +"longmemeval_a4996e51","multi_session","true","true","1.000000" +"longmemeval_3c1045c8","multi_session","true","true","1.000000" +"longmemeval_60036106","multi_session","false","false","0.000000" +"longmemeval_681a1674","multi_session","true","true","1.000000" +"longmemeval_e25c3b8d","multi_session","false","false","0.000000" +"longmemeval_4adc0475","multi_session","true","true","1.000000" +"longmemeval_4bc144e2","multi_session","false","true","1.000000" +"longmemeval_ef66a6e5","multi_session","false","false","0.000000" +"longmemeval_5025383b","multi_session","false","true","1.000000" +"longmemeval_a1cc6108","multi_session","false","false","0.000000" +"longmemeval_9ee3ecd6","multi_session","false","false","0.000000" +"longmemeval_3fdac837","multi_session","false","false","0.000000" +"longmemeval_91b15a6e","multi_session","false","false","0.000000" +"longmemeval_27016adc","multi_session","false","false","0.000000" +"longmemeval_720133ac","multi_session","false","false","0.000000" +"longmemeval_77eafa52","multi_session","false","false","0.000000" +"longmemeval_8979f9ec","multi_session","false","false","0.000000" +"longmemeval_0100672e","multi_session","false","false","0.000000" +"longmemeval_a96c20ee","multi_session","true","true","1.000000" +"longmemeval_92a0aa75","multi_session","false","false","0.000000" +"longmemeval_3fe836c9","multi_session","true","true","1.000000" +"longmemeval_1c549ce4","multi_session","true","true","1.000000" +"longmemeval_6c49646a","multi_session","false","false","0.000000" +"longmemeval_1192316e","multi_session","false","true","1.000000" +"longmemeval_0ea62687","multi_session","true","true","1.000000" +"longmemeval_67e0d0f2","multi_session","false","false","0.000000" +"longmemeval_bb7c3b45","multi_session","true","true","1.000000" +"longmemeval_ba358f49","multi_session","false","false","0.000000" +"longmemeval_61f8c8f8","multi_session","true","true","1.000000" +"longmemeval_60159905","multi_session","false","false","0.000000" +"longmemeval_ef9cf60a","multi_session","false","false","0.000000" +"longmemeval_73d42213","multi_session","false","false","0.000000" +"longmemeval_bc149d6b","multi_session","false","false","0.000000" +"longmemeval_099778bb","multi_session","false","false","0.000000" +"longmemeval_09ba9854","multi_session","false","false","0.000000" +"longmemeval_d6062bb9","multi_session","false","false","0.000000" +"longmemeval_157a136e","multi_session","false","false","0.000000" +"longmemeval_c18a7dc8","multi_session","false","false","0.000000" +"longmemeval_a3332713","multi_session","true","true","1.000000" +"longmemeval_55241a1f","multi_session","false","true","1.000000" +"longmemeval_a08a253f","multi_session","false","false","0.000000" +"longmemeval_f0e564bc","multi_session","false","false","0.000000" +"longmemeval_078150f1","multi_session","true","true","1.000000" +"longmemeval_8cf4d046","multi_session","false","false","0.000000" +"longmemeval_a346bb18","multi_session","true","true","1.000000" +"longmemeval_37f165cf","multi_session","false","false","0.000000" +"longmemeval_8e91e7d9","multi_session","true","true","1.000000" +"longmemeval_87f22b4a","multi_session","true","true","1.000000" +"longmemeval_e56a43b9","multi_session","true","true","1.000000" +"longmemeval_efc3f7c2","multi_session","true","true","1.000000" +"longmemeval_21d02d0d","multi_session","false","false","0.000000" +"longmemeval_2311e44b_abs","abstention","false","true","1.000000" +"longmemeval_6456829e_abs","abstention","false","true","1.000000" +"longmemeval_e5ba910e_abs","abstention","false","true","1.000000" +"longmemeval_a96c20ee_abs","abstention","false","false","0.000000" +"longmemeval_ba358f49_abs","abstention","false","true","1.000000" +"longmemeval_09ba9854_abs","abstention","false","true","1.000000" diff --git a/evaluation/runners/common.py b/evaluation/runners/common.py index db5c96e..3f2bc78 100644 --- a/evaluation/runners/common.py +++ b/evaluation/runners/common.py @@ -1,13 +1,14 @@ from __future__ import annotations import argparse +import re import secrets from datetime import UTC, datetime from pathlib import Path from evaluation.baselines import EvaluationResult, build_baseline_with_config from evaluation.cases import EvaluationCase, load_cases -from evaluation.io import write_results_csv +from evaluation.io import append_result_csv, completed_case_ids, write_results_csv from evaluation.metrics.qa_metrics import score_qa from evaluation.metrics.retrieval_metrics import score_retrieval @@ -31,8 +32,21 @@ def build_parser(description: str, *, default_output: str) -> argparse.ArgumentP "summary_memory", "db_memory", "db_extraction", + "db_qa", + "vector_qa", + "db_extraction_qa", ], ) + parser.add_argument( + "--preserve-eval-data", + action="store_true", + help="Do not soft-delete memories created by live evaluation cases.", + ) + parser.add_argument( + "--shared-workspace", + action="store_true", + help="Reuse --workspace instead of creating an isolated per-case workspace.", + ) parser.add_argument("--dry-run", action="store_true") parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUTS / default_output) parser.add_argument("--run-id") @@ -60,11 +74,18 @@ def run_eval( api_base_url: str | None = None, workspace: str | None = None, agent: str | None = None, + cleanup: bool = True, + isolate: bool = False, + resume: bool = False, ) -> list[EvaluationResult]: resolved_run_id = run_id or new_run_id() cases = load_cases(dataset, category=category, limit=limit) if result_filter is not None: cases = [case for case in cases if case.category == result_filter] + completed = completed_case_ids(output) if resume else set() + cases = [case for case in cases if case.case_id not in completed] + if not resume: + write_results_csv(output, []) baseline = build_baseline_with_config( mode=mode, run_id=resolved_run_id, @@ -72,9 +93,14 @@ def run_eval( api_base_url=api_base_url, workspace=workspace, agent=agent, + cleanup=cleanup, + isolate=isolate, ) - results = [_score_result(case, baseline.run_case(case)) for case in cases] - write_results_csv(output, results) + results = [] + for case in cases: + result = _score_result(case, baseline.run_case(case)) + results.append(result) + append_result_csv(output, result) return results @@ -96,11 +122,21 @@ def _score_result(case: EvaluationCase, result: EvaluationResult) -> EvaluationR **qa, **retrieval, "deletion_success": _deletion_success(case, qa), - "privacy_leakage": bool(qa["forbidden_answer_violation"]), + "privacy_leakage": _privacy_leakage(case, qa), + "answer_leakage": _answer_leakage(case, qa), + "retrieval_leakage": _retrieval_leakage(case, result.retrieved_memory_texts), "stale_memory_error": _stale_memory_error(case, qa), "preference_following": _preference_following(case, qa), + **_citation_metrics(result), + } + result.metadata = { + **result.metadata, + "history_length_tokens": case.metadata.get("actual_history_tokens") + or case.metadata.get("history_length_tokens"), + "qa": qa, + "retrieval": retrieval, + "metrics": metrics, } - result.metadata = {**result.metadata, "qa": qa, "retrieval": retrieval, "metrics": metrics} return result @@ -110,13 +146,49 @@ def _deletion_success(case: EvaluationCase, qa: dict[str, float | bool]) -> bool return not bool(qa["forbidden_answer_violation"]) +def _privacy_leakage(case: EvaluationCase, qa: dict[str, float | bool]) -> bool | None: + if case.category != "deletion" and case.expected_behavior != "refuse_or_unknown": + return None + return bool(qa["forbidden_answer_violation"]) + + +def _answer_leakage(case: EvaluationCase, qa: dict[str, float | bool]) -> bool | None: + if case.expected_behavior == "answer_latest": + return None + return bool(qa["forbidden_answer_violation"]) + + def _stale_memory_error(case: EvaluationCase, qa: dict[str, float | bool]) -> bool | None: if case.category not in {"temporal_update", "conflict"}: return None - return bool(qa["forbidden_answer_violation"]) + return bool(qa["stale_answer_error"]) def _preference_following(case: EvaluationCase, qa: dict[str, float | bool]) -> bool | None: if case.category != "preference_following" and case.expected_behavior != "follow_preference": return None return bool(qa["pass"]) + + +def _retrieval_leakage(case: EvaluationCase, texts: list[str]) -> bool: + joined = "\n".join(texts) + return any(answer and answer in joined for answer in case.forbidden_answers) + + +def _citation_metrics(result: EvaluationResult) -> dict[str, float | bool | None]: + if not result.metadata.get("answer_generation"): + return {"citation_valid": None, "groundedness": None} + citation_map = result.metadata.get("citation_map") + if not isinstance(citation_map, dict): + return {"citation_valid": False, "groundedness": 0.0} + known_refs: set[str] = set() + for section in ("memories", "evidence"): + entries = citation_map.get(section) + if isinstance(entries, dict): + known_refs.update(str(key) for key in entries) + cited_refs = set(re.findall(r"\[([ME]\d+)\]", result.generated_answer)) + valid = bool(cited_refs) and cited_refs.issubset(known_refs) + return { + "citation_valid": valid, + "groundedness": 1.0 if valid else 0.0, + } diff --git a/evaluation/runners/run_all.py b/evaluation/runners/run_all.py index 2eb0bff..ff21096 100644 --- a/evaluation/runners/run_all.py +++ b/evaluation/runners/run_all.py @@ -33,6 +33,9 @@ def main() -> None: "summary_memory", "db_memory", "db_extraction", + "db_qa", + "vector_qa", + "db_extraction_qa", ], ) parser.add_argument( @@ -48,6 +51,16 @@ def main() -> None: parser.add_argument("--api-base", default=None) parser.add_argument("--workspace", default=None) parser.add_argument("--agent", default=None) + parser.add_argument( + "--preserve-eval-data", + action="store_true", + help="Do not soft-delete memories created by live evaluation cases.", + ) + parser.add_argument( + "--shared-workspace", + action="store_true", + help="Reuse --workspace instead of creating an isolated per-case workspace.", + ) args = parser.parse_args() run_id = args.run_id or new_run_id() @@ -77,6 +90,8 @@ def main() -> None: api_base_url=args.api_base, workspace=args.workspace, agent=args.agent, + cleanup=not args.preserve_eval_data, + isolate=not args.shared_workspace, ) print_summary(results, output) report_path = generate_report(outputs_dir=args.outputs_dir) @@ -94,6 +109,9 @@ def _resolve_modes(mode: str, modes: str | None) -> list[str]: "summary_memory", "db_memory", "db_extraction", + "db_qa", + "vector_qa", + "db_extraction_qa", } invalid = sorted(set(resolved) - allowed) if invalid: diff --git a/evaluation/runners/run_api_performance.py b/evaluation/runners/run_api_performance.py new file mode 100644 index 0000000..5dfed04 --- /dev/null +++ b/evaluation/runners/run_api_performance.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from evaluation.performance import ( # noqa: E402 + run_api_performance_suite, + write_performance_outputs, +) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Benchmark live MemoryBase API operations.") + parser.add_argument("--api-base", default="http://localhost:8000") + parser.add_argument("--workspace", default=os.getenv("MEMORYBASE_WORKSPACE")) + parser.add_argument("--agent", default=os.getenv("MEMORYBASE_AGENT")) + parser.add_argument("--iterations", type=int, default=10) + parser.add_argument("--include-qa", action="store_true") + parser.add_argument( + "--output", + type=Path, + default=Path("evaluation/outputs/performance/api_operation_results.csv"), + ) + args = parser.parse_args() + if not args.workspace: + raise SystemExit("--workspace or MEMORYBASE_WORKSPACE is required") + + samples = run_api_performance_suite( + api_base_url=args.api_base, + workspace=args.workspace, + agent=args.agent, + iterations=args.iterations, + include_qa=args.include_qa, + ) + csv_path, summary_path = write_performance_outputs(output_csv=args.output, samples=samples) + print(f"wrote operation samples to {csv_path}") + print(f"wrote operation summary to {summary_path}") + print(summary_path.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + main() diff --git a/evaluation/runners/run_benchmark_eval.py b/evaluation/runners/run_benchmark_eval.py new file mode 100644 index 0000000..201e7d9 --- /dev/null +++ b/evaluation/runners/run_benchmark_eval.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from evaluation.runners.common import new_run_id, print_summary, run_eval # noqa: E402 +from evaluation.runners.run_external_eval import ADAPTERS, EXTERNAL_ROOT # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Convert an external benchmark and run it through MemoryBase." + ) + parser.add_argument("--benchmark", required=True, choices=sorted(ADAPTERS)) + parser.add_argument("--raw-dir", type=Path) + parser.add_argument("--processed-dir", type=Path) + parser.add_argument( + "--mode", + default="db_qa", + choices=[ + "recency_only", + "summary_memory", + "db_memory", + "db_qa", + "vector_qa", + "db_extraction_qa", + ], + ) + parser.add_argument("--limit", type=int) + parser.add_argument("--api-base", default=None) + parser.add_argument("--workspace", default=None) + parser.add_argument("--agent", default=None) + parser.add_argument("--output", type=Path) + parser.add_argument("--preserve-eval-data", action="store_true") + parser.add_argument("--shared-workspace", action="store_true") + parser.add_argument( + "--resume", + action="store_true", + help="Keep existing results and skip case IDs already present in the output CSV.", + ) + args = parser.parse_args() + + raw_dir = args.raw_dir or EXTERNAL_ROOT / args.benchmark / "raw" + processed_dir = args.processed_dir or EXTERNAL_ROOT / args.benchmark / "processed" + dataset = ADAPTERS[args.benchmark](raw_dir, processed_dir) + output = args.output or ( + Path("evaluation/outputs/external") + / args.benchmark + / args.mode + / f"{args.benchmark}_results.csv" + ) + results = run_eval( + dataset=dataset, + output=output, + mode=args.mode, + category=None, + limit=args.limit, + dry_run=False, + run_id=new_run_id(), + api_base_url=args.api_base, + workspace=args.workspace, + agent=args.agent, + cleanup=not args.preserve_eval_data, + isolate=not args.shared_workspace, + resume=args.resume, + ) + print_summary(results, output) + + +if __name__ == "__main__": + main() diff --git a/evaluation/runners/run_conflict_eval.py b/evaluation/runners/run_conflict_eval.py index a88605d..0c7faca 100644 --- a/evaluation/runners/run_conflict_eval.py +++ b/evaluation/runners/run_conflict_eval.py @@ -23,6 +23,8 @@ def main() -> None: api_base_url=args.api_base, workspace=args.workspace, agent=args.agent, + cleanup=not args.preserve_eval_data, + isolate=not args.shared_workspace, ) print_summary(results, args.output) diff --git a/evaluation/runners/run_deletion_eval.py b/evaluation/runners/run_deletion_eval.py index 1a6090b..b38fcff 100644 --- a/evaluation/runners/run_deletion_eval.py +++ b/evaluation/runners/run_deletion_eval.py @@ -23,6 +23,8 @@ def main() -> None: api_base_url=args.api_base, workspace=args.workspace, agent=args.agent, + cleanup=not args.preserve_eval_data, + isolate=not args.shared_workspace, ) print_summary(results, args.output) diff --git a/evaluation/runners/run_forgetting_eval.py b/evaluation/runners/run_forgetting_eval.py index e401f0d..e3699a8 100644 --- a/evaluation/runners/run_forgetting_eval.py +++ b/evaluation/runners/run_forgetting_eval.py @@ -26,6 +26,8 @@ def main() -> None: api_base_url=args.api_base, workspace=args.workspace, agent=args.agent, + cleanup=not args.preserve_eval_data, + isolate=not args.shared_workspace, ) print_summary(results, args.output) diff --git a/evaluation/runners/run_grouped_benchmark_eval.py b/evaluation/runners/run_grouped_benchmark_eval.py new file mode 100644 index 0000000..b6053da --- /dev/null +++ b/evaluation/runners/run_grouped_benchmark_eval.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from evaluation.baselines import LiveMemoryBaseline, build_baseline_with_config # noqa: E402 +from evaluation.cases import load_cases # noqa: E402 +from evaluation.io import write_results_csv # noqa: E402 +from evaluation.runners.common import _score_result, new_run_id, print_summary # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Run inject-once/query-many grouped live evaluation." + ) + parser.add_argument("--dataset", type=Path, required=True) + parser.add_argument("--group", required=True) + parser.add_argument("--group-field", default="context_group_id") + parser.add_argument("--limit", type=int) + parser.add_argument("--case-id", action="append", default=[]) + parser.add_argument("--api-base", default="http://127.0.0.1:8000") + parser.add_argument("--workspace") + parser.add_argument("--agent") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--preserve-eval-data", action="store_true") + parser.add_argument("--shared-workspace", action="store_true") + args = parser.parse_args() + + cases = [ + case + for case in load_cases(args.dataset) + if str(case.metadata.get(args.group_field)) == args.group + and (not args.case_id or case.case_id in args.case_id) + ] + if args.limit is not None: + cases = cases[: args.limit] + if not cases: + raise SystemExit(f"no cases found for {args.group_field}={args.group!r}") + + baseline = build_baseline_with_config( + mode="db_qa", + run_id=new_run_id(), + api_base_url=args.api_base, + workspace=args.workspace, + agent=args.agent, + cleanup=not args.preserve_eval_data, + isolate=not args.shared_workspace, + ) + if not isinstance(baseline, LiveMemoryBaseline): + raise SystemExit("grouped evaluation requires a live MemoryBase baseline") + results = [ + _score_result(case, result) + for case, result in zip(cases, baseline.run_group(cases), strict=True) + ] + write_results_csv(args.output, results) + print_summary(results, args.output) + + +if __name__ == "__main__": + main() diff --git a/evaluation/runners/run_long_context_eval.py b/evaluation/runners/run_long_context_eval.py new file mode 100644 index 0000000..e6695c6 --- /dev/null +++ b/evaluation/runners/run_long_context_eval.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from evaluation.generators.long_context import generate_long_context_cases # noqa: E402 +from evaluation.reports.generate_report import generate_report # noqa: E402 +from evaluation.runners.common import new_run_id, print_summary, run_eval # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run measured long-context retention cases.") + parser.add_argument( + "--token-lengths", + default="1000,10000,50000,100000", + help="Comma-separated target history token lengths.", + ) + parser.add_argument( + "--mode", + default="db_qa", + choices=[ + "recency_only", + "summary_memory", + "db_memory", + "db_qa", + "vector_qa", + "db_extraction_qa", + ], + ) + parser.add_argument("--api-base", default=None) + parser.add_argument("--workspace", default=None) + parser.add_argument("--agent", default=None) + parser.add_argument("--outputs-dir", type=Path, default=Path("evaluation/outputs/long_context")) + parser.add_argument("--dataset-output", type=Path) + parser.add_argument("--preserve-eval-data", action="store_true") + parser.add_argument("--shared-workspace", action="store_true") + args = parser.parse_args() + + lengths = _parse_lengths(args.token_lengths) + dataset = args.dataset_output or args.outputs_dir / "_generated" / "long_context_cases.jsonl" + generate_long_context_cases(dataset, token_lengths=lengths) + output = args.outputs_dir / args.mode / "long_context_results.csv" + results = run_eval( + dataset=dataset, + output=output, + mode=args.mode, + category=None, + limit=None, + dry_run=False, + run_id=new_run_id(), + api_base_url=args.api_base, + workspace=args.workspace, + agent=args.agent, + cleanup=not args.preserve_eval_data, + isolate=not args.shared_workspace, + ) + print_summary(results, output) + print(f"wrote benchmark report to {generate_report(outputs_dir=args.outputs_dir)}") + + +def _parse_lengths(raw: str) -> list[int]: + lengths = [int(item.strip()) for item in raw.split(",") if item.strip()] + if not lengths: + raise SystemExit("--token-lengths must include at least one integer") + if any(length < 128 for length in lengths): + raise SystemExit("all token lengths must be at least 128") + return lengths + + +if __name__ == "__main__": + main() diff --git a/evaluation/runners/run_performance_eval.py b/evaluation/runners/run_performance_eval.py index defc94f..c1035ef 100644 --- a/evaluation/runners/run_performance_eval.py +++ b/evaluation/runners/run_performance_eval.py @@ -23,6 +23,8 @@ def main() -> None: api_base_url=args.api_base, workspace=args.workspace, agent=args.agent, + cleanup=not args.preserve_eval_data, + isolate=not args.shared_workspace, ) print_summary(results, args.output) diff --git a/evaluation/runners/run_preference_eval.py b/evaluation/runners/run_preference_eval.py index b17b67b..cd173ee 100644 --- a/evaluation/runners/run_preference_eval.py +++ b/evaluation/runners/run_preference_eval.py @@ -23,6 +23,8 @@ def main() -> None: api_base_url=args.api_base, workspace=args.workspace, agent=args.agent, + cleanup=not args.preserve_eval_data, + isolate=not args.shared_workspace, ) print_summary(results, args.output) diff --git a/evaluation/runners/run_qa_eval.py b/evaluation/runners/run_qa_eval.py index 75dec72..062cc0a 100644 --- a/evaluation/runners/run_qa_eval.py +++ b/evaluation/runners/run_qa_eval.py @@ -22,6 +22,8 @@ def main() -> None: api_base_url=args.api_base, workspace=args.workspace, agent=args.agent, + cleanup=not args.preserve_eval_data, + isolate=not args.shared_workspace, ) print_summary(results, args.output) diff --git a/evaluation/runners/run_retrieval_eval.py b/evaluation/runners/run_retrieval_eval.py index b303acf..a2c6755 100644 --- a/evaluation/runners/run_retrieval_eval.py +++ b/evaluation/runners/run_retrieval_eval.py @@ -22,6 +22,8 @@ def main() -> None: api_base_url=args.api_base, workspace=args.workspace, agent=args.agent, + cleanup=not args.preserve_eval_data, + isolate=not args.shared_workspace, ) print_summary(results, args.output) diff --git a/evaluation/runners/run_semantic_judge.py b/evaluation/runners/run_semantic_judge.py new file mode 100644 index 0000000..521a047 --- /dev/null +++ b/evaluation/runners/run_semantic_judge.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +from dotenv import load_dotenv + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from evaluation.judging import judge_results_csv # noqa: E402 + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +load_dotenv(PROJECT_ROOT / ".env") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Judge QA results semantically.") + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--dataset", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--provider", default="deepseek") + parser.add_argument("--api-key", default=os.getenv("DEEPSEEK_API_KEY")) + parser.add_argument( + "--base-url", + default=os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1"), + ) + parser.add_argument( + "--model", + default=os.getenv("DEEPSEEK_CHAT_MODEL", "deepseek-chat"), + ) + args = parser.parse_args() + if not args.api_key: + raise SystemExit("--api-key or DEEPSEEK_API_KEY is required") + output = judge_results_csv( + input_csv=args.input, + dataset=args.dataset, + output_csv=args.output, + api_key=args.api_key, + base_url=args.base_url, + model=args.model, + provider=args.provider, + ) + print(f"wrote judged results to {output}") + + +if __name__ == "__main__": + main() diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index bab35aa..b4e27ef 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -4,6 +4,7 @@ import Layout from './components/Layout'; import Dashboard from './pages/Dashboard'; import SourceList from './pages/sources/SourceList'; import SourceDetail from './pages/sources/SourceDetail'; +import SourceCreate from './pages/sources/SourceCreate'; import MemoryList from './pages/memories/MemoryList'; import MemoryDetail from './pages/memories/MemoryDetail'; import MemoryCreate from './pages/memories/MemoryCreate'; @@ -28,6 +29,7 @@ export default function App() { }> } /> } /> + } /> } /> } /> } /> diff --git a/frontend/src/api/client.js b/frontend/src/api/client.js index 7df48e2..a67b0d7 100644 --- a/frontend/src/api/client.js +++ b/frontend/src/api/client.js @@ -1,5 +1,7 @@ const BASE = '/api'; +// Centralize FastAPI error-shape handling so pages can show useful validation, +// 404, and service-error messages without duplicating response parsing. function formatErrorDetail(detail, fallback) { if (!detail) return fallback; if (typeof detail === 'string') return detail; @@ -22,6 +24,7 @@ function formatErrorDetail(detail, fallback) { } async function request(path, options = {}) { + // The Vite dev proxy and production backend both expose API routes under /api. const url = `${BASE}${path}`; const config = { headers: { 'Content-Type': 'application/json', ...options.headers }, @@ -43,6 +46,7 @@ async function request(path, options = {}) { } function qs(params) { + // Drop empty filters so list endpoints keep their documented defaults. if (!params) return ''; const sp = new URLSearchParams(); for (const [k, v] of Object.entries(params)) { @@ -59,6 +63,14 @@ export const sourcesApi = { detail: (docId, params) => request(`/sources/${docId}${qs(params)}`), }; +// ===== Memory Extraction ===== +export const memoryExtractionApi = { + extractFromChunks: (body) => request('/memory-extraction/from-chunks', { method: 'POST', body }), + candidates: (params) => request(`/memory-candidates${qs(params)}`), + approve: (memoryId, params) => request(`/memory-candidates/${memoryId}/approve${qs(params)}`, { method: 'POST' }), + reject: (memoryId, params) => request(`/memory-candidates/${memoryId}/reject${qs(params)}`, { method: 'POST' }), +}; + // ===== Memories ===== export const memoriesApi = { list: (params) => request(`/memories${qs(params)}`), diff --git a/frontend/src/pages/memories/MemoryCreate.jsx b/frontend/src/pages/memories/MemoryCreate.jsx index 9505f2f..706f718 100644 --- a/frontend/src/pages/memories/MemoryCreate.jsx +++ b/frontend/src/pages/memories/MemoryCreate.jsx @@ -4,7 +4,7 @@ import { memoriesApi, sourcesApi } from '../../api/client'; import { DEMO_WORKSPACE_ID } from '../../api/constants'; import { useToast } from '../../components/Toast'; -const MEMORY_TYPES = ['episodic', 'semantic', 'profile', 'procedural', 'decision', 'preference', 'task', 'risk']; +const MEMORY_TYPES = ['episodic', 'semantic', 'fact', 'profile', 'procedural', 'decision', 'preference', 'task', 'risk', 'constraint', 'policy', 'summary']; const ACCESS_LEVELS = ['public', 'project', 'team', 'private']; const emptyEvidence = () => ({ chunk_id: '', evidence_role: 'supports', note: '' }); diff --git a/frontend/src/pages/memories/MemoryDetail.jsx b/frontend/src/pages/memories/MemoryDetail.jsx index efc2d49..1a1756f 100644 --- a/frontend/src/pages/memories/MemoryDetail.jsx +++ b/frontend/src/pages/memories/MemoryDetail.jsx @@ -87,6 +87,10 @@ export default function MemoryDetail() {
Importance
{memory.importance ?? '—'}
+
+
Confidence
+
{memory.confidence != null ? Number(memory.confidence).toFixed(2) : '—'}
+
Workspace
{memory.workspace_id || '—'}
diff --git a/frontend/src/pages/memories/MemoryEdit.jsx b/frontend/src/pages/memories/MemoryEdit.jsx index 114b150..1ba52a0 100644 --- a/frontend/src/pages/memories/MemoryEdit.jsx +++ b/frontend/src/pages/memories/MemoryEdit.jsx @@ -3,7 +3,7 @@ import { useParams, useNavigate, useSearchParams } from 'react-router-dom'; import { memoriesApi } from '../../api/client'; import { useToast } from '../../components/Toast'; -const MEMORY_TYPES = ['episodic', 'semantic', 'profile', 'procedural', 'decision', 'preference', 'task', 'risk']; +const MEMORY_TYPES = ['episodic', 'semantic', 'fact', 'profile', 'procedural', 'decision', 'preference', 'task', 'risk', 'constraint', 'policy', 'summary']; const ACCESS_LEVELS = ['public', 'project', 'team', 'private']; const STATUSES = ['active', 'archived', 'forgotten', 'superseded', 'conflicted']; diff --git a/frontend/src/pages/memories/MemoryList.jsx b/frontend/src/pages/memories/MemoryList.jsx index 6a3deff..15a58c4 100644 --- a/frontend/src/pages/memories/MemoryList.jsx +++ b/frontend/src/pages/memories/MemoryList.jsx @@ -3,8 +3,18 @@ import { Link } from 'react-router-dom'; import { memoriesApi } from '../../api/client'; import { useToast } from '../../components/Toast'; -const MEMORY_TYPES = ['', 'episodic', 'semantic', 'profile', 'procedural', 'decision', 'preference', 'task', 'risk']; -const STATUSES = ['', 'active', 'archived', 'forgotten', 'superseded', 'conflicted']; +const MEMORY_TYPES = ['', 'episodic', 'semantic', 'fact', 'profile', 'procedural', 'decision', 'preference', 'task', 'risk', 'constraint', 'policy', 'summary']; +const STATUS_OPTIONS = [ + { value: '', label: 'Active (default)' }, + { value: 'all', label: 'All Statuses' }, + { value: 'candidate', label: 'candidate' }, + { value: 'active', label: 'active' }, + { value: 'archived', label: 'archived' }, + { value: 'forgotten', label: 'forgotten' }, + { value: 'superseded', label: 'superseded' }, + { value: 'rejected', label: 'rejected' }, + { value: 'conflicted', label: 'conflicted' }, +]; export default function MemoryList() { const [memories, setMemories] = useState([]); @@ -58,8 +68,7 @@ export default function MemoryList() { updateFilter('workspace_id', e.target.value)} style={{ minWidth: 160 }} /> @@ -83,6 +92,7 @@ export default function MemoryList() { Title Type Status + Confidence Importance Workspace Updated @@ -103,6 +113,7 @@ export default function MemoryList() { {m.status || '—'} + {m.confidence != null ? Number(m.confidence).toFixed(2) : '—'} {m.importance ?? '—'} {m.workspace_id || '—'} {m.updated_at ? new Date(m.updated_at).toLocaleDateString() : '—'} diff --git a/frontend/src/pages/recall/Recall.jsx b/frontend/src/pages/recall/Recall.jsx index 2d2847b..332a647 100644 --- a/frontend/src/pages/recall/Recall.jsx +++ b/frontend/src/pages/recall/Recall.jsx @@ -31,6 +31,8 @@ export default function Recall() { } async function handleSubmit(e) { + // Search, context pack, and QA share the same form so the demo can show the + // same query moving from raw recall to agent-ready context to optional answer. e.preventDefault(); if (!form.query_text.trim()) { toast.error('Please enter a query'); diff --git a/frontend/src/pages/sources/SourceCreate.jsx b/frontend/src/pages/sources/SourceCreate.jsx new file mode 100644 index 0000000..03bd40c --- /dev/null +++ b/frontend/src/pages/sources/SourceCreate.jsx @@ -0,0 +1,115 @@ +import { useState } from 'react'; +import { Link, useNavigate } from 'react-router-dom'; +import { sourcesApi } from '../../api/client'; +import { DEMO_WORKSPACE_ID } from '../../api/constants'; +import { useToast } from '../../components/Toast'; + +const DOC_TYPES = ['markdown', 'txt', 'meeting', 'chat', 'note', 'report']; + +export default function SourceCreate() { + const navigate = useNavigate(); + const toast = useToast(); + const [submitting, setSubmitting] = useState(false); + const [form, setForm] = useState({ + workspace_id: DEMO_WORKSPACE_ID, + title: '', + doc_type: 'markdown', + source_path: '', + raw_text: '', + }); + + function updateField(key, value) { + setForm((current) => ({ ...current, [key]: value })); + } + + async function handleSubmit(event) { + event.preventDefault(); + setSubmitting(true); + try { + const created = await sourcesApi.create({ + workspace_id: form.workspace_id, + title: form.title, + doc_type: form.doc_type, + source_path: form.source_path || null, + raw_text: form.raw_text, + }); + toast.success(`Source created with ${created.chunk_count} chunks`); + navigate(`/sources/${created.doc_id}?workspace_id=${form.workspace_id}`); + } catch (err) { + toast.error(err.message || 'Failed to create source'); + } finally { + setSubmitting(false); + } + } + + return ( +
+
+

+ New Source

+ Back to Sources +
+ +
+
+
+ + updateField('workspace_id', event.target.value)} + required + /> +
+
+ + +
+
+ + updateField('title', event.target.value)} + placeholder="Discussion notes, design draft, meeting recap..." + required + /> +
+
+ + updateField('source_path', event.target.value)} + placeholder="Optional logical path or filename" + /> +
+
+ +
+ +