diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a6c819d..f0718764 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -606,3 +606,40 @@ jobs: echo "::error::any dependency, run 'uv lock' and commit the result in the SAME commit." echo "::error::A stale lock means the committed pins are not what this project resolves." exit 1 + + prior-art: + name: Prior art evidence corpus + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.11" + + - name: Validate canonical prior art records + run: python -m tools.prior_art validate + + - name: Verify generated prior art reports + run: python -m tools.prior_art render --check + + - name: Check prior work declarations in changed experiments + shell: bash + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + exit 0 + fi + if [ "${{ github.event_name }}" = "pull_request" ]; then + base="${{ github.event.pull_request.base.sha }}" + else + base="${{ github.event.before }}" + fi + if [ -z "$base" ] || [[ "$base" =~ ^0+$ ]]; then + exit 0 + fi + mapfile -t files < <(git diff --name-only --diff-filter=AM "$base" "${{ github.sha }}" -- 'benchmarks/**/*.py' 'results/**/*.py') + if ((${#files[@]})); then + python -m tools.prior_art check-experiments "${files[@]}" + fi diff --git a/benchmarks/EXPERIMENT-CONVENTION.md b/benchmarks/EXPERIMENT-CONVENTION.md index 390950c8..ac366c92 100644 --- a/benchmarks/EXPERIMENT-CONVENTION.md +++ b/benchmarks/EXPERIMENT-CONVENTION.md @@ -12,6 +12,12 @@ or, when nothing was found: Prior work: none found (docs_search "", source_type=memory) ``` +For changed or newly added modules, run the repository check with the affected paths: + +```text +python -m tools.prior_art check-experiments benchmarks/my_probe.py +``` + ## Why this exists rather than a rule someone remembers On 2026-07-28 an abstention investigation was run that had already been run on 2026-07-24, and diff --git a/docs/PRIOR_ART.md b/docs/PRIOR_ART.md index ec3bc132..7804ce86 100644 --- a/docs/PRIOR_ART.md +++ b/docs/PRIOR_ART.md @@ -1,5 +1,26 @@ # Prior art, and where this comes from +## Structured prior art evidence system + +This page remains the public narrative, while the canonical comparison data now lives in the +[RE-call prior art evidence system](prior_art/README.md). The structured corpus records sources, +systems, capability definitions, atomic claims, evidence locations, and human review decisions. + +The generated [capability matrix](prior_art/generated_matrix.md) and [gap report](prior_art/generated_gap_report.md) +are reproducible from the checked in JSON and JSON Lines records. They preserve `verified`, +`partial`, `not_evidenced`, `contradicted`, and `unknown` as separate states. Missing evidence is +not treated as proof that a system lacks a capability. + +The current RE-call research hypothesis combines evidence backed claims, explicit validity and +supersession, reversible provenance lineage, authority and scope enforcement, deletion propagation +through derived artifacts, support based abstention, and action outcome feedback. The matrix may +identify this as an unverified research gap, but it does not automatically claim that RE-call is +first, unique, or unprecedented. + +The collection scope covers agent memory systems, agent runtimes, benchmarks, privacy, governance, +and continual learning. The search cutoff and evidence rules are recorded in +[report_config.json](prior_art/report_config.json). + ## Prior art — and where this genuinely differs Agent memory is a crowded field. Everything below is Apache-2.0 and further along than this diff --git a/docs/prior_art/README.md b/docs/prior_art/README.md new file mode 100644 index 00000000..67822393 --- /dev/null +++ b/docs/prior_art/README.md @@ -0,0 +1,43 @@ +# RE-call prior art evidence system + +This directory is the canonical, reviewable evidence corpus behind the public prior art matrix. + +The corpus separates five things: + +1. Sources, which are external papers, repositories, documentation, benchmarks, protocols, or standards. +2. Systems, which are the products or research systems being compared. +3. Capabilities, which are stable, defined matrix dimensions. +4. Claims, which are atomic statements about one system and one capability. +5. Reviews, which record the human decision that allows a claim into the generated report. + +The accepted capability values are `verified`, `partial`, `not_evidenced`, `contradicted`, and +`unknown`. `not_evidenced` means that reviewed sources did not establish a capability. `unknown` +means that the investigation is incomplete. Neither value proves that a system lacks a capability. + +## Workflow + +Run validation and rendering from the repository root: + +```text +python -m tools.prior_art validate +python -m tools.prior_art render +python -m tools.prior_art render --check +python -m tools.prior_art check-experiments benchmarks/my_probe.py +python -m tools.prior_art check-links +``` + +The link check is intentionally manual because external availability is unstable. It does not +rewrite source records. + +## Evidence policy + +Secondary sources may discover candidates, but accepted `verified` claims require primary evidence. +Each accepted claim has a source, an evidence locator, a human review record, and a short evidence +note. The renderer never creates a claim from silence and never emits an automatic claim of novelty. + +## Generated reports + +* [Capability matrix](generated_matrix.md) +* [Gap report](generated_gap_report.md) +* [Machine readable summary](generated_summary.json) +* [Public prior art position](../PRIOR_ART.md) diff --git a/docs/prior_art/claims.jsonl b/docs/prior_art/claims.jsonl new file mode 100644 index 00000000..dd558223 --- /dev/null +++ b/docs/prior_art/claims.jsonl @@ -0,0 +1,51 @@ +{"claim_id":"clm_graphiti_graph_001","system_id":"sys_graphiti","capability_id":"representation.knowledge_graph","value":"verified","claim_text":"Graphiti represents agent memory as a temporal knowledge graph of entities and relations.","source_id":"src_graphiti_docs","evidence_locator":{"kind":"section","value":"Graphiti overview"},"evidence_note":"The official overview describes temporal context graphs built from conversations and structured data.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_graphiti_graph_001"} +{"claim_id":"clm_graphiti_traversal_001","system_id":"sys_graphiti","capability_id":"retrieval.graph_traversal","value":"verified","claim_text":"Graphiti combines semantic, full text, and graph based retrieval.","source_id":"src_graphiti_docs","evidence_locator":{"kind":"section","value":"What Graphiti does"},"evidence_note":"The official overview names vector, full text, and graph traversal retrieval.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_graphiti_traversal_001"} +{"claim_id":"clm_graphiti_valid_time_001","system_id":"sys_graphiti","capability_id":"time.valid_time","value":"verified","claim_text":"Graphiti records when a relation became valid and when it stopped being valid.","source_id":"src_graphiti_docs","evidence_locator":{"kind":"section","value":"Temporal by design"},"evidence_note":"The documentation describes valid time on graph edges.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_graphiti_valid_time_001"} +{"claim_id":"clm_graphiti_ingestion_time_001","system_id":"sys_graphiti","capability_id":"time.ingestion_time","value":"verified","claim_text":"Graphiti records when it learned a fact and when it learned that the fact was no longer true.","source_id":"src_graphiti_docs","evidence_locator":{"kind":"section","value":"Temporal by design"},"evidence_note":"The documentation describes ingestion and invalidation times in addition to valid time.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_graphiti_ingestion_time_001"} +{"claim_id":"clm_mem0_facts_001","system_id":"sys_mem0","capability_id":"representation.structured_facts","value":"verified","claim_text":"Mem0 extracts and stores structured persistent memories from ongoing conversations.","source_id":"src_mem0_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The paper describes dynamically extracting, consolidating, and retrieving salient information.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_mem0_facts_001"} +{"claim_id":"clm_mem0_extraction_001","system_id":"sys_mem0","capability_id":"write_path.extraction","value":"verified","claim_text":"Mem0 uses an extraction and consolidation pipeline for ongoing conversations.","source_id":"src_mem0_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The paper explicitly presents dynamic extraction and consolidation.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_mem0_extraction_001"} +{"claim_id":"clm_mem0_graph_001","system_id":"sys_mem0","capability_id":"representation.knowledge_graph","value":"partial","claim_text":"Mem0 provides a graph based variant for representing relationships among conversational elements.","source_id":"src_mem0_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The paper presents an enhanced graph based variant, without establishing that every deployment uses it.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_mem0_graph_001"} +{"claim_id":"clm_mem0_dense_001","system_id":"sys_mem0","capability_id":"retrieval.dense","value":"verified","claim_text":"Mem0 evaluates memory retrieval as part of a persistent memory architecture.","source_id":"src_mem0_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The paper evaluates retrieval from persistent structured memory, while this claim does not assert a specific vector backend.","evidence_type":"explicit","evidence_excerpt":null,"review_status":"accepted","review_id":"rev_mem0_dense_001"} +{"claim_id":"clm_letta_raw_001","system_id":"sys_letta","capability_id":"representation.raw_episodes","value":"verified","claim_text":"Letta supports persistent agent memory through externally managed memory blocks and archival storage.","source_id":"src_letta_repository","evidence_locator":{"kind":"repository_path","value":"README.md:memory and archival storage"},"evidence_note":"The repository documents memory blocks and archival memory as part of the agent runtime.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_letta_raw_001"} +{"claim_id":"clm_letta_agent_authored_001","system_id":"sys_letta","capability_id":"write_path.agent_authored","value":"verified","claim_text":"Letta agents can manage memory through tool calls as part of the agent runtime.","source_id":"src_letta_benchmark","evidence_locator":{"kind":"section","value":"Benchmarking Agentic Memory"},"evidence_note":"The benchmark describes agentic memory management through tool calling.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_letta_agent_authored_001"} +{"claim_id":"clm_letta_eval_001","system_id":"sys_letta","capability_id":"evaluation.conflict_resolution","value":"not_evidenced","claim_text":"The reviewed Letta benchmark sources do not establish a dedicated conflict resolution evaluation cell.","source_id":"src_letta_benchmark","evidence_locator":{"kind":"section","value":"Letta Leaderboard"},"evidence_note":"The reviewed source describes read, write, and update memory benchmarking but does not establish this specific capability.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_letta_eval_001"} +{"claim_id":"clm_langmem_extraction_001","system_id":"sys_langmem","capability_id":"write_path.extraction","value":"verified","claim_text":"LangMem provides memory management components for extracting and managing long term memory in LangGraph applications.","source_id":"src_langmem_docs","evidence_locator":{"kind":"section","value":"LangMem documentation"},"evidence_note":"The official documentation presents LangMem as a memory management toolkit.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_langmem_extraction_001"} +{"claim_id":"clm_langmem_revision_001","system_id":"sys_langmem","capability_id":"validity_and_revision.contradiction","value":"unknown","claim_text":"The reviewed LangMem documentation is insufficient to establish its contradiction handling behavior.","source_id":"src_langmem_docs","evidence_locator":{"kind":"section","value":"LangMem documentation"},"evidence_note":"The source was recorded for discovery, but the reviewed documentation does not establish this capability.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_langmem_revision_001"} +{"claim_id":"clm_recall_source_001","system_id":"sys_recall","capability_id":"provenance.source_attribution","value":"verified","claim_text":"RE-call returns provenance and source metadata with retrieved memory evidence.","source_id":"src_recall_readme","evidence_locator":{"kind":"section","value":"Why RE-call, Validity-aware retrieval"},"evidence_note":"The README describes verdicts, confidence, provenance, tenant scope, and validity metadata on returned hits.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_recall_source_001"} +{"claim_id":"clm_recall_lineage_001","system_id":"sys_recall","capability_id":"provenance.transformation_lineage","value":"partial","claim_text":"RE-call projects typed semantic entities and relations with supporting chunk identifiers and generation identity.","source_id":"src_recall_reasoning_graph","evidence_locator":{"kind":"section","value":"Evidence Graph V1"},"evidence_note":"The reasoning graph documents evidence linked graph projections, while the feature remains opt in and does not replace authored corpus truth.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_recall_lineage_001"} +{"claim_id":"clm_recall_abstention_001","system_id":"sys_recall","capability_id":"uncertainty.abstention","value":"verified","claim_text":"RE-call can return an explicit abstention when no valid result clears its calibrated threshold.","source_id":"src_recall_readme","evidence_locator":{"kind":"section","value":"Why RE-call, Explicit abstention"},"evidence_note":"The README describes abstention with a reason instead of a nearest neighbor guess.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_recall_abstention_001"} +{"claim_id":"clm_recall_supersession_001","system_id":"sys_recall","capability_id":"validity_and_revision.supersession","value":"verified","claim_text":"RE-call uses declared supersession and validity metadata to demote stale memories during retrieval.","source_id":"src_recall_readme","evidence_locator":{"kind":"section","value":"Why RE-call, Validity-aware retrieval"},"evidence_note":"The README describes superseded and expired hits as verdicts rather than ordinary results.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_recall_supersession_001"} +{"claim_id":"clm_recall_tenant_001","system_id":"sys_recall","capability_id":"authority_and_scope.tenant_scope","value":"verified","claim_text":"RE-call includes tenant identifiers and row level security in its production boundaries.","source_id":"src_recall_readme","evidence_locator":{"kind":"section","value":"Why RE-call, Production boundaries"},"evidence_note":"The README lists tenant IDs and row level security as shipped production boundaries.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_recall_tenant_001"} +{"claim_id":"clm_amp_egress_001","system_id":"sys_amp","capability_id":"security.egress_control","value":"verified","claim_text":"The Agent-Memory Protocol defines deterministic redaction, purpose limited packing, and hydration operations to keep personal identifiers inside the user boundary.","source_id":"src_amp_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly defines the three operations and their confidentiality goal.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_amp_egress_001"} +{"claim_id":"clm_neo4j_graph_001","system_id":"sys_neo4j_agent_memory","capability_id":"representation.knowledge_graph","value":"verified","claim_text":"Neo4j Agent Memory represents agent memory as a knowledge graph with entity and relationship structure.","source_id":"src_neo4j_agent_memory","evidence_locator":{"kind":"section","value":"README.md:What It Does"},"evidence_note":"The official repository describes graph native memory and a knowledge graph representation.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_neo4j_graph_001"} +{"claim_id":"clm_neo4j_provenance_001","system_id":"sys_neo4j_agent_memory","capability_id":"provenance.source_attribution","value":"verified","claim_text":"Neo4j Agent Memory tracks where entities were extracted from and which extractor produced them.","source_id":"src_neo4j_agent_memory","evidence_locator":{"kind":"section","value":"README.md:Production features"},"evidence_note":"The official repository explicitly lists provenance tracking for extracted entities.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_neo4j_provenance_001"} +{"claim_id":"clm_neo4j_time_001","system_id":"sys_neo4j_agent_memory","capability_id":"time.valid_time","value":"verified","claim_text":"Neo4j Agent Memory tracks when facts become valid or invalid.","source_id":"src_neo4j_agent_memory","evidence_locator":{"kind":"section","value":"README.md:Production features"},"evidence_note":"The official repository explicitly describes temporal relationships and fact validity changes.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_neo4j_time_001"} +{"claim_id":"clm_neo4j_consolidation_001","system_id":"sys_neo4j_agent_memory","capability_id":"write_path.consolidation","value":"verified","claim_text":"Neo4j Agent Memory provides consolidation primitives for deduplicating entities.","source_id":"src_neo4j_agent_memory","evidence_locator":{"kind":"section","value":"README.md:Production features"},"evidence_note":"The official repository explicitly lists consolidation and entity deduplication.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_neo4j_consolidation_001"} +{"claim_id":"clm_neo4j_trace_001","system_id":"sys_neo4j_agent_memory","capability_id":"action_feedback.outcome_storage","value":"partial","claim_text":"Neo4j Agent Memory stores reasoning traces, tool usage, and audit edges linking reasoning steps to entities.","source_id":"src_neo4j_agent_memory","evidence_locator":{"kind":"section","value":"README.md:Production features"},"evidence_note":"The repository describes trace and audit storage, but this does not establish outcome driven belief revision.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_neo4j_trace_001"} +{"claim_id":"clm_quipu_episodes_001","system_id":"sys_quipu","capability_id":"representation.raw_episodes","value":"verified","claim_text":"Quipu preserves raw evidence and links derived memories back to the supporting messages.","source_id":"src_quipu_repository","evidence_locator":{"kind":"section","value":"README.md:What It Does"},"evidence_note":"The official repository describes raw evidence preservation and evidence linked derived facts.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_quipu_episodes_001"} +{"claim_id":"clm_quipu_lineage_001","system_id":"sys_quipu","capability_id":"provenance.transformation_lineage","value":"verified","claim_text":"Quipu records supporting evidence for derived facts and exposes provenance inspection for those derivations.","source_id":"src_quipu_repository","evidence_locator":{"kind":"section","value":"README.md:Why Quipu"},"evidence_note":"The official repository describes evidence backed derivation and provenance inspection.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_quipu_lineage_001"} +{"claim_id":"clm_quipu_time_001","system_id":"sys_quipu","capability_id":"time.valid_time","value":"verified","claim_text":"Quipu stores current facts with temporal supersession so old values remain historical rather than silently replacing the record.","source_id":"src_quipu_repository","evidence_locator":{"kind":"section","value":"README.md:Why Quipu"},"evidence_note":"The official repository explicitly describes temporal facts and historical old values.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_quipu_time_001"} +{"claim_id":"clm_quipu_forgetting_001","system_id":"sys_quipu","capability_id":"deletion_and_forgetting.derived_propagation","value":"verified","claim_text":"Quipu propagates forgetting from raw evidence to derived memories and reports the resulting closure.","source_id":"src_quipu_repository","evidence_locator":{"kind":"section","value":"README.md:Why Quipu"},"evidence_note":"The official repository describes provenance aware forgetting, suppression of derived memories, and closure reports.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_quipu_forgetting_001"} +{"claim_id":"clm_quipu_scope_001","system_id":"sys_quipu","capability_id":"authority_and_scope.tenant_scope","value":"verified","claim_text":"Quipu supports scoped retrieval so memory access can be constrained to the requested scope.","source_id":"src_quipu_repository","evidence_locator":{"kind":"section","value":"README.md:What It Does"},"evidence_note":"The official repository explicitly describes scoped and budgeted retrieval.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_quipu_scope_001"} +{"claim_id":"clm_memmachine_episodes_001","system_id":"sys_memmachine","capability_id":"representation.raw_episodes","value":"verified","claim_text":"MemMachine stores entire conversational episodes to preserve ground truth for later memory use.","source_id":"src_memmachine_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper describes a ground truth preserving architecture that stores entire conversational episodes.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_memmachine_episodes_001"} +{"claim_id":"clm_memmachine_context_001","system_id":"sys_memmachine","capability_id":"retrieval.query_time_context","value":"verified","claim_text":"MemMachine expands nucleus retrieval matches with surrounding conversational context at retrieval time.","source_id":"src_memmachine_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly describes contextualized retrieval that expands nucleus matches with surrounding context.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_memmachine_context_001"} +{"claim_id":"clm_agent_memory_bench_continual_001","system_id":"sys_agent_memory_bench","capability_id":"evaluation.continual_learning","value":"verified","claim_text":"AgentMemoryBench jointly evaluates system and personal agent memory in a continual learning framework covering online learning, transfer, and forgetting.","source_id":"src_agent_memory_bench","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary benchmark paper defines a unified continual evaluation for system and personal memory.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_agent_memory_bench_continual_001"} +{"claim_id":"clm_memorybank_forgetting_001","system_id":"sys_memorybank","capability_id":"deletion_and_forgetting.selective_forgetting","value":"verified","claim_text":"MemoryBank selectively forgets or reinforces memories using elapsed time and relative significance.","source_id":"src_memorybank_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly describes selective preservation, forgetting, and reinforcement based on time and significance.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_memorybank_forgetting_001"} +{"claim_id":"clm_amem_agent_authored_001","system_id":"sys_amem","capability_id":"write_path.agent_authored","value":"verified","claim_text":"A-MEM uses an agentic process to create structured memory notes, link new memories to historical ones, and evolve existing memory attributes.","source_id":"src_amem_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly describes agent driven note construction, linking, and memory evolution.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_amem_agent_authored_001"} +{"claim_id":"clm_hipporag_graph_001","system_id":"sys_hipporag","capability_id":"retrieval.graph_traversal","value":"verified","claim_text":"HippoRAG combines a knowledge graph with Personalized PageRank to expand retrieval across related information for multi hop questions.","source_id":"src_hipporag_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly identifies knowledge graphs and Personalized PageRank as the retrieval components.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_hipporag_graph_001"} +{"claim_id":"clm_longmemeval_temporal_001","system_id":"sys_longmemeval","capability_id":"evaluation.temporal_reasoning","value":"verified","claim_text":"LongMemEval evaluates long term memory systems on temporal reasoning and knowledge updates across timestamped interaction histories.","source_id":"src_longmemeval_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary benchmark paper explicitly lists temporal reasoning and knowledge updates among its core abilities and uses timestamped histories.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_longmemeval_temporal_001"} +{"claim_id":"clm_longmemeval_v2_action_001","system_id":"sys_longmemeval_v2","capability_id":"evaluation.action_impact","value":"partial","claim_text":"LongMemEval V2 evaluates whether compact memory evidence improves downstream agent question answering about environment state, workflows, and recurring failure modes.","source_id":"src_longmemeval_v2_repo","evidence_locator":{"kind":"section","value":"README.md:Overview"},"evidence_note":"The official repository evaluates downstream answer accuracy and query latency, but does not establish improvement in executed action outcomes.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_longmemeval_v2_action_001"} +{"claim_id":"clm_memory_r1_delete_001","system_id":"sys_memory_r1","capability_id":"deletion_and_forgetting.source_deletion","value":"verified","claim_text":"Memory R1 trains a Memory Manager to perform structured DELETE operations on external memory entries.","source_id":"src_memory_r1_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly lists DELETE among the learned memory operations.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_memory_r1_delete_001"} +{"claim_id":"clm_memory_r1_agent_authored_001","system_id":"sys_memory_r1","capability_id":"write_path.agent_authored","value":"verified","claim_text":"Memory R1 uses a learned Memory Manager to choose whether to add, update, delete, or leave memory unchanged.","source_id":"src_memory_r1_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper describes an active memory manager that selects structured memory operations.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_memory_r1_agent_authored_001"} +{"claim_id":"clm_agemem_agent_authored_001","system_id":"sys_agemem","capability_id":"write_path.agent_authored","value":"verified","claim_text":"AgeMem exposes memory operations as tool actions so the agent can decide when to store, retrieve, update, summarize, or discard information.","source_id":"src_agemem_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly describes autonomous agent decisions over the unified memory operations.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_agemem_agent_authored_001"} +{"claim_id":"clm_agemem_forgetting_001","system_id":"sys_agemem","capability_id":"deletion_and_forgetting.selective_forgetting","value":"verified","claim_text":"AgeMem includes discard as an agent controlled operation for removing selected information from memory.","source_id":"src_agemem_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly lists discard among the memory operations available to the agent.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_agemem_forgetting_001"} +{"claim_id":"clm_mindmemos_consolidation_001","system_id":"sys_mindmemos","capability_id":"write_path.consolidation","value":"verified","claim_text":"MindMemOS consolidates accumulated memories by merging redundant records and resolving conflicts.","source_id":"src_mindmemos_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly describes dreaming based consolidation with redundancy merging and conflict resolution.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_mindmemos_consolidation_001"} +{"claim_id":"clm_mindmemos_contradiction_001","system_id":"sys_mindmemos","capability_id":"validity_and_revision.contradiction","value":"verified","claim_text":"MindMemOS resolves conflicting accumulated memory records during its consolidation process.","source_id":"src_mindmemos_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly identifies conflict resolution as part of memory consolidation.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_mindmemos_contradiction_001"} +{"claim_id":"clm_chronomem_rollback_001","system_id":"sys_chronomem","capability_id":"validity_and_revision.rollback","value":"verified","claim_text":"ChronoMem snapshots whole memory states and maps natural language rollback requests to historical versions.","source_id":"src_chronomem_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly describes whole memory snapshots, version histories, and semantic rollback requests.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_chronomem_rollback_001"} +{"claim_id":"clm_dependency_rollback_derived_001","system_id":"sys_dependency_rollback","capability_id":"deletion_and_forgetting.derived_propagation","value":"verified","claim_text":"Dependency Guided Rollback Repair traces memory to action dependencies, deactivates unsupported memory state, and selectively replays affected computation while preserving unaffected state.","source_id":"src_dependency_rollback_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly describes typed dependency tracing, affected state invalidation, and preservation of benign state.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_dependency_rollback_derived_001"} +{"claim_id":"clm_merit_outcome_storage_001","system_id":"sys_merit","capability_id":"action_feedback.outcome_storage","value":"verified","claim_text":"MERIT stores oracle verified repair corrections and observed unsuccessful directions in an online dual polarity episodic memory.","source_id":"src_merit_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly describes persistent memory for finalized repair outcomes and unsuccessful directions.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_merit_outcome_storage_001"} +{"claim_id":"clm_reflexion_policy_revision_001","system_id":"sys_reflexion","capability_id":"action_feedback.policy_revision","value":"verified","claim_text":"Reflexion stores verbal reflections derived from task feedback in episodic memory to improve later decision making.","source_id":"src_reflexion_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly links feedback, episodic memory, and improved subsequent decisions.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_reflexion_policy_revision_001"} +{"claim_id":"clm_bad_memory_security_eval_001","system_id":"sys_bad_memory","capability_id":"evaluation.memory_security","value":"verified","claim_text":"Bad Memory evaluates whether prompt injection payloads planted in persistent memory influence current and future agent sessions.","source_id":"src_bad_memory_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly evaluates planted memory payloads across current and future sessions.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_bad_memory_security_eval_001"} +{"claim_id":"clm_memsecbench_security_eval_001","system_id":"sys_memsecbench","capability_id":"evaluation.memory_security","value":"verified","claim_text":"MemSecBench evaluates the lifecycle of malicious memory from persistence through downstream consequence and selective repair.","source_id":"src_memsecbench_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary benchmark paper explicitly defines a Write Execute Forget protocol and lifecycle checkpoints.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_memsecbench_security_eval_001"} +{"claim_id":"clm_mpbench_security_eval_001","system_id":"sys_mpbench","capability_id":"evaluation.memory_security","value":"verified","claim_text":"MPBench evaluates memory poisoning attacks across multiple memory write channels and attack classes.","source_id":"src_mpbench_paper","evidence_locator":{"kind":"section","value":"Abstract"},"evidence_note":"The primary paper explicitly introduces MPBench for evaluating memory poisoning attacks and identifies multiple write channels and attack classes.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_mpbench_security_eval_001"} +{"claim_id":"clm_quipu_receipts_001","system_id":"sys_quipu","capability_id":"provenance.evidence_receipts","value":"partial","claim_text":"Quipu exposes provenance inspection for derived memories, including the supporting evidence used for a memory result.","source_id":"src_quipu_repository","evidence_locator":{"kind":"section","value":"README.md:Why Quipu"},"evidence_note":"The official repository describes provenance inspection and evidence backed derivations, but this record does not independently establish the exact serialized receipt schema.","evidence_excerpt":null,"evidence_type":"explicit","review_status":"accepted","review_id":"rev_quipu_receipts_001"} diff --git a/docs/prior_art/generated_gap_report.md b/docs/prior_art/generated_gap_report.md new file mode 100644 index 00000000..d39d5af3 --- /dev/null +++ b/docs/prior_art/generated_gap_report.md @@ -0,0 +1,357 @@ +# RE-call prior art gap report + +This report identifies research candidates. It does not assert that a capability has never been implemented. + +## Capability status + +### representation.raw_episodes + +Group: `representation`. Status: **established**. + +Definition: The system preserves source episodes or turns as retrievable memory. + +Systems with reviewed claims: `sys_letta`, `sys_memmachine`, `sys_quipu`. + +### representation.structured_facts + +Group: `representation`. Status: **emerging**. + +Definition: The system represents memory as structured facts or attributes rather than only raw text. + +Systems with reviewed claims: `sys_mem0`. + +### representation.knowledge_graph + +Group: `representation`. Status: **established**. + +Definition: The system represents entities and relations as a graph that can be queried. + +Systems with reviewed claims: `sys_graphiti`, `sys_mem0`, `sys_neo4j_agent_memory`. + +### write_path.extraction + +Group: `write_path`. Status: **established**. + +Definition: The system extracts persistent memory from conversations or agent traces. + +Systems with reviewed claims: `sys_langmem`, `sys_mem0`. + +### write_path.consolidation + +Group: `write_path`. Status: **established**. + +Definition: The system merges, compresses, or organizes multiple memories during maintenance. + +Systems with reviewed claims: `sys_mindmemos`, `sys_neo4j_agent_memory`. + +### write_path.agent_authored + +Group: `write_path`. Status: **established**. + +Definition: The agent can deliberately decide what to write or update in persistent memory. + +Systems with reviewed claims: `sys_agemem`, `sys_amem`, `sys_letta`, `sys_memory_r1`. + +### retrieval.dense + +Group: `retrieval`. Status: **emerging**. + +Definition: The system retrieves memory using vector similarity. + +Systems with reviewed claims: `sys_mem0`. + +### retrieval.lexical + +Group: `retrieval`. Status: **unverified_gap**. + +Definition: The system retrieves memory using lexical or full text search. + +No reviewed system claim exists for this capability. + +### retrieval.graph_traversal + +Group: `retrieval`. Status: **established**. + +Definition: The system expands retrieval through graph relations or traversals. + +Systems with reviewed claims: `sys_graphiti`, `sys_hipporag`. + +### retrieval.query_time_context + +Group: `retrieval`. Status: **emerging**. + +Definition: The system constructs or expands the memory context at query time using the retrieved evidence. + +Systems with reviewed claims: `sys_memmachine`. + +### validity_and_revision.supersession + +Group: `validity_and_revision`. Status: **emerging**. + +Definition: The system represents that a newer memory replaces an older memory. + +Systems with reviewed claims: `sys_recall`. + +### validity_and_revision.contradiction + +Group: `validity_and_revision`. Status: **emerging**. + +Definition: The system detects or represents conflicting memories. + +Systems with reviewed claims: `sys_langmem`, `sys_mindmemos`. + +### validity_and_revision.rollback + +Group: `validity_and_revision`. Status: **emerging**. + +Definition: The system can restore or select an earlier memory state. + +Systems with reviewed claims: `sys_chronomem`. + +### time.valid_time + +Group: `time`. Status: **established**. + +Definition: The system records when a fact was true in the world. + +Systems with reviewed claims: `sys_graphiti`, `sys_neo4j_agent_memory`, `sys_quipu`. + +### time.ingestion_time + +Group: `time`. Status: **emerging**. + +Definition: The system records when it learned or ingested a fact. + +Systems with reviewed claims: `sys_graphiti`. + +### time.point_in_time + +Group: `time`. Status: **unverified_gap**. + +Definition: The system can answer what was known or valid at a requested historical time. + +No reviewed system claim exists for this capability. + +### provenance.source_attribution + +Group: `provenance`. Status: **established**. + +Definition: A returned memory can identify the source evidence supporting it. + +Systems with reviewed claims: `sys_neo4j_agent_memory`, `sys_recall`. + +### provenance.transformation_lineage + +Group: `provenance`. Status: **emerging**. + +Definition: Derived memories link back through transformations to their source evidence. + +Systems with reviewed claims: `sys_quipu`, `sys_recall`. + +### provenance.evidence_receipts + +Group: `provenance`. Status: **emerging**. + +Definition: The system returns a machine readable record of evidence used for a memory result. + +Systems with reviewed claims: `sys_quipu`. + +### uncertainty.abstention + +Group: `uncertainty`. Status: **emerging**. + +Definition: The system can return an explicit refusal when memory support is insufficient. + +Systems with reviewed claims: `sys_recall`. + +### uncertainty.entailment + +Group: `uncertainty`. Status: **unverified_gap**. + +Definition: The system checks whether retrieved evidence actually supports the requested claim. + +No reviewed system claim exists for this capability. + +### uncertainty.calibration + +Group: `uncertainty`. Status: **unverified_gap**. + +Definition: The system calibrates a trust or confidence decision against labeled examples. + +No reviewed system claim exists for this capability. + +### authority_and_scope.user_consent + +Group: `authority_and_scope`. Status: **unverified_gap**. + +Definition: The system makes persistent memory subject to an explicit user consent or control decision. + +No reviewed system claim exists for this capability. + +### authority_and_scope.tenant_scope + +Group: `authority_and_scope`. Status: **established**. + +Definition: The system enforces tenant boundaries on memory storage and retrieval. + +Systems with reviewed claims: `sys_quipu`, `sys_recall`. + +### authority_and_scope.multi_agent_handoff + +Group: `authority_and_scope`. Status: **unverified_gap**. + +Definition: Memory can be intentionally shared or handed from one agent to another with scope semantics. + +No reviewed system claim exists for this capability. + +### deletion_and_forgetting.source_deletion + +Group: `deletion_and_forgetting`. Status: **emerging**. + +Definition: A source memory can be deleted from the system. + +Systems with reviewed claims: `sys_memory_r1`. + +### deletion_and_forgetting.derived_propagation + +Group: `deletion_and_forgetting`. Status: **established**. + +Definition: Deleting a source removes or invalidates derived memories, graph edges, embeddings, and caches that depend on it. + +Systems with reviewed claims: `sys_dependency_rollback`, `sys_quipu`. + +### deletion_and_forgetting.selective_forgetting + +Group: `deletion_and_forgetting`. Status: **established**. + +Definition: The system can forget selected memories without clearing unrelated memory. + +Systems with reviewed claims: `sys_agemem`, `sys_memorybank`. + +### action_feedback.outcome_storage + +Group: `action_feedback`. Status: **emerging**. + +Definition: The system stores observed results of agent actions as reusable memory. + +Systems with reviewed claims: `sys_merit`, `sys_neo4j_agent_memory`. + +### action_feedback.outcome_linked_revision + +Group: `action_feedback`. Status: **unverified_gap**. + +Definition: An observed action result updates the memory claim that influenced that action. + +No reviewed system claim exists for this capability. + +### action_feedback.policy_revision + +Group: `action_feedback`. Status: **emerging**. + +Definition: Remembered outcomes modify future agent behavior or policy selection. + +Systems with reviewed claims: `sys_reflexion`. + +### security.prompt_injection_resistance + +Group: `security`. Status: **unverified_gap**. + +Definition: The system limits memory content from changing agent instructions or tool authority through injection. + +No reviewed system claim exists for this capability. + +### security.memory_poisoning_resistance + +Group: `security`. Status: **unverified_gap**. + +Definition: The system detects or limits malicious or misleading persistent memories. + +No reviewed system claim exists for this capability. + +### security.egress_control + +Group: `security`. Status: **emerging**. + +Definition: The system controls which memory fields can leave the memory boundary for a model or tool. + +Systems with reviewed claims: `sys_amp`. + +### evaluation.temporal_reasoning + +Group: `evaluation`. Status: **emerging**. + +Definition: The system is evaluated on changing facts and historical validity. + +Systems with reviewed claims: `sys_longmemeval`. + +### evaluation.conflict_resolution + +Group: `evaluation`. Status: **unverified_gap**. + +Definition: The system is evaluated on contradictory memories or conflicting claims. + +Systems with reviewed claims: `sys_letta`. + +### evaluation.action_impact + +Group: `evaluation`. Status: **emerging**. + +Definition: The system is evaluated by whether memory improves downstream agent actions or outcomes. + +Systems with reviewed claims: `sys_longmemeval_v2`. + +### evaluation.continual_learning + +Group: `evaluation`. Status: **emerging**. + +Definition: The system is evaluated over an ordered sequence of tasks or episodes where memory must support transfer and forgetting. + +Systems with reviewed claims: `sys_agent_memory_bench`. + +### evaluation.memory_security + +Group: `evaluation`. Status: **established**. + +Definition: The system or benchmark is evaluated on persistent memory poisoning, prompt injection, downstream consequences, or selective repair. + +Systems with reviewed claims: `sys_bad_memory`, `sys_memsecbench`, `sys_mpbench`. + +## RE-call research hypothesis + +The current hypothesis is a combination of evidence backed claims, explicit validity and supersession, reversible provenance lineage, authority and scope enforcement, deletion propagation through derived artifacts, abstention based on support and conflict, and action outcome feedback into future belief state. + +The matrix must establish the evidence boundary before this combination is described as novel. + +## Target combination analysis + +This section reports coverage of the configured RE-call hypothesis. It is not a novelty claim. A missing cell means that this corpus has not accepted evidence for that capability in that system. + +Target capabilities: `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision`. + +| System | Combination status | Verified support | Partial support | Missing evidence | Conflicting evidence | +| --- | --- | --- | --- | --- | --- | +| `sys_agemem` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_agent_memory_bench` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_amem` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_amp` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_bad_memory` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_chronomem` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_dependency_rollback` | `partial_combination` | `deletion_and_forgetting.derived_propagation` | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_graphiti` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_hipporag` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_langmem` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_letta` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_longmemeval` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_longmemeval_v2` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_mem0` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_memmachine` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_memory_r1` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_memorybank` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_memsecbench` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_merit` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_mindmemos` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_mpbench` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_neo4j_agent_memory` | `partial_combination` | `provenance.source_attribution` | none | `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_quipu` | `partial_combination` | `provenance.transformation_lineage`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation` | none | `provenance.source_attribution`, `validity_and_revision.supersession`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| +| `sys_recall` | `partial_combination` | `provenance.source_attribution`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `uncertainty.abstention` | `provenance.transformation_lineage` | `deletion_and_forgetting.derived_propagation`, `action_feedback.outcome_linked_revision` | none| +| `sys_reflexion` | `unverified_combination` | none | none | `provenance.source_attribution`, `provenance.transformation_lineage`, `validity_and_revision.supersession`, `authority_and_scope.tenant_scope`, `deletion_and_forgetting.derived_propagation`, `uncertainty.abstention`, `action_feedback.outcome_linked_revision` | none| diff --git a/docs/prior_art/generated_matrix.md b/docs/prior_art/generated_matrix.md new file mode 100644 index 00000000..68a4875a --- /dev/null +++ b/docs/prior_art/generated_matrix.md @@ -0,0 +1,1298 @@ +# RE-call prior art capability matrix + +Generated from the canonical evidence records in `docs/prior_art/`. + +This matrix records evidence, not absolute absence. `unknown` means the investigation is incomplete; `not_evidenced` means the reviewed sources did not establish the capability. + +## System overview + +This overview has one row per system and one column per capability group. Group cells aggregate the detailed capability records below; `partial` means the group is not completely evidenced. + +| System | representation | write_path | retrieval | validity_and_revision | time | provenance | uncertainty | authority_and_scope | deletion_and_forgetting | action_feedback | security | evaluation | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| AgeMem | `unknown` | `partial` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `partial` | `unknown` | `unknown` | `unknown` | +| AgentMemoryBench | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `partial` | +| A-MEM | `unknown` | `partial` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | +| Agent-Memory Protocol | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `partial` | `unknown` | +| Bad Memory | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `partial` | +| ChronoMem | `unknown` | `unknown` | `unknown` | `partial` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | +| Dependency Guided Rollback Repair | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `partial` | `unknown` | `unknown` | `unknown` | +| Graphiti | `partial` | `unknown` | `partial` | `unknown` | `partial` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | +| HippoRAG | `unknown` | `unknown` | `partial` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | +| LangMem | `unknown` | `partial` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | +| Letta | `partial` | `partial` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | +| LongMemEval | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `partial` | +| LongMemEval V2 | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `partial` | +| Mem0 | `partial` | `partial` | `partial` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | +| MemMachine | `partial` | `unknown` | `partial` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | +| Memory R1 | `unknown` | `partial` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `partial` | `unknown` | `unknown` | `unknown` | +| MemoryBank | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `partial` | `unknown` | `unknown` | `unknown` | +| MemSecBench | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `partial` | +| MERIT | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `partial` | `unknown` | `unknown` | +| MindMemOS | `unknown` | `partial` | `unknown` | `partial` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | +| MPBench | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `partial` | +| Neo4j Agent Memory | `partial` | `partial` | `unknown` | `unknown` | `partial` | `partial` | `unknown` | `unknown` | `unknown` | `partial` | `unknown` | `unknown` | +| Quipu | `partial` | `unknown` | `unknown` | `unknown` | `partial` | `partial` | `unknown` | `partial` | `partial` | `unknown` | `unknown` | `unknown` | +| RE-call | `unknown` | `unknown` | `unknown` | `partial` | `unknown` | `partial` | `partial` | `partial` | `unknown` | `unknown` | `unknown` | `unknown` | +| Reflexion | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `unknown` | `partial` | `unknown` | `unknown` | + +## representation: Raw episodes + +The system preserves source episodes or turns as retrievable memory. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `verified` | [src_letta_repository](https://github.com/letta-ai/letta) | Letta supports persistent agent memory through externally managed memory blocks and archival storage. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `verified` | [src_memmachine_paper](https://arxiv.org/abs/2604.04853) | MemMachine stores entire conversational episodes to preserve ground truth for later memory use. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `verified` | [src_quipu_repository](https://github.com/jeffhajewski/quipu) | Quipu preserves raw evidence and links derived memories back to the supporting messages. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## representation: Structured facts + +The system represents memory as structured facts or attributes rather than only raw text. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `verified` | [src_mem0_paper](https://arxiv.org/abs/2504.19413) | Mem0 extracts and stores structured persistent memories from ongoing conversations. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## representation: Knowledge graph + +The system represents entities and relations as a graph that can be queried. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `verified` | [src_graphiti_docs](https://help.getzep.com/graphiti/getting-started/overview) | Graphiti represents agent memory as a temporal knowledge graph of entities and relations. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `partial` | [src_mem0_paper](https://arxiv.org/abs/2504.19413) | Mem0 provides a graph based variant for representing relationships among conversational elements. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `verified` | [src_neo4j_agent_memory](https://github.com/neo4j-labs/agent-memory) | Neo4j Agent Memory represents agent memory as a knowledge graph with entity and relationship structure. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## write_path: Memory extraction + +The system extracts persistent memory from conversations or agent traces. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `verified` | [src_langmem_docs](https://langchain-ai.github.io/langmem/) | LangMem provides memory management components for extracting and managing long term memory in LangGraph applications. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `verified` | [src_mem0_paper](https://arxiv.org/abs/2504.19413) | Mem0 uses an extraction and consolidation pipeline for ongoing conversations. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## write_path: Memory consolidation + +The system merges, compresses, or organizes multiple memories during maintenance. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `verified` | [src_mindmemos_paper](https://arxiv.org/abs/2608.12428) | MindMemOS consolidates accumulated memories by merging redundant records and resolving conflicts. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `verified` | [src_neo4j_agent_memory](https://github.com/neo4j-labs/agent-memory) | Neo4j Agent Memory provides consolidation primitives for deduplicating entities. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## write_path: Agent authored memory + +The agent can deliberately decide what to write or update in persistent memory. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `verified` | [src_agemem_paper](https://arxiv.org/abs/2601.01885) | AgeMem exposes memory operations as tool actions so the agent can decide when to store, retrieve, update, summarize, or discard information. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `verified` | [src_amem_paper](https://arxiv.org/abs/2502.12110) | A-MEM uses an agentic process to create structured memory notes, link new memories to historical ones, and evolve existing memory attributes. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `verified` | [src_letta_benchmark](https://www.letta.com/blog/letta-leaderboard/) | Letta agents can manage memory through tool calls as part of the agent runtime. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `verified` | [src_memory_r1_paper](https://arxiv.org/abs/2508.19828) | Memory R1 uses a learned Memory Manager to choose whether to add, update, delete, or leave memory unchanged. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## retrieval: Dense retrieval + +The system retrieves memory using vector similarity. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `verified` | [src_mem0_paper](https://arxiv.org/abs/2504.19413) | Mem0 evaluates memory retrieval as part of a persistent memory architecture. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## retrieval: Lexical retrieval + +The system retrieves memory using lexical or full text search. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## retrieval: Graph traversal + +The system expands retrieval through graph relations or traversals. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `verified` | [src_graphiti_docs](https://help.getzep.com/graphiti/getting-started/overview) | Graphiti combines semantic, full text, and graph based retrieval. | +| HippoRAG | `verified` | [src_hipporag_paper](https://arxiv.org/abs/2405.14831) | HippoRAG combines a knowledge graph with Personalized PageRank to expand retrieval across related information for multi hop questions. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## retrieval: Query time context construction + +The system constructs or expands the memory context at query time using the retrieved evidence. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `verified` | [src_memmachine_paper](https://arxiv.org/abs/2604.04853) | MemMachine expands nucleus retrieval matches with surrounding conversational context at retrieval time. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## validity_and_revision: Supersession + +The system represents that a newer memory replaces an older memory. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `verified` | [src_recall_readme](https://github.com/GiulioDER/RE-call/blob/master/README.md) | RE-call uses declared supersession and validity metadata to demote stale memories during retrieval. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## validity_and_revision: Contradiction handling + +The system detects or represents conflicting memories. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | [src_langmem_docs](https://langchain-ai.github.io/langmem/) | The reviewed LangMem documentation is insufficient to establish its contradiction handling behavior. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `verified` | [src_mindmemos_paper](https://arxiv.org/abs/2608.12428) | MindMemOS resolves conflicting accumulated memory records during its consolidation process. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## validity_and_revision: Rollback + +The system can restore or select an earlier memory state. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `verified` | [src_chronomem_paper](https://arxiv.org/abs/2607.27773) | ChronoMem snapshots whole memory states and maps natural language rollback requests to historical versions. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## time: Valid time + +The system records when a fact was true in the world. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `verified` | [src_graphiti_docs](https://help.getzep.com/graphiti/getting-started/overview) | Graphiti records when a relation became valid and when it stopped being valid. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `verified` | [src_neo4j_agent_memory](https://github.com/neo4j-labs/agent-memory) | Neo4j Agent Memory tracks when facts become valid or invalid. | +| Quipu | `verified` | [src_quipu_repository](https://github.com/jeffhajewski/quipu) | Quipu stores current facts with temporal supersession so old values remain historical rather than silently replacing the record. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## time: Ingestion time + +The system records when it learned or ingested a fact. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `verified` | [src_graphiti_docs](https://help.getzep.com/graphiti/getting-started/overview) | Graphiti records when it learned a fact and when it learned that the fact was no longer true. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## time: Point in time retrieval + +The system can answer what was known or valid at a requested historical time. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## provenance: Source attribution + +A returned memory can identify the source evidence supporting it. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `verified` | [src_neo4j_agent_memory](https://github.com/neo4j-labs/agent-memory) | Neo4j Agent Memory tracks where entities were extracted from and which extractor produced them. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `verified` | [src_recall_readme](https://github.com/GiulioDER/RE-call/blob/master/README.md) | RE-call returns provenance and source metadata with retrieved memory evidence. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## provenance: Transformation lineage + +Derived memories link back through transformations to their source evidence. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `verified` | [src_quipu_repository](https://github.com/jeffhajewski/quipu) | Quipu records supporting evidence for derived facts and exposes provenance inspection for those derivations. | +| RE-call | `partial` | [src_recall_reasoning_graph](https://github.com/GiulioDER/RE-call/blob/master/docs/REASONING_GRAPH.md) | RE-call projects typed semantic entities and relations with supporting chunk identifiers and generation identity. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## provenance: Evidence receipts + +The system returns a machine readable record of evidence used for a memory result. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `partial` | [src_quipu_repository](https://github.com/jeffhajewski/quipu) | Quipu exposes provenance inspection for derived memories, including the supporting evidence used for a memory result. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## uncertainty: Abstention + +The system can return an explicit refusal when memory support is insufficient. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `verified` | [src_recall_readme](https://github.com/GiulioDER/RE-call/blob/master/README.md) | RE-call can return an explicit abstention when no valid result clears its calibrated threshold. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## uncertainty: Entailment checking + +The system checks whether retrieved evidence actually supports the requested claim. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## uncertainty: Confidence calibration + +The system calibrates a trust or confidence decision against labeled examples. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## authority_and_scope: User consent + +The system makes persistent memory subject to an explicit user consent or control decision. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## authority_and_scope: Tenant scope + +The system enforces tenant boundaries on memory storage and retrieval. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `verified` | [src_quipu_repository](https://github.com/jeffhajewski/quipu) | Quipu supports scoped retrieval so memory access can be constrained to the requested scope. | +| RE-call | `verified` | [src_recall_readme](https://github.com/GiulioDER/RE-call/blob/master/README.md) | RE-call includes tenant identifiers and row level security in its production boundaries. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## authority_and_scope: Multi agent handoff + +Memory can be intentionally shared or handed from one agent to another with scope semantics. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## deletion_and_forgetting: Source deletion + +A source memory can be deleted from the system. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `verified` | [src_memory_r1_paper](https://arxiv.org/abs/2508.19828) | Memory R1 trains a Memory Manager to perform structured DELETE operations on external memory entries. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## deletion_and_forgetting: Derived deletion propagation + +Deleting a source removes or invalidates derived memories, graph edges, embeddings, and caches that depend on it. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `verified` | [src_dependency_rollback_paper](https://arxiv.org/abs/2608.10502) | Dependency Guided Rollback Repair traces memory to action dependencies, deactivates unsupported memory state, and selectively replays affected computation while preserving unaffected state. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `verified` | [src_quipu_repository](https://github.com/jeffhajewski/quipu) | Quipu propagates forgetting from raw evidence to derived memories and reports the resulting closure. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## deletion_and_forgetting: Selective forgetting + +The system can forget selected memories without clearing unrelated memory. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `verified` | [src_agemem_paper](https://arxiv.org/abs/2601.01885) | AgeMem includes discard as an agent controlled operation for removing selected information from memory. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `verified` | [src_memorybank_paper](https://arxiv.org/abs/2305.10250) | MemoryBank selectively forgets or reinforces memories using elapsed time and relative significance. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## action_feedback: Outcome storage + +The system stores observed results of agent actions as reusable memory. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `verified` | [src_merit_paper](https://arxiv.org/abs/2608.05906) | MERIT stores oracle verified repair corrections and observed unsuccessful directions in an online dual polarity episodic memory. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `partial` | [src_neo4j_agent_memory](https://github.com/neo4j-labs/agent-memory) | Neo4j Agent Memory stores reasoning traces, tool usage, and audit edges linking reasoning steps to entities. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## action_feedback: Outcome linked revision + +An observed action result updates the memory claim that influenced that action. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## action_feedback: Policy revision + +Remembered outcomes modify future agent behavior or policy selection. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `verified` | [src_reflexion_paper](https://arxiv.org/abs/2303.11366) | Reflexion stores verbal reflections derived from task feedback in episodic memory to improve later decision making. | + +## security: Prompt injection resistance + +The system limits memory content from changing agent instructions or tool authority through injection. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## security: Memory poisoning resistance + +The system detects or limits malicious or misleading persistent memories. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## security: Egress control + +The system controls which memory fields can leave the memory boundary for a model or tool. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `verified` | [src_amp_paper](https://proceedings.mlr.press/v317/wu26a.html) | The Agent-Memory Protocol defines deterministic redaction, purpose limited packing, and hydration operations to keep personal identifiers inside the user boundary. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## evaluation: Temporal evaluation + +The system is evaluated on changing facts and historical validity. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `verified` | [src_longmemeval_paper](https://arxiv.org/abs/2410.10813) | LongMemEval evaluates long term memory systems on temporal reasoning and knowledge updates across timestamped interaction histories. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## evaluation: Conflict evaluation + +The system is evaluated on contradictory memories or conflicting claims. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `not_evidenced` | [src_letta_benchmark](https://www.letta.com/blog/letta-leaderboard/) | The reviewed Letta benchmark sources do not establish a dedicated conflict resolution evaluation cell. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## evaluation: Action impact evaluation + +The system is evaluated by whether memory improves downstream agent actions or outcomes. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `partial` | [src_longmemeval_v2_repo](https://github.com/xiaowu0162/LongMemEval-V2) | LongMemEval V2 evaluates whether compact memory evidence improves downstream agent question answering about environment state, workflows, and recurring failure modes. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## evaluation: Continual learning evaluation + +The system is evaluated over an ordered sequence of tasks or episodes where memory must support transfer and forgetting. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `verified` | [src_agent_memory_bench](https://openreview.net/pdf?id=MSXbrNExax) | AgentMemoryBench jointly evaluates system and personal agent memory in a continual learning framework covering online learning, transfer, and forgetting. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `unknown` | | No accepted claim in the corpus. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `unknown` | | No accepted claim in the corpus. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `unknown` | | No accepted claim in the corpus. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## evaluation: Memory security evaluation + +The system or benchmark is evaluated on persistent memory poisoning, prompt injection, downstream consequences, or selective repair. + +| System | Value | Evidence | Claim | +| --- | --- | --- | --- | +| AgeMem | `unknown` | | No accepted claim in the corpus. | +| AgentMemoryBench | `unknown` | | No accepted claim in the corpus. | +| A-MEM | `unknown` | | No accepted claim in the corpus. | +| Agent-Memory Protocol | `unknown` | | No accepted claim in the corpus. | +| Bad Memory | `verified` | [src_bad_memory_paper](https://arxiv.org/abs/2607.14611) | Bad Memory evaluates whether prompt injection payloads planted in persistent memory influence current and future agent sessions. | +| ChronoMem | `unknown` | | No accepted claim in the corpus. | +| Dependency Guided Rollback Repair | `unknown` | | No accepted claim in the corpus. | +| Graphiti | `unknown` | | No accepted claim in the corpus. | +| HippoRAG | `unknown` | | No accepted claim in the corpus. | +| LangMem | `unknown` | | No accepted claim in the corpus. | +| Letta | `unknown` | | No accepted claim in the corpus. | +| LongMemEval | `unknown` | | No accepted claim in the corpus. | +| LongMemEval V2 | `unknown` | | No accepted claim in the corpus. | +| Mem0 | `unknown` | | No accepted claim in the corpus. | +| MemMachine | `unknown` | | No accepted claim in the corpus. | +| Memory R1 | `unknown` | | No accepted claim in the corpus. | +| MemoryBank | `unknown` | | No accepted claim in the corpus. | +| MemSecBench | `verified` | [src_memsecbench_paper](https://arxiv.org/abs/2607.27080) | MemSecBench evaluates the lifecycle of malicious memory from persistence through downstream consequence and selective repair. | +| MERIT | `unknown` | | No accepted claim in the corpus. | +| MindMemOS | `unknown` | | No accepted claim in the corpus. | +| MPBench | `verified` | [src_mpbench_paper](https://arxiv.org/abs/2606.04329) | MPBench evaluates memory poisoning attacks across multiple memory write channels and attack classes. | +| Neo4j Agent Memory | `unknown` | | No accepted claim in the corpus. | +| Quipu | `unknown` | | No accepted claim in the corpus. | +| RE-call | `unknown` | | No accepted claim in the corpus. | +| Reflexion | `unknown` | | No accepted claim in the corpus. | + +## Value legend + +`verified` means primary evidence directly supports the claim. `partial` means the evidence covers a narrower case. `not_evidenced` means the reviewed sources did not establish the capability. `unknown` means research is incomplete. `contradicted` means available evidence conflicts with the claim. `contested` is rendered when accepted claims for one cell conflict. + +## Incomplete or unresolved claims + +* `clm_langmem_revision_001` for `sys_langmem` and `validity_and_revision.contradiction` is `unknown`. +* `clm_letta_eval_001` for `sys_letta` and `evaluation.conflict_resolution` is `not_evidenced`. + +## Conflicting evidence + +No conflicting evidence is recorded. diff --git a/docs/prior_art/generated_summary.json b/docs/prior_art/generated_summary.json new file mode 100644 index 00000000..0988f022 --- /dev/null +++ b/docs/prior_art/generated_summary.json @@ -0,0 +1,446 @@ +{ + "capability_status": { + "action_feedback.outcome_linked_revision": "unverified_gap", + "action_feedback.outcome_storage": "emerging", + "action_feedback.policy_revision": "emerging", + "authority_and_scope.multi_agent_handoff": "unverified_gap", + "authority_and_scope.tenant_scope": "established", + "authority_and_scope.user_consent": "unverified_gap", + "deletion_and_forgetting.derived_propagation": "established", + "deletion_and_forgetting.selective_forgetting": "established", + "deletion_and_forgetting.source_deletion": "emerging", + "evaluation.action_impact": "emerging", + "evaluation.conflict_resolution": "unverified_gap", + "evaluation.continual_learning": "emerging", + "evaluation.memory_security": "established", + "evaluation.temporal_reasoning": "emerging", + "provenance.evidence_receipts": "emerging", + "provenance.source_attribution": "established", + "provenance.transformation_lineage": "emerging", + "representation.knowledge_graph": "established", + "representation.raw_episodes": "established", + "representation.structured_facts": "emerging", + "retrieval.dense": "emerging", + "retrieval.graph_traversal": "established", + "retrieval.lexical": "unverified_gap", + "retrieval.query_time_context": "emerging", + "security.egress_control": "emerging", + "security.memory_poisoning_resistance": "unverified_gap", + "security.prompt_injection_resistance": "unverified_gap", + "time.ingestion_time": "emerging", + "time.point_in_time": "unverified_gap", + "time.valid_time": "established", + "uncertainty.abstention": "emerging", + "uncertainty.calibration": "unverified_gap", + "uncertainty.entailment": "unverified_gap", + "validity_and_revision.contradiction": "emerging", + "validity_and_revision.rollback": "emerging", + "validity_and_revision.supersession": "emerging", + "write_path.agent_authored": "established", + "write_path.consolidation": "established", + "write_path.extraction": "established" + }, + "claim_count": 51, + "review_count": 51, + "schema_version": "1.0", + "search_cutoff_date": "2026-08-24", + "source_count": 29, + "system_count": 25, + "target_combination": { + "capability_ids": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "systems": { + "sys_agemem": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_agent_memory_bench": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_amem": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_amp": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_bad_memory": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_chronomem": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_dependency_rollback": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "partial_combination", + "verified": [ + "deletion_and_forgetting.derived_propagation" + ] + }, + "sys_graphiti": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_hipporag": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_langmem": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_letta": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_longmemeval": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_longmemeval_v2": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_mem0": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_memmachine": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_memory_r1": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_memorybank": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_memsecbench": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_merit": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_mindmemos": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_mpbench": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + }, + "sys_neo4j_agent_memory": { + "conflicts": [], + "missing": [ + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "partial_combination", + "verified": [ + "provenance.source_attribution" + ] + }, + "sys_quipu": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "validity_and_revision.supersession", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "partial_combination", + "verified": [ + "provenance.transformation_lineage", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation" + ] + }, + "sys_recall": { + "conflicts": [], + "missing": [ + "deletion_and_forgetting.derived_propagation", + "action_feedback.outcome_linked_revision" + ], + "partial": [ + "provenance.transformation_lineage" + ], + "status": "partial_combination", + "verified": [ + "provenance.source_attribution", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "uncertainty.abstention" + ] + }, + "sys_reflexion": { + "conflicts": [], + "missing": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ], + "partial": [], + "status": "unverified_combination", + "verified": [] + } + } + }, + "unresolved_claim_ids": [ + "clm_langmem_revision_001", + "clm_letta_eval_001" + ] +} diff --git a/docs/prior_art/report_config.json b/docs/prior_art/report_config.json new file mode 100644 index 00000000..f82b3c31 --- /dev/null +++ b/docs/prior_art/report_config.json @@ -0,0 +1,16 @@ +{ + "schema_version": "1.0", + "report_title": "RE-call prior art capability matrix", + "search_cutoff_date": "2026-08-24", + "research_scope": "Agent memory systems, agent runtimes, benchmarks, privacy, governance, and continual learning.", + "novelty_policy": "The renderer may identify unverified gaps but may not assert first, unique, or never done.", + "target_hypothesis": [ + "provenance.source_attribution", + "provenance.transformation_lineage", + "validity_and_revision.supersession", + "authority_and_scope.tenant_scope", + "deletion_and_forgetting.derived_propagation", + "uncertainty.abstention", + "action_feedback.outcome_linked_revision" + ] +} diff --git a/docs/prior_art/reviews.jsonl b/docs/prior_art/reviews.jsonl new file mode 100644 index 00000000..faab92ff --- /dev/null +++ b/docs/prior_art/reviews.jsonl @@ -0,0 +1,51 @@ +{"review_id":"rev_graphiti_graph_001","claim_id":"clm_graphiti_graph_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"Official documentation explicitly describes temporal context graphs.","conflict_claim_ids":[]} +{"review_id":"rev_graphiti_traversal_001","claim_id":"clm_graphiti_traversal_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"Official documentation explicitly describes graph based retrieval.","conflict_claim_ids":[]} +{"review_id":"rev_graphiti_valid_time_001","claim_id":"clm_graphiti_valid_time_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"Official documentation explicitly describes relation validity windows.","conflict_claim_ids":[]} +{"review_id":"rev_graphiti_ingestion_time_001","claim_id":"clm_graphiti_ingestion_time_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"Official documentation explicitly describes ingestion and invalidation timestamps.","conflict_claim_ids":[]} +{"review_id":"rev_mem0_facts_001","claim_id":"clm_mem0_facts_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"Primary paper explicitly describes structured persistent memory.","conflict_claim_ids":[]} +{"review_id":"rev_mem0_extraction_001","claim_id":"clm_mem0_extraction_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"Primary paper explicitly describes extraction and consolidation.","conflict_claim_ids":[]} +{"review_id":"rev_mem0_graph_001","claim_id":"clm_mem0_graph_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"Primary paper describes a graph based variant, so the claim is partial rather than universal.","conflict_claim_ids":[]} +{"review_id":"rev_mem0_dense_001","claim_id":"clm_mem0_dense_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The paper evaluates retrieval from persistent memory, without overclaiming a specific backend.","conflict_claim_ids":[]} +{"review_id":"rev_letta_raw_001","claim_id":"clm_letta_raw_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The official repository documents persistent memory blocks and archival storage.","conflict_claim_ids":[]} +{"review_id":"rev_letta_agent_authored_001","claim_id":"clm_letta_agent_authored_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"Official benchmark documentation describes agent controlled memory management.","conflict_claim_ids":[]} +{"review_id":"rev_letta_eval_001","claim_id":"clm_letta_eval_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The reviewed benchmark does not establish this specific capability, so it remains not evidenced.","conflict_claim_ids":[]} +{"review_id":"rev_langmem_extraction_001","claim_id":"clm_langmem_extraction_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"Official documentation identifies LangMem as a memory management toolkit.","conflict_claim_ids":[]} +{"review_id":"rev_langmem_revision_001","claim_id":"clm_langmem_revision_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The reviewed documentation does not establish contradiction handling.","conflict_claim_ids":[]} +{"review_id":"rev_recall_source_001","claim_id":"clm_recall_source_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The public README explicitly lists provenance and source metadata on returned evidence.","conflict_claim_ids":[]} +{"review_id":"rev_recall_lineage_001","claim_id":"clm_recall_lineage_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The public reasoning graph design describes supporting chunks and generation identity, but the projection is opt in, so the claim is partial.","conflict_claim_ids":[]} +{"review_id":"rev_recall_abstention_001","claim_id":"clm_recall_abstention_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The public README explicitly documents calibrated abstention.","conflict_claim_ids":[]} +{"review_id":"rev_recall_supersession_001","claim_id":"clm_recall_supersession_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The public README explicitly documents superseded and expired verdicts.","conflict_claim_ids":[]} +{"review_id":"rev_recall_tenant_001","claim_id":"clm_recall_tenant_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The public README explicitly lists tenant IDs and row level security.","conflict_claim_ids":[]} +{"review_id":"rev_amp_egress_001","claim_id":"clm_amp_egress_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper defines redaction, purpose limited packing, and hydration as privacy preserving operations.","conflict_claim_ids":[]} +{"review_id":"rev_neo4j_graph_001","claim_id":"clm_neo4j_graph_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The official repository explicitly describes graph native memory and a knowledge graph.","conflict_claim_ids":[]} +{"review_id":"rev_neo4j_provenance_001","claim_id":"clm_neo4j_provenance_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The official repository explicitly describes provenance tracking for extracted entities.","conflict_claim_ids":[]} +{"review_id":"rev_neo4j_time_001","claim_id":"clm_neo4j_time_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The official repository explicitly describes temporal fact validity.","conflict_claim_ids":[]} +{"review_id":"rev_neo4j_consolidation_001","claim_id":"clm_neo4j_consolidation_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The official repository explicitly lists consolidation and entity deduplication.","conflict_claim_ids":[]} +{"review_id":"rev_neo4j_trace_001","claim_id":"clm_neo4j_trace_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The official repository describes trace and audit storage, but not outcome driven belief revision, so the claim is partial.","conflict_claim_ids":[]} +{"review_id":"rev_quipu_episodes_001","claim_id":"clm_quipu_episodes_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The official repository explicitly describes raw evidence preservation and evidence linked derived facts.","conflict_claim_ids":[]} +{"review_id":"rev_quipu_lineage_001","claim_id":"clm_quipu_lineage_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The official repository explicitly describes evidence backed derivation and provenance inspection.","conflict_claim_ids":[]} +{"review_id":"rev_quipu_time_001","claim_id":"clm_quipu_time_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The official repository explicitly describes temporal supersession and historical values.","conflict_claim_ids":[]} +{"review_id":"rev_quipu_forgetting_001","claim_id":"clm_quipu_forgetting_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The official repository explicitly describes forgetting propagation and closure reports.","conflict_claim_ids":[]} +{"review_id":"rev_quipu_scope_001","claim_id":"clm_quipu_scope_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The official repository explicitly describes scoped retrieval.","conflict_claim_ids":[]} +{"review_id":"rev_memmachine_episodes_001","claim_id":"clm_memmachine_episodes_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly describes storage of entire conversational episodes.","conflict_claim_ids":[]} +{"review_id":"rev_memmachine_context_001","claim_id":"clm_memmachine_context_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly describes contextualized retrieval with surrounding context.","conflict_claim_ids":[]} +{"review_id":"rev_agent_memory_bench_continual_001","claim_id":"clm_agent_memory_bench_continual_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary benchmark paper defines continual evaluation across online learning, transfer, and forgetting.","conflict_claim_ids":[]} +{"review_id":"rev_memorybank_forgetting_001","claim_id":"clm_memorybank_forgetting_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly describes selective forgetting and reinforcement based on elapsed time and significance.","conflict_claim_ids":[]} +{"review_id":"rev_amem_agent_authored_001","claim_id":"clm_amem_agent_authored_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly describes agent driven note construction, linking, and memory evolution.","conflict_claim_ids":[]} +{"review_id":"rev_hipporag_graph_001","claim_id":"clm_hipporag_graph_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly describes knowledge graph and Personalized PageRank retrieval.","conflict_claim_ids":[]} +{"review_id":"rev_longmemeval_temporal_001","claim_id":"clm_longmemeval_temporal_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary benchmark paper explicitly evaluates temporal reasoning and knowledge updates over timestamped histories.","conflict_claim_ids":[]} +{"review_id":"rev_longmemeval_v2_action_001","claim_id":"clm_longmemeval_v2_action_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The official repository documents downstream answer evaluation, but not executed action outcomes, so the claim is partial.","conflict_claim_ids":[]} +{"review_id":"rev_memory_r1_delete_001","claim_id":"clm_memory_r1_delete_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly lists DELETE as a learned operation of the memory manager.","conflict_claim_ids":[]} +{"review_id":"rev_memory_r1_agent_authored_001","claim_id":"clm_memory_r1_agent_authored_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly describes a memory manager selecting structured memory operations.","conflict_claim_ids":[]} +{"review_id":"rev_agemem_agent_authored_001","claim_id":"clm_agemem_agent_authored_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly describes autonomous agent decisions over memory tool actions.","conflict_claim_ids":[]} +{"review_id":"rev_agemem_forgetting_001","claim_id":"clm_agemem_forgetting_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly lists discard as an agent controlled memory operation.","conflict_claim_ids":[]} +{"review_id":"rev_mindmemos_consolidation_001","claim_id":"clm_mindmemos_consolidation_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly describes consolidation through redundancy merging and conflict resolution.","conflict_claim_ids":[]} +{"review_id":"rev_mindmemos_contradiction_001","claim_id":"clm_mindmemos_contradiction_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly identifies conflict resolution during memory consolidation.","conflict_claim_ids":[]} +{"review_id":"rev_chronomem_rollback_001","claim_id":"clm_chronomem_rollback_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly describes version histories and semantic rollback to historical memory states.","conflict_claim_ids":[]} +{"review_id":"rev_dependency_rollback_derived_001","claim_id":"clm_dependency_rollback_derived_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly describes dependency tracing, affected state invalidation, selective replay, and preservation of benign state.","conflict_claim_ids":[]} +{"review_id":"rev_merit_outcome_storage_001","claim_id":"clm_merit_outcome_storage_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly describes memory for finalized corrections and unsuccessful directions.","conflict_claim_ids":[]} +{"review_id":"rev_reflexion_policy_revision_001","claim_id":"clm_reflexion_policy_revision_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly links feedback derived reflections in episodic memory to later decision making.","conflict_claim_ids":[]} +{"review_id":"rev_bad_memory_security_eval_001","claim_id":"clm_bad_memory_security_eval_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly evaluates persistent memory payloads across current and future sessions.","conflict_claim_ids":[]} +{"review_id":"rev_memsecbench_security_eval_001","claim_id":"clm_memsecbench_security_eval_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary benchmark paper explicitly evaluates persistence, downstream consequence, and selective repair across its lifecycle protocol.","conflict_claim_ids":[]} +{"review_id":"rev_mpbench_security_eval_001","claim_id":"clm_mpbench_security_eval_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The primary paper explicitly introduces MPBench for memory poisoning attacks across write channels and attack classes.","conflict_claim_ids":[]} +{"review_id":"rev_quipu_receipts_001","claim_id":"clm_quipu_receipts_001","decision":"accepted","reviewer":"Giulio","reviewed_at":"2026-08-24","reason":"The official repository describes provenance inspection and evidence backed derivation, but the exact serialized receipt schema is not independently established, so the claim is partial.","conflict_claim_ids":[]} diff --git a/docs/prior_art/sources.jsonl b/docs/prior_art/sources.jsonl new file mode 100644 index 00000000..19aa2056 --- /dev/null +++ b/docs/prior_art/sources.jsonl @@ -0,0 +1,29 @@ +{"source_id":"src_graphiti_repository","source_type":"repository","title":"Graphiti repository","authors_or_org":["Zep"],"canonical_url":"https://github.com/getzep/graphiti","published_at":null,"updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Seeded from the existing RE-call prior art review.","supersedes_source_id":null} +{"source_id":"src_graphiti_docs","source_type":"official_docs","title":"Graphiti documentation overview","authors_or_org":["Zep"],"canonical_url":"https://help.getzep.com/graphiti/getting-started/overview","published_at":null,"updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Official documentation for temporal graph behavior.","supersedes_source_id":null} +{"source_id":"src_mem0_repository","source_type":"repository","title":"Mem0 repository","authors_or_org":["Mem0"],"canonical_url":"https://github.com/mem0ai/mem0","published_at":null,"updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Seeded from the existing RE-call prior art review.","supersedes_source_id":null} +{"source_id":"src_mem0_paper","source_type":"paper","title":"Mem0: Building Production-Ready AI Agents with Scalable Long-Term Memory","authors_or_org":["Mem0 authors"],"canonical_url":"https://arxiv.org/abs/2504.19413","published_at":"2025-04-28","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary research paper describing extraction, consolidation, and graph memory.","supersedes_source_id":null} +{"source_id":"src_letta_repository","source_type":"repository","title":"Letta repository","authors_or_org":["Letta"],"canonical_url":"https://github.com/letta-ai/letta","published_at":null,"updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Seeded from the existing RE-call prior art review.","supersedes_source_id":null} +{"source_id":"src_letta_benchmark","source_type":"benchmark","title":"Letta Leaderboard: Benchmarking LLMs on Agentic Memory","authors_or_org":["Letta"],"canonical_url":"https://www.letta.com/blog/letta-leaderboard/","published_at":"2025-05-29","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Official benchmark documentation.","supersedes_source_id":null} +{"source_id":"src_langmem_docs","source_type":"official_docs","title":"LangMem documentation","authors_or_org":["LangChain"],"canonical_url":"https://langchain-ai.github.io/langmem/","published_at":null,"updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Seeded from the existing RE-call prior art review.","supersedes_source_id":null} +{"source_id":"src_recall_readme","source_type":"repository","title":"RE-call README","authors_or_org":["Giulio D'Erme"],"canonical_url":"https://github.com/GiulioDER/RE-call/blob/master/README.md","published_at":null,"updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Public project documentation.","supersedes_source_id":null} +{"source_id":"src_recall_reasoning_graph","source_type":"official_docs","title":"RE-call Reasoning Graph Projection","authors_or_org":["Giulio D'Erme"],"canonical_url":"https://github.com/GiulioDER/RE-call/blob/master/docs/REASONING_GRAPH.md","published_at":null,"updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Public design document for typed evidence graph projection.","supersedes_source_id":null} +{"source_id":"src_amp_paper","source_type":"protocol","title":"Agent-Memory Protocol: A Privacy-Focused Protocol for LLM Agents and User Memory Interaction","authors_or_org":["Junde Wu","Minhao Hu","Jiayuan Zhu","Jiaye Wang","Yueming Jin"],"canonical_url":"https://proceedings.mlr.press/v317/wu26a.html","published_at":"2026-01-20","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary protocol paper identified during the frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_neo4j_agent_memory","source_type":"repository","title":"Neo4j Agent Memory","authors_or_org":["Neo4j Labs"],"canonical_url":"https://github.com/neo4j-labs/agent-memory","published_at":null,"updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Official repository with graph, provenance, temporal, consolidation, and reasoning trace features.","supersedes_source_id":null} +{"source_id":"src_quipu_repository","source_type":"repository","title":"Quipu: Evidence-backed temporal memory for long-horizon AI agents","authors_or_org":["Jeff Hajewski"],"canonical_url":"https://github.com/jeffhajewski/quipu","published_at":null,"updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Official repository identified during the frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_memmachine_paper","source_type":"paper","title":"MemMachine: A Ground-Truth-Preserving Memory System for Personalized AI Agents","authors_or_org":["Shu Wang","Edwin Yu","Oscar Love","Tom Zhang","Tom Wong","Steve Scargall","Charles Fan"],"canonical_url":"https://arxiv.org/abs/2604.04853","published_at":"2026-04-06","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary paper identified during the frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_agent_memory_bench","source_type":"benchmark","title":"Benchmarking Continual Agent Memory for Online Learning, Transfer, and Forgetting","authors_or_org":["Zihang Ma","Jinyi Liu","Hongyao Tang","Yi Ma","Ruitao Wang","Yifu Yuan","Yan Zheng","Jianye Hao"],"canonical_url":"https://openreview.net/pdf?id=MSXbrNExax","published_at":null,"updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"AgentMemoryBench workshop paper identified during the frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_memorybank_paper","source_type":"paper","title":"MemoryBank: Enhancing Large Language Models with Long-Term Memory","authors_or_org":["Wanjun Zhong","Lianghong Guo","Qiqi Gao","He Ye","Yanlin Wang"],"canonical_url":"https://arxiv.org/abs/2305.10250","published_at":"2023-05-17","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary paper identified during the second frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_amem_paper","source_type":"paper","title":"A-MEM: Agentic Memory for LLM Agents","authors_or_org":["Wujiang Xu","Zujie Liang","Kai Mei","Hang Gao","Juntao Tan","Yongfeng Zhang"],"canonical_url":"https://arxiv.org/abs/2502.12110","published_at":"2025-02-17","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary paper identified during the second frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_hipporag_paper","source_type":"paper","title":"HippoRAG: Neurobiologically Inspired Long-Term Memory for Large Language Models","authors_or_org":["Bernal Jiménez Gutiérrez","Yiheng Shu","Yu Gu","Michihiro Yasunaga","Yu Su"],"canonical_url":"https://arxiv.org/abs/2405.14831","published_at":"2024-05-23","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary paper identified during the second frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_longmemeval_paper","source_type":"benchmark","title":"LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory","authors_or_org":["Di Wu","Hongwei Wang","Wenhao Yu","Yuwei Zhang","Kai-Wei Chang","Dong Yu"],"canonical_url":"https://arxiv.org/abs/2410.10813","published_at":"2024-10-14","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary benchmark paper identified during the second frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_longmemeval_v2_repo","source_type":"benchmark","title":"LongMemEval-V2: Evaluating Long-Term Agent Memory Toward Experienced Colleagues","authors_or_org":["Di Wu","Zixiang Ji","Asmi Kawatkar","Bryan Kwan","Jia-Chen Gu","Nanyun Peng","Kai-Wei Chang"],"canonical_url":"https://github.com/xiaowu0162/LongMemEval-V2","published_at":"2026-05-12","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Official benchmark repository identified during the second frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_memory_r1_paper","source_type":"paper","title":"Memory-R1: Enhancing Large Language Model Agents to Manage and Utilize Memories via Reinforcement Learning","authors_or_org":["Sikuan Yan","Xiufeng Yang","Zuchao Huang","Ercong Nie","Zifeng Ding","Zonggen Li","Xiaowen Ma","Hinrich Schütze","Volker Tresp","Yunpu Ma"],"canonical_url":"https://arxiv.org/abs/2508.19828","published_at":"2025-08-27","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary paper identified during the third frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_agemem_paper","source_type":"paper","title":"Agentic Memory: Learning Unified Long-Term and Short-Term Memory Management for Large Language Model Agents","authors_or_org":["Yi Yu","Liuyi Yao","Yuexiang Xie","Qingquan Tan","Jiaqi Feng","Yaliang Li","Libing Wu"],"canonical_url":"https://arxiv.org/abs/2601.01885","published_at":"2026-01-05","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary paper identified during the third frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_mindmemos_paper","source_type":"paper","title":"MindMemOS: A Portable and Self-Evolving Memory Operating Layer for AI Agents","authors_or_org":["Kaichao Liang","Yuqi Cui","Hao Kong","Xinyuan Huang","Guohaotian Hou","Qingcan Kang","Liang Chen","Yiyang Yin","Ke Ye","Jiaquan Guo","Da Chen","Lingan Zeng","Yixing Peng","Rong Yao","Shixiong Kai","Mingxuan Yuan"],"canonical_url":"https://arxiv.org/abs/2608.12428","published_at":"2026-08-12","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary paper identified during the third frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_chronomem_paper","source_type":"paper","title":"ChronoMem: Version Control and Semantic Rollback for Large Language Model Agent Memory","authors_or_org":["Yongye Su","Wujiang Xu","Chaoji Zuo","Elisa Bertino"],"canonical_url":"https://arxiv.org/abs/2607.27773","published_at":"2026-07-30","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary paper identified during the fourth frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_dependency_rollback_paper","source_type":"paper","title":"From Faulty Memories to Corrected Actions: Dependency-Guided Rollback Repair for Memory-Augmented Agents","authors_or_org":["Caili Yu","Yiqi Wang","Jiaqi Zhang","Yiqun Duan","Mingkai Zheng","Zhangkai Wu","Kaize Shi","Taotao Cai"],"canonical_url":"https://arxiv.org/abs/2608.10502","published_at":"2026-08-11","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary paper identified during the fourth frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_merit_paper","source_type":"paper","title":"Causal Episodic Memory for Feedback-Driven Agent Repair","authors_or_org":["Khang Nhat Hoang Vo","Tam Minh Chu","Anh Trac Duc Dinh","Thuyen Vinh Ha Bui","Tho Quan"],"canonical_url":"https://arxiv.org/abs/2608.05906","published_at":"2026-08-06","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary paper identified during the fourth frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_reflexion_paper","source_type":"paper","title":"Reflexion: Language Agents with Verbal Reinforcement Learning","authors_or_org":["Noah Shinn","Federico Cassano","Edward Berman","Ashwin Gopinath","Karthik Narasimhan","Shunyu Yao"],"canonical_url":"https://arxiv.org/abs/2303.11366","published_at":"2023-03-20","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary paper identified during the fourth frontier expansion pass.","supersedes_source_id":null} +{"source_id":"src_bad_memory_paper","source_type":"benchmark","title":"Bad Memory: Evaluating Prompt Injection Risks from Memory in Agentic Systems","authors_or_org":["Soham Gadgil","David Alexander","Sai Sunku","Franziska Roesner"],"canonical_url":"https://arxiv.org/abs/2607.14611","published_at":"2026-07-16","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary security evaluation paper identified during the governance expansion pass.","supersedes_source_id":null} +{"source_id":"src_memsecbench_paper","source_type":"benchmark","title":"MemSecBench: Tracking Agent Memory Poisoning from Persistence to Consequence and Repair","authors_or_org":["Xuanze Chen","Xukang Xie","Wentao Fu","Jiajun Zhou","Shanqing Yu","Qi Xuan"],"canonical_url":"https://arxiv.org/abs/2607.27080","published_at":"2026-07-29","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary security benchmark paper identified during the governance expansion pass.","supersedes_source_id":null} +{"source_id":"src_mpbench_paper","source_type":"benchmark","title":"From Untrusted Input to Trusted Memory: A Systematic Study of Memory Poisoning Attacks in LLM Agents","authors_or_org":["Pritam Dash","Tongyu Ge","Aditi Jain","Tanmay Shah","Zhiwei Shang"],"canonical_url":"https://arxiv.org/abs/2606.04329","published_at":"2026-06-03","updated_at":null,"accessed_at":"2026-08-24","source_tier":"primary","license_or_access":"public","status":"accepted","discovery_notes":"Primary memory poisoning study identified during the governance expansion pass.","supersedes_source_id":null} diff --git a/docs/prior_art/systems.jsonl b/docs/prior_art/systems.jsonl new file mode 100644 index 00000000..b704e710 --- /dev/null +++ b/docs/prior_art/systems.jsonl @@ -0,0 +1,25 @@ +{"system_id":"sys_graphiti","display_name":"Graphiti","organization":"Zep","system_type":"open_source_library","homepage_url":"https://github.com/getzep/graphiti","source_ids":["src_graphiti_repository","src_graphiti_docs"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Temporal knowledge graph framework for agent memory."} +{"system_id":"sys_mem0","display_name":"Mem0","organization":"Mem0","system_type":"open_source_library","homepage_url":"https://github.com/mem0ai/mem0","source_ids":["src_mem0_repository","src_mem0_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Memory layer with extraction, consolidation, and graph based variants."} +{"system_id":"sys_letta","display_name":"Letta","organization":"Letta","system_type":"agent_runtime","homepage_url":"https://github.com/letta-ai/letta","source_ids":["src_letta_repository","src_letta_benchmark"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Stateful agent runtime with agent controlled memory management."} +{"system_id":"sys_langmem","display_name":"LangMem","organization":"LangChain","system_type":"open_source_library","homepage_url":"https://langchain-ai.github.io/langmem/","source_ids":["src_langmem_docs"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Memory management toolkit used with LangGraph applications."} +{"system_id":"sys_recall","display_name":"RE-call","organization":"Giulio D'Erme","system_type":"open_source_library","homepage_url":"https://github.com/GiulioDER/RE-call","source_ids":["src_recall_readme","src_recall_reasoning_graph"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Local retrieval layer with validity, provenance, calibration, and explicit abstention."} +{"system_id":"sys_amp","display_name":"Agent-Memory Protocol","organization":"Wu et al.","system_type":"protocol","homepage_url":"https://proceedings.mlr.press/v317/wu26a.html","source_ids":["src_amp_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Privacy focused protocol for memory interaction at the language and computation boundary."} +{"system_id":"sys_neo4j_agent_memory","display_name":"Neo4j Agent Memory","organization":"Neo4j Labs","system_type":"open_source_library","homepage_url":"https://github.com/neo4j-labs/agent-memory","source_ids":["src_neo4j_agent_memory"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Graph native agent memory with provenance, temporal relations, consolidation, and reasoning traces."} +{"system_id":"sys_quipu","display_name":"Quipu","organization":"Jeff Hajewski","system_type":"open_source_library","homepage_url":"https://github.com/jeffhajewski/quipu","source_ids":["src_quipu_repository"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Local evidence backed temporal memory with provenance aware forgetting."} +{"system_id":"sys_memmachine","display_name":"MemMachine","organization":"MemMachine authors","system_type":"research_system","homepage_url":"https://arxiv.org/abs/2604.04853","source_ids":["src_memmachine_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Ground truth preserving episodic and profile memory with contextualized retrieval."} +{"system_id":"sys_agent_memory_bench","display_name":"AgentMemoryBench","organization":"Ma et al.","system_type":"benchmark","homepage_url":"https://openreview.net/pdf?id=MSXbrNExax","source_ids":["src_agent_memory_bench"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Benchmark for continual system and personal agent memory."} +{"system_id":"sys_memorybank","display_name":"MemoryBank","organization":"MemoryBank authors","system_type":"research_system","homepage_url":"https://arxiv.org/abs/2305.10250","source_ids":["src_memorybank_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Long term memory mechanism with time and significance based memory updates."} +{"system_id":"sys_amem","display_name":"A-MEM","organization":"A-MEM authors","system_type":"research_system","homepage_url":"https://arxiv.org/abs/2502.12110","source_ids":["src_amem_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Agentic memory system with dynamic linking and memory evolution."} +{"system_id":"sys_hipporag","display_name":"HippoRAG","organization":"HippoRAG authors","system_type":"research_system","homepage_url":"https://arxiv.org/abs/2405.14831","source_ids":["src_hipporag_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Knowledge graph and Personalized PageRank retrieval framework for long term memory."} +{"system_id":"sys_longmemeval","display_name":"LongMemEval","organization":"LongMemEval authors","system_type":"benchmark","homepage_url":"https://github.com/xiaowu0162/LongMemEval","source_ids":["src_longmemeval_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Benchmark for extraction, multi session reasoning, temporal reasoning, knowledge updates, and abstention."} +{"system_id":"sys_longmemeval_v2","display_name":"LongMemEval V2","organization":"LongMemEval V2 authors","system_type":"benchmark","homepage_url":"https://github.com/xiaowu0162/LongMemEval-V2","source_ids":["src_longmemeval_v2_repo"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Benchmark for memory supported experience in web and enterprise environments."} +{"system_id":"sys_memory_r1","display_name":"Memory R1","organization":"Memory R1 authors","system_type":"research_system","homepage_url":"https://arxiv.org/abs/2508.19828","source_ids":["src_memory_r1_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Reinforcement learned memory manager with structured add, update, delete, and no operation actions."} +{"system_id":"sys_agemem","display_name":"AgeMem","organization":"AgeMem authors","system_type":"research_system","homepage_url":"https://arxiv.org/abs/2601.01885","source_ids":["src_agemem_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Unified long term and short term memory management integrated into agent policy."} +{"system_id":"sys_mindmemos","display_name":"MindMemOS","organization":"MindMemOS authors","system_type":"research_system","homepage_url":"https://arxiv.org/abs/2608.12428","source_ids":["src_mindmemos_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Portable self evolving memory operating layer with consolidation and conflict resolution."} +{"system_id":"sys_chronomem","display_name":"ChronoMem","organization":"ChronoMem authors","system_type":"research_system","homepage_url":"https://arxiv.org/abs/2607.27773","source_ids":["src_chronomem_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Version control layer for agent memory with semantic rollback and counterfactual evaluation."} +{"system_id":"sys_dependency_rollback","display_name":"Dependency Guided Rollback Repair","organization":"Rollback Repair authors","system_type":"research_system","homepage_url":"https://arxiv.org/abs/2608.10502","source_ids":["src_dependency_rollback_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Dependency guided recovery for faulty memories and affected actions while preserving benign state."} +{"system_id":"sys_merit","display_name":"MERIT","organization":"MERIT authors","system_type":"research_system","homepage_url":"https://arxiv.org/abs/2608.05906","source_ids":["src_merit_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Feedback driven causal episodic memory for repair outcomes and unsuccessful directions."} +{"system_id":"sys_reflexion","display_name":"Reflexion","organization":"Reflexion authors","system_type":"research_system","homepage_url":"https://arxiv.org/abs/2303.11366","source_ids":["src_reflexion_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Verbal reinforcement framework that stores feedback in episodic memory for later decisions."} +{"system_id":"sys_bad_memory","display_name":"Bad Memory","organization":"Bad Memory authors","system_type":"benchmark","homepage_url":"https://arxiv.org/abs/2607.14611","source_ids":["src_bad_memory_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Benchmark for prompt injection payloads planted in persistent agent memory."} +{"system_id":"sys_memsecbench","display_name":"MemSecBench","organization":"MemSecBench authors","system_type":"benchmark","homepage_url":"https://arxiv.org/abs/2607.27080","source_ids":["src_memsecbench_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Lifecycle benchmark covering memory poisoning persistence, execution consequence, and selective repair."} +{"system_id":"sys_mpbench","display_name":"MPBench","organization":"MPBench authors","system_type":"benchmark","homepage_url":"https://arxiv.org/abs/2606.04329","source_ids":["src_mpbench_paper"],"review_status":"accepted","as_of_date":"2026-08-24","notes":"Benchmark and taxonomy for memory poisoning attacks across agent memory write channels."} diff --git a/docs/prior_art/taxonomy.json b/docs/prior_art/taxonomy.json new file mode 100644 index 00000000..158cb663 --- /dev/null +++ b/docs/prior_art/taxonomy.json @@ -0,0 +1,116 @@ +{ + "schema_version": "1.0", + "groups": [ + { + "group_id": "representation", + "name": "Representation", + "capabilities": [ + {"capability_id": "representation.raw_episodes", "name": "Raw episodes", "definition": "The system preserves source episodes or turns as retrievable memory.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe or demonstrate preservation of source episodes."}, + {"capability_id": "representation.structured_facts", "name": "Structured facts", "definition": "The system represents memory as structured facts or attributes rather than only raw text.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe structured fact representation."}, + {"capability_id": "representation.knowledge_graph", "name": "Knowledge graph", "definition": "The system represents entities and relations as a graph that can be queried.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe graph construction or graph retrieval."} + ] + }, + { + "group_id": "write_path", + "name": "Write path", + "capabilities": [ + {"capability_id": "write_path.extraction", "name": "Memory extraction", "definition": "The system extracts persistent memory from conversations or agent traces.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe extraction from an input trace."}, + {"capability_id": "write_path.consolidation", "name": "Memory consolidation", "definition": "The system merges, compresses, or organizes multiple memories during maintenance.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe a consolidation or maintenance operation."}, + {"capability_id": "write_path.agent_authored", "name": "Agent authored memory", "definition": "The agent can deliberately decide what to write or update in persistent memory.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must show agent controlled memory writes."} + ] + }, + { + "group_id": "retrieval", + "name": "Retrieval", + "capabilities": [ + {"capability_id": "retrieval.dense", "name": "Dense retrieval", "definition": "The system retrieves memory using vector similarity.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary implementation or documentation must identify vector retrieval."}, + {"capability_id": "retrieval.lexical", "name": "Lexical retrieval", "definition": "The system retrieves memory using lexical or full text search.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary implementation or documentation must identify lexical retrieval."}, + {"capability_id": "retrieval.graph_traversal", "name": "Graph traversal", "definition": "The system expands retrieval through graph relations or traversals.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe relation traversal during retrieval."}, + {"capability_id": "retrieval.query_time_context", "name": "Query time context construction", "definition": "The system constructs or expands the memory context at query time using the retrieved evidence.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe query time context expansion or construction."} + ] + }, + { + "group_id": "validity_and_revision", + "name": "Validity and revision", + "capabilities": [ + {"capability_id": "validity_and_revision.supersession", "name": "Supersession", "definition": "The system represents that a newer memory replaces an older memory.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe supersession or replacement behavior."}, + {"capability_id": "validity_and_revision.contradiction", "name": "Contradiction handling", "definition": "The system detects or represents conflicting memories.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe conflict detection or representation."}, + {"capability_id": "validity_and_revision.rollback", "name": "Rollback", "definition": "The system can restore or select an earlier memory state.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe restoration of a previous memory state."} + ] + }, + { + "group_id": "time", + "name": "Time", + "capabilities": [ + {"capability_id": "time.valid_time", "name": "Valid time", "definition": "The system records when a fact was true in the world.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must identify fact validity intervals or equivalent semantics."}, + {"capability_id": "time.ingestion_time", "name": "Ingestion time", "definition": "The system records when it learned or ingested a fact.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must identify ingestion or observation timestamps."}, + {"capability_id": "time.point_in_time", "name": "Point in time retrieval", "definition": "The system can answer what was known or valid at a requested historical time.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must demonstrate or document historical time queries."} + ] + }, + { + "group_id": "provenance", + "name": "Provenance", + "capabilities": [ + {"capability_id": "provenance.source_attribution", "name": "Source attribution", "definition": "A returned memory can identify the source evidence supporting it.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must show source or document attribution in the returned memory."}, + {"capability_id": "provenance.transformation_lineage", "name": "Transformation lineage", "definition": "Derived memories link back through transformations to their source evidence.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe links from derived memory to source material."}, + {"capability_id": "provenance.evidence_receipts", "name": "Evidence receipts", "definition": "The system returns a machine readable record of evidence used for a memory result.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must show a structured evidence result or receipt."} + ] + }, + { + "group_id": "uncertainty", + "name": "Uncertainty", + "capabilities": [ + {"capability_id": "uncertainty.abstention", "name": "Abstention", "definition": "The system can return an explicit refusal when memory support is insufficient.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe or demonstrate an explicit abstention result."}, + {"capability_id": "uncertainty.entailment", "name": "Entailment checking", "definition": "The system checks whether retrieved evidence actually supports the requested claim.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe an entailment or support check."}, + {"capability_id": "uncertainty.calibration", "name": "Confidence calibration", "definition": "The system calibrates a trust or confidence decision against labeled examples.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe calibration data or a calibrated decision boundary."} + ] + }, + { + "group_id": "authority_and_scope", + "name": "Authority and scope", + "capabilities": [ + {"capability_id": "authority_and_scope.user_consent", "name": "User consent", "definition": "The system makes persistent memory subject to an explicit user consent or control decision.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe user control over persistence."}, + {"capability_id": "authority_and_scope.tenant_scope", "name": "Tenant scope", "definition": "The system enforces tenant boundaries on memory storage and retrieval.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary implementation or documentation must describe tenant isolation."}, + {"capability_id": "authority_and_scope.multi_agent_handoff", "name": "Multi agent handoff", "definition": "Memory can be intentionally shared or handed from one agent to another with scope semantics.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe cross agent memory sharing or handoff."} + ] + }, + { + "group_id": "deletion_and_forgetting", + "name": "Deletion and forgetting", + "capabilities": [ + {"capability_id": "deletion_and_forgetting.source_deletion", "name": "Source deletion", "definition": "A source memory can be deleted from the system.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe source deletion."}, + {"capability_id": "deletion_and_forgetting.derived_propagation", "name": "Derived deletion propagation", "definition": "Deleting a source removes or invalidates derived memories, graph edges, embeddings, and caches that depend on it.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe deletion across derived artifacts."}, + {"capability_id": "deletion_and_forgetting.selective_forgetting", "name": "Selective forgetting", "definition": "The system can forget selected memories without clearing unrelated memory.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must demonstrate selective deletion or forgetting."} + ] + }, + { + "group_id": "action_feedback", + "name": "Action feedback", + "capabilities": [ + {"capability_id": "action_feedback.outcome_storage", "name": "Outcome storage", "definition": "The system stores observed results of agent actions as reusable memory.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe action outcome persistence."}, + {"capability_id": "action_feedback.outcome_linked_revision", "name": "Outcome linked revision", "definition": "An observed action result updates the memory claim that influenced that action.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe the link between action outcome and later memory behavior."}, + {"capability_id": "action_feedback.policy_revision", "name": "Policy revision", "definition": "Remembered outcomes modify future agent behavior or policy selection.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must demonstrate outcome driven behavior change."} + ] + }, + { + "group_id": "security", + "name": "Security", + "capabilities": [ + {"capability_id": "security.prompt_injection_resistance", "name": "Prompt injection resistance", "definition": "The system limits memory content from changing agent instructions or tool authority through injection.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe an injection defense or evaluation."}, + {"capability_id": "security.memory_poisoning_resistance", "name": "Memory poisoning resistance", "definition": "The system detects or limits malicious or misleading persistent memories.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe poisoning detection or prevention."}, + {"capability_id": "security.egress_control", "name": "Egress control", "definition": "The system controls which memory fields can leave the memory boundary for a model or tool.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must describe purpose or policy aware memory egress."} + ] + }, + { + "group_id": "evaluation", + "name": "Evaluation", + "capabilities": [ + {"capability_id": "evaluation.temporal_reasoning", "name": "Temporal evaluation", "definition": "The system is evaluated on changing facts and historical validity.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must identify a temporal memory evaluation."}, + {"capability_id": "evaluation.conflict_resolution", "name": "Conflict evaluation", "definition": "The system is evaluated on contradictory memories or conflicting claims.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must identify a conflict resolution evaluation."}, + {"capability_id": "evaluation.action_impact", "name": "Action impact evaluation", "definition": "The system is evaluated by whether memory improves downstream agent actions or outcomes.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must identify an outcome or action based memory evaluation."}, + {"capability_id": "evaluation.continual_learning", "name": "Continual learning evaluation", "definition": "The system is evaluated over an ordered sequence of tasks or episodes where memory must support transfer and forgetting.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must identify an online, continual, transfer, or forgetting evaluation."}, + {"capability_id": "evaluation.memory_security", "name": "Memory security evaluation", "definition": "The system or benchmark is evaluated on persistent memory poisoning, prompt injection, downstream consequences, or selective repair.", "allowed_values": ["verified", "partial", "not_evidenced", "contradicted", "unknown"], "minimum_evidence": "Primary evidence must identify a memory security attack, lifecycle evaluation, or repair benchmark."} + ] + } + ] +} diff --git a/tests/test_prior_art_convention.py b/tests/test_prior_art_convention.py new file mode 100644 index 00000000..72b6de5f --- /dev/null +++ b/tests/test_prior_art_convention.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from pathlib import Path + +from tools.prior_art.convention import check_prior_work_declarations + + +def test_prior_work_declaration_accepts_first_module_docstring(tmp_path: Path) -> None: + path = tmp_path / "probe.py" + path.write_text('"""Prior work: [[memory-note]] established the baseline."""\n', encoding="utf-8") + assert check_prior_work_declarations([path]) == [] + + +def test_prior_work_declaration_rejects_missing_docstring(tmp_path: Path) -> None: + path = tmp_path / "probe.py" + path.write_text("VALUE = 1\n", encoding="utf-8") + errors = check_prior_work_declarations([path]) + assert any("must start with a module docstring" in error for error in errors) + + +def test_prior_work_declaration_rejects_wrong_docstring_prefix(tmp_path: Path) -> None: + path = tmp_path / "probe.py" + path.write_text('"""Experiment details."""\n', encoding="utf-8") + errors = check_prior_work_declarations([path]) + assert any("must contain a Prior work:" in error for error in errors) diff --git a/tests/test_prior_art_render.py b/tests/test_prior_art_render.py new file mode 100644 index 00000000..4cf8b582 --- /dev/null +++ b/tests/test_prior_art_render.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from tools.prior_art.loader import load_dataset +from tools.prior_art.render import render_gap_report, render_matrix, render_files, render_summary + + +def test_matrix_render_is_deterministic() -> None: + dataset = load_dataset() + assert render_matrix(dataset) == render_matrix(dataset) + assert render_gap_report(dataset) == render_gap_report(dataset) + + +def test_matrix_contains_uncertainty_states_and_systems() -> None: + dataset = load_dataset() + matrix = render_matrix(dataset) + assert "`unknown`" in matrix + assert "`not_evidenced`" in matrix + assert "Graphiti" in matrix + assert "RE-call" in matrix + assert "## System overview" in matrix + assert "| System | representation | write_path | retrieval |" in matrix + assert "## Incomplete or unresolved claims" in matrix + assert "clm_langmem_revision_001" in matrix + assert "## Conflicting evidence" in matrix + + +def test_summary_contains_unresolved_claims_and_statuses() -> None: + summary = render_summary(load_dataset()) + assert '"unresolved_claim_ids"' in summary + assert '"unverified_gap"' in summary + assert '"target_combination"' in summary + + +def test_gap_report_renders_target_combination_without_uniqueness_claim() -> None: + report = render_gap_report(load_dataset()) + assert "## Target combination analysis" in report + assert "`sys_recall`" in report + assert "`unverified_combination`" in report or "`partial_combination`" in report + assert "not a novelty claim" in report + + +def test_render_check_passes_against_committed_reports() -> None: + dataset = load_dataset() + assert render_files(dataset, check=True) == [] diff --git a/tests/test_prior_art_schema.py b/tests/test_prior_art_schema.py new file mode 100644 index 00000000..b5a3549c --- /dev/null +++ b/tests/test_prior_art_schema.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from datetime import date + +from tools.prior_art.loader import load_dataset +from tools.prior_art.validate import validate_dataset + + +def test_checked_in_prior_art_corpus_is_valid() -> None: + assert validate_dataset(load_dataset(), today=date(2026, 8, 24)) == [] + + +def test_taxonomy_contains_target_hypothesis_capabilities() -> None: + dataset = load_dataset() + capabilities = { + capability["capability_id"] + for group in dataset["taxonomy"]["groups"] + for capability in group["capabilities"] + } + assert { + "provenance.transformation_lineage", + "deletion_and_forgetting.derived_propagation", + "action_feedback.outcome_linked_revision", + } <= capabilities diff --git a/tests/test_prior_art_validation.py b/tests/test_prior_art_validation.py new file mode 100644 index 00000000..9c6f3421 --- /dev/null +++ b/tests/test_prior_art_validation.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from copy import deepcopy +from datetime import date + +from tools.prior_art.loader import load_dataset +from tools.prior_art.validate import validate_dataset + + +def test_verified_claim_requires_primary_source() -> None: + dataset = deepcopy(load_dataset()) + source = next(source for source in dataset["sources"] if source["source_id"] == "src_langmem_docs") + source["source_tier"] = "secondary" + errors = validate_dataset(dataset, today=date(2026, 8, 24)) + assert any("verified claims require a primary source" in error for error in errors) + + +def test_missing_review_is_rejected() -> None: + dataset = deepcopy(load_dataset()) + dataset["claims"][0]["review_id"] = "missing_review" + errors = validate_dataset(dataset, today=date(2026, 8, 24)) + assert any("review_id references a missing review" in error for error in errors) + + +def test_unknown_and_not_evidenced_are_valid_distinct_values() -> None: + dataset = deepcopy(load_dataset()) + values = {claim["value"] for claim in dataset["claims"]} + assert "unknown" in values + assert "not_evidenced" in values + + +def test_future_access_date_is_rejected() -> None: + dataset = deepcopy(load_dataset()) + dataset["sources"][0]["accessed_at"] = "2026-08-25" + errors = validate_dataset(dataset, today=date(2026, 8, 24)) + assert any("accessed_at is in the future" in error for error in errors) + + +def test_long_evidence_excerpt_is_rejected() -> None: + dataset = deepcopy(load_dataset()) + dataset["claims"][0]["evidence_excerpt"] = "word " * 26 + errors = validate_dataset(dataset, today=date(2026, 8, 24)) + assert any("evidence_excerpt exceeds" in error for error in errors) + + +def test_unknown_target_hypothesis_capability_is_rejected() -> None: + dataset = deepcopy(load_dataset()) + dataset["report_config"]["target_hypothesis"].append("missing.capability") + errors = validate_dataset(dataset, today=date(2026, 8, 24)) + assert any("target_hypothesis references a missing capability" in error for error in errors) diff --git a/tools/prior_art/__init__.py b/tools/prior_art/__init__.py new file mode 100644 index 00000000..9078dc6b --- /dev/null +++ b/tools/prior_art/__init__.py @@ -0,0 +1,6 @@ +"""Structured prior art evidence tooling for RE-call.""" + +from .loader import DATA_ROOT, load_dataset +from .validate import validate_dataset + +__all__ = ["DATA_ROOT", "load_dataset", "validate_dataset"] diff --git a/tools/prior_art/__main__.py b/tools/prior_art/__main__.py new file mode 100644 index 00000000..eb53e2f3 --- /dev/null +++ b/tools/prior_art/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/tools/prior_art/cli.py b/tools/prior_art/cli.py new file mode 100644 index 00000000..6ffadf0d --- /dev/null +++ b/tools/prior_art/cli.py @@ -0,0 +1,78 @@ +"""Command line interface for the prior art evidence corpus.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from urllib.request import Request, urlopen + +from .convention import check_prior_work_declarations +from .loader import load_dataset +from .render import render_files +from .validate import validate_dataset + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Validate and render RE-call prior art evidence") + parser.add_argument("--root", type=Path, default=None, help="prior art data directory") + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("validate") + render = subparsers.add_parser("render") + render.add_argument("--check", action="store_true") + subparsers.add_parser("report") + experiments = subparsers.add_parser("check-experiments") + experiments.add_argument("paths", nargs="+", type=Path) + links = subparsers.add_parser("check-links") + links.add_argument("--timeout", type=float, default=10.0) + return parser + + +def _check_links(dataset: dict[str, object], timeout: float) -> int: + failures = 0 + for source in dataset["sources"]: # type: ignore[index] + url = source["canonical_url"] # type: ignore[index] + request = Request(str(url), method="HEAD", headers={"User-Agent": "RE-call-prior-art/1"}) + try: + with urlopen(request, timeout=timeout) as response: + print(f"{response.status} {url}") + except Exception as exc: # pragma: no cover, network is environment dependent + failures += 1 + print(f"FAIL {url}: {exc}", file=sys.stderr) + return 1 if failures else 0 + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + dataset = load_dataset(args.root) if args.root else load_dataset() + if args.command == "validate": + errors = validate_dataset(dataset) + if errors: + print("\n".join(errors), file=sys.stderr) + return 1 + print("prior art corpus valid") + return 0 + if args.command in {"render", "report"}: + try: + changed = render_files(dataset, check=getattr(args, "check", False)) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 1 + if changed: + for path in changed: + print(f"generated file is stale: {path}", file=sys.stderr) + return 1 + print("prior art reports are current") + return 0 + if args.command == "check-experiments": + errors = check_prior_work_declarations(args.paths) + if errors: + print("\n".join(errors), file=sys.stderr) + return 1 + print("prior work declarations are present") + return 0 + return _check_links(dataset, args.timeout) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/prior_art/convention.py b/tools/prior_art/convention.py new file mode 100644 index 00000000..2736e000 --- /dev/null +++ b/tools/prior_art/convention.py @@ -0,0 +1,39 @@ +"""Check prior work declarations on newly added experiment modules.""" + +from __future__ import annotations + +import ast +from pathlib import Path + + +def check_prior_work_declarations(paths: list[Path]) -> list[str]: + """Return errors for Python modules without a first statement prior work declaration.""" + + errors: list[str] = [] + for path in paths: + if path.suffix.lower() != ".py": + errors.append(f"{path} is not a Python experiment module") + continue + try: + source = path.read_text(encoding="utf-8") + except OSError as exc: + errors.append(f"{path} cannot be read: {exc}") + continue + try: + module = ast.parse(source, filename=str(path)) + except SyntaxError as exc: + errors.append(f"{path} has invalid Python syntax: {exc}") + continue + if not module.body or not isinstance(module.body[0], ast.Expr): + errors.append(f"{path} must start with a module docstring containing Prior work:") + continue + value = module.body[0].value + if not isinstance(value, ast.Constant) or not isinstance(value.value, str): + errors.append(f"{path} must start with a module docstring containing Prior work:") + continue + has_declaration = any( + line.strip().startswith("Prior work:") for line in value.value.splitlines() + ) + if not has_declaration: + errors.append(f"{path} module docstring must contain a Prior work: declaration") + return errors diff --git a/tools/prior_art/loader.py b/tools/prior_art/loader.py new file mode 100644 index 00000000..73811437 --- /dev/null +++ b/tools/prior_art/loader.py @@ -0,0 +1,43 @@ +"""Load the checked in prior art evidence corpus.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +DATA_ROOT = REPO_ROOT / "docs" / "prior_art" + + +def _load_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path} must contain a JSON object") + return value + + +def _load_jsonl(path: Path) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + value = json.loads(line) + if not isinstance(value, dict): + raise ValueError(f"{path}:{line_number} must contain a JSON object") + records.append(value) + return records + + +def load_dataset(root: Path = DATA_ROOT) -> dict[str, Any]: + """Load all canonical prior art files from ``root``.""" + + return { + "root": root, + "taxonomy": _load_json(root / "taxonomy.json"), + "report_config": _load_json(root / "report_config.json"), + "sources": _load_jsonl(root / "sources.jsonl"), + "systems": _load_jsonl(root / "systems.jsonl"), + "claims": _load_jsonl(root / "claims.jsonl"), + "reviews": _load_jsonl(root / "reviews.jsonl"), + } diff --git a/tools/prior_art/render.py b/tools/prior_art/render.py new file mode 100644 index 00000000..01d4ecaa --- /dev/null +++ b/tools/prior_art/render.py @@ -0,0 +1,377 @@ +"""Deterministic Markdown rendering for the prior art evidence corpus.""" + +from __future__ import annotations + +from collections import defaultdict +import json +from pathlib import Path +from typing import Any + +from .loader import DATA_ROOT +from .validate import validate_dataset + + +def _capabilities(dataset: dict[str, Any]) -> list[tuple[str, dict[str, Any]]]: + values: list[tuple[str, dict[str, Any]]] = [] + for group in dataset["taxonomy"]["groups"]: + for capability in group["capabilities"]: + values.append((group["group_id"], capability)) + return values + + +def _groups(dataset: dict[str, Any]) -> list[tuple[str, list[str]]]: + return [ + ( + group["group_id"], + [capability["capability_id"] for capability in group["capabilities"]], + ) + for group in dataset["taxonomy"]["groups"] + ] + + +def _cell(claims: list[dict[str, Any]]) -> str: + if not claims: + return "unknown" + values = {claim["value"] for claim in claims} + if "contradicted" in values and ("verified" in values or "partial" in values): + return "contested" + for value in ("verified", "partial", "not_evidenced", "unknown", "contradicted"): + if value in values: + return value + return "unknown" + + +def _group_cell(values: list[str]) -> str: + """Aggregate capability values without turning missing evidence into a positive result.""" + + if any(value in {"contested", "contradicted"} for value in values): + if any(value in {"verified", "partial"} for value in values): + return "contested" + return "contradicted" + if values and all(value == "verified" for value in values): + return "verified" + if any(value in {"verified", "partial"} for value in values): + return "partial" + if values and all(value == "not_evidenced" for value in values): + return "not_evidenced" + return "unknown" + + +def _source_link(source: dict[str, Any]) -> str: + return f"[{source['source_id']}]({source['canonical_url']})" + + +def _target_hypothesis(dataset: dict[str, Any]) -> list[str]: + target = dataset["report_config"].get("target_hypothesis", []) + return [capability_id for capability_id in target if isinstance(capability_id, str)] + + +def _combination_status(values: list[str]) -> str: + if any(value in {"contested", "contradicted"} for value in values): + return "contested_combination" + if values and all(value == "verified" for value in values): + return "verified_combination" + if any(value in {"verified", "partial"} for value in values): + return "partial_combination" + return "unverified_combination" + + +def _combination_rows(dataset: dict[str, Any]) -> list[dict[str, Any]]: + target = _target_hypothesis(dataset) + claims_by_cell: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for claim in dataset["claims"]: + claims_by_cell[(claim["system_id"], claim["capability_id"])].append(claim) + + rows: list[dict[str, Any]] = [] + for system in sorted(dataset["systems"], key=lambda item: item["system_id"]): + values = { + capability_id: _cell(claims_by_cell.get((system["system_id"], capability_id), [])) + for capability_id in target + } + rows.append( + { + "system_id": system["system_id"], + "status": _combination_status(list(values.values())), + "verified": [capability_id for capability_id, value in values.items() if value == "verified"], + "partial": [capability_id for capability_id, value in values.items() if value == "partial"], + "missing": [ + capability_id + for capability_id, value in values.items() + if value in {"unknown", "not_evidenced"} + ], + "conflicts": [ + capability_id + for capability_id, value in values.items() + if value in {"contested", "contradicted"} + ], + } + ) + return rows + + +def _incomplete_claims(dataset: dict[str, Any]) -> list[dict[str, Any]]: + return sorted( + [ + claim + for claim in dataset["claims"] + if claim["value"] in {"unknown", "not_evidenced"} + or claim.get("review_status") != "accepted" + ], + key=lambda claim: claim["claim_id"], + ) + + +def _conflicting_cells(dataset: dict[str, Any]) -> list[tuple[str, str, list[str]]]: + claims_by_cell: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for claim in dataset["claims"]: + claims_by_cell[(claim["system_id"], claim["capability_id"])].append(claim) + conflicts: list[tuple[str, str, list[str]]] = [] + for (system_id, capability_id), claims in sorted(claims_by_cell.items()): + linked_conflict = any( + conflict_id + for review in dataset["reviews"] + if review.get("claim_id") in {claim["claim_id"] for claim in claims} + for conflict_id in review.get("conflict_claim_ids", []) + ) + if _cell(claims) == "contested" or linked_conflict: + conflicts.append((system_id, capability_id, sorted(claim["claim_id"] for claim in claims))) + return conflicts + + +def render_matrix(dataset: dict[str, Any]) -> str: + systems = sorted(dataset["systems"], key=lambda item: item["system_id"]) + sources = {source["source_id"]: source for source in dataset["sources"]} + claims_by_cell: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for claim in dataset["claims"]: + claims_by_cell[(claim["system_id"], claim["capability_id"])].append(claim) + + lines = [ + "# RE-call prior art capability matrix", + "", + "Generated from the canonical evidence records in `docs/prior_art/`.", + "", + "This matrix records evidence, not absolute absence. `unknown` means the investigation is incomplete; `not_evidenced` means the reviewed sources did not establish the capability.", + "", + "## System overview", + "", + "This overview has one row per system and one column per capability group. Group cells aggregate the detailed capability records below; `partial` means the group is not completely evidenced.", + "", + ] + group_columns = [group_id for group_id, _ in _groups(dataset)] + lines.append("| System | " + " | ".join(group_columns) + " |") + lines.append("| --- | " + " | ".join("---" for _ in group_columns) + " |") + for system in systems: + group_values = [] + for _, capability_ids in _groups(dataset): + group_values.append( + _group_cell( + [ + _cell(claims_by_cell.get((system["system_id"], capability_id), [])) + for capability_id in capability_ids + ] + ) + ) + lines.append( + f"| {system['display_name']} | " + + " | ".join(f"`{value}`" for value in group_values) + + " |" + ) + lines.append("") + for group_id, capability in _capabilities(dataset): + capability_id = capability["capability_id"] + lines.extend( + [ + f"## {group_id}: {capability['name']}", + "", + capability["definition"], + "", + "| System | Value | Evidence | Claim |", + "| --- | --- | --- | --- |", + ] + ) + for system in systems: + claims = claims_by_cell.get((system["system_id"], capability_id), []) + value = _cell(claims) + if claims: + evidence = "; ".join( + _source_link(sources[claim["source_id"]]) for claim in sorted(claims, key=lambda item: item["claim_id"]) + ) + claim_text = "
".join(claim["claim_text"] for claim in sorted(claims, key=lambda item: item["claim_id"])) + else: + evidence = "" + claim_text = "No accepted claim in the corpus." + lines.append(f"| {system['display_name']} | `{value}` | {evidence} | {claim_text} |") + lines.append("") + + lines.extend( + [ + "## Value legend", + "", + "`verified` means primary evidence directly supports the claim. `partial` means the evidence covers a narrower case. `not_evidenced` means the reviewed sources did not establish the capability. `unknown` means research is incomplete. `contradicted` means available evidence conflicts with the claim. `contested` is rendered when accepted claims for one cell conflict.", + "", + "## Incomplete or unresolved claims", + "", + ] + ) + incomplete_claims = _incomplete_claims(dataset) + if incomplete_claims: + for claim in incomplete_claims: + lines.append( + f"* `{claim['claim_id']}` for `{claim['system_id']}` and `{claim['capability_id']}` is `{claim['value']}`." + ) + else: + lines.append("No incomplete or unresolved claims are recorded.") + lines.extend(["", "## Conflicting evidence", ""]) + conflicts = _conflicting_cells(dataset) + if conflicts: + for system_id, capability_id, claim_ids in conflicts: + lines.append( + f"* `{system_id}` and `{capability_id}` has conflicting evidence across " + + ", ".join(f"`{claim_id}`" for claim_id in claim_ids) + + "." + ) + else: + lines.append("No conflicting evidence is recorded.") + lines.append("") + return "\n".join(lines).rstrip() + "\n" + + +def _gap_classification(values: list[str]) -> str: + if "contradicted" in values and ("verified" in values or "partial" in values): + return "contested" + if values.count("verified") >= 2: + return "established" + if "verified" in values or "partial" in values: + return "emerging" + return "unverified_gap" + + +def render_gap_report(dataset: dict[str, Any]) -> str: + capabilities = _capabilities(dataset) + claims_by_capability: dict[str, list[dict[str, Any]]] = defaultdict(list) + for claim in dataset["claims"]: + claims_by_capability[claim["capability_id"]].append(claim) + lines = [ + "# RE-call prior art gap report", + "", + "This report identifies research candidates. It does not assert that a capability has never been implemented.", + "", + "## Capability status", + "", + ] + for group_id, capability in capabilities: + claims = claims_by_capability.get(capability["capability_id"], []) + values = [claim["value"] for claim in claims] + classification = _gap_classification(values) + systems = sorted({claim["system_id"] for claim in claims}) + lines.append(f"### {capability['capability_id']}") + lines.append("") + lines.append(f"Group: `{group_id}`. Status: **{classification}**.") + lines.append("") + lines.append(f"Definition: {capability['definition']}") + lines.append("") + if systems: + lines.append("Systems with reviewed claims: " + ", ".join(f"`{system}`" for system in systems) + ".") + else: + lines.append("No reviewed system claim exists for this capability.") + lines.append("") + + lines.extend( + [ + "## RE-call research hypothesis", + "", + "The current hypothesis is a combination of evidence backed claims, explicit validity and supersession, reversible provenance lineage, authority and scope enforcement, deletion propagation through derived artifacts, abstention based on support and conflict, and action outcome feedback into future belief state.", + "", + "The matrix must establish the evidence boundary before this combination is described as novel.", + "", + "## Target combination analysis", + "", + "This section reports coverage of the configured RE-call hypothesis. It is not a novelty claim. A missing cell means that this corpus has not accepted evidence for that capability in that system.", + "", + "Target capabilities: " + ", ".join(f"`{capability_id}`" for capability_id in _target_hypothesis(dataset)) + ".", + "", + "| System | Combination status | Verified support | Partial support | Missing evidence | Conflicting evidence |", + "| --- | --- | --- | --- | --- | --- |", + ] + ) + for row in _combination_rows(dataset): + lines.append( + "| " + + " | ".join( + [ + f"`{row['system_id']}`", + f"`{row['status']}`", + ", ".join(f"`{value}`" for value in row["verified"]) or "none", + ", ".join(f"`{value}`" for value in row["partial"]) or "none", + ", ".join(f"`{value}`" for value in row["missing"]) or "none", + ", ".join(f"`{value}`" for value in row["conflicts"]) or "none", + ] + ) + + "|" + ) + lines.append("") + return "\n".join(lines).rstrip() + "\n" + + +def render_summary(dataset: dict[str, Any]) -> str: + claims_by_capability: dict[str, list[dict[str, Any]]] = defaultdict(list) + for claim in dataset["claims"]: + claims_by_capability[claim["capability_id"]].append(claim) + capability_status: dict[str, str] = {} + for _, capability in _capabilities(dataset): + capability_id = capability["capability_id"] + values = [claim["value"] for claim in claims_by_capability.get(capability_id, [])] + capability_status[capability_id] = _gap_classification(values) + unresolved_claim_ids = sorted( + claim["claim_id"] + for claim in dataset["claims"] + if claim["value"] in {"unknown", "not_evidenced"} + ) + combination = { + row["system_id"]: { + "status": row["status"], + "verified": row["verified"], + "partial": row["partial"], + "missing": row["missing"], + "conflicts": row["conflicts"], + } + for row in _combination_rows(dataset) + } + return json.dumps( + { + "schema_version": "1.0", + "search_cutoff_date": dataset["report_config"]["search_cutoff_date"], + "system_count": len(dataset["systems"]), + "source_count": len(dataset["sources"]), + "claim_count": len(dataset["claims"]), + "review_count": len(dataset["reviews"]), + "unresolved_claim_ids": unresolved_claim_ids, + "capability_status": capability_status, + "target_combination": { + "capability_ids": _target_hypothesis(dataset), + "systems": combination, + }, + }, + indent=2, + sort_keys=True, + ) + "\n" + + +def render_files(dataset: dict[str, Any], *, check: bool = False) -> list[Path]: + errors = validate_dataset(dataset) + if errors: + raise ValueError("prior art corpus is invalid:\n" + "\n".join(f" {error}" for error in errors)) + root = Path(dataset.get("root", DATA_ROOT)) + outputs = { + root / "generated_matrix.md": render_matrix(dataset), + root / "generated_gap_report.md": render_gap_report(dataset), + root / "generated_summary.json": render_summary(dataset), + } + changed: list[Path] = [] + for path, content in outputs.items(): + if check: + if not path.exists() or path.read_text(encoding="utf-8") != content: + changed.append(path) + else: + path.write_text(content, encoding="utf-8", newline="\n") + return changed diff --git a/tools/prior_art/schema.py b/tools/prior_art/schema.py new file mode 100644 index 00000000..fbe4f2f8 --- /dev/null +++ b/tools/prior_art/schema.py @@ -0,0 +1,61 @@ +"""Schema constants and small helpers for the prior art evidence corpus.""" + +from __future__ import annotations + +from datetime import date +from typing import Any, Mapping + +SOURCE_TYPES = frozenset( + { + "paper", + "repository", + "official_docs", + "product_docs", + "benchmark", + "protocol", + "standard", + "survey", + } +) +SOURCE_TIERS = frozenset({"primary", "secondary", "discovery_only"}) +SOURCE_STATUSES = frozenset({"candidate", "accepted", "superseded", "inaccessible"}) +SYSTEM_TYPES = frozenset( + { + "open_source_library", + "agent_runtime", + "commercial_service", + "research_system", + "benchmark", + "protocol", + } +) +REVIEW_STATUSES = frozenset({"draft", "accepted", "rejected", "disputed"}) +CLAIM_VALUES = frozenset( + {"verified", "partial", "not_evidenced", "contradicted", "unknown"} +) +REVIEW_DECISIONS = frozenset({"accepted", "rejected", "disputed"}) +EVIDENCE_TYPES = frozenset({"explicit", "demonstrated", "vendor_claim", "inferred"}) +LOCATOR_KINDS = frozenset({"section", "page", "anchor", "line", "repository_path", "api"}) + + +def as_date(value: Any) -> date | None: + """Parse an ISO date, returning None for an explicit null value.""" + + if value is None: + return None + if not isinstance(value, str): + raise ValueError("date must be an ISO string or null") + return date.fromisoformat(value) + + +def is_mapping(value: Any) -> bool: + return isinstance(value, Mapping) + + +def nonempty_string(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def record_id(record: Mapping[str, Any], field: str) -> str | None: + value = record.get(field) + return value if isinstance(value, str) and value else None diff --git a/tools/prior_art/validate.py b/tools/prior_art/validate.py new file mode 100644 index 00000000..acf3f334 --- /dev/null +++ b/tools/prior_art/validate.py @@ -0,0 +1,273 @@ +"""Fail closed validation for the prior art evidence corpus.""" + +from __future__ import annotations + +from collections import defaultdict +from datetime import date +from typing import Any +from urllib.parse import urlparse + +from .schema import ( + CLAIM_VALUES, + EVIDENCE_TYPES, + LOCATOR_KINDS, + REVIEW_DECISIONS, + REVIEW_STATUSES, + SOURCE_STATUSES, + SOURCE_TIERS, + SOURCE_TYPES, + SYSTEM_TYPES, + as_date, + is_mapping, + nonempty_string, + record_id, +) + +MAX_EVIDENCE_WORDS = 25 + + +def _ids(records: list[dict[str, Any]], field: str, label: str, errors: list[str]) -> set[str]: + values: set[str] = set() + for index, record in enumerate(records): + value = record_id(record, field) + if value is None: + errors.append(f"{label}[{index}] missing nonempty {field}") + continue + if value in values: + errors.append(f"duplicate {label} identifier: {value}") + values.add(value) + return values + + +def _check_date( + value: Any, + location: str, + today: date, + errors: list[str], + *, + required: bool = False, +) -> None: + if value is None: + if required: + errors.append(f"{location} is required") + return + try: + parsed = as_date(value) + except ValueError as exc: + errors.append(f"{location} is invalid: {exc}") + return + if parsed is not None and parsed > today: + errors.append(f"{location} is in the future: {value}") + + +def _check_url(value: Any, location: str, errors: list[str]) -> None: + if not nonempty_string(value): + errors.append(f"{location} must be a nonempty URL") + return + parsed = urlparse(value) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + errors.append(f"{location} must use http or https: {value}") + + +def _taxonomy_capabilities(taxonomy: dict[str, Any], errors: list[str]) -> dict[str, dict[str, Any]]: + groups = taxonomy.get("groups") + if not isinstance(groups, list) or not groups: + errors.append("taxonomy.groups must be a nonempty list") + return {} + capabilities: dict[str, dict[str, Any]] = {} + for group_index, group in enumerate(groups): + if not is_mapping(group): + errors.append(f"taxonomy.groups[{group_index}] must be an object") + continue + group_id = group.get("group_id") + entries = group.get("capabilities") + if not nonempty_string(group_id): + errors.append(f"taxonomy.groups[{group_index}] missing group_id") + if not isinstance(entries, list) or not entries: + errors.append(f"taxonomy group {group_id!r} must have capabilities") + continue + for capability_index, capability in enumerate(entries): + if not is_mapping(capability): + errors.append(f"taxonomy capability {group_id}[{capability_index}] must be an object") + continue + capability_id = capability.get("capability_id") + if not nonempty_string(capability_id): + errors.append(f"taxonomy capability {group_id}[{capability_index}] missing capability_id") + continue + if not capability_id.startswith(f"{group_id}."): + errors.append(f"capability {capability_id} must be namespaced under {group_id}") + if capability_id in capabilities: + errors.append(f"duplicate capability identifier: {capability_id}") + allowed_values = capability.get("allowed_values") + if set(allowed_values or []) != set(CLAIM_VALUES): + errors.append(f"capability {capability_id} must declare all claim values") + for field in ("name", "definition", "minimum_evidence"): + if not nonempty_string(capability.get(field)): + errors.append(f"capability {capability_id} missing {field}") + capabilities[capability_id] = dict(capability) + return capabilities + + +def validate_dataset(dataset: dict[str, Any], today: date | None = None) -> list[str]: + """Return all validation errors. An empty list means the corpus is valid.""" + + today = today or date.today() + errors: list[str] = [] + taxonomy = dataset.get("taxonomy") + if not is_mapping(taxonomy): + errors.append("taxonomy must be an object") + taxonomy = {} + capabilities = _taxonomy_capabilities(dict(taxonomy), errors) + + report_config = dataset.get("report_config") + if not is_mapping(report_config): + errors.append("report_config must be an object") + report_config = {} + target_hypothesis = report_config.get("target_hypothesis", []) + if not isinstance(target_hypothesis, list) or not target_hypothesis: + errors.append("report_config.target_hypothesis must be a nonempty list") + else: + for capability_id in target_hypothesis: + if capability_id not in capabilities: + errors.append( + f"report_config.target_hypothesis references a missing capability {capability_id}" + ) + + sources = dataset.get("sources", []) + systems = dataset.get("systems", []) + claims = dataset.get("claims", []) + reviews = dataset.get("reviews", []) + for label, records in (("sources", sources), ("systems", systems), ("claims", claims), ("reviews", reviews)): + if not isinstance(records, list): + errors.append(f"{label} must be a list") + + if not all(isinstance(records, list) for records in (sources, systems, claims, reviews)): + return errors + + source_ids = _ids(sources, "source_id", "source", errors) + system_ids = _ids(systems, "system_id", "system", errors) + claim_ids = _ids(claims, "claim_id", "claim", errors) + review_ids = _ids(reviews, "review_id", "review", errors) + + source_by_id = {record.get("source_id"): record for record in sources} + review_by_claim: dict[str, list[dict[str, Any]]] = defaultdict(list) + + for index, source in enumerate(sources): + prefix = f"sources[{index}]" + if source.get("source_type") not in SOURCE_TYPES: + errors.append(f"{prefix}.source_type is invalid") + if source.get("source_tier") not in SOURCE_TIERS: + errors.append(f"{prefix}.source_tier is invalid") + if source.get("status") not in SOURCE_STATUSES: + errors.append(f"{prefix}.status is invalid") + for field in ("title", "canonical_url", "accessed_at"): + if field != "canonical_url" and not nonempty_string(source.get(field)): + errors.append(f"{prefix}.{field} must be nonempty") + _check_url(source.get("canonical_url"), f"{prefix}.canonical_url", errors) + _check_date(source.get("published_at"), f"{prefix}.published_at", today, errors) + _check_date(source.get("updated_at"), f"{prefix}.updated_at", today, errors) + _check_date(source.get("accessed_at"), f"{prefix}.accessed_at", today, errors, required=True) + if source.get("supersedes_source_id") is not None and source.get("supersedes_source_id") not in source_ids: + errors.append(f"{prefix}.supersedes_source_id does not exist") + + for index, system in enumerate(systems): + prefix = f"systems[{index}]" + if system.get("system_type") not in SYSTEM_TYPES: + errors.append(f"{prefix}.system_type is invalid") + if system.get("review_status") not in REVIEW_STATUSES: + errors.append(f"{prefix}.review_status is invalid") + for field in ("display_name", "organization", "homepage_url", "as_of_date"): + if not nonempty_string(system.get(field)): + errors.append(f"{prefix}.{field} must be nonempty") + _check_url(system.get("homepage_url"), f"{prefix}.homepage_url", errors) + _check_date(system.get("as_of_date"), f"{prefix}.as_of_date", today, errors, required=True) + source_refs = system.get("source_ids") + if not isinstance(source_refs, list) or not source_refs: + errors.append(f"{prefix}.source_ids must be a nonempty list") + else: + for source_id in source_refs: + if source_id not in source_ids: + errors.append(f"{prefix}.source_ids references missing source {source_id}") + + claims_by_cell: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for index, claim in enumerate(claims): + prefix = f"claims[{index}]" + claim_id = claim.get("claim_id") + if claim_id not in claim_ids: + errors.append(f"{prefix}.claim_id is invalid") + if claim.get("system_id") not in system_ids: + errors.append(f"{prefix}.system_id references a missing system") + capability_id = claim.get("capability_id") + if capability_id not in capabilities: + errors.append(f"{prefix}.capability_id references a missing capability") + if claim.get("value") not in CLAIM_VALUES: + errors.append(f"{prefix}.value is invalid") + if claim.get("review_status") not in REVIEW_STATUSES: + errors.append(f"{prefix}.review_status is invalid") + if claim.get("evidence_type") not in EVIDENCE_TYPES: + errors.append(f"{prefix}.evidence_type is invalid") + if not nonempty_string(claim.get("claim_text")): + errors.append(f"{prefix}.claim_text must be nonempty") + source_id = claim.get("source_id") + if source_id not in source_ids: + errors.append(f"{prefix}.source_id references a missing source") + elif claim.get("value") == "verified" and source_by_id[source_id].get("source_tier") != "primary": + errors.append(f"{prefix}.verified claims require a primary source") + locator = claim.get("evidence_locator") + if not is_mapping(locator): + errors.append(f"{prefix}.evidence_locator must be an object") + else: + if locator.get("kind") not in LOCATOR_KINDS: + errors.append(f"{prefix}.evidence_locator.kind is invalid") + if not nonempty_string(locator.get("value")): + errors.append(f"{prefix}.evidence_locator.value must be nonempty") + excerpt = claim.get("evidence_excerpt") + if excerpt is not None: + if not isinstance(excerpt, str): + errors.append(f"{prefix}.evidence_excerpt must be a string or null") + elif len(excerpt.split()) > MAX_EVIDENCE_WORDS: + errors.append(f"{prefix}.evidence_excerpt exceeds {MAX_EVIDENCE_WORDS} words") + review_id = claim.get("review_id") + if review_id not in review_ids: + errors.append(f"{prefix}.review_id references a missing review") + claims_by_cell[(claim.get("system_id"), capability_id)].append(claim) + + for index, review in enumerate(reviews): + prefix = f"reviews[{index}]" + claim_id = review.get("claim_id") + if claim_id not in claim_ids: + errors.append(f"{prefix}.claim_id references a missing claim") + if review.get("decision") not in REVIEW_DECISIONS: + errors.append(f"{prefix}.decision is invalid") + if not nonempty_string(review.get("reviewer")): + errors.append(f"{prefix}.reviewer must be nonempty") + if not nonempty_string(review.get("reason")): + errors.append(f"{prefix}.reason must be nonempty") + _check_date(review.get("reviewed_at"), f"{prefix}.reviewed_at", today, errors, required=True) + conflict_ids = review.get("conflict_claim_ids", []) + if not isinstance(conflict_ids, list): + errors.append(f"{prefix}.conflict_claim_ids must be a list") + else: + for conflict_id in conflict_ids: + if conflict_id not in claim_ids: + errors.append(f"{prefix}.conflict_claim_ids references a missing claim") + review_by_claim[claim_id].append(review) + + for claim_id, claim_reviews in review_by_claim.items(): + if len(claim_reviews) > 1: + errors.append(f"claim {claim_id} has multiple review records") + + for (system_id, capability_id), cell_claims in claims_by_cell.items(): + if len(cell_claims) <= 1: + continue + claim_ids_in_cell = {claim["claim_id"] for claim in cell_claims} + linked = set() + for claim in cell_claims: + for review in review_by_claim.get(claim["claim_id"], []): + linked.update(review.get("conflict_claim_ids", [])) + if not claim_ids_in_cell.issubset(linked | {next(iter(claim_ids_in_cell))}): + errors.append( + f"duplicate claims for {system_id}/{capability_id} require explicit conflict_claim_ids" + ) + + return errors