From 25c8facd2f9525846abc581ad74ed9be87502c3f Mon Sep 17 00:00:00 2001 From: shaohang <15062640367@163.com> Date: Tue, 20 Oct 2026 18:08:08 +0800 Subject: [PATCH 1/2] feat(evaluation): measure scoped recall lineage occupancy Refs oceanbase/powercontext#1977. --- evaluation/memory/longmemeval_v2/README.md | 5 + .../longmemeval_v2/docs/lineage-occupancy.md | 131 ++++ .../docs/lineage-occupancy.zh.md | 55 ++ .../powercontext_eval_longmemeval_v2/cli.py | 52 ++ .../lineage.py | 244 ++++++++ .../lineage_fixtures.py | 164 +++++ .../lineage_run.py | 563 ++++++++++++++++++ .../longmemeval_v2/tests/test_lineage.py | 161 +++++ .../longmemeval_v2/tests/test_lineage_run.py | 115 ++++ 9 files changed, 1490 insertions(+) create mode 100644 evaluation/memory/longmemeval_v2/docs/lineage-occupancy.md create mode 100644 evaluation/memory/longmemeval_v2/docs/lineage-occupancy.zh.md create mode 100644 evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage.py create mode 100644 evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_fixtures.py create mode 100644 evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_run.py create mode 100644 evaluation/memory/longmemeval_v2/tests/test_lineage.py create mode 100644 evaluation/memory/longmemeval_v2/tests/test_lineage_run.py diff --git a/evaluation/memory/longmemeval_v2/README.md b/evaluation/memory/longmemeval_v2/README.md index 1f27cb77a4..45f0d82902 100644 --- a/evaluation/memory/longmemeval_v2/README.md +++ b/evaluation/memory/longmemeval_v2/README.md @@ -3,6 +3,11 @@ Source, data locks, configuration, documentation, and tests for the LongMemEval-V2 suite live here. The suite is installed through the shared `evaluation/` environment and provides the independent `longmemeval-v2` command. +The evaluation-only `lineage-fixture` command measures Scope-local lineage concentration, +near-duplicate occupancy and final Context fact coverage using disposable native backends. +See [the fixture and report contract](docs/lineage-occupancy.md) / +[中文说明](docs/lineage-occupancy.zh.md) for prerequisites, commands and interpretation. + ## LongMemEval-V2 smoke workload The LongMemEval-V2 commands run a fixed ten-question smoke subset end to end: preflight diff --git a/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.md b/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.md new file mode 100644 index 0000000000..837421ef84 --- /dev/null +++ b/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.md @@ -0,0 +1,131 @@ +# Scope-local lineage and near-duplicate occupancy + +`longmemeval-v2 lineage-fixture` implements the evaluation-only fixture mode for +issue #1977. It uses real native Atomic Memory FTS, exact scoped revision/evidence +reads, and the in-process `PreparedContextBuilder`. It does not exercise HTTP/MCP +or a host's delivery workflow. No Runtime configuration, public schema, ranking, +write behavior, lifecycle policy, or Context Receipt is added. + +## Prerequisites and execution + +The loaded Runtime must include the deterministic FTS baseline from #1976 and +the exact historical Source-evidence API from #1975. A missing `evidence` API +refuses execution before fixture writes. It is never replaced with inferred +trust, Source names, current registration, or adapter audit correlations. + +Install the evaluation suite and the pinned `powercontext[builtin]` source in +the same Python environment, then run: + +```bash +longmemeval-v2 lineage-fixture \ + --backend sqlite \ + --runtime-revision EXACT_40_CHARACTER_RUNTIME_COMMIT \ + --dependency-state merged \ + --output-dir /path/outside/repository/new-lineage-report +``` + +Use `--dependency-state unmerged` when executing a pinned dependency snapshot +that has not merged. This state remains visible in the manifest, report and +decision. A snapshot run is useful local evidence, not merged-main acceptance. +The command records digests of the loaded Runtime and evaluation source trees; +the declared commit must identify the installed source. Existing report +directories are refused. The SQLite database is temporary, allocated beside the +report directory on the selected output filesystem, and removed on exit. + +For OceanBase, configure `POWERCONTEXT_TEST_OCEANBASE_URL` and pass +`--backend oceanbase`. The account needs database creation/deletion privileges. +Only a newly created `pc_lineage_` database is populated and deleted; the +configured database is never populated or deleted. Credentials, connection +strings, machine paths, entry text, Source text, individual content digests and +exact entry/Source/Scope identifiers are not exported. Backend failures must not +be reported as successful or equivalent SQLite execution. + +## Workload and measurement + +`scope-local-lineage-fixtures-v1` fixes small, separate Scope partitions for: + +- byte-identical concentration and a less-represented relevant fact; +- paraphrases grounded in shared evidence; +- distinct facts that share a Source and must all remain recallable; +- revised and historical entries with exact retained revision reads; +- recurring automation with distinct event facts; +- entries without Source evidence; +- all six mandatory CJK forms from #1976, literal English and negative controls; +- held-out shared-source facts and held-out byte-budget pressure. + +Oracles are declared synthetic fact markers, not model decisions or Source +authority claims. Held-out cases do not tune lexical thresholds. Each query +retrieves one immutable, post-admission RRF pool of at most 16 entries, with no +reranker or model. All alternatives reuse that pool and its recorded order. +Source and Artifact bodies/annotations are read under the same native Scope +boundary. Exact historical reads and before/after authority snapshots verify +that measurement does not alter retained revisions, lineage or state. + +The versioned grouping keeps exact Artifact versions and declared Artifact +edges separate from exact Source materialization identity. A Source identity +includes its Scope, type, ID, digest and historical Definition manifest binding; +equal text or equal digests across different Sources do not merge identities. +Missing evidence stays unknown. Shared evidence measures concentration only; +it is not proof of factual duplication or a reason to discard an entry. + +Three registered fixture arms are evaluated: + +| Arm | Measurement | +| --- | --- | +| `lineage-baseline-v1` | Original pool and native final Context build. | +| `lineage-exact-collapse-measurement-v1` | First `(kind, exact UTF-8 text)` representative per equality class, projected only in memory into the same Builder. The original pool is unchanged. | +| `lineage-lexical-annotations-v1` | NFC/casefold/whitespace-normalized character-trigram Jaccard annotations at threshold 0.75. No suppression, reordering or different Context input. | + +Equality projection is a counterfactual measurement, not a replacement pool or +a production deduper. No arm admits an entry from outside the original pool. +Jev is optional and absent here: manifests explicitly record `not_provided` and +no authority. No model call is made in this fixture suite. + +Pool and delivered-origin aggregates include top-k Source/Artifact concentration, +exact redundant slots, lexical-neighbour occupancy, unique evidence/facts, +unknown evidence and neutral declarations. Delivered-origin fact counts describe +selected records; actual answer recall is measured against the rendered Context +bytes, so truncation cannot be mistaken for successful factual delivery. +For held-out fixtures, disjoint declared facts in a lexical-neighbour pair count +as a false relation annotation, independently of actual suppression. + +Task outcomes separately record retrieval misses, final omitted facts, explicit +byte-budget drops/truncations, entry-limit omissions, baseline facts/evidence +lost by an alternative, task regressions, and negative-control false hits. +Latency separates retrieval, exact-evidence reads, and Builder work. Model calls +and model cost are zero. Counts sum across each Scope's queries; concentration, +occupancy and recall ratios are means over applicable samples. No per-entry +diagnostic or cross-Scope identifiers are exported. + +## Artifacts and comparisons + +One output directory contains the pinned baseline manifest, per-arm manifests, +aggregate `report.json` and `report.md`, `backend-matrix.json`, a path-free replay +command, `decision.md`, and artifact hashes. Fixture and source digests, backend +capabilities, dependency state, Python/library versions and all algorithm knobs +are comparison conditions. Timing is observed output, not a pinned input. + +```bash +longmemeval-v2 lineage-compare \ + --baseline /reports/lineage-baseline-v1.manifest.json \ + --alternative /reports/lineage-exact-collapse-measurement-v1.manifest.json +``` + +Comparisons reject missing manifests, unknown arms, changed knobs, and every +manifest difference outside `experiment_arm`, including newly introduced fields. +SQLite and OceanBase are reported separately; their manifests cannot be compared +as if backend differences were an evaluation arm. An unexecuted backend is +explicitly `not_run`. The report's recommendation is **no product change**: +bounded synthetic results alone do not justify production suppression or ranking. + +## Validation + +```bash +python -m pytest -c evaluation/pyproject.toml evaluation/memory/longmemeval_v2/tests -q +ruff check evaluation/memory/longmemeval_v2/src evaluation/memory/longmemeval_v2/tests +``` + +Run native acceptance with the pinned evidence-capable Runtime. On an older +Runtime, the prerequisite-refusal test executes and the evidence-capable SQLite +acceptance is explicitly skipped. This does not establish backend acceptance. +Keep report directories and raw local replay evidence outside Git. diff --git a/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.zh.md b/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.zh.md new file mode 100644 index 0000000000..7e52511d45 --- /dev/null +++ b/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.zh.md @@ -0,0 +1,55 @@ +# Scope 内谱系集中度与近似重复占比评测 + +`longmemeval-v2 lineage-fixture` 提供 #1977 的独立 fixture 模式，使用真实 Atomic Memory FTS、精确历史证据读取和进程内 `PreparedContextBuilder`。它不修改生产排序、写入、生命周期、公共 API 或交付契约，不产生 Context Receipt，也不代表 HTTP/MCP 或宿主交付验收。 + +## 前置条件与运行 + +加载的 Runtime 必须包含 #1976 的确定性 FTS 基线和 #1975 的精确 Source 证据读取接口。缺少接口时，在写入 fixture 前明确拒绝运行，不根据正文、Source 名称、当前注册配置或 adapter 审计关联猜测历史证据与可信度。 + +在同一 Python 环境安装评测包和固定的 `powercontext[builtin]` 源码后执行： + +```bash +longmemeval-v2 lineage-fixture \ + --backend sqlite \ + --runtime-revision 固定的40位Runtime提交SHA \ + --dependency-state merged \ + --output-dir /仓库外/新的谱系报告目录 +``` + +验证未合并的依赖快照时使用 `--dependency-state unmerged`。该状态进入 manifest、报告和决策记录，不能解释为主分支验收通过。命令记录实际加载的 Runtime 和评测源码摘要；提交 SHA 应对应安装的源码。已有输出目录不会被覆盖。SQLite 在报告目录所在文件系统创建临时数据库，退出后清理。 + +OceanBase 使用 `--backend oceanbase`，通过 `POWERCONTEXT_TEST_OCEANBASE_URL` 提供测试连接。账号需有创建和删除数据库的权限。只写入并清理本次创建的 `pc_lineage_` 数据库，不写入或删除连接配置中的原数据库。连接字符串、凭据、机器路径、条目和 Source 正文、单条内容摘要及精确 entry/Source/Scope 标识不会导出。 + +## 固定场景与指标 + +`scope-local-lineage-fixtures-v1` 包括独立 Scope 中的完全相同条目、同证据改写、同 Source 不同事实、修订与历史事实、周期自动来源、未知证据、#1976 的六种中文正例及英文和无关查询对照。另有不参与阈值调整的共享来源与字节预算保留验证案例。 + +事实答案由合成 fixture 显式声明，不是模型判断或可信度推断。每个查询只取一次最多 16 条的准入后 RRF 候选序列，所有实验复用其成员和顺序。精确历史读取及评测前后的权威记录快照用于验证内容、谱系和状态没有被评测修改。 + +谱系分组保留精确 Artifact 版本及其声明的前驱/证据边，Source 分组使用 Scope、类型、ID、material digest 和历史 Definition manifest 绑定。不同 Source 即使正文或摘要相同也不会合并。没有安全证据时明确记为未知。共享来源只是集中度观察，不代表事实重复。 + +| 实验组 | 行为 | +| --- | --- | +| `lineage-baseline-v1` | 原始候选池与原生最终 Context 构建。 | +| `lineage-exact-collapse-measurement-v1` | 按 kind 与精确 UTF-8 正文保留首个代表，仅在内存测量投影上调用相同 Builder，原池保持不变。 | +| `lineage-lexical-annotations-v1` | NFC、大小写及空白规范化后的字符三元组 Jaccard 标注，阈值 0.75；不抑制、不重排。 | + +Jev 不是必需依赖，本模式明确记录为 `not_provided`，不作为权威。模型调用和模型费用均为零。 + +候选池和最终选中来源分别统计 top-k 集中度、精确冗余槽位、词汇邻近占比、独立证据及事实数、未知证据和中性声明。选中条目的事实数不等同于实际交付事实数：答案召回直接检查最终渲染字节，避免将截断后的内容当作完整交付。 + +检索漏召回、关系不确定、预算丢弃/截断、条数上限遗漏、相对基线的事实与证据损失、任务退化、负例误命中分别记录。耗时拆分为检索、精确证据读取和构建。每个类别对应一个 Scope，只导出该 Scope 查询的聚合计数和比率，不提供逐条诊断或跨 Scope 标识。 + +## 报告与比较 + +单个目录包含固定 manifest、各实验组 manifest、聚合 JSON/Markdown 报告、后端能力矩阵、无机器路径的重跑命令、Markdown 决策记录和文件摘要。比较条件包括 fixture 与源码摘要、后端能力、依赖状态、Python/库版本及算法参数。 + +```bash +longmemeval-v2 lineage-compare \ + --baseline /报告/lineage-baseline-v1.manifest.json \ + --alternative /报告/lineage-exact-collapse-measurement-v1.manifest.json +``` + +除了声明的实验组，任何 manifest 差异都会被拒绝，包括后端、未知新增字段和缺失条件。SQLite 与 OceanBase 分开执行、分开报告，未执行的后端标记 `not_run`。本评测建议保持产品行为：合成小规模结果不足以批准生产抑制或重排。 + +验证命令及完整指标语义见 [英文说明](lineage-occupancy.md)。在缺少 #1975 的 Runtime 上，只能验证拒绝行为；证据能力验收会显式跳过，不能据此宣称后端通过。 diff --git a/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/cli.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/cli.py index bab17f8a35..19b7a30c91 100644 --- a/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/cli.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/cli.py @@ -75,6 +75,58 @@ app = typer.Typer(no_args_is_help=True, help="Pinned LongMemEval-V2 evaluation.") +@app.command("lineage-fixture") +def lineage_fixture( + output_dir: Annotated[Path, typer.Option("--output-dir")], + runtime_revision: Annotated[str, typer.Option("--runtime-revision")], + backend: Annotated[str, typer.Option("--backend")] = "sqlite", + dependency_state: Annotated[str, typer.Option("--dependency-state")] = "unknown", + oceanbase_url_env: Annotated[str, typer.Option("--oceanbase-url-env")] = "POWERCONTEXT_TEST_OCEANBASE_URL", +) -> None: + """Measure lineage/occupancy on disposable native-backend fixtures, without models.""" + import asyncio + + from powercontext_eval_longmemeval_v2.lineage import LineageEvaluationError + from powercontext_eval_longmemeval_v2.lineage_run import run_lineage_fixture + + try: + result = asyncio.run( + run_lineage_fixture( + output_dir=output_dir, + backend=backend, + runtime_revision=runtime_revision, + dependency_state=dependency_state, + oceanbase_url_env=oceanbase_url_env, + ) + ) + except LineageEvaluationError as error: + typer.echo(f"Lineage evaluation refused: {error}", err=True) + raise typer.Exit(code=1) from None + typer.echo( + json.dumps({"report": str(result.report_path), "manifest": str(result.manifest_path)}, ensure_ascii=False) + ) + + +@app.command("lineage-compare") +def lineage_compare( + baseline: Annotated[Path, typer.Option("--baseline")], + alternative: Annotated[Path, typer.Option("--alternative")], +) -> None: + """Validate two pinned fixture manifests before interpreting an arm comparison.""" + from powercontext_eval_longmemeval_v2.lineage import LineageEvaluationError, ensure_comparable_lineage_runs + + try: + first = json.loads(baseline.read_text(encoding="utf-8")) + second = json.loads(alternative.read_text(encoding="utf-8")) + if not isinstance(first, dict) or not isinstance(second, dict): + raise LineageEvaluationError("manifest must be a JSON object") + ensure_comparable_lineage_runs(first, second) + except (OSError, ValueError, LineageEvaluationError): + typer.echo("Lineage comparison refused: invalid or incompatible manifests", err=True) + raise typer.Exit(code=1) from None + typer.echo("Comparable: only the declared evaluation arm differs.") + + def _parse_price_policy_option(value: str | None, *, label: str) -> ModelPricePolicy | None: """Parse an explicitly passed JSON price policy, or keep costs unconfigured as null.""" diff --git a/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage.py new file mode 100644 index 0000000000..175068af53 --- /dev/null +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage.py @@ -0,0 +1,244 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bounded, content-free measurements over an immutable, Scope-local recall pool. + +This module has no Runtime writes, model calls, or product ranking policy. Exact +evidence identity and lexical similarity are deliberately separate observations. +""" + +from __future__ import annotations + +import copy +import unicodedata +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from itertools import combinations + +from powercontext_eval_longmemeval_v2.errors import LongMemEvalError + +GROUPING_VERSION = "exact-scoped-evidence-v1" +LEXICAL_VERSION = "unicode-character-trigram-jaccard-v1" +LEXICAL_THRESHOLD = 0.75 +MANIFEST_SCHEMA = "powercontext.lineage-evaluation-manifest.v1" +ARMS = ("lineage-baseline-v1", "lineage-exact-collapse-measurement-v1", "lineage-lexical-annotations-v1") +MAX_POOL = 16 + + +class LineageEvaluationError(LongMemEvalError): + """An evaluation cannot preserve its evidence, isolation, or replay contract.""" + + +@dataclass(frozen=True, order=True) +class EntryRef: + scope: str + artifact: str + revision: int + family: str = "atomic-memory" + + +@dataclass(frozen=True) +class Material: + scope: str + source_type: str + source_id: str + digest: str + definition: str + definition_version: str + fingerprint: str | None + declaration_version: str + authority: str + verification: str + + @property + def identity(self) -> tuple[object, ...]: + # Never coalesce different Source identities merely because bodies hash equally. + return ( + self.scope, + self.source_type, + self.source_id, + self.digest, + self.definition, + self.definition_version, + self.fingerprint, + ) + + +@dataclass(frozen=True) +class Observation: + ref: EntryRef + kind: str + text: str + sources: tuple[Material, ...] = () + artifacts: tuple[EntryRef, ...] = () + facts: frozenset[str] = frozenset() + + +def validate_pool(pool: Sequence[Observation]) -> None: + if len(pool) > MAX_POOL: + raise LineageEvaluationError("candidate pool exceeds the fixed evaluation bound") + if len({entry.ref for entry in pool}) != len(pool): + raise LineageEvaluationError("candidate pool repeats an exact entry revision") + scopes = {entry.ref.scope for entry in pool} + if len(scopes) > 1: + raise LineageEvaluationError("a recall pool must belong to one authorized Scope") + for entry in pool: + if entry.ref.revision < 1 or any(ref.scope != entry.ref.scope for ref in entry.artifacts): + raise LineageEvaluationError("invalid or cross-Scope Artifact evidence") + for source in entry.sources: + if source.scope != entry.ref.scope or not source.digest or not source.declaration_version: + raise LineageEvaluationError("unbound or cross-Scope Source materialization") + + +def exact_representatives(pool: Sequence[Observation]) -> tuple[int, ...]: + """Stable equality projection for measurement; the original pool remains intact.""" + validate_pool(pool) + seen: set[tuple[str, bytes]] = set() + kept = [] + for index, entry in enumerate(pool): + key = (entry.kind, entry.text.encode("utf-8")) + if key not in seen: + kept.append(index) + seen.add(key) + return tuple(kept) + + +def _shingles(text: str) -> frozenset[str]: + value = " ".join(unicodedata.normalize("NFC", text).casefold().split()) + return frozenset(value[index : index + 3] for index in range(len(value) - 2)) + + +def lexical_pairs(pool: Sequence[Observation]) -> tuple[tuple[int, int], ...]: + """Annotate lexical neighbours; this is neither semantic truth nor lineage.""" + validate_pool(pool) + shingles = tuple(_shingles(entry.text) for entry in pool) + pairs = [] + for left, right in combinations(range(len(pool)), 2): + if pool[left].text == pool[right].text or pool[left].kind != pool[right].kind: + continue + a, b = shingles[left], shingles[right] + if a and b and len(a & b) / len(a | b) >= LEXICAL_THRESHOLD: + pairs.append((left, right)) + return tuple(pairs) + + +def measure_pool(pool: Sequence[Observation], *, top_k: int = 8) -> dict[str, object]: + """Export only counts and ratios, with unknown evidence kept explicit.""" + validate_pool(pool) + top = pool[:top_k] + material_counts: Counter[tuple[object, ...]] = Counter() + artifact_counts: Counter[tuple[str, str]] = Counter() + exact_counts: Counter[tuple[str, bytes]] = Counter() + referenced_artifacts: Counter[EntryRef] = Counter() + parent: dict[EntryRef, EntryRef] = {} + + def root(ref: EntryRef) -> EntryRef: + while ref in parent: + ref = parent[ref] + return ref + + # Connect only exact declared predecessor/evidence references. These components + # describe observed lineage, not an equivalence relation between factual claims. + for entry in pool: + for ref in entry.artifacts: + left, right = root(entry.ref), root(ref) + if left != right: + parent[max(left, right)] = min(left, right) + components = Counter(root(entry.ref) for entry in top) + known = 0 + unknown_declarations = 0 + for entry in top: + identities = {material.identity for material in entry.sources} + material_counts.update(identities) + known += bool(identities) + unknown_declarations += sum( + material.authority == "untrusted" or material.verification == "unknown" for material in entry.sources + ) + artifact_counts[(entry.ref.scope, entry.ref.artifact)] += 1 + referenced_artifacts.update(set(entry.artifacts)) + exact_counts[(entry.kind, entry.text.encode("utf-8"))] += 1 + neighbours = lexical_pairs(pool) + annotated = {index for pair in neighbours for index in pair} + false_pairs = sum( + bool(pool[a].facts and pool[b].facts) and not (pool[a].facts & pool[b].facts) for a, b in neighbours + ) + unknown_pairs = sum(not pool[a].facts or not pool[b].facts for a, b in neighbours) + facts = set().union(*(entry.facts for entry in pool)) if pool else set() + return { + "candidate_count": len(pool), + "top_k_count": len(top), + "same_source_top_k_count": max(material_counts.values(), default=0), + "same_source_top_k_concentration": max(material_counts.values(), default=0) / len(top) if top else None, + "same_entry_lineage_top_k_count": max(artifact_counts.values(), default=0), + "exact_lineage_component_top_k_count": max(components.values(), default=0), + "exact_lineage_top_k_concentration": max(components.values(), default=0) / len(top) if top else None, + "shared_exact_artifact_top_k_count": max(referenced_artifacts.values(), default=0), + "unknown_source_evidence_count": len(top) - known, + "neutral_or_unknown_declaration_count": unknown_declarations, + "exact_redundant_top_k_slots": sum(count - 1 for count in exact_counts.values()), + "exact_redundant_top_k_occupancy": sum(count - 1 for count in exact_counts.values()) / len(top) + if top + else None, + "lexical_neighbour_pairs": len(neighbours), + "lexical_neighbour_slots": len(annotated), + "lexical_neighbour_occupancy": len(annotated) / len(pool) if pool else None, + "lexical_false_relation_pairs": false_pairs, + "lexical_unknown_relation_pairs": unknown_pairs, + "advisory_unverified_relation_pairs": len(neighbours), + "unique_evidence_count": len({source.identity for entry in pool for source in entry.sources}), + "unique_fact_count": len(facts), + } + + +def ensure_comparable_lineage_runs(first: Mapping[str, object], second: Mapping[str, object]) -> None: + """Refuse every undeclared manifest difference, including newly added fields.""" + if any( + value.get("schema") != MANIFEST_SCHEMA or value.get("experiment_arm") not in ARMS for value in (first, second) + ): + raise LineageEvaluationError("unknown lineage manifest schema or experiment arm") + required = { + "fixture_version", + "fixture_sha256", + "runtime_revision", + "runtime_source_sha256", + "evaluation_source_sha256", + "backend", + "capabilities", + "dependency_state", + "grouping_version", + "lexical_version", + "lexical_threshold", + "candidate_limit", + "top_k", + "model", + "advisory_jev", + "environment", + } + for value in (first, second): + if not required <= value.keys() or any(value[key] is None for key in required - {"model"}): + raise LineageEvaluationError("lineage manifest lacks pinned comparison conditions") + if ( + value["grouping_version"] != GROUPING_VERSION + or value["lexical_version"] != LEXICAL_VERSION + or value["candidate_limit"] != MAX_POOL + or value["top_k"] != 8 + or value["lexical_threshold"] != LEXICAL_THRESHOLD + ): + raise LineageEvaluationError("unregistered evaluation policy configuration") + a, b = copy.deepcopy(dict(first)), copy.deepcopy(dict(second)) + a.pop("experiment_arm") + b.pop("experiment_arm") + if a != b: + raise LineageEvaluationError("run manifests differ outside the declared evaluation arm") diff --git a/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_fixtures.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_fixtures.py new file mode 100644 index 0000000000..b697a87927 --- /dev/null +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_fixtures.py @@ -0,0 +1,164 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Versioned synthetic oracles, never inferred from Source trust or model text.""" + +from __future__ import annotations + +from dataclasses import dataclass + +FIXTURE_VERSION = "scope-local-lineage-fixtures-v1" + + +@dataclass(frozen=True) +class Entry: + key: str + text: str + facts: tuple[str, ...] + source: str | None = "evidence" + previous: str | None = None + + +@dataclass(frozen=True) +class Query: + text: str + answers: tuple[str, ...] + category: str = "literal" + max_bytes: int = 8000 + + +@dataclass(frozen=True) +class Case: + category: str + entries: tuple[Entry, ...] + queries: tuple[Query, ...] + held_out: bool = False + + +def fixture_cases() -> tuple[Case, ...]: + """All cases fit the native 16-candidate bound; held-out thresholds never tune it.""" + repeated = "Atlas deployment endpoint is PORT8000." + wrappers = ( + "PowerContext Server 地址", + "PowerContext Server 地址是", + "PowerContext Server 地址是什么", + "PowerContext Server 地址是什么？", + "的 PowerContext Server 地址是什么？", + "我之前让你记住的 PowerContext Server 地址是什么？", + ) + return ( + Case( + "exact_concentration", + ( + *(Entry(f"copy-{i:02}", repeated, ("PORT8000",)) for i in range(9)), + Entry("region", "Atlas deployment region is REGIONWEST.", ("REGIONWEST",), "region"), + Entry("owner", "Atlas deployment owner is TEAMBLUE.", ("TEAMBLUE",), "owner"), + ), + (Query("Atlas deployment", ("PORT8000", "REGIONWEST", "TEAMBLUE")),), + ), + Case( + "common_evidence_paraphrases", + ( + Entry("one", "Atlas deployment endpoint uses PORT8000.", ("PORT8000",)), + Entry("two", "Atlas deployment endpoint runs on PORT8000.", ("PORT8000",)), + Entry("three", "Atlas deployment endpoint listens at PORT8000.", ("PORT8000",)), + Entry("four", "Atlas deployment endpoint is PORT8000.", ("PORT8000",)), + ), + (Query("Atlas deployment endpoint", ("PORT8000",)),), + ), + Case( + "legitimate_shared_source", + ( + Entry("port", repeated, ("PORT8000",)), + Entry("region", "Atlas deployment region is REGIONWEST.", ("REGIONWEST",)), + Entry("owner", "Atlas deployment owner is TEAMBLUE.", ("TEAMBLUE",)), + ), + (Query("Atlas deployment", ("PORT8000", "REGIONWEST", "TEAMBLUE")),), + ), + Case( + "revision_history", + ( + Entry("endpoint", "Atlas deployment endpoint was PORT7000.", ("PORT7000",), "old"), + Entry("endpoint", repeated, ("PORT8000",), "new"), + Entry( + "audit", "Atlas deployment historical endpoint was PORT7000.", ("PORT7000",), "old", "endpoint:1" + ), + ), + ( + Query("Atlas deployment", ("PORT8000", "PORT7000")), + Query("Atlas deployment endpoint", ("PORT8000",), "current_revision"), + ), + ), + Case( + "recurring_automation", + tuple( + Entry( + f"cycle-{i}", + f"Atlas deployment automation reported healthy at CYCLE{i}.", + (f"CYCLE{i}",), + f"cycle-{i}", + ) + for i in range(6) + ), + (Query("Atlas deployment automation", tuple(f"CYCLE{i}" for i in range(6))),), + ), + Case( + "unknown_evidence", + ( + Entry("one", "Atlas deployment unknown entry has UNKNOWNONE.", ("UNKNOWNONE",), None), + Entry("two", "Atlas deployment unknown entry has UNKNOWNTWO.", ("UNKNOWNTWO",), None), + ), + (Query("Atlas deployment unknown", ("UNKNOWNONE", "UNKNOWNTWO")),), + ), + Case( + "query_form_baseline", + ( + Entry( + "server", + "本项目的 PowerContext Server 地址固定在 127.0.0.1:8000；测试标记 SERVER8000。", + ("SERVER8000",), + ), + ), + ( + *(Query(text, ("SERVER8000",), "cjk_positive") for text in wrappers), + Query("PowerContext Server", ("SERVER8000",), "english_literal"), + Query("量子香蕉烘焙配方", (), "negative"), + Query("Where do I connect to the service?", (), "english_no_literal_evidence"), + ), + ), + Case( + "held_out_shared_source", + ( + Entry("read", "Atlas deployment read endpoint is READ8100.", ("READ8100",)), + Entry("write", "Atlas deployment write endpoint is WRITE8200.", ("WRITE8200",)), + Entry("archive", "Atlas deployment archive endpoint is ARCHIVE8300.", ("ARCHIVE8300",)), + ), + (Query("Atlas deployment endpoint", ("READ8100", "WRITE8200", "ARCHIVE8300")),), + True, + ), + Case( + "held_out_byte_budget", + tuple( + Entry( + f"item-{i}", + f"Atlas deployment evidence {i}: " + "bounded detail " * 30 + f" ANSWER{i}.", + (f"ANSWER{i}",), + f"item-{i}", + ) + for i in range(5) + ), + (Query("Atlas deployment evidence", tuple(f"ANSWER{i}" for i in range(5)), max_bytes=1000),), + True, + ), + ) diff --git a/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_run.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_run.py new file mode 100644 index 0000000000..19a37c05bb --- /dev/null +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_run.py @@ -0,0 +1,563 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Real disposable-backend fixture replay with aggregate-only report artifacts.""" + +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +import os +import platform +import sqlite3 +import time +from collections import defaultdict +from collections.abc import AsyncIterator, Mapping +from contextlib import asynccontextmanager +from dataclasses import asdict, dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any +from uuid import uuid4 + +from powercontext_eval_longmemeval_v2.lineage import ( + ARMS, + GROUPING_VERSION, + LEXICAL_THRESHOLD, + LEXICAL_VERSION, + MANIFEST_SCHEMA, + EntryRef, + LineageEvaluationError, + Material, + Observation, + ensure_comparable_lineage_runs, + exact_representatives, + measure_pool, +) +from powercontext_eval_longmemeval_v2.lineage_fixtures import FIXTURE_VERSION, Case, fixture_cases + + +def _canonical(value: object) -> bytes: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def _digest(value: object) -> str: + return hashlib.sha256(_canonical(value)).hexdigest() + + +def _tree_digest(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + if path.is_file() and path.suffix in {".py", ".json", ".txt", ".yaml", ".toml"}: + digest.update(path.relative_to(root).as_posix().encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +@dataclass(frozen=True) +class LineageRun: + output_dir: Path + report_path: Path + manifest_path: Path + + +@asynccontextmanager +async def _database(backend: str, directory: Path, oceanbase_url_env: str) -> AsyncIterator[Any]: + from powercontext.builtin.persistence.sqlite import SQLiteConfig + + if backend == "sqlite": + yield SQLiteConfig(url=f"sqlite+aiosqlite:///{directory / 'fixture.db'}") + return + if backend != "oceanbase": + raise LineageEvaluationError("supported backends are sqlite and oceanbase") + configured = os.environ.get(oceanbase_url_env) + if not configured: + raise LineageEvaluationError("OceanBase is not configured; no backend execution was performed") + from pydantic import SecretStr + from sqlalchemy.engine import make_url + + from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile + + # Never reuse or delete the configured database. Only a newly created fixture DB is owned here. + name = f"pc_lineage_{uuid4().hex}" + async with OceanBaseProfile.open(OceanBaseConfig(url=SecretStr(configured)), tables=()) as profile: + async with profile.database.transaction() as connection: + await connection.exec_driver_sql(f"CREATE DATABASE `{name}`") + try: + url = make_url(configured).set(database=name).render_as_string(hide_password=False) + yield OceanBaseConfig(url=SecretStr(url)) + finally: + async with profile.database.transaction() as connection: + await connection.exec_driver_sql(f"DROP DATABASE `{name}`") + + +async def _seed(contexts: Any, case: Case) -> dict[EntryRef, tuple[str, ...]]: + from powercontext.artifacts import ArtifactLineage, ArtifactRef + from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent + from powercontext.builtin.sources.content import ContentSource + from powercontext.sources import SourceMaterialization, SourceRef + + scope = f"lineage-fixture-{case.category}" + context = await contexts.get(scope) + for source in sorted({entry.source for entry in case.entries if entry.source is not None}): + body = "\n".join(entry.text for entry in case.entries if entry.source == source) + await context.sources.add( + ContentSource(name=source, materialization=SourceMaterialization.CAPTURED, content=body) + ) + revisions: dict[str, int] = {} + facts = {} + service = contexts.atomic_memory.service + for entry in case.entries: + previous = revisions.get(entry.key) + evidence = () + if entry.previous: + key, revision = entry.previous.rsplit(":", 1) + evidence = (ArtifactRef(family="atomic-memory", artifact_id=key, revision=int(revision)),) + lineage = ArtifactLineage( + sources=() if entry.source is None else (SourceRef(source_type="content", source_id=entry.source),), + artifacts=evidence, + ) + async with contexts.database.transaction() as connection: + plan = await service.inspect_change( + connection, + scope, + entry.key, + AtomicMemoryContent(kind="fact", text=entry.text), + None, + expected_revision=previous, + lineage=lineage, + ) + prepared = await service.prepare_change(plan) + async with contexts.database.transaction() as connection: + result = await service.commit(connection, prepared, None) + revision = result.primary.ref.revision + revisions[entry.key] = revision + facts[EntryRef(scope, entry.key, revision)] = entry.facts + return facts + + +async def _observe(memory: Any, hit: Any, scope: str, facts: Mapping[EntryRef, tuple[str, ...]]) -> Observation: + ref = hit.hit.artifact_ref + key = EntryRef(scope, ref.artifact_id, ref.revision) + if key not in facts: + raise LineageEvaluationError("search returned an entry outside the authorized fixture Scope") + record = await memory.get(ref.artifact_id, revision=ref.revision) + annotation = await memory.evidence(ref.artifact_id, revision=ref.revision) + if ( + annotation.scope_id != scope + or annotation.artifact_ref != ref + or record.ref != ref + or record.artifact.content.text != hit.text + ): + raise LineageEvaluationError("evidence does not bind to the exact authorized entry revision") + materials = tuple( + Material( + scope, + value.source_ref.source_type, + value.source_ref.source_id, + value.material_digest, + value.definition_name, + value.definition_version, + value.manifest_fingerprint, + value.declaration.declaration_version, + value.declaration.authority, + value.declaration.verification, + ) + for value in annotation.sources + ) + return Observation( + key, + record.artifact.content.kind, + hit.text, + materials, + tuple( + EntryRef(scope, value.artifact_id, value.revision, value.family) + for value in record.artifact.lineage.artifacts + ), + frozenset(facts[key]), + ) + + +def _outcome( + pool: tuple[Observation, ...], content: str | None, answers: tuple[str, ...] +) -> dict[str, int | float | None]: + expected = set(answers) + retrieved = set().union(*(entry.facts for entry in pool)) if pool else set() + delivered = {answer for answer in expected if content is not None and answer in content} + return { + "required_facts": len(expected), + "retrieved_required_facts": len(expected & retrieved), + "delivered_required_facts": len(delivered), + "retrieval_missed_facts": len(expected - retrieved), + "post_retrieval_omitted_facts": len((expected & retrieved) - delivered), + "task_success": int(expected <= delivered and (bool(expected) or not pool)), + "negative_control_false_hits": int(not expected and bool(pool)), + "retrieval_recall": len(expected & retrieved) / len(expected) if expected else None, + "delivered_recall": len(delivered) / len(expected) if expected else None, + } + + +def _aggregate(samples: list[dict[str, Any]]) -> list[dict[str, Any]]: + grouped: dict[tuple[str, str, bool], list[dict[str, Any]]] = defaultdict(list) + for sample in samples: + grouped[(sample["case_category"], sample["arm"], sample["held_out"])].append(sample) + rows = [] + for (category, arm, held_out), group in sorted(grouped.items()): + row: dict[str, Any] = { + "case_category": category, + "experiment_arm": arm, + "held_out": held_out, + "samples": len(group), + } + for section in ("pool", "delivered", "outcome", "omissions", "comparison", "timing_ms"): + keys = set().union(*(sample[section] for sample in group)) + values = {} + for key in sorted(keys): + numbers = [sample[section][key] for sample in group if sample[section][key] is not None] + if key.endswith(("concentration", "occupancy", "_recall")): + values[f"mean_{key}"] = sum(numbers) / len(numbers) if numbers else None + else: + values[key] = sum(numbers) if numbers else None + row[section] = values + rows.append(row) + return rows + + +async def _replay(config: Any, cases: tuple[Case, ...]) -> tuple[list[dict[str, Any]], dict[str, object]]: + from powercontext.artifacts import ArtifactAddress + from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts + from powercontext.builtin.runtime.models import PrepareContextRequest + from powercontext.builtin.runtime.prepared_context import PreparedContextBuilder + + rows = [] + async with open_builtin_contexts(BuiltinConfig(database=config)) as contexts: + index = contexts.atomic_memory.index + if not index.capabilities.fts: + raise LineageEvaluationError("backend has no FTS capability; no semantic equivalence is assumed") + if not callable(getattr(contexts.atomic_memory.for_scope("capability-probe"), "evidence", None)): + raise LineageEvaluationError("requires #1975 exact Source evidence API; no inferred replacement is allowed") + facts = {} + for case in cases: + facts.update(await _seed(contexts, case)) + + # Read exact historical revisions before/after measurement, including all superseded bodies. + async def authority_snapshot() -> tuple[str, ...]: + values = [] + for ref in sorted(facts): + memory = contexts.atomic_memory.for_scope(ref.scope) + record = await memory.get(ref.artifact, revision=ref.revision) + evidence = await memory.evidence(ref.artifact, revision=ref.revision) + values.append(record.model_dump_json() + evidence.model_dump_json()) + return tuple(values) + + before = await authority_snapshot() + builder = PreparedContextBuilder() + for case in cases: + scope = f"lineage-fixture-{case.category}" + memory = contexts.atomic_memory.for_scope(scope) + for query in case.queries: + started = time.perf_counter_ns() + page = await memory.search(query.text, mode="text", limit=builder.memory_candidate_limit) + search_ms = (time.perf_counter_ns() - started) / 1_000_000 + started = time.perf_counter_ns() + pool = tuple([await _observe(memory, hit, scope, facts) for hit in page.hits]) + evidence_ms = (time.perf_counter_ns() - started) / 1_000_000 + started = time.perf_counter_ns() + pool_metrics = measure_pool(pool) + pool_measurement_ms = (time.perf_counter_ns() - started) / 1_000_000 + baseline = None + baseline_pool = tuple(hit.hit.artifact_ref for hit in page.hits) + for arm in ARMS: + started = time.perf_counter_ns() + indices = exact_representatives(pool) if arm == ARMS[1] else tuple(range(len(pool))) + projected = tuple(pool[index] for index in indices) + hits = tuple(page.hits[index] for index in indices) + built = builder.build_result( + request=PrepareContextRequest(query=query.text, max_bytes=query.max_bytes), + scope_id=scope, + hits=hits, + ) + build_ms = (time.perf_counter_ns() - started) / 1_000_000 + selected = set() + for origin in built.origins: + if isinstance(origin, ArtifactAddress) and origin.scope_id != scope: + raise LineageEvaluationError("Builder returned a foreign Scope origin") + ref = origin.artifact if isinstance(origin, ArtifactAddress) else origin + selected.add((ref.artifact_id, ref.revision)) + delivered = tuple( + entry for entry in projected if (entry.ref.artifact, entry.ref.revision) in selected + ) + started = time.perf_counter_ns() + delivered_metrics = measure_pool(delivered) + measurement_ms = pool_measurement_ms + (time.perf_counter_ns() - started) / 1_000_000 + outcome = _outcome(pool, built.context.content, query.answers) + if query.category == "cjk_positive" and outcome["retrieval_missed_facts"]: + raise LineageEvaluationError( + "mandatory #1976 positive baseline failed; diversity comparison refused" + ) + if baseline is None: + baseline = (outcome, built.context.content, delivered) + base_outcome, base_content, base_delivered = baseline + lost = sum( + answer in (base_content or "") and answer not in (built.context.content or "") + for answer in query.answers + ) + base_evidence = {source.identity for entry in base_delivered for source in entry.sources} + delivered_evidence = {source.identity for entry in delivered for source in entry.sources} + rows.append( + { + "case_category": case.category, + "held_out": case.held_out, + "arm": arm, + "pool": pool_metrics, + "delivered": delivered_metrics, + "outcome": outcome, + "omissions": { + **asdict(built.omissions), + "content_bytes": built.context.content_bytes, + "equality_measurement_excluded_candidates": len(pool) - len(projected), + "entry_limit_omitted_candidates": len(projected) + - len(delivered) + - built.omissions.dropped_items, + }, + "comparison": { + "false_suppressed_baseline_facts": lost, + "lost_baseline_evidence": len(base_evidence - delivered_evidence), + "task_regressions": int( + (outcome["task_success"] or 0) < (base_outcome["task_success"] or 0) + ), + "task_improvements": int( + (outcome["task_success"] or 0) > (base_outcome["task_success"] or 0) + ), + }, + "timing_ms": { + "retrieval": search_ms, + "evidence": evidence_ms, + "build": build_ms, + "measurement": measurement_ms, + }, + } + ) + if tuple(hit.hit.artifact_ref for hit in page.hits) != baseline_pool: + raise LineageEvaluationError("an alternative mutated fixed RRF pool membership or order") + if await authority_snapshot() != before: + raise LineageEvaluationError("evaluation mutated authoritative fixture records") + return rows, { + "fts": True, + "vector": bool(index.capabilities.vector), + "hybrid": bool(index.capabilities.hybrid), + "exact_source_evidence_api": True, + "historical_revision_reads": True, + "historical_revisions_checked": len(facts), + "authoritative_records_unchanged": True, + "production_ranking_changed": False, + "context_receipts_emitted": 0, + "runtime_events_emitted": 0, + "measurement_path": "native-scoped-search-and-in-process-builder", + } + + +async def run_lineage_fixture( + *, + output_dir: Path, + backend: str, + runtime_revision: str, + dependency_state: str = "unknown", + oceanbase_url_env: str = "POWERCONTEXT_TEST_OCEANBASE_URL", +) -> LineageRun: + """Execute bounded fixtures, never writing entry/Source content to a report.""" + if output_dir.exists(): + raise LineageEvaluationError("refusing to overwrite an existing report directory") + if dependency_state not in {"merged", "unmerged", "unknown"}: + raise LineageEvaluationError("dependency state must be merged, unmerged, or unknown") + if len(runtime_revision) != 40 or any(char not in "0123456789abcdef" for char in runtime_revision): + raise LineageEvaluationError("runtime revision must be an exact lowercase Git SHA") + try: + import powercontext + except ImportError: + raise LineageEvaluationError( + "install the pinned powercontext[builtin] runtime in the evaluation environment" + ) from None + cases = fixture_cases() + from sqlalchemy.exc import SQLAlchemyError + + try: + output_dir.parent.mkdir(parents=True, exist_ok=True) + with TemporaryDirectory(prefix="pc-lineage-", dir=output_dir.parent) as directory: + async with _database(backend, Path(directory), oceanbase_url_env) as config: + samples, capabilities = await _replay(config, cases) + except LineageEvaluationError: + raise + except (SQLAlchemyError, OSError, ValueError): + # Provider values and URLs are not reportable diagnostic data. Preserve cancellation. + raise LineageEvaluationError("backend fixture execution failed; no successful report was produced") from None + package_root = Path(__file__).parent + runtime_root = Path(powercontext.__file__).parent + manifest = { + "schema": MANIFEST_SCHEMA, + "fixture_version": FIXTURE_VERSION, + "fixture_sha256": _digest([asdict(case) for case in cases]), + "runtime_revision": runtime_revision, + "runtime_source_sha256": _tree_digest(runtime_root), + "evaluation_source_sha256": _tree_digest(package_root), + "backend": backend, + "capabilities": capabilities, + "dependency_state": dependency_state, + "grouping_version": GROUPING_VERSION, + "lexical_version": LEXICAL_VERSION, + "lexical_threshold": LEXICAL_THRESHOLD, + "candidate_limit": 16, + "top_k": 8, + "model": None, + "advisory_jev": {"status": "not_provided", "authority": "none"}, + "environment": { + "python": platform.python_version(), + "platform": platform.system(), + "sqlite": sqlite3.sqlite_version, + "packages": {name: importlib.metadata.version(name) for name in ("SQLAlchemy", "jieba", "pydantic")}, + }, + "experiment_arm": ARMS[0], + } + manifests = [{**manifest, "experiment_arm": arm} for arm in ARMS] + for value in manifests[1:]: + ensure_comparable_lineage_runs(manifests[0], value) + aggregates = _aggregate(samples) + backend_matrix = [ + {"backend": backend, "status": "executed", "capabilities": capabilities}, + { + "backend": "oceanbase" if backend == "sqlite" else "sqlite", + "status": "not_run", + "reason": "environment_not_configured" + if backend == "sqlite" and not os.environ.get(oceanbase_url_env) + else "separate_backend_run_required", + }, + ] + report = { + "schema": "powercontext.lineage-evaluation-report.v1", + "classification": "bounded-synthetic-fixture-evaluation", + "recommendation": "no_product_change", + "dependency_state": dependency_state, + "workload": { + "scope_partitions": len(cases), + "queries": sum(len(case.queries) for case in cases), + "held_out_partitions": sum(case.held_out for case in cases), + }, + "backend_matrix": backend_matrix, + "aggregates": aggregates, + "findings": { + "sampled_lexical_false_relation_pairs": sum( + row["pool"]["lexical_false_relation_pairs"] for row in aggregates if row["experiment_arm"] == ARMS[0] + ), + "equality_measurement_task_improvements": sum( + row["comparison"]["task_improvements"] for row in aggregates if row["experiment_arm"] == ARMS[1] + ), + "alternative_task_regressions": sum(row["comparison"]["task_regressions"] for row in aggregates), + "alternative_false_suppressed_facts": sum( + row["comparison"]["false_suppressed_baseline_facts"] for row in aggregates + ), + }, + "cost": {"model_calls": 0, "model_cost_usd": 0, "currency": "USD"}, + "limits": [ + "Synthetic fixture oracles are not a full benchmark or production task acceptance.", + "Shared evidence is not duplicate-fact proof; lexical annotations are advisory only.", + "Exact-equality collapse is an in-memory measurement projection of the unchanged pool.", + "Native scoped reads and the pure Builder were exercised; HTTP/MCP and host delivery were not.", + "Backend results must be executed and reported separately.", + "Unmerged dependency snapshots do not establish merged-main acceptance.", + ], + } + output_dir.mkdir(parents=True, exist_ok=False) + _write(output_dir / "manifest.json", manifest) + _write(output_dir / "arm-manifests.json", manifests) + for value in manifests: + _write(output_dir / f"{value['experiment_arm']}.manifest.json", value) + _write(output_dir / "report.json", report) + _write(output_dir / "backend-matrix.json", backend_matrix) + _write( + output_dir / "command.json", + { + "argv": [ + "longmemeval-v2", + "lineage-fixture", + "--backend", + backend, + "--runtime-revision", + runtime_revision, + "--dependency-state", + dependency_state, + "--oceanbase-url-env", + oceanbase_url_env, + "--output-dir", + "", + ], + "prerequisites": "Install the exact recorded Runtime source and evaluation source; compare their digests.", + }, + ) + (output_dir / "report.md").write_text(_markdown(report, aggregates), encoding="utf-8") + (output_dir / "decision.md").write_text( + "# Decision: no product change\n\n" + "This bounded fixture evaluation measures concentration, lexical relationship uncertainty, fact recall, " + "false suppression, and final byte-budget omissions separately. Shared Source evidence does not justify " + "suppressing distinct facts. Neither equality projection nor lexical annotations are a production policy.\n\n" + f"Measured lexical false-relation pairs across query samples: {report['findings']['sampled_lexical_false_relation_pairs']}. " + f"Equality-measurement task improvements: {report['findings']['equality_measurement_task_improvements']}. " + f"Alternative task regressions: {report['findings']['alternative_task_regressions']}. " + f"False-suppressed baseline facts: {report['findings']['alternative_false_suppressed_facts']}.\n\n" + "A later fixed-pool stable reorder experiment or review-gated relation proposal requires broader held-out " + "task evidence, merged prerequisites, and separately executed backend validation. No production ranking, " + "write behavior, lifecycle state, citations, public APIs, or delivery contract is changed.\n\n" + f"Dependency state: {dependency_state}. See report.json for measured outcomes and explicit limits.\n", + encoding="utf-8", + ) + _write( + output_dir / "artifact-hashes.json", + { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(output_dir.iterdir()) + if path.is_file() + }, + ) + return LineageRun(output_dir, output_dir / "report.json", output_dir / "manifest.json") + + +def _write(path: Path, value: object) -> None: + path.write_text(json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _markdown(report: Mapping[str, Any], rows: list[dict[str, Any]]) -> str: + lines = [ + "# Scope-local lineage and occupancy evaluation", + "", + "Recommendation: **no product change**. Classification: bounded synthetic fixtures.", + "", + f"Dependency state: `{report['dependency_state']}`. Models: 0 calls / USD 0.", + "", + "Each category is one isolated Scope; only aggregates are exported. Counts sum over that Scope's queries.", + "", + "| Case | Arm | Queries | Retrieved facts | Delivered facts | Required facts | False suppression | Task regressions |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |", + ] + for row in rows: + result, comparison = row["outcome"], row["comparison"] + lines.append( + f"| {row['case_category']} | {row['experiment_arm']} | {row['samples']} | " + f"{result['retrieved_required_facts']} | {result['delivered_required_facts']} | {result['required_facts']} | " + f"{comparison['false_suppressed_baseline_facts']} | {comparison['task_regressions']} |" + ) + lines.extend(["", "## Interpretation and limits", ""]) + lines.extend(f"- {value}" for value in report["limits"]) + return "\n".join(lines) + "\n" diff --git a/evaluation/memory/longmemeval_v2/tests/test_lineage.py b/evaluation/memory/longmemeval_v2/tests/test_lineage.py new file mode 100644 index 0000000000..de6d27086b --- /dev/null +++ b/evaluation/memory/longmemeval_v2/tests/test_lineage.py @@ -0,0 +1,161 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exact evidence, conservative equality, and replay-comparison contracts.""" + +from __future__ import annotations + +import copy +from dataclasses import replace + +import pytest + +from powercontext_eval_longmemeval_v2.lineage import ( + ARMS, + GROUPING_VERSION, + LEXICAL_THRESHOLD, + LEXICAL_VERSION, + MANIFEST_SCHEMA, + EntryRef, + LineageEvaluationError, + Material, + Observation, + ensure_comparable_lineage_runs, + exact_representatives, + lexical_pairs, + measure_pool, +) + + +def material(scope="scope", source="source", digest="a" * 64): + return Material(scope, "content", source, digest, "content", "1", "b" * 64, "1", "untrusted", "unknown") + + +def observation(key, text="same fact", *, sources=(), facts=(), scope="scope", artifacts=(), revision=1): + return Observation(EntryRef(scope, key, revision), "fact", text, sources, artifacts, frozenset(facts)) + + +def test_shared_source_preserves_distinct_facts_and_unknown_trust(): + pool = ( + observation("a", "Atlas deployment port is 8000.", sources=(material(),), facts=("port",)), + observation("b", "Atlas deployment port is 9000.", sources=(material(),), facts=("historical-port",)), + ) + metrics = measure_pool(pool) + assert metrics["same_source_top_k_concentration"] == 1 + assert metrics["unique_fact_count"] == 2 + assert metrics["neutral_or_unknown_declaration_count"] == 2 + assert metrics["lexical_false_relation_pairs"] == 1 + assert exact_representatives(pool) == (0, 1) + assert lexical_pairs(pool) == ((0, 1),) + + +def test_equality_projection_is_stable_byte_exact_and_keeps_pool_intact(): + pool = ( + observation("a", sources=(material(),)), + observation("b", sources=(material(source="other"),)), + observation("c", "same FACT"), + observation("d", "same fact "), + ) + before = copy.deepcopy(pool) + assert exact_representatives(pool) == (0, 2, 3) + assert measure_pool(pool)["unique_evidence_count"] == 2 + assert pool == before + + +def test_equal_material_digest_does_not_merge_source_identities(): + pool = (observation("a", sources=(material(source="one"),)), observation("b", sources=(material(source="two"),))) + assert measure_pool(pool)["same_source_top_k_count"] == 1 + assert measure_pool(pool)["unique_evidence_count"] == 2 + + +def test_unknown_source_is_not_one_shared_lineage(): + metrics = measure_pool((observation("a"), observation("b"))) + assert metrics["unknown_source_evidence_count"] == 2 + assert metrics["same_source_top_k_count"] == 0 + assert metrics["exact_lineage_component_top_k_count"] == 1 + assert measure_pool(())["same_source_top_k_concentration"] is None + + +def test_missing_relation_oracle_is_unknown_rather_than_a_false_relation(): + pool = (observation("a", "Atlas deployment port is 8000."), observation("b", "Atlas deployment port is 9000.")) + metrics = measure_pool(pool) + assert metrics["lexical_unknown_relation_pairs"] == 1 + assert metrics["lexical_false_relation_pairs"] == 0 + assert metrics["advisory_unverified_relation_pairs"] == 1 + + +def test_exact_historical_artifact_edges_do_not_use_todays_version(): + predecessor = EntryRef("scope", "predecessor", 1) + first = observation("a", artifacts=(predecessor,)) + second = observation("b", artifacts=(predecessor,)) + different_version = observation("c", artifacts=(replace(predecessor, revision=2),)) + assert measure_pool((first, second, different_version))["exact_lineage_component_top_k_count"] == 2 + + +@pytest.mark.parametrize( + "pool", + [ + (observation("a"), observation("b", scope="foreign")), + (observation("a", sources=(material(scope="foreign"),)),), + (observation("a", artifacts=(EntryRef("foreign", "a", 1),)),), + (observation("a"), observation("a")), + tuple(observation(str(index)) for index in range(17)), + ], +) +def test_unbounded_ambiguous_or_cross_scope_evidence_is_rejected(pool): + with pytest.raises(LineageEvaluationError): + measure_pool(pool) + + +def manifest(): + return { + "schema": MANIFEST_SCHEMA, + "experiment_arm": ARMS[0], + "fixture_version": "fixture-v1", + "fixture_sha256": "a" * 64, + "runtime_revision": "b" * 40, + "runtime_source_sha256": "c" * 64, + "evaluation_source_sha256": "d" * 64, + "backend": "sqlite", + "capabilities": {"fts": True}, + "dependency_state": "unmerged", + "grouping_version": GROUPING_VERSION, + "lexical_version": LEXICAL_VERSION, + "lexical_threshold": LEXICAL_THRESHOLD, + "candidate_limit": 16, + "top_k": 8, + "model": None, + "advisory_jev": {"status": "not_provided"}, + "environment": {"python": "3.11"}, + } + + +def test_comparison_allows_only_registered_arm_difference(): + first = manifest() + ensure_comparable_lineage_runs(first, {**first, "experiment_arm": ARMS[1]}) + for key, value in first.items(): + if key == "experiment_arm": + continue + with pytest.raises(LineageEvaluationError): + ensure_comparable_lineage_runs(first, {**first, key: str(value) + "changed"}) + with pytest.raises(LineageEvaluationError): + ensure_comparable_lineage_runs(first, {**first, "new_unpinned_condition": True}) + with pytest.raises(LineageEvaluationError): + ensure_comparable_lineage_runs(first, {**first, "experiment_arm": "unregistered"}) + + +def test_comparison_refuses_two_equally_incomplete_manifests(): + incomplete = {"schema": MANIFEST_SCHEMA, "experiment_arm": ARMS[0]} + with pytest.raises(LineageEvaluationError): + ensure_comparable_lineage_runs(incomplete, incomplete) diff --git a/evaluation/memory/longmemeval_v2/tests/test_lineage_run.py b/evaluation/memory/longmemeval_v2/tests/test_lineage_run.py new file mode 100644 index 0000000000..b856ab8725 --- /dev/null +++ b/evaluation/memory/longmemeval_v2/tests/test_lineage_run.py @@ -0,0 +1,115 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native SQLite acceptance requires the exact Source-evidence prerequisite.""" + +from __future__ import annotations + +import asyncio +import json + +import pytest +from typer.testing import CliRunner + +from powercontext_eval_longmemeval_v2.cli import app +from powercontext_eval_longmemeval_v2.lineage import ARMS, LineageEvaluationError +from powercontext_eval_longmemeval_v2.lineage_fixtures import fixture_cases +from powercontext_eval_longmemeval_v2.lineage_run import run_lineage_fixture + + +def test_native_fixture_refuses_missing_evidence_prerequisite(tmp_path): + runtime = pytest.importorskip("powercontext.builtin.runtime.atomic_memory") + if hasattr(runtime.ScopedAtomicMemory, "evidence"): + pytest.skip("this runtime provides the prerequisite; covered by native acceptance") + output = tmp_path / "report" + with pytest.raises(LineageEvaluationError, match="#1975"): + asyncio.run(run_lineage_fixture(output_dir=output, backend="sqlite", runtime_revision="a" * 40)) + assert not output.exists() + + +def test_native_sqlite_replay_exports_only_aggregates_and_preserves_shared_facts(tmp_path): + runtime = pytest.importorskip("powercontext.builtin.runtime.atomic_memory") + if not hasattr(runtime.ScopedAtomicMemory, "evidence"): + pytest.skip("requires the exact #1975 evidence-read contract") + output = tmp_path / "report" + asyncio.run( + run_lineage_fixture( + output_dir=output, + backend="sqlite", + runtime_revision="a" * 40, + dependency_state="unmerged", + ) + ) + report = json.loads((output / "report.json").read_text(encoding="utf-8")) + assert report["cost"]["model_calls"] == 0 + assert report["recommendation"] == "no_product_change" + assert report["backend_matrix"][0]["capabilities"]["authoritative_records_unchanged"] + assert report["backend_matrix"][1]["status"] == "not_run" + for row in report["aggregates"]: + if row["case_category"] in {"legitimate_shared_source", "held_out_shared_source"}: + assert row["outcome"]["task_success"] == row["samples"] + assert row["comparison"]["false_suppressed_baseline_facts"] == 0 + if row["case_category"] == "held_out_byte_budget": + assert row["outcome"]["retrieval_missed_facts"] == 0 + assert row["outcome"]["post_retrieval_omitted_facts"] > 0 + assert row["omissions"]["truncated_items"] + row["omissions"]["dropped_items"] > 0 + published = "\n".join(path.read_text(encoding="utf-8") for path in output.iterdir()) + for case in fixture_cases(): + for entry in case.entries: + assert entry.text not in published + for fact in entry.facts: + assert fact not in published + assert f"lineage-fixture-{case.category}" not in published + runner = CliRunner() + result = runner.invoke( + app, + [ + "lineage-compare", + "--baseline", + str(output / f"{ARMS[0]}.manifest.json"), + "--alternative", + str(output / f"{ARMS[1]}.manifest.json"), + ], + ) + assert result.exit_code == 0, result.output + with pytest.raises(LineageEvaluationError, match="overwrite"): + asyncio.run(run_lineage_fixture(output_dir=output, backend="sqlite", runtime_revision="a" * 40)) + + +def test_invalid_comparison_fails_without_echoing_manifest_content(tmp_path): + path = tmp_path / "invalid.json" + path.write_text('{"secret-content": "never-echo-this"}', encoding="utf-8") + result = CliRunner().invoke(app, ["lineage-compare", "--baseline", str(path), "--alternative", str(path)]) + assert result.exit_code == 1 + assert "never-echo-this" not in result.output + + +def test_backend_configuration_error_does_not_echo_connection_values(tmp_path): + pytest.importorskip("powercontext.builtin.persistence.oceanbase") + result = CliRunner().invoke( + app, + [ + "lineage-fixture", + "--backend", + "oceanbase", + "--runtime-revision", + "a" * 40, + "--output-dir", + str(tmp_path / "report"), + ], + env={"POWERCONTEXT_TEST_OCEANBASE_URL": "private-invalid-connection-value"}, + ) + assert result.exit_code == 1 + assert "private-invalid-connection-value" not in result.output + assert not (tmp_path / "report").exists() From b8087fce0ea2e9591e92d3749e2b65af5cc1d07e Mon Sep 17 00:00:00 2001 From: shaohang <15062640367@163.com> Date: Thu, 22 Oct 2026 20:54:44 +0800 Subject: [PATCH 2/2] fix(evaluation): account for exact Artifact evidence Include scoped Artifact family and revision in evidence coverage and loss. Add Artifact-only native regression fixtures and version the evidence accounting contract. Refs oceanbase/powercontext#1977. --- .../longmemeval_v2/docs/lineage-occupancy.md | 14 ++++++- .../docs/lineage-occupancy.zh.md | 4 +- .../lineage.py | 22 ++++++++++- .../lineage_fixtures.py | 17 ++++++++- .../lineage_run.py | 38 +++++++++++++++++-- .../longmemeval_v2/tests/test_lineage.py | 33 ++++++++++++++++ .../longmemeval_v2/tests/test_lineage_run.py | 18 +++++++++ 7 files changed, 139 insertions(+), 7 deletions(-) diff --git a/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.md b/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.md index 837421ef84..34f055f240 100644 --- a/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.md +++ b/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.md @@ -42,7 +42,7 @@ be reported as successful or equivalent SQLite execution. ## Workload and measurement -`scope-local-lineage-fixtures-v1` fixes small, separate Scope partitions for: +`scope-local-lineage-fixtures-v2` fixes small, separate Scope partitions for: - byte-identical concentration and a less-represented relevant fact; - paraphrases grounded in shared evidence; @@ -50,6 +50,7 @@ be reported as successful or equivalent SQLite execution. - revised and historical entries with exact retained revision reads; - recurring automation with distinct event facts; - entries without Source evidence; +- held-out byte-identical entries without Sources citing different retained Artifact revisions; - all six mandatory CJK forms from #1976, literal English and negative controls; - held-out shared-source facts and held-out byte-budget pressure. @@ -89,6 +90,17 @@ bytes, so truncation cannot be mistaken for successful factual delivery. For held-out fixtures, disjoint declared facts in a lexical-neighbour pair count as a false relation annotation, independently of actual suppression. +Evidence accounting uses `scoped-source-artifact-evidence-v1`, pinned as +`evidence_version` in manifests and reports. `unique_evidence_count` is the +deduplicated union of direct Source materialization identities and exact scoped +Artifact citations, including family and revision. `unique_source_evidence_count` +and `unique_artifact_evidence_count` expose each part. Coverage and +`lost_baseline_evidence` use the same identity rules; Source and Artifact losses +are also reported separately. Artifact references are counted at the retained +cited version, without following current heads or recursively expanding lineage. +These are distinct direct references, not a claim of independent factual content. +Manifests without the evidence-accounting version are rejected for comparison. + Task outcomes separately record retrieval misses, final omitted facts, explicit byte-budget drops/truncations, entry-limit omissions, baseline facts/evidence lost by an alternative, task regressions, and negative-control false hits. diff --git a/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.zh.md b/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.zh.md index 7e52511d45..acb279a114 100644 --- a/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.zh.md +++ b/evaluation/memory/longmemeval_v2/docs/lineage-occupancy.zh.md @@ -22,7 +22,7 @@ OceanBase 使用 `--backend oceanbase`，通过 `POWERCONTEXT_TEST_OCEANBASE_URL ## 固定场景与指标 -`scope-local-lineage-fixtures-v1` 包括独立 Scope 中的完全相同条目、同证据改写、同 Source 不同事实、修订与历史事实、周期自动来源、未知证据、#1976 的六种中文正例及英文和无关查询对照。另有不参与阈值调整的共享来源与字节预算保留验证案例。 +`scope-local-lineage-fixtures-v2` 包括独立 Scope 中的完全相同条目、同证据改写、同 Source 不同事实、修订与历史事实、周期自动来源、未知证据、#1976 的六种中文正例及英文和无关查询对照。另有不参与阈值调整的共享来源、字节预算，以及无直接 Source、正文相同但引用不同历史 Artifact 修订的保留验证案例。 事实答案由合成 fixture 显式声明，不是模型判断或可信度推断。每个查询只取一次最多 16 条的准入后 RRF 候选序列，所有实验复用其成员和顺序。精确历史读取及评测前后的权威记录快照用于验证内容、谱系和状态没有被评测修改。 @@ -38,6 +38,8 @@ Jev 不是必需依赖，本模式明确记录为 `not_provided`，不作为权 候选池和最终选中来源分别统计 top-k 集中度、精确冗余槽位、词汇邻近占比、独立证据及事实数、未知证据和中性声明。选中条目的事实数不等同于实际交付事实数：答案召回直接检查最终渲染字节，避免将截断后的内容当作完整交付。 +证据计数使用 `scoped-source-artifact-evidence-v1`，通过 manifest 和报告中的 `evidence_version` 固定。`unique_evidence_count` 是直接 Source materialization 与精确 Artifact 引用的去重并集，Artifact 身份保留 Scope、family、ID 和 revision。`unique_source_evidence_count` 与 `unique_artifact_evidence_count` 分别报告两类数量。覆盖和 `lost_baseline_evidence` 使用相同身份规则，损失也按 Source 与 Artifact 拆分。统计不追随当前 head，也不递归展开引用；不同直接引用不代表不同事实。缺少证据计数版本的 manifest 无法参与比较。 + 检索漏召回、关系不确定、预算丢弃/截断、条数上限遗漏、相对基线的事实与证据损失、任务退化、负例误命中分别记录。耗时拆分为检索、精确证据读取和构建。每个类别对应一个 Scope，只导出该 Scope 查询的聚合计数和比率，不提供逐条诊断或跨 Scope 标识。 ## 报告与比较 diff --git a/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage.py index 175068af53..d376b1ee0f 100644 --- a/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage.py @@ -30,6 +30,7 @@ from powercontext_eval_longmemeval_v2.errors import LongMemEvalError GROUPING_VERSION = "exact-scoped-evidence-v1" +EVIDENCE_VERSION = "scoped-source-artifact-evidence-v1" LEXICAL_VERSION = "unicode-character-trigram-jaccard-v1" LEXICAL_THRESHOLD = 0.75 MANIFEST_SCHEMA = "powercontext.lineage-evaluation-manifest.v1" @@ -115,6 +116,20 @@ def exact_representatives(pool: Sequence[Observation]) -> tuple[int, ...]: return tuple(kept) +def evidence_identities(pool: Sequence[Observation]) -> frozenset[tuple[object, ...]]: + """Distinct direct evidence, with separate Source and exact Artifact identity domains. + + Artifact evidence is the cited historical revision, not its current head or + recursively expanded lineage. Identities stay in-process; only counts leave. + """ + validate_pool(pool) + sources = frozenset(("source", *source.identity) for entry in pool for source in entry.sources) + artifacts = frozenset( + ("artifact", ref.scope, ref.family, ref.artifact, ref.revision) for entry in pool for ref in entry.artifacts + ) + return sources | artifacts + + def _shingles(text: str) -> frozenset[str]: value = " ".join(unicodedata.normalize("NFC", text).casefold().split()) return frozenset(value[index : index + 3] for index in range(len(value) - 2)) @@ -176,6 +191,7 @@ def root(ref: EntryRef) -> EntryRef: ) unknown_pairs = sum(not pool[a].facts or not pool[b].facts for a, b in neighbours) facts = set().union(*(entry.facts for entry in pool)) if pool else set() + evidence = evidence_identities(pool) return { "candidate_count": len(pool), "top_k_count": len(top), @@ -197,7 +213,9 @@ def root(ref: EntryRef) -> EntryRef: "lexical_false_relation_pairs": false_pairs, "lexical_unknown_relation_pairs": unknown_pairs, "advisory_unverified_relation_pairs": len(neighbours), - "unique_evidence_count": len({source.identity for entry in pool for source in entry.sources}), + "unique_evidence_count": len(evidence), + "unique_source_evidence_count": sum(identity[0] == "source" for identity in evidence), + "unique_artifact_evidence_count": sum(identity[0] == "artifact" for identity in evidence), "unique_fact_count": len(facts), } @@ -218,6 +236,7 @@ def ensure_comparable_lineage_runs(first: Mapping[str, object], second: Mapping[ "capabilities", "dependency_state", "grouping_version", + "evidence_version", "lexical_version", "lexical_threshold", "candidate_limit", @@ -231,6 +250,7 @@ def ensure_comparable_lineage_runs(first: Mapping[str, object], second: Mapping[ raise LineageEvaluationError("lineage manifest lacks pinned comparison conditions") if ( value["grouping_version"] != GROUPING_VERSION + or value["evidence_version"] != EVIDENCE_VERSION or value["lexical_version"] != LEXICAL_VERSION or value["candidate_limit"] != MAX_POOL or value["top_k"] != 8 diff --git a/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_fixtures.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_fixtures.py index b697a87927..b480bc6af0 100644 --- a/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_fixtures.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_fixtures.py @@ -18,7 +18,7 @@ from dataclasses import dataclass -FIXTURE_VERSION = "scope-local-lineage-fixtures-v1" +FIXTURE_VERSION = "scope-local-lineage-fixtures-v2" @dataclass(frozen=True) @@ -121,6 +121,21 @@ def fixture_cases() -> tuple[Case, ...]: ), (Query("Atlas deployment unknown", ("UNKNOWNONE", "UNKNOWNTWO")),), ), + Case( + "artifact_only_evidence", + ( + Entry("retained", "Retained ledger reference version HISTORYONE.", ("HISTORYONE",), None), + Entry("retained", "Retained ledger reference version HISTORYTWO.", ("HISTORYTWO",), None), + Entry( + "copy-a", "Atlas deployment artifact fact is ARTIFACTFACT.", ("ARTIFACTFACT",), None, "retained:1" + ), + Entry( + "copy-b", "Atlas deployment artifact fact is ARTIFACTFACT.", ("ARTIFACTFACT",), None, "retained:2" + ), + ), + (Query("Atlas deployment artifact", ("ARTIFACTFACT",)),), + True, + ), Case( "query_form_baseline", ( diff --git a/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_run.py b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_run.py index 19a37c05bb..d88d6a864d 100644 --- a/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_run.py +++ b/evaluation/memory/longmemeval_v2/src/powercontext_eval_longmemeval_v2/lineage_run.py @@ -34,6 +34,7 @@ from powercontext_eval_longmemeval_v2.lineage import ( ARMS, + EVIDENCE_VERSION, GROUPING_VERSION, LEXICAL_THRESHOLD, LEXICAL_VERSION, @@ -43,6 +44,7 @@ Material, Observation, ensure_comparable_lineage_runs, + evidence_identities, exact_representatives, measure_pool, ) @@ -316,8 +318,9 @@ async def authority_snapshot() -> tuple[str, ...]: answer in (base_content or "") and answer not in (built.context.content or "") for answer in query.answers ) - base_evidence = {source.identity for entry in base_delivered for source in entry.sources} - delivered_evidence = {source.identity for entry in delivered for source in entry.sources} + base_evidence = evidence_identities(base_delivered) + delivered_evidence = evidence_identities(delivered) + lost_evidence = base_evidence - delivered_evidence rows.append( { "case_category": case.category, @@ -336,7 +339,13 @@ async def authority_snapshot() -> tuple[str, ...]: }, "comparison": { "false_suppressed_baseline_facts": lost, - "lost_baseline_evidence": len(base_evidence - delivered_evidence), + "lost_baseline_evidence": len(lost_evidence), + "lost_baseline_source_evidence": sum( + identity[0] == "source" for identity in lost_evidence + ), + "lost_baseline_artifact_evidence": sum( + identity[0] == "artifact" for identity in lost_evidence + ), "task_regressions": int( (outcome["task_success"] or 0) < (base_outcome["task_success"] or 0) ), @@ -418,6 +427,7 @@ async def run_lineage_fixture( "capabilities": capabilities, "dependency_state": dependency_state, "grouping_version": GROUPING_VERSION, + "evidence_version": EVIDENCE_VERSION, "lexical_version": LEXICAL_VERSION, "lexical_threshold": LEXICAL_THRESHOLD, "candidate_limit": 16, @@ -448,6 +458,7 @@ async def run_lineage_fixture( ] report = { "schema": "powercontext.lineage-evaluation-report.v1", + "evidence_version": EVIDENCE_VERSION, "classification": "bounded-synthetic-fixture-evaluation", "recommendation": "no_product_change", "dependency_state": dependency_state, @@ -469,6 +480,7 @@ async def run_lineage_fixture( "alternative_false_suppressed_facts": sum( row["comparison"]["false_suppressed_baseline_facts"] for row in aggregates ), + "alternative_lost_evidence": sum(row["comparison"]["lost_baseline_evidence"] for row in aggregates), }, "cost": {"model_calls": 0, "model_cost_usd": 0, "currency": "USD"}, "limits": [ @@ -517,6 +529,8 @@ async def run_lineage_fixture( f"Equality-measurement task improvements: {report['findings']['equality_measurement_task_improvements']}. " f"Alternative task regressions: {report['findings']['alternative_task_regressions']}. " f"False-suppressed baseline facts: {report['findings']['alternative_false_suppressed_facts']}.\n\n" + f"Lost baseline evidence references: {report['findings']['alternative_lost_evidence']}. " + "Evidence loss is independent of whether identical text still delivers the same fact.\n\n" "A later fixed-pool stable reorder experiment or review-gated relation proposal requires broader held-out " "task evidence, merged prerequisites, and separately executed backend validation. No production ranking, " "write behavior, lifecycle state, citations, public APIs, or delivery contract is changed.\n\n" @@ -558,6 +572,24 @@ def _markdown(report: Mapping[str, Any], rows: list[dict[str, Any]]) -> str: f"{result['retrieved_required_facts']} | {result['delivered_required_facts']} | {result['required_facts']} | " f"{comparison['false_suppressed_baseline_facts']} | {comparison['task_regressions']} |" ) + lines.extend( + [ + "", + "## Direct evidence coverage", + "", + "Source materializations and exact Artifact citations are distinct evidence identities; counts sum over queries.", + "", + "| Case | Arm | Delivered Sources | Delivered Artifacts | Lost Sources | Lost Artifacts |", + "| --- | --- | ---: | ---: | ---: | ---: |", + ] + ) + for row in rows: + delivered, comparison = row["delivered"], row["comparison"] + lines.append( + f"| {row['case_category']} | {row['experiment_arm']} | " + f"{delivered['unique_source_evidence_count']} | {delivered['unique_artifact_evidence_count']} | " + f"{comparison['lost_baseline_source_evidence']} | {comparison['lost_baseline_artifact_evidence']} |" + ) lines.extend(["", "## Interpretation and limits", ""]) lines.extend(f"- {value}" for value in report["limits"]) return "\n".join(lines) + "\n" diff --git a/evaluation/memory/longmemeval_v2/tests/test_lineage.py b/evaluation/memory/longmemeval_v2/tests/test_lineage.py index de6d27086b..8145187d05 100644 --- a/evaluation/memory/longmemeval_v2/tests/test_lineage.py +++ b/evaluation/memory/longmemeval_v2/tests/test_lineage.py @@ -23,6 +23,7 @@ from powercontext_eval_longmemeval_v2.lineage import ( ARMS, + EVIDENCE_VERSION, GROUPING_VERSION, LEXICAL_THRESHOLD, LEXICAL_VERSION, @@ -79,6 +80,30 @@ def test_equal_material_digest_does_not_merge_source_identities(): assert measure_pool(pool)["unique_evidence_count"] == 2 +@pytest.mark.parametrize( + "second", + [EntryRef("scope", "retained", 2), EntryRef("scope", "retained", 1, "profile")], +) +def test_artifact_only_coverage_preserves_exact_family_and_revision(second): + first = EntryRef("scope", "retained", 1) + pool = (observation("a", artifacts=(first, first)), observation("b", artifacts=(second,))) + assert measure_pool(pool)["unique_evidence_count"] == 2 + assert measure_pool(pool[:1])["unique_evidence_count"] == 1 + assert measure_pool(pool)["unique_source_evidence_count"] == 0 + assert measure_pool(pool)["unique_artifact_evidence_count"] == 2 + + +def test_source_and_artifact_evidence_share_one_deduplicated_coverage_contract(): + ref = EntryRef("scope", "retained", 1) + pool = ( + observation("a", sources=(material(),), artifacts=(ref,)), + observation("b", sources=(material(),), artifacts=(ref,)), + ) + assert measure_pool(pool)["unique_evidence_count"] == 2 + assert measure_pool(pool)["unique_source_evidence_count"] == 1 + assert measure_pool(pool)["unique_artifact_evidence_count"] == 1 + + def test_unknown_source_is_not_one_shared_lineage(): metrics = measure_pool((observation("a"), observation("b"))) assert metrics["unknown_source_evidence_count"] == 2 @@ -131,6 +156,7 @@ def manifest(): "capabilities": {"fts": True}, "dependency_state": "unmerged", "grouping_version": GROUPING_VERSION, + "evidence_version": EVIDENCE_VERSION, "lexical_version": LEXICAL_VERSION, "lexical_threshold": LEXICAL_THRESHOLD, "candidate_limit": 16, @@ -159,3 +185,10 @@ def test_comparison_refuses_two_equally_incomplete_manifests(): incomplete = {"schema": MANIFEST_SCHEMA, "experiment_arm": ARMS[0]} with pytest.raises(LineageEvaluationError): ensure_comparable_lineage_runs(incomplete, incomplete) + + +def test_comparison_refuses_source_only_accounting_manifests(): + source_only = manifest() + source_only.pop("evidence_version") + with pytest.raises(LineageEvaluationError): + ensure_comparable_lineage_runs(source_only, source_only) diff --git a/evaluation/memory/longmemeval_v2/tests/test_lineage_run.py b/evaluation/memory/longmemeval_v2/tests/test_lineage_run.py index b856ab8725..91dc5aea69 100644 --- a/evaluation/memory/longmemeval_v2/tests/test_lineage_run.py +++ b/evaluation/memory/longmemeval_v2/tests/test_lineage_run.py @@ -57,6 +57,24 @@ def test_native_sqlite_replay_exports_only_aggregates_and_preserves_shared_facts assert report["backend_matrix"][0]["capabilities"]["authoritative_records_unchanged"] assert report["backend_matrix"][1]["status"] == "not_run" for row in report["aggregates"]: + for section in ("pool", "delivered"): + assert row[section]["unique_evidence_count"] == ( + row[section]["unique_source_evidence_count"] + row[section]["unique_artifact_evidence_count"] + ) + assert row["comparison"]["lost_baseline_evidence"] == ( + row["comparison"]["lost_baseline_source_evidence"] + row["comparison"]["lost_baseline_artifact_evidence"] + ) + if row["case_category"] == "artifact_only_evidence": + assert row["pool"]["candidate_count"] == 2 + assert row["pool"]["unique_evidence_count"] == 2 + assert row["pool"]["unique_source_evidence_count"] == 0 + assert row["pool"]["unique_artifact_evidence_count"] == 2 + collapsed = row["experiment_arm"] == ARMS[1] + assert row["delivered"]["candidate_count"] == (1 if collapsed else 2) + assert row["delivered"]["unique_evidence_count"] == (1 if collapsed else 2) + assert row["comparison"]["lost_baseline_evidence"] == int(collapsed) + assert row["comparison"]["lost_baseline_source_evidence"] == 0 + assert row["comparison"]["lost_baseline_artifact_evidence"] == int(collapsed) if row["case_category"] in {"legitimate_shared_source", "held_out_shared_source"}: assert row["outcome"]["task_success"] == row["samples"] assert row["comparison"]["false_suppressed_baseline_facts"] == 0