From 1f894266755814ce4da2f2c88089c0ceb0af998c Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Tue, 20 Oct 2026 10:48:35 +0800 Subject: [PATCH 1/9] perf(search): simplify retrieval queries and lifecycle checks --- docs/en/docs/workflows/atomic-memory.md | 9 +- docs/en/docs/workflows/topic-memory.md | 12 + docs/en/rfcs/1417_topic_memory.md | 8 +- docs/zh/docs/workflows/atomic-memory.md | 7 +- docs/zh/docs/workflows/topic-memory.md | 11 + docs/zh/rfcs/1417_topic_memory.md | 7 +- .../builtin/artifacts/experience/search.py | 12 +- src/powercontext/builtin/artifacts/search.py | 13 +- .../artifacts/topic_memory/chunking.py | 7 +- .../persistence/atomic_memory_index.py | 83 +++- .../builtin/persistence/experience_index.py | 134 ++++++ .../oceanbase/atomic_memory_index.py | 71 +++- .../persistence/oceanbase/experience_index.py | 74 +--- .../oceanbase/topic_memory_index.py | 272 ++++++------- .../persistence/sqlite/atomic_memory_index.py | 189 ++++++++- .../persistence/sqlite/experience_index.py | 60 +-- .../persistence/sqlite/topic_memory_index.py | 381 +++++++----------- .../builtin/persistence/topic_memory.py | 101 +++-- .../builtin/persistence/topic_memory_index.py | 32 -- .../builtin/runtime/application.py | 17 +- .../builtin/runtime/atomic_memory.py | 40 +- .../runtime/atomic_memory_processing.py | 83 ++-- .../builtin/runtime/composition.py | 8 +- .../runtime/topic_memory_processing.py | 63 ++- .../builtin/runtime/topic_memory_search.py | 14 +- .../persistence/test_experience_index.py | 17 +- .../builtin/persistence/test_topic_memory.py | 208 ++++++++-- .../test_topic_memory_oceanbase.py | 65 +-- .../runtime/test_composition_embedding.py | 16 +- .../runtime/test_topic_memory_processing.py | 15 + .../runtime/test_topic_memory_search.py | 26 +- tests/e2e/test_atomic_memory_vector_index.py | 314 +++++++++++++++ tests/e2e/test_lexical_admission.py | 204 ++++++++++ tests/e2e/test_recall_sufficiency_gate.py | 1 - tests/e2e/test_sqlite_vec.py | 13 +- 35 files changed, 1711 insertions(+), 876 deletions(-) create mode 100644 tests/e2e/test_atomic_memory_vector_index.py create mode 100644 tests/e2e/test_lexical_admission.py diff --git a/docs/en/docs/workflows/atomic-memory.md b/docs/en/docs/workflows/atomic-memory.md index 1caf1e47d5..a06c95e69e 100644 --- a/docs/en/docs/workflows/atomic-memory.md +++ b/docs/en/docs/workflows/atomic-memory.md @@ -91,8 +91,13 @@ Submit to `POST /v1/atomic-memory/search`: `mode` accepts `text`, `vector` or `hybrid` and defaults to `text`. Vector and hybrid modes require an available, matching Embedding profile. Search returns active memories only. Each `hits[].memory` contains the content, exact Artifact reference and state version; the hit also has `score` and `matched_by`. Preserve `memory.artifact` for citations. -Ordinary vector search applies eligibility first, computes exact L2, then ranks and limits results. This path does not -use ANN and does not establish production backend performance or acceptance. +Ordinary vector search on seekdb and OceanBase uses native approximate nearest-neighbor retrieval, with Scope, kind +and tag eligibility inside the candidate query before its limit. ANN can omit exact nearest neighbors; tune and measure +recall and latency for the deployment's embedding model and filter selectivity. SQLite uses Scope-partitioned +sqlite-vec native exact KNN for bounded search. Normal startup audits vector deployment readiness; ordinary Top-K +selects usable vectors and validates selected results rather than auditing the whole eligible collection. Complete +threshold enumeration used by generation and exact recovery probes still reject incomplete eligible vectors. +Enumeration remains exact on every backend. For complete Artifact results and advanced retrieval controls, use `POST /v1/scopes/S/artifacts/atomic-memory/search`. This unified route requires `scope.read`, defaults to text, diff --git a/docs/en/docs/workflows/topic-memory.md b/docs/en/docs/workflows/topic-memory.md index 43d49f01e1..7562a9691e 100644 --- a/docs/en/docs/workflows/topic-memory.md +++ b/docs/en/docs/workflows/topic-memory.md @@ -117,6 +117,18 @@ For complete Artifact results, explicit text/vector/hybrid modes, admission, sco [Fusion algorithms and parameters](search-fusion.md). The dedicated route keeps its existing summary response and `0`–`100` score scale. +Explicit vector/hybrid search uses the configured query embedding provider even in an empty Scope; provider errors and +fallback depend on the requested mode. SQLite ranks exact nearest Topics across both title/summary and detail vectors. +It first checks a bounded exact neighbor prefix. If that prefix contains enough eligible distinct Topics or is exhausted, +it returns; otherwise one full exact query replaces the prefix. Worst-case latency grows with eligible chunks. At exactly equal +distances, boundary Topic membership and the choice of an equally near chunk need not follow global identity or +ordinal order. These choices can change fused candidates and scores; the final fused-score tie rules still apply. +seekdb/OceanBase use approximate title/summary retrieval. Detail search ranks current chunks by each Topic's minimum +distance before the candidate limit and payload reads. This scans eligible chunks and has no constant-latency guarantee. +The native fused result remains approximate because its title channel uses ANN; measure recall and tail latency for +the deployment. Startup leaves existing detail HNSW indexes intact and does not create a detail graph for new databases. +Detail queries do not use that graph; an existing graph still incurs database maintenance costs. + ## Read exact detail Do not reconstruct the latest version from a title. Pass the `artifact` from a search hit (`family`, `artifact_id`, and diff --git a/docs/en/rfcs/1417_topic_memory.md b/docs/en/rfcs/1417_topic_memory.md index 6e454a1de9..6e18cbd0a4 100644 --- a/docs/en/rfcs/1417_topic_memory.md +++ b/docs/en/rfcs/1417_topic_memory.md @@ -425,9 +425,11 @@ Chunk policy: 6. Exact get always returns the complete Detail. The concrete chunk length, tail threshold, and overlap ratio are internal constants associated with a policy version, -not public configuration. The implementation also enforces a hard per-Topic chunk count so the vector channel's -worst-case neighbor scan remains within the backend limit; adversarial Markdown falls back to bounded overlapping -windows. Changing the Chunk policy requires rebuilding the corresponding retrieval projections. +not public configuration. A hard per-Topic chunk count bounds chunk projection writes and storage per Topic; adversarial +Markdown falls back to bounded overlapping windows. This budget does not guarantee Topic-level Top-K completeness +within a backend's raw neighbor limit or bound the total retrieval work. Backend retrieval behavior and cost boundaries +are described in the [Topic Memory workflow](../docs/workflows/topic-memory.md#search-current-topics). +Changing the Chunk policy requires rebuilding the corresponding retrieval projections. Each retrieval channel first collapses by Topic. When multiple Detail chunks from one Topic match, only the best position is retained for the snippet. The two or four channels enabled for the current deployment are fused by RRF diff --git a/docs/zh/docs/workflows/atomic-memory.md b/docs/zh/docs/workflows/atomic-memory.md index cfa93a7d23..5843f164de 100644 --- a/docs/zh/docs/workflows/atomic-memory.md +++ b/docs/zh/docs/workflows/atomic-memory.md @@ -88,8 +88,11 @@ Atomic Memory 将每条事实、偏好或决策保存为独立的 `atomic-memory `mode` 可选 `text`、`vector`、`hybrid`，默认 `text`。向量和混合模式要求可用且匹配的 Embedding profile。 搜索只返回 active 记忆，`hits[].memory` 包含正文、精确 Artifact 引用和状态版本；还返回 `score`、 -`matched_by`。引用结果时保留 `memory.artifact`。普通向量搜索在资格过滤后计算精确 L2，再排序和限制条数； -当前路径不使用 ANN，不能据此推断生产后端性能或验收结果。 +`matched_by`。引用结果时保留 `memory.artifact`。seekdb 和 OceanBase 的普通向量搜索使用原生近似最近邻检索， +Scope、kind、tag 的资格条件位于候选查询内部，在限制条数前执行。ANN 可能遗漏精确最近邻，需要针对部署的 +Embedding 模型和过滤选择率测量召回与延迟。SQLite 的有界检索使用按 Scope 分区的 sqlite-vec 原生精确 KNN。 +正常启动时核查向量部署是否完整；普通 Top-K 选择可用向量并校验选中结果，不在每次请求中审计整个候选集合。 +生成使用的完整阈值枚举和精确恢复探测仍拒绝资格范围内不完整的向量；完整枚举在所有后端上保持精确。 需要完整 Artifact 结果和高级检索参数时，使用 `POST /v1/scopes/S/artifacts/atomic-memory/search`。 统一入口要求 `scope.read`，默认文本检索，支持 `filters`、准入、RRF 排名常数与权重、`min_score` 和可选评分元数据。 diff --git a/docs/zh/docs/workflows/topic-memory.md b/docs/zh/docs/workflows/topic-memory.md index d81da38b9a..22f15399f6 100644 --- a/docs/zh/docs/workflows/topic-memory.md +++ b/docs/zh/docs/workflows/topic-memory.md @@ -103,6 +103,17 @@ Content-Type: application/json 需要完整 Artifact 结果、显式文本/向量/混合模式、准入、评分阈值或通道原分时，使用[检索 Artifact](search-artifacts.md)。 Topic 的融合参数见[融合算法与参数](search-fusion.md)。专用入口保留原来的摘要响应和 `0`–`100` 评分。 +显式向量或混合检索在空 Scope 中也使用配置的 query embedding provider；模型错误及回退行为由请求模式决定。 +SQLite 对标题/摘要和详情两个向量通道计算精确最近主题。先检查有界的精确邻居前缀； +取得足够的合格不同 Topic 或已穷尽时即返回，否则用一次完整精确查询替换前缀。 +最坏延迟会随资格范围内的 chunk 数增长。距离完全相等时， +边界 Topic 的入选和等距最佳 chunk 的选择不保证遵循全局身份或 ordinal 顺序；这可能改变融合候选和得分， +最终融合得分并列时的排序规则仍然适用。 +seekdb/OceanBase 的标题/摘要使用近似检索；详情按各 Topic 当前 chunk 的最小距离排序， +再截取候选和读取正文。该路径扫描资格范围内的 chunk，不保证固定时延；标题通道仍是 ANN， +需要测量实际部署的融合召回率与尾时延。启动保留已有 detail HNSW，新库不创建 detail 图。 +详情查询不使用该图；旧库中的已有图仍产生数据库维护成本。 + ## 读取精确详情 不要根据标题自行拼接最新版本。把搜索结果中的 `artifact`（`family`、`artifact_id`、`revision`）原样传给 diff --git a/docs/zh/rfcs/1417_topic_memory.md b/docs/zh/rfcs/1417_topic_memory.md index e6a5df3a4a..ab49ef2393 100644 --- a/docs/zh/rfcs/1417_topic_memory.md +++ b/docs/zh/rfcs/1417_topic_memory.md @@ -377,9 +377,10 @@ Chunk policy： 5. 搜索命中后围绕最佳位置动态扩展 snippet； 6. exact get 始终返回完整 Detail。 -具体 chunk 长度、尾块阈值和 overlap 比例是带 policy version 的内部常量，不是公共配置。实现还会强制每个 Topic -的 chunk 数量上限，保证向量通道最坏情况下的近邻扫描不超过后端限制；对抗性 Markdown 会回退为有界重叠窗口。 -Chunk 策略变化需要重建对应检索投影。 +具体 chunk 长度、尾块阈值和 overlap 比例是带 policy version 的内部常量，不是公共配置。每个 Topic 的 chunk +数量上限约束单个 Topic 的 chunk 投影写入量和存储量；对抗性 Markdown 会回退为有界重叠窗口。该预算不保证在后端原始 +近邻数量限制内获得完整的 Topic 级 Top-K，也不限制整个检索的工作量。后端检索行为与成本边界见 +[Topic Memory 工作流](../docs/workflows/topic-memory.md#搜索当前主题)。Chunk 策略变化需要重建对应检索投影。 每个检索通道先按 Topic collapse，同一 Topic 的多个 detail 命中只保留最佳位置用于 snippet。当前部署启用的 两个或四个通道通过 RRF 排名融合，不比较全文分数和向量距离的原始数值。最终一个 Topic 最多占一个结果位置，并保留 diff --git a/src/powercontext/builtin/artifacts/experience/search.py b/src/powercontext/builtin/artifacts/experience/search.py index 53a4673005..7308b46f78 100644 --- a/src/powercontext/builtin/artifacts/experience/search.py +++ b/src/powercontext/builtin/artifacts/experience/search.py @@ -38,15 +38,11 @@ class ExperienceSearchRequest(ArtifactSearchQuery): @dataclass(frozen=True) class ExperienceSearchOutcome: - """Experience hits plus the admission accounting for the search that produced them. + """Experience hits and bounded admission observations for Recall recovery planning. - It replaces the bare ``tuple[ExperienceSearchHit, ...]`` the ``experience_recall`` callback - used to return, because the Runtime's recall gate needs the retrieved/admitted pair and the - callback signature is part of the Runtime's construction surface. - - ``admission`` is ``None`` when the search did not measure — for example a configured index - with nothing to report — rather than a fabricated ``(0, 0)``. The counters are aggregates - with no hit identity, so carrying them costs nothing the callback did not already compute. + ``admission`` is ``None`` when the index did not measure candidates. Built-in + indexes observe selected qualified matches and at most one ranked-prefix rejection witness; + these counters do not represent the whole matching collection. """ hits: tuple[ExperienceSearchHit, ...] = () diff --git a/src/powercontext/builtin/artifacts/search.py b/src/powercontext/builtin/artifacts/search.py index 67d6b608eb..94ddb3abd0 100644 --- a/src/powercontext/builtin/artifacts/search.py +++ b/src/powercontext/builtin/artifacts/search.py @@ -271,12 +271,13 @@ def __post_init__(self) -> None: class AdmissionCounts: """Per-family, per-scope admission accounting for one search. - ``retrieved`` is what the backend returned *before* the admission floor was applied; - ``admitted`` is what survived it. A family may also provide ``rejected`` when those two - counts are not measured on a comparable basis, for example when the trace keeps an - unbounded pre-admission count but the delivered candidate pool is capped. ``rejected`` - is the number of same-search candidates rejected by the admission floor, excluding - candidate-pool truncation. + ``retrieved`` counts candidates actually observed by this search, including + any bounded rejection evidence. ``admitted`` counts observed candidates that + pass the floor. These are not collection totals. A Family may provide + ``rejected`` when the two observations use different bounded pools; it counts + actual rejection evidence, excluding candidate-pool truncation. Topic FTS + observes at most one rejected Topic per channel; Experience observes at most + one rejected match in its ranked prefix. Both support recovery planning without collection counts. These are plain aggregate integers with no candidate identity, no query text and no per-entry attribution, so the value is safe to carry in a trace and safe to hand to the diff --git a/src/powercontext/builtin/artifacts/topic_memory/chunking.py b/src/powercontext/builtin/artifacts/topic_memory/chunking.py index 29da2b8bef..64f09cb665 100644 --- a/src/powercontext/builtin/artifacts/topic_memory/chunking.py +++ b/src/powercontext/builtin/artifacts/topic_memory/chunking.py @@ -34,10 +34,9 @@ TOPIC_MEMORY_CHUNK_MAX_CHARACTERS = 1_800 TOPIC_MEMORY_CHUNK_MIN_TAIL_CHARACTERS = 300 TOPIC_MEMORY_CHUNK_OVERLAP_CHARACTERS = 160 -# The first-release search contract admits at most 20 Topics per channel. -# Keeping each Topic at or below 200 chunks therefore bounds sqlite-vec k at -# 4,000, below its 4,096-neighbor ceiling. Pathological Markdown that would -# exceed this limit uses the deterministic fixed-window fallback below. +# Bound per-Topic projection and inference work. Retrieval must independently +# handle distinct Topics and native neighbor limits; many close sibling chunks +# can exhaust a raw neighbor pool. Oversized Markdown uses fixed windows below. TOPIC_MEMORY_CHUNK_MAX_COUNT = 200 _BLOCK_BOUNDARY = re.compile(r"(?:\n[ \t]*\n+)|(?=^#{1,6}[ \t]+)|(?=^[ \t]*(?:[-*+] |\d+[.)] ))", re.MULTILINE) diff --git a/src/powercontext/builtin/persistence/atomic_memory_index.py b/src/powercontext/builtin/persistence/atomic_memory_index.py index 052874761e..88d3e0021b 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/atomic_memory_index.py @@ -23,7 +23,7 @@ from typing import Any, Literal, Protocol from pydantic import BaseModel -from sqlalchemy import Table, bindparam, delete, insert, inspect, select, text, update +from sqlalchemy import Table, bindparam, delete, func, insert, inspect, or_, select, text, update from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.artifacts import ArtifactRef, ChannelScore @@ -51,6 +51,17 @@ async def _projection_columns(connection: AsyncConnection, table: Table) -> set[str]: + if connection.dialect.name == "mysql": + return set( + await connection.scalars( + text( + "SELECT column_name FROM information_schema.columns " + "WHERE table_schema = DATABASE() AND table_name = :table_name" + ), + {"table_name": table.name}, + ) + ) + def read(sync: Any) -> set[str]: inspector = inspect(sync) if not inspector.has_table(table.name): @@ -188,6 +199,29 @@ async def refresh_tags( ) -> None: ... +async def validate_atomic_memory_vectors(connection: AsyncConnection, index: AtomicMemoryIndex) -> None: + """Audit vector deployment readiness at startup, outside ordinary bounded search.""" + profile = index.capabilities.embedding_profile + if profile is None: + return + table = index.table + invalid = [ + table.c.embedding.is_(None), + table.c.embedding_input_hash.is_(None), + table.c.profile_fingerprint.is_(None), + table.c.profile_fingerprint != atomic_memory_profile_fingerprint(profile), + table.c.projection_format != ATOMIC_MEMORY_PROJECTION_FORMAT, + ] + if connection.dialect.name == "sqlite": + invalid.append(func.length(table.c.embedding) != profile.dimension * 4) + if await connection.scalar(select(table.c.artifact_id).where(or_(*invalid)).limit(1)) is not None: + raise AtomicMemoryIndexError( + "incomplete-vector", + "Current Atomic Memory vectors are incomplete or use another profile. Stop writers and run " + "powercontext server atomic-memory-rebuild-projection --maintenance-confirmed.", + ) + + class AtomicMemoryProjectionPublisher: """Prepare vectors outside writes and publish with the authoritative transaction.""" @@ -368,13 +402,11 @@ def atomic_memory_filter_sql( def atomic_memory_vector_sql( eligibility: str, dialect: Literal["sqlite", "mysql"], /, *, bounded: bool, threshold: bool ) -> str: - """One statement checks readiness and retrieves matches at one channel snapshot. + """Exact Top-K over usable rows, or complete enumeration with a readiness sentinel. - The sentinel reports any incomplete eligible row even when no vector meets - the threshold. Bounded results limit only after same-row eligibility and exact - scoring. Explicit L2 thresholds belong to complete related enumeration; - ordinary search uses cosine admission on unit-normalized L2 vectors. - There is no business-table join, ANN or enumeration result cap. + Bounded search is not a collection audit. Complete related enumeration rejects + incomplete eligible vectors and has no result cap. Both apply same-row filters + before truncation and keep business-table joins out of candidate scoring. """ distance = "vec_distance_l2" if dialect == "sqlite" else "l2_distance" @@ -388,6 +420,17 @@ def atomic_memory_vector_sql( " AND distance <= :max_distance" if threshold else f" AND {clamped_similarity} >= :min_semantic_similarity" ) limit_sql = " LIMIT :result_limit" if bounded else "" + if bounded and not threshold: + return ( + "WITH scored AS (SELECT artifact_id, revision, state_version, kind, text, " # noqa: S608 + f"{distance}(embedding, :query_vector) AS distance FROM pc_atomic_memory_current " + f"WHERE scope_id = :scope_id AND ({eligibility}) " + "AND embedding IS NOT NULL AND profile_fingerprint = :profile AND embedding_input_hash IS NOT NULL " + f"AND projection_format = :projection_format{valid_length}) " + "SELECT artifact_id, revision, state_version, kind, text, distance, 0 AS invalid_vectors " + f"FROM scored WHERE {clamped_similarity} >= :min_semantic_similarity " + "ORDER BY distance, artifact_id LIMIT :result_limit" + ) # eligibility is built by atomic_memory_filter_sql from fixed grammar. return ( "WITH eligible AS (SELECT artifact_id, revision, state_version, kind, text, embedding, " # noqa: S608 @@ -642,13 +685,34 @@ async def probe_recoverable( if not recoverable: return False new_channels = [f"({lower} AND NOT {admitted})" for admitted, lower in zip(current, recoverable, strict=True)] - statement = text( + probe = ( "SELECT 1 FROM pc_atomic_memory_current WHERE scope_id = :scope_id " # noqa: S608 f"AND ({eligibility}) AND ({' OR '.join(new_channels)}) LIMIT 1" ) + invalid = "0 = 1" + profile = self.profile + if request.mode in {"vector", "hybrid"} and profile is not None: + parameters.update( + profile=atomic_memory_profile_fingerprint(profile), + projection_format=ATOMIC_MEMORY_PROJECTION_FORMAT, + embedding_bytes=profile.dimension * 4, + ) + length = " OR LENGTH(embedding) <> :embedding_bytes" if dialect == "sqlite" else "" + invalid = ( + "EXISTS (SELECT 1 FROM pc_atomic_memory_current WHERE scope_id = :scope_id " # noqa: S608 + f"AND ({eligibility}) AND (embedding IS NULL OR embedding_input_hash IS NULL " + "OR profile_fingerprint IS NULL OR profile_fingerprint <> :profile " + f"OR projection_format <> :projection_format{length}) LIMIT 1)" + ) + statement = text(f"SELECT CASE WHEN {invalid} THEN -1 ELSE EXISTS ({probe}) END") if request.mode in {"vector", "hybrid"} and dialect == "mysql": statement = statement.bindparams(bindparam("probe_vector", type_=self.table.c.embedding.type)) - return await connection.scalar(statement, parameters) is not None + result = await connection.scalar(statement, parameters) + if result == -1: + raise AtomicMemoryIndexError( + "incomplete-vector", "Eligible Atomic Memory vectors are incomplete or use another profile" + ) + return bool(result) def _require_vectors( self, @@ -708,4 +772,5 @@ def validate_request(request: AtomicMemorySearchRequest | AtomicMemoryRelatedReq "load_atomic_memory_tags", "oceanbase_fts_coverage_sql", "sqlite_fts_coverage_sql", + "validate_atomic_memory_vectors", ] diff --git a/src/powercontext/builtin/persistence/experience_index.py b/src/powercontext/builtin/persistence/experience_index.py index bf3c6a161b..44c9fd5960 100644 --- a/src/powercontext/builtin/persistence/experience_index.py +++ b/src/powercontext/builtin/persistence/experience_index.py @@ -38,7 +38,10 @@ InvalidSearchScore, admits_fts_projection, admits_fts_text, + analyze_fts_query, analyze_text, + fts_match_query, + fts_query_requirements, lexical_search_score, ) from powercontext.builtin.artifacts.skill import ( @@ -50,6 +53,7 @@ skill_searchable_text, ) from powercontext.builtin.artifacts.skill.package import capture_skill_archive +from powercontext.builtin.persistence.atomic_memory_index import oceanbase_fts_coverage_sql, sqlite_fts_coverage_sql from powercontext.builtin.persistence.codec import load_model, stored_bytes from powercontext.builtin.persistence.errors import RepositoryNotFoundError from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE, SKILL_PACKAGES_TABLE @@ -357,6 +361,133 @@ async def replace_skill_projection( ) +async def search_approved_artifact_rows( + connection: AsyncConnection, + scope_id: str, + family: str, + query: str, + limit: int, + *, + admission: AdmissionFloor | None, +) -> tuple[Mapping[Any, Any], ...]: + """Qualify active heads before truncation; hydrate bounded candidates and decode selected rows. + + Admission observes the ranked prefix, not rejected rows beyond that prefix. + """ + terms, required = fts_query_requirements(query, floor=admission) + if not terms: + return () + parameters: dict[str, Any] = { + "scope_id": scope_id, + "family": family, + "result_limit": limit, + "required": min(required, len(terms) + 1), + } + if connection.dialect.name == "sqlite": + coverage, values = sqlite_fts_coverage_sql(terms) if required > 1 else ("1", {}) + parameters["query"] = fts_match_query(query) + source = ( + "pc_artifact_fts AS f JOIN pc_artifact_heads AS h ON " + "h.scope_id=f.scope_id AND h.family=f.family AND h.artifact_id=f.artifact_id " + "AND h.revision=f.revision" + ) + coverage = coverage.replace("searchable_text", "f.searchable_text") + score, predicate, direction, metric = ( + "bm25(pc_artifact_fts)", + "pc_artifact_fts MATCH :query", + "ASC", + "sqlite_bm25", + ) + identity = "f.rowid AS projection_id, " + projection_join = "JOIN pc_artifact_fts AS p ON p.rowid=c.projection_id " + else: + coverage, values = oceanbase_fts_coverage_sql(terms) if required > 1 else ("1", {}) + parameters["query"] = analyze_fts_query(query) + source = "pc_artifact_heads AS h" + score = "MATCH(h.searchable_text) AGAINST (:query)" + predicate, direction, metric = score, "DESC", "oceanbase_match" + identity = "" + projection_join = ( + "JOIN pc_artifact_heads AS p ON p.scope_id=:scope_id AND p.family=:family " + "AND p.artifact_id=c.artifact_id AND p.revision=c.revision " + ) + parameters.update(values) + eligibility = f"WHERE h.scope_id=:scope_id AND h.family=:family AND h.lifecycle_state='active' AND {predicate}" + matches = f"FROM {source} {eligibility}" + witness: tuple[Mapping[Any, Any], ...] = () + if connection.dialect.name != "sqlite" and required > 1: + parameters["probe_limit"] = limit * 8 + probe = text( + f"WITH raw_probe AS (SELECT h.artifact_id, h.revision, h.searchable_text, {score} AS raw_score " # noqa: S608 + f"{matches} ORDER BY raw_score DESC, h.artifact_id, h.revision LIMIT :probe_limit) " + "SELECT r.*, a.content AS content, 0 AS is_rejected FROM raw_probe AS r " + "LEFT JOIN pc_artifacts AS a ON a.scope_id=:scope_id AND a.family=:family " + "AND a.artifact_id=r.artifact_id AND a.revision=r.revision " + "ORDER BY raw_score DESC, artifact_id, revision" + ) + observed = tuple((await connection.execute(probe, parameters)).mappings()) + qualified = tuple( + row for row in observed if admits_fts_projection(query, str(row["searchable_text"]), floor=admission) + ) + witness = ( + ({"is_rejected": 1, "score_metric": metric},) + if family == Experience.family and len(qualified) < len(observed) + else () + ) + if len(qualified) >= limit or len(observed) < parameters["probe_limit"]: + return witness + tuple({**row, "score_metric": metric} for row in qualified[:limit]) + prefix = "" + candidate_source = source + if connection.dialect.name == "sqlite" and required > 1: + parameters["probe_limit"] = limit * 4 + prefix = ( + f"raw_probe AS MATERIALIZED (SELECT {identity}h.artifact_id, h.revision, " # noqa: S608 + f"{score} AS raw_score " + f"{matches} ORDER BY raw_score, h.artifact_id, h.revision LIMIT :probe_limit), " + "qualified_probe AS MATERIALIZED (SELECT r.* FROM raw_probe AS r " + "CROSS JOIN pc_artifact_fts AS f ON f.rowid=r.projection_id " + f"WHERE {coverage} >= :required), " + "fallback_gate AS MATERIALIZED (SELECT 1 " + "WHERE (SELECT COUNT(*) FROM raw_probe)=:probe_limit " + "AND (SELECT COUNT(*) FROM qualified_probe)<:result_limit), " + ) + # Keep the bounded prefix and the zero-or-one-row fallback gate outside FTS loops. + candidate_source = "fallback_gate CROSS JOIN " + source.replace(" JOIN ", " CROSS JOIN ", 1) + candidates = ( + f"SELECT {identity}h.artifact_id, h.revision, {score} AS raw_score " # noqa: S608 + f"FROM {candidate_source} {eligibility} AND {coverage} >= :required " + f"ORDER BY raw_score {direction}, h.artifact_id, h.revision LIMIT :result_limit" + ) + if prefix: + # An exact sorted prefix proves Top-K when enough qualify; otherwise retain the full query. + candidates = ( + "SELECT * FROM (SELECT * FROM qualified_probe " # noqa: S608 + "WHERE NOT EXISTS (SELECT 1 FROM fallback_gate) " + "ORDER BY raw_score, artifact_id, revision LIMIT :result_limit) " + f"UNION ALL SELECT * FROM ({candidates})" + ) + rejected = ( + ( + " UNION ALL SELECT NULL, NULL, NULL, NULL, NULL, 1 FROM " + "(SELECT 1 WHERE (SELECT COUNT(*) FROM raw_probe) > " + "(SELECT COUNT(*) FROM qualified_probe)) AS rejected" + ) + if family == Experience.family and prefix + else "" + ) + statement = text( + f"WITH {prefix}candidates AS ({candidates}) " # noqa: S608 + "SELECT c.artifact_id AS artifact_id, c.revision AS revision, p.searchable_text AS searchable_text, " + f"a.content AS content, c.raw_score AS raw_score, 0 AS is_rejected FROM candidates AS c {projection_join}" + "LEFT JOIN pc_artifacts AS a ON a.scope_id=:scope_id AND a.family=:family " + f"AND a.artifact_id=c.artifact_id AND a.revision=c.revision{rejected} " + f"ORDER BY is_rejected DESC, raw_score {direction}, artifact_id, revision" + ) + rows = (await connection.execute(statement, parameters)).mappings() + result = tuple({**row, "score_metric": metric} for row in rows) + return witness + result + + def experience_search_hits( rows: Iterable[Mapping[Any, Any]], query: str, @@ -384,6 +515,8 @@ def experience_search_hits( admitted = 0 for row in rows: examined += 1 + if row.get("is_rejected"): + continue content = _content(row["content"]) if not admits_fts_text(query, experience_search_text(content), floor=admission): continue @@ -547,5 +680,6 @@ async def _skill_projection_texts(connection, rows): "remove_experience_projection", "replace_experience_projection", "replace_skill_projection", + "search_approved_artifact_rows", "skill_search_hits", ] diff --git a/src/powercontext/builtin/persistence/oceanbase/atomic_memory_index.py b/src/powercontext/builtin/persistence/oceanbase/atomic_memory_index.py index 39d77c0671..b833e99668 100644 --- a/src/powercontext/builtin/persistence/oceanbase/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/oceanbase/atomic_memory_index.py @@ -56,16 +56,36 @@ _HIT_COLUMNS = "artifact_id, revision, state_version, kind, text" +def oceanbase_atomic_memory_vector_sql(eligibility: str, /) -> str: + """Bounded ANN over current usable vectors with same-row qualification.""" + return ( + f"WITH candidates AS (SELECT {_HIT_COLUMNS}, l2_distance(embedding, :query_vector) AS distance " # noqa: S608 + "FROM pc_atomic_memory_current " + f"WHERE scope_id = :scope_id AND ({eligibility}) " + "AND embedding IS NOT NULL AND embedding_input_hash IS NOT NULL " + "AND profile_fingerprint = :profile AND projection_format = :projection_format " + "ORDER BY l2_distance(embedding, :query_vector) APPROXIMATE LIMIT :result_limit " + "PARAMETERS(ef_search=:ef_search)) " + f"SELECT {_HIT_COLUMNS}, distance, 0 AS invalid_vectors FROM candidates " + "WHERE greatest(-1.0, least(1.0, 1.0 - distance * distance / 2.0)) >= :min_semantic_similarity " + "ORDER BY distance, artifact_id" + ) + + class OceanBaseAtomicMemoryIndex(RelationalAtomicMemoryIndex): """Native same-row retrieval; complete enumeration always uses exact L2. - Ordinary vector search also uses exact distance, preserving same-row kind and tag filters - before LIMIT without relying on ANN filtering. - The native index exists for backend-native support and future bounded modes. + Ordinary vector search uses native ANN with eligibility inside the candidate + query. Complete related enumeration retains the separate exact-distance SQL. + Search breadth is statement-local and conservative: aggregate neighbor recall + alone can hide missed multilingual outliers. Calibrate it for the deployment. """ - def __init__(self, profile: EmbeddingProfile | None = None) -> None: + def __init__(self, profile: EmbeddingProfile | None = None, *, ef_search: int = 1000) -> None: + if isinstance(ef_search, bool) or not 1 <= ef_search <= 1000: + raise AtomicMemoryIndexError("vector-search", "Vector search breadth must be between 1 and 1000") super().__init__(atomic_memory_current_table(None if profile is None else VECTOR(profile.dimension)), profile) + self._ef_search = ef_search self._vector_index = ( None if profile is None @@ -94,23 +114,29 @@ async def initialize(self, connection: AsyncConnection, /) -> None: "oceanbase-fts", "OceanBase Atomic Memory requires native FULLTEXT SPACE parser and inline JSON predicates", ) from error - if self.profile is not None: - column_type = await connection.scalar(_VECTOR_TYPE) - expected = f"VECTOR({self.profile.dimension})" - if str(column_type).upper() != expected: + await self._initialize_vectors(connection) + + async def _initialize_vectors(self, connection: AsyncConnection) -> None: + if self.profile is None: + return + column_type = await connection.scalar(_VECTOR_TYPE) + expected = f"VECTOR({self.profile.dimension})" + if str(column_type).upper() != expected: + raise AtomicMemoryIndexError( + "vector-schema", + f"Atomic Memory uses {column_type!r}; expected {expected}. " + "Reconfigure the current vector column offline and rebuild vectors before vector search.", + ) + if self._vector_index is not None and not await connection.scalar( + _INDEX_EXISTS, {"index_name": _VECTOR_INDEX_NAME} + ): + vector_index = self._vector_index + try: + await connection.run_sync(lambda sync: vector_index.create(sync, checkfirst=False)) + except SQLAlchemyError as error: raise AtomicMemoryIndexError( - "vector-schema", - f"Atomic Memory uses {column_type!r}; expected {expected}. " - "Reconfigure the current vector column offline and rebuild vectors before vector search.", - ) - if self._vector_index is not None: - vector_index = self._vector_index - try: - await connection.run_sync(lambda sync: vector_index.create(sync, checkfirst=True)) - except SQLAlchemyError as error: - raise AtomicMemoryIndexError( - "oceanbase-vector", "OceanBase native Atomic Memory vector index is unavailable" - ) from error + "oceanbase-vector", "OceanBase native Atomic Memory vector index is unavailable" + ) from error async def reconfigure_vector_column(self, connection: AsyncConnection, /) -> None: """Offline derived-schema operation, followed by authoritative vector rebuild. @@ -196,13 +222,16 @@ async def _channels( query_vector=query_vector, profile=atomic_memory_profile_fingerprint(profile), projection_format=ATOMIC_MEMORY_PROJECTION_FORMAT, + ef_search=self._ef_search, max_distance=request.max_distance if isinstance(request, AtomicMemoryRelatedRequest) else None, min_semantic_similarity=( DEFAULT_ADMISSION_FLOOR if request.admission is None else request.admission ).min_semantic_similarity, ) statement = text( - atomic_memory_vector_sql( + oceanbase_atomic_memory_vector_sql(eligibility) + if limit is not None and limit <= 16384 + else atomic_memory_vector_sql( eligibility, "mysql", bounded=limit is not None, diff --git a/src/powercontext/builtin/persistence/oceanbase/experience_index.py b/src/powercontext/builtin/persistence/oceanbase/experience_index.py index 467475ff5d..5180569be5 100644 --- a/src/powercontext/builtin/persistence/oceanbase/experience_index.py +++ b/src/powercontext/builtin/persistence/oceanbase/experience_index.py @@ -16,13 +16,13 @@ from __future__ import annotations -from sqlalchemy import literal, select, text +from sqlalchemy import select, text from sqlalchemy.dialects.mysql import match from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.builtin.artifacts.experience import Experience, ExperienceSearchOutcome from powercontext.builtin.artifacts.memory import CapabilityNotSupportedError -from powercontext.builtin.artifacts.search import AdmissionFloor, analyze_fts_query +from powercontext.builtin.artifacts.search import AdmissionFloor from powercontext.builtin.artifacts.skill import Skill, SkillPackageSnapshot, SkillSearchHit from powercontext.builtin.persistence.experience_index import ( ensure_artifact_head_searchable_text, @@ -32,9 +32,10 @@ remove_experience_projection, replace_experience_projection, replace_skill_projection, + search_approved_artifact_rows, skill_search_hits, ) -from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE +from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE _OCEANBASE_FTS_INDEX_NAME = "ix_pc_artifact_heads_fts" _OCEANBASE_CREATE_FTS_SQL = f""" @@ -94,36 +95,9 @@ async def search( min_score: float | None = None, require_scores: bool = False, ) -> ExperienceSearchOutcome: - analyzed = analyze_fts_query(query) - if not analyzed: - return ExperienceSearchOutcome() - score = match(ARTIFACT_HEADS_TABLE.c.searchable_text, against=analyzed) - rows = ( - await connection.execute( - select( - ARTIFACT_HEADS_TABLE.c.artifact_id, - ARTIFACT_HEADS_TABLE.c.revision, - ARTIFACTS_TABLE.c.content, - score.label("raw_score"), - literal("oceanbase_match").label("score_metric"), - ) - .join( - ARTIFACTS_TABLE, - (ARTIFACTS_TABLE.c.scope_id == ARTIFACT_HEADS_TABLE.c.scope_id) - & (ARTIFACTS_TABLE.c.family == ARTIFACT_HEADS_TABLE.c.family) - & (ARTIFACTS_TABLE.c.artifact_id == ARTIFACT_HEADS_TABLE.c.artifact_id) - & (ARTIFACTS_TABLE.c.revision == ARTIFACT_HEADS_TABLE.c.revision), - ) - .where( - ARTIFACT_HEADS_TABLE.c.scope_id == scope_id, - ARTIFACT_HEADS_TABLE.c.family == Experience.family, - ARTIFACT_HEADS_TABLE.c.lifecycle_state == "active", - score, - ) - .order_by(score.desc(), ARTIFACT_HEADS_TABLE.c.artifact_id, ARTIFACT_HEADS_TABLE.c.revision) - .limit(limit * 4) - ) - ).mappings() + rows = await search_approved_artifact_rows( + connection, scope_id, Experience.family, query, limit, admission=admission + ) return experience_search_hits( rows, query, limit, scope_id, admission=admission, min_score=min_score, require_scores=require_scores ) @@ -150,37 +124,9 @@ async def search_skills( min_score: float | None = None, require_scores: bool = False, ) -> tuple[SkillSearchHit, ...]: - analyzed = analyze_fts_query(query) - if not analyzed: - return () - score = match(ARTIFACT_HEADS_TABLE.c.searchable_text, against=analyzed) - rows = ( - await connection.execute( - select( - ARTIFACT_HEADS_TABLE.c.artifact_id, - ARTIFACT_HEADS_TABLE.c.revision, - ARTIFACT_HEADS_TABLE.c.searchable_text, - ARTIFACTS_TABLE.c.content, - score.label("raw_score"), - literal("oceanbase_match").label("score_metric"), - ) - .join( - ARTIFACTS_TABLE, - (ARTIFACTS_TABLE.c.scope_id == ARTIFACT_HEADS_TABLE.c.scope_id) - & (ARTIFACTS_TABLE.c.family == ARTIFACT_HEADS_TABLE.c.family) - & (ARTIFACTS_TABLE.c.artifact_id == ARTIFACT_HEADS_TABLE.c.artifact_id) - & (ARTIFACTS_TABLE.c.revision == ARTIFACT_HEADS_TABLE.c.revision), - ) - .where( - ARTIFACT_HEADS_TABLE.c.scope_id == scope_id, - ARTIFACT_HEADS_TABLE.c.family == Skill.family, - ARTIFACT_HEADS_TABLE.c.lifecycle_state == "active", - score, - ) - .order_by(score.desc(), ARTIFACT_HEADS_TABLE.c.artifact_id, ARTIFACT_HEADS_TABLE.c.revision) - .limit(limit * 4) - ) - ).mappings() + rows = await search_approved_artifact_rows( + connection, scope_id, Skill.family, query, limit, admission=admission + ) return skill_search_hits( rows, query, limit, admission=admission, min_score=min_score, require_scores=require_scores ) diff --git a/src/powercontext/builtin/persistence/oceanbase/topic_memory_index.py b/src/powercontext/builtin/persistence/oceanbase/topic_memory_index.py index 7e60cebe0b..7434ba7904 100644 --- a/src/powercontext/builtin/persistence/oceanbase/topic_memory_index.py +++ b/src/powercontext/builtin/persistence/oceanbase/topic_memory_index.py @@ -30,11 +30,13 @@ bindparam, case, delete, + desc, func, insert, literal, select, text, + union_all, ) from sqlalchemy.dialects.mysql import match from sqlalchemy.ext.asyncio import AsyncConnection @@ -44,7 +46,6 @@ from powercontext.builtin.artifacts.memory.canonical import canonical_embedding from powercontext.builtin.artifacts.search import fts_query_requirements from powercontext.builtin.artifacts.topic_memory import ( - TOPIC_MEMORY_CHUNK_MAX_COUNT, TopicMemoryCapabilities, TopicMemoryCapabilityError, TopicMemoryChannelHit, @@ -52,6 +53,7 @@ TopicMemoryProjection, TopicMemorySearchChannels, TopicMemorySearchRequest, + TopicMemoryStorageInvariantError, ) from powercontext.builtin.persistence.tables import ( TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE, @@ -69,7 +71,6 @@ _TOPIC_VECTOR_TABLE = "pc_topic_memory_vector_topics" _CHUNK_VECTOR_TABLE = "pc_topic_memory_vector_chunks" _TOPIC_VECTOR_INDEX = "ix_pc_topic_memory_vector_topics_embedding" -_CHUNK_VECTOR_INDEX = "ix_pc_topic_memory_vector_chunks_embedding" _INDEX_EXISTS_SQL = text( """ @@ -85,7 +86,8 @@ ) # Keep ANN selection join-free: CE 4.3.5.6 can lose neighbors when its # distance-ordered vector scan participates directly in a merge join. -# The LIMIT boundary precedes active-head hydration and Topic collapse. +# Hydrate only the selected Topic identities. Chunk retrieval must collapse +# before LIMIT: a bounded ANN chunk pool cannot guarantee distinct Topic hits. _TOPIC_VECTOR_SEARCH = """ WITH candidates AS ( SELECT v.scope_id, v.artifact_id, v.revision, @@ -96,41 +98,44 @@ ORDER BY l2_distance(v.embedding, :query_vector) APPROXIMATE LIMIT :candidate_limit ) -SELECT a.artifact_id, a.revision, a.title, a.summary, n.distance +SELECT n.scope_id, n.artifact_id, n.revision, a.revision AS active_revision, + a.title, a.summary, n.distance FROM candidates AS n -JOIN pc_topic_memory_active_topics AS a +LEFT JOIN pc_topic_memory_active_topics AS a ON a.scope_id = n.scope_id AND a.artifact_id = n.artifact_id AND a.revision = n.revision ORDER BY distance, artifact_id, revision DESC """ _CHUNK_VECTOR_SEARCH = """ -WITH neighbors AS ( +WITH distances AS ( SELECT v.scope_id, v.artifact_id, v.revision, v.chunk_ordinal, l2_distance(v.embedding, :query_vector) AS distance FROM pc_topic_memory_vector_chunks AS v WHERE v.scope_id = :scope_id AND v.profile_fingerprint = :profile_fingerprint - ORDER BY l2_distance(v.embedding, :query_vector) APPROXIMATE - LIMIT :neighbor_limit ), ranked AS ( - SELECT a.artifact_id, a.revision, a.title, a.summary, - c.chunk_ordinal, c.start_offset, c.chunk_text, n.distance, + SELECT n.scope_id, n.artifact_id, n.revision, n.chunk_ordinal, n.distance, row_number() OVER ( PARTITION BY n.artifact_id, n.revision ORDER BY n.distance, n.chunk_ordinal ) AS topic_rank - FROM neighbors AS n - JOIN pc_topic_memory_active_topics AS a - ON a.scope_id = n.scope_id AND a.artifact_id = n.artifact_id AND a.revision = n.revision - JOIN pc_topic_memory_active_chunks AS c - ON c.scope_id = n.scope_id AND c.artifact_id = n.artifact_id - AND c.revision = n.revision AND c.chunk_ordinal = n.chunk_ordinal + FROM distances AS n +), candidates AS ( + SELECT scope_id, artifact_id, revision, chunk_ordinal, distance + FROM ranked + WHERE topic_rank = 1 + ORDER BY distance, artifact_id, revision DESC, chunk_ordinal + LIMIT :candidate_limit ) -SELECT artifact_id, revision, title, summary, - chunk_ordinal, start_offset, chunk_text, distance -FROM ranked -WHERE topic_rank = 1 -ORDER BY distance, artifact_id, revision DESC, chunk_ordinal -LIMIT :candidate_limit +SELECT n.scope_id, n.artifact_id, n.revision, a.revision AS active_revision, + a.title, a.summary, n.chunk_ordinal, c.chunk_ordinal AS active_chunk_ordinal, + c.start_offset, c.chunk_text, n.distance +FROM candidates AS n +LEFT JOIN pc_topic_memory_active_topics AS a + ON a.scope_id = n.scope_id AND a.artifact_id = n.artifact_id AND a.revision = n.revision +LEFT JOIN pc_topic_memory_active_chunks AS c + ON c.scope_id = n.scope_id AND c.artifact_id = n.artifact_id + AND c.revision = n.revision AND c.chunk_ordinal = n.chunk_ordinal +ORDER BY n.distance, n.artifact_id, n.revision DESC, n.chunk_ordinal """ @@ -182,45 +187,42 @@ async def search( query_terms, coverage_required, ) - topic_retrieved = await connection.scalar( - select(func.count()) - .select_from(TOPIC_MEMORY_ACTIVE_TOPICS_TABLE) - .where( - TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.scope_id == scope_id, - topic_score, + topic_matches = ( + select( + TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.artifact_id, + TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.revision, + TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.title, + TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.summary, + topic_score.label("raw_score"), + topic_coverage.label("coverage"), ) + .where(TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.scope_id == scope_id, topic_score) + .cte("topic_matches") ) - topic_eligible = await connection.scalar( - select(func.count()) - .select_from(TOPIC_MEMORY_ACTIVE_TOPICS_TABLE) - .where( - TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.scope_id == scope_id, - topic_score, - topic_coverage, - ) + topic_candidates = ( + select(*(topic_matches.c[name] for name in ("artifact_id", "revision", "title", "summary", "raw_score"))) + .where(topic_matches.c.coverage) + .order_by(topic_matches.c.raw_score.desc(), topic_matches.c.artifact_id, topic_matches.c.revision.desc()) + .limit(request.candidate_limit) + .cte("topic_candidates") ) - topic_rows = ( - await connection.execute( - select( - TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.artifact_id, - TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.revision, - TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.title, - TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.summary, - topic_score.label("raw_score"), - ) - .where( - TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.scope_id == scope_id, - topic_score, - topic_coverage, - ) - .order_by( - topic_score.desc(), - TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.artifact_id, - TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.revision.desc(), + topic_rejected = ( + select(literal(1)) + .select_from(topic_matches) + .where(~topic_matches.c.coverage) + .limit(1) + .cte("topic_rejected") + ) + topic_rows = tuple( + ( + await connection.execute( + union_all( + select(topic_candidates, literal(0).label("is_rejected")), + select(*(literal(None) for _ in range(5)), literal(1)).select_from(topic_rejected), + ).order_by("is_rejected", desc("raw_score"), "artifact_id", desc("revision")) ) - .limit(request.candidate_limit) - ) - ).mappings() + ).mappings() + ) chunk_score = match(TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE.c.searchable_text, against=request.analyzed_query) chunk_coverage = _coverage_expression( TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE.c.searchable_text, @@ -265,54 +267,61 @@ async def search( ) .subquery() ) - detail_retrieved = await connection.scalar( - select(func.count()).select_from(chunk_candidates).where(chunk_candidates.c.topic_rank == 1) + chunk_matches = select(chunk_candidates).where(chunk_candidates.c.topic_rank == 1).cte("chunk_matches") + chunk_selected = ( + select( + *( + chunk_matches.c[name] + for name in ( + "artifact_id", + "revision", + "title", + "summary", + "chunk_ordinal", + "start_offset", + "chunk_text", + ) + ), + chunk_matches.c.score.label("raw_score"), + ) + .where(chunk_matches.c.coverage) + .order_by( + chunk_matches.c.score.desc(), + chunk_matches.c.artifact_id, + chunk_matches.c.revision.desc(), + chunk_matches.c.chunk_ordinal, + ) + .limit(request.candidate_limit) + .cte("chunk_candidates") ) - detail_eligible = await connection.scalar( - select(func.count()) - .select_from(chunk_candidates) - .where(chunk_candidates.c.topic_rank == 1, chunk_candidates.c.coverage) + chunk_rejected = ( + select(literal(1)) + .select_from(chunk_matches) + .where(~chunk_matches.c.coverage) + .limit(1) + .cte("chunk_rejected") ) - chunk_rows = ( - await connection.execute( - select( - chunk_candidates.c.artifact_id, - chunk_candidates.c.revision, - chunk_candidates.c.title, - chunk_candidates.c.summary, - chunk_candidates.c.chunk_ordinal, - chunk_candidates.c.start_offset, - chunk_candidates.c.chunk_text, - chunk_candidates.c.score.label("raw_score"), - ) - .where(chunk_candidates.c.topic_rank == 1, chunk_candidates.c.coverage) - .order_by( - chunk_candidates.c.score.desc(), - chunk_candidates.c.artifact_id, - chunk_candidates.c.revision.desc(), - chunk_candidates.c.chunk_ordinal, + chunk_rows = tuple( + ( + await connection.execute( + union_all( + select(chunk_selected, literal(0).label("is_rejected")), + select(*(literal(None) for _ in range(8)), literal(1)).select_from(chunk_rejected), + ).order_by("is_rejected", desc("raw_score"), "artifact_id", desc("revision"), "chunk_ordinal") ) - .limit(request.candidate_limit) - ) - ).mappings() + ).mappings() + ) + topic_hits = tuple(_channel_hit(row, "topic_fts") for row in topic_rows if not row["is_rejected"]) + chunk_hits = tuple(_channel_hit(row, "detail_fts") for row in chunk_rows if not row["is_rejected"]) return TopicMemorySearchChannels( - topic_fts=tuple(_channel_hit(row, "topic_fts") for row in topic_rows), - detail_fts=tuple(_channel_hit(row, "detail_fts") for row in chunk_rows), - topic_fts_retrieved=int(topic_retrieved or 0), - detail_fts_retrieved=int(detail_retrieved or 0), - topic_fts_eligible=int(topic_eligible or 0), - detail_fts_eligible=int(detail_eligible or 0), + topic_fts=topic_hits, + detail_fts=chunk_hits, + topic_fts_retrieved=len(topic_rows), + detail_fts_retrieved=len(chunk_rows), + topic_fts_eligible=len(topic_hits), + detail_fts_eligible=len(chunk_hits), ) - async def vector_complete( - self, - _connection: AsyncConnection, - _scope_id: str, - _topic_ref: ArtifactRef, - /, - ) -> bool: - return False - @staticmethod async def _ensure_index(connection: AsyncConnection, table_name: str, index_name: str) -> None: count = await connection.scalar( @@ -326,7 +335,7 @@ async def _ensure_index(connection: AsyncConnection, table_name: str, index_name class OceanBaseTopicMemoryVectorIndex: - """Use native HNSW indexes for complete Topic and detail embeddings.""" + """Use title HNSW and exact per-Topic detail ranking over current vectors.""" def __init__(self, profile: EmbeddingProfile) -> None: if profile.dimension < 1 or profile.distance != "l2" or profile.normalization != "unit": @@ -372,11 +381,6 @@ def __init__(self, profile: EmbeddingProfile) -> None: self.topic_table.c.embedding, params="distance=l2,type=hnsw", ) - self._chunk_index = VectorIndex( - _CHUNK_VECTOR_INDEX, - self.chunk_table.c.embedding, - params="distance=l2,type=hnsw", - ) vector_type = VECTOR(profile.dimension) self._topic_search = text(_TOPIC_VECTOR_SEARCH).bindparams(bindparam("query_vector", type_=vector_type)) self._chunk_search = text(_CHUNK_VECTOR_SEARCH).bindparams(bindparam("query_vector", type_=vector_type)) @@ -393,7 +397,6 @@ async def initialize(self, connection: AsyncConnection, /) -> None: f"OceanBase projection {table.name} uses {actual!r}; expected {expected}", ) await connection.run_sync(lambda sync: self._topic_index.create(sync, checkfirst=True)) - await connection.run_sync(lambda sync: self._chunk_index.create(sync, checkfirst=True)) async def validate_current(self, connection: AsyncConnection, /) -> None: await validate_current_topic_vectors(connection, self.topic_table, self.chunk_table, self._fingerprint) @@ -472,7 +475,6 @@ async def search( normalization=self.profile.normalization, ), "candidate_limit": request.candidate_limit, - "neighbor_limit": request.candidate_limit * TOPIC_MEMORY_CHUNK_MAX_COUNT, } topic_rows = (await connection.execute(self._topic_search, parameters)).mappings() chunk_rows = (await connection.execute(self._chunk_search, parameters)).mappings() @@ -481,51 +483,6 @@ async def search( detail_vector=tuple(_channel_hit(row, "detail_vector") for row in chunk_rows), ) - async def vector_complete( - self, - connection: AsyncConnection, - scope_id: str, - topic_ref: ArtifactRef, - /, - ) -> bool: - topic_count = int( - await connection.scalar( - select(func.count()) - .select_from(self.topic_table) - .where( - self.topic_table.c.scope_id == scope_id, - self.topic_table.c.artifact_id == topic_ref.artifact_id, - self.topic_table.c.revision == topic_ref.revision, - self.topic_table.c.profile_fingerprint == self._fingerprint, - ) - ) - or 0 - ) - expected_ordinals = set( - ( - await connection.execute( - select(TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE.c.chunk_ordinal).where( - TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE.c.scope_id == scope_id, - TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE.c.artifact_id == topic_ref.artifact_id, - TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE.c.revision == topic_ref.revision, - ) - ) - ).scalars() - ) - stored_ordinals = set( - ( - await connection.execute( - select(self.chunk_table.c.chunk_ordinal).where( - self.chunk_table.c.scope_id == scope_id, - self.chunk_table.c.artifact_id == topic_ref.artifact_id, - self.chunk_table.c.revision == topic_ref.revision, - self.chunk_table.c.profile_fingerprint == self._fingerprint, - ) - ) - ).scalars() - ) - return topic_count == 1 and bool(expected_ordinals) and stored_ordinals == expected_ordinals - def _coverage_expression(column: Any, terms: tuple[str, ...], required: int) -> Any: matches = ( @@ -542,12 +499,13 @@ def _coverage_expression(column: Any, terms: tuple[str, ...], required: int) -> def _channel_hit(row: Mapping[Any, Any], channel: TopicMemoryMatchedBy) -> TopicMemoryChannelHit: + ref = ArtifactRef(family="topic-memory", artifact_id=str(row["artifact_id"]), revision=int(row["revision"])) + if channel.endswith("_vector") and row["active_revision"] is None: + raise TopicMemoryStorageInvariantError("missing-active-head", (str(row["scope_id"]), ref.artifact_id)) + if channel == "detail_vector" and row["active_chunk_ordinal"] is None: + raise TopicMemoryStorageInvariantError("missing-active-chunks", (str(row["scope_id"]), ref)) return TopicMemoryChannelHit( - artifact_ref=ArtifactRef( - family="topic-memory", - artifact_id=str(row["artifact_id"]), - revision=int(row["revision"]), - ), + artifact_ref=ref, title=str(row["title"]), summary=str(row["summary"]), channel=channel, diff --git a/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py b/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py index 91d40b5d9f..c3552d31fb 100644 --- a/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""SQLite Atomic Memory: one current row, an FTS5 helper and exact L2.""" +"""SQLite Atomic Memory: transactional FTS5 and partitioned sqlite-vec KNN helpers.""" from __future__ import annotations +import re import struct +import sqlite_vec from aiosqlite import Connection as SQLiteConnection from sqlalchemy import text from sqlalchemy.exc import SQLAlchemyError @@ -62,6 +64,10 @@ ) _HIT_COLUMNS = "artifact_id, revision, state_version, kind, text" _FTS_IDENTITY_SQL = "'i' || lower(hex(scope_id)) || 'x' || lower(hex(artifact_id))" +_VECTOR_TABLE = "pc_atomic_memory_current_vec" +# Avoid reserving 1024 vector slots for every small Scope. +_VECTOR_CHUNK_SIZE = 128 +_VECTOR_COLUMNS = "identity, scope_id, kind, embedding, artifact_id, revision, state_version, text" class SQLiteAtomicMemoryIndex(RelationalAtomicMemoryIndex): @@ -72,7 +78,8 @@ class SQLiteAtomicMemoryIndex(RelationalAtomicMemoryIndex): neither a business JOIN nor post-limit authorization. Its indexed identity token encodes the exact Scope/Artifact pair independently of SQLite rowids, which VACUUM may change on current's composite-primary-key table. Vectors - stay in current. + use a Scope-partitioned vec0 helper with the same stable identity. Current + remains the rebuild source; helper writes and rollback share its transaction. """ def __init__(self, profile: EmbeddingProfile | None = None) -> None: @@ -105,6 +112,117 @@ async def initialize(self, connection: AsyncConnection, /) -> None: "sqlite-vector", "SQLite Atomic Memory vectors require loaded sqlite-vec" ) from error await self._initialize_fts(connection) + await self._initialize_vectors(connection, driver) + + def _invalid_vectors_sql(self) -> str: + profile = self.profile + if profile is None: + raise AtomicMemoryIndexError("embedding-profile", "Vector profile is unavailable") + # Constants are owned schema/profile values, never request input. + # Only usable current vectors are copied into the native helper. + return ( + "(embedding IS NULL OR embedding_input_hash IS NULL OR profile_fingerprint IS NULL " + f"OR profile_fingerprint <> '{atomic_memory_profile_fingerprint(profile)}' " + f"OR projection_format <> '{ATOMIC_MEMORY_PROJECTION_FORMAT}' " + f"OR LENGTH(embedding) <> {profile.dimension * 4})" + ) + + async def _initialize_vectors(self, connection: AsyncConnection, driver: SQLiteConnection) -> None: + existing = await connection.scalar( + text("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = :name"), {"name": _VECTOR_TABLE} + ) + profile = self.profile + definition = ( + None + if profile is None + else ( + f"CREATE VIRTUAL TABLE {_VECTOR_TABLE} USING vec0(" + "identity TEXT PRIMARY KEY, scope_id TEXT PARTITION KEY, kind TEXT, " + f"embedding float[{profile.dimension}], +artifact_id TEXT, +revision INTEGER, " + f"+state_version INTEGER, +text TEXT, chunk_size={_VECTOR_CHUNK_SIZE} " + f"/* {atomic_memory_profile_fingerprint(profile)} */)" + ) + ) + if existing and existing != definition: + # Disabling embeddings must also remove persisted vector triggers. + # Load only on this connection to drop the old virtual helper safely. + if profile is None: + await driver.enable_load_extension(True) + try: + await driver.load_extension(sqlite_vec.loadable_path()) + finally: + await driver.enable_load_extension(False) + for event in ("insert", "update", "delete"): + await connection.exec_driver_sql(f"DROP TRIGGER IF EXISTS pc_atomic_memory_vec_{event}") + await connection.exec_driver_sql(f"DROP TABLE {_VECTOR_TABLE}") + existing = None + if definition is None: + return + invalid = self._invalid_vectors_sql() + if not existing: + await connection.exec_driver_sql(definition) + await connection.exec_driver_sql( + f"INSERT INTO {_VECTOR_TABLE} ({_VECTOR_COLUMNS}) " # noqa: S608 + f"SELECT {_FTS_IDENTITY_SQL}, scope_id, kind, embedding, artifact_id, revision, state_version, text " + f"FROM pc_atomic_memory_current WHERE NOT {invalid}" + ) + await self._initialize_vector_triggers(connection, invalid) + + async def _initialize_vector_triggers(self, connection: AsyncConnection, invalid: str) -> None: + new_identity = _FTS_IDENTITY_SQL.replace("scope_id", "new.scope_id").replace("artifact_id", "new.artifact_id") + old_identity = _FTS_IDENTITY_SQL.replace("scope_id", "old.scope_id").replace("artifact_id", "old.artifact_id") + new_values = ", ".join( + new_identity if name == "identity" else f"new.{name}" for name in _VECTOR_COLUMNS.split(", ") + ) + for event in ("INSERT", "UPDATE", "DELETE"): + statements = [] + if event in {"UPDATE", "DELETE"}: + statements.append(f"DELETE FROM {_VECTOR_TABLE} WHERE identity = ({old_identity});") # noqa: S608 + if event in {"INSERT", "UPDATE"}: + valid_new = re.sub( + r"\b(embedding_input_hash|profile_fingerprint|projection_format|embedding)\b", r"new.\1", invalid + ) + statements.append( + f"INSERT INTO {_VECTOR_TABLE} ({_VECTOR_COLUMNS}) SELECT {new_values} WHERE NOT {valid_new};" + ) + name = f"pc_atomic_memory_vec_{event.lower()}" + if event == "UPDATE": + dependencies = ( + "scope_id, artifact_id, kind, embedding, revision, state_version, text, " + "embedding_input_hash, profile_fingerprint, projection_format" + ) + changed = " OR ".join(f"old.{column} IS NOT new.{column}" for column in dependencies.split(", ")) + definition = ( + f"CREATE TRIGGER {name} AFTER UPDATE OF {dependencies} ON pc_atomic_memory_current " + f"WHEN {changed} BEGIN {' '.join(statements)} END" + ) + existing_trigger = await connection.scalar( + text("SELECT sql FROM sqlite_master WHERE type = 'trigger' AND name = :name"), {"name": name} + ) + if existing_trigger == definition: + continue + await connection.exec_driver_sql(f"DROP TRIGGER IF EXISTS {name}") + await connection.exec_driver_sql(definition) + else: + await connection.exec_driver_sql( + f"CREATE TRIGGER IF NOT EXISTS {name} " + f"AFTER {event} ON pc_atomic_memory_current BEGIN {' '.join(statements)} END" + ) + + def _knn_sql(self, request: AtomicMemorySearchRequest | AtomicMemoryRelatedRequest, eligibility: str) -> str: + filters = " AND kind = :kind_filter" if request.filters.kind is not None else "" + if request.filters.tag_filter is not None: + filters += ( + f" AND identity IN (SELECT {_FTS_IDENTITY_SQL} FROM pc_atomic_memory_current " # noqa: S608 + f"WHERE scope_id = :scope_id AND ({eligibility}))" + ) + return ( + f"WITH nearest AS MATERIALIZED (SELECT {_HIT_COLUMNS}, distance FROM {_VECTOR_TABLE} " # noqa: S608 + f"WHERE embedding MATCH :query_vector AND k = :neighbor_limit AND scope_id = :scope_id{filters}) " + f"SELECT {_HIT_COLUMNS}, distance, 0 AS invalid_vectors FROM nearest " + "WHERE max(-1.0, min(1.0, 1.0 - distance * distance / 2.0)) >= :min_semantic_similarity " + "ORDER BY distance, artifact_id" + ) async def _initialize_fts(self, connection: AsyncConnection) -> None: existing = await connection.scalar( @@ -199,13 +317,23 @@ async def _channels( if match_query is not None: coverage, coverage_parameters = sqlite_fts_coverage_sql(terms) parameters.update(coverage_parameters) - parameters.update(fts_query=match_query, fts_required=required) + grouped = f"searchable_text:({match_query})" + parameters.update( + fts_query=grouped, + fts_scope_query=f'({grouped}) AND identity_token:"i{scope_id.encode("utf-8").hex()}x"*', + fts_required=required, + ) + # Prefix expansion is cheap only for a small Scope. The PK probe + # reads at most 65 identities; both branches have identical ranks. + rank = f"bm25({_FTS_TABLE}, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0)" rows = ( await connection.execute( text( - f"SELECT {_HIT_COLUMNS}, -bm25({_FTS_TABLE}) AS score, " # noqa: S608 - f"bm25({_FTS_TABLE}) AS raw_score, 'sqlite_bm25' AS score_metric FROM {_FTS_TABLE} " - "WHERE searchable_text MATCH :fts_query AND scope_id = :scope_id " + f"SELECT {_HIT_COLUMNS}, -{rank} AS score, " # noqa: S608 + f"{rank} AS raw_score, 'sqlite_bm25' AS score_metric FROM {_FTS_TABLE} " + f"WHERE {_FTS_TABLE} MATCH (CASE WHEN EXISTS (SELECT 1 " + "FROM pc_atomic_memory_current WHERE scope_id = :scope_id LIMIT 1 OFFSET 64) " + "THEN :fts_query ELSE :fts_scope_query END) AND scope_id = :scope_id " f"AND ({eligibility}) AND {coverage} >= :fts_required " f"ORDER BY score DESC, artifact_id{limit_sql}" ), @@ -228,21 +356,42 @@ async def _channels( DEFAULT_ADMISSION_FLOOR if request.admission is None else request.admission ).min_semantic_similarity, ) - rows = ( - await connection.execute( - text( - atomic_memory_vector_sql( - eligibility, - "sqlite", - bounded=limit is not None, - threshold=isinstance(request, AtomicMemoryRelatedRequest), - ) - ), - parameters, - ) - ).mappings() - vector = atomic_memory_channel_hits(rows, vector=True) + vector = await self._vector_hits(connection, request, eligibility, parameters, limit) return AtomicMemorySearchChannels(fts=fts, vector=vector) + async def _vector_hits( + self, + connection: AsyncConnection, + request: AtomicMemorySearchRequest | AtomicMemoryRelatedRequest, + eligibility: str, + parameters: dict[str, object], + limit: int | None, + ) -> tuple[AtomicMemoryIndexHit, ...]: + """Run exact native KNN, expanding boundary ties without scanning current bodies.""" + exact_sql = atomic_memory_vector_sql( + eligibility, + "sqlite", + bounded=limit is not None, + threshold=isinstance(request, AtomicMemoryRelatedRequest), + ) + # vec0's native k ceiling is 4096. Expand only boundary ties until + # every tied candidate is available for the Artifact-ID tie break. + knn = isinstance(request, AtomicMemorySearchRequest) and limit is not None and limit < 4096 + neighbor_limit = (limit or 0) + 1 + if knn: + parameters["neighbor_limit"] = neighbor_limit + sql = self._knn_sql(request, eligibility) if knn else exact_sql + rows = tuple((await connection.execute(text(sql), parameters)).mappings()) + if knn and limit is not None: + while len(rows) == neighbor_limit and rows[limit - 1]["distance"] == rows[-1]["distance"]: + if neighbor_limit == 4096: + rows = tuple((await connection.execute(text(exact_sql), parameters)).mappings()) + break + neighbor_limit = min(4096, neighbor_limit * 2) + parameters["neighbor_limit"] = neighbor_limit + rows = tuple((await connection.execute(text(sql), parameters)).mappings()) + rows = rows[:limit] + return atomic_memory_channel_hits(rows, vector=True) + __all__ = ["SQLiteAtomicMemoryIndex"] diff --git a/src/powercontext/builtin/persistence/sqlite/experience_index.py b/src/powercontext/builtin/persistence/sqlite/experience_index.py index 6fa3766b27..03e5430bd9 100644 --- a/src/powercontext/builtin/persistence/sqlite/experience_index.py +++ b/src/powercontext/builtin/persistence/sqlite/experience_index.py @@ -28,7 +28,7 @@ experience_searchable_text, ) from powercontext.builtin.artifacts.memory import CapabilityNotSupportedError -from powercontext.builtin.artifacts.search import AdmissionFloor, fts_match_query +from powercontext.builtin.artifacts.search import AdmissionFloor from powercontext.builtin.artifacts.skill import Skill, SkillPackageSnapshot, SkillSearchHit, skill_searchable_text from powercontext.builtin.persistence.experience_index import ( ensure_artifact_head_searchable_text, @@ -38,6 +38,7 @@ remove_experience_projection, replace_experience_projection, replace_skill_projection, + search_approved_artifact_rows, skill_search_hits, ) @@ -84,29 +85,6 @@ VALUES (:scope_id, :family, :artifact_id, :revision, :searchable_text) """ ) -_SEARCH_FTS_SQL = text( - """ - SELECT f.artifact_id, f.revision, f.searchable_text, a.content, - bm25(pc_artifact_fts) AS raw_score, 'sqlite_bm25' AS score_metric - FROM pc_artifact_fts AS f - JOIN pc_artifacts AS a - ON a.scope_id = f.scope_id - AND a.family = f.family - AND a.artifact_id = f.artifact_id - AND a.revision = f.revision - JOIN pc_artifact_heads AS h - ON h.scope_id = f.scope_id - AND h.family = f.family - AND h.artifact_id = f.artifact_id - AND h.revision = f.revision - WHERE pc_artifact_fts MATCH :query - AND f.scope_id = :scope_id - AND f.family = :family - AND h.lifecycle_state = 'active' - ORDER BY bm25(pc_artifact_fts), f.artifact_id, f.revision - LIMIT :candidate_limit - """ -) class SQLiteExperienceFTSIndex: @@ -158,20 +136,9 @@ async def search( min_score: float | None = None, require_scores: bool = False, ) -> ExperienceSearchOutcome: - match_query = fts_match_query(query) - if match_query is None: - return ExperienceSearchOutcome() - rows = ( - await connection.execute( - _SEARCH_FTS_SQL, - { - "query": match_query, - "scope_id": scope_id, - "family": Experience.family, - "candidate_limit": limit * 4, - }, - ) - ).mappings() + rows = await search_approved_artifact_rows( + connection, scope_id, Experience.family, query, limit, admission=admission + ) return experience_search_hits( rows, query, limit, scope_id, admission=admission, min_score=min_score, require_scores=require_scores ) @@ -219,20 +186,9 @@ async def search_skills( min_score: float | None = None, require_scores: bool = False, ) -> tuple[SkillSearchHit, ...]: - match_query = fts_match_query(query) - if match_query is None: - return () - rows = ( - await connection.execute( - _SEARCH_FTS_SQL, - { - "query": match_query, - "scope_id": scope_id, - "family": Skill.family, - "candidate_limit": limit * 4, - }, - ) - ).mappings() + rows = await search_approved_artifact_rows( + connection, scope_id, Skill.family, query, limit, admission=admission + ) return skill_search_hits( rows, query, limit, admission=admission, min_score=min_score, require_scores=require_scores ) diff --git a/src/powercontext/builtin/persistence/sqlite/topic_memory_index.py b/src/powercontext/builtin/persistence/sqlite/topic_memory_index.py index 0bb4559c31..608f9d52e3 100644 --- a/src/powercontext/builtin/persistence/sqlite/topic_memory_index.py +++ b/src/powercontext/builtin/persistence/sqlite/topic_memory_index.py @@ -43,7 +43,6 @@ from powercontext.builtin.artifacts.memory.canonical import canonical_embedding from powercontext.builtin.artifacts.search import analyze_text, fts_match_query, fts_query_requirements from powercontext.builtin.artifacts.topic_memory import ( - TOPIC_MEMORY_CHUNK_MAX_COUNT, TopicMemoryCapabilities, TopicMemoryCapabilityError, TopicMemoryChannelHit, @@ -107,7 +106,6 @@ SQLITE_TOPIC_MEMORY_VECTOR_TOPICS_TABLE, SQLITE_TOPIC_MEMORY_VECTOR_CHUNKS_TABLE, ) -_SQLITE_VEC_MAX_K = 4096 _CREATE_TOPIC_FTS_SQL = """ CREATE VIRTUAL TABLE IF NOT EXISTS pc_topic_memory_topic_fts USING fts5( @@ -148,89 +146,65 @@ """ ) _SEARCH_TOPIC_FTS_SQL = """ - WITH scored AS ( - SELECT artifact_id, revision, title, summary, - bm25(pc_topic_memory_topic_fts) AS score, ({coverage}) AS coverage - FROM pc_topic_memory_topic_fts - WHERE pc_topic_memory_topic_fts MATCH :query AND scope_id = :scope_id - ) - SELECT artifact_id, revision, title, summary, score AS raw_score - FROM scored - WHERE coverage >= :required_matches - ORDER BY score, artifact_id, revision DESC - LIMIT :candidate_limit - """ -_COUNT_TOPIC_FTS_SQL = """ - SELECT count(*) +WITH scored AS MATERIALIZED ( + SELECT rowid, artifact_id, revision, + bm25(pc_topic_memory_topic_fts) AS raw_score, + ({coverage}) AS coverage FROM pc_topic_memory_topic_fts WHERE pc_topic_memory_topic_fts MATCH :query AND scope_id = :scope_id - """ -_COUNT_TOPIC_FTS_ELIGIBLE_SQL = """ - WITH scored AS ( - SELECT ({coverage}) AS coverage - FROM pc_topic_memory_topic_fts - WHERE pc_topic_memory_topic_fts MATCH :query AND scope_id = :scope_id - ) - SELECT count(*) +), candidates AS ( + SELECT rowid, artifact_id, revision, raw_score FROM scored WHERE coverage >= :required_matches - """ -_SEARCH_CHUNK_FTS_SQL = """ - WITH scored AS ( - SELECT artifact_id, revision, title, summary, chunk_ordinal, start_offset, chunk_text, - bm25(pc_topic_memory_chunk_fts) AS score, ({coverage}) AS coverage - FROM pc_topic_memory_chunk_fts - WHERE pc_topic_memory_chunk_fts MATCH :query AND scope_id = :scope_id - ), ranked AS ( - SELECT scored.*, - row_number() OVER ( - PARTITION BY artifact_id, revision - ORDER BY coverage DESC, score, chunk_ordinal - ) AS topic_rank - FROM scored - ) - SELECT artifact_id, revision, title, summary, chunk_ordinal, start_offset, chunk_text, score AS raw_score - FROM ranked - WHERE topic_rank = 1 AND coverage >= :required_matches - ORDER BY score, artifact_id, revision DESC, chunk_ordinal + ORDER BY raw_score, artifact_id, revision DESC LIMIT :candidate_limit - """ -_COUNT_CHUNK_FTS_SQL = """ - WITH scored AS ( - SELECT artifact_id, revision, title, summary, chunk_ordinal, start_offset, chunk_text, - bm25(pc_topic_memory_chunk_fts) AS score, ({coverage}) AS coverage - FROM pc_topic_memory_chunk_fts - WHERE pc_topic_memory_chunk_fts MATCH :query AND scope_id = :scope_id - ), ranked AS ( - SELECT scored.*, - row_number() OVER ( - PARTITION BY artifact_id, revision - ORDER BY coverage DESC, score, chunk_ordinal - ) AS topic_rank - FROM scored - ) - SELECT count(*) +), rejected AS ( + SELECT 1 FROM scored WHERE coverage < :required_matches LIMIT 1 +) +SELECT c.artifact_id AS artifact_id, c.revision AS revision, f.title, f.summary, + c.raw_score AS raw_score, 0 AS is_rejected +FROM candidates AS c +JOIN pc_topic_memory_topic_fts AS f ON f.rowid = c.rowid +UNION ALL +SELECT NULL, NULL, NULL, NULL, NULL, 1 FROM rejected +ORDER BY is_rejected, raw_score, artifact_id, revision DESC +""" +_SEARCH_CHUNK_FTS_SQL = """ +WITH scored AS MATERIALIZED ( + SELECT rowid, artifact_id, revision, chunk_ordinal, + bm25(pc_topic_memory_chunk_fts) AS raw_score, + ({coverage}) AS coverage + FROM pc_topic_memory_chunk_fts + WHERE pc_topic_memory_chunk_fts MATCH :query AND scope_id = :scope_id +), ranked AS ( + SELECT scored.*, + row_number() OVER ( + PARTITION BY artifact_id, revision + ORDER BY coverage DESC, raw_score, chunk_ordinal + ) AS topic_rank + FROM scored +), matched AS MATERIALIZED ( + SELECT rowid, artifact_id, revision, chunk_ordinal, raw_score, coverage FROM ranked WHERE topic_rank = 1 - """ -_COUNT_CHUNK_FTS_ELIGIBLE_SQL = """ - WITH scored AS ( - SELECT artifact_id, revision, chunk_ordinal, - bm25(pc_topic_memory_chunk_fts) AS score, ({coverage}) AS coverage - FROM pc_topic_memory_chunk_fts - WHERE pc_topic_memory_chunk_fts MATCH :query AND scope_id = :scope_id - ), ranked AS ( - SELECT scored.*, - row_number() OVER ( - PARTITION BY artifact_id, revision - ORDER BY coverage DESC, score, chunk_ordinal - ) AS topic_rank - FROM scored - ) - SELECT count(*) - FROM ranked - WHERE topic_rank = 1 AND coverage >= :required_matches - """ +), candidates AS ( + SELECT rowid, artifact_id, revision, chunk_ordinal, raw_score + FROM matched + WHERE coverage >= :required_matches + ORDER BY raw_score, artifact_id, revision DESC, chunk_ordinal + LIMIT :candidate_limit +), rejected AS ( + SELECT 1 FROM matched WHERE coverage < :required_matches LIMIT 1 +) +SELECT c.artifact_id AS artifact_id, c.revision AS revision, f.title, f.summary, + c.chunk_ordinal AS chunk_ordinal, f.start_offset, f.chunk_text, + c.raw_score AS raw_score, 0 AS is_rejected +FROM candidates AS c +JOIN pc_topic_memory_chunk_fts AS f ON f.rowid = c.rowid +UNION ALL +SELECT NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 1 FROM rejected +ORDER BY is_rejected, raw_score, artifact_id, revision DESC, chunk_ordinal +""" _DELETE_TOPIC_VECTOR_SQL = text("DELETE FROM pc_topic_memory_topic_vec WHERE rowid = :vector_id") _DELETE_CHUNK_VECTOR_SQL = text("DELETE FROM pc_topic_memory_chunk_vec WHERE rowid = :vector_id") @@ -246,59 +220,65 @@ _PROBE_CHUNK_VECTOR_SQL = ( "SELECT rowid FROM pc_topic_memory_chunk_vec WHERE scope_id = ? AND embedding MATCH ? AND k = 1" ) -_TOPIC_VECTOR_SEARCH_SQL = text( - """ - WITH nearest AS ( - SELECT rowid, distance FROM pc_topic_memory_topic_vec - WHERE scope_id = :scope_id AND embedding MATCH :query_vector AND k = :neighbor_limit - AND rowid IN ( - SELECT vector_id FROM pc_topic_memory_vector_topics - WHERE scope_id = :scope_id AND profile_fingerprint = :profile_fingerprint - ) + + +def _vector_search_sql(*, detail: bool, exact: bool = False) -> str: + """Rank vector identities first; hydrate only the selected exact Topics/chunks.""" + channel = "chunk" if detail else "topic" + metadata = f"pc_topic_memory_vector_{'chunks' if detail else 'topics'}" + vectors = f"pc_topic_memory_{channel}_vec" + # Startup validates each metadata ID's native Scope; publication maintains + # that mapping atomically. Exact point reads need only the scoped metadata. + nearest = ( + f"SELECT v.vector_id AS rowid, vec_distance_l2(r.embedding, :query_vector) AS distance " # noqa: S608 + f"FROM {metadata} AS v JOIN {vectors} AS r ON r.rowid=v.vector_id " + "WHERE v.scope_id=:scope_id AND v.profile_fingerprint=:profile_fingerprint" + if exact + else f"SELECT rowid, distance FROM {vectors} " # noqa: S608 + "WHERE scope_id=:scope_id AND embedding MATCH :query_vector AND k=:neighbor_limit " + f"AND rowid IN (SELECT vector_id FROM {metadata} " + "WHERE scope_id=:scope_id AND profile_fingerprint=:profile_fingerprint)" ) - SELECT a.artifact_id, a.revision, a.title, a.summary, nearest.distance - FROM nearest - JOIN pc_topic_memory_vector_topics AS v ON v.vector_id = nearest.rowid - JOIN pc_topic_memory_active_topics AS a - ON a.scope_id = v.scope_id AND a.artifact_id = v.artifact_id AND a.revision = v.revision - WHERE v.scope_id = :scope_id - ORDER BY nearest.distance, a.artifact_id, a.revision DESC - LIMIT :candidate_limit - """ -) -_CHUNK_VECTOR_SEARCH_SQL = text( - """ - WITH nearest AS ( - SELECT rowid, distance FROM pc_topic_memory_chunk_vec - WHERE scope_id = :scope_id AND embedding MATCH :query_vector AND k = :neighbor_limit - AND rowid IN ( - SELECT vector_id FROM pc_topic_memory_vector_chunks - WHERE scope_id = :scope_id AND profile_fingerprint = :profile_fingerprint - ) - ), ranked AS ( - SELECT a.artifact_id, a.revision, a.title, a.summary, - c.chunk_ordinal, c.start_offset, c.chunk_text, nearest.distance, - row_number() OVER ( - PARTITION BY a.artifact_id, a.revision - ORDER BY nearest.distance, c.chunk_ordinal - ) AS topic_rank - FROM nearest - JOIN pc_topic_memory_vector_chunks AS v ON v.vector_id = nearest.rowid - JOIN pc_topic_memory_active_topics AS a - ON a.scope_id = v.scope_id AND a.artifact_id = v.artifact_id AND a.revision = v.revision - JOIN pc_topic_memory_active_chunks AS c - ON c.scope_id = v.scope_id AND c.artifact_id = v.artifact_id - AND c.revision = v.revision AND c.chunk_ordinal = v.chunk_ordinal - WHERE v.scope_id = :scope_id + observations = "0 AS neighbor_count" if exact else "COUNT(*) OVER () AS neighbor_count" + ordinal = ", v.chunk_ordinal" if detail else "" + rank = ( + ", row_number() OVER (PARTITION BY v.artifact_id, v.revision " + "ORDER BY n.distance, v.chunk_ordinal) AS topic_rank" + if detail + else "" + ) + order = "distance, artifact_id, revision DESC" + (", chunk_ordinal" if detail else "") + chunk_columns = ", c.chunk_ordinal, c.start_offset, c.chunk_text" if detail else "" + chunk_join = ( + "JOIN pc_topic_memory_active_chunks AS c ON c.scope_id=:scope_id " + "AND c.artifact_id=s.artifact_id AND c.revision=s.revision AND c.chunk_ordinal=s.chunk_ordinal " + if detail + else "" + ) + eligible_chunks = chunk_join.replace("s.", "v.") + representative = "WHERE topic_rank=1" if detail else "" + final_order = "s.distance, s.artifact_id, s.revision DESC" + (", s.chunk_ordinal" if detail else "") + return ( + f"WITH nearest AS MATERIALIZED ({nearest}), " # noqa: S608 + f"observed AS (SELECT rowid, distance, {observations} FROM nearest), " + f"ranked AS (SELECT v.artifact_id, v.revision{ordinal}, n.distance, " + f"n.neighbor_count{rank} FROM observed AS n " + f"JOIN {metadata} AS v ON v.vector_id=n.rowid " + "JOIN pc_topic_memory_active_topics AS a ON a.scope_id=v.scope_id " + "AND a.artifact_id=v.artifact_id AND a.revision=v.revision " + f"{eligible_chunks}WHERE v.scope_id=:scope_id), " + f"selected AS (SELECT * FROM ranked {representative} ORDER BY {order} LIMIT :candidate_limit) " + f"SELECT s.artifact_id, s.revision, a.title, a.summary{chunk_columns}, " + "s.distance, s.neighbor_count FROM selected AS s " + "JOIN pc_topic_memory_active_topics AS a ON a.scope_id=:scope_id " + f"AND a.artifact_id=s.artifact_id AND a.revision=s.revision {chunk_join}ORDER BY {final_order}" ) - SELECT artifact_id, revision, title, summary, - chunk_ordinal, start_offset, chunk_text, distance - FROM ranked - WHERE topic_rank = 1 - ORDER BY distance, artifact_id, revision DESC, chunk_ordinal - LIMIT :candidate_limit - """ -) + + +_TOPIC_VECTOR_SEARCH_SQL = text(_vector_search_sql(detail=False)) +_CHUNK_VECTOR_SEARCH_SQL = text(_vector_search_sql(detail=True)) +_TOPIC_VECTOR_EXACT_SQL = text(_vector_search_sql(detail=False, exact=True)) +_CHUNK_VECTOR_EXACT_SQL = text(_vector_search_sql(detail=True, exact=True)) class SQLiteTopicMemoryFTSIndex: @@ -410,40 +390,23 @@ async def search( "required_matches": required_matches, **{f"coverage_term_{position}": f" {term} " for position, term in enumerate(query_terms)}, } - topic_retrieved = await connection.scalar(text(_COUNT_TOPIC_FTS_SQL), parameters) - topic_eligible = await connection.scalar( - text(_COUNT_TOPIC_FTS_ELIGIBLE_SQL.format(coverage=coverage)), - parameters, + topic_rows = tuple( + (await connection.execute(text(_SEARCH_TOPIC_FTS_SQL.format(coverage=coverage)), parameters)).mappings() ) - topic_rows = ( - await connection.execute(text(_SEARCH_TOPIC_FTS_SQL.format(coverage=coverage)), parameters) - ).mappings() - detail_retrieved = await connection.scalar(text(_COUNT_CHUNK_FTS_SQL.format(coverage=coverage)), parameters) - detail_eligible = await connection.scalar( - text(_COUNT_CHUNK_FTS_ELIGIBLE_SQL.format(coverage=coverage)), - parameters, + chunk_rows = tuple( + (await connection.execute(text(_SEARCH_CHUNK_FTS_SQL.format(coverage=coverage)), parameters)).mappings() ) - chunk_rows = ( - await connection.execute(text(_SEARCH_CHUNK_FTS_SQL.format(coverage=coverage)), parameters) - ).mappings() + topic_hits = tuple(_channel_hit(row, "topic_fts") for row in topic_rows if not row["is_rejected"]) + chunk_hits = tuple(_channel_hit(row, "detail_fts") for row in chunk_rows if not row["is_rejected"]) return TopicMemorySearchChannels( - topic_fts=tuple(_channel_hit(row, "topic_fts") for row in topic_rows), - detail_fts=tuple(_channel_hit(row, "detail_fts") for row in chunk_rows), - topic_fts_retrieved=int(topic_retrieved or 0), - detail_fts_retrieved=int(detail_retrieved or 0), - topic_fts_eligible=int(topic_eligible or 0), - detail_fts_eligible=int(detail_eligible or 0), + topic_fts=topic_hits, + detail_fts=chunk_hits, + topic_fts_retrieved=len(topic_rows), + detail_fts_retrieved=len(chunk_rows), + topic_fts_eligible=len(topic_hits), + detail_fts_eligible=len(chunk_hits), ) - async def vector_complete( - self, - _connection: AsyncConnection, - _scope_id: str, - _topic_ref: ArtifactRef, - /, - ) -> bool: - return False - class SQLiteTopicMemoryVectorIndex: """Maintain complete active Topic and chunk embeddings in sqlite-vec.""" @@ -624,96 +587,26 @@ async def search( "profile_fingerprint": self._fingerprint, "candidate_limit": request.candidate_limit, } - topic_rows = ( - await connection.execute( - _TOPIC_VECTOR_SEARCH_SQL, - {**parameters, "neighbor_limit": request.candidate_limit}, - ) - ).mappings() - chunk_rows = ( - await connection.execute( - _CHUNK_VECTOR_SEARCH_SQL, - { - **parameters, - "neighbor_limit": min( - request.candidate_limit * TOPIC_MEMORY_CHUNK_MAX_COUNT, - _SQLITE_VEC_MAX_K, - ), - }, - ) - ).mappings() return TopicMemorySearchChannels( - topic_vector=tuple(_channel_hit(row, "topic_vector") for row in topic_rows), - detail_vector=tuple(_channel_hit(row, "detail_vector") for row in chunk_rows), + topic_vector=await self._vector_hits(connection, parameters, detail=False), + detail_vector=await self._vector_hits(connection, parameters, detail=True), ) - async def vector_complete( - self, - connection: AsyncConnection, - scope_id: str, - topic_ref: ArtifactRef, - /, - ) -> bool: - topic = ( - await connection.execute( - select( - SQLITE_TOPIC_MEMORY_VECTOR_TOPICS_TABLE.c.vector_id, - SQLITE_TOPIC_MEMORY_VECTOR_TOPICS_TABLE.c.profile_fingerprint, - ).where( - SQLITE_TOPIC_MEMORY_VECTOR_TOPICS_TABLE.c.scope_id == scope_id, - SQLITE_TOPIC_MEMORY_VECTOR_TOPICS_TABLE.c.artifact_id == topic_ref.artifact_id, - SQLITE_TOPIC_MEMORY_VECTOR_TOPICS_TABLE.c.revision == topic_ref.revision, - ) - ) - ).one_or_none() - if topic is None or str(topic[1]) != self._fingerprint: - return False - expected_ordinals = set( - ( - await connection.execute( - select(TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE.c.chunk_ordinal).where( - TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE.c.scope_id == scope_id, - TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE.c.artifact_id == topic_ref.artifact_id, - TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE.c.revision == topic_ref.revision, - ) - ) - ).scalars() - ) - chunks = ( - await connection.execute( - select( - SQLITE_TOPIC_MEMORY_VECTOR_CHUNKS_TABLE.c.vector_id, - SQLITE_TOPIC_MEMORY_VECTOR_CHUNKS_TABLE.c.profile_fingerprint, - SQLITE_TOPIC_MEMORY_VECTOR_CHUNKS_TABLE.c.chunk_ordinal, - ).where( - SQLITE_TOPIC_MEMORY_VECTOR_CHUNKS_TABLE.c.scope_id == scope_id, - SQLITE_TOPIC_MEMORY_VECTOR_CHUNKS_TABLE.c.artifact_id == topic_ref.artifact_id, - SQLITE_TOPIC_MEMORY_VECTOR_CHUNKS_TABLE.c.revision == topic_ref.revision, - ) + async def _vector_hits( + self, connection: AsyncConnection, parameters: Mapping[str, Any], *, detail: bool + ) -> tuple[TopicMemoryChannelHit, ...]: + limit = parameters["candidate_limit"] + neighbor_limit = limit * 4 if detail else limit + statement = _CHUNK_VECTOR_SEARCH_SQL if detail else _TOPIC_VECTOR_SEARCH_SQL + rows = tuple((await connection.execute(statement, {**parameters, "neighbor_limit": neighbor_limit})).mappings()) + if rows and rows[0]["neighbor_count"] == neighbor_limit and len(rows) < limit: + rows = tuple( + ( + await connection.execute(_CHUNK_VECTOR_EXACT_SQL if detail else _TOPIC_VECTOR_EXACT_SQL, parameters) + ).mappings() ) - ).all() - if not expected_ordinals or {int(row[2]) for row in chunks} != expected_ordinals: - return False - if str(topic[1]) != self._fingerprint or any(str(row[1]) != self._fingerprint for row in chunks): - return False - topic_vector = await connection.scalar( - text("SELECT rowid FROM pc_topic_memory_topic_vec WHERE rowid = :vector_id AND scope_id = :scope_id"), - {"vector_id": int(topic[0]), "scope_id": scope_id}, - ) - if topic_vector is None: - return False - for row in chunks: - if ( - await connection.scalar( - text( - "SELECT rowid FROM pc_topic_memory_chunk_vec WHERE rowid = :vector_id AND scope_id = :scope_id" - ), - {"vector_id": int(row[0]), "scope_id": scope_id}, - ) - is None - ): - return False - return True + channel = "detail_vector" if detail else "topic_vector" + return tuple(_channel_hit(row, channel) for row in rows) async def _delete_existing(self, connection: AsyncConnection, scope_id: str, artifact_id: str) -> None: topics = ( diff --git a/src/powercontext/builtin/persistence/topic_memory.py b/src/powercontext/builtin/persistence/topic_memory.py index 7d93fb7ec9..5c82656497 100644 --- a/src/powercontext/builtin/persistence/topic_memory.py +++ b/src/powercontext/builtin/persistence/topic_memory.py @@ -19,7 +19,7 @@ from datetime import UTC, datetime from typing import cast -from sqlalchemy import and_, delete, func, insert, or_, select, update +from sqlalchemy import and_, delete, func, insert, or_, select, tuple_, update from sqlalchemy.dialects.mysql import insert as mysql_insert from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.ext.asyncio import AsyncConnection @@ -315,39 +315,70 @@ async def get_exact( ) -> PublishedTopicMemory: """Read one exact immutable Revision even after its active Head changes.""" - if ref.family != TopicMemory.family: + return (await self.get_exact_many(connection, scope_id, (ref,)))[0] + + async def get_exact_many( + self, + connection: AsyncConnection, + scope_id: str, + refs: tuple[ArtifactRef, ...], + /, + ) -> tuple[PublishedTopicMemory, ...]: + """Read exact revisions, publication evidence and active identities in bounded batches.""" + if any(ref.family != TopicMemory.family for ref in refs): raise InvalidRepositoryArgumentError("artifact_ref", "must reference topic-memory") + if not refs: + return () await self._check_retrieval_shape(connection) - topic = await self.artifacts.get(connection, scope_id, ref) - if not isinstance(topic, TopicMemory): - raise TopicMemoryStorageInvariantError("artifact-type", ref) - published_at = await connection.scalar( - select(TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE.c.published_at).where( - TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE.c.scope_id == scope_id, - TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE.c.family == TopicMemory.family, - TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE.c.artifact_id == ref.artifact_id, - TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE.c.revision == ref.revision, - ) - ) - if not isinstance(published_at, datetime): - raise TopicMemoryStorageInvariantError("missing-publication", (scope_id, ref)) - active_revision = await connection.scalar( - select(TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.revision).where( - TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.scope_id == scope_id, - TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.family == TopicMemory.family, - TOPIC_MEMORY_ACTIVE_TOPICS_TABLE.c.artifact_id == ref.artifact_id, + topics = await self.artifacts.get_many(connection, scope_id, refs) + publications, active = TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE, TOPIC_MEMORY_ACTIVE_TOPICS_TABLE + keys = tuple(dict.fromkeys((ref.artifact_id, ref.revision) for ref in refs)) + published = {} + for offset in range(0, len(keys), 100): + rows = ( + await connection.execute( + select( + publications.c.artifact_id, + publications.c.revision, + publications.c.published_at, + active.c.revision.label("active_revision"), + ) + .outerjoin( + active, + and_( + *(active.c[name] == publications.c[name] for name in ("scope_id", "family", "artifact_id")) + ), + ) + .where( + publications.c.scope_id == scope_id, + publications.c.family == TopicMemory.family, + tuple_(publications.c.artifact_id, publications.c.revision).in_(keys[offset : offset + 100]), + ) + ) + ).mappings() + published.update({(row["artifact_id"], row["revision"]): row for row in rows}) + result = [] + for ref, topic in zip(refs, topics, strict=True): + if not isinstance(topic, TopicMemory): + raise TopicMemoryStorageInvariantError("artifact-type", ref) + row = published.get((ref.artifact_id, ref.revision)) + if row is None or not isinstance(row["published_at"], datetime): + raise TopicMemoryStorageInvariantError("missing-publication", (scope_id, ref)) + if row["active_revision"] is None: + raise TopicMemoryStorageInvariantError("missing-active-head", (scope_id, ref.artifact_id)) + current = ArtifactRef( + family=TopicMemory.family, artifact_id=ref.artifact_id, revision=int(row["active_revision"]) + ) + result.append( + PublishedTopicMemory( + topic=topic, + published_at=_aware_utc(row["published_at"]), + is_current=current == ref, + current_artifact=current, + ) ) - ) - if active_revision is None: - raise TopicMemoryStorageInvariantError("missing-active-head", (scope_id, ref.artifact_id)) - current = ArtifactRef(family=TopicMemory.family, artifact_id=ref.artifact_id, revision=int(active_revision)) await self._check_retrieval_shape(connection) - return PublishedTopicMemory( - topic=topic, - published_at=_aware_utc(published_at), - is_current=current == ref, - current_artifact=current, - ) + return tuple(result) async def browse_current( self, @@ -466,6 +497,7 @@ async def search( ) -> TopicMemorySearchResult: """Search current complete projections and fuse two or four logical channels. + The caller owns a consistent read snapshot across shape, channels and exact results. ``admission=None`` applies the historical fusion-time thresholds bit for bit. """ @@ -509,7 +541,6 @@ async def search( admission=admission, ) channels = await self.index.search(connection, scope_id, request) - await self._check_retrieval_shape(connection) outcome = _fuse_topic_memory_rankings( query, channels, @@ -685,10 +716,10 @@ def _require_retrieval_shape(self, stored_shape: str, stored_fingerprint: str | async def _check_retrieval_shape(self, connection: AsyncConnection) -> None: # Reads must neither initialize/reconfigure the store nor take its write - # lock. Check before and after hydration: SQLite's legacy SELECT mode - # need not hold a read snapshot across statements. A newly published - # Topic freezes the shape, so a racing switch cannot silently label - # another profile's results with this runtime's startup capabilities. + # lock. Search holds a snapshot; other reads check again after hydration + # because legacy SQLite SELECT need not begin one. A published Topic + # freezes the shape, preventing results from being labeled with another + # profile's startup capabilities. if not self.index.capabilities.fts: return stored = ( diff --git a/src/powercontext/builtin/persistence/topic_memory_index.py b/src/powercontext/builtin/persistence/topic_memory_index.py index fe53737068..e126f9518d 100644 --- a/src/powercontext/builtin/persistence/topic_memory_index.py +++ b/src/powercontext/builtin/persistence/topic_memory_index.py @@ -63,14 +63,6 @@ async def search( /, ) -> TopicMemorySearchChannels: ... - async def vector_complete( - self, - connection: AsyncConnection, - scope_id: str, - topic_ref: ArtifactRef, - /, - ) -> bool: ... - class NoTopicMemoryIndex: """Support exact audit reads without claiming searchable publication.""" @@ -103,15 +95,6 @@ async def search( ) -> TopicMemorySearchChannels: return TopicMemorySearchChannels() - async def vector_complete( - self, - _connection: AsyncConnection, - _scope_id: str, - _topic_ref: ArtifactRef, - /, - ) -> bool: - return False - class CompositeTopicMemoryIndex: """Compose one FTS adapter and an optional matching vector adapter.""" @@ -188,21 +171,6 @@ async def search( ) return channels - async def vector_complete( - self, - connection: AsyncConnection, - scope_id: str, - topic_ref: ArtifactRef, - /, - ) -> bool: - vector_indexes = tuple(index for index in self.indexes if index.capabilities.vector) - if not vector_indexes: - return False - for index in vector_indexes: - if not await index.vector_complete(connection, scope_id, topic_ref): - return False - return True - def _combine_retrieved(left: int | None, right: int | None) -> int | None: if left is None and right is None: diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index 80c36bc4c3..3d90d3a2ac 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -1467,6 +1467,7 @@ async def _recall_round( embedding_calls = 0 generation_calls = 0 memory_recoverable = False + shared_embedding = next((embedding for embedding in reuse.values() if embedding is not None), None) for scope_id in scope_ids: memory_limit = ( builder.memory_candidate_limit + sum(identity[0] == scope_id for identity in excluded_memory) @@ -1479,7 +1480,7 @@ async def _recall_round( memory_limit=memory_limit, experience_limit=builder.experience_candidate_limit if EXPERIENCE_FAMILY in families else 0, admission=admission, - reuse=reuse.get(scope_id), + reuse=reuse.get(scope_id, shared_embedding), allow_embedding=scope_id not in reuse or reuse[scope_id] is not None, ) memory_recoverable |= outcome.memory_recoverable @@ -1497,6 +1498,8 @@ async def _recall_round( ) experience_candidates.append(outcome.experience) reuse[scope_id] = outcome.memory_query_embedding + if outcome.memory_query_embedding is not None: + shared_embedding = outcome.memory_query_embedding admissions.extend( count for count in (outcome.memory_admission, outcome.experience_admission) if count is not None ) @@ -1507,12 +1510,21 @@ async def _recall_round( experience_candidates, builder.experience_candidate_limit, ) + atomic = self._runtime.atomic_memory + topic_embedding = topic_reuse.get(self.scope_id) + if ( + self.scope_id not in topic_reuse + and atomic is not None + and self._runtime._topic_memory_embedding_model is atomic._application.embedding_model + and bounded_topic_memory_query(request.query.strip()) == request.query + ): + topic_embedding = shared_embedding topic_outcome = ( await self._topic_memory_hits( request.query.strip(), builder.topic_memory_candidate_limit, admission=admission, - reuse=topic_reuse.get(self.scope_id), + reuse=topic_embedding, allow_embedding=self.scope_id not in topic_reuse or topic_reuse[self.scope_id] is not None, ) if TOPIC_MEMORY_FAMILY in families @@ -3729,7 +3741,6 @@ def __init__( self._topic_memory_searcher = TopicMemorySearcher( search=topic_memory_search, get=topic_memory_get, - browse=topic_memory_browse, embedding_model=topic_memory_embedding_model, observer=topic_memory_search_observer, ) diff --git a/src/powercontext/builtin/runtime/atomic_memory.py b/src/powercontext/builtin/runtime/atomic_memory.py index 0aad55967b..03cacb5648 100644 --- a/src/powercontext/builtin/runtime/atomic_memory.py +++ b/src/powercontext/builtin/runtime/atomic_memory.py @@ -571,34 +571,34 @@ async def _validate_search_hits(self, connection, hits, *, verify_vectors) -> tu "profile_fingerprint", "projection_format", ) - projected = { - row["artifact_id"]: row + heads = ARTIFACT_HEADS_TABLE + rows = { + row["projection_artifact_id"]: row for row in ( await connection.execute( - select(*(table.c[name] for name in columns)).where( - table.c.scope_id == self.scope_id, table.c.artifact_id.in_(ids) - ) - ) - ).mappings() - } - heads = { - row["artifact_id"]: row - for row in ( - await connection.execute( - select(ARTIFACT_HEADS_TABLE).where( - ARTIFACT_HEADS_TABLE.c.scope_id == self.scope_id, - ARTIFACT_HEADS_TABLE.c.family == "atomic-memory", - ARTIFACT_HEADS_TABLE.c.artifact_id.in_(ids), - ) + with_artifact_content( + select(heads, *(table.c[name].label(f"projection_{name}") for name in columns)).select_from( + table.outerjoin( + heads, + and_( + heads.c.scope_id == table.c.scope_id, + heads.c.family == "atomic-memory", + heads.c.artifact_id == table.c.artifact_id, + ), + ) + ) + ).where(table.c.scope_id == self.scope_id, table.c.artifact_id.in_(ids)) ) ).mappings() } + projected = {identity: {name: row[f"projection_{name}"] for name in columns} for identity, row in rows.items()} for hit in hits: identity = hit.artifact_ref.artifact_id - projection, head = projected.get(identity), heads.get(identity) + projection, head = projected.get(identity), rows.get(identity) if ( projection is None or head is None + or head["artifact_id"] is None or hit.artifact_ref.family != "atomic-memory" or head["revision"] != hit.artifact_ref.revision or projection["revision"] != hit.artifact_ref.revision @@ -609,8 +609,8 @@ async def _validate_search_hits(self, connection, hits, *, verify_vectors) -> tu or head["governance_generation"] != hit.state_version ): raise AtomicMemoryIndexError("stale-projection", "Atomic Memory projection identity is inconsistent") - artifacts = await application.artifacts.get_many( - connection, self.scope_id, tuple(hit.artifact_ref for hit in hits) + artifacts = await application.artifacts.decode_many( + connection, self.scope_id, tuple(rows[hit.artifact_ref.artifact_id] for hit in hits) ) for hit, artifact in zip(hits, artifacts, strict=True): if not isinstance(artifact, AtomicMemory): diff --git a/src/powercontext/builtin/runtime/atomic_memory_processing.py b/src/powercontext/builtin/runtime/atomic_memory_processing.py index 237852d6dd..2a7a493d21 100644 --- a/src/powercontext/builtin/runtime/atomic_memory_processing.py +++ b/src/powercontext/builtin/runtime/atomic_memory_processing.py @@ -60,10 +60,11 @@ InvalidInferenceOutputError, embed_query, ) +from powercontext.builtin.persistence.artifacts import with_artifact_content from powercontext.builtin.persistence.atomic_memory_index import AtomicMemoryIndexError, AtomicMemoryRelatedRequest from powercontext.builtin.persistence.cursors import SourceCursorRepository from powercontext.builtin.persistence.decision_observations import DecisionObservationRepository -from powercontext.builtin.persistence.errors import GenerationConflictError +from powercontext.builtin.persistence.errors import GenerationConflictError, RepositoryNotFoundError from powercontext.builtin.persistence.memory_windows import MemorySourceWindowRepository from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, SOURCE_JOURNAL_HEADS_TABLE @@ -483,13 +484,47 @@ async def _load_related_items(self, scope_id, hits, context, workset, artifacts) from powercontext.server.authz import AccessDeniedError recalled = [] - for hit in hits: - try: - async with self._read_transaction(context) as connection: - record = await self.application.service.get( - connection, scope_id, hit.artifact_ref.artifact_id, context + heads = ARTIFACT_HEADS_TABLE + for offset in range(0, len(hits), 100): + async with self._read_transaction(context) as connection: + allowed = [] + for hit in hits[offset : offset + 100]: + try: + await self.application.security.authorize( + connection, + scope_id, + context, + "read", + ArtifactRef(family="atomic-memory", artifact_id=hit.artifact_ref.artifact_id, revision=1), + ) + except AccessDeniedError: + continue + allowed.append(hit) + if not allowed: + continue + rows = ( + await connection.execute( + with_artifact_content(select(heads)).where( + heads.c.scope_id == scope_id, + heads.c.family == "atomic-memory", + heads.c.artifact_id.in_(tuple(hit.artifact_ref.artifact_id for hit in allowed)), + ) ) - await self.application.security.authorize(connection, scope_id, context, "write", record.ref) + ).mappings() + by_id = {row["artifact_id"]: row for row in rows} + for hit in allowed: + if hit.artifact_ref.artifact_id not in by_id: + raise RepositoryNotFoundError( + "artifact", (scope_id, "atomic-memory", hit.artifact_ref.artifact_id) + ) + records = await self.application.service.read_many( + connection, scope_id, tuple(by_id[hit.artifact_ref.artifact_id] for hit in allowed) + ) + for hit, record in zip(allowed, records, strict=True): + try: + await self.application.security.authorize(connection, scope_id, context, "write", record.ref) + except AccessDeniedError: + continue if ( record.ref != hit.artifact_ref or record.state.state_version != hit.state_version @@ -498,26 +533,24 @@ async def _load_related_items(self, scope_id, hits, context, workset, artifacts) or record.artifact.content.text != hit.text ): _conflict("Related memory changed before comparison") - except AccessDeniedError: - continue - content = AtomicMemoryContent(kind=hit.kind, text=hit.text) - supported = AtomicMemoryArtifactEvidence( - evidence_id=f"artifact:{record.ref.artifact_id}@{record.ref.revision}", - artifact_ref=record.ref, - content=content, - ) - artifacts[(record.ref.family, record.ref.artifact_id, record.ref.revision)] = record.ref - recalled.append( - workset.add( - AtomicMemoryWorkingItem( - key=f"memory:{hit.artifact_ref.artifact_id}", + content = AtomicMemoryContent(kind=hit.kind, text=hit.text) + supported = AtomicMemoryArtifactEvidence( + evidence_id=f"artifact:{record.ref.artifact_id}@{record.ref.revision}", + artifact_ref=record.ref, content=content, - origins=(record.as_read(),), - evidence=(supported,), - artifacts=(record.ref,), ) - ) - ) + artifacts[(record.ref.family, record.ref.artifact_id, record.ref.revision)] = record.ref + recalled.append( + workset.add( + AtomicMemoryWorkingItem( + key=f"memory:{hit.artifact_ref.artifact_id}", + content=content, + origins=(record.as_read(),), + evidence=(supported,), + artifacts=(record.ref,), + ) + ) + ) return recalled async def _recall(self, scope_id, query, context): diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index f3e1b03f99..e5d48fe64b 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -96,7 +96,10 @@ UsageReportingEmbeddingModel, UsageReportingStructuredGenerator, ) -from powercontext.builtin.persistence.atomic_memory_index import AtomicMemoryIndexCapabilities +from powercontext.builtin.persistence.atomic_memory_index import ( + AtomicMemoryIndexCapabilities, + validate_atomic_memory_vectors, +) from powercontext.builtin.persistence.atomic_memory_readiness import ( assert_atomic_memory_migration_ready, ensure_supported_atomic_memory_schema, @@ -740,7 +743,6 @@ async def run_profile(scope_id, high): topic_memory_searcher = TopicMemorySearcher( search=contexts.search_topic_memories, get=contexts.get_topic_memory, - browse=contexts.browse_topic_memories, embedding_model=configured_embedding, observer=topic_memory_search_observer, capabilities=contexts.topic_memory_index.capabilities, @@ -1113,6 +1115,7 @@ async def open_builtin_contexts( if not _topic_memory_worker: await _initialize_atomic_memory_authority(connection) await atomic_index.initialize(connection) + await validate_atomic_memory_vectors(connection, atomic_index) await refresh_lexical_projections( connection, atomic_index.table if atomic_index.table in atomic_index.tables else None ) @@ -1193,6 +1196,7 @@ async def open_builtin_contexts( if not _topic_memory_worker: await _initialize_atomic_memory_authority(connection) await atomic_index.initialize(connection) + await validate_atomic_memory_vectors(connection, atomic_index) await refresh_lexical_projections( connection, atomic_index.table if atomic_index.table in atomic_index.tables else None ) diff --git a/src/powercontext/builtin/runtime/topic_memory_processing.py b/src/powercontext/builtin/runtime/topic_memory_processing.py index d26932f675..77bb4614a2 100644 --- a/src/powercontext/builtin/runtime/topic_memory_processing.py +++ b/src/powercontext/builtin/runtime/topic_memory_processing.py @@ -83,7 +83,9 @@ from powercontext.builtin.artifacts.topic_memory.search import bounded_topic_memory_query from powercontext.builtin.inference import ( EmbeddingModel, + EmbeddingResult, GenerationResult, + InvalidInferenceOutputError, StructuredGenerator, TokenEstimator, character_token_estimator, @@ -478,6 +480,9 @@ def __init__( ) self._publisher = publisher self._embedding_model = embedding_model + self._document_vectors: ContextVar[dict[tuple[str, str], tuple[float, ...]] | None] = ContextVar( + "topic_memory_document_vectors", default=None + ) self._usage_reporter = usage_reporter self._history_max = history_max_candidates self._history_threshold = history_rrf_threshold @@ -521,6 +526,7 @@ async def process( fence=assignment.fence, ) token = self._work_budget.set(budget) + vector_token = self._document_vectors.set({}) self._used_stages.clear() try: await budget.begin() @@ -544,6 +550,7 @@ async def process( except RevisionConflictError: return ArtifactProcessingWorkerCompletion(ArtifactProcessingWorkerOutcome.HEAD_CONFLICT) finally: + self._document_vectors.reset(vector_token) self._work_budget.reset(token) return ArtifactProcessingWorkerCompletion() @@ -575,13 +582,24 @@ async def _reserve(self, *, requests: int, tokens: int) -> None: self._local_tokens += tokens async def _embed(self, texts: tuple[str, ...]): - if self._embedding_model is None: + model = self._embedding_model + if model is None: raise TopicMemoryGenerationError("embedding_unavailable") + cached = self._document_vectors.get() + if cached is None: + cached = {} + profile = model.profile.model_dump_json() + pending = tuple(dict.fromkeys(text for text in texts if (profile, text) not in cached)) # At most one provider request per text (the adapter may batch them). - await self._reserve( - requests=max(1, len(texts)), tokens=max(1, sum(self._stages.estimator.estimate(text) for text in texts)) - ) - return await self._embedding_model.embed(texts) + if pending: + await self._reserve( + requests=len(pending), tokens=max(1, sum(self._stages.estimator.estimate(text) for text in pending)) + ) + result = await model.embed(pending) + if len(result.vectors) != len(pending): + raise InvalidInferenceOutputError("topic-memory.index", "requires one vector per input") + cached.update(((profile, text), vector) for text, vector in zip(pending, result.vectors, strict=True)) + return EmbeddingResult(vectors=tuple(cached[profile, text] for text in texts)) async def _embed_query(self, texts: tuple[str, ...]): if self._embedding_model is None: @@ -821,10 +839,12 @@ async def _history( )[: self._history_max] id_by_key = {key: f"candidate-{index:04d}" for index, key in enumerate(ordered_keys, start=1)} candidates: dict[str, PublishedTopicMemory] = {} - async with self._database.transaction() as connection: - for key in ordered_keys: - ref = ranked[key][2].artifact_ref - candidates[id_by_key[key]] = await self._topics.get_exact(connection, scope_id, ref) + if ordered_keys: + async with self._database.transaction(consistent_snapshot=True) as connection: + publications = await self._topics.get_exact_many( + connection, scope_id, tuple(ranked[key][2].artifact_ref for key in ordered_keys) + ) + candidates = {id_by_key[key]: value for key, value in zip(ordered_keys, publications, strict=True)} projected = tuple( TopicMemoryProbeCandidates( probe_id=f"probe-{index:04d}", @@ -864,7 +884,7 @@ async def _search( embedded = await self._embed_query((query if semantic_query is None else semantic_query,)) query_vector = embedded.vectors[0] profile = self._embedding_model.profile - async with self._database.transaction() as connection: + async with self._database.transaction(consistent_snapshot=True) as connection: result = await self._topics.search( connection, scope_id, @@ -1009,7 +1029,7 @@ async def _coordinate( or f"{proposal.content.title}\n{proposal.content.summary}"[:MAX_TOPIC_MEMORY_QUERY_LENGTH].strip() ) mode: TopicMemorySearchMode = "fts" if vector is None else "hybrid" if signature else "vector" - async with self._database.transaction() as connection: + async with self._database.transaction(consistent_snapshot=True) as connection: result = await self._topics.search( connection, scope_id, @@ -1019,16 +1039,19 @@ async def _coordinate( query_vector=vector, embedding_profile=profile, ) - ids: set[str] = set() - for hit in result.hits: - candidate_id = next( - (key for key, value in candidates.items() if value.topic.as_ref() == hit.artifact_ref), - None, - ) - if candidate_id is None: + known = {_artifact_key(publication.topic.as_ref()): key for key, publication in candidates.items()} + missing = { + _artifact_key(hit.artifact_ref): hit.artifact_ref + for hit in result.hits + if _artifact_key(hit.artifact_ref) not in known + } + if missing: + publications = await self._topics.get_exact_many(connection, scope_id, tuple(missing.values())) + for ref, publication in zip(missing, publications, strict=True): candidate_id = f"candidate-{len(candidates) + 1:04d}" - candidates[candidate_id] = await self._topics.get_exact(connection, scope_id, hit.artifact_ref) - ids.add(candidate_id) + candidates[candidate_id] = publication + known[ref] = candidate_id + ids = {known[_artifact_key(hit.artifact_ref)] for hit in result.hits} secondary.append(frozenset(ids)) vectors.append(vector) components = topic_memory_related_components( diff --git a/src/powercontext/builtin/runtime/topic_memory_search.py b/src/powercontext/builtin/runtime/topic_memory_search.py index 6c3c99cc04..83ab75756e 100644 --- a/src/powercontext/builtin/runtime/topic_memory_search.py +++ b/src/powercontext/builtin/runtime/topic_memory_search.py @@ -31,7 +31,6 @@ PublishedTopicMemory, TopicArtifactSearchRequest, TopicMemoryCapabilities, - TopicMemoryCurrentItem, TopicMemorySearchResult, ) from powercontext.builtin.artifacts.topic_memory.search import TopicSearchPlan, plan_topic_search, topic_search_error @@ -53,7 +52,7 @@ class TopicMemorySearcher: The calling Runtime facade owns scoped operation and model usage accounting. Public execution forwards the Family request to the composed backend. Legacy - execution preserves its existing callback keywords and complete-read cost. + execution preserves its existing callback keywords. """ family = "topic-memory" @@ -64,14 +63,12 @@ def __init__( *, search: Callable[..., Awaitable[TopicMemorySearchResult]], get: Callable[[str, ArtifactRef], Awaitable[PublishedTopicMemory]] | None = None, - browse: Callable[..., Awaitable[tuple[TopicMemoryCurrentItem, ...]]] | None = None, embedding_model: EmbeddingModel | None = None, observer: Callable[[str, bool], None] | None = None, capabilities: TopicMemoryCapabilities | None = None, ) -> None: self._search = search self._get = get - self._browse = browse self._embedding = embedding_model self._observer = observer self._capabilities = capabilities @@ -182,15 +179,6 @@ async def _execute( keywords["artifact_request"] = artifact_request if execution_context is not None: keywords["execution_context"] = execution_context - if ( - embedding is not None - and self._browse is not None - and (execution_context is None or (execution_context.access is None and execution_context.trusted_local)) - and not await self._browse(scope_id, limit=1, after=None) - ): - if artifact_request is not None and plan is not None and not plan.fallback_allowed: - return TopicMemorySearchResult(mode=plan.mode, artifacts=()), False - embedding = None if embedding is None: result = await self._search(scope_id, query, mode="fts", **keywords) return result.model_copy(update={"embedding_calls": 0}), False diff --git a/tests/builtin/persistence/test_experience_index.py b/tests/builtin/persistence/test_experience_index.py index c04662bae6..3ba140e7bf 100644 --- a/tests/builtin/persistence/test_experience_index.py +++ b/tests/builtin/persistence/test_experience_index.py @@ -465,22 +465,17 @@ def test_skill_threshold_uses_same_score_and_admission_contract() -> None: assert hits[0].channel_scores["text"].raw == -1 -def test_oceanbase_search_selects_match_raw_and_preserves_candidate_window() -> None: +def test_oceanbase_search_preserves_match_raw_score() -> None: rows = [_experience_row("one", 2.0, "oceanbase_match")] - connection = SimpleNamespace(execute=AsyncMock(return_value=SimpleNamespace(mappings=lambda: rows))) + connection = SimpleNamespace( + dialect=SimpleNamespace(name="mysql"), + execute=AsyncMock(return_value=SimpleNamespace(mappings=lambda: rows)), + ) outcome = asyncio.run( OceanBaseExperienceFTSIndex().search( cast(AsyncConnection, connection), "project", "client", 8, require_scores=True ) ) - statement = connection.execute.await_args.args[0] - compiled = statement.compile(dialect=mysql.dialect()) - sql = str(compiled) - - assert "AS raw_score" in sql - assert "MATCH (pc_artifact_heads.searchable_text) AGAINST" in sql - assert "ORDER BY MATCH" in sql - assert " DESC, pc_artifact_heads.artifact_id, pc_artifact_heads.revision" in sql - assert 32 in compiled.params.values() assert outcome.hits[0].channel_scores is not None assert outcome.hits[0].channel_scores["text"].metric == "oceanbase_match" + assert outcome.hits[0].channel_scores["text"].raw == 2.0 diff --git a/tests/builtin/persistence/test_topic_memory.py b/tests/builtin/persistence/test_topic_memory.py index 1ce6044027..91a8f60761 100644 --- a/tests/builtin/persistence/test_topic_memory.py +++ b/tests/builtin/persistence/test_topic_memory.py @@ -114,15 +114,6 @@ async def search( ) -> TopicMemorySearchChannels: return await self.delegate.search(connection, scope_id, request) - async def vector_complete( - self, - connection: AsyncConnection, - scope_id: str, - topic_ref: ArtifactRef, - /, - ) -> bool: - return await self.delegate.vector_complete(connection, scope_id, topic_ref) - def _content(label: str, detail_term: str) -> TopicMemoryContent: return TopicMemoryContent( @@ -249,15 +240,6 @@ async def search( ) -> TopicMemorySearchChannels: return channels - async def vector_complete( - self, - _connection: AsyncConnection, - _scope_id: str, - _topic_ref: ArtifactRef, - /, - ) -> bool: - return True - topics = { ref.artifact_id: TopicMemory( artifact_id=ref.artifact_id, @@ -550,6 +532,9 @@ async def scenario() -> None: async with profile.database.transaction() as connection: old = await repository.get_exact(connection, "scope-a", first.topic.as_ref()) current = await repository.get_exact(connection, "scope-a", second.topic.as_ref()) + batch = await repository.get_exact_many( + connection, "scope-a", (second.topic.as_ref(), first.topic.as_ref(), second.topic.as_ref()) + ) browse = await repository.browse_current(connection, "scope-a", limit=10) old_search = await repository.search(connection, "scope-a", "saffron", limit=10) current_search = await repository.search(connection, "scope-a", "zircon", limit=10) @@ -567,6 +552,7 @@ async def scenario() -> None: assert not old.is_current assert old.current_artifact == second.topic.as_ref() assert current.is_current + assert batch == (current, old, current) assert current.topic.lineage.sources == (source.ref,) assert current.topic.lineage.artifacts == (first.topic.as_ref(),) assert tuple(item.artifact_ref for item in browse) == (second.topic.as_ref(),) @@ -914,8 +900,8 @@ async def scenario() -> None: assert tuple(hit.artifact_ref for hit in result.hits) == (published.topic.as_ref(),) assert result.admission is not None - assert result.admission.retrieved > MAX_TOPIC_MEMORY_SEARCH_LIMIT - assert result.admission.admitted >= 1 + assert result.admission.admitted > 0 + assert result.admission.rejected is not None and result.admission.rejected > 0 asyncio.run(scenario()) @@ -1421,7 +1407,7 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_search_rechecks_shape_after_a_concurrent_empty_store_switch(tmp_path: Path) -> None: +def test_search_snapshot_excludes_a_concurrent_shape_switch_and_rejects_it_on_the_next_read(tmp_path: Path) -> None: async def scenario() -> None: embedding_profile = EmbeddingProfile( profile_id="old", model="test", dimension=2, distance="l2", normalization="unit" @@ -1448,18 +1434,21 @@ async def search(self, connection, scope_id, request, /): old = TopicMemoryRepository(index=SwitchDuringSearch(hybrid)) async with profile.database.transaction() as connection: await old.initialize(connection) + async with profile.database.transaction(consistent_snapshot=True) as reader: + result = await old.search( + reader, + "scope-a", + "evidence", + limit=2, + mode="hybrid", + query_vector=(1.0, 0.0), + embedding_profile=embedding_profile, + ) + assert result.mode == "hybrid" and not result.hits with pytest.raises(TopicMemoryCapabilityError, match="retrieval-shape"): - async with profile.database.transaction() as reader: - await old.search( - reader, - "scope-a", - "evidence", - limit=2, - mode="hybrid", - query_vector=(1.0, 0.0), - embedding_profile=embedding_profile, - ) - async with profile.database.transaction() as reader: + async with profile.database.transaction(consistent_snapshot=True) as reader: + await old.search(reader, "scope-a", "evidence", limit=2) + async with profile.database.transaction(consistent_snapshot=True) as reader: result = await current.search(reader, "scope-a", "evidence", limit=2) assert result.mode == "fts" assert [hit.artifact_ref.artifact_id for hit in result.hits] == ["new"] @@ -1731,6 +1720,39 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("channel", ["topic", "chunk"]) +def test_startup_rejects_native_vector_scope_mapping_corruption(channel: str) -> None: + async def scenario() -> None: + embedding = EmbeddingProfile(profile_id="scope-mapping", model="fixture", dimension=2) + index = CompositeTopicMemoryIndex(SQLiteTopicMemoryFTSIndex(), SQLiteTopicMemoryVectorIndex(embedding)) + repository = TopicMemoryRepository(index=index) + content = _content("Evidence", "scope") + projection = prepare_topic_memory_projection( + content, + topic_embedding=(1.0, 0.0), + chunk_embeddings=((1.0, 0.0),), + embedding_profile=embedding, + ) + async with SQLiteProfile.open( + SQLiteConfig(), tables=BUILTIN_TABLES + index.tables, load_vector_extension=True + ) as profile: + async with profile.database.transaction() as connection: + await repository.initialize(connection) + await repository.publish_create(connection, "scope-a", "topic-1", _draft(content), projection) + native = f"pc_topic_memory_{channel}_vec" + row = (await connection.exec_driver_sql(f"SELECT rowid, embedding FROM {native}")).one() # noqa: S608 + await connection.exec_driver_sql(f"DELETE FROM {native} WHERE rowid=?", (row[0],)) # noqa: S608 + await connection.exec_driver_sql( + f"INSERT INTO {native} (rowid, scope_id, embedding) VALUES (?, ?, ?)", # noqa: S608 + (row[0], "foreign", row[1]), + ) + with pytest.raises(TopicMemoryStorageInvariantError, match="incomplete-vector"): + async with profile.database.transaction() as connection: + await repository.initialize(connection) + + asyncio.run(scenario()) + + def test_unit_vector_contract_is_applied_at_publication_and_query_boundaries() -> None: async def scenario() -> None: embedding_profile = EmbeddingProfile( @@ -2190,3 +2212,125 @@ async def scenario() -> None: assert (await connection.exec_driver_sql(query)).scalar_one() == 0 asyncio.run(scenario()) + + +def test_dense_detail_neighbors_do_not_hide_a_topic_after_chunk_collapse() -> None: + import math + + async def scenario() -> None: + embedding = EmbeddingProfile(profile_id="dense-topics", model="fixture", dimension=2) + index = CompositeTopicMemoryIndex(SQLiteTopicMemoryFTSIndex(), SQLiteTopicMemoryVectorIndex(embedding)) + repository = TopicMemoryRepository(index=index) + async with SQLiteProfile.open( + SQLiteConfig(), tables=BUILTIN_TABLES + index.tables, load_vector_extension=True + ) as profile: + async with profile.database.transaction() as connection: + await repository.initialize(connection) + dense = TopicMemoryContent( + title="Dense siblings", + summary="Many close detail chunks", + detail="\n\n".join(["small", "a" * 1801] * 44), + ) + unembedded = prepare_topic_memory_projection(dense) + projection = prepare_topic_memory_projection( + dense, + topic_embedding=(0.9, math.sqrt(0.19)), + chunk_embeddings=((1.0, 0.0),) * len(unembedded.chunks), + embedding_profile=embedding, + ) + for number in range(32): + await repository.publish_create( + connection, "scope", f"dense-{number:02}", _draft(dense), projection + ) + background = TopicMemoryContent(title="Background", summary="Other evidence", detail="Other detail") + projection = prepare_topic_memory_projection( + background, + topic_embedding=(0.95, math.sqrt(1 - 0.95**2)), + chunk_embeddings=((-1.0, 0.0),), + embedding_profile=embedding, + ) + for number in range(40): + await repository.publish_create( + connection, "scope", f"background-{number:02}", _draft(background), projection + ) + target = TopicMemoryContent(title="Target", summary="Complementary evidence", detail="Target detail") + await repository.publish_create( + connection, + "scope", + "target", + _draft(target), + prepare_topic_memory_projection( + target, + topic_embedding=(1.0, 0.0), + chunk_embeddings=((0.6, 0.8),), + embedding_profile=embedding, + ), + ) + async with profile.database.transaction(consistent_snapshot=True) as connection: + result = await repository.search( + connection, + "scope", + "semantic", + limit=20, + mode="vector", + query_vector=(1.0, 0.0), + embedding_profile=embedding, + ) + assert result.hits[0].artifact_ref.artifact_id == "target" + assert set(result.hits[0].matched_by) == {"topic_vector", "detail_vector"} + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("reverse", [False, True]) +def test_vector_boundary_ties_preserve_capacity_and_strictly_nearer_topics(reverse: bool) -> None: + async def scenario() -> None: + embedding = EmbeddingProfile(profile_id="topic-ties", model="fixture", dimension=2) + index = CompositeTopicMemoryIndex(SQLiteTopicMemoryFTSIndex(), SQLiteTopicMemoryVectorIndex(embedding)) + repository = TopicMemoryRepository(index=index) + content = TopicMemoryContent(title="Equal topic", summary="Equal evidence", detail="Equal detail") + projection = prepare_topic_memory_projection( + content, + topic_embedding=(0.6, 0.8), + chunk_embeddings=((0.6, 0.8),), + embedding_profile=embedding, + ) + async with SQLiteProfile.open( + SQLiteConfig(), tables=BUILTIN_TABLES + index.tables, load_vector_extension=True + ) as profile: + async with profile.database.transaction() as connection: + await repository.initialize(connection) + for number in reversed(range(100)) if reverse else range(100): + await repository.publish_create( + connection, "scope", f"topic-{number:03}", _draft(content), projection + ) + await repository.publish_create( + connection, + "scope", + "target", + _draft(content), + prepare_topic_memory_projection( + content, + topic_embedding=(1.0, 0.0), + chunk_embeddings=((1.0, 0.0),), + embedding_profile=embedding, + ), + ) + async with profile.database.transaction(consistent_snapshot=True) as connection: + result = await repository.search( + connection, + "scope", + "equal", + limit=8, + mode="vector", + query_vector=(1.0, 0.0), + embedding_profile=embedding, + ) + identities = [hit.artifact_ref.artifact_id for hit in result.hits] + assert len(set(identities)) == 8 + assert identities[0] == "target" + scores = [hit.score for hit in result.hits] + assert scores == sorted(scores, reverse=True) + assert all(identity.startswith("topic-") for identity in identities[1:]) + + asyncio.run(scenario()) diff --git a/tests/builtin/persistence/test_topic_memory_oceanbase.py b/tests/builtin/persistence/test_topic_memory_oceanbase.py index 2dc5a266d6..2c06408430 100644 --- a/tests/builtin/persistence/test_topic_memory_oceanbase.py +++ b/tests/builtin/persistence/test_topic_memory_oceanbase.py @@ -16,7 +16,6 @@ import asyncio import os -import re from typing import cast from unittest.mock import AsyncMock, MagicMock from uuid import uuid4 @@ -52,7 +51,6 @@ from powercontext.builtin.persistence.topic_memory import TopicMemoryRepository from powercontext.builtin.persistence.topic_memory_index import ( CompositeTopicMemoryIndex, - topic_memory_embedding_profile_fingerprint, ) @@ -191,8 +189,7 @@ async def scenario() -> None: ) as profile: async with profile.database.transaction() as connection: await repository.initialize(connection) - # Reverse insertion order, equal distances, multiple chunks: - # tie-break and collapse must occur outside the bounded ANN. + # Equal distances across multi-chunk Topics retain ordering and Scope isolation. for scope, label in (("scope-a", "zulu"), ("scope-a", "alpha"), ("scope-b", "hidden")): content = TopicMemoryContent( title=f"{label} recovery", @@ -240,66 +237,12 @@ async def scenario() -> None: ) for hits in (channels.topic_vector, channels.detail_vector): assert [hit.artifact_ref.artifact_id for hit in hits] == ["topic-alpha", "topic-zulu"] - assert [hit.chunk_ordinal for hit in channels.detail_vector] == [0, 0] + assert all( + hit.chunk_ordinal is not None and hit.distance == 0 for hit in channels.detail_vector + ) finally: if database_created: async with server_profile.database.transaction() as connection: await connection.exec_driver_sql(f"DROP DATABASE `{database_name}`") asyncio.run(scenario()) - - -@pytest.mark.parametrize("mode", ["vector", "hybrid"]) -def test_oceanbase_detail_vector_collapses_topics_before_the_channel_limit(mode) -> None: - async def scenario() -> None: - connection = AsyncMock(spec=AsyncConnection) - connection.execute.return_value.mappings = MagicMock(return_value=()) - index = OceanBaseTopicMemoryVectorIndex(_embedding_profile()) - - result = await index.search( - cast(AsyncConnection, connection), - "scope-a", - TopicMemorySearchRequest( - query="semantic evidence", - analyzed_query="semantic evidence", - candidate_limit=2, - mode=mode, - query_vector=(1.0, 0.0, 0.0), - embedding_profile=_embedding_profile(), - ), - ) - - statements = [str(call.args[0]) for call in connection.execute.await_args_list] - # OceanBase's grammar is `order_by opt_approx limit_clause`: the - # modifier follows the ENTIRE sort list, never an individual key. - for statement in statements: - assert re.search(r"\bAPPROXIMATE\s+LIMIT\s+:(?:candidate|neighbor)_limit\b", statement) - assert not re.search(r"\bAPPROXIMATE\s*,", statement) - # The vector executor (not just the parser) rejects secondary - # sort keys on ANN queries. Tie-break only AFTER bounded KNN. - assert re.search( - r"ORDER BY\s+l2_distance\([^()]+\)\s+APPROXIMATE\s+LIMIT\s+:(?:candidate|neighbor)_limit", - statement, - ) - # CE 4.3.5.6 loses same-scope neighbors if ANN itself joins the - # active tables. Hydrate only AFTER bounded, join-free selection. - ann_selection = statement[: statement.index("APPROXIMATE")] - assert "JOIN" not in ann_selection - assert "WHERE v.scope_id = :scope_id" in ann_selection - assert "v.profile_fingerprint = :profile_fingerprint" in ann_selection - assert statement.index("JOIN pc_topic_memory_active_topics") > statement.index("APPROXIMATE") - assert statement.rfind("ORDER BY distance, artifact_id, revision DESC") > statement.index("APPROXIMATE") - assert statements[0].index("LIMIT :candidate_limit") < statements[0].rfind("ORDER BY distance") - assert "row_number() OVER" in statements[1] - assert "WHERE topic_rank = 1" in statements[1] - assert statements[1].find("LIMIT :neighbor_limit") < statements[1].find("row_number() OVER") - assert statements[1].rfind("LIMIT :candidate_limit") > statements[1].rfind("WHERE topic_rank = 1") - assert connection.execute.await_args_list[1].args[1]["neighbor_limit"] == 400 - for call in connection.execute.await_args_list: - assert call.args[1]["profile_fingerprint"] == topic_memory_embedding_profile_fingerprint( - _embedding_profile() - ) - assert result.topic_vector == () - assert result.detail_vector == () - - asyncio.run(scenario()) diff --git a/tests/builtin/runtime/test_composition_embedding.py b/tests/builtin/runtime/test_composition_embedding.py index 7ca08bfdc5..33cc02e906 100644 --- a/tests/builtin/runtime/test_composition_embedding.py +++ b/tests/builtin/runtime/test_composition_embedding.py @@ -30,6 +30,7 @@ prepare_topic_memory_projection, ) from powercontext.builtin.inference import EmbeddingResult +from powercontext.builtin.persistence.atomic_memory_index import AtomicMemoryIndexError from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.records import ArtifactWrite from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts @@ -112,7 +113,9 @@ async def scenario() -> None: @pytest.mark.parametrize("existing_memory", [False, True]) -def test_empty_topic_database_allows_embedding_configuration_changes(tmp_path: Path, existing_memory: bool) -> None: +def test_embedding_configuration_changes_respect_each_family_projection_lifecycle( + tmp_path: Path, existing_memory: bool +) -> None: class Embedding: def __init__(self, profile_id: str) -> None: self.profile = EmbeddingProfile( @@ -133,12 +136,15 @@ async def scenario() -> None: ArtifactWrite(content={"kind": "decision", "text": "Preserve ordinary Memory."}), ) - # Reopening an unused Topic store must not lock out ordinary Memory - # embeddings, disabling embeddings, or changing a compatible profile. + # An empty Topic store imposes no profile lock. Existing Atomic rows + # without vectors require offline rebuilding before enabling embeddings. for embedding in (Embedding("first"), Embedding("second"), None, Embedding("third")): + if existing_memory and embedding is not None: + with pytest.raises(AtomicMemoryIndexError, match="rebuild-projection"): + async with open_builtin_contexts(config, embedding_model=embedding): + pass + continue async with open_builtin_contexts(config, embedding_model=embedding) as contexts: - assert contexts.atomic_memory.index.capabilities.vector is (embedding is not None) - assert contexts.topic_memory_index.capabilities.vector is (embedding is not None) if existing_memory: assert memory is not None result = await contexts.atomic_memory.for_scope("project").search("ordinary", mode="text") diff --git a/tests/builtin/runtime/test_topic_memory_processing.py b/tests/builtin/runtime/test_topic_memory_processing.py index 20948209ec..2db62f68e7 100644 --- a/tests/builtin/runtime/test_topic_memory_processing.py +++ b/tests/builtin/runtime/test_topic_memory_processing.py @@ -1087,6 +1087,9 @@ async def search(self, _connection, _scope_id, query, **_kwargs): async def get_exact(self, _connection, _scope_id, ref): return stored[ref.artifact_id] + async def get_exact_many(self, connection, scope_id, refs): + return tuple([await self.get_exact(connection, scope_id, ref) for ref in refs]) + processor = TopicMemoryProcessor( database=profile.database, sources=sources, @@ -1176,6 +1179,9 @@ async def search(self, _connection, _scope_id, _query, **_kwargs): async def get_exact(self, _connection, _scope_id, _ref): return published + async def get_exact_many(self, connection, scope_id, refs): + return tuple([await self.get_exact(connection, scope_id, ref) for ref in refs]) + global_proposals = tuple( TopicMemoryProposal( candidate_id=target, @@ -1588,6 +1594,9 @@ async def search(self, _connection, _scope_id, _query, **_kwargs): async def get_exact(self, _connection, _scope_id, ref): return published[ref.artifact_id] + async def get_exact_many(self, connection, scope_id, refs): + return tuple([await self.get_exact(connection, scope_id, ref) for ref in refs]) + try: stages = _stages(probe=TopicMemoryProbeOutput(), global_output=TopicMemoryGlobalOutput()) processor = TopicMemoryProcessor( @@ -1673,6 +1682,9 @@ async def search(self, _connection, _scope_id, _query, **_kwargs): async def get_exact(self, _connection, _scope_id, _ref): return published + async def get_exact_many(self, connection, scope_id, refs): + return tuple([await self.get_exact(connection, scope_id, ref) for ref in refs]) + class UnexpectedPublisher: async def publish(self, *_args, **_kwargs): raise AssertionError("unexpected publish") # noqa: TRY003 @@ -1764,6 +1776,9 @@ async def search(self, _connection, _scope_id, _query, **_kwargs): async def get_exact(self, _connection, _scope_id, ref): return published[ref.artifact_id] + async def get_exact_many(self, connection, scope_id, refs): + return tuple([await self.get_exact(connection, scope_id, ref) for ref in refs]) + leases = ArtifactProcessingLeaseRepository() async with profile.database.transaction() as connection: await sources.add( diff --git a/tests/builtin/runtime/test_topic_memory_search.py b/tests/builtin/runtime/test_topic_memory_search.py index 95603d0565..7ebccc911d 100644 --- a/tests/builtin/runtime/test_topic_memory_search.py +++ b/tests/builtin/runtime/test_topic_memory_search.py @@ -43,7 +43,7 @@ async def embed(self, texts, /): return EmbeddingResult(vectors=((1.0, 0.0),)) -def _service(*, embedding=None, capabilities=None, browse=None, calls=None, observer=None): +def _service(*, embedding=None, capabilities=None, calls=None, observer=None): from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher async def search(scope_id, query, /, **kwargs): @@ -52,7 +52,7 @@ async def search(scope_id, query, /, **kwargs): return TopicMemorySearchResult(mode=kwargs["mode"], artifacts=() if "artifact_request" in kwargs else None) return TopicMemorySearcher( - search=search, get=None, browse=browse, embedding_model=embedding, capabilities=capabilities, observer=observer + search=search, get=None, embedding_model=embedding, capabilities=capabilities, observer=observer ) @@ -81,10 +81,7 @@ def test_topic_service_dispatches_explicit_modes(mode): def test_topic_service_validates_vector_requirements_before_empty_scope(fields): calls = [] - async def browse(scope_id, *, limit, after): - raise AssertionError("invalid request must fail before browsing") # noqa: TRY003 - - service = _service(capabilities=TopicMemoryCapabilities(fts=True), browse=browse, calls=calls) + service = _service(capabilities=TopicMemoryCapabilities(fts=True), calls=calls) with pytest.raises(ValidationError): asyncio.run(service.search("scope", TopicArtifactSearchRequest.model_validate({"query": "needle", **fields}))) assert calls == [] @@ -179,7 +176,6 @@ async def scenario(): async with open_builtin_contexts(BuiltinConfig(), embedding_model=embedding) as contexts: service = TopicMemorySearcher( search=contexts.search_topic_memories, - browse=contexts.browse_topic_memories, embedding_model=embedding, capabilities=contexts.topic_memory_index.capabilities, ) @@ -208,13 +204,9 @@ def test_topic_service_rejects_mismatched_embedding_profile_before_empty_scope() embedding = Embedding() profile = EmbeddingProfile(profile_id="other", model="other", dimension=2) - async def browse(scope, *, limit, after): - return () - service = _service( embedding=embedding, capabilities=TopicMemoryCapabilities(fts=True, vector=True, hybrid=True, embedding_profile=profile), - browse=browse, ) with pytest.raises(ValidationError): asyncio.run(service.search("scope", TopicArtifactSearchRequest(query="needle"))) @@ -233,7 +225,7 @@ async def browse(scope, *, limit, after): ({"fusion": {"method": "unknown"}}, ("fusion", "method")), ], ) -def test_topic_service_reports_mode_parameter_conflicts_before_browsing(fields, path): +def test_topic_service_reports_mode_parameter_conflicts_before_inference(fields, path): embedding = Embedding() capabilities = TopicMemoryCapabilities(fts=True, vector=True, hybrid=True, embedding_profile=embedding.profile) service = _service(embedding=embedding, capabilities=capabilities) @@ -242,18 +234,14 @@ def test_topic_service_reports_mode_parameter_conflicts_before_browsing(fields, assert error.value.errors()[0]["loc"] == path -def test_topic_service_default_empty_scope_skips_embedding_but_validates_explicit_parameters(): +def test_topic_service_empty_results_use_the_requested_mode_and_validate_parameters(): embedding = Embedding() calls = [] - async def browse(scope, *, limit, after): - return () - capabilities = TopicMemoryCapabilities(fts=True, vector=True, hybrid=True, embedding_profile=embedding.profile) - service = _service(embedding=embedding, capabilities=capabilities, browse=browse, calls=calls) + service = _service(embedding=embedding, capabilities=capabilities, calls=calls) result = asyncio.run(service.search("scope", TopicArtifactSearchRequest(query="needle"))) - assert result.mode == "fts" - assert embedding.calls == 0 + assert result.mode == "hybrid" with pytest.raises(ValidationError): asyncio.run( service.search( diff --git a/tests/e2e/test_atomic_memory_vector_index.py b/tests/e2e/test_atomic_memory_vector_index.py new file mode 100644 index 0000000000..43af570d86 --- /dev/null +++ b/tests/e2e/test_atomic_memory_vector_index.py @@ -0,0 +1,314 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bounded recall and complete enumeration keep eligibility and readiness contracts.""" + +import asyncio +from dataclasses import replace + +import pytest +from sqlalchemy import update + +from powercontext.artifacts import ArtifactRef +from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent +from powercontext.builtin.artifacts.memory import EmbeddingProfile +from powercontext.builtin.artifacts.search import AdmissionFloor +from powercontext.builtin.inference import EmbeddingResult +from powercontext.builtin.persistence.atomic_memory_index import ( + AtomicMemoryIndexError, + AtomicMemoryIndexFilter, + AtomicMemoryProjection, + AtomicMemoryProjectionPublisher, + AtomicMemoryRelatedRequest, + AtomicMemorySearchRequest, +) +from powercontext.builtin.tags import TagFilter +from tests.e2e.atomic_memory_migration_backend import MigrationBackend, migration_index, migration_profile + + +class Embedding: + profile = EmbeddingProfile(profile_id="indexed-recall", model="fixture", dimension=3, normalization="unit") + + async def embed(self, texts, /): + return EmbeddingResult(vectors=tuple((1.0, 0.0, 0.0) for _ in texts)) + + +@pytest.mark.parametrize("size", [3, 80]) +def test_sqlite_lexical_scope_filter_preserves_global_raw_scores_and_complete_results(size: int) -> None: + from powercontext.builtin.persistence.sqlite import SQLiteConfig + from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex + + async def scenario() -> None: + index = SQLiteAtomicMemoryIndex() + preparer = AtomicMemoryProjectionPublisher(index) + scope = "项目a" + async with migration_profile(SQLiteConfig()) as profile, profile.database.transaction() as connection: + await index.initialize(connection) + for target in (scope, scope + "b", scope.upper()): + for number in range(size): + prepared = await preparer.prepare( + AtomicMemoryContent( + kind="fact" if number % 2 else "preference", + text=f"alpha beta {target} evidence {number}", + ) + ) + await index.replace( + connection, + target, + AtomicMemoryProjection( + artifact_ref=ArtifactRef( + family="atomic-memory", artifact_id=f"memory-{number:02}", revision=1 + ), + state_version=0, + prepared=prepared, + tag_keys=("keep",), + ), + ) + all_matches = await index.enumerate_related( + connection, scope, AtomicMemoryRelatedRequest(query="alpha beta", filters=AtomicMemoryIndexFilter()) + ) + assert len(all_matches) == size + assert {hit.text for hit in all_matches} == { + f"alpha beta {scope} evidence {number}" for number in range(size) + } + expected = [hit for hit in all_matches if hit.kind == "fact"] + filters = AtomicMemoryIndexFilter(kind="fact", tag_filter=TagFilter(tags=("keep",), match="all")) + request = AtomicMemorySearchRequest(query="alpha beta", filters=filters, mode="fts", limit=2) + hits = (await index.search(connection, scope, request)).fts + complete = await index.enumerate_related( + connection, scope, AtomicMemoryRelatedRequest(query="alpha beta", filters=filters) + ) + assert [(hit.artifact_ref, hit.channel_scores) for hit in hits] == [ + (hit.artifact_ref, hit.channel_scores) for hit in expected[:2] + ] + assert [(hit.artifact_ref, hit.channel_scores) for hit in complete] == [ + (hit.artifact_ref, hit.channel_scores) for hit in expected + ] + + asyncio.run(scenario()) + + +def test_vector_search_filters_and_complete_threshold_enumeration(migration_backend: MigrationBackend) -> None: + async def scenario(): + model = Embedding() + index = migration_index(migration_backend.config, model.profile) + preparer = AtomicMemoryProjectionPublisher(index, embedding_model=model) + async with migration_profile(migration_backend.config, load_vector_extension=True) as profile: + async with profile.database.transaction() as connection: + await index.initialize(connection) + for n in range(40): + prepared = await preparer.prepare( + AtomicMemoryContent(kind="fact" if n % 2 else "preference", text=f"Semantic fact {n}.") + ) + await index.replace( + connection, + "scope", + AtomicMemoryProjection( + artifact_ref=ArtifactRef(family="atomic-memory", artifact_id=f"memory-{n:02}", revision=1), + state_version=0, + prepared=prepared, + tag_keys=("keep" if n % 3 else "other",), + ), + ) + await index.replace( + connection, + "foreign", + AtomicMemoryProjection( + artifact_ref=ArtifactRef(family="atomic-memory", artifact_id="foreign", revision=1), + state_version=0, + prepared=prepared, + tag_keys=("keep",), + ), + ) + filters = AtomicMemoryIndexFilter(kind="fact", tag_filter=TagFilter(tags=("keep",), match="all")) + expected = {f"memory-{n:02}" for n in range(40) if n % 2 and n % 3} + request = AtomicMemorySearchRequest( + query="semantic", + filters=filters, + mode="vector", + limit=5, + query_vector=(1.0, 0.0, 0.0), + embedding_profile=model.profile, + ) + related = AtomicMemoryRelatedRequest( + query="semantic", + filters=filters, + mode="vector", + query_vector=(1.0, 0.0, 0.0), + embedding_profile=model.profile, + max_distance=0.1, + ) + async with profile.database.transaction() as connection: + hits = (await index.search(connection, "scope", request)).vector + assert len(hits) == 5 + assert {h.artifact_ref.artifact_id for h in hits} <= expected + all_hits = await index.enumerate_related(connection, "scope", related) + assert {h.artifact_ref.artifact_id for h in all_hits} == expected + expanded = (await index.search(connection, "scope", replace(request, limit=20000))).vector + assert {h.artifact_ref.artifact_id for h in expanded} == expected + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("corruption", ["missing", "older-profile", "newer-profile", "older-format", "newer-format"]) +def test_bounded_search_skips_unusable_vectors_while_complete_operations_reject_them( + migration_backend: MigrationBackend, corruption +): + async def scenario(): + model = Embedding() + index = migration_index(migration_backend.config, model.profile) + preparer = AtomicMemoryProjectionPublisher(index, embedding_model=model) + async with migration_profile(migration_backend.config, load_vector_extension=True) as profile: + async with profile.database.transaction() as connection: + await index.initialize(connection) + prepared = await preparer.prepare(AtomicMemoryContent(kind="fact", text="Semantic fact.")) + projections = tuple( + AtomicMemoryProjection( + artifact_ref=ArtifactRef(family="atomic-memory", artifact_id=identity, revision=1), + state_version=0, + prepared=prepared, + tag_keys=("keep",), + ) + for identity in ("broken", "healthy") + ) + for projection in projections: + await index.replace(connection, "scope", projection) + values = ( + {"embedding": None, "profile_fingerprint": None, "embedding_input_hash": None} + if corruption == "missing" + else { + "profile_fingerprint" if "profile" in corruption else "projection_format": "0" + if "older" in corruption + else "z" + } + ) + await connection.execute( + update(index.table).where(index.table.c.artifact_id == "broken").values(**values) + ) + request = AtomicMemorySearchRequest( + query="semantic", + filters=AtomicMemoryIndexFilter(), + mode="vector", + limit=5, + query_vector=(1.0, 0.0, 0.0), + embedding_profile=model.profile, + ) + related = AtomicMemoryRelatedRequest( + query=request.query, + filters=request.filters, + mode="vector", + query_vector=request.query_vector, + embedding_profile=model.profile, + max_distance=0.1, + ) + async with profile.database.transaction() as connection: + for limit in (5, 20000): + hits = (await index.search(connection, "scope", replace(request, limit=limit))).vector + assert [hit.artifact_ref.artifact_id for hit in hits] == ["healthy"] + for operation in ( + index.enumerate_related(connection, "scope", related), + index.probe_recoverable(connection, "scope", request, AdmissionFloor(min_semantic_similarity=-1)), + ): + with pytest.raises(AtomicMemoryIndexError, match="incomplete"): + await operation + assert not await index.enumerate_related( + connection, "scope", replace(related, filters=AtomicMemoryIndexFilter(kind="other")) + ) + assert not await index.enumerate_related(connection, "foreign", related) + savepoint = await connection.begin_nested() + await index.replace(connection, "scope", projections[0]) + assert len(await index.enumerate_related(connection, "scope", related)) == 2 + await savepoint.rollback() + with pytest.raises(AtomicMemoryIndexError, match="incomplete"): + await index.enumerate_related(connection, "scope", related) + + asyncio.run(scenario()) + + +def test_sqlite_reopen_tag_updates_rollback_and_disabling_embeddings_preserve_search_results(tmp_path): + from powercontext.builtin.persistence.sqlite import SQLiteConfig + from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex + + async def scenario(): + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'vectors.db'}") + model = Embedding() + index = SQLiteAtomicMemoryIndex(model.profile) + preparer = AtomicMemoryProjectionPublisher(index, embedding_model=model) + request = AtomicMemorySearchRequest( + query="semantic", + filters=AtomicMemoryIndexFilter(), + mode="vector", + limit=5, + query_vector=(1.0, 0.0, 0.0), + embedding_profile=model.profile, + ) + async with migration_profile(config, load_vector_extension=True) as profile: + async with profile.database.transaction() as connection: + await index.initialize(connection) + prepared = await preparer.prepare(AtomicMemoryContent(kind="fact", text="Semantic fact.")) + for number in reversed(range(13)): + await index.replace( + connection, + "scope", + AtomicMemoryProjection( + artifact_ref=ArtifactRef(family="atomic-memory", artifact_id=f"m{number:02}", revision=1), + state_version=0, + prepared=prepared, + tag_keys=(), + ), + ) + before = (await index.search(connection, "scope", request)).vector + assert [hit.artifact_ref.artifact_id for hit in before] == [f"m{n:02}" for n in range(5)] + async with profile.database.connection() as connection: + await connection.exec_driver_sql("VACUUM") + async with ( + migration_profile(config, load_vector_extension=True) as profile, + profile.database.transaction() as connection, + ): + await index.initialize(connection) + assert (await index.search(connection, "scope", request)).vector == before + await index.refresh_tags(connection, "scope", "m00", ("keep",)) + filtered = replace(request, filters=AtomicMemoryIndexFilter(tag_filter=TagFilter(tags=("keep",)))) + assert [ + hit.artifact_ref.artifact_id for hit in (await index.search(connection, "scope", filtered)).vector + ] == ["m00"] + savepoint = await connection.begin_nested() + changed = await preparer.prepare(AtomicMemoryContent(kind="preference", text="Changed fact.")) + await index.replace( + connection, + "scope", + AtomicMemoryProjection( + artifact_ref=ArtifactRef(family="atomic-memory", artifact_id="m00", revision=2), + state_version=1, + prepared=changed, + tag_keys=("keep",), + ), + ) + (updated,) = (await index.search(connection, "scope", filtered)).vector + assert (updated.artifact_ref.revision, updated.kind, updated.text) == (2, "preference", "Changed fact.") + await savepoint.rollback() + assert (await index.search(connection, "scope", request)).vector == before + async with migration_profile(config) as profile, profile.database.transaction() as connection: + lexical = SQLiteAtomicMemoryIndex() + await lexical.initialize(connection) + await lexical.delete(connection, "scope", "m00") + async with ( + migration_profile(config, load_vector_extension=True) as profile, + profile.database.transaction() as connection, + ): + await index.initialize(connection) + after = (await index.search(connection, "scope", request)).vector + assert [hit.artifact_ref.artifact_id for hit in after] == [f"m{n:02}" for n in range(1, 6)] + + asyncio.run(scenario()) diff --git a/tests/e2e/test_lexical_admission.py b/tests/e2e/test_lexical_admission.py new file mode 100644 index 0000000000..90892c2640 --- /dev/null +++ b/tests/e2e/test_lexical_admission.py @@ -0,0 +1,204 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lexical qualification must precede bounded candidate selection on real backends.""" + +import asyncio + +import pytest +from fastapi.testclient import TestClient +from pydantic import JsonValue +from sqlalchemy import update + +from powercontext.builtin.artifacts.search import AdmissionFloor +from powercontext.builtin.persistence.tables import ARTIFACTS_TABLE +from powercontext.builtin.records import ArtifactWrite +from powercontext.builtin.runtime import BuiltinConfig, RuntimeConfig, open_builtin_contexts +from powercontext.server.factory import create_server_app +from powercontext.server.settings import AccessControlConfig, BearerAuthConfig, McpConfig, ServerSettings +from tests.e2e.atomic_memory_migration_backend import MigrationBackend +from tests.e2e.test_topic_memory_generic_api import _scope + + +def _content(family: str, body: str, number: int) -> dict[str, JsonValue]: + if family == "experience": + return { + "situation": body, + "action": "Act carefully.", + "outcome": "Worked.", + "lesson": "Remember the evidence.", + } + return { + "name": f"skill-{number}", + "description": "Useful advice", + "instructions": body, + "validation": ["Validated"], + } + + +def _lexical_app(backend: MigrationBackend): + return create_server_app( + settings=ServerSettings( + database=backend.config, + runtime=RuntimeConfig(artifact_processing_families=()), + auth=BearerAuthConfig(enabled=False), + access=AccessControlConfig(mode="disabled"), + mcp=McpConfig(enabled=False), + ), + scheduler_path=backend.directory / "lexical-scheduler.db", + ) + + +@pytest.mark.parametrize("family", ["experience", "skill"]) +def test_qualified_long_text_survives_high_scoring_low_coverage_matches( + migration_backend: MigrationBackend, family: str +) -> None: + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=migration_backend.config)) as contexts: + await contexts.get("project") + + target = await contexts.records.create_artifact( + "project", + family, + ArtifactWrite(content=_content(family, "alpha beta " + ("filler " * 300).rstrip(), 999)), + ) + for number in range(80): + await contexts.records.create_artifact( + "project", + family, + ArtifactWrite(content=_content(family, "alpha" if number % 2 else "beta", number)), + ) + async with contexts.database.transaction() as connection: + if family == "experience": + result = await contexts.experience_index.search(connection, "project", "alpha beta gamma", 1) + hits = result.hits + relaxed = await contexts.experience_index.search( + connection, + "project", + "alpha beta gamma", + 1, + admission=AdmissionFloor(lexical_min_matched_terms=1, lexical_coverage=0), + ) + assert relaxed.hits and relaxed.hits[0].artifact_ref.artifact_id != target.artifact_id + else: + hits = await contexts.experience_index.search_skills(connection, "project", "alpha beta gamma", 1) + assert len(hits) == 1 + assert hits[0].artifact_ref.artifact_id == target.artifact_id + assert ( + hits[0] + .content.model_dump(mode="json")["situation" if family == "experience" else "instructions"] + .startswith("alpha beta ") + ) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("family", ["experience", "skill"]) +def test_unqualified_corrupt_body_does_not_break_public_search( + migration_backend: MigrationBackend, family: str +) -> None: + app = _lexical_app(migration_backend) + with TestClient(app, raise_server_exceptions=False) as client: + scope = _scope(client, "unqualified-body") + created = [] + for number, body in enumerate(("alpha", "alpha beta " + ("filler " * 300).rstrip())): + response = client.post( + f"/v1/scopes/{scope}/artifacts", json={"family": family, "content": _content(family, body, number)} + ) + assert response.status_code == 201, response.text + created.append(response.json()) + path = f"/v1/scopes/{scope}/artifacts/{family}/search" + request = {"query": "alpha beta gamma", "limit": 1, "include_scores": True} + strict = client.post(path, json=request) + assert strict.status_code == 200, strict.text + assert strict.json()["results"][0]["artifact_id"] == created[1]["artifact_id"] + single_term = client.post(path, json={"query": "alpha", "limit": 1}) + assert single_term.status_code == 200, single_term.text + assert single_term.json()["results"][0]["artifact_id"] == created[0]["artifact_id"] + + async def corrupt_unqualified_body() -> None: + async with app.state.application.primary_database.transaction() as connection: + await connection.execute( + update(ARTIFACTS_TABLE) + .where( + ARTIFACTS_TABLE.c.scope_id == scope, + ARTIFACTS_TABLE.c.family == family, + ARTIFACTS_TABLE.c.artifact_id == created[0]["artifact_id"], + ARTIFACTS_TABLE.c.revision == created[0]["revision"], + ) + .values(content=b"not-json") + ) + + assert client.portal is not None + client.portal.call(corrupt_unqualified_body) + after = client.post(path, json=request) + assert after.status_code == 200, after.text + assert after.json() == strict.json() + + +@pytest.mark.parametrize("family", ["experience", "skill"]) +def test_lower_ranked_rejections_do_not_change_public_top_hit(migration_backend: MigrationBackend, family: str) -> None: + with TestClient(_lexical_app(migration_backend)) as client: + scope = _scope(client, "lower-ranked-rejection") + for number in range(12): + created = client.post( + f"/v1/scopes/{scope}/artifacts", + json={"family": family, "content": _content(family, "alpha beta gamma", number)}, + ) + assert created.status_code == 201, created.text + path = f"/v1/scopes/{scope}/artifacts/{family}/search" + request = {"query": "alpha beta gamma", "limit": 1} + before = client.post(path, json=request) + assert before.status_code == 200, before.text + assert len(before.json()["results"]) == 1 + distractor = client.post( + f"/v1/scopes/{scope}/artifacts", + json={"family": family, "content": _content(family, "alpha " + ("filler " * 300).rstrip(), 999)}, + ) + assert distractor.status_code == 201, distractor.text + strict = client.post(path, json=request) + relaxed = client.post( + path, json={**request, "admission": {"lexical_min_matched_terms": 1, "lexical_coverage": 0}} + ) + assert strict.status_code == relaxed.status_code == 200 + assert strict.json() == relaxed.json() + assert strict.json() == before.json() + + +@pytest.mark.parametrize("family", ["experience", "skill"]) +def test_selected_missing_exact_content_is_a_search_contract_failure(tmp_path, family: str) -> None: + import sqlite3 + + from tests.e2e.test_topic_memory_generic_api import _app + + content = ( + {"situation": "alpha evidence", "action": "Act", "outcome": "Worked", "lesson": "Remember"} + if family == "experience" + else {"name": "alpha", "description": "Evidence", "instructions": "alpha evidence", "validation": ["Checked"]} + ) + with TestClient(_app(tmp_path), raise_server_exceptions=False) as client: + scope = _scope(client, "missing-selected") + created = client.post(f"/v1/scopes/{scope}/artifacts", json={"family": family, "content": content}) + assert created.status_code == 201, created.text + url = f"/v1/scopes/{scope}/artifacts/{family}/search" + ordinary = client.post(url, json={"query": "alpha", "limit": 1}) + assert ordinary.status_code == 200 and len(ordinary.json()["results"]) == 1 + with sqlite3.connect(tmp_path / "topics.db") as connection: + connection.execute("PRAGMA foreign_keys=OFF") + connection.execute( + "DELETE FROM pc_artifacts WHERE scope_id=? AND family=? AND artifact_id=? AND revision=1", + (scope, family, created.json()["artifact_id"]), + ) + damaged = client.post(url, json={"query": "alpha", "limit": 1}) + assert damaged.status_code == 500, damaged.text diff --git a/tests/e2e/test_recall_sufficiency_gate.py b/tests/e2e/test_recall_sufficiency_gate.py index 5e3ca41b8d..41392ca803 100644 --- a/tests/e2e/test_recall_sufficiency_gate.py +++ b/tests/e2e/test_recall_sufficiency_gate.py @@ -774,7 +774,6 @@ async def scenario() -> None: runtime._topic_memory_searcher = TopicMemorySearcher( search=runtime._topic_memory_search, get=runtime._topic_memory_get, - browse=runtime._topic_memory_browse, embedding_model=embedding, observer=runtime._topic_memory_search_observer, ) diff --git a/tests/e2e/test_sqlite_vec.py b/tests/e2e/test_sqlite_vec.py index d5d2327b74..07504399ef 100644 --- a/tests/e2e/test_sqlite_vec.py +++ b/tests/e2e/test_sqlite_vec.py @@ -149,8 +149,17 @@ async def scenario() -> None: else {"revision": created.revision + 1} ) await connection.execute(update(table).values(**values)) - with pytest.raises(AtomicMemoryIndexError): - await memory.search("alpha", mode="vector") + if corruption == "missing-vector": + assert not (await memory.search("alpha", mode="vector")).hits + else: + with pytest.raises(AtomicMemoryIndexError): + await memory.search("alpha", mode="vector") + if corruption == "missing-vector": + with pytest.raises(AtomicMemoryIndexError, match="rebuild-projection"): + async with open_builtin_contexts( + BuiltinConfig(database=config), embedding_model=_KeywordEmbeddingModel() + ): + pass asyncio.run(scenario()) From 10d7e2302803399ae3cb73e8dc41dd9ac600cc8f Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Tue, 20 Oct 2026 10:58:43 +0800 Subject: [PATCH 2/9] fix(search): audit only vector tables owned by the index --- src/powercontext/builtin/persistence/atomic_memory_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/powercontext/builtin/persistence/atomic_memory_index.py b/src/powercontext/builtin/persistence/atomic_memory_index.py index 88d3e0021b..2ac5149059 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/atomic_memory_index.py @@ -202,7 +202,7 @@ async def refresh_tags( async def validate_atomic_memory_vectors(connection: AsyncConnection, index: AtomicMemoryIndex) -> None: """Audit vector deployment readiness at startup, outside ordinary bounded search.""" profile = index.capabilities.embedding_profile - if profile is None: + if profile is None or index.table not in index.tables: return table = index.table invalid = [ From 6c9fec8941f8efa7317e1e321e9faa71ed222d97 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Tue, 20 Oct 2026 10:58:44 +0800 Subject: [PATCH 3/9] fix(harness): render the installed Kimi scope helper path --- e2e/bub/src/powercontext_e2e/harbor_kimi_code.py | 4 +++- e2e/bub/tests/test_harbor_job_config.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/e2e/bub/src/powercontext_e2e/harbor_kimi_code.py b/e2e/bub/src/powercontext_e2e/harbor_kimi_code.py index ec44453b3a..b287ab4570 100644 --- a/e2e/bub/src/powercontext_e2e/harbor_kimi_code.py +++ b/e2e/bub/src/powercontext_e2e/harbor_kimi_code.py @@ -86,7 +86,9 @@ write(manifestPath, manifest) const skill = path.join(plugin, 'skills/powercontext-project-context/SKILL.md') const guidance = fs.readFileSync(skill, 'utf8') -fs.writeFileSync(skill, guidance.replaceAll('__POWERCONTEXT_SCOPE_JSON__', () => JSON.stringify(scopeId))) +fs.writeFileSync(skill, guidance + .replaceAll('__POWERCONTEXT_SCOPE_JSON__', () => JSON.stringify(scopeId)) + .replaceAll('__POWERCONTEXT_SCOPE_SCRIPT_JSON__', () => JSON.stringify(path.join(plugin, 'scripts/scope.mjs')))) const now = new Date().toISOString() write(path.join(home, 'plugins/installed.json'), {{ version: 1, plugins: [{{ id: 'powercontext', root: plugin, source: 'local-path', enabled: true, diff --git a/e2e/bub/tests/test_harbor_job_config.py b/e2e/bub/tests/test_harbor_job_config.py index 025cab79ae..a7ab32ea8f 100644 --- a/e2e/bub/tests/test_harbor_job_config.py +++ b/e2e/bub/tests/test_harbor_job_config.py @@ -842,9 +842,13 @@ def _install_kimi_plugin(tmp_path: Path, plugin_url: str, *, token: str | None) def _kimi_installation(home: Path) -> dict[str, object]: files: dict[str, object] = {} + scope_script = json.dumps( + str(home.resolve() / "plugins/managed/powercontext/scripts/scope.mjs"), ensure_ascii=False + ) for path in sorted((home / "plugins").rglob("*")): if path.is_file(): text = path.read_text(encoding="utf-8") + text = text.replace(scope_script, '""') files[path.relative_to(home).as_posix()] = json.loads(text) if path.suffix == ".json" else text for record in files["plugins/installed.json"]["plugins"]: # type: ignore[index] # Each installation records its own time, and its source and root inside this test's directories. From 572537519211f59b3e54f5d6ba0307eae2d22b7f Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Tue, 20 Oct 2026 12:05:01 +0800 Subject: [PATCH 4/9] refactor(search): unify index lifecycle and exact result handling Let indexes own startup validation and declare supported search families. Share Experience/Skill candidate reads and materialize exact results without rereading bodies; require complete Topic results in the retrieval snapshot. --- src/powercontext/artifacts/search.py | 2 +- .../builtin/artifacts/experience/search.py | 2 +- .../builtin/artifacts/skill/search.py | 4 +- .../builtin/artifacts/topic_memory/models.py | 2 +- .../persistence/atomic_memory_index.py | 50 +++++------ .../builtin/persistence/experience_index.py | 82 +++++++++---------- .../persistence/oceanbase/experience_index.py | 46 +++-------- .../persistence/sqlite/experience_index.py | 44 ++-------- .../builtin/persistence/topic_memory.py | 6 +- .../builtin/runtime/application.py | 1 - .../builtin/runtime/atomic_memory.py | 13 ++- .../builtin/runtime/composition.py | 6 +- .../builtin/runtime/experience_search.py | 44 +++++----- .../builtin/runtime/relational.py | 15 +++- .../builtin/runtime/skill_search.py | 55 ++++++++----- .../builtin/runtime/topic_memory_search.py | 22 +---- src/powercontext/server/mapping.py | 2 - .../persistence/test_experience_index.py | 17 ++-- tests/builtin/persistence/test_records.py | 13 ++- .../builtin/persistence/test_topic_memory.py | 2 +- tests/builtin/review/test_service.py | 34 +------- tests/builtin/runtime/test_artifact_search.py | 63 ++++++-------- .../runtime/test_topic_memory_search.py | 15 +--- tests/builtin/test_publication.py | 14 +--- .../e2e/test_atomic_artifact_search_domain.py | 1 - tests/e2e/test_lexical_admission.py | 26 +++--- tests/e2e/test_observability.py | 3 + tests/e2e/test_recall_sufficiency_gate.py | 1 - 28 files changed, 245 insertions(+), 340 deletions(-) diff --git a/src/powercontext/artifacts/search.py b/src/powercontext/artifacts/search.py index 64ec275f77..a6e005f261 100644 --- a/src/powercontext/artifacts/search.py +++ b/src/powercontext/artifacts/search.py @@ -111,7 +111,7 @@ class ArtifactSearchOutcome(Protocol): def matches(self) -> Sequence[ArtifactSearchMatch]: ... @property - def artifacts(self) -> Sequence[Artifact[Any]] | None: ... + def artifacts(self) -> Sequence[Artifact[Any]]: ... RequestT = TypeVar("RequestT", bound=ArtifactSearchQuery) diff --git a/src/powercontext/builtin/artifacts/experience/search.py b/src/powercontext/builtin/artifacts/experience/search.py index 7308b46f78..f09a5cac7c 100644 --- a/src/powercontext/builtin/artifacts/experience/search.py +++ b/src/powercontext/builtin/artifacts/experience/search.py @@ -48,7 +48,7 @@ class ExperienceSearchOutcome: hits: tuple[ExperienceSearchHit, ...] = () admission: AdmissionCounts | None = None matches: tuple[ArtifactSearchMatch, ...] = () - artifacts: tuple[Experience, ...] | None = None + artifacts: tuple[Experience, ...] = () class ExperienceSearchHit(BaseModel): diff --git a/src/powercontext/builtin/artifacts/skill/search.py b/src/powercontext/builtin/artifacts/skill/search.py index dd3d05b1b3..eb9e1f22bf 100644 --- a/src/powercontext/builtin/artifacts/skill/search.py +++ b/src/powercontext/builtin/artifacts/skill/search.py @@ -51,11 +51,11 @@ class SkillSearchHit(BaseModel): @dataclass(frozen=True) class SkillSearchOutcome: - """Managed Skill hits and their optional exact Artifact materialization.""" + """Managed Skill hits and their exact Artifact materialization for public search.""" hits: tuple[SkillSearchHit, ...] = () matches: tuple[ArtifactSearchMatch, ...] = () - artifacts: tuple[Skill, ...] | None = None + artifacts: tuple[Skill, ...] = () def skill_search_text(content: SkillContent, package: SkillPackageSnapshot | None = None, /) -> str: diff --git a/src/powercontext/builtin/artifacts/topic_memory/models.py b/src/powercontext/builtin/artifacts/topic_memory/models.py index 879ef13013..784b08282c 100644 --- a/src/powercontext/builtin/artifacts/topic_memory/models.py +++ b/src/powercontext/builtin/artifacts/topic_memory/models.py @@ -248,7 +248,7 @@ class TopicMemorySearchResult(BaseModel): query_embedding: MemoryQueryEmbedding | None = Field(default=None, exclude=True) embedding_calls: int = Field(default=0, exclude=True) matches: tuple[ArtifactSearchMatch, ...] = Field(default=(), exclude=True) - artifacts: tuple[TopicMemory, ...] | None = Field(default=None, exclude=True) + artifacts: tuple[TopicMemory, ...] = Field(default=(), exclude=True) class PublishedTopicMemory(BaseModel): diff --git a/src/powercontext/builtin/persistence/atomic_memory_index.py b/src/powercontext/builtin/persistence/atomic_memory_index.py index 2ac5149059..d0661b9add 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/atomic_memory_index.py @@ -170,12 +170,16 @@ class AtomicMemoryRelatedRequest: class AtomicMemoryIndex(Protocol): + """Caller-owned projection transactions, including initialization and current-row validation.""" + table: Table tables: tuple[Table, ...] capabilities: AtomicMemoryIndexCapabilities async def initialize(self, connection: AsyncConnection, /) -> None: ... + async def validate_current(self, connection: AsyncConnection, /) -> None: ... + async def replace( self, connection: AsyncConnection, scope_id: str, projection: AtomicMemoryProjection, / ) -> None: ... @@ -199,29 +203,6 @@ async def refresh_tags( ) -> None: ... -async def validate_atomic_memory_vectors(connection: AsyncConnection, index: AtomicMemoryIndex) -> None: - """Audit vector deployment readiness at startup, outside ordinary bounded search.""" - profile = index.capabilities.embedding_profile - if profile is None or index.table not in index.tables: - return - table = index.table - invalid = [ - table.c.embedding.is_(None), - table.c.embedding_input_hash.is_(None), - table.c.profile_fingerprint.is_(None), - table.c.profile_fingerprint != atomic_memory_profile_fingerprint(profile), - table.c.projection_format != ATOMIC_MEMORY_PROJECTION_FORMAT, - ] - if connection.dialect.name == "sqlite": - invalid.append(func.length(table.c.embedding) != profile.dimension * 4) - if await connection.scalar(select(table.c.artifact_id).where(or_(*invalid)).limit(1)) is not None: - raise AtomicMemoryIndexError( - "incomplete-vector", - "Current Atomic Memory vectors are incomplete or use another profile. Stop writers and run " - "powercontext server atomic-memory-rebuild-projection --maintenance-confirmed.", - ) - - class AtomicMemoryProjectionPublisher: """Prepare vectors outside writes and publish with the authoritative transaction.""" @@ -564,6 +545,28 @@ def __init__(self, table: Table, profile: EmbeddingProfile | None) -> None: vector=profile is not None, hybrid=profile is not None, embedding_profile=profile ) + async def validate_current(self, connection: AsyncConnection, /) -> None: + """Audit vector deployment readiness at startup, outside ordinary bounded search.""" + profile = self.capabilities.embedding_profile + if profile is None: + return + table = self.table + invalid = [ + table.c.embedding.is_(None), + table.c.embedding_input_hash.is_(None), + table.c.profile_fingerprint.is_(None), + table.c.profile_fingerprint != atomic_memory_profile_fingerprint(profile), + table.c.projection_format != ATOMIC_MEMORY_PROJECTION_FORMAT, + ] + if connection.dialect.name == "sqlite": + invalid.append(func.length(table.c.embedding) != profile.dimension * 4) + if await connection.scalar(select(table.c.artifact_id).where(or_(*invalid)).limit(1)) is not None: + raise AtomicMemoryIndexError( + "incomplete-vector", + "Current Atomic Memory vectors are incomplete or use another profile. Stop writers and run " + "powercontext server atomic-memory-rebuild-projection --maintenance-confirmed.", + ) + def _encode_embedding(self, vector: tuple[float, ...]) -> object: return vector @@ -772,5 +775,4 @@ def validate_request(request: AtomicMemorySearchRequest | AtomicMemoryRelatedReq "load_atomic_memory_tags", "oceanbase_fts_coverage_sql", "sqlite_fts_coverage_sql", - "validate_atomic_memory_vectors", ] diff --git a/src/powercontext/builtin/persistence/experience_index.py b/src/powercontext/builtin/persistence/experience_index.py index 44c9fd5960..f51f182532 100644 --- a/src/powercontext/builtin/persistence/experience_index.py +++ b/src/powercontext/builtin/persistence/experience_index.py @@ -17,7 +17,7 @@ from __future__ import annotations from collections.abc import Iterable, Mapping -from typing import Any, Protocol +from typing import Any, Literal, Protocol from sqlalchemy import select, text, tuple_, update from sqlalchemy.ext.asyncio import AsyncConnection @@ -88,7 +88,9 @@ class ExperienceIndex(Protocol): - """A rebuildable query projection updated with approved Artifact heads.""" + """Shared approved Experience/Skill projection with Family-owned result decoding.""" + + families: frozenset[str] async def initialize(self, connection: AsyncConnection, /) -> None: ... @@ -106,14 +108,19 @@ async def search( self, connection: AsyncConnection, scope_id: str, + family: Literal["experience", "skill"], query: str, limit: int, /, *, admission: AdmissionFloor | None = None, - min_score: float | None = None, - require_scores: bool = False, - ) -> ExperienceSearchOutcome: ... + ) -> tuple[Mapping[Any, Any], ...]: + """Return ranked qualified rows with exact body and revision metadata. + + Rows may include a bounded rejection witness. Family services apply + score controls and materialize selected rows with their lineage. + """ + ... async def replace_skill( self, @@ -124,22 +131,11 @@ async def replace_skill( /, ) -> None: ... - async def search_skills( - self, - connection: AsyncConnection, - scope_id: str, - query: str, - limit: int, - /, - *, - admission: AdmissionFloor | None = None, - min_score: float | None = None, - require_scores: bool = False, - ) -> tuple[SkillSearchHit, ...]: ... - class NoExperienceIndex: - """Allow exact Experience reads when no recall projection is configured.""" + """Retain exact Artifact reads when approved-head search is disabled.""" + + families: frozenset[str] = frozenset() async def initialize(self, _connection: AsyncConnection, /) -> None: pass @@ -157,15 +153,14 @@ async def search( self, _connection: AsyncConnection, _scope_id: str, + _family: Literal["experience", "skill"], _query: str, _limit: int, /, *, admission: AdmissionFloor | None = None, - min_score: float | None = None, - require_scores: bool = False, - ) -> ExperienceSearchOutcome: - return ExperienceSearchOutcome() + ) -> tuple[Mapping[Any, Any], ...]: + return () async def remove(self, _connection: AsyncConnection, _scope_id: str, _artifact_id: str, /) -> None: pass @@ -180,20 +175,6 @@ async def replace_skill( ) -> None: pass - async def search_skills( - self, - _connection: AsyncConnection, - _scope_id: str, - _query: str, - _limit: int, - /, - *, - admission: AdmissionFloor | None = None, - min_score: float | None = None, - require_scores: bool = False, - ) -> tuple[SkillSearchHit, ...]: - return () - async def ensure_artifact_head_searchable_text(connection: AsyncConnection, /) -> None: """Upgrade existing shared heads and lineage with their current persisted columns.""" @@ -420,7 +401,8 @@ async def search_approved_artifact_rows( probe = text( f"WITH raw_probe AS (SELECT h.artifact_id, h.revision, h.searchable_text, {score} AS raw_score " # noqa: S608 f"{matches} ORDER BY raw_score DESC, h.artifact_id, h.revision LIMIT :probe_limit) " - "SELECT r.*, a.content AS content, 0 AS is_rejected FROM raw_probe AS r " + "SELECT r.*, a.content, a.created_at, a.created_by, a.restored_from_revision, " + "a.rollback_reason, 0 AS is_rejected FROM raw_probe AS r " "LEFT JOIN pc_artifacts AS a ON a.scope_id=:scope_id AND a.family=:family " "AND a.artifact_id=r.artifact_id AND a.revision=r.revision " "ORDER BY raw_score DESC, artifact_id, revision" @@ -435,7 +417,7 @@ async def search_approved_artifact_rows( else () ) if len(qualified) >= limit or len(observed) < parameters["probe_limit"]: - return witness + tuple({**row, "score_metric": metric} for row in qualified[:limit]) + return witness + tuple({**row, "family": family, "score_metric": metric} for row in qualified[:limit]) prefix = "" candidate_source = source if connection.dialect.name == "sqlite" and required > 1: @@ -468,7 +450,7 @@ async def search_approved_artifact_rows( ) rejected = ( ( - " UNION ALL SELECT NULL, NULL, NULL, NULL, NULL, 1 FROM " + " UNION ALL SELECT NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 1 FROM " "(SELECT 1 WHERE (SELECT COUNT(*) FROM raw_probe) > " "(SELECT COUNT(*) FROM qualified_probe)) AS rejected" ) @@ -478,13 +460,14 @@ async def search_approved_artifact_rows( statement = text( f"WITH {prefix}candidates AS ({candidates}) " # noqa: S608 "SELECT c.artifact_id AS artifact_id, c.revision AS revision, p.searchable_text AS searchable_text, " - f"a.content AS content, c.raw_score AS raw_score, 0 AS is_rejected FROM candidates AS c {projection_join}" + "a.content, a.created_at, a.created_by, a.restored_from_revision, a.rollback_reason, " + f"c.raw_score AS raw_score, 0 AS is_rejected FROM candidates AS c {projection_join}" "LEFT JOIN pc_artifacts AS a ON a.scope_id=:scope_id AND a.family=:family " f"AND a.artifact_id=c.artifact_id AND a.revision=c.revision{rejected} " f"ORDER BY is_rejected DESC, raw_score {direction}, artifact_id, revision" ) rows = (await connection.execute(statement, parameters)).mappings() - result = tuple({**row, "score_metric": metric} for row in rows) + result = tuple({**row, "family": family, "score_metric": metric} for row in rows) return witness + result @@ -517,6 +500,16 @@ def experience_search_hits( examined += 1 if row.get("is_rejected"): continue + if row["content"] is None: + raise RepositoryNotFoundError( + "artifact", + ( + scope_id, + ArtifactRef( + family=Experience.family, artifact_id=str(row["artifact_id"]), revision=int(row["revision"]) + ), + ), + ) content = _content(row["content"]) if not admits_fts_text(query, experience_search_text(content), floor=admission): continue @@ -563,6 +556,11 @@ def skill_search_hits( hits: list[SkillSearchHit] = [] for row in rows: + if row["content"] is None: + raise RepositoryNotFoundError( + "artifact", + ArtifactRef(family=Skill.family, artifact_id=str(row["artifact_id"]), revision=int(row["revision"])), + ) content = _skill_content(row["content"]) searchable = row.get("searchable_text") or analyze_text(skill_search_text(content)) if not admits_fts_projection(query, str(searchable), floor=admission): diff --git a/src/powercontext/builtin/persistence/oceanbase/experience_index.py b/src/powercontext/builtin/persistence/oceanbase/experience_index.py index 5180569be5..91892507bf 100644 --- a/src/powercontext/builtin/persistence/oceanbase/experience_index.py +++ b/src/powercontext/builtin/persistence/oceanbase/experience_index.py @@ -12,28 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""OceanBase FULLTEXT projection for approved Experience heads.""" +"""OceanBase FULLTEXT projection shared by approved Experience and Skill heads.""" from __future__ import annotations +from collections.abc import Mapping +from typing import Any, Literal + from sqlalchemy import select, text from sqlalchemy.dialects.mysql import match from sqlalchemy.ext.asyncio import AsyncConnection -from powercontext.builtin.artifacts.experience import Experience, ExperienceSearchOutcome +from powercontext.builtin.artifacts.experience import Experience from powercontext.builtin.artifacts.memory import CapabilityNotSupportedError from powercontext.builtin.artifacts.search import AdmissionFloor -from powercontext.builtin.artifacts.skill import Skill, SkillPackageSnapshot, SkillSearchHit +from powercontext.builtin.artifacts.skill import Skill, SkillPackageSnapshot from powercontext.builtin.persistence.experience_index import ( ensure_artifact_head_searchable_text, - experience_search_hits, rebuild_experience_projections, rebuild_skill_projections, remove_experience_projection, replace_experience_projection, replace_skill_projection, search_approved_artifact_rows, - skill_search_hits, ) from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE @@ -54,7 +55,9 @@ class OceanBaseExperienceFTSIndex: - """Maintain and query approved Experience heads with OceanBase FULLTEXT.""" + """Maintain and query approved Experience and Skill heads with OceanBase FULLTEXT.""" + + families: frozenset[str] = frozenset({Experience.family, Skill.family}) async def initialize(self, connection: AsyncConnection, /) -> None: if connection.dialect.name != "mysql": @@ -87,20 +90,14 @@ async def search( self, connection: AsyncConnection, scope_id: str, + family: Literal["experience", "skill"], query: str, limit: int, /, *, admission: AdmissionFloor | None = None, - min_score: float | None = None, - require_scores: bool = False, - ) -> ExperienceSearchOutcome: - rows = await search_approved_artifact_rows( - connection, scope_id, Experience.family, query, limit, admission=admission - ) - return experience_search_hits( - rows, query, limit, scope_id, admission=admission, min_score=min_score, require_scores=require_scores - ) + ) -> tuple[Mapping[Any, Any], ...]: + return await search_approved_artifact_rows(connection, scope_id, family, query, limit, admission=admission) async def replace_skill( self, @@ -112,24 +109,5 @@ async def replace_skill( ) -> None: await replace_skill_projection(connection, scope_id, skill, package) - async def search_skills( - self, - connection: AsyncConnection, - scope_id: str, - query: str, - limit: int, - /, - *, - admission: AdmissionFloor | None = None, - min_score: float | None = None, - require_scores: bool = False, - ) -> tuple[SkillSearchHit, ...]: - rows = await search_approved_artifact_rows( - connection, scope_id, Skill.family, query, limit, admission=admission - ) - return skill_search_hits( - rows, query, limit, admission=admission, min_score=min_score, require_scores=require_scores - ) - __all__ = ["OceanBaseExperienceFTSIndex"] diff --git a/src/powercontext/builtin/persistence/sqlite/experience_index.py b/src/powercontext/builtin/persistence/sqlite/experience_index.py index 03e5430bd9..e04a8d84e4 100644 --- a/src/powercontext/builtin/persistence/sqlite/experience_index.py +++ b/src/powercontext/builtin/persistence/sqlite/experience_index.py @@ -12,34 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""SQLite FTS5 projection for approved Experience heads.""" +"""SQLite FTS5 projection shared by approved Experience and Skill heads.""" from __future__ import annotations from collections.abc import Mapping -from typing import Any +from typing import Any, Literal from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.builtin.artifacts.experience import ( Experience, - ExperienceSearchOutcome, experience_searchable_text, ) from powercontext.builtin.artifacts.memory import CapabilityNotSupportedError from powercontext.builtin.artifacts.search import AdmissionFloor -from powercontext.builtin.artifacts.skill import Skill, SkillPackageSnapshot, SkillSearchHit, skill_searchable_text +from powercontext.builtin.artifacts.skill import Skill, SkillPackageSnapshot, skill_searchable_text from powercontext.builtin.persistence.experience_index import ( ensure_artifact_head_searchable_text, - experience_search_hits, rebuild_experience_projections, rebuild_skill_projections, remove_experience_projection, replace_experience_projection, replace_skill_projection, search_approved_artifact_rows, - skill_search_hits, ) _CREATE_FTS_SQL = """ @@ -88,7 +85,9 @@ class SQLiteExperienceFTSIndex: - """Maintain and query approved Experience heads with SQLite FTS5.""" + """Maintain and query approved Experience and Skill heads with SQLite FTS5.""" + + families: frozenset[str] = frozenset({Experience.family, Skill.family}) async def initialize(self, connection: AsyncConnection, /) -> None: if connection.dialect.name != "sqlite": @@ -128,20 +127,14 @@ async def search( self, connection: AsyncConnection, scope_id: str, + family: Literal["experience", "skill"], query: str, limit: int, /, *, admission: AdmissionFloor | None = None, - min_score: float | None = None, - require_scores: bool = False, - ) -> ExperienceSearchOutcome: - rows = await search_approved_artifact_rows( - connection, scope_id, Experience.family, query, limit, admission=admission - ) - return experience_search_hits( - rows, query, limit, scope_id, admission=admission, min_score=min_score, require_scores=require_scores - ) + ) -> tuple[Mapping[Any, Any], ...]: + return await search_approved_artifact_rows(connection, scope_id, family, query, limit, admission=admission) async def remove(self, connection: AsyncConnection, scope_id: str, artifact_id: str, /) -> None: await remove_experience_projection(connection, scope_id, artifact_id) @@ -174,25 +167,6 @@ async def replace_skill( }, ) - async def search_skills( - self, - connection: AsyncConnection, - scope_id: str, - query: str, - limit: int, - /, - *, - admission: AdmissionFloor | None = None, - min_score: float | None = None, - require_scores: bool = False, - ) -> tuple[SkillSearchHit, ...]: - rows = await search_approved_artifact_rows( - connection, scope_id, Skill.family, query, limit, admission=admission - ) - return skill_search_hits( - rows, query, limit, admission=admission, min_score=min_score, require_scores=require_scores - ) - @staticmethod async def _insert_row(connection: AsyncConnection, row: Mapping[Any, Any]) -> None: await connection.execute( diff --git a/src/powercontext/builtin/persistence/topic_memory.py b/src/powercontext/builtin/persistence/topic_memory.py index 5c82656497..9bf1f6a925 100644 --- a/src/powercontext/builtin/persistence/topic_memory.py +++ b/src/powercontext/builtin/persistence/topic_memory.py @@ -528,9 +528,7 @@ async def search( query_vector = self._canonical_query_vector(used_mode, query_vector) await self._check_retrieval_shape(connection) if not query_terms and used_mode == "fts": - return TopicMemorySearchResult( - mode=used_mode, hits=(), artifacts=() if artifact_request is not None else None - ) + return TopicMemorySearchResult(mode=used_mode) request = TopicMemorySearchRequest( query=query, analyzed_query=" ".join(query_terms), @@ -551,7 +549,7 @@ async def search( min_score=None if artifact_request is None else artifact_request.min_score, include_scores=artifact_request is not None and artifact_request.include_scores, ) - artifacts = None + artifacts: tuple[TopicMemory, ...] = () if artifact_request is not None: try: artifacts = cast( diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index 3d90d3a2ac..3c90f4ec44 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -3740,7 +3740,6 @@ def __init__( if self._topic_memory_searcher is None and topic_memory_search is not None: self._topic_memory_searcher = TopicMemorySearcher( search=topic_memory_search, - get=topic_memory_get, embedding_model=topic_memory_embedding_model, observer=topic_memory_search_observer, ) diff --git a/src/powercontext/builtin/runtime/atomic_memory.py b/src/powercontext/builtin/runtime/atomic_memory.py index 03cacb5648..e074413d96 100644 --- a/src/powercontext/builtin/runtime/atomic_memory.py +++ b/src/powercontext/builtin/runtime/atomic_memory.py @@ -191,7 +191,7 @@ class AtomicMemorySearchPage: rerank: AtomicMemoryRerankTrace | None = None recoverable: bool = False matches: tuple[ArtifactSearchMatch, ...] = () - artifacts: tuple[AtomicMemory, ...] | None = None + artifacts: tuple[AtomicMemory, ...] = () class AtomicMemoryApplication: @@ -491,7 +491,7 @@ async def search( # noqa: C901 if artifact_request is None: raise raise ArtifactSearchContractError("atomic-memory", "Selected exact revision is unavailable") from exc - artifacts = validated if artifact_request is not None else None + artifacts = validated if artifact_request is not None else () recoverable = recovery_admission is not None and await application.index.probe_recoverable( connection, self.scope_id, request, recovery_admission ) @@ -725,7 +725,7 @@ async def _rerank( context, *, artifact_request=None, - artifacts=None, + artifacts=(), ): application = self.application reranker = application.reranker @@ -735,7 +735,7 @@ async def _rerank( candidates[:limit], query_embedding, embedding_calls, - artifacts=(() if artifacts is None else artifacts) if artifact_request is not None else None, + artifacts=artifacts, ) # Reauthorize the exact candidate bodies immediately before an external rank model. try: @@ -750,12 +750,9 @@ async def _rerank( (), query_embedding, embedding_calls, - artifacts=() if artifact_request is not None else None, ) complete_artifacts = [] if artifact_request is not None: - if artifacts is None: - raise ArtifactSearchContractError("atomic-memory", "Selected exact Artifacts are unavailable") artifacts_by_ref = {_artifact_key(artifact.as_ref()): artifact for artifact in artifacts} complete_artifacts = [ artifacts_by_ref[_artifact_key(candidate.hit.artifact_ref)] for candidate in candidates @@ -790,7 +787,7 @@ async def _rerank( embedding_calls, 1, rerank=trace, - artifacts=tuple(complete_artifacts[rank - 1] for rank in ranks) if artifact_request is not None else None, + artifacts=tuple(complete_artifacts[rank - 1] for rank in ranks) if artifact_request is not None else (), ) async def merge( diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index e5d48fe64b..dfc584842b 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -98,7 +98,6 @@ ) from powercontext.builtin.persistence.atomic_memory_index import ( AtomicMemoryIndexCapabilities, - validate_atomic_memory_vectors, ) from powercontext.builtin.persistence.atomic_memory_readiness import ( assert_atomic_memory_migration_ready, @@ -742,7 +741,6 @@ async def run_profile(scope_id, high): artifact_search.register(contexts.list_skills) topic_memory_searcher = TopicMemorySearcher( search=contexts.search_topic_memories, - get=contexts.get_topic_memory, embedding_model=configured_embedding, observer=topic_memory_search_observer, capabilities=contexts.topic_memory_index.capabilities, @@ -1115,7 +1113,7 @@ async def open_builtin_contexts( if not _topic_memory_worker: await _initialize_atomic_memory_authority(connection) await atomic_index.initialize(connection) - await validate_atomic_memory_vectors(connection, atomic_index) + await atomic_index.validate_current(connection) await refresh_lexical_projections( connection, atomic_index.table if atomic_index.table in atomic_index.tables else None ) @@ -1196,7 +1194,7 @@ async def open_builtin_contexts( if not _topic_memory_worker: await _initialize_atomic_memory_authority(connection) await atomic_index.initialize(connection) - await validate_atomic_memory_vectors(connection, atomic_index) + await atomic_index.validate_current(connection) await refresh_lexical_projections( connection, atomic_index.table if atomic_index.table in atomic_index.tables else None ) diff --git a/src/powercontext/builtin/runtime/experience_search.py b/src/powercontext/builtin/runtime/experience_search.py index c496359b19..445e702df7 100644 --- a/src/powercontext/builtin/runtime/experience_search.py +++ b/src/powercontext/builtin/runtime/experience_search.py @@ -24,11 +24,10 @@ ArtifactSearchUnsupported, ) from powercontext.builtin.artifacts.experience import Experience, ExperienceSearchOutcome, ExperienceSearchRequest -from powercontext.builtin.artifacts.search import InvalidSearchScore from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.errors import RepositoryNotFoundError -from powercontext.builtin.persistence.experience_index import ExperienceIndex, NoExperienceIndex +from powercontext.builtin.persistence.experience_index import ExperienceIndex, experience_search_hits from powercontext.builtin.sources import validate_scope_id @@ -51,31 +50,38 @@ async def search( *, execution_context: ArtifactSearchExecutionContext | None = None, ) -> ExperienceSearchOutcome: - if isinstance(self._index, NoExperienceIndex): + if self.family not in self._index.families: raise ArtifactSearchUnsupported(self.family, field="mode") scope = validate_scope_id(scope_id) async with self._database.transaction(consistent_snapshot=True) as connection: - outcome = await self._index.search( - connection, - scope, - request.query, - request.limit, - admission=request.admission.as_floor(), - min_score=request.min_score, - require_scores=True, + rows = await self._index.search( + connection, scope, "experience", request.query, request.limit, admission=request.admission.as_floor() ) - matches: list[ArtifactSearchMatch] = [] - for hit in outcome.hits: - if hit.retrieval_score is None or hit.channel_scores is None or "text" not in hit.channel_scores: - raise InvalidSearchScore("public Experience search requires true text scoring metadata") # noqa: TRY003 - matches.append(ArtifactSearchMatch(hit.artifact_ref, hit.retrieval_score, hit.channel_scores)) try: - artifacts = await self._artifacts.get_many( - connection, scope, tuple(hit.artifact_ref for hit in outcome.hits) + outcome = experience_search_hits( + rows, + request.query, + request.limit, + scope, + admission=request.admission.as_floor(), + min_score=request.min_score, + require_scores=True, + ) + by_ref = { + (str(row["artifact_id"]), int(row["revision"])): row for row in rows if not row.get("is_rejected") + } + artifacts = await self._artifacts.decode_many( + connection, + scope, + tuple(by_ref[(hit.artifact_ref.artifact_id, hit.artifact_ref.revision)] for hit in outcome.hits), ) except RepositoryNotFoundError as error: raise ArtifactSearchContractError(self.family, "selected exact Artifact revision is missing") from error - return replace(outcome, matches=tuple(matches), artifacts=cast(tuple[Experience, ...], artifacts)) + matches = tuple( + ArtifactSearchMatch(hit.artifact_ref, cast(float, hit.retrieval_score), hit.channel_scores) + for hit in outcome.hits + ) + return replace(outcome, matches=matches, artifacts=cast(tuple[Experience, ...], artifacts)) __all__ = ["ExperienceArtifactSearcher"] diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 38044c34fa..5f31638802 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -141,7 +141,12 @@ from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.decision_observations import DecisionObservationRepository from powercontext.builtin.persistence.errors import RepositoryNotFoundError, StoredPayloadConflictError -from powercontext.builtin.persistence.experience_index import ExperienceIndex, NoExperienceIndex +from powercontext.builtin.persistence.experience_index import ( + ExperienceIndex, + NoExperienceIndex, + experience_search_hits, + skill_search_hits, +) from powercontext.builtin.persistence.external_skills import ExternalSkillRepository from powercontext.builtin.persistence.family_management import ( AtomicMemoryManagementWriter, @@ -967,7 +972,10 @@ async def search_experience_outcome( raise ValueError("Experience search limit must be positive") # noqa: TRY003 scope = validate_scope_id(scope_id) async with self.database.transaction() as connection: - return await self.experience_index.search(connection, scope, query, limit, admission=admission) + rows = await self.experience_index.search( + connection, scope, "experience", query, limit, admission=admission + ) + return experience_search_hits(rows, query, limit, scope, admission=admission) async def get_topic_memory( self, @@ -1088,7 +1096,8 @@ async def search_skills( raise ValueError("Skill search limit must be positive") # noqa: TRY003 scope = validate_scope_id(scope_id) async with self.database.transaction() as connection: - return await self.experience_index.search_skills(connection, scope, query, limit) + rows = await self.experience_index.search(connection, scope, "skill", query, limit) + return skill_search_hits(rows, query, limit) async def get_skill_governance( self, diff --git a/src/powercontext/builtin/runtime/skill_search.py b/src/powercontext/builtin/runtime/skill_search.py index e44b5496f1..965759e8f9 100644 --- a/src/powercontext/builtin/runtime/skill_search.py +++ b/src/powercontext/builtin/runtime/skill_search.py @@ -25,7 +25,6 @@ ArtifactSearchMatch, ArtifactSearchUnsupported, ) -from powercontext.builtin.artifacts.search import InvalidSearchScore from powercontext.builtin.artifacts.skill import Skill, SkillSearchHit, SkillSearchOutcome, SkillSearchRequest from powercontext.builtin.persistence.artifact_governance import ( ArtifactGovernance, @@ -35,7 +34,7 @@ from powercontext.builtin.persistence.artifacts import ArtifactRepository, with_artifact_content from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.errors import RepositoryNotFoundError -from powercontext.builtin.persistence.experience_index import ExperienceIndex, NoExperienceIndex +from powercontext.builtin.persistence.experience_index import ExperienceIndex, skill_search_hits from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE from powercontext.builtin.runtime.models import GetSkillRequest from powercontext.builtin.sources import validate_scope_id @@ -119,13 +118,21 @@ async def __call__( async with self._database.transaction(consistent_snapshot=bool(query)) as connection: values: list[tuple[Skill, ArtifactGovernance]] = [] if query: - hits = await self._index.search_skills(connection, scope, query, limit) + rows = await self._index.search(connection, scope, "skill", query, limit) + hits = skill_search_hits(rows, query, limit) refs = tuple(hit.artifact_ref for hit in hits) invalid_ref = next((ref for ref in refs if ref.family != Skill.family), None) if invalid_ref is not None: raise ArtifactNotFoundError(invalid_ref) try: - skills = await self._artifacts.get_many(connection, scope, refs) + by_ref = { + (str(row["artifact_id"]), int(row["revision"])): row + for row in rows + if not row.get("is_rejected") + } + skills = await self._artifacts.decode_many( + connection, scope, tuple(by_ref[(ref.artifact_id, ref.revision)] for ref in refs) + ) except RepositoryNotFoundError as error: raise ArtifactNotFoundError(cast(tuple[str, ArtifactRef], error.identity)[1]) from None governance = {} @@ -181,29 +188,37 @@ async def search( *, execution_context: ArtifactSearchExecutionContext | None = None, ) -> SkillSearchOutcome: - if isinstance(self._index, NoExperienceIndex): + if self.family not in self._index.families: raise ArtifactSearchUnsupported(self.family, field="mode") scope = validate_scope_id(scope_id) async with self._database.transaction(consistent_snapshot=True) as connection: - hits = await self._index.search_skills( - connection, - scope, - request.query, - request.limit, - admission=request.admission.as_floor(), - min_score=request.min_score, - require_scores=True, + rows = await self._index.search( + connection, scope, "skill", request.query, request.limit, admission=request.admission.as_floor() ) - matches: list[ArtifactSearchMatch] = [] - for hit in hits: - if hit.retrieval_score is None or hit.channel_scores is None or "text" not in hit.channel_scores: - raise InvalidSearchScore("public Skill search requires true text scoring metadata") # noqa: TRY003 - matches.append(ArtifactSearchMatch(hit.artifact_ref, hit.retrieval_score, hit.channel_scores)) try: - artifacts = await self._artifacts.get_many(connection, scope, tuple(hit.artifact_ref for hit in hits)) + hits = skill_search_hits( + rows, + request.query, + request.limit, + admission=request.admission.as_floor(), + min_score=request.min_score, + require_scores=True, + ) + by_ref = { + (str(row["artifact_id"]), int(row["revision"])): row for row in rows if not row.get("is_rejected") + } + artifacts = await self._artifacts.decode_many( + connection, + scope, + tuple(by_ref[(hit.artifact_ref.artifact_id, hit.artifact_ref.revision)] for hit in hits), + ) except RepositoryNotFoundError as error: raise ArtifactSearchContractError(self.family, "selected exact Artifact revision is missing") from error - return SkillSearchOutcome(hits=hits, matches=tuple(matches), artifacts=cast(tuple[Skill, ...], artifacts)) + matches = tuple( + ArtifactSearchMatch(hit.artifact_ref, cast(float, hit.retrieval_score), hit.channel_scores) + for hit in hits + ) + return SkillSearchOutcome(hits=hits, matches=matches, artifacts=cast(tuple[Skill, ...], artifacts)) __all__ = ["SkillArtifactSearcher", "search_skill_library"] diff --git a/src/powercontext/builtin/runtime/topic_memory_search.py b/src/powercontext/builtin/runtime/topic_memory_search.py index 83ab75756e..0228dda443 100644 --- a/src/powercontext/builtin/runtime/topic_memory_search.py +++ b/src/powercontext/builtin/runtime/topic_memory_search.py @@ -22,13 +22,11 @@ from typing import Any from powercontext._logging import log_safely -from powercontext.artifacts import ArtifactRef from powercontext.artifacts.search import ArtifactSearchContractError, ArtifactSearchExecutionContext from powercontext.builtin.artifacts.memory import MemoryQueryEmbedding from powercontext.builtin.artifacts.search import AdmissionFloor, analyze_text from powercontext.builtin.artifacts.topic_memory import ( MAX_TOPIC_MEMORY_QUERY_TERMS, - PublishedTopicMemory, TopicArtifactSearchRequest, TopicMemoryCapabilities, TopicMemorySearchResult, @@ -42,7 +40,6 @@ InvalidInferenceOutputError, ) from powercontext.builtin.inference.protocols import embed_query -from powercontext.builtin.persistence.errors import RepositoryNotFoundError logger = logging.getLogger(__name__) @@ -62,13 +59,11 @@ def __init__( self, *, search: Callable[..., Awaitable[TopicMemorySearchResult]], - get: Callable[[str, ArtifactRef], Awaitable[PublishedTopicMemory]] | None = None, embedding_model: EmbeddingModel | None = None, observer: Callable[[str, bool], None] | None = None, capabilities: TopicMemoryCapabilities | None = None, ) -> None: self._search = search - self._get = get self._embedding = embedding_model self._observer = observer self._capabilities = capabilities @@ -111,21 +106,8 @@ async def search( embedding_model=embedding, execution_context=execution_context, ) - if result.artifacts is None: - if self._get is None: - raise ArtifactSearchContractError( - self.family, "public Topic search did not materialize exact Artifacts" - ) - if len(result.matches) != len(result.hits): - raise ArtifactSearchContractError(self.family, "public Topic hits and matches must have the same count") - artifacts = [] - for match in result.matches: - try: - publication = await self._get(scope_id, match.artifact_ref) - except RepositoryNotFoundError as error: - raise ArtifactSearchContractError(self.family, "selected exact Artifact is missing") from error - artifacts.append(publication.topic) - result = result.model_copy(update={"artifacts": tuple(artifacts)}) + if len(result.matches) != len(result.hits) or len(result.artifacts) != len(result.matches): + raise ArtifactSearchContractError(self.family, "public Topic search requires complete exact Artifacts") self._observe(result, fallback) return result diff --git a/src/powercontext/server/mapping.py b/src/powercontext/server/mapping.py index d074ad8b28..7858c25eba 100644 --- a/src/powercontext/server/mapping.py +++ b/src/powercontext/server/mapping.py @@ -343,8 +343,6 @@ def artifact_search_response(outcome: ArtifactSearchOutcome, /, *, include_scores: bool) -> SearchArtifactsResponse: """Serialize actual Artifact content while retaining the Family's final match order.""" - if outcome.artifacts is None: - raise ValueError("public Artifact search requires complete Artifacts") # noqa: TRY003 results: list[dict[str, Any]] = [] for match, artifact in zip(outcome.matches, outcome.artifacts, strict=True): item: dict[str, Any] = { diff --git a/tests/builtin/persistence/test_experience_index.py b/tests/builtin/persistence/test_experience_index.py index 3ba140e7bf..230a76b8ea 100644 --- a/tests/builtin/persistence/test_experience_index.py +++ b/tests/builtin/persistence/test_experience_index.py @@ -218,10 +218,12 @@ async def scenario() -> None: assert raw.raw < 0 assert raw.higher_is_better is False assert scored_hit.retrieval_score == -raw.raw / (1 - raw.raw) - async with contexts.database.transaction() as connection: - excluded = await contexts.experience_index.search( - connection, scope_id, "hamsterlegacy", 8, min_score=1, require_scores=True - ) + from powercontext.builtin.artifacts.experience import ExperienceSearchRequest + from powercontext.builtin.runtime.experience_search import ExperienceArtifactSearcher + + excluded = await ExperienceArtifactSearcher( + contexts.database, contexts.repositories.artifacts, contexts.experience_index + ).search(scope_id, ExperienceSearchRequest(query="hamsterlegacy", limit=8, min_score=1)) assert excluded.hits == () assert excluded.admission is not None assert excluded.admission.admitted == 1 @@ -471,11 +473,10 @@ def test_oceanbase_search_preserves_match_raw_score() -> None: dialect=SimpleNamespace(name="mysql"), execute=AsyncMock(return_value=SimpleNamespace(mappings=lambda: rows)), ) - outcome = asyncio.run( - OceanBaseExperienceFTSIndex().search( - cast(AsyncConnection, connection), "project", "client", 8, require_scores=True - ) + selected = asyncio.run( + OceanBaseExperienceFTSIndex().search(cast(AsyncConnection, connection), "project", "experience", "client", 8) ) + outcome = experience_search_hits(selected, "client", 8, "project", require_scores=True) assert outcome.hits[0].channel_scores is not None assert outcome.hits[0].channel_scores["text"].metric == "oceanbase_match" assert outcome.hits[0].channel_scores["text"].raw == 2.0 diff --git a/tests/builtin/persistence/test_records.py b/tests/builtin/persistence/test_records.py index af705717ee..887f5cac12 100644 --- a/tests/builtin/persistence/test_records.py +++ b/tests/builtin/persistence/test_records.py @@ -30,7 +30,12 @@ from powercontext.builtin.artifacts.handoff import Handoff from powercontext.builtin.artifacts.skill import Skill from powercontext.builtin.persistence.artifacts import ArtifactRepository -from powercontext.builtin.persistence.experience_index import ExperienceIndex, NoExperienceIndex +from powercontext.builtin.persistence.experience_index import ( + ExperienceIndex, + NoExperienceIndex, + experience_search_hits, + skill_search_hits, +) from powercontext.builtin.persistence.family_management import ( ExperienceManagementWriter, FamilyManagementWriterRegistry, @@ -740,8 +745,10 @@ async def scenario() -> None: ) async with profile.database.transaction() as connection: - experience_hits = await index.search(connection, "scope-a", "rollback", 10) - skill_hits = await index.search_skills(connection, "scope-a", "matrix", 10) + experience_rows = await index.search(connection, "scope-a", "experience", "rollback", 10) + skill_rows = await index.search(connection, "scope-a", "skill", "matrix", 10) + experience_hits = experience_search_hits(experience_rows, "rollback", 10, "scope-a") + skill_hits = skill_search_hits(skill_rows, "matrix", 10) package_count = await connection.scalar(select(func.count()).select_from(SKILL_PACKAGES_TABLE)) assert [hit.artifact_ref for hit in experience_hits.hits] == [ ArtifactRef( diff --git a/tests/builtin/persistence/test_topic_memory.py b/tests/builtin/persistence/test_topic_memory.py index 91a8f60761..a595af3352 100644 --- a/tests/builtin/persistence/test_topic_memory.py +++ b/tests/builtin/persistence/test_topic_memory.py @@ -169,7 +169,7 @@ async def scenario() -> None: assert plain.artifacts is not None assert plain.artifacts[0] == publication.topic assert plain.artifacts[0].as_ref() == scored.matches[0].artifact_ref - assert legacy.artifacts is None + assert tuple(hit.artifact_ref for hit in legacy.hits) == tuple(hit.artifact_ref for hit in plain.hits) assert empty.artifacts == () assert plain.matches[0].channel_scores is None assert scored.matches[0].channel_scores is not None diff --git a/tests/builtin/review/test_service.py b/tests/builtin/review/test_service.py index 0589294177..3cfe252923 100644 --- a/tests/builtin/review/test_service.py +++ b/tests/builtin/review/test_service.py @@ -25,13 +25,13 @@ from powercontext.builtin.artifacts.experience import ( Experience, ExperienceContent, - ExperienceSearchOutcome, FailureRecord, FailureSignature, FailureVerification, ) -from powercontext.builtin.artifacts.skill import Skill, SkillContent, SkillPackageSnapshot, SkillSearchHit +from powercontext.builtin.artifacts.skill import Skill, SkillContent, SkillPackageSnapshot from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.experience_index import NoExperienceIndex from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import ARTIFACTS_TABLE, BUILTIN_TABLES from powercontext.builtin.records import ArtifactWrite @@ -80,7 +80,7 @@ class _InjectedExperienceProjectionError(RuntimeError): pass -class _FailingExperienceIndex: +class _FailingExperienceIndex(NoExperienceIndex): async def initialize(self, _connection: AsyncConnection, /) -> None: pass @@ -96,20 +96,6 @@ async def replace( ) -> None: raise _InjectedExperienceProjectionError - async def search( - self, - _connection: AsyncConnection, - _scope_id: str, - _query: str, - _limit: int, - /, - *, - admission: object = None, - min_score: float | None = None, - require_scores: bool = False, - ) -> ExperienceSearchOutcome: - return ExperienceSearchOutcome() - async def replace_skill( self, _connection: AsyncConnection, @@ -120,20 +106,6 @@ async def replace_skill( ) -> None: pass - async def search_skills( - self, - _connection: AsyncConnection, - _scope_id: str, - _query: str, - _limit: int, - /, - *, - admission: object = None, - min_score: float | None = None, - require_scores: bool = False, - ) -> tuple[SkillSearchHit, ...]: - return () - def _proposal(lesson: str = "Regenerate the Client before contract tests.") -> ExperienceContent: return ExperienceContent( diff --git a/tests/builtin/runtime/test_artifact_search.py b/tests/builtin/runtime/test_artifact_search.py index fe59270e31..d6cfe85a50 100644 --- a/tests/builtin/runtime/test_artifact_search.py +++ b/tests/builtin/runtime/test_artifact_search.py @@ -28,9 +28,8 @@ ArtifactSearchMatch, ArtifactSearchQuery, ArtifactSearchUnsupported, - ChannelScore, ) -from powercontext.builtin.artifacts.experience import ExperienceContent, ExperienceSearchHit, ExperienceSearchOutcome +from powercontext.builtin.artifacts.experience import ExperienceContent from powercontext.builtin.artifacts.search import InvalidSearchScore from powercontext.builtin.artifacts.skill import Skill, SkillContent, SkillSearchHit from powercontext.builtin.persistence.artifact_governance import ArtifactGovernance, ArtifactLifecycleState @@ -139,13 +138,17 @@ async def scenario() -> None: @pytest.mark.parametrize("family", ["experience", "skill"]) -def test_public_lexical_search_reports_disabled_index_while_legacy_stays_empty(family: str) -> None: +@pytest.mark.parametrize("custom_index", [False, True]) +def test_public_lexical_search_reports_disabled_index_while_legacy_stays_empty(family: str, custom_index: bool) -> None: + class DisabledIndex(SQLiteExperienceFTSIndex): + families = frozenset() + async def scenario() -> None: from powercontext.builtin.runtime.experience_search import ExperienceArtifactSearcher from powercontext.builtin.runtime.skill_search import SkillArtifactSearcher async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: - index = NoExperienceIndex() + index = DisabledIndex() if custom_index else NoExperienceIndex() service = ArtifactSearchService(known_families={family}) if family == "experience": service.register(ExperienceArtifactSearcher(contexts.database, contexts.repositories.artifacts, index)) @@ -154,8 +157,8 @@ async def scenario() -> None: with pytest.raises(ArtifactSearchUnsupported): await service.search("project", family, {"query": "client"}) async with contexts.database.transaction() as connection: - assert (await index.search(connection, "project", "", 500)).hits == () - assert await index.search_skills(connection, "project", "", 500) == () + assert await index.search(connection, "project", "experience", "", 500) == () + assert await index.search(connection, "project", "skill", "", 500) == () asyncio.run(scenario()) @@ -237,15 +240,14 @@ def test_public_experience_requires_raw_metadata_even_when_scores_are_hidden(inc from powercontext.builtin.runtime.experience_search import ExperienceArtifactSearcher class Index(SQLiteExperienceFTSIndex): - async def search(self, *_args: Any, **_kwargs: Any) -> ExperienceSearchOutcome: - return ExperienceSearchOutcome( - hits=( - ExperienceSearchHit( - artifact_ref=ArtifactRef(family="experience", artifact_id="missing-raw", revision=1), - content=cast(ExperienceContent, _content("experience", "needle")), - retrieval_score=0.5, - ), - ) + async def search(self, *_args: Any, **_kwargs: Any) -> tuple[dict[str, Any], ...]: + return ( + { + "family": "experience", + "artifact_id": "missing-raw", + "revision": 1, + "content": _content("experience", "needle").model_dump_json().encode(), + }, ) async def scenario() -> None: @@ -333,30 +335,17 @@ def test_public_search_missing_selected_revision_is_an_internal_contract_failure from powercontext.builtin.runtime.experience_search import ExperienceArtifactSearcher from powercontext.builtin.runtime.skill_search import SkillArtifactSearcher - ref = ArtifactRef(family=family, artifact_id="missing-selected-revision", revision=1) - scores = {"text": ChannelScore(-1.0, "sqlite_bm25", False)} - class Index(SQLiteExperienceFTSIndex): - async def search(self, *_args: Any, **_kwargs: Any) -> ExperienceSearchOutcome: - return ExperienceSearchOutcome( - hits=( - ExperienceSearchHit( - artifact_ref=ref, - content=cast(ExperienceContent, _content(family, "needle")), - retrieval_score=0.5, - channel_scores=scores, - ), - ) - ) - - async def search_skills(self, *_args: Any, **_kwargs: Any) -> tuple[SkillSearchHit, ...]: + async def search(self, *_args: Any, **_kwargs: Any) -> tuple[dict[str, Any], ...]: return ( - SkillSearchHit( - artifact_ref=ref, - content=cast(SkillContent, _content(family, "needle")), - retrieval_score=0.5, - channel_scores=scores, - ), + { + "family": family, + "artifact_id": "missing-selected-revision", + "revision": 1, + "content": None, + "raw_score": -1.0, + "score_metric": "sqlite_bm25", + }, ) async def scenario() -> None: diff --git a/tests/builtin/runtime/test_topic_memory_search.py b/tests/builtin/runtime/test_topic_memory_search.py index 7ebccc911d..91d79209b9 100644 --- a/tests/builtin/runtime/test_topic_memory_search.py +++ b/tests/builtin/runtime/test_topic_memory_search.py @@ -49,11 +49,9 @@ def _service(*, embedding=None, capabilities=None, calls=None, observer=None): async def search(scope_id, query, /, **kwargs): if calls is not None: calls.append(kwargs) - return TopicMemorySearchResult(mode=kwargs["mode"], artifacts=() if "artifact_request" in kwargs else None) + return TopicMemorySearchResult(mode=kwargs["mode"], artifacts=()) - return TopicMemorySearcher( - search=search, get=None, embedding_model=embedding, capabilities=capabilities, observer=observer - ) + return TopicMemorySearcher(search=search, embedding_model=embedding, capabilities=capabilities, observer=observer) @pytest.mark.parametrize("mode", ["text", "vector", "hybrid"]) @@ -133,7 +131,7 @@ async def legacy(scope, query, /, *, limit, mode, query_vector, embedding_profil from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher - service = TopicMemorySearcher(search=legacy, get=None, embedding_model=embedding) + service = TopicMemorySearcher(search=legacy, embedding_model=embedding) result = asyncio.run( service.search_legacy( "scope", @@ -143,7 +141,6 @@ async def legacy(scope, query, /, *, limit, mode, query_vector, embedding_profil ) ) assert result.embedding_calls == 0 - assert result.artifacts is None assert embedding.calls == 0 assert calls[0][2] == (1.0, 0.0) @@ -293,7 +290,6 @@ def test_topic_service_missing_selected_exact_revision_is_a_contract_failure(): from powercontext.artifacts import ArtifactRef from powercontext.artifacts.search import ArtifactSearchContractError, ArtifactSearchMatch from powercontext.builtin.artifacts.topic_memory import TopicMemorySearchHit - from powercontext.builtin.persistence.errors import RepositoryNotFoundError from powercontext.builtin.runtime.topic_memory_search import TopicMemorySearcher ref = ArtifactRef(family="topic-memory", artifact_id="selected", revision=1) @@ -309,9 +305,6 @@ async def search(scope, query, /, **kwargs): matches=(ArtifactSearchMatch(ref, 0.5),), ) - async def get(scope, artifact_ref): - raise RepositoryNotFoundError("artifact", artifact_ref) - - service = TopicMemorySearcher(search=search, get=get, capabilities=TopicMemoryCapabilities(fts=True)) + service = TopicMemorySearcher(search=search, capabilities=TopicMemoryCapabilities(fts=True)) with pytest.raises(ArtifactSearchContractError): asyncio.run(service.search("scope", TopicArtifactSearchRequest(query="needle", mode="text"))) diff --git a/tests/builtin/test_publication.py b/tests/builtin/test_publication.py index 7776ff7328..59289bdc07 100644 --- a/tests/builtin/test_publication.py +++ b/tests/builtin/test_publication.py @@ -20,7 +20,7 @@ from sqlalchemy import update from powercontext.artifacts import ArtifactAddress, ArtifactRef -from powercontext.builtin.artifacts.experience import ExperienceContent, ExperienceDraft, ExperienceSearchOutcome +from powercontext.builtin.artifacts.experience import ExperienceContent, ExperienceDraft from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.errors import InvalidPublicationLineageError, RepositoryNotFoundError from powercontext.builtin.persistence.experience_index import NoExperienceIndex @@ -42,7 +42,7 @@ class _IndexUnavailableError(RuntimeError): pass -class _FailingExperienceIndex: +class _FailingExperienceIndex(NoExperienceIndex): async def initialize(self, _connection, /) -> None: pass @@ -52,19 +52,9 @@ async def remove(self, _connection, _scope_id, _artifact_id, /) -> None: async def replace(self, _connection, _scope_id, _experience, /) -> None: raise _IndexUnavailableError - async def search( - self, _connection, _scope_id, _query, _limit, /, *, admission=None, min_score=None, require_scores=False - ): - return ExperienceSearchOutcome() - async def replace_skill(self, _connection, _scope_id, _skill, _package, /) -> None: pass - async def search_skills( - self, _connection, _scope_id, _query, _limit, /, *, admission=None, min_score=None, require_scores=False - ): - return () - def test_publication_copies_one_exact_revision_with_original_provenance(tmp_path) -> None: async def scenario() -> None: diff --git a/tests/e2e/test_atomic_artifact_search_domain.py b/tests/e2e/test_atomic_artifact_search_domain.py index a1511653a4..354fb2bf9c 100644 --- a/tests/e2e/test_atomic_artifact_search_domain.py +++ b/tests/e2e/test_atomic_artifact_search_domain.py @@ -125,7 +125,6 @@ async def scenario(): ] assert len(kept.matches) < 3 legacy = await contexts.atomic_memory.for_scope("project").search("alpha", mode=mode) - assert legacy.artifacts is None and legacy.matches == () assert [hit.hit.artifact_ref for hit in legacy.hits] == [match.artifact_ref for match in scored.matches] asyncio.run(scenario()) diff --git a/tests/e2e/test_lexical_admission.py b/tests/e2e/test_lexical_admission.py index 90892c2640..3d7a782431 100644 --- a/tests/e2e/test_lexical_admission.py +++ b/tests/e2e/test_lexical_admission.py @@ -79,20 +79,18 @@ async def scenario() -> None: family, ArtifactWrite(content=_content(family, "alpha" if number % 2 else "beta", number)), ) - async with contexts.database.transaction() as connection: - if family == "experience": - result = await contexts.experience_index.search(connection, "project", "alpha beta gamma", 1) - hits = result.hits - relaxed = await contexts.experience_index.search( - connection, - "project", - "alpha beta gamma", - 1, - admission=AdmissionFloor(lexical_min_matched_terms=1, lexical_coverage=0), - ) - assert relaxed.hits and relaxed.hits[0].artifact_ref.artifact_id != target.artifact_id - else: - hits = await contexts.experience_index.search_skills(connection, "project", "alpha beta gamma", 1) + if family == "experience": + result = await contexts.search_experience_outcome("project", "alpha beta gamma", 1) + hits = result.hits + relaxed = await contexts.search_experience_outcome( + "project", + "alpha beta gamma", + 1, + admission=AdmissionFloor(lexical_min_matched_terms=1, lexical_coverage=0), + ) + assert relaxed.hits and relaxed.hits[0].artifact_ref.artifact_id != target.artifact_id + else: + hits = await contexts.search_skills("project", "alpha beta gamma", 1) assert len(hits) == 1 assert hits[0].artifact_ref.artifact_id == target.artifact_id assert ( diff --git a/tests/e2e/test_observability.py b/tests/e2e/test_observability.py index 35d33a864f..110bf27262 100644 --- a/tests/e2e/test_observability.py +++ b/tests/e2e/test_observability.py @@ -245,6 +245,9 @@ def __init__(self, profile=None): async def initialize(self, connection: AsyncConnection, /): pass + async def validate_current(self, connection: AsyncConnection, /): + pass + async def replace(self, connection: AsyncConnection, scope_id: str, projection: AtomicMemoryProjection, /): pass diff --git a/tests/e2e/test_recall_sufficiency_gate.py b/tests/e2e/test_recall_sufficiency_gate.py index 41392ca803..4e7c6da419 100644 --- a/tests/e2e/test_recall_sufficiency_gate.py +++ b/tests/e2e/test_recall_sufficiency_gate.py @@ -773,7 +773,6 @@ async def scenario() -> None: assert runtime._topic_memory_search is not None runtime._topic_memory_searcher = TopicMemorySearcher( search=runtime._topic_memory_search, - get=runtime._topic_memory_get, embedding_model=embedding, observer=runtime._topic_memory_search_observer, ) From defe7f115f1d62337d884bf0aafd326bf0b0f1cd Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Tue, 20 Oct 2026 12:05:23 +0800 Subject: [PATCH 5/9] perf(search): reuse FTS projections across restarts Record completed refreshes in the existing lexical version table. Reuse stored text and native indexes on warm startup, and retain explicit projection repair. Keep inactive Skill heads indexed so reactivation remains searchable after a restart. --- .../builtin/persistence/experience_index.py | 11 +++ .../builtin/persistence/lexical_projection.py | 46 +++++++---- .../persistence/oceanbase/experience_index.py | 18 +++- .../persistence/sqlite/experience_index.py | 56 +++++++------ .../persistence/test_experience_index.py | 2 +- tests/e2e/test_lexical_projection_upgrade.py | 82 ++++++++++++++++++- 6 files changed, 171 insertions(+), 44 deletions(-) diff --git a/src/powercontext/builtin/persistence/experience_index.py b/src/powercontext/builtin/persistence/experience_index.py index f51f182532..3360cb258f 100644 --- a/src/powercontext/builtin/persistence/experience_index.py +++ b/src/powercontext/builtin/persistence/experience_index.py @@ -56,8 +56,12 @@ from powercontext.builtin.persistence.atomic_memory_index import oceanbase_fts_coverage_sql, sqlite_fts_coverage_sql from powercontext.builtin.persistence.codec import load_model, stored_bytes from powercontext.builtin.persistence.errors import RepositoryNotFoundError +from powercontext.builtin.persistence.lexical_projection import LEXICAL_ANALYZER_VERSION from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE, SKILL_PACKAGES_TABLE +# Bump the suffix when Family searchable fields or package evidence rules change. +APPROVED_ARTIFACT_PROJECTION_VERSION = f"{LEXICAL_ANALYZER_VERSION}:approved-v1" + _SQLITE_SEARCHABLE_TEXT_EXISTS_SQL = text( """ SELECT COUNT(*) @@ -94,6 +98,10 @@ class ExperienceIndex(Protocol): async def initialize(self, connection: AsyncConnection, /) -> None: ... + async def rebuild(self, connection: AsyncConnection, /) -> None: + """Repair initialized projections from authority with writers stopped.""" + ... + async def replace( self, connection: AsyncConnection, @@ -140,6 +148,9 @@ class NoExperienceIndex: async def initialize(self, _connection: AsyncConnection, /) -> None: pass + async def rebuild(self, _connection: AsyncConnection, /) -> None: + pass + async def replace( self, _connection: AsyncConnection, diff --git a/src/powercontext/builtin/persistence/lexical_projection.py b/src/powercontext/builtin/persistence/lexical_projection.py index 4175c25ed3..b1f564e3ed 100644 --- a/src/powercontext/builtin/persistence/lexical_projection.py +++ b/src/powercontext/builtin/persistence/lexical_projection.py @@ -27,7 +27,7 @@ ) # Bump when document segmentation or token encoding changes; query stopwords do not affect projections. -_ANALYZER_VERSION = "jieba-0.42.1-words-v1" +LEXICAL_ANALYZER_VERSION = "jieba-0.42.1-words-v1" async def refresh_lexical_projections(connection: AsyncConnection, atomic_table: Table | None) -> None: @@ -41,30 +41,44 @@ async def refresh_lexical_projections(connection: AsyncConnection, atomic_table: projections.append((atomic_table, ("kind", "text"))) topic_changed = False for table, fields in projections: - if versions.get(table.name) == _ANALYZER_VERSION: + if versions.get(table.name) == LEXICAL_ANALYZER_VERSION: continue changed = await _refresh_projection(connection, table, fields) topic_changed |= changed and table is not atomic_table - values = {"projection": table.name, "analyzer_version": _ANALYZER_VERSION} - if connection.dialect.name == "sqlite": - marker = ( - sqlite_insert(LEXICAL_PROJECTION_VERSIONS_TABLE) - .values(values) - .on_conflict_do_update(index_elements=["projection"], set_={"analyzer_version": _ANALYZER_VERSION}) - ) - else: - marker = ( - mysql_insert(LEXICAL_PROJECTION_VERSIONS_TABLE) - .values(values) - .on_duplicate_key_update(analyzer_version=_ANALYZER_VERSION) - ) - await connection.execute(marker) + await record_lexical_projection_version(connection, table.name) if topic_changed and connection.dialect.name == "sqlite": from powercontext.builtin.persistence.sqlite.topic_memory_index import SQLITE_TOPIC_MEMORY_FTS_MARKER_TABLE await connection.execute(SQLITE_TOPIC_MEMORY_FTS_MARKER_TABLE.delete()) +async def lexical_projection_is_current( + connection: AsyncConnection, projection: str, *, version: str = LEXICAL_ANALYZER_VERSION +) -> bool: + table = LEXICAL_PROJECTION_VERSIONS_TABLE + return await connection.scalar(select(table.c.analyzer_version).where(table.c.projection == projection)) == version + + +async def record_lexical_projection_version( + connection: AsyncConnection, projection: str, *, version: str = LEXICAL_ANALYZER_VERSION +) -> None: + """Record a completed refresh in the same transaction as its derived data.""" + values = {"projection": projection, "analyzer_version": version} + if connection.dialect.name == "sqlite": + statement = ( + sqlite_insert(LEXICAL_PROJECTION_VERSIONS_TABLE) + .values(values) + .on_conflict_do_update(index_elements=["projection"], set_={"analyzer_version": version}) + ) + else: + statement = ( + mysql_insert(LEXICAL_PROJECTION_VERSIONS_TABLE) + .values(values) + .on_duplicate_key_update(analyzer_version=version) + ) + await connection.execute(statement) + + async def _refresh_projection(connection: AsyncConnection, table: Table, fields: tuple[str, ...]) -> bool: """Page one projection and write only changed tokens in the caller's transaction.""" keys = tuple(table.primary_key.columns) diff --git a/src/powercontext/builtin/persistence/oceanbase/experience_index.py b/src/powercontext/builtin/persistence/oceanbase/experience_index.py index 91892507bf..e53c2f2512 100644 --- a/src/powercontext/builtin/persistence/oceanbase/experience_index.py +++ b/src/powercontext/builtin/persistence/oceanbase/experience_index.py @@ -28,6 +28,7 @@ from powercontext.builtin.artifacts.search import AdmissionFloor from powercontext.builtin.artifacts.skill import Skill, SkillPackageSnapshot from powercontext.builtin.persistence.experience_index import ( + APPROVED_ARTIFACT_PROJECTION_VERSION, ensure_artifact_head_searchable_text, rebuild_experience_projections, rebuild_skill_projections, @@ -36,6 +37,10 @@ replace_skill_projection, search_approved_artifact_rows, ) +from powercontext.builtin.persistence.lexical_projection import ( + lexical_projection_is_current, + record_lexical_projection_version, +) from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE _OCEANBASE_FTS_INDEX_NAME = "ix_pc_artifact_heads_fts" @@ -63,17 +68,26 @@ async def initialize(self, connection: AsyncConnection, /) -> None: if connection.dialect.name != "mysql": raise CapabilityNotSupportedError("oceanbase-experience-fts") await ensure_artifact_head_searchable_text(connection) - await rebuild_experience_projections(connection) - await rebuild_skill_projections(connection) count = await connection.scalar( _OCEANBASE_FTS_INDEX_EXISTS_SQL, {"index_name": _OCEANBASE_FTS_INDEX_NAME}, ) if count == 0: await connection.exec_driver_sql(_OCEANBASE_CREATE_FTS_SQL) + if not await lexical_projection_is_current( + connection, ARTIFACT_HEADS_TABLE.name, version=APPROVED_ARTIFACT_PROJECTION_VERSION + ): + await self.rebuild(connection) probe = match(ARTIFACT_HEADS_TABLE.c.searchable_text, against="powercontext") await connection.execute(select(ARTIFACT_HEADS_TABLE.c.artifact_id).where(probe).limit(1)) + async def rebuild(self, connection: AsyncConnection, /) -> None: + await rebuild_experience_projections(connection) + await rebuild_skill_projections(connection) + await record_lexical_projection_version( + connection, ARTIFACT_HEADS_TABLE.name, version=APPROVED_ARTIFACT_PROJECTION_VERSION + ) + async def replace( self, connection: AsyncConnection, diff --git a/src/powercontext/builtin/persistence/sqlite/experience_index.py b/src/powercontext/builtin/persistence/sqlite/experience_index.py index e04a8d84e4..e945ae820d 100644 --- a/src/powercontext/builtin/persistence/sqlite/experience_index.py +++ b/src/powercontext/builtin/persistence/sqlite/experience_index.py @@ -19,6 +19,7 @@ from collections.abc import Mapping from typing import Any, Literal +from aiosqlite import Connection as SQLiteConnection from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncConnection @@ -30,6 +31,7 @@ from powercontext.builtin.artifacts.search import AdmissionFloor from powercontext.builtin.artifacts.skill import Skill, SkillPackageSnapshot, skill_searchable_text from powercontext.builtin.persistence.experience_index import ( + APPROVED_ARTIFACT_PROJECTION_VERSION, ensure_artifact_head_searchable_text, rebuild_experience_projections, rebuild_skill_projections, @@ -38,6 +40,11 @@ replace_skill_projection, search_approved_artifact_rows, ) +from powercontext.builtin.persistence.lexical_projection import ( + lexical_projection_is_current, + record_lexical_projection_version, +) +from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE _CREATE_FTS_SQL = """ CREATE VIRTUAL TABLE IF NOT EXISTS pc_artifact_fts USING fts5( @@ -49,30 +56,14 @@ tokenize='unicode61' ) """ -_RECONCILE_DELETE_FTS_SQL = """ -DELETE FROM pc_artifact_fts -WHERE NOT EXISTS ( - SELECT 1 FROM pc_artifact_heads AS h - WHERE h.scope_id = pc_artifact_fts.scope_id AND h.family = pc_artifact_fts.family - AND h.artifact_id = pc_artifact_fts.artifact_id AND h.revision = pc_artifact_fts.revision - AND h.family IN ('experience', 'skill') AND h.lifecycle_state = 'active' - AND h.searchable_text = pc_artifact_fts.searchable_text -) -OR rowid NOT IN ( - SELECT min(rowid) FROM pc_artifact_fts GROUP BY scope_id, family, artifact_id -) -""" -_RECONCILE_INSERT_FTS_SQL = """ +_BACKFILL_FTS_SQL = """ INSERT INTO pc_artifact_fts (scope_id, family, artifact_id, revision, searchable_text) SELECT h.scope_id, h.family, h.artifact_id, h.revision, h.searchable_text FROM pc_artifact_heads AS h -WHERE h.family IN ('experience', 'skill') AND h.lifecycle_state = 'active' +WHERE h.family IN ('experience', 'skill') AND h.searchable_text IS NOT NULL -EXCEPT -SELECT scope_id, family, artifact_id, revision, searchable_text -FROM pc_artifact_fts """ -_PROBE_FTS_SQL = "SELECT rowid FROM pc_artifact_fts WHERE pc_artifact_fts MATCH 'powercontext'" +_PROBE_FTS_SQL = "SELECT rowid FROM pc_artifact_fts WHERE pc_artifact_fts MATCH 'powercontext' LIMIT 1" _DELETE_FTS_SQL = text( "DELETE FROM pc_artifact_fts WHERE scope_id = :scope_id AND family = :family AND artifact_id = :artifact_id" ) @@ -92,13 +83,32 @@ class SQLiteExperienceFTSIndex: async def initialize(self, connection: AsyncConnection, /) -> None: if connection.dialect.name != "sqlite": raise CapabilityNotSupportedError("sqlite-experience-fts") + driver = (await connection.get_raw_connection()).driver_connection + if isinstance(driver, SQLiteConnection) and not driver.in_transaction: + await connection.exec_driver_sql("BEGIN") await ensure_artifact_head_searchable_text(connection) + existing = await connection.scalar( + text("SELECT 1 FROM sqlite_master WHERE type='table' AND name='pc_artifact_fts'") + ) + if not existing: + await connection.exec_driver_sql(_CREATE_FTS_SQL) + if not await lexical_projection_is_current( + connection, ARTIFACT_HEADS_TABLE.name, version=APPROVED_ARTIFACT_PROJECTION_VERSION + ): + await self.rebuild(connection) + elif not existing: + # A missing helper can be restored from heads without decoding packages. + await connection.exec_driver_sql(_BACKFILL_FTS_SQL) + await connection.exec_driver_sql(_PROBE_FTS_SQL) + + async def rebuild(self, connection: AsyncConnection, /) -> None: await rebuild_experience_projections(connection) await rebuild_skill_projections(connection) - await connection.exec_driver_sql(_CREATE_FTS_SQL) - await connection.exec_driver_sql(_RECONCILE_DELETE_FTS_SQL) - await connection.exec_driver_sql(_RECONCILE_INSERT_FTS_SQL) - await connection.exec_driver_sql(_PROBE_FTS_SQL) + await connection.exec_driver_sql("DELETE FROM pc_artifact_fts") + await connection.exec_driver_sql(_BACKFILL_FTS_SQL) + await record_lexical_projection_version( + connection, ARTIFACT_HEADS_TABLE.name, version=APPROVED_ARTIFACT_PROJECTION_VERSION + ) async def replace( self, diff --git a/tests/builtin/persistence/test_experience_index.py b/tests/builtin/persistence/test_experience_index.py index 230a76b8ea..721f65f2df 100644 --- a/tests/builtin/persistence/test_experience_index.py +++ b/tests/builtin/persistence/test_experience_index.py @@ -331,7 +331,7 @@ async def scenario() -> None: assert await contexts.search_experience(scope_id, "falconcurrent", 8) == () async with contexts.database.transaction() as connection: - await contexts.experience_index.initialize(connection) + await contexts.experience_index.rebuild(connection) rebuilt = await contexts.search_experience(scope_id, "falconcurrent", 8) assert tuple(hit.artifact_ref for hit in rebuilt) == (replaced.result_artifact,) assert await contexts.search_experience(scope_id, "staleprojection", 8) == () diff --git a/tests/e2e/test_lexical_projection_upgrade.py b/tests/e2e/test_lexical_projection_upgrade.py index 155dae193a..aa54eaebd0 100644 --- a/tests/e2e/test_lexical_projection_upgrade.py +++ b/tests/e2e/test_lexical_projection_upgrade.py @@ -16,12 +16,15 @@ import asyncio +import pytest from pydantic import JsonValue -from sqlalchemy import update +from sqlalchemy import Engine, event, update from powercontext.builtin.artifacts.memory import EmbeddingProfile from powercontext.builtin.inference import EmbeddingResult +from powercontext.builtin.persistence.artifact_governance import ArtifactLifecycleState from powercontext.builtin.persistence.atomic_memory_index_schema import atomic_memory_current_table +from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.persistence.tables import ( ARTIFACT_HEADS_TABLE, LEXICAL_PROJECTION_VERSIONS_TABLE, @@ -29,7 +32,7 @@ TOPIC_MEMORY_ACTIVE_TOPICS_TABLE, ) from powercontext.builtin.records import ArtifactWrite -from powercontext.builtin.runtime import BuiltinConfig, RuntimeConfig, open_builtin_runtime +from powercontext.builtin.runtime import BuiltinConfig, RuntimeConfig, open_builtin_contexts, open_builtin_runtime from powercontext.builtin.scope import ScopeDraft from tests.e2e.atomic_memory_migration_backend import migration_profile @@ -41,6 +44,81 @@ async def embed(self, texts, /): return EmbeddingResult(vectors=tuple((1.0, 0.0, 0.0) for _ in texts)) +@pytest.mark.parametrize("missing_helper", (False, True)) +def test_restart_reuses_search_text_and_preserves_skill_reactivation(migration_backend, missing_helper): + if missing_helper and not isinstance(migration_backend.config, SQLiteConfig): + pytest.skip("Only SQLite has a separately stored FTS helper") + config = BuiltinConfig(database=migration_backend.config) + + async def scenario(): + async with open_builtin_contexts(config) as contexts: + scope = ( + await contexts.scopes.create( + ScopeDraft(title="Restart", summary="Search lifecycle", idempotency_key="restart") + ) + ).scope_id + experience = await contexts.records.create_artifact( + scope, + "experience", + ArtifactWrite( + content={ + "situation": "restartmarker", + "action": "Restart", + "outcome": "Ready", + "lesson": "Reuse text", + } + ), + ) + skill = await contexts.records.create_artifact( + scope, + "skill", + ArtifactWrite( + content={ + "name": "restart-skill", + "description": "restartmarker", + "instructions": "Restart", + "validation": ["Search finds the current revision"], + } + ), + ) + await contexts.update_skill_lifecycle(scope, skill.artifact_id, 0, ArtifactLifecycleState.DEPRECATED, None) + if missing_helper: + async with contexts.database.transaction() as connection: + await connection.exec_driver_sql("DROP TABLE pc_artifact_fts") + + starting = True + payload_reads = [] + + def observe(connection, cursor, statement, parameters, context, executemany): + sql = statement.lower().replace('"', "").replace("`", "") + if ( + starting + and sql.lstrip().startswith("select") + and ("pc_artifacts.content" in sql or "from pc_skill_packages" in sql) + ): + payload_reads.append(statement) + + event.listen(Engine, "before_cursor_execute", observe) + try: + async with open_builtin_contexts(config) as contexts: + starting = False + # Healthy warm startup must not read Artifact bodies or package archives. + assert payload_reads == [] + assert [ + hit.artifact_ref.artifact_id for hit in await contexts.search_experience(scope, "restartmarker", 8) + ] == [experience.artifact_id] + assert await contexts.search_skills(scope, "restartmarker", 8) == () + await contexts.update_skill_lifecycle(scope, skill.artifact_id, 1, ArtifactLifecycleState.ACTIVE, None) + hits = await contexts.search_skills(scope, "restartmarker", 8) + assert [(hit.artifact_ref.artifact_id, hit.artifact_ref.revision) for hit in hits] == [ + (skill.artifact_id, skill.revision) + ] + finally: + event.remove(Engine, "before_cursor_execute", observe) + + asyncio.run(scenario()) + + def test_upgrade_preserves_search_exact_records_and_vectors(migration_backend): config = BuiltinConfig(database=migration_backend.config, runtime=RuntimeConfig(artifact_processing_families=())) embedding = _Embedding() From 6695d70026bc6a415857855715bd23fc20a00d38 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Tue, 20 Oct 2026 12:08:10 +0800 Subject: [PATCH 6/9] chore(docs): exclude RFC edits from retrieval changes --- docs/en/rfcs/1417_topic_memory.md | 8 +++----- docs/zh/rfcs/1417_topic_memory.md | 7 +++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/docs/en/rfcs/1417_topic_memory.md b/docs/en/rfcs/1417_topic_memory.md index 6e18cbd0a4..6e454a1de9 100644 --- a/docs/en/rfcs/1417_topic_memory.md +++ b/docs/en/rfcs/1417_topic_memory.md @@ -425,11 +425,9 @@ Chunk policy: 6. Exact get always returns the complete Detail. The concrete chunk length, tail threshold, and overlap ratio are internal constants associated with a policy version, -not public configuration. A hard per-Topic chunk count bounds chunk projection writes and storage per Topic; adversarial -Markdown falls back to bounded overlapping windows. This budget does not guarantee Topic-level Top-K completeness -within a backend's raw neighbor limit or bound the total retrieval work. Backend retrieval behavior and cost boundaries -are described in the [Topic Memory workflow](../docs/workflows/topic-memory.md#search-current-topics). -Changing the Chunk policy requires rebuilding the corresponding retrieval projections. +not public configuration. The implementation also enforces a hard per-Topic chunk count so the vector channel's +worst-case neighbor scan remains within the backend limit; adversarial Markdown falls back to bounded overlapping +windows. Changing the Chunk policy requires rebuilding the corresponding retrieval projections. Each retrieval channel first collapses by Topic. When multiple Detail chunks from one Topic match, only the best position is retained for the snippet. The two or four channels enabled for the current deployment are fused by RRF diff --git a/docs/zh/rfcs/1417_topic_memory.md b/docs/zh/rfcs/1417_topic_memory.md index ab49ef2393..e6a5df3a4a 100644 --- a/docs/zh/rfcs/1417_topic_memory.md +++ b/docs/zh/rfcs/1417_topic_memory.md @@ -377,10 +377,9 @@ Chunk policy： 5. 搜索命中后围绕最佳位置动态扩展 snippet； 6. exact get 始终返回完整 Detail。 -具体 chunk 长度、尾块阈值和 overlap 比例是带 policy version 的内部常量，不是公共配置。每个 Topic 的 chunk -数量上限约束单个 Topic 的 chunk 投影写入量和存储量；对抗性 Markdown 会回退为有界重叠窗口。该预算不保证在后端原始 -近邻数量限制内获得完整的 Topic 级 Top-K，也不限制整个检索的工作量。后端检索行为与成本边界见 -[Topic Memory 工作流](../docs/workflows/topic-memory.md#搜索当前主题)。Chunk 策略变化需要重建对应检索投影。 +具体 chunk 长度、尾块阈值和 overlap 比例是带 policy version 的内部常量，不是公共配置。实现还会强制每个 Topic +的 chunk 数量上限，保证向量通道最坏情况下的近邻扫描不超过后端限制；对抗性 Markdown 会回退为有界重叠窗口。 +Chunk 策略变化需要重建对应检索投影。 每个检索通道先按 Topic collapse，同一 Topic 的多个 detail 命中只保留最佳位置用于 snippet。当前部署启用的 两个或四个通道通过 RRF 排名融合，不比较全文分数和向量距离的原始数值。最终一个 Topic 最多占一个结果位置，并保留 From cedfbfdd4aa3a1f355150665cd1682e1d0a236d3 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Tue, 20 Oct 2026 19:38:37 +0800 Subject: [PATCH 7/9] fix(sqlite): reject unsupported versions before initialization Require SQLite 3.35.0 for materialized retrieval CTEs before creating the engine or accepting writes. Report the linked library version, preserve existing storage on rejection, and document the backend minimum. --- docs/en/docs/operate/configuration.md | 4 ++++ docs/zh/docs/operate/configuration.md | 4 ++++ .../builtin/persistence/sqlite/profile.py | 6 +++++ .../persistence/sqlite/topic_memory_index.py | 1 - .../persistence/test_sqlite_profile.py | 24 +++++++++++++++++++ 5 files changed, 38 insertions(+), 1 deletion(-) diff --git a/docs/en/docs/operate/configuration.md b/docs/en/docs/operate/configuration.md index 0bb5f8ec8f..dfa1e78a1f 100644 --- a/docs/en/docs/operate/configuration.md +++ b/docs/en/docs/operate/configuration.md @@ -79,6 +79,10 @@ Without an override, the default is: The default SQLite database is `powercontext.db` in this directory. Built-in background processors persist intents and scheduling checkpoints in the same database. Existing installations require [offline migration](artifact-processing-migration.md). +The built-in SQLite backend requires Python's linked SQLite library (`sqlite3.sqlite_version`) to be **3.35.0 or newer** +for [materialized retrieval CTEs](https://www.sqlite.org/lang_with.html#materialization_hints). +Older libraries are rejected before database initialization or accepting writes. + ## Server Server settings use the `POWERCONTEXT_SERVER_` prefix. diff --git a/docs/zh/docs/operate/configuration.md b/docs/zh/docs/operate/configuration.md index 0f7080fc8a..990eb0e0b6 100644 --- a/docs/zh/docs/operate/configuration.md +++ b/docs/zh/docs/operate/configuration.md @@ -70,6 +70,10 @@ export POWERCONTEXT_HOME=/srv/powercontext 默认 SQLite 数据库是该目录下的 `powercontext.db`。后台处理器的意图与调度检查点保存在同一数据库中。 已有部署须先完成[停机迁移](artifact-processing-migration.md)。 +内置 SQLite 后端要求 Python 链接的 SQLite 库（`sqlite3.sqlite_version`）为 **3.35.0 或更新版本**， +以支持[检索使用的物化 CTE](https://www.sqlite.org/lang_with.html#materialization_hints)。 +旧版本会在初始化数据库或接受写入之前被拒绝。 + ## Server Server 配置使用 `POWERCONTEXT_SERVER_` 前缀。 diff --git a/src/powercontext/builtin/persistence/sqlite/profile.py b/src/powercontext/builtin/persistence/sqlite/profile.py index 7bb56cb91b..97a7c308a8 100644 --- a/src/powercontext/builtin/persistence/sqlite/profile.py +++ b/src/powercontext/builtin/persistence/sqlite/profile.py @@ -39,6 +39,7 @@ from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.dream_schema import assert_dream_schema_ready +from powercontext.builtin.persistence.errors import PersistenceError from powercontext.builtin.persistence.schema import create_tables _WARMUP_RETRY_SECONDS = 0.05 @@ -98,6 +99,11 @@ async def open( ) -> AsyncIterator[SQLiteProfile]: """Create, initialize and exclusively own one SQLite engine.""" + if sqlite3.sqlite_version_info < (3, 35, 0): + raise PersistenceError( # noqa: TRY003 + "SQLite 3.35.0 or newer is required for retrieval; " + f"this Python uses SQLite {sqlite3.sqlite_version}. Upgrade Python's SQLite library." + ) _create_database_directory(config.url) engine_options: dict[str, object] = {"echo": config.echo, "hide_parameters": True} if not config.is_persistent: diff --git a/src/powercontext/builtin/persistence/sqlite/topic_memory_index.py b/src/powercontext/builtin/persistence/sqlite/topic_memory_index.py index 608f9d52e3..23dd3631f2 100644 --- a/src/powercontext/builtin/persistence/sqlite/topic_memory_index.py +++ b/src/powercontext/builtin/persistence/sqlite/topic_memory_index.py @@ -521,7 +521,6 @@ async def replace( profile_fingerprint=self._fingerprint, ) ) - # SQLite before 3.35 supports lastrowid, but not INSERT ... RETURNING. topic_id = inserted.lastrowid await connection.execute( _INSERT_TOPIC_VECTOR_SQL, diff --git a/tests/builtin/persistence/test_sqlite_profile.py b/tests/builtin/persistence/test_sqlite_profile.py index b7223ed8b8..6f0f30c04b 100644 --- a/tests/builtin/persistence/test_sqlite_profile.py +++ b/tests/builtin/persistence/test_sqlite_profile.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +import sqlite3 from functools import partial from threading import Event as ThreadEvent from typing import cast @@ -30,6 +31,7 @@ from sqlalchemy.pool import QueuePool from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.errors import PersistenceError from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.sqlite import profile as sqlite_profile from powercontext.builtin.persistence.tables import ( @@ -39,6 +41,28 @@ ) +@pytest.mark.parametrize("existing", (False, True)) +def test_unsupported_sqlite_is_rejected_without_changing_storage(tmp_path, monkeypatch, existing) -> None: + database = tmp_path / "legacy.db" + if existing: + with sqlite3.connect(database) as connection: + connection.execute("CREATE TABLE preserved (value TEXT)") + connection.execute("INSERT INTO preserved VALUES ('keep')") + before = database.read_bytes() if existing else None + # Simulate the linked-library version; the test host still runs its installed SQLite. + monkeypatch.setattr(sqlite3, "sqlite_version_info", (3, 34, 1)) + monkeypatch.setattr(sqlite3, "sqlite_version", "3.34.1") + + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{database}") + with pytest.raises(PersistenceError, match=r"SQLite 3\.35\.0.*SQLite 3\.34\.1"): + async with SQLiteProfile.open(config, tables=SHARED_TABLES): + pytest.fail("Unsupported SQLite must not accept database operations") + + asyncio.run(scenario()) + assert (database.read_bytes() if database.exists() else None) == before + + def test_sqlite_stop_waiters_finish_when_another_waiter_is_cancelled(tmp_path) -> None: async def scenario() -> None: config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'stop.db'}") From 4093f5315beb2d427346c00b2d797f6c9fb13e78 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Tue, 20 Oct 2026 20:41:03 +0800 Subject: [PATCH 8/9] fix(sqlite): support retrieval on SQLite 3.38 --- docs/en/docs/operate/configuration.md | 5 +++-- docs/zh/docs/operate/configuration.md | 5 +++-- .../builtin/persistence/atomic_memory_index.py | 4 +++- .../persistence/sqlite/atomic_memory_index.py | 12 ++++++------ .../builtin/persistence/sqlite/profile.py | 4 ++-- .../builtin/persistence/test_sqlite_profile.py | 9 +++++---- tests/e2e/test_atomic_memory_vector_index.py | 17 +++++++++++++++-- 7 files changed, 37 insertions(+), 19 deletions(-) diff --git a/docs/en/docs/operate/configuration.md b/docs/en/docs/operate/configuration.md index dfa1e78a1f..1da699b9b3 100644 --- a/docs/en/docs/operate/configuration.md +++ b/docs/en/docs/operate/configuration.md @@ -79,8 +79,9 @@ Without an override, the default is: The default SQLite database is `powercontext.db` in this directory. Built-in background processors persist intents and scheduling checkpoints in the same database. Existing installations require [offline migration](artifact-processing-migration.md). -The built-in SQLite backend requires Python's linked SQLite library (`sqlite3.sqlite_version`) to be **3.35.0 or newer** -for [materialized retrieval CTEs](https://www.sqlite.org/lang_with.html#materialization_hints). +The built-in SQLite backend requires Python's linked SQLite library (`sqlite3.sqlite_version`) to be **3.38.0 or newer**, +with FTS5, JSON functions, and loadable sqlite-vec for vector search. This retains the backend minimum specified in +[the Memory design](../../rfcs/0014_memory_layer_design.md#sqlite-adapter), including native vector `IN` filtering. Older libraries are rejected before database initialization or accepting writes. ## Server diff --git a/docs/zh/docs/operate/configuration.md b/docs/zh/docs/operate/configuration.md index 990eb0e0b6..fdd98245a4 100644 --- a/docs/zh/docs/operate/configuration.md +++ b/docs/zh/docs/operate/configuration.md @@ -70,8 +70,9 @@ export POWERCONTEXT_HOME=/srv/powercontext 默认 SQLite 数据库是该目录下的 `powercontext.db`。后台处理器的意图与调度检查点保存在同一数据库中。 已有部署须先完成[停机迁移](artifact-processing-migration.md)。 -内置 SQLite 后端要求 Python 链接的 SQLite 库（`sqlite3.sqlite_version`）为 **3.35.0 或更新版本**， -以支持[检索使用的物化 CTE](https://www.sqlite.org/lang_with.html#materialization_hints)。 +内置 SQLite 后端要求 Python 链接的 SQLite 库（`sqlite3.sqlite_version`）为 **3.38.0 或更新版本**， +启用 FTS5、JSON 函数，并为向量检索加载 sqlite-vec。这沿用 +[Memory 设计](../../rfcs/0014_memory_layer_design.md#sqlite-adapter)约定的后端下限，包括原生向量 `IN` 过滤能力。 旧版本会在初始化数据库或接受写入之前被拒绝。 ## Server diff --git a/src/powercontext/builtin/persistence/atomic_memory_index.py b/src/powercontext/builtin/persistence/atomic_memory_index.py index d0661b9add..7cb942c861 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/atomic_memory_index.py @@ -682,7 +682,9 @@ async def probe_recoverable( probe_lower_similarity=floor.min_semantic_similarity, ) distance = "vec_distance_l2" if dialect == "sqlite" else "l2_distance" - similarity = f"(1.0 - POWER({distance}(embedding, :probe_vector), 2) / 2.0)" + distance_sql = f"{distance}(embedding, :probe_vector)" + squared = f"({distance_sql} * {distance_sql})" if dialect == "sqlite" else f"POWER({distance_sql}, 2)" + similarity = f"(1.0 - {squared} / 2.0)" current.append(f"({similarity} >= :probe_similarity)") recoverable.append(f"({similarity} >= :probe_lower_similarity)") if not recoverable: diff --git a/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py b/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py index c3552d31fb..84e0d34a6a 100644 --- a/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py @@ -217,11 +217,10 @@ def _knn_sql(self, request: AtomicMemorySearchRequest | AtomicMemoryRelatedReque f"WHERE scope_id = :scope_id AND ({eligibility}))" ) return ( - f"WITH nearest AS MATERIALIZED (SELECT {_HIT_COLUMNS}, distance FROM {_VECTOR_TABLE} " # noqa: S608 - f"WHERE embedding MATCH :query_vector AND k = :neighbor_limit AND scope_id = :scope_id{filters}) " - f"SELECT {_HIT_COLUMNS}, distance, 0 AS invalid_vectors FROM nearest " - "WHERE max(-1.0, min(1.0, 1.0 - distance * distance / 2.0)) >= :min_semantic_similarity " - "ORDER BY distance, artifact_id" + f"SELECT {_HIT_COLUMNS}, distance, 0 AS invalid_vectors FROM {_VECTOR_TABLE} " # noqa: S608 + f"WHERE embedding MATCH :query_vector AND k = :neighbor_limit AND scope_id = :scope_id{filters} " + "AND max(-1.0, min(1.0, 1.0 - distance * distance / 2.0)) >= :min_semantic_similarity " + "ORDER BY distance" ) async def _initialize_fts(self, connection: AsyncConnection) -> None: @@ -390,7 +389,8 @@ async def _vector_hits( neighbor_limit = min(4096, neighbor_limit * 2) parameters["neighbor_limit"] = neighbor_limit rows = tuple((await connection.execute(text(sql), parameters)).mappings()) - rows = rows[:limit] + # vec0 orders only by distance; apply the ID tie break after collecting boundary ties. + rows = tuple(sorted(rows, key=lambda row: (row["distance"], row["artifact_id"])))[:limit] return atomic_memory_channel_hits(rows, vector=True) diff --git a/src/powercontext/builtin/persistence/sqlite/profile.py b/src/powercontext/builtin/persistence/sqlite/profile.py index 97a7c308a8..1fe25873d5 100644 --- a/src/powercontext/builtin/persistence/sqlite/profile.py +++ b/src/powercontext/builtin/persistence/sqlite/profile.py @@ -99,9 +99,9 @@ async def open( ) -> AsyncIterator[SQLiteProfile]: """Create, initialize and exclusively own one SQLite engine.""" - if sqlite3.sqlite_version_info < (3, 35, 0): + if sqlite3.sqlite_version_info < (3, 38, 0): raise PersistenceError( # noqa: TRY003 - "SQLite 3.35.0 or newer is required for retrieval; " + "SQLite 3.38.0 or newer is required for retrieval; " f"this Python uses SQLite {sqlite3.sqlite_version}. Upgrade Python's SQLite library." ) _create_database_directory(config.url) diff --git a/tests/builtin/persistence/test_sqlite_profile.py b/tests/builtin/persistence/test_sqlite_profile.py index 6f0f30c04b..f042726324 100644 --- a/tests/builtin/persistence/test_sqlite_profile.py +++ b/tests/builtin/persistence/test_sqlite_profile.py @@ -42,7 +42,8 @@ @pytest.mark.parametrize("existing", (False, True)) -def test_unsupported_sqlite_is_rejected_without_changing_storage(tmp_path, monkeypatch, existing) -> None: +@pytest.mark.parametrize("version", ((3, 34, 1), (3, 37, 2))) +def test_unsupported_sqlite_is_rejected_without_changing_storage(tmp_path, monkeypatch, existing, version) -> None: database = tmp_path / "legacy.db" if existing: with sqlite3.connect(database) as connection: @@ -50,12 +51,12 @@ def test_unsupported_sqlite_is_rejected_without_changing_storage(tmp_path, monke connection.execute("INSERT INTO preserved VALUES ('keep')") before = database.read_bytes() if existing else None # Simulate the linked-library version; the test host still runs its installed SQLite. - monkeypatch.setattr(sqlite3, "sqlite_version_info", (3, 34, 1)) - monkeypatch.setattr(sqlite3, "sqlite_version", "3.34.1") + monkeypatch.setattr(sqlite3, "sqlite_version_info", version) + monkeypatch.setattr(sqlite3, "sqlite_version", ".".join(map(str, version))) async def scenario() -> None: config = SQLiteConfig(url=f"sqlite+aiosqlite:///{database}") - with pytest.raises(PersistenceError, match=r"SQLite 3\.35\.0.*SQLite 3\.34\.1"): + with pytest.raises(PersistenceError, match=r"SQLite 3\.38\.0.*this Python uses SQLite"): async with SQLiteProfile.open(config, tables=SHARED_TABLES): pytest.fail("Unsupported SQLite must not accept database operations") diff --git a/tests/e2e/test_atomic_memory_vector_index.py b/tests/e2e/test_atomic_memory_vector_index.py index 43af570d86..8d1c33b269 100644 --- a/tests/e2e/test_atomic_memory_vector_index.py +++ b/tests/e2e/test_atomic_memory_vector_index.py @@ -33,6 +33,7 @@ AtomicMemoryRelatedRequest, AtomicMemorySearchRequest, ) +from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.tags import TagFilter from tests.e2e.atomic_memory_migration_backend import MigrationBackend, migration_index, migration_profile @@ -115,7 +116,9 @@ async def scenario(): connection, "scope", AtomicMemoryProjection( - artifact_ref=ArtifactRef(family="atomic-memory", artifact_id=f"memory-{n:02}", revision=1), + artifact_ref=ArtifactRef( + family="atomic-memory", artifact_id=f"memory-{39 - n:02}", revision=1 + ), state_version=0, prepared=prepared, tag_keys=("keep" if n % 3 else "other",), @@ -132,7 +135,7 @@ async def scenario(): ), ) filters = AtomicMemoryIndexFilter(kind="fact", tag_filter=TagFilter(tags=("keep",), match="all")) - expected = {f"memory-{n:02}" for n in range(40) if n % 2 and n % 3} + expected = {f"memory-{39 - n:02}" for n in range(40) if n % 2 and n % 3} request = AtomicMemorySearchRequest( query="semantic", filters=filters, @@ -150,9 +153,19 @@ async def scenario(): max_distance=0.1, ) async with profile.database.transaction() as connection: + if isinstance(migration_backend.config, SQLiteConfig): + # Supported SQLite builds need not enable optional math functions. + driver = (await connection.get_raw_connection()).driver_connection + assert driver is not None + await driver.create_function("power", 2, None) hits = (await index.search(connection, "scope", request)).vector assert len(hits) == 5 assert {h.artifact_ref.artifact_id for h in hits} <= expected + if isinstance(migration_backend.config, SQLiteConfig): + assert [hit.artifact_ref.artifact_id for hit in hits] == sorted(expected)[:5] + assert not await index.probe_recoverable( + connection, "scope", request, AdmissionFloor(min_semantic_similarity=-1) + ) all_hits = await index.enumerate_related(connection, "scope", related) assert {h.artifact_ref.artifact_id for h in all_hits} == expected expanded = (await index.search(connection, "scope", replace(request, limit=20000))).vector From de99968dc8f0a6e375736ea160238e552a740824 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Tue, 20 Oct 2026 20:41:09 +0800 Subject: [PATCH 9/9] perf(sqlite): bound vector tie fallback queries --- .../persistence/sqlite/atomic_memory_index.py | 16 +++++----------- tests/e2e/test_sqlite_vec.py | 11 ++++++----- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py b/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py index 84e0d34a6a..f11d9298e0 100644 --- a/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py @@ -366,15 +366,14 @@ async def _vector_hits( parameters: dict[str, object], limit: int | None, ) -> tuple[AtomicMemoryIndexHit, ...]: - """Run exact native KNN, expanding boundary ties without scanning current bodies.""" + """Use native KNN, with one exact fallback when boundary ties affect Top-K.""" exact_sql = atomic_memory_vector_sql( eligibility, "sqlite", bounded=limit is not None, threshold=isinstance(request, AtomicMemoryRelatedRequest), ) - # vec0's native k ceiling is 4096. Expand only boundary ties until - # every tied candidate is available for the Artifact-ID tie break. + # vec0's native k ceiling is 4096. One extra neighbor detects boundary ties. knn = isinstance(request, AtomicMemorySearchRequest) and limit is not None and limit < 4096 neighbor_limit = (limit or 0) + 1 if knn: @@ -382,14 +381,9 @@ async def _vector_hits( sql = self._knn_sql(request, eligibility) if knn else exact_sql rows = tuple((await connection.execute(text(sql), parameters)).mappings()) if knn and limit is not None: - while len(rows) == neighbor_limit and rows[limit - 1]["distance"] == rows[-1]["distance"]: - if neighbor_limit == 4096: - rows = tuple((await connection.execute(text(exact_sql), parameters)).mappings()) - break - neighbor_limit = min(4096, neighbor_limit * 2) - parameters["neighbor_limit"] = neighbor_limit - rows = tuple((await connection.execute(text(sql), parameters)).mappings()) - # vec0 orders only by distance; apply the ID tie break after collecting boundary ties. + if len(rows) == neighbor_limit and rows[limit - 1]["distance"] == rows[-1]["distance"]: + # Avoid repeatedly scanning vectors when a large tie group needs the ID tie break. + rows = tuple((await connection.execute(text(exact_sql), parameters)).mappings()) rows = tuple(sorted(rows, key=lambda row: (row["distance"], row["artifact_id"])))[:limit] return atomic_memory_channel_hits(rows, vector=True) diff --git a/tests/e2e/test_sqlite_vec.py b/tests/e2e/test_sqlite_vec.py index 07504399ef..420cb5b45b 100644 --- a/tests/e2e/test_sqlite_vec.py +++ b/tests/e2e/test_sqlite_vec.py @@ -103,7 +103,8 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_sqlite_vector_completeness_uses_a_fixed_sql_budget(tmp_path) -> None: +@pytest.mark.parametrize("size", [3, 100]) +def test_sqlite_vector_completeness_uses_a_fixed_sql_budget(tmp_path, size: int) -> None: async def scenario() -> None: config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'sql-budget.db'}") async with open_builtin_contexts( @@ -112,12 +113,12 @@ async def scenario() -> None: await contexts.get("project") memory = contexts.atomic_memory.for_scope("project") await contexts.records.create_atomic_memories( - "project", tuple({"kind": "fact", "text": f"Alpha record {index}."} for index in range(100)) + "project", tuple({"kind": "fact", "text": f"Alpha record {index}."} for index in range(size)) ) with _sql_counter() as statements: - result = await memory.search("alpha", mode="vector") - assert result.hits - assert len(statements) < 40 + result = await memory.search("alpha", mode="vector", limit=5) + assert len(result.hits) == min(size, 5) + assert len(statements) <= 6 asyncio.run(scenario())