From ac670f8f2707df7eba4ae67e27ec4e005d52cc08 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Tue, 22 Sep 2026 19:36:17 +0800 Subject: [PATCH 01/30] feat(memory): add source evidence declarations and lifecycle projections --- docs/en/docs/operate/troubleshoot.md | 2 +- docs/zh/docs/operate/troubleshoot.md | 2 +- openapi/powercontext.yaml | 24 ++ src/powercontext/__init__.py | 6 + .../builtin/artifacts/memory/__init__.py | 8 + .../builtin/artifacts/memory/models.py | 57 +++- .../builtin/artifacts/memory/protocols.py | 7 + .../builtin/artifacts/memory/service.py | 57 ++++ .../builtin/persistence/memory.py | 293 ++++++++++++++++-- .../builtin/persistence/tables.py | 90 +++++- .../builtin/runtime/relational.py | 8 + src/powercontext/context.py | 1 + src/powercontext/http/__init__.py | 6 + src/powercontext/http/_generated/models.py | 24 ++ src/powercontext/http/_generated/schema.py | 17 + src/powercontext/server/mapping.py | 8 +- src/powercontext/sources/__init__.py | 13 +- src/powercontext/sources/catalog.py | 15 +- src/powercontext/sources/definitions.py | 25 +- src/powercontext/sources/models.py | 39 +++ src/powercontext/sources/observations.py | 12 +- tests/builtin/persistence/test_memory.py | 71 ++++- tests/e2e/real_experience_skill/harness.py | 2 + tests/test_source_observations.py | 84 ++++- 24 files changed, 829 insertions(+), 42 deletions(-) diff --git a/docs/en/docs/operate/troubleshoot.md b/docs/en/docs/operate/troubleshoot.md index 07d931b60d..abf4857c44 100644 --- a/docs/en/docs/operate/troubleshoot.md +++ b/docs/en/docs/operate/troubleshoot.md @@ -285,7 +285,7 @@ so the previous database remains available for recovery: ```bash obloader -D --csv \ - --table 'pc_artifact_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_heads,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ + --table 'pc_artifact_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_evidence,pc_memory_entry_heads,pc_memory_entry_lifecycle_projections,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ -f ``` diff --git a/docs/zh/docs/operate/troubleshoot.md b/docs/zh/docs/operate/troubleshoot.md index 4aba608a5f..1b6a8e5c68 100644 --- a/docs/zh/docs/operate/troubleshoot.md +++ b/docs/zh/docs/operate/troubleshoot.md @@ -276,7 +276,7 @@ collation，但不会包含数据库 URL 或凭据。 ```bash obloader -D --csv \ - --table 'pc_artifact_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_heads,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ + --table 'pc_artifact_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_evidence,pc_memory_entry_heads,pc_memory_entry_lifecycle_projections,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ -f ``` diff --git a/openapi/powercontext.yaml b/openapi/powercontext.yaml index 7186b6ef3a..f63d487329 100644 --- a/openapi/powercontext.yaml +++ b/openapi/powercontext.yaml @@ -6482,6 +6482,26 @@ components: schema: type: object additionalProperties: true + MemoryEvidenceAuthority: + type: string + enum: [untrusted, user_asserted, repository_attested, system_attested] + MemoryEvidenceVerification: + type: string + enum: [unknown, verified, not_verified] + MemoryEvidenceDeclaration: + type: object + additionalProperties: false + required: [authority, verification, declaration_version] + properties: + authority: + $ref: "#/components/schemas/MemoryEvidenceAuthority" + verification: + $ref: "#/components/schemas/MemoryEvidenceVerification" + declaration_version: + type: string + minLength: 1 + maxLength: 128 + pattern: '.*\S.*' SourceDefinitionManifest: type: object additionalProperties: false @@ -6508,6 +6528,8 @@ components: maxItems: 16 items: $ref: "#/components/schemas/SourceProjectionManifest" + memory_evidence: + $ref: "#/components/schemas/MemoryEvidenceDeclaration" RegisterSourceDefinitionRequest: type: object additionalProperties: false @@ -6591,6 +6613,8 @@ components: definition_fingerprint: type: string pattern: '^sha256:[0-9a-f]{64}$' + memory_evidence: + $ref: "#/components/schemas/MemoryEvidenceDeclaration" payload: type: object additionalProperties: true diff --git a/src/powercontext/__init__.py b/src/powercontext/__init__.py index 7448a04133..4dd4515b0a 100644 --- a/src/powercontext/__init__.py +++ b/src/powercontext/__init__.py @@ -51,6 +51,9 @@ ) from powercontext.sources import ( AdapterSourceDefinition, + MemoryEvidenceAuthority, + MemoryEvidenceDeclaration, + MemoryEvidenceVerification, Source, SourceAdapter, SourceCatalog, @@ -90,6 +93,9 @@ "InvalidSourceProjectionError", "InvalidSourceReferenceError", "InvalidSourceResultError", + "MemoryEvidenceAuthority", + "MemoryEvidenceDeclaration", + "MemoryEvidenceVerification", "PolicyTransition", "PowerContext", "PowerContextError", diff --git a/src/powercontext/builtin/artifacts/memory/__init__.py b/src/powercontext/builtin/artifacts/memory/__init__.py index 7b4142f825..b50442c0ef 100644 --- a/src/powercontext/builtin/artifacts/memory/__init__.py +++ b/src/powercontext/builtin/artifacts/memory/__init__.py @@ -53,10 +53,14 @@ MemoryCompactionPolicy, MemoryCompactionResult, MemoryContent, + MemoryContextAnnotation, MemoryEntryInput, MemoryEntryState, MemoryEntryVersion, + MemoryEvidenceSnapshot, MemoryHit, + MemoryLifecycleProjection, + MemoryLifecycleValidity, MemoryManifest, MemoryManifestEntry, MemoryMatchedBy, @@ -140,12 +144,14 @@ "MemoryCompactionPolicy", "MemoryCompactionResult", "MemoryContent", + "MemoryContextAnnotation", "MemoryEntryInactiveError", "MemoryEntryInput", "MemoryEntryNotFoundError", "MemoryEntryState", "MemoryEntryVersion", "MemoryEvidenceProjector", + "MemoryEvidenceSnapshot", "MemoryExtractionCandidate", "MemoryExtractionCurrentEntry", "MemoryExtractionEvidence", @@ -155,6 +161,8 @@ "MemoryExtractionProfile", "MemoryHit", "MemoryLayerError", + "MemoryLifecycleProjection", + "MemoryLifecycleValidity", "MemoryManifest", "MemoryManifestEntry", "MemoryMatchedBy", diff --git a/src/powercontext/builtin/artifacts/memory/models.py b/src/powercontext/builtin/artifacts/memory/models.py index e7b5c553a6..c7a9a8e88a 100644 --- a/src/powercontext/builtin/artifacts/memory/models.py +++ b/src/powercontext/builtin/artifacts/memory/models.py @@ -25,7 +25,7 @@ from powercontext.artifacts import MemoryCitation as MemoryCitation from powercontext.builtin.artifacts.search import AdmissionCounts from powercontext.builtin.inference.models import InferenceUsage -from powercontext.sources import Source, SourceRef +from powercontext.sources import MemoryEvidenceDeclaration, Source, SourceRef MemoryEntryState: TypeAlias = Literal["active", "inactive"] MemoryChangeOp: TypeAlias = Literal["add", "revise", "deactivate", "reactivate", "compact"] @@ -33,6 +33,20 @@ MemorySearchMode: TypeAlias = Literal["fts", "vector", "hybrid", "auto"] MemoryUsedSearchMode: TypeAlias = Literal["fts", "vector", "hybrid"] MemoryMatchedBy: TypeAlias = Literal["fts", "vector"] +MemoryLifecycleValidity: TypeAlias = Literal["current", "inactive", "superseded", "unresolved_conflict"] +MemoryTimeState: TypeAlias = Literal["unknown"] + + +class MemoryEvidenceSnapshot(BaseModel): + """The definition-owned evidence declaration attached to an entry version. + + The snapshot is intentionally independent of the entry content hash. It is + immutable provenance metadata that can be replayed into lifecycle + projections without changing the authoritative Memory body or citation. + """ + + source: SourceRef + declaration: MemoryEvidenceDeclaration class EmbeddingProfile(BaseModel): @@ -184,9 +198,50 @@ class MemoryEntryVersion(BaseModel): entry_content_hash: str created_in_revision: int sources: tuple[SourceRef, ...] = () + source_evidence: tuple[MemoryEvidenceSnapshot, ...] = () artifacts: tuple[ArtifactRef, ...] = () +class MemoryLifecycleProjection(BaseModel): + """Rebuildable neutral lifecycle state for one Memory manifest entry.""" + + memory_ref: ArtifactRef + entry_id: str + entry_version_id: str + validity: MemoryLifecycleValidity + validity_reason: str | None = None + successor_entry_id: str | None = None + source_evidence: tuple[MemoryEvidenceSnapshot, ...] = () + rule_version: str = "memory-lifecycle-v1" + quality_policy: Literal["neutral"] = "neutral" + + @property + def context_annotation(self) -> MemoryContextAnnotation: + """Return the non-public annotation available to Context Pack assembly.""" + + return MemoryContextAnnotation( + validity=self.validity, + validity_reason=self.validity_reason, + successor_entry_id=self.successor_entry_id, + source_evidence=self.source_evidence, + source_count=len(self.source_evidence), + time_state="unknown", + quality_policy=self.quality_policy, + ) + + +class MemoryContextAnnotation(BaseModel): + """Internal-only lifecycle context available to later Context Pack rendering.""" + + validity: MemoryLifecycleValidity + validity_reason: str | None = None + successor_entry_id: str | None = None + source_evidence: tuple[MemoryEvidenceSnapshot, ...] = () + source_count: int = Field(ge=0, default=0) + time_state: MemoryTimeState = "unknown" + quality_policy: Literal["neutral"] = "neutral" + + class MemoryRevisionChanges(BaseModel): """The compact changes stored by one exact Memory Revision.""" diff --git a/src/powercontext/builtin/artifacts/memory/protocols.py b/src/powercontext/builtin/artifacts/memory/protocols.py index 933425d961..c3730270f6 100644 --- a/src/powercontext/builtin/artifacts/memory/protocols.py +++ b/src/powercontext/builtin/artifacts/memory/protocols.py @@ -32,6 +32,7 @@ MemoryEntryInput, MemoryEntryVersion, MemoryHit, + MemoryLifecycleProjection, MemoryRevisionChanges, MemoryUsedSearchMode, ) @@ -55,6 +56,7 @@ class MemoryProjection(BaseModel): searchable_text: str embedding: EmbeddingVector | None = None embedding_content_hash: str | None = None + lifecycle: MemoryLifecycleProjection | None = None class MemoryCommit(BaseModel): @@ -211,6 +213,11 @@ async def projections(self, memory: ArtifactRef, /) -> tuple[MemoryProjection, . ... + async def lifecycle_projections(self, memory: ArtifactRef, /) -> tuple[MemoryLifecycleProjection, ...]: + """Load internal lifecycle metadata for one exact Memory Revision.""" + + ... + async def rebuild_projections(self, embedding_model: EmbeddingModel | None = None, /) -> None: """Rebuild active-head and search projections from authoritative revisions.""" diff --git a/src/powercontext/builtin/artifacts/memory/service.py b/src/powercontext/builtin/artifacts/memory/service.py index c77af62782..a2995015a7 100644 --- a/src/powercontext/builtin/artifacts/memory/service.py +++ b/src/powercontext/builtin/artifacts/memory/service.py @@ -73,6 +73,7 @@ MemoryContent, MemoryEntryInput, MemoryEntryVersion, + MemoryEvidenceSnapshot, MemoryHit, MemoryManifest, MemoryManifestEntry, @@ -111,6 +112,7 @@ from powercontext.errors import RevisionConflictError from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, + MemoryEvidenceDeclaration, Source, SourceObservation, SourceProjectionKey, @@ -135,6 +137,8 @@ def as_ref(self, source: Source, /) -> SourceRef: ... def project(self, source: Source, key: SourceProjectionKey, /) -> object: ... + def memory_evidence(self, source: Source, /) -> MemoryEvidenceDeclaration: ... + class _ArtifactResolver(Protocol): async def get(self, artifact: Artifact[object], /) -> Artifact[object]: ... @@ -153,6 +157,7 @@ class _EntryMaterial: kind: str text: str sources: tuple[SourceRef, ...] + source_evidence: tuple[MemoryEvidenceSnapshot, ...] artifacts: tuple[ArtifactRef, ...] content_bytes: bytes content_hash: str @@ -1641,6 +1646,7 @@ async def _material_from_candidate( previous_artifacts = () if previous is None else previous.artifacts candidate_sources = await self._canonical_candidate_sources(candidate.sources, allowed_sources) candidate_source_refs = self._source_refs(candidate_sources) + candidate_source_evidence = self._source_evidence(candidate_sources) candidate_artifacts = await self._canonical_candidate_artifacts( candidate.artifacts, allowed_artifacts, @@ -1648,16 +1654,19 @@ async def _material_from_candidate( ) if previous is None: sources = candidate_source_refs + source_evidence = candidate_source_evidence artifacts = candidate_artifacts else: # Revises retain predecessor evidence and add current candidate evidence. sources = (*previous.sources, *candidate_source_refs) + source_evidence = (*previous.source_evidence, *candidate_source_evidence) artifacts = (*previous.artifacts, *candidate_artifacts) try: return self._entry_material( kind=candidate.kind, text=candidate.text, sources=sources, + source_evidence=source_evidence, artifacts=artifacts, ) except (TypeError, ValueError) as error: @@ -1705,6 +1714,7 @@ def _material_from_version(self, version: MemoryEntryVersion) -> _EntryMaterial: kind=version.kind, text=version.text, sources=version.sources, + source_evidence=version.source_evidence, artifacts=version.artifacts, ) @@ -1714,11 +1724,13 @@ def _entry_material( kind: str, text: str, sources: Sequence[SourceRef], + source_evidence: Sequence[MemoryEvidenceSnapshot], artifacts: Sequence[ArtifactRef], ) -> _EntryMaterial: normalized_kind = normalize_kind(kind) normalized_text = normalize_text(text) canonical_sources = _canonical_source_refs(sources) + canonical_evidence = _canonical_source_evidence(canonical_sources, source_evidence) canonical_artifacts = _canonical_artifact_refs(artifacts) source_refs = tuple(source.model_dump(mode="json") for source in canonical_sources) artifact_refs = tuple(artifact.model_dump(mode="json") for artifact in canonical_artifacts) @@ -1732,6 +1744,7 @@ def _entry_material( kind=normalized_kind, text=normalized_text, sources=canonical_sources, + source_evidence=canonical_evidence, artifacts=canonical_artifacts, content_bytes=content_bytes, content_hash=entry_content_hash( @@ -1756,6 +1769,25 @@ def _source_refs(self, sources: Sequence[Source]) -> tuple[SourceRef, ...]: keyed.setdefault(canonical_json(reference.model_dump(mode="json")), reference) return tuple(keyed[key] for key in sorted(keyed)) + def _source_evidence(self, sources: Sequence[Source]) -> tuple[MemoryEvidenceSnapshot, ...]: + """Snapshot Definition-owned declarations without changing content identity.""" + + if not sources: + return () + if self._source_resolver is None: + raise InvalidMemoryEvidenceError("source-adapter") + resolver = getattr(self._source_resolver, "memory_evidence", None) + keyed: dict[bytes, MemoryEvidenceSnapshot] = {} + for source in sources: + try: + reference = self._source_resolver.as_ref(source) + declaration = MemoryEvidenceDeclaration() if resolver is None else resolver(source) + snapshot = MemoryEvidenceSnapshot(source=reference, declaration=declaration) + except (KeyError, LookupError, TypeError, ValueError): + raise InvalidMemoryEvidenceError("source-adapter") from None + keyed.setdefault(canonical_json(reference.model_dump(mode="json")), snapshot) + return tuple(keyed[key] for key in sorted(keyed)) + def _new_entry_version( self, *, @@ -1774,6 +1806,7 @@ def _new_entry_version( kind=material.kind, text=material.text, sources=material.sources, + source_evidence=material.source_evidence, artifacts=material.artifacts, entry_content_hash=material.content_hash, created_in_revision=created_in_revision, @@ -1913,6 +1946,30 @@ def _canonical_source_refs(values: Sequence[SourceRef]) -> tuple[SourceRef, ...] return tuple(keyed[key] for key in sorted(keyed)) +def _canonical_source_evidence( + sources: Sequence[SourceRef], + values: Sequence[MemoryEvidenceSnapshot], +) -> tuple[MemoryEvidenceSnapshot, ...]: + """Align provenance snapshots with canonical source identity. + + Entries written before the lifecycle projection existed have no snapshots. + They deliberately derive the neutral declaration instead of acquiring a + trust claim from a current adapter configuration. + """ + + declared = {canonical_json(value.source.model_dump(mode="json")): value for value in values} + canonical: list[MemoryEvidenceSnapshot] = [] + for source in sources: + key = canonical_json(source.model_dump(mode="json")) + canonical.append( + declared.get( + key, + MemoryEvidenceSnapshot(source=source, declaration=MemoryEvidenceDeclaration()), + ) + ) + return tuple(canonical) + + def _canonical_artifact_refs(values: Sequence[ArtifactRef]) -> tuple[ArtifactRef, ...]: keyed = {canonical_json(value.model_dump(mode="json")): value for value in values} return tuple(keyed[key] for key in sorted(keyed)) diff --git a/src/powercontext/builtin/persistence/memory.py b/src/powercontext/builtin/persistence/memory.py index 8ce878cc09..e65a8e488f 100644 --- a/src/powercontext/builtin/persistence/memory.py +++ b/src/powercontext/builtin/persistence/memory.py @@ -19,10 +19,10 @@ from collections.abc import AsyncIterator, Mapping from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass, replace -from typing import Any, ClassVar +from typing import Any, ClassVar, Literal, cast from pydantic import RootModel -from sqlalchemy import delete, insert, select +from sqlalchemy import delete, insert, select, tuple_ from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.artifacts import ArtifactDraft, ArtifactRef @@ -34,7 +34,10 @@ MemoryCommit, MemoryContent, MemoryEntryVersion, + MemoryEvidenceSnapshot, MemoryHit, + MemoryLifecycleProjection, + MemoryLifecycleValidity, MemoryProjection, MemoryRevisionChanges, MemorySearchChannels, @@ -62,13 +65,20 @@ from powercontext.builtin.persistence.tables import ( ARTIFACT_HEADS_TABLE, ARTIFACT_TAGS_TABLE, + MEMORY_ENTRY_EVIDENCE_TABLE, MEMORY_ENTRY_HEADS_TABLE, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE, MEMORY_ENTRY_VERSIONS_TABLE, ) from powercontext.builtin.persistence.tags import tag_predicate from powercontext.builtin.tags import TagFilter from powercontext.errors import ArtifactNotFoundError -from powercontext.sources import SourceRef +from powercontext.sources import ( + MemoryEvidenceAuthority, + MemoryEvidenceDeclaration, + MemoryEvidenceVerification, + SourceRef, +) class _SourceRefs(RootModel[tuple[SourceRef, ...]]): @@ -87,6 +97,7 @@ class _MemoryDraft(ArtifactDraft[MemoryContent]): class _RebuildSnapshot: memory_ref: ArtifactRef projections: tuple[MemoryProjection, ...] + lifecycle: tuple[MemoryLifecycleProjection, ...] class _InvalidMemoryCommitError(MemoryBackendConfigurationError): @@ -212,42 +223,85 @@ async def entries(self, memory: ArtifactRef, /) -> tuple[MemoryEntryVersion, ... ) ) ).mappings() - by_id = {str(row["entry_version_id"]): _decode_entry(row) for row in rows} + versions = await self._hydrate_source_evidence(connection, tuple(_decode_entry(row) for row in rows)) + by_id = {version.entry_version_id: version for version in versions} if set(by_id) != set(version_ids): raise InvalidMemoryCitationError("missing-version") return tuple(by_id[version_id] for version_id in version_ids) - async def projections(self, memory: ArtifactRef, /) -> tuple[MemoryProjection, ...]: + async def lifecycle_projections(self, memory: ArtifactRef, /) -> tuple[MemoryLifecycleProjection, ...]: + """Load the internal neutral lifecycle projection for one exact Memory head.""" + canonical = await self.get(memory) + version_ids = tuple(item.entry_version_id for item in canonical.content.manifest.entries) + if not version_ids: + return () async with self._database.connection(self._bound_connection) as connection: - rows = ( + entry_rows = ( await connection.execute( - select(MEMORY_ENTRY_HEADS_TABLE, MEMORY_ENTRY_VERSIONS_TABLE) - .join( - MEMORY_ENTRY_VERSIONS_TABLE, - (MEMORY_ENTRY_VERSIONS_TABLE.c.scope_id == MEMORY_ENTRY_HEADS_TABLE.c.scope_id) - & ( - MEMORY_ENTRY_VERSIONS_TABLE.c.memory_artifact_id - == MEMORY_ENTRY_HEADS_TABLE.c.memory_artifact_id - ) - & ( - MEMORY_ENTRY_VERSIONS_TABLE.c.entry_version_id - == MEMORY_ENTRY_HEADS_TABLE.c.entry_version_id - ), + select(MEMORY_ENTRY_VERSIONS_TABLE).where( + MEMORY_ENTRY_VERSIONS_TABLE.c.scope_id == self._scope_id, + MEMORY_ENTRY_VERSIONS_TABLE.c.memory_artifact_id == memory.artifact_id, + MEMORY_ENTRY_VERSIONS_TABLE.c.entry_version_id.in_(version_ids), ) + ) + ).mappings() + versions = await self._hydrate_source_evidence(connection, tuple(_decode_entry(row) for row in entry_rows)) + by_version = {entry.entry_version_id: entry for entry in versions} + rows = ( + await connection.execute( + select(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE) .where( - MEMORY_ENTRY_HEADS_TABLE.c.scope_id == self._scope_id, - MEMORY_ENTRY_HEADS_TABLE.c.memory_artifact_id == memory.artifact_id, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.scope_id == self._scope_id, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.memory_artifact_id == memory.artifact_id, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.head_revision == memory.revision, ) - .order_by(MEMORY_ENTRY_HEADS_TABLE.c.entry_id) + .order_by(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.entry_id) ) ).mappings() + projections = tuple( + _decode_lifecycle_projection(memory, row, by_version.get(str(row["entry_version_id"]))) for row in rows + ) + if {item.entry_id for item in projections} != {item.entry_id for item in canonical.content.manifest.entries}: + raise InvalidMemoryCitationError("lifecycle-projection") + return projections + + async def projections(self, memory: ArtifactRef, /) -> tuple[MemoryProjection, ...]: + canonical = await self.get(memory) + async with self._database.connection(self._bound_connection) as connection: + rows = ( + ( + await connection.execute( + select(MEMORY_ENTRY_HEADS_TABLE, MEMORY_ENTRY_VERSIONS_TABLE) + .join( + MEMORY_ENTRY_VERSIONS_TABLE, + (MEMORY_ENTRY_VERSIONS_TABLE.c.scope_id == MEMORY_ENTRY_HEADS_TABLE.c.scope_id) + & ( + MEMORY_ENTRY_VERSIONS_TABLE.c.memory_artifact_id + == MEMORY_ENTRY_HEADS_TABLE.c.memory_artifact_id + ) + & ( + MEMORY_ENTRY_VERSIONS_TABLE.c.entry_version_id + == MEMORY_ENTRY_HEADS_TABLE.c.entry_version_id + ), + ) + .where( + MEMORY_ENTRY_HEADS_TABLE.c.scope_id == self._scope_id, + MEMORY_ENTRY_HEADS_TABLE.c.memory_artifact_id == memory.artifact_id, + ) + .order_by(MEMORY_ENTRY_HEADS_TABLE.c.entry_id) + ) + ) + .mappings() + .all() + ) + entries = await self._hydrate_source_evidence(connection, tuple(_decode_entry(row) for row in rows)) projections = tuple( MemoryProjection( - entry_version=_decode_entry(row), + entry_version=entry, searchable_text=str(row["searchable_text"]), ) - for row in rows + for row, entry in zip(rows, entries, strict=True) ) projections = await self._index.hydrate(connection, self._scope_id, projections) active_ids = {item.entry_version_id for item in canonical.content.manifest.entries if item.state == "active"} @@ -283,6 +337,11 @@ async def rebuild_projections(self, embedding_model: EmbeddingModel | None = Non await connection.execute( delete(MEMORY_ENTRY_HEADS_TABLE).where(MEMORY_ENTRY_HEADS_TABLE.c.scope_id == self._scope_id) ) + await connection.execute( + delete(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE).where( + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.scope_id == self._scope_id + ) + ) for snapshot in rebuilt: if snapshot.projections: await connection.execute( @@ -292,6 +351,11 @@ async def rebuild_projections(self, embedding_model: EmbeddingModel | None = Non for projection in snapshot.projections ], ) + if snapshot.lifecycle: + await connection.execute( + insert(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE), + [_lifecycle_values(self._scope_id, lifecycle) for lifecycle in snapshot.lifecycle], + ) await self._index.replace( connection, self._scope_id, @@ -379,7 +443,87 @@ async def expand(self, hits: tuple[MemoryHit, ...], /) -> tuple[MemoryEntryVersi if row is None: raise InvalidMemoryCitationError("expand-anchor") expanded.append(_decode_entry(row)) - return tuple(expanded) + return await self._hydrate_source_evidence(connection, tuple(expanded)) + + async def _hydrate_source_evidence( + self, + connection: AsyncConnection, + entries: tuple[MemoryEntryVersion, ...], + ) -> tuple[MemoryEntryVersion, ...]: + if not entries: + return () + identities = tuple((entry.memory_artifact_id, entry.entry_id, entry.entry_version_id) for entry in entries) + rows = ( + await connection.execute( + select(MEMORY_ENTRY_EVIDENCE_TABLE) + .where( + MEMORY_ENTRY_EVIDENCE_TABLE.c.scope_id == self._scope_id, + tuple_( + MEMORY_ENTRY_EVIDENCE_TABLE.c.memory_artifact_id, + MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_id, + MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_version_id, + ).in_(identities), + ) + .order_by( + MEMORY_ENTRY_EVIDENCE_TABLE.c.memory_artifact_id, + MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_id, + MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_version_id, + MEMORY_ENTRY_EVIDENCE_TABLE.c.ordinal, + ) + ) + ).mappings() + snapshots: dict[tuple[str, str, str], list[MemoryEvidenceSnapshot]] = {} + for row in rows: + identity = (str(row["memory_artifact_id"]), str(row["entry_id"]), str(row["entry_version_id"])) + snapshots.setdefault(identity, []).append( + MemoryEvidenceSnapshot( + source=SourceRef(source_type=str(row["source_type"]), source_id=str(row["source_id"])), + declaration=MemoryEvidenceDeclaration( + authority=MemoryEvidenceAuthority(str(row["authority"])), + verification=MemoryEvidenceVerification(str(row["verification"])), + declaration_version=str(row["declaration_version"]), + ), + ) + ) + hydrated: list[MemoryEntryVersion] = [] + for entry in entries: + identity = (entry.memory_artifact_id, entry.entry_id, entry.entry_version_id) + evidence = tuple(snapshots.get(identity, ())) + if evidence and tuple(snapshot.source for snapshot in evidence) != entry.sources: + raise InvalidMemoryCitationError("evidence-source") + hydrated.append( + entry.model_copy(update={"source_evidence": evidence or _neutral_source_evidence(entry.sources)}) + ) + return tuple(hydrated) + + async def _lifecycle_for_memory( + self, + connection: AsyncConnection, + memory: Memory, + ) -> tuple[MemoryLifecycleProjection, ...]: + manifest = memory.content.manifest.entries + if not manifest: + return () + version_ids = tuple(item.entry_version_id for item in manifest) + rows = ( + await connection.execute( + select(MEMORY_ENTRY_VERSIONS_TABLE).where( + MEMORY_ENTRY_VERSIONS_TABLE.c.scope_id == self._scope_id, + MEMORY_ENTRY_VERSIONS_TABLE.c.memory_artifact_id == memory.artifact_id, + MEMORY_ENTRY_VERSIONS_TABLE.c.entry_version_id.in_(version_ids), + ) + ) + ).mappings() + versions = await self._hydrate_source_evidence(connection, tuple(_decode_entry(row) for row in rows)) + by_version = {entry.entry_version_id: entry for entry in versions} + lifecycle: list[MemoryLifecycleProjection] = [] + for item in manifest: + entry = by_version.get(item.entry_version_id) + if entry is None: + raise InvalidMemoryCitationError("missing-version") + _validate_rebuild_entry(memory.as_ref(), item.entry_id, item.entry_content_hash, entry) + lifecycle.append(_lifecycle_projection(memory.as_ref(), item.state, entry)) + return tuple(lifecycle) async def _authoritative_projections( self, @@ -406,9 +550,10 @@ async def _authoritative_projections( revision=int(revision), ) memory = _require_memory(await self._artifacts.get(connection, self._scope_id, ref)) + lifecycle = await self._lifecycle_for_memory(connection, memory) active = tuple(item for item in memory.content.manifest.entries if item.state == "active") if not active: - snapshots.append(_RebuildSnapshot(memory_ref=ref, projections=())) + snapshots.append(_RebuildSnapshot(memory_ref=ref, projections=(), lifecycle=lifecycle)) continue version_ids = tuple(item.entry_version_id for item in active) rows = ( @@ -420,7 +565,8 @@ async def _authoritative_projections( ) ) ).mappings() - versions = {str(row["entry_version_id"]): _decode_entry(row) for row in rows} + hydrated = await self._hydrate_source_evidence(connection, tuple(_decode_entry(row) for row in rows)) + versions = {entry.entry_version_id: entry for entry in hydrated} projections: list[MemoryProjection] = [] for item in active: version = versions.get(item.entry_version_id) @@ -437,6 +583,7 @@ async def _authoritative_projections( _RebuildSnapshot( memory_ref=ref, projections=tuple(projections), + lifecycle=lifecycle, ) ) return tuple(snapshots) @@ -522,6 +669,9 @@ async def _commit(self, connection: AsyncConnection, value: MemoryCommit) -> Mem insert(MEMORY_ENTRY_VERSIONS_TABLE), [_entry_values(self._scope_id, entry) for entry in value.entry_versions], ) + evidence_rows = [row for entry in value.entry_versions for row in _evidence_values(self._scope_id, entry)] + if evidence_rows: + await connection.execute(insert(MEMORY_ENTRY_EVIDENCE_TABLE), evidence_rows) # Only entries whose pointer or state changed need projection work; the # rest of the active head stays exactly as the previous revision left it. previous_active = ( @@ -572,6 +722,20 @@ async def _commit(self, connection: AsyncConnection, value: MemoryCommit) -> Mem value.memory.as_ref(), upserts, ) + # Lifecycle rows are keyed by entry, not by revision, so the artifact's rows + # are replaced wholesale: lifecycle_projections reads them back by revision. + await connection.execute( + delete(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE).where( + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.scope_id == self._scope_id, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.memory_artifact_id == value.memory.artifact_id, + ) + ) + lifecycle = await self._lifecycle_for_memory(connection, committed) + if lifecycle: + await connection.execute( + insert(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE), + [_lifecycle_values(self._scope_id, projection) for projection in lifecycle], + ) return committed @@ -639,6 +803,24 @@ def _entry_values(scope_id: str, value: MemoryEntryVersion) -> dict[str, object] } +def _evidence_values(scope_id: str, value: MemoryEntryVersion) -> tuple[dict[str, object], ...]: + return tuple( + { + "scope_id": scope_id, + "memory_artifact_id": value.memory_artifact_id, + "entry_id": value.entry_id, + "entry_version_id": value.entry_version_id, + "ordinal": ordinal, + "source_type": snapshot.source.source_type, + "source_id": snapshot.source.source_id, + "authority": snapshot.declaration.authority.value, + "verification": snapshot.declaration.verification.value, + "declaration_version": snapshot.declaration.declaration_version, + } + for ordinal, snapshot in enumerate(value.source_evidence) + ) + + def _projection_values( scope_id: str, memory_ref: ArtifactRef, @@ -657,6 +839,23 @@ def _projection_values( } +def _lifecycle_values(scope_id: str, value: MemoryLifecycleProjection) -> dict[str, object]: + return { + "scope_id": scope_id, + "family": Memory.family, + "memory_artifact_id": value.memory_ref.artifact_id, + "head_revision": value.memory_ref.revision, + "entry_id": value.entry_id, + "entry_version_id": value.entry_version_id, + "validity": value.validity, + "validity_reason": value.validity_reason, + "successor_entry_id": value.successor_entry_id, + "source_count": len(value.source_evidence), + "rule_version": value.rule_version, + "quality_policy": value.quality_policy, + } + + def _decode_entry(row: Mapping[Any, Any]) -> MemoryEntryVersion: entry_version_id = str(row["entry_version_id"]) return MemoryEntryVersion( @@ -684,6 +883,48 @@ def _decode_entry(row: Mapping[Any, Any]) -> MemoryEntryVersion: ) +def _neutral_source_evidence(sources: tuple[SourceRef, ...]) -> tuple[MemoryEvidenceSnapshot, ...]: + return tuple(MemoryEvidenceSnapshot(source=source, declaration=MemoryEvidenceDeclaration()) for source in sources) + + +def _lifecycle_projection( + memory_ref: ArtifactRef, + state: str, + entry: MemoryEntryVersion, +) -> MemoryLifecycleProjection: + validity = "current" if state == "active" else "inactive" + return MemoryLifecycleProjection( + memory_ref=memory_ref, + entry_id=entry.entry_id, + entry_version_id=entry.entry_version_id, + validity=validity, + source_evidence=entry.source_evidence, + ) + + +def _decode_lifecycle_projection( + memory_ref: ArtifactRef, + row: Mapping[Any, Any], + entry: MemoryEntryVersion | None, +) -> MemoryLifecycleProjection: + if entry is None or entry.entry_id != str(row["entry_id"]): + raise InvalidMemoryCitationError("lifecycle-version") + projection = MemoryLifecycleProjection( + memory_ref=memory_ref, + entry_id=str(row["entry_id"]), + entry_version_id=str(row["entry_version_id"]), + validity=cast(MemoryLifecycleValidity, str(row["validity"])), + validity_reason=(None if row["validity_reason"] is None else str(row["validity_reason"])), + successor_entry_id=(None if row["successor_entry_id"] is None else str(row["successor_entry_id"])), + source_evidence=entry.source_evidence, + rule_version=str(row["rule_version"]), + quality_policy=cast(Literal["neutral"], str(row["quality_policy"])), + ) + if int(row["source_count"]) != len(projection.source_evidence): + raise InvalidMemoryCitationError("lifecycle-source-count") + return projection + + def _validate_rebuild_entry( memory_ref: ArtifactRef, entry_id: str, diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index 530d3f13de..b4fc0e125b 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -978,6 +978,45 @@ def _entry_text_type(): ), ) +MEMORY_ENTRY_EVIDENCE_TABLE = Table( + "pc_memory_entry_evidence", + SHARED_METADATA, + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("memory_artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), + Column("entry_id", identity_string(MAX_MEMORY_ENTRY_ID_LENGTH), primary_key=True), + Column("entry_version_id", identity_string(MAX_MEMORY_ENTRY_ID_LENGTH), primary_key=True), + Column("ordinal", Integer, primary_key=True), + Column("source_type", identity_string(MAX_SOURCE_TYPE_LENGTH), nullable=False), + Column("source_id", identity_string(MAX_SOURCE_ID_LENGTH), nullable=False), + Column("authority", identity_string(32), nullable=False), + Column("verification", identity_string(32), nullable=False), + Column("declaration_version", identity_string(MAX_SOURCE_TYPE_LENGTH), nullable=False), + ForeignKeyConstraint( + ("scope_id", "memory_artifact_id", "entry_id", "entry_version_id"), + ( + "pc_memory_entry_versions.scope_id", + "pc_memory_entry_versions.memory_artifact_id", + "pc_memory_entry_versions.entry_id", + "pc_memory_entry_versions.entry_version_id", + ), + ondelete="CASCADE", + ), + ForeignKeyConstraint( + ("scope_id", "source_type", "source_id"), + ("pc_sources.scope_id", "pc_sources.source_type", "pc_sources.source_id"), + ondelete="RESTRICT", + ), + CheckConstraint("ordinal >= 0", name="ck_pc_memory_entry_evidence_ordinal_nonnegative"), + CheckConstraint( + "authority IN ('untrusted', 'user_asserted', 'repository_attested', 'system_attested')", + name="ck_pc_memory_entry_evidence_authority", + ), + CheckConstraint( + "verification IN ('unknown', 'verified', 'not_verified')", + name="ck_pc_memory_entry_evidence_verification", + ), +) + MEMORY_ENTRY_HEADS_TABLE = Table( "pc_memory_entry_heads", SHARED_METADATA, @@ -1012,8 +1051,57 @@ def _entry_text_type(): CheckConstraint("head_revision > 0", name="ck_pc_memory_entry_heads_revision_positive"), ) +MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE = Table( + "pc_memory_entry_lifecycle_projections", + SHARED_METADATA, + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("family", identity_string(MAX_ARTIFACT_FAMILY_LENGTH), nullable=False), + Column("memory_artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), + Column("head_revision", Integer, nullable=False), + Column("entry_id", identity_string(MAX_MEMORY_ENTRY_ID_LENGTH), primary_key=True), + Column("entry_version_id", identity_string(MAX_MEMORY_ENTRY_ID_LENGTH), nullable=False), + Column("validity", identity_string(32), nullable=False), + Column("validity_reason", identity_string(256)), + Column("successor_entry_id", identity_string(MAX_MEMORY_ENTRY_ID_LENGTH)), + Column("source_count", Integer, nullable=False), + Column("rule_version", identity_string(64), nullable=False), + Column("quality_policy", identity_string(16), nullable=False), + ForeignKeyConstraint( + ("scope_id", "family", "memory_artifact_id", "head_revision"), + ( + "pc_artifacts.scope_id", + "pc_artifacts.family", + "pc_artifacts.artifact_id", + "pc_artifacts.revision", + ), + ondelete="RESTRICT", + ), + ForeignKeyConstraint( + ("scope_id", "memory_artifact_id", "entry_id", "entry_version_id"), + ( + "pc_memory_entry_versions.scope_id", + "pc_memory_entry_versions.memory_artifact_id", + "pc_memory_entry_versions.entry_id", + "pc_memory_entry_versions.entry_version_id", + ), + ondelete="RESTRICT", + ), + CheckConstraint("head_revision > 0", name="ck_pc_memory_lifecycle_projection_revision_positive"), + CheckConstraint("source_count >= 0", name="ck_pc_memory_lifecycle_projection_sources_nonnegative"), + CheckConstraint( + "validity IN ('current', 'inactive', 'superseded', 'unresolved_conflict')", + name="ck_pc_memory_lifecycle_projection_validity", + ), + CheckConstraint("quality_policy = 'neutral'", name="ck_pc_memory_lifecycle_projection_quality_neutral"), +) + -MEMORY_TABLES = (MEMORY_ENTRY_VERSIONS_TABLE, MEMORY_ENTRY_HEADS_TABLE) +MEMORY_TABLES = ( + MEMORY_ENTRY_VERSIONS_TABLE, + MEMORY_ENTRY_EVIDENCE_TABLE, + MEMORY_ENTRY_HEADS_TABLE, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE, +) TOPIC_MEMORY_TABLES = ( TOPIC_MEMORY_RETRIEVAL_SHAPE_TABLE, diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 90e0b8db9a..223a89e1f8 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -231,6 +231,7 @@ from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, ConnectorBinding, + MemoryEvidenceDeclaration, Source, SourceCatalog, SourceDefinitionManifest, @@ -1486,6 +1487,11 @@ async def get_ref(self, ref: SourceRef, /) -> Source: raise SourceNotFoundError(ref) from None return stored.value + def memory_evidence(self, source: Source, /) -> MemoryEvidenceDeclaration: + """Resolve Definition-owned evidence metadata for a stored Source.""" + + return self._catalog.memory_evidence(source) + async def get(self, source: Source, /) -> Source: try: async with self._database.connection(self._connection) as connection: @@ -1987,6 +1993,8 @@ def _validate_source_observation(source: SourceObservation, manifest: SourceDefi raise InvalidSourceObservationError("definition", "does not match the registered manifest identity") if source.definition_fingerprint != manifest.fingerprint: raise InvalidSourceObservationError("fingerprint", "does not match the registered manifest") + if "memory_evidence" in source.__pydantic_fields_set__ and source.memory_evidence != manifest.memory_evidence: + raise InvalidSourceObservationError("memory_evidence", "does not match the registered manifest") if len(source.model_dump_json().encode()) > MAX_SOURCE_OBSERVATION_BYTES: raise InvalidSourceObservationError("size", "must not exceed 4 MiB") _validate_schema_value(manifest.name, manifest.source_schema, source.payload) diff --git a/src/powercontext/context.py b/src/powercontext/context.py index cab0aa20aa..b7f3988f75 100644 --- a/src/powercontext/context.py +++ b/src/powercontext/context.py @@ -55,6 +55,7 @@ async def resolve(self, value: object, /) -> Source: async def add(self, source: Source, /) -> Source: """Persist a resolved Source without deriving Artifacts.""" + source = self.catalog.materialize(source) self.catalog.as_ref(source) stored = await self.store.add(source) self.catalog.as_ref(stored) diff --git a/src/powercontext/http/__init__.py b/src/powercontext/http/__init__.py index 302628a54f..36f1d60597 100644 --- a/src/powercontext/http/__init__.py +++ b/src/powercontext/http/__init__.py @@ -236,6 +236,9 @@ MemoryEntryInventoryStatistics, MemoryEntryState, MemoryEntryTagTarget, + MemoryEvidenceAuthority, + MemoryEvidenceDeclaration, + MemoryEvidenceVerification, MemoryInventoryStatistics, MemoryKindCount, MemoryMatchedBy, @@ -634,6 +637,9 @@ "MemoryEntryInventoryStatistics", "MemoryEntryState", "MemoryEntryTagTarget", + "MemoryEvidenceAuthority", + "MemoryEvidenceDeclaration", + "MemoryEvidenceVerification", "MemoryInventoryStatistics", "MemoryKindCount", "MemoryMatchedBy", diff --git a/src/powercontext/http/_generated/models.py b/src/powercontext/http/_generated/models.py index 4ff5c26339..0dabeb2931 100644 --- a/src/powercontext/http/_generated/models.py +++ b/src/powercontext/http/_generated/models.py @@ -723,6 +723,28 @@ class SourceProjectionManifest(BaseModel): schema_: Annotated[dict[str, Any], Field(alias="schema")] +class MemoryEvidenceAuthority(StrEnum): + UNTRUSTED = "untrusted" + USER_ASSERTED = "user_asserted" + REPOSITORY_ATTESTED = "repository_attested" + SYSTEM_ATTESTED = "system_attested" + + +class MemoryEvidenceVerification(StrEnum): + UNKNOWN = "unknown" + VERIFIED = "verified" + NOT_VERIFIED = "not_verified" + + +class MemoryEvidenceDeclaration(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + authority: MemoryEvidenceAuthority + verification: MemoryEvidenceVerification + declaration_version: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern=".*\\S.*")] + + class SourceDefinitionManifest(BaseModel): model_config = ConfigDict( extra="forbid", @@ -732,6 +754,7 @@ class SourceDefinitionManifest(BaseModel): fingerprint: Annotated[StrictStr, Field(pattern="^sha256:[0-9a-f]{64}$")] source_schema: dict[str, Any] projections: Annotated[list[SourceProjectionManifest], Field(max_length=16)] + memory_evidence: MemoryEvidenceDeclaration | None = None class RegisterSourceDefinitionRequest(BaseModel): @@ -788,6 +811,7 @@ class SourceObservation(BaseModel): description: StrictStr | None = None source_type: Annotated[StrictStr, Field(max_length=128, min_length=1)] definition_fingerprint: Annotated[StrictStr, Field(pattern="^sha256:[0-9a-f]{64}$")] + memory_evidence: MemoryEvidenceDeclaration | None = None payload: dict[str, Any] projections: Annotated[list[SourceProjectionValue], Field(max_length=16)] diff --git a/src/powercontext/http/_generated/schema.py b/src/powercontext/http/_generated/schema.py index c91522b25d..5fcbd79f01 100644 --- a/src/powercontext/http/_generated/schema.py +++ b/src/powercontext/http/_generated/schema.py @@ -6103,6 +6103,21 @@ "type": "object", "required": ["key", "schema"], }, + "MemoryEvidenceAuthority": { + "type": "string", + "enum": ["untrusted", "user_asserted", "repository_attested", "system_attested"], + }, + "MemoryEvidenceVerification": {"type": "string", "enum": ["unknown", "verified", "not_verified"]}, + "MemoryEvidenceDeclaration": { + "properties": { + "authority": {"$ref": "#/components/schemas/MemoryEvidenceAuthority"}, + "verification": {"$ref": "#/components/schemas/MemoryEvidenceVerification"}, + "declaration_version": {"type": "string", "maxLength": 128, "minLength": 1, "pattern": ".*\\S.*"}, + }, + "additionalProperties": False, + "type": "object", + "required": ["authority", "verification", "declaration_version"], + }, "SourceDefinitionManifest": { "properties": { "name": {"type": "string", "maxLength": 128, "minLength": 1, "pattern": ".*\\S.*"}, @@ -6114,6 +6129,7 @@ "type": "array", "maxItems": 16, }, + "memory_evidence": {"$ref": "#/components/schemas/MemoryEvidenceDeclaration"}, }, "additionalProperties": False, "type": "object", @@ -6165,6 +6181,7 @@ "description": {"type": "string", "nullable": True}, "source_type": {"type": "string", "maxLength": 128, "minLength": 1}, "definition_fingerprint": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "memory_evidence": {"$ref": "#/components/schemas/MemoryEvidenceDeclaration"}, "payload": {"additionalProperties": True, "type": "object"}, "projections": { "items": {"$ref": "#/components/schemas/SourceProjectionValue"}, diff --git a/src/powercontext/server/mapping.py b/src/powercontext/server/mapping.py index bd13026be5..39368df8c1 100644 --- a/src/powercontext/server/mapping.py +++ b/src/powercontext/server/mapping.py @@ -495,7 +495,9 @@ def capture_response(value: SourceReceipt) -> CaptureContentSourceResponse: def runtime_source_definition_manifest(value: SourceDefinitionManifest) -> RuntimeSourceDefinitionManifest: try: - return RuntimeSourceDefinitionManifest.model_validate(value.model_dump(mode="json", by_alias=True)) + return RuntimeSourceDefinitionManifest.model_validate( + value.model_dump(mode="json", by_alias=True, exclude_none=True) + ) except ValidationError as error: raise InvalidRuntimeRequestError("source-definition-manifest") from error @@ -519,7 +521,9 @@ def submit_source_observation_request(value: SubmitSourceObservationRequest) -> try: return RuntimeSubmitSourceObservation( scope_id=value.scope_id, - observation=RuntimeSourceObservation.model_validate(value.observation.model_dump(mode="json")), + observation=RuntimeSourceObservation.model_validate( + value.observation.model_dump(mode="json", exclude_none=True) + ), ) except ValidationError as error: raise InvalidRuntimeRequestError("source-observation") from error diff --git a/src/powercontext/sources/__init__.py b/src/powercontext/sources/__init__.py index 91e1a27b08..25f7663d59 100644 --- a/src/powercontext/sources/__init__.py +++ b/src/powercontext/sources/__init__.py @@ -30,7 +30,15 @@ SourceDefinitionRegistry, SourceProjection, ) -from powercontext.sources.models import Source, SourceMaterialization, SourceProjectionKey, SourceRef +from powercontext.sources.models import ( + MemoryEvidenceAuthority, + MemoryEvidenceDeclaration, + MemoryEvidenceVerification, + Source, + SourceMaterialization, + SourceProjectionKey, + SourceRef, +) from powercontext.sources.observations import ( SourceDefinitionManifest, SourceObservation, @@ -53,6 +61,9 @@ "ConnectorRunSession", "ConnectorRunStatus", "ConnectorSubmissionStatus", + "MemoryEvidenceAuthority", + "MemoryEvidenceDeclaration", + "MemoryEvidenceVerification", "Source", "SourceAdapter", "SourceCatalog", diff --git a/src/powercontext/sources/catalog.py b/src/powercontext/sources/catalog.py index 1691d33509..4b6f68d011 100644 --- a/src/powercontext/sources/catalog.py +++ b/src/powercontext/sources/catalog.py @@ -24,7 +24,7 @@ ) from powercontext.sources.adapters import SourceAdapter from powercontext.sources.definitions import SourceDefinitionRegistry -from powercontext.sources.models import Source, SourceProjectionKey, SourceRef +from powercontext.sources.models import MemoryEvidenceDeclaration, Source, SourceProjectionKey, SourceRef from powercontext.sources.observations import SourceObservation from powercontext.sources.protocols import SourceCatalogBackend @@ -67,6 +67,19 @@ def as_ref(self, source: Source, /) -> SourceRef: definition = self._registry.definition_for_source(source) return SourceRef(source_type=definition.name, source_id=source.name) + def memory_evidence(self, source: Source, /) -> MemoryEvidenceDeclaration: + """Return the immutable declaration carried by the exact Source value.""" + + return self._registry.memory_evidence(source) + + def materialize(self, source: Source, /) -> Source: + """Attach the Definition-owned declaration before a native Source is stored.""" + + if isinstance(source, SourceObservation): + return source + definition = self._registry.definition_for_source(source) + return source.model_copy(update={"memory_evidence": definition.memory_evidence}) + async def resolve(self, value: object, /) -> Source: return await self._registry.resolve(value) diff --git a/src/powercontext/sources/definitions.py b/src/powercontext/sources/definitions.py index d9c54572dc..e5e8c009ed 100644 --- a/src/powercontext/sources/definitions.py +++ b/src/powercontext/sources/definitions.py @@ -17,7 +17,7 @@ from __future__ import annotations from collections.abc import Iterable, Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, Generic, Protocol, TypeVar, cast from pydantic import BaseModel, JsonValue, TypeAdapter @@ -34,7 +34,7 @@ SourceProjectionNotFoundError, ) from powercontext.sources.adapters import SourceAdapter -from powercontext.sources.models import Source, SourceProjectionKey +from powercontext.sources.models import MemoryEvidenceDeclaration, Source, SourceProjectionKey InputT = TypeVar("InputT") SourceT = TypeVar("SourceT", bound=Source) @@ -60,6 +60,7 @@ class SourceDefinition(SourceAdapter[InputT, SourceT, ValueT_co], Protocol[Input version: str projections: tuple[SourceProjection[SourceT], ...] + memory_evidence: MemoryEvidenceDeclaration @dataclass(frozen=True, slots=True) @@ -69,6 +70,7 @@ class AdapterSourceDefinition(Generic[InputT, SourceT, ValueT_co]): adapter: SourceAdapter[InputT, SourceT, ValueT_co] version: str = "1" projections: tuple[SourceProjection[SourceT], ...] = () + memory_evidence: MemoryEvidenceDeclaration = field(default_factory=MemoryEvidenceDeclaration) @property def input_class(self) -> type[InputT]: @@ -171,7 +173,7 @@ async def resolve(self, value: object, /) -> Source: if type(source) is not definition.source_class: raise InvalidSourceResultError(definition.name, "resolve", definition.source_class, type(source)) self.definition_for_source(source) - return cast(Source, source) + return cast(Source, source.model_copy(update={"memory_evidence": definition.memory_evidence})) async def read(self, source: Source, /) -> object: definition = self.definition_for_source(source) @@ -202,6 +204,16 @@ def project(self, source: Source, key: SourceProjectionKey, /) -> JsonValue: except (TypeError, ValueError) as error: raise InvalidSourceProjectionError(key.name, "result", "must match the declared output schema") from error + def memory_evidence(self, source: Source, /) -> MemoryEvidenceDeclaration: + """Return the declaration that applies to one exact Source value.""" + + from powercontext.sources.observations import SourceObservation + + if isinstance(source, SourceObservation): + return source.memory_evidence + self.definition_for_source(source) + return source.memory_evidence + def _validate_definition(definition: object) -> tuple[type[object], type[Source]]: definition_type = type(definition) @@ -220,6 +232,13 @@ def _validate_definition(definition: object) -> tuple[type[object], type[Source] projections = getattr(definition, "projections", None) if not isinstance(projections, tuple): raise InvalidSourceDefinitionError(definition_type, "projections", "must be a tuple") + memory_evidence = getattr(definition, "memory_evidence", None) + if not isinstance(memory_evidence, MemoryEvidenceDeclaration): + raise InvalidSourceDefinitionError( + definition_type, + "memory_evidence", + "must be a MemoryEvidenceDeclaration", + ) for method_name in ("resolve", "read"): if not callable(getattr(definition, method_name, None)): raise InvalidSourceAdapterError(definition_type, method_name, "must be callable") diff --git a/src/powercontext/sources/models.py b/src/powercontext/sources/models.py index 6e2cd48df6..3b4f221b30 100644 --- a/src/powercontext/sources/models.py +++ b/src/powercontext/sources/models.py @@ -29,6 +29,44 @@ class SourceMaterialization(StrEnum): REFERENCED = "referenced" +class MemoryEvidenceAuthority(StrEnum): + """The declared authority class of evidence produced by a Source Definition.""" + + UNTRUSTED = "untrusted" + USER_ASSERTED = "user_asserted" + REPOSITORY_ATTESTED = "repository_attested" + SYSTEM_ATTESTED = "system_attested" + + +class MemoryEvidenceVerification(StrEnum): + """Whether a Source Definition's authority declaration has been verified.""" + + UNKNOWN = "unknown" + VERIFIED = "verified" + NOT_VERIFIED = "not_verified" + + +class MemoryEvidenceDeclaration(BaseModel): + """A versioned, reconstructible declaration for Memory evidence. + + This is Definition-owned metadata, not a claim inferred from a ``SourceRef`` + or from Source content. Its neutral default deliberately contributes no + ranking preference until a later policy opts in. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + authority: MemoryEvidenceAuthority = MemoryEvidenceAuthority.UNTRUSTED + verification: MemoryEvidenceVerification = MemoryEvidenceVerification.UNKNOWN + declaration_version: str = "1" + + @field_validator("declaration_version") + @classmethod + def validate_declaration_version(cls, value: str) -> str: + _validate_reference_part("declaration_version", value) + return value + + class SourceRef(BaseModel): """A stable reference to one Source in the current catalog view.""" @@ -64,6 +102,7 @@ class Source(BaseModel): definition_version: str = "1" materialization: SourceMaterialization description: str | None = None + memory_evidence: MemoryEvidenceDeclaration = MemoryEvidenceDeclaration() @field_validator("definition_version") @classmethod diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 576e29fb79..d416f666c7 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -29,7 +29,7 @@ ) from powercontext.limits import MAX_SOURCE_TYPE_LENGTH from powercontext.sources.definitions import SourceDefinition, SourceDefinitionRegistry -from powercontext.sources.models import Source, SourceMaterialization, SourceProjectionKey +from powercontext.sources.models import MemoryEvidenceDeclaration, Source, SourceMaterialization, SourceProjectionKey _JSON_VALUE = TypeAdapter(JsonValue) @@ -59,6 +59,7 @@ class SourceDefinitionManifest(BaseModel): fingerprint: str source_schema: dict[str, JsonValue] projections: tuple[SourceProjectionManifest, ...] = () + memory_evidence: MemoryEvidenceDeclaration = MemoryEvidenceDeclaration() @field_validator("name", "version") @classmethod @@ -106,6 +107,10 @@ def validate_manifest(self) -> SourceDefinitionManifest: version=self.version, source_schema=self.source_schema, projections=self.projections, + # Accept persisted and remote manifests emitted before the evidence + # declaration existed. They resolve to the neutral declaration, but + # retain their historical content-addressed identity. + memory_evidence=self.memory_evidence if "memory_evidence" in self.__pydantic_fields_set__ else None, ) if self.fingerprint != expected: raise ValueError("manifest fingerprint does not match its declaration") # noqa: TRY003 @@ -179,9 +184,11 @@ def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> S version=definition.version, source_schema=source_schema, projections=projections, + memory_evidence=definition.memory_evidence, ), source_schema=source_schema, projections=projections, + memory_evidence=definition.memory_evidence, ) @@ -210,6 +217,7 @@ def project_source_for_transport( description=source.description, source_type=definition.name, definition_fingerprint=manifest.fingerprint, + memory_evidence=manifest.memory_evidence, payload=payload, projections=projections, ) @@ -221,12 +229,14 @@ def _source_definition_fingerprint( version: str, source_schema: dict[str, JsonValue], projections: tuple[SourceProjectionManifest, ...], + memory_evidence: MemoryEvidenceDeclaration | None, ) -> str: declaration = { "name": name, "version": version, "source_schema": source_schema, "projections": [projection.model_dump(mode="json", by_alias=True) for projection in projections], + **({} if memory_evidence is None else {"memory_evidence": memory_evidence.model_dump(mode="json")}), } encoded = rfc8785.dumps(_JSON_VALUE.validate_python(declaration)) return f"sha256:{hashlib.sha256(encoded).hexdigest()}" diff --git a/tests/builtin/persistence/test_memory.py b/tests/builtin/persistence/test_memory.py index 86a96b8b86..e94f08ab50 100644 --- a/tests/builtin/persistence/test_memory.py +++ b/tests/builtin/persistence/test_memory.py @@ -16,14 +16,17 @@ import asyncio -from sqlalchemy import BigInteger, DateTime, Integer, String +from sqlalchemy import BigInteger, DateTime, Integer, String, delete from sqlalchemy.dialects import mysql from sqlalchemy.schema import CreateTable, ForeignKeyConstraint, PrimaryKeyConstraint, UniqueConstraint from powercontext.builtin.artifacts.memory import MemoryEntryInput +from powercontext.builtin.persistence.memory import RelationalMemoryBackend from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.persistence.tables import ( + MEMORY_ENTRY_EVIDENCE_TABLE, MEMORY_ENTRY_HEADS_TABLE, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE, MEMORY_ENTRY_VERSIONS_TABLE, ) from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts @@ -53,22 +56,31 @@ def _key_budget(constraint: PrimaryKeyConstraint | UniqueConstraint | ForeignKey def test_memory_schema_is_mysql_compilable_and_respects_key_and_payload_limits() -> None: dialect = mysql.dialect() versions = str(CreateTable(MEMORY_ENTRY_VERSIONS_TABLE).compile(dialect=dialect)) + evidence = str(CreateTable(MEMORY_ENTRY_EVIDENCE_TABLE).compile(dialect=dialect)) heads = str(CreateTable(MEMORY_ENTRY_HEADS_TABLE).compile(dialect=dialect)) + lifecycle = str(CreateTable(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE).compile(dialect=dialect)) assert "scope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL" in versions assert "text MEDIUMTEXT NOT NULL" in versions assert "source_refs MEDIUMBLOB NOT NULL" in versions assert "artifact_refs MEDIUMBLOB NOT NULL" in versions + assert "declaration_version VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL" in evidence assert "searchable_text MEDIUMTEXT NOT NULL" in heads + assert "quality_policy VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL" in lifecycle budgets = [ _key_budget(constraint) - for table in (MEMORY_ENTRY_VERSIONS_TABLE, MEMORY_ENTRY_HEADS_TABLE) + for table in ( + MEMORY_ENTRY_VERSIONS_TABLE, + MEMORY_ENTRY_EVIDENCE_TABLE, + MEMORY_ENTRY_HEADS_TABLE, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE, + ) for constraint in table.constraints if isinstance(constraint, PrimaryKeyConstraint | UniqueConstraint | ForeignKeyConstraint) ] assert budgets - assert max(budgets) == 2560 + assert max(budgets) <= 2568 assert all(budget < _INNODB_MAX_INDEX_BYTES for budget in budgets) @@ -174,6 +186,59 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_memory_evidence_snapshots_and_neutral_lifecycle_projection_are_rebuildable() -> None: + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + context = await contexts.get("project") + source, _ = await context.sources.capture( + ContentCapture(source_id="turn-1", content="A durable source declaration must be snapshotted.") + ) + memory = await context.artifacts.memory.remember( + memory=None, + sources=(source,), + entries=( + MemoryEntryInput( + kind="decision", + text="Keep provenance metadata outside the Memory body.", + sources=(source,), + ), + ), + mode="append", + ) + assert memory is not None + (entry,) = await context.artifacts.memory.entries(memory) + assert entry.source_evidence[0].source.source_id == "turn-1" + assert entry.source_evidence[0].declaration.authority == "untrusted" + assert entry.source_evidence[0].declaration.verification == "unknown" + + backend = RelationalMemoryBackend( + database=contexts.database, + scope_id="project", + artifacts=contexts.repositories.artifacts, + index=contexts.index, + ) + current_lifecycle = (await backend.lifecycle_projections(memory.as_ref()))[0] + assert current_lifecycle.validity == "current" + assert current_lifecycle.context_annotation.time_state == "unknown" + assert current_lifecycle.context_annotation.quality_policy == "neutral" + + inactive = await context.artifacts.memory.forget(memory, entries=(entry,), reason="superseded elsewhere") + assert inactive is not None + lifecycle = await backend.lifecycle_projections(inactive.as_ref()) + assert lifecycle[0].validity == "inactive" + assert lifecycle[0].source_evidence == entry.source_evidence + + async with contexts.database.transaction() as connection: + await connection.execute(delete(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE)) + + await context.artifacts.memory.rebuild_projections() + + rebuilt = await backend.lifecycle_projections(inactive.as_ref()) + assert rebuilt == lifecycle + + asyncio.run(scenario()) + + def test_scope_bound_contexts_do_not_share_rows() -> None: async def scenario() -> None: async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: diff --git a/tests/e2e/real_experience_skill/harness.py b/tests/e2e/real_experience_skill/harness.py index c03de72cc8..004b8058d4 100644 --- a/tests/e2e/real_experience_skill/harness.py +++ b/tests/e2e/real_experience_skill/harness.py @@ -105,7 +105,9 @@ "pc_access_owners", "pc_access_relationships", "pc_memory_vector_entries", + "pc_memory_entry_lifecycle_projections", "pc_memory_entry_heads", + "pc_memory_entry_evidence", "pc_memory_entry_versions", "pc_skill_publications", "pc_artifact_candidate_heads", diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index c313ae7962..2b5295f325 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -15,16 +15,26 @@ from __future__ import annotations import asyncio +import hashlib import urllib.request +from dataclasses import replace import pytest +import rfc8785 from pydantic import ValidationError from referencing.exceptions import Unresolvable -from powercontext.builtin.runtime.relational import _json_schema_validator +from powercontext.builtin.runtime.relational import _json_schema_validator, _validate_source_observation from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION, ContentCapture +from powercontext.http import SourceDefinitionManifest as HttpSourceDefinitionManifest +from powercontext.http import SourceObservation as HttpSourceObservation +from powercontext.http import SubmitSourceObservationRequest +from powercontext.server.mapping import runtime_source_definition_manifest, submit_source_observation_request from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, + MemoryEvidenceAuthority, + MemoryEvidenceDeclaration, + MemoryEvidenceVerification, Source, SourceCatalog, SourceDefinitionManifest, @@ -54,6 +64,78 @@ def test_definition_manifest_has_a_stable_content_addressed_identity() -> None: SourceDefinitionManifest.model_validate(first.model_dump(mode="json", by_alias=True) | {"version": "2"}) +def test_definition_manifest_carries_versioned_memory_evidence_to_remote_observations() -> None: + async def scenario() -> None: + definition = replace( + CONTENT_SOURCE_DEFINITION, + memory_evidence=MemoryEvidenceDeclaration( + authority=MemoryEvidenceAuthority.SYSTEM_ATTESTED, + verification=MemoryEvidenceVerification.VERIFIED, + declaration_version="attestation-v2", + ), + ) + registry = SourceDefinitionRegistry((definition,)) + manifest = manifest_for_definition(definition) + source = await registry.resolve(ContentCapture(source_id="turn-1", content="Keep this declaration.")) + observation = project_source_for_transport(registry, source) + + assert manifest.memory_evidence == definition.memory_evidence + assert source.memory_evidence == definition.memory_evidence + assert observation.memory_evidence == definition.memory_evidence + assert registry.memory_evidence(observation) == definition.memory_evidence + assert manifest.fingerprint != manifest_for_definition(CONTENT_SOURCE_DEFINITION).fingerprint + + runtime = submit_source_observation_request( + SubmitSourceObservationRequest( + scope_id="scope", + observation=HttpSourceObservation.model_validate(observation.model_dump(mode="json")), + ) + ) + assert runtime.observation.memory_evidence == definition.memory_evidence + _validate_source_observation(runtime.observation, manifest) + + asyncio.run(scenario()) + + +def test_legacy_remote_definition_manifest_remains_neutral_and_accepted() -> None: + current = manifest_for_definition(CONTENT_SOURCE_DEFINITION) + legacy_payload = { + "name": current.name, + "version": current.version, + "source_schema": current.source_schema, + "projections": [projection.model_dump(mode="json", by_alias=True) for projection in current.projections], + } + legacy = SourceDefinitionManifest( + name=current.name, + version=current.version, + source_schema=current.source_schema, + projections=current.projections, + fingerprint=f"sha256:{hashlib.sha256(rfc8785.dumps(legacy_payload)).hexdigest()}", + ) + transported = HttpSourceDefinitionManifest.model_validate( + legacy.model_dump(mode="json", by_alias=True, exclude={"memory_evidence"}) + ) + + assert runtime_source_definition_manifest(transported) == legacy + assert legacy.memory_evidence.authority == "untrusted" + assert legacy.memory_evidence.verification == "unknown" + + async def legacy_source() -> SourceObservation: + registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) + resolved = await registry.resolve(ContentCapture(source_id="turn-1", content="Legacy transport.")) + return project_source_for_transport(registry, resolved) + + source = asyncio.run(legacy_source()).model_copy(update={"definition_fingerprint": legacy.fingerprint}) + legacy_observation = HttpSourceObservation.model_validate( + source.model_dump(mode="json", exclude={"memory_evidence"}) + ) + runtime_observation = submit_source_observation_request( + SubmitSourceObservationRequest(scope_id="scope", observation=legacy_observation) + ).observation + assert "memory_evidence" not in runtime_observation.__pydantic_fields_set__ + _validate_source_observation(runtime_observation, legacy) + + def test_source_observation_remains_usable_without_worker_definition_code() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) From cc91c9f13bca63908ed29a76c008274b797d63e6 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Tue, 22 Sep 2026 21:46:36 +0800 Subject: [PATCH 02/30] fix(opendal): declare memory evidence for the snapshot definition --- .../opendal/src/powercontext_connector_opendal/source.py | 2 ++ integrations/opendal/tests/test_opendal_connector.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/integrations/opendal/src/powercontext_connector_opendal/source.py b/integrations/opendal/src/powercontext_connector_opendal/source.py index 5ab48cf7d6..6b310601f3 100644 --- a/integrations/opendal/src/powercontext_connector_opendal/source.py +++ b/integrations/opendal/src/powercontext_connector_opendal/source.py @@ -23,6 +23,7 @@ from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, + MemoryEvidenceDeclaration, Source, SourceMaterialization, SourceProjection, @@ -92,6 +93,7 @@ class TextFileSnapshotSourceDefinition: name = TEXT_FILE_SNAPSHOT_SOURCE_NAME version = "1" source_class = TextFileSnapshotSource + memory_evidence = MemoryEvidenceDeclaration() def __init__(self) -> None: projection = cast(SourceProjection[TextFileSnapshotSource], TextFileEvidenceProjection()) diff --git a/integrations/opendal/tests/test_opendal_connector.py b/integrations/opendal/tests/test_opendal_connector.py index a20bf3069b..a89a81bb3a 100644 --- a/integrations/opendal/tests/test_opendal_connector.py +++ b/integrations/opendal/tests/test_opendal_connector.py @@ -234,6 +234,8 @@ def test_text_file_definition_declares_its_executable_contract() -> None: assert definition.version == "1" assert definition.input_class is TextFileSnapshotCapture assert len(definition.projections) == 1 + assert definition.memory_evidence.authority == "untrusted" + assert definition.memory_evidence.verification == "unknown" def test_remote_opendal_worker_completes_the_source_to_memory_loop(tmp_path: Path) -> None: From 56897ff4b20404c781bbb889d720b444aa460719 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Wed, 23 Sep 2026 21:19:04 +0800 Subject: [PATCH 03/30] fix(sources): keep a pre-declaration manifest readable after storage The fingerprint covers exactly the fields a manifest was emitted with, so an absent memory_evidence resolves to the neutral declaration while keeping its historical content-addressed identity. Serialization re-introduced the model default, so a legacy manifest validated on the way in, was stored with the field, and then failed its own fingerprint check on readback (InvalidStoredPayloadError). Serialize the field set the manifest actually validated instead, keeping registration, readback, repeat registration and observation submission consistent. --- src/powercontext/sources/observations.py | 27 +++++++++++- tests/builtin/persistence/test_sources.py | 50 +++++++++++++++++++++++ tests/test_source_observations.py | 47 +++++++++++++++------ 3 files changed, 110 insertions(+), 14 deletions(-) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index d416f666c7..0cdc5a9258 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -20,7 +20,17 @@ from typing import Any, Literal import rfc8785 -from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, field_validator, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + JsonValue, + SerializerFunctionWrapHandler, + TypeAdapter, + field_validator, + model_serializer, + model_validator, +) from powercontext.errors import ( InvalidSourceDefinitionError, @@ -116,6 +126,21 @@ def validate_manifest(self) -> SourceDefinitionManifest: raise ValueError("manifest fingerprint does not match its declaration") # noqa: TRY003 return self + @model_serializer(mode="wrap") + def serialize_manifest(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: + """Preserve the field set the fingerprint actually covers. + + A manifest received without a declaration keeps its historical identity, so + it must also serialize without that field. Writing the model default would + make the stored payload validate against a different fingerprint than the + one it was registered with, which makes registration unreadable. + """ + + dumped = handler(self) + if "memory_evidence" not in self.__pydantic_fields_set__: + dumped.pop("memory_evidence", None) + return dumped + class SourceProjectionValue(BaseModel): """One named projection computed by the worker that owns the Definition.""" diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 22e080499f..ed8e387a83 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -29,8 +29,10 @@ from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import SHARED_TABLES, SOURCES_TABLE +from powercontext.builtin.runtime.models import SubmitSourceObservation from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.sources import ( + SourceDefinitionManifest, SourceDefinitionRegistry, SourceMaterialization, SourceObservation, @@ -38,14 +40,34 @@ manifest_for_definition, project_source_for_transport, ) +from powercontext.sources.observations import _source_definition_fingerprint from tests.builtin.persistence.contract import ( SOURCE_ADAPTERS, CommitSource, + NoteInput, NoteSource, repository_profile, ) +def _legacy_manifest(current: SourceDefinitionManifest) -> SourceDefinitionManifest: + """Return the declaration shape registered before the evidence field existed.""" + + return SourceDefinitionManifest( + name=current.name, + version=current.version, + fingerprint=_source_definition_fingerprint( + name=current.name, + version=current.version, + source_schema=current.source_schema, + projections=current.projections, + memory_evidence=None, + ), + source_schema=current.source_schema, + projections=current.projections, + ) + + def test_two_source_adapters_share_one_repository_and_journal() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): @@ -178,6 +200,34 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_legacy_definition_manifest_registers_reads_back_and_accepts_an_observation() -> None: + async def scenario() -> None: + worker_registry = SourceDefinitionRegistry.from_adapters(SOURCE_ADAPTERS) + definition = worker_registry.definition_for_name("note") + legacy = _legacy_manifest(manifest_for_definition(definition)) + source = await worker_registry.resolve(NoteInput(note_id="note-1", body="Registered before the declaration.")) + projected = project_source_for_transport(worker_registry, source) + observation = SourceObservation.model_validate( + projected.model_dump(mode="json", exclude={"memory_evidence"}) + | {"definition_fingerprint": legacy.fingerprint} + ) + + async with repository_profile() as (profile, _repositories): + # The server owns its own registry, so the worker definition stays remote. + contexts = RelationalContexts(database=profile.database) + first = await contexts.register_source_definition(legacy) + second = await contexts.register_source_definition(legacy) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + + assert first == legacy + assert second == first + assert receipt.source_ref == SourceRef(source_type="note", source_id="note-1") + + asyncio.run(scenario()) + + def test_stored_observation_representation_does_not_depend_on_the_active_registry() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry.from_adapters(SOURCE_ADAPTERS) diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 2b5295f325..3364d4d758 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -16,6 +16,7 @@ import asyncio import hashlib +import json import urllib.request from dataclasses import replace @@ -54,6 +55,24 @@ async def get(self, source: Source, /) -> Source: raise AssertionError(source) +def _legacy_manifest(current: SourceDefinitionManifest) -> SourceDefinitionManifest: + """Return the declaration shape emitted before the evidence field existed.""" + + payload = { + "name": current.name, + "version": current.version, + "source_schema": current.source_schema, + "projections": [projection.model_dump(mode="json", by_alias=True) for projection in current.projections], + } + return SourceDefinitionManifest( + name=current.name, + version=current.version, + source_schema=current.source_schema, + projections=current.projections, + fingerprint=f"sha256:{hashlib.sha256(rfc8785.dumps(payload)).hexdigest()}", + ) + + def test_definition_manifest_has_a_stable_content_addressed_identity() -> None: first = manifest_for_definition(CONTENT_SOURCE_DEFINITION) second = manifest_for_definition(CONTENT_SOURCE_DEFINITION) @@ -99,19 +118,7 @@ async def scenario() -> None: def test_legacy_remote_definition_manifest_remains_neutral_and_accepted() -> None: current = manifest_for_definition(CONTENT_SOURCE_DEFINITION) - legacy_payload = { - "name": current.name, - "version": current.version, - "source_schema": current.source_schema, - "projections": [projection.model_dump(mode="json", by_alias=True) for projection in current.projections], - } - legacy = SourceDefinitionManifest( - name=current.name, - version=current.version, - source_schema=current.source_schema, - projections=current.projections, - fingerprint=f"sha256:{hashlib.sha256(rfc8785.dumps(legacy_payload)).hexdigest()}", - ) + legacy = _legacy_manifest(current) transported = HttpSourceDefinitionManifest.model_validate( legacy.model_dump(mode="json", by_alias=True, exclude={"memory_evidence"}) ) @@ -136,6 +143,20 @@ async def legacy_source() -> SourceObservation: _validate_source_observation(runtime_observation, legacy) +def test_legacy_definition_manifest_round_trips_without_gaining_the_evidence_field() -> None: + current = manifest_for_definition(CONTENT_SOURCE_DEFINITION) + legacy = _legacy_manifest(current) + + stored = SourceDefinitionManifest.model_validate_json(legacy.model_dump_json(by_alias=True)) + transported = HttpSourceDefinitionManifest.model_validate(legacy.model_dump(mode="json", by_alias=True)) + + assert stored == legacy + assert transported.memory_evidence is None + assert "memory_evidence" not in json.loads(legacy.model_dump_json(by_alias=True)) + assert SourceDefinitionManifest.model_validate_json(current.model_dump_json(by_alias=True)) == current + assert "memory_evidence" in current.model_dump(mode="json", by_alias=True) + + def test_source_observation_remains_usable_without_worker_definition_code() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) From d9213c381d93d2c3c636bdf17729c78f981477f5 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Wed, 23 Sep 2026 21:19:07 +0800 Subject: [PATCH 04/30] perf(memory): rewrite only changed lifecycle projection rows Lifecycle rows now follow the same per-entry rule as the active-head and vector rows: a commit diffs each manifest entry's identity and state against its base revision and rewrites only what changed, so appending one entry to an N-entry Memory writes one row instead of N deletes plus N+1 inserts. The read no longer filters rows by head_revision, which was the only reason every row had to be re-stamped; it selects the artifact's rows and proves completeness against the authoritative manifest by entry identity. head_revision keeps its meaning as the revision that last wrote a row. --- .../builtin/persistence/memory.py | 62 ++++++++++--- .../builtin/persistence/tables.py | 4 + tests/builtin/persistence/test_memory.py | 90 ++++++++++++++++++- 3 files changed, 144 insertions(+), 12 deletions(-) diff --git a/src/powercontext/builtin/persistence/memory.py b/src/powercontext/builtin/persistence/memory.py index e65a8e488f..1c08d8a374 100644 --- a/src/powercontext/builtin/persistence/memory.py +++ b/src/powercontext/builtin/persistence/memory.py @@ -230,7 +230,12 @@ async def entries(self, memory: ArtifactRef, /) -> tuple[MemoryEntryVersion, ... return tuple(by_id[version_id] for version_id in version_ids) async def lifecycle_projections(self, memory: ArtifactRef, /) -> tuple[MemoryLifecycleProjection, ...]: - """Load the internal neutral lifecycle projection for one exact Memory head.""" + """Load the internal neutral lifecycle projection for one exact Memory head. + + Rows are keyed by entry and re-stamped only by the revision that changes + them, so the read selects the artifact's rows and proves completeness by + entry identity, exactly as the active-head projection does. + """ canonical = await self.get(memory) version_ids = tuple(item.entry_version_id for item in canonical.content.manifest.entries) @@ -254,7 +259,6 @@ async def lifecycle_projections(self, memory: ArtifactRef, /) -> tuple[MemoryLif .where( MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.scope_id == self._scope_id, MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.memory_artifact_id == memory.artifact_id, - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.head_revision == memory.revision, ) .order_by(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.entry_id) ) @@ -501,7 +505,22 @@ async def _lifecycle_for_memory( connection: AsyncConnection, memory: Memory, ) -> tuple[MemoryLifecycleProjection, ...]: - manifest = memory.content.manifest.entries + return await self._lifecycle_for_entries( + connection, + memory, + tuple(item.entry_id for item in memory.content.manifest.entries), + ) + + async def _lifecycle_for_entries( + self, + connection: AsyncConnection, + memory: Memory, + entry_ids: tuple[str, ...], + ) -> tuple[MemoryLifecycleProjection, ...]: + """Derive lifecycle rows for the named manifest entries of one Memory.""" + + wanted = frozenset(entry_ids) + manifest = tuple(item for item in memory.content.manifest.entries if item.entry_id in wanted) if not manifest: return () version_ids = tuple(item.entry_version_id for item in manifest) @@ -722,16 +741,37 @@ async def _commit(self, connection: AsyncConnection, value: MemoryCommit) -> Mem value.memory.as_ref(), upserts, ) - # Lifecycle rows are keyed by entry, not by revision, so the artifact's rows - # are replaced wholesale: lifecycle_projections reads them back by revision. - await connection.execute( - delete(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE).where( - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.scope_id == self._scope_id, - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.memory_artifact_id == value.memory.artifact_id, + # Lifecycle rows cover every manifest entry, including inactive ones, and + # follow the same per-entry rule as the active head: only the entries whose + # identity or state changed are rewritten, so repeated appends stay bounded + # by the appended entries instead of the manifest size. + previous_entries = ( + {} + if value.base is None + else {item.entry_id: (item.entry_version_id, item.state) for item in value.base.content.manifest.entries} + ) + current_entries = { + item.entry_id: (item.entry_version_id, item.state) for item in value.memory.content.manifest.entries + } + lifecycle_removed = tuple(sorted(entry_id for entry_id in previous_entries if entry_id not in current_entries)) + lifecycle_changed = tuple( + sorted( + entry_id for entry_id, identity in current_entries.items() if previous_entries.get(entry_id) != identity ) ) - lifecycle = await self._lifecycle_for_memory(connection, committed) - if lifecycle: + lifecycle_drop = tuple(sorted({*lifecycle_removed, *lifecycle_changed})) + if lifecycle_drop: + await connection.execute( + delete(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE).where( + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.scope_id == self._scope_id, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.memory_artifact_id == value.memory.artifact_id, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.entry_id.in_(lifecycle_drop), + ) + ) + if lifecycle_changed: + lifecycle = await self._lifecycle_for_entries(connection, committed, lifecycle_changed) + if len(lifecycle) != len(lifecycle_changed): + raise _InvalidMemoryCommitError("lifecycle-projection") await connection.execute( insert(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE), [_lifecycle_values(self._scope_id, projection) for projection in lifecycle], diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index b4fc0e125b..3418f300c2 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -1051,6 +1051,10 @@ def _entry_text_type(): CheckConstraint("head_revision > 0", name="ck_pc_memory_entry_heads_revision_positive"), ) +# Lifecycle rows are maintained per entry and re-stamped only by the revision that +# changes them, like the active-head and vector rows. ``head_revision`` records the +# revision that last wrote a row; a read identifies rows by entry and proves +# completeness against the authoritative manifest rather than by revision. MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE = Table( "pc_memory_entry_lifecycle_projections", SHARED_METADATA, diff --git a/tests/builtin/persistence/test_memory.py b/tests/builtin/persistence/test_memory.py index e94f08ab50..9b3059a320 100644 --- a/tests/builtin/persistence/test_memory.py +++ b/tests/builtin/persistence/test_memory.py @@ -16,7 +16,7 @@ import asyncio -from sqlalchemy import BigInteger, DateTime, Integer, String, delete +from sqlalchemy import BigInteger, DateTime, Integer, String, delete, select from sqlalchemy.dialects import mysql from sqlalchemy.schema import CreateTable, ForeignKeyConstraint, PrimaryKeyConstraint, UniqueConstraint @@ -30,6 +30,7 @@ MEMORY_ENTRY_VERSIONS_TABLE, ) from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts +from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.builtin.sources import ContentCapture, ContentSource _INNODB_MAX_INDEX_BYTES = 3072 @@ -239,6 +240,93 @@ async def scenario() -> None: asyncio.run(scenario()) +async def _lifecycle_rows(contexts: RelationalContexts) -> dict[str, tuple[int, str]]: + async with contexts.database.transaction() as connection: + rows = ( + await connection.execute( + select( + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.entry_id, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.head_revision, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.validity, + ) + ) + ).all() + return {str(entry_id): (int(revision), str(validity)) for entry_id, revision, validity in rows} + + +def test_lifecycle_projection_rows_are_rewritten_only_for_changed_entries() -> None: + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + context = await contexts.get("project") + source, _ = await context.sources.capture( + ContentCapture(source_id="turn-1", content="Lifecycle rows must follow their own entry.") + ) + opened = await context.artifacts.memory.remember( + memory=None, + sources=(source,), + entries=( + MemoryEntryInput(kind="decision", text="Keep the first entry.", sources=(source,)), + MemoryEntryInput(kind="constraint", text="Keep the second entry.", sources=(source,)), + ), + mode="append", + ) + assert opened is not None + opened_rows = await _lifecycle_rows(contexts) + assert len(opened_rows) == 2 + assert {revision for revision, _ in opened_rows.values()} == {opened.revision} + + appended = await context.artifacts.memory.remember( + memory=opened, + sources=(source,), + entries=(MemoryEntryInput(kind="agent-note", text="Append exactly one entry.", sources=(source,)),), + mode="append", + ) + assert appended is not None + assert appended.revision == opened.revision + 1 + + backend = RelationalMemoryBackend( + database=contexts.database, + scope_id="project", + artifacts=contexts.repositories.artifacts, + index=contexts.index, + ) + appended_rows = await _lifecycle_rows(contexts) + + # Appending one entry rewrites that entry alone: every row that was + # already current keeps the revision that last wrote it. + assert len(appended_rows) == 3 + assert { + entry_id: value for entry_id, value in appended_rows.items() if entry_id in opened_rows + } == opened_rows + assert {validity for _, validity in appended_rows.values()} == {"current"} + + forgotten = (await context.artifacts.memory.entries(opened))[0] + inactive = await context.artifacts.memory.forget( + appended, + entries=(forgotten,), + reason="superseded elsewhere", + ) + assert inactive is not None + inactive_rows = await _lifecycle_rows(contexts) + + assert inactive_rows[forgotten.entry_id] == (inactive.revision, "inactive") + assert {entry_id: value for entry_id, value in inactive_rows.items() if entry_id != forgotten.entry_id} == { + entry_id: value for entry_id, value in appended_rows.items() if entry_id != forgotten.entry_id + } + + incremental = await backend.lifecycle_projections(inactive.as_ref()) + assert {item.validity for item in incremental if item.entry_id == forgotten.entry_id} == {"inactive"} + assert {item.validity for item in incremental if item.entry_id != forgotten.entry_id} == {"current"} + + async with contexts.database.transaction() as connection: + await connection.execute(delete(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE)) + await context.artifacts.memory.rebuild_projections() + + assert await backend.lifecycle_projections(inactive.as_ref()) == incremental + + asyncio.run(scenario()) + + def test_scope_bound_contexts_do_not_share_rows() -> None: async def scenario() -> None: async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: From 68e83c30baa7384d5ad9023b20b8483acb04c828 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Wed, 23 Sep 2026 21:48:44 +0800 Subject: [PATCH 05/30] fix(server): preserve null projection values when mapping observations exclude_none was used to keep an absent evidence declaration out of the runtime model, but it also dropped required-but-nullable values: a projection whose value is JSON null became projections.0.value Field required, so a valid observation was rejected with 422. Drop only the absent declaration and keep every other value as sent. --- src/powercontext/server/mapping.py | 17 +++++++++++++++-- tests/test_source_observations.py | 26 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/powercontext/server/mapping.py b/src/powercontext/server/mapping.py index 39368df8c1..8e1280d998 100644 --- a/src/powercontext/server/mapping.py +++ b/src/powercontext/server/mapping.py @@ -493,10 +493,23 @@ def capture_response(value: SourceReceipt) -> CaptureContentSourceResponse: ) +def _without_absent_evidence(payload: dict[str, Any]) -> dict[str, Any]: + """Drop only an absent evidence declaration from a serialized request model. + + A worker whose Definition pre-dates the declaration must not gain one during + mapping, but ``exclude_none`` would also drop required values that are + legitimately JSON null, such as an explicit null projection value. + """ + + if payload.get("memory_evidence") is None: + payload.pop("memory_evidence", None) + return payload + + def runtime_source_definition_manifest(value: SourceDefinitionManifest) -> RuntimeSourceDefinitionManifest: try: return RuntimeSourceDefinitionManifest.model_validate( - value.model_dump(mode="json", by_alias=True, exclude_none=True) + _without_absent_evidence(value.model_dump(mode="json", by_alias=True)) ) except ValidationError as error: raise InvalidRuntimeRequestError("source-definition-manifest") from error @@ -522,7 +535,7 @@ def submit_source_observation_request(value: SubmitSourceObservationRequest) -> return RuntimeSubmitSourceObservation( scope_id=value.scope_id, observation=RuntimeSourceObservation.model_validate( - value.observation.model_dump(mode="json", exclude_none=True) + _without_absent_evidence(value.observation.model_dump(mode="json")) ), ) except ValidationError as error: diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 3364d4d758..835420432b 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -157,6 +157,32 @@ def test_legacy_definition_manifest_round_trips_without_gaining_the_evidence_fie assert "memory_evidence" in current.model_dump(mode="json", by_alias=True) +def test_source_observation_request_mapping_preserves_null_projection_values() -> None: + async def scenario() -> None: + registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) + resolved = await registry.resolve(ContentCapture(source_id="turn-1", content="Null is a value.")) + projected = project_source_for_transport(registry, resolved) + (projection,) = projected.projections + transport = HttpSourceObservation.model_validate( + projected.model_dump(mode="json", exclude={"memory_evidence"}) + | { + "payload": projected.payload | {"nullable_field": None}, + "projections": [{"key": projection.key.model_dump(mode="json"), "value": None}], + } + ) + + observation = submit_source_observation_request( + SubmitSourceObservationRequest(scope_id="scope", observation=transport) + ).observation + + assert observation.projections[0].value is None + assert observation.payload == transport.payload + assert observation.payload["nullable_field"] is None + assert "memory_evidence" not in observation.__pydantic_fields_set__ + + asyncio.run(scenario()) + + def test_source_observation_remains_usable_without_worker_definition_code() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) From 0cb6ec18494922bd3c49a18a9bff951f03aa4c63 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Wed, 23 Sep 2026 21:48:48 +0800 Subject: [PATCH 06/30] fix(memory): batch evidence lookups within the backend bind budget The evidence lookup bound three identity columns per entry plus the scope, so a manifest beyond roughly a third of SQLite's variable ceiling could no longer be read or appended to, even when no entry carried sources. Chunk the lookup at the shared SELECTION_BATCH_SIZE bind budget, deduplicating identities so batches stay disjoint, and pin the budget with a regression test that measures the bound values of each statement. --- .../builtin/persistence/memory.py | 67 +++++++++++-------- tests/builtin/persistence/test_memory.py | 58 +++++++++++++++- 2 files changed, 95 insertions(+), 30 deletions(-) diff --git a/src/powercontext/builtin/persistence/memory.py b/src/powercontext/builtin/persistence/memory.py index 1c08d8a374..b30b02d40e 100644 --- a/src/powercontext/builtin/persistence/memory.py +++ b/src/powercontext/builtin/persistence/memory.py @@ -59,7 +59,7 @@ from powercontext.builtin.inference import EmbeddingModel from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.codec import dump_model, load_model, stored_bytes -from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.database import SELECTION_BATCH_SIZE, AsyncDatabase from powercontext.builtin.persistence.errors import RepositoryNotFoundError from powercontext.builtin.persistence.memory_index import MemoryIndex, NoMemoryIndex from powercontext.builtin.persistence.tables import ( @@ -89,6 +89,11 @@ class _ArtifactRefs(RootModel[tuple[ArtifactRef, ...]]): pass +# An evidence lookup binds three identity columns per entry plus the scope, so it +# chunks at the shared bind budget instead of the shared selection row count. +_EVIDENCE_SELECTION_BATCH_SIZE = max(1, SELECTION_BATCH_SIZE // 3) + + class _MemoryDraft(ArtifactDraft[MemoryContent]): family: ClassVar[str] = Memory.family @@ -456,39 +461,43 @@ async def _hydrate_source_evidence( ) -> tuple[MemoryEntryVersion, ...]: if not entries: return () - identities = tuple((entry.memory_artifact_id, entry.entry_id, entry.entry_version_id) for entry in entries) - rows = ( - await connection.execute( - select(MEMORY_ENTRY_EVIDENCE_TABLE) - .where( - MEMORY_ENTRY_EVIDENCE_TABLE.c.scope_id == self._scope_id, - tuple_( + identities = tuple( + dict.fromkeys((entry.memory_artifact_id, entry.entry_id, entry.entry_version_id) for entry in entries) + ) + snapshots: dict[tuple[str, str, str], list[MemoryEvidenceSnapshot]] = {} + for start in range(0, len(identities), _EVIDENCE_SELECTION_BATCH_SIZE): + batch = identities[start : start + _EVIDENCE_SELECTION_BATCH_SIZE] + rows = ( + await connection.execute( + select(MEMORY_ENTRY_EVIDENCE_TABLE) + .where( + MEMORY_ENTRY_EVIDENCE_TABLE.c.scope_id == self._scope_id, + tuple_( + MEMORY_ENTRY_EVIDENCE_TABLE.c.memory_artifact_id, + MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_id, + MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_version_id, + ).in_(batch), + ) + .order_by( MEMORY_ENTRY_EVIDENCE_TABLE.c.memory_artifact_id, MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_id, MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_version_id, - ).in_(identities), - ) - .order_by( - MEMORY_ENTRY_EVIDENCE_TABLE.c.memory_artifact_id, - MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_id, - MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_version_id, - MEMORY_ENTRY_EVIDENCE_TABLE.c.ordinal, + MEMORY_ENTRY_EVIDENCE_TABLE.c.ordinal, + ) ) - ) - ).mappings() - snapshots: dict[tuple[str, str, str], list[MemoryEvidenceSnapshot]] = {} - for row in rows: - identity = (str(row["memory_artifact_id"]), str(row["entry_id"]), str(row["entry_version_id"])) - snapshots.setdefault(identity, []).append( - MemoryEvidenceSnapshot( - source=SourceRef(source_type=str(row["source_type"]), source_id=str(row["source_id"])), - declaration=MemoryEvidenceDeclaration( - authority=MemoryEvidenceAuthority(str(row["authority"])), - verification=MemoryEvidenceVerification(str(row["verification"])), - declaration_version=str(row["declaration_version"]), - ), + ).mappings() + for row in rows: + identity = (str(row["memory_artifact_id"]), str(row["entry_id"]), str(row["entry_version_id"])) + snapshots.setdefault(identity, []).append( + MemoryEvidenceSnapshot( + source=SourceRef(source_type=str(row["source_type"]), source_id=str(row["source_id"])), + declaration=MemoryEvidenceDeclaration( + authority=MemoryEvidenceAuthority(str(row["authority"])), + verification=MemoryEvidenceVerification(str(row["verification"])), + declaration_version=str(row["declaration_version"]), + ), + ) ) - ) hydrated: list[MemoryEntryVersion] = [] for entry in entries: identity = (entry.memory_artifact_id, entry.entry_id, entry.entry_version_id) diff --git a/tests/builtin/persistence/test_memory.py b/tests/builtin/persistence/test_memory.py index 9b3059a320..f478c9dad9 100644 --- a/tests/builtin/persistence/test_memory.py +++ b/tests/builtin/persistence/test_memory.py @@ -16,11 +16,12 @@ import asyncio -from sqlalchemy import BigInteger, DateTime, Integer, String, delete, select +from sqlalchemy import BigInteger, DateTime, Integer, String, delete, event, select from sqlalchemy.dialects import mysql from sqlalchemy.schema import CreateTable, ForeignKeyConstraint, PrimaryKeyConstraint, UniqueConstraint from powercontext.builtin.artifacts.memory import MemoryEntryInput +from powercontext.builtin.persistence.database import SELECTION_BATCH_SIZE from powercontext.builtin.persistence.memory import RelationalMemoryBackend from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.persistence.tables import ( @@ -34,6 +35,7 @@ from powercontext.builtin.sources import ContentCapture, ContentSource _INNODB_MAX_INDEX_BYTES = 3072 +_EVIDENCE_BUDGET_ENTRIES = SELECTION_BATCH_SIZE class _UnbudgetedColumnTypeError(TypeError): @@ -327,6 +329,60 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_manifest_evidence_lookups_stay_within_the_shared_bind_budget() -> None: + """A manifest read must batch evidence lookups instead of binding every entry. + + The ceiling that matters depends on the SQLite build (32,766 on most system + builds, 250,000 in CPython's bundled Windows library), so this pins the bind + budget the backend controls rather than one build's variable limit. + """ + + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + context = await contexts.get("project") + bound: list[int] = [] + + def record_statement( + _connection: object, + _cursor: object, + statement: str, + parameters: object, + *_rest: object, + ) -> None: + if not statement.lstrip().upper().startswith("SELECT"): + return + if "pc_memory_entry_evidence" in statement: + assert isinstance(parameters, tuple | list) + bound.append(len(parameters)) + + engine = contexts.database.engine.sync_engine + event.listen(engine, "before_cursor_execute", record_statement) + try: + opened = await context.artifacts.memory.remember( + memory=None, + entries=tuple( + MemoryEntryInput(kind="agent-note", text=f"Bounded entry {index}.", sources=()) + for index in range(_EVIDENCE_BUDGET_ENTRIES) + ), + mode="append", + ) + assert opened is not None + appended = await context.artifacts.memory.remember( + memory=opened, + entries=(MemoryEntryInput(kind="agent-note", text="Appended within budget.", sources=()),), + mode="append", + ) + assert appended is not None + assert len(await context.artifacts.memory.entries(appended)) == _EVIDENCE_BUDGET_ENTRIES + 1 + finally: + event.remove(engine, "before_cursor_execute", record_statement) + + assert bound + assert max(bound) <= SELECTION_BATCH_SIZE + + asyncio.run(scenario()) + + def test_scope_bound_contexts_do_not_share_rows() -> None: async def scenario() -> None: async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: From fad033087d1e075b04231c44e089503a373cae31 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Tue, 29 Sep 2026 11:16:12 +0800 Subject: [PATCH 07/30] fix(memory): keep the evidence declaration out of Definition identity ... --- docs/en/docs/operate/troubleshoot.md | 18 ++ docs/zh/docs/operate/troubleshoot.md | 15 ++ .../builtin/artifacts/memory/models.py | 17 +- .../builtin/persistence/memory.py | 162 ++++++++++-------- .../builtin/persistence/tables.py | 6 + .../builtin/runtime/relational.py | 25 ++- src/powercontext/sources/__init__.py | 2 + src/powercontext/sources/catalog.py | 4 +- src/powercontext/sources/definitions.py | 20 ++- src/powercontext/sources/observations.py | 50 ++++-- tests/builtin/persistence/test_memory.py | 110 ++++++++++-- tests/builtin/persistence/test_sources.py | 96 ++++++++++- tests/test_source_observations.py | 58 ++++++- 13 files changed, 475 insertions(+), 108 deletions(-) diff --git a/docs/en/docs/operate/troubleshoot.md b/docs/en/docs/operate/troubleshoot.md index abf4857c44..a29f31c6ac 100644 --- a/docs/en/docs/operate/troubleshoot.md +++ b/docs/en/docs/operate/troubleshoot.md @@ -310,6 +310,24 @@ so the previous database remains available for recovery: If records were previously merged because the old collation considered their identities equal, changing the schema cannot reconstruct them. Resolve those records from an authoritative source before accepting writes. +## A remote Source Definition stops registering + +A Source Definition manifest is immutable and keyed by its `name` and `version`. Its `fingerprint` covers exactly the +fields the manifest was emitted with, so a Definition that is emitted differently for the same `name` and `version` +no longer matches the registration already in the database. A worker re-registers its Definition on every run, which +makes an upgrade the ordinary way to hit this: + +- Registering the Definition returns HTTP 409 `source_conflict`. +- Submitting an observation whose `definition_fingerprint` no longer matches returns HTTP 422 + `invalid_source_ingestion`. + +Register the changed Definition under a new `version`, then restart the worker so it submits observations against the +new manifest. Do not edit a stored manifest in place, and do not reuse a `version` with different content. + +A declaration that carries no ranking preference — an absent `memory_evidence` or the neutral default — is excluded +from the fingerprint, so upgrading across the introduction of that field does not change any Definition's identity. +Only a declaration that asserts an authority or verification level changes it. + ## An inference readiness check fails When generation or embedding is configured, Server readiness makes one minimal real provider request. This catches diff --git a/docs/zh/docs/operate/troubleshoot.md b/docs/zh/docs/operate/troubleshoot.md index 1b6a8e5c68..6d00d1d969 100644 --- a/docs/zh/docs/operate/troubleshoot.md +++ b/docs/zh/docs/operate/troubleshoot.md @@ -298,6 +298,21 @@ collation，但不会包含数据库 URL 或凭据。 如果旧 collation 曾因 identity 相等而合并记录，重建 schema 无法恢复这些记录。接受新的写入之前，请从权威数据源 修复它们。 +## 远程 Source Definition 无法注册 + +Source Definition manifest 不可变，并以 `name` 和 `version` 作为键。它的 `fingerprint` 只覆盖 manifest 发出时 +实际携带的字段，因此同一个 `name` 和 `version` 若发出内容不同，就无法匹配数据库中已存在的注册记录。worker +每次运行都会重新注册自己的 Definition，所以升级是最常见的触发方式： + +- 注册该 Definition 返回 HTTP 409 `source_conflict`。 +- 提交的 observation 其 `definition_fingerprint` 不再匹配时，返回 HTTP 422 `invalid_source_ingestion`。 + +请用新的 `version` 注册变更后的 Definition，然后重启 worker，让它基于新的 manifest 提交 observation。不要就地 +修改已存储的 manifest，也不要用同一个 `version` 承载不同内容。 + +不带任何排序偏好的声明——缺失的 `memory_evidence` 或中性默认值——不参与 fingerprint 计算，因此跨越该字段引入的 +升级不会改变任何 Definition 的 identity。只有真正声明了 authority 或 verification 等级的声明才会改变它。 + ## 推理服务 readiness 检查失败 配置 generation 或 embedding 后，Server readiness 会向 provider 发起一次最小化真实请求。这样可以发现只有 diff --git a/src/powercontext/builtin/artifacts/memory/models.py b/src/powercontext/builtin/artifacts/memory/models.py index c7a9a8e88a..d25a8ddfc1 100644 --- a/src/powercontext/builtin/artifacts/memory/models.py +++ b/src/powercontext/builtin/artifacts/memory/models.py @@ -203,7 +203,14 @@ class MemoryEntryVersion(BaseModel): class MemoryLifecycleProjection(BaseModel): - """Rebuildable neutral lifecycle state for one Memory manifest entry.""" + """Rebuildable lifecycle state for one Memory manifest entry. + + Reserved infrastructure for the stage-2 quality policy. The rules are + deterministic so the rows can be rebuilt from the authoritative manifest, but + no production path consumes them yet: the materialized table has no reader + outside the rebuild, and the annotation below is not wired into Context Pack + assembly. + """ memory_ref: ArtifactRef entry_id: str @@ -217,7 +224,7 @@ class MemoryLifecycleProjection(BaseModel): @property def context_annotation(self) -> MemoryContextAnnotation: - """Return the non-public annotation available to Context Pack assembly.""" + """Return the internal annotation reserved for Context Pack assembly.""" return MemoryContextAnnotation( validity=self.validity, @@ -231,7 +238,11 @@ def context_annotation(self) -> MemoryContextAnnotation: class MemoryContextAnnotation(BaseModel): - """Internal-only lifecycle context available to later Context Pack rendering.""" + """Internal lifecycle context reserved for later Context Pack rendering. + + Nothing renders it yet, and it is deliberately absent from the public transport + models. + """ validity: MemoryLifecycleValidity validity_reason: str | None = None diff --git a/src/powercontext/builtin/persistence/memory.py b/src/powercontext/builtin/persistence/memory.py index b30b02d40e..10e1011b0c 100644 --- a/src/powercontext/builtin/persistence/memory.py +++ b/src/powercontext/builtin/persistence/memory.py @@ -16,10 +16,10 @@ from __future__ import annotations -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Iterable, Iterator, Mapping from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass, replace -from typing import Any, ClassVar, Literal, cast +from typing import Any, ClassVar, Literal, TypeVar, cast from pydantic import RootModel from sqlalchemy import delete, insert, select, tuple_ @@ -89,9 +89,37 @@ class _ArtifactRefs(RootModel[tuple[ArtifactRef, ...]]): pass -# An evidence lookup binds three identity columns per entry plus the scope, so it -# chunks at the shared bind budget instead of the shared selection row count. -_EVIDENCE_SELECTION_BATCH_SIZE = max(1, SELECTION_BATCH_SIZE // 3) +# The evidence lookup binds three identity columns per entry, so its rows chunk at +# the scope's share of the shared bind budget. The single-column entry-version +# lookups bind one value per row and chunk at the row budget less the fixed +# parameters that accompany the batch: the scope, the artifact identity, and the +# correlated tag predicate, which binds about two values per requested label. +# ``TagFilter`` admits at most 16 labels, so the reserve covers that maximum with +# headroom instead of landing exactly on the budget. +_EVIDENCE_BIND_RESERVE = 1 +_ENTRY_BIND_RESERVE = 40 +_ENTRY_SELECTION_BATCH_SIZE = max(1, SELECTION_BATCH_SIZE - _ENTRY_BIND_RESERVE) +_EVIDENCE_SELECTION_BATCH_SIZE = max(1, (SELECTION_BATCH_SIZE - _EVIDENCE_BIND_RESERVE) // 3) + +_BatchItem = TypeVar("_BatchItem") + + +def _batched(items: Iterable[_BatchItem], size: int, /) -> Iterator[tuple[_BatchItem, ...]]: + """Yield disjoint chunks of at most ``size`` items. + + One statement may bind only a bounded number of values, so a lookup over an + unbounded manifest has to be chunked. Sharing this helper keeps every lookup + on the same budget instead of letting them drift apart. + """ + + batch: list[_BatchItem] = [] + for item in items: + batch.append(item) + if len(batch) >= size: + yield tuple(batch) + batch = [] + if batch: + yield tuple(batch) class _MemoryDraft(ArtifactDraft[MemoryContent]): @@ -178,23 +206,26 @@ async def tagged_entry_ids(self, memory: ArtifactRef, tag_filter: TagFilter) -> canonical = await self.get(memory) versions = tuple(entry.entry_version_id for entry in canonical.content.manifest.entries) table = MEMORY_ENTRY_VERSIONS_TABLE + tagged: set[str] = set() async with self._database.connection(self._bound_connection) as connection: - rows = await connection.scalars( - select(table.c.entry_id).where( - table.c.scope_id == self._scope_id, - table.c.memory_artifact_id == memory.artifact_id, - table.c.entry_version_id.in_(versions), - tag_predicate( - self._scope_id, - "memory", - table.c.memory_artifact_id, - "memory_entry", - table.c.entry_id, - tag_filter, - ), + for batch in _batched(dict.fromkeys(versions), _ENTRY_SELECTION_BATCH_SIZE): + rows = await connection.scalars( + select(table.c.entry_id).where( + table.c.scope_id == self._scope_id, + table.c.memory_artifact_id == memory.artifact_id, + table.c.entry_version_id.in_(batch), + tag_predicate( + self._scope_id, + "memory", + table.c.memory_artifact_id, + "memory_entry", + table.c.entry_id, + tag_filter, + ), + ) ) - ) - return frozenset(rows) + tagged.update(rows) + return frozenset(tagged) async def any_tagged_entry_ids(self, memory: ArtifactRef, /) -> frozenset[str]: await self.get(memory) @@ -219,20 +250,7 @@ async def entries(self, memory: ArtifactRef, /) -> tuple[MemoryEntryVersion, ... if not version_ids: return () async with self._database.connection(self._bound_connection) as connection: - rows = ( - await connection.execute( - select(MEMORY_ENTRY_VERSIONS_TABLE).where( - MEMORY_ENTRY_VERSIONS_TABLE.c.scope_id == self._scope_id, - MEMORY_ENTRY_VERSIONS_TABLE.c.memory_artifact_id == memory.artifact_id, - MEMORY_ENTRY_VERSIONS_TABLE.c.entry_version_id.in_(version_ids), - ) - ) - ).mappings() - versions = await self._hydrate_source_evidence(connection, tuple(_decode_entry(row) for row in rows)) - by_id = {version.entry_version_id: version for version in versions} - if set(by_id) != set(version_ids): - raise InvalidMemoryCitationError("missing-version") - return tuple(by_id[version_id] for version_id in version_ids) + return await self._hydrate_entry_versions(connection, memory, version_ids) async def lifecycle_projections(self, memory: ArtifactRef, /) -> tuple[MemoryLifecycleProjection, ...]: """Load the internal neutral lifecycle projection for one exact Memory head. @@ -247,16 +265,7 @@ async def lifecycle_projections(self, memory: ArtifactRef, /) -> tuple[MemoryLif if not version_ids: return () async with self._database.connection(self._bound_connection) as connection: - entry_rows = ( - await connection.execute( - select(MEMORY_ENTRY_VERSIONS_TABLE).where( - MEMORY_ENTRY_VERSIONS_TABLE.c.scope_id == self._scope_id, - MEMORY_ENTRY_VERSIONS_TABLE.c.memory_artifact_id == memory.artifact_id, - MEMORY_ENTRY_VERSIONS_TABLE.c.entry_version_id.in_(version_ids), - ) - ) - ).mappings() - versions = await self._hydrate_source_evidence(connection, tuple(_decode_entry(row) for row in entry_rows)) + versions = await self._hydrate_entry_versions(connection, memory, version_ids) by_version = {entry.entry_version_id: entry for entry in versions} rows = ( await connection.execute( @@ -454,6 +463,42 @@ async def expand(self, hits: tuple[MemoryHit, ...], /) -> tuple[MemoryEntryVersi expanded.append(_decode_entry(row)) return await self._hydrate_source_evidence(connection, tuple(expanded)) + async def _hydrate_entry_versions( + self, + connection: AsyncConnection, + memory_ref: ArtifactRef, + version_ids: tuple[str, ...], + ) -> tuple[MemoryEntryVersion, ...]: + """Load the named entry versions in request order with their evidence. + + A manifest may name more entries than one statement can bind, so the lookup + chunks at the shared budget. Every caller needs the same completeness + guarantee, so it lives here rather than being restated at each call site. + """ + + if not version_ids: + return () + by_version: dict[str, MemoryEntryVersion] = {} + for batch in _batched(dict.fromkeys(version_ids), _ENTRY_SELECTION_BATCH_SIZE): + rows = ( + await connection.execute( + select(MEMORY_ENTRY_VERSIONS_TABLE).where( + MEMORY_ENTRY_VERSIONS_TABLE.c.scope_id == self._scope_id, + MEMORY_ENTRY_VERSIONS_TABLE.c.memory_artifact_id == memory_ref.artifact_id, + MEMORY_ENTRY_VERSIONS_TABLE.c.entry_version_id.in_(batch), + ) + ) + ).mappings() + hydrated = await self._hydrate_source_evidence(connection, tuple(_decode_entry(row) for row in rows)) + by_version.update({entry.entry_version_id: entry for entry in hydrated}) + entries: list[MemoryEntryVersion] = [] + for version_id in version_ids: + entry = by_version.get(version_id) + if entry is None: + raise InvalidMemoryCitationError("missing-version") + entries.append(entry) + return tuple(entries) + async def _hydrate_source_evidence( self, connection: AsyncConnection, @@ -461,12 +506,11 @@ async def _hydrate_source_evidence( ) -> tuple[MemoryEntryVersion, ...]: if not entries: return () - identities = tuple( - dict.fromkeys((entry.memory_artifact_id, entry.entry_id, entry.entry_version_id) for entry in entries) + identities = dict.fromkeys( + (entry.memory_artifact_id, entry.entry_id, entry.entry_version_id) for entry in entries ) snapshots: dict[tuple[str, str, str], list[MemoryEvidenceSnapshot]] = {} - for start in range(0, len(identities), _EVIDENCE_SELECTION_BATCH_SIZE): - batch = identities[start : start + _EVIDENCE_SELECTION_BATCH_SIZE] + for batch in _batched(identities, _EVIDENCE_SELECTION_BATCH_SIZE): rows = ( await connection.execute( select(MEMORY_ENTRY_EVIDENCE_TABLE) @@ -533,16 +577,7 @@ async def _lifecycle_for_entries( if not manifest: return () version_ids = tuple(item.entry_version_id for item in manifest) - rows = ( - await connection.execute( - select(MEMORY_ENTRY_VERSIONS_TABLE).where( - MEMORY_ENTRY_VERSIONS_TABLE.c.scope_id == self._scope_id, - MEMORY_ENTRY_VERSIONS_TABLE.c.memory_artifact_id == memory.artifact_id, - MEMORY_ENTRY_VERSIONS_TABLE.c.entry_version_id.in_(version_ids), - ) - ) - ).mappings() - versions = await self._hydrate_source_evidence(connection, tuple(_decode_entry(row) for row in rows)) + versions = await self._hydrate_entry_versions(connection, memory.as_ref(), version_ids) by_version = {entry.entry_version_id: entry for entry in versions} lifecycle: list[MemoryLifecycleProjection] = [] for item in manifest: @@ -584,16 +619,7 @@ async def _authoritative_projections( snapshots.append(_RebuildSnapshot(memory_ref=ref, projections=(), lifecycle=lifecycle)) continue version_ids = tuple(item.entry_version_id for item in active) - rows = ( - await connection.execute( - select(MEMORY_ENTRY_VERSIONS_TABLE).where( - MEMORY_ENTRY_VERSIONS_TABLE.c.scope_id == self._scope_id, - MEMORY_ENTRY_VERSIONS_TABLE.c.memory_artifact_id == ref.artifact_id, - MEMORY_ENTRY_VERSIONS_TABLE.c.entry_version_id.in_(version_ids), - ) - ) - ).mappings() - hydrated = await self._hydrate_source_evidence(connection, tuple(_decode_entry(row) for row in rows)) + hydrated = await self._hydrate_entry_versions(connection, ref, version_ids) versions = {entry.entry_version_id: entry for entry in hydrated} projections: list[MemoryProjection] = [] for item in active: diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index 3418f300c2..abb7360807 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -1055,6 +1055,12 @@ def _entry_text_type(): # changes them, like the active-head and vector rows. ``head_revision`` records the # revision that last wrote a row; a read identifies rows by entry and proves # completeness against the authoritative manifest rather than by revision. +# +# The CHECK constraints below describe the stage-1 vocabulary, which is why +# ``quality_policy`` admits only ``neutral``. Stage 2 has to widen them in one DDL +# migration alongside the model literals; the columns are narrow enough that an +# allow-list is the right shape, so the migration is a constraint swap rather than +# a rebuild. MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE = Table( "pc_memory_entry_lifecycle_projections", SHARED_METADATA, diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 223a89e1f8..3d238edad1 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -849,7 +849,7 @@ async def submit_source_observation( ) except RepositoryNotFoundError: raise SourceDefinitionNotFoundError(observation.source_type, observation.definition_version) from None - _validate_source_observation(observation, manifest) + observation = _validate_source_observation(observation, manifest) services = self._services_for(request.scope_id) source_store, source_catalog = services.sources() stored = await source_store.add(observation) @@ -1988,19 +1988,31 @@ def _validate_source_definition_manifest( ) -def _validate_source_observation(source: SourceObservation, manifest: SourceDefinitionManifest) -> None: +def _validate_source_observation(source: SourceObservation, manifest: SourceDefinitionManifest) -> SourceObservation: + """Validate one observation and stamp the Definition-owned evidence declaration. + + The declaration is Definition-owned metadata, so the registered manifest is + authoritative for it: an observation cannot manufacture a stronger attestation + than the Definition it was produced by. A remote worker does not necessarily + read the registered manifest, so a declaration that contradicts it is rejected + rather than silently rewritten. An omitted field resolves to the manifest's + declaration instead, which is what makes the check impossible to bypass by + leaving the field out. + """ + if source.source_type != manifest.name or source.definition_version != manifest.version: raise InvalidSourceObservationError("definition", "does not match the registered manifest identity") if source.definition_fingerprint != manifest.fingerprint: raise InvalidSourceObservationError("fingerprint", "does not match the registered manifest") if "memory_evidence" in source.__pydantic_fields_set__ and source.memory_evidence != manifest.memory_evidence: raise InvalidSourceObservationError("memory_evidence", "does not match the registered manifest") - if len(source.model_dump_json().encode()) > MAX_SOURCE_OBSERVATION_BYTES: + stamped = source.model_copy(update={"memory_evidence": manifest.memory_evidence}) + if len(stamped.model_dump_json().encode()) > MAX_SOURCE_OBSERVATION_BYTES: raise InvalidSourceObservationError("size", "must not exceed 4 MiB") - _validate_schema_value(manifest.name, manifest.source_schema, source.payload) + _validate_schema_value(manifest.name, manifest.source_schema, stamped.payload) declarations = {projection.key: projection for projection in manifest.projections} - supplied = {projection.key: projection.value for projection in source.projections} + supplied = {projection.key: projection.value for projection in stamped.projections} if declarations.keys() != supplied.keys(): raise InvalidSourceObservationError("projections", "must exactly match the registered manifest") for key, declaration in declarations.items(): @@ -2008,11 +2020,12 @@ def _validate_source_observation(source: SourceObservation, manifest: SourceDefi _validate_schema_value(key.name, declaration.schema_, value) if key == TEXT_EVIDENCE_PROJECTION_KEY: evidence = TextEvidence.model_validate(value) - if evidence.source_type != source.source_type or evidence.source_id != source.name: + if evidence.source_type != stamped.source_type or evidence.source_id != stamped.name: raise InvalidSourceObservationError( "text-evidence", "source identity does not match the observation envelope", ) + return stamped def _json_schema_validator(name: str, schema: Mapping[str, Any]) -> Validator: diff --git a/src/powercontext/sources/__init__.py b/src/powercontext/sources/__init__.py index 25f7663d59..2408c7b64d 100644 --- a/src/powercontext/sources/__init__.py +++ b/src/powercontext/sources/__init__.py @@ -29,6 +29,7 @@ SourceDefinition, SourceDefinitionRegistry, SourceProjection, + definition_memory_evidence, ) from powercontext.sources.models import ( MemoryEvidenceAuthority, @@ -80,6 +81,7 @@ "SourceRef", "SourceStore", "TextEvidence", + "definition_memory_evidence", "manifest_for_definition", "project_source_for_transport", ] diff --git a/src/powercontext/sources/catalog.py b/src/powercontext/sources/catalog.py index 4b6f68d011..8207301d1d 100644 --- a/src/powercontext/sources/catalog.py +++ b/src/powercontext/sources/catalog.py @@ -23,7 +23,7 @@ SourceNotFoundError, ) from powercontext.sources.adapters import SourceAdapter -from powercontext.sources.definitions import SourceDefinitionRegistry +from powercontext.sources.definitions import SourceDefinitionRegistry, definition_memory_evidence from powercontext.sources.models import MemoryEvidenceDeclaration, Source, SourceProjectionKey, SourceRef from powercontext.sources.observations import SourceObservation from powercontext.sources.protocols import SourceCatalogBackend @@ -78,7 +78,7 @@ def materialize(self, source: Source, /) -> Source: if isinstance(source, SourceObservation): return source definition = self._registry.definition_for_source(source) - return source.model_copy(update={"memory_evidence": definition.memory_evidence}) + return source.model_copy(update={"memory_evidence": definition_memory_evidence(definition)}) async def resolve(self, value: object, /) -> Source: return await self._registry.resolve(value) diff --git a/src/powercontext/sources/definitions.py b/src/powercontext/sources/definitions.py index e5e8c009ed..dcb732f58a 100644 --- a/src/powercontext/sources/definitions.py +++ b/src/powercontext/sources/definitions.py @@ -173,7 +173,7 @@ async def resolve(self, value: object, /) -> Source: if type(source) is not definition.source_class: raise InvalidSourceResultError(definition.name, "resolve", definition.source_class, type(source)) self.definition_for_source(source) - return cast(Source, source.model_copy(update={"memory_evidence": definition.memory_evidence})) + return cast(Source, source.model_copy(update={"memory_evidence": definition_memory_evidence(definition)})) async def read(self, source: Source, /) -> object: definition = self.definition_for_source(source) @@ -215,6 +215,18 @@ def memory_evidence(self, source: Source, /) -> MemoryEvidenceDeclaration: return source.memory_evidence +def definition_memory_evidence(definition: object, /) -> MemoryEvidenceDeclaration: + """Return a Definition's evidence declaration, or the neutral one when absent. + + Adapters built against the contract before the declaration existed do not carry + the attribute at all. Resolving that case here keeps every consumer of the + declaration on one answer instead of requiring the caller to re-check it. + """ + + declared = getattr(definition, "memory_evidence", None) + return declared if isinstance(declared, MemoryEvidenceDeclaration) else MemoryEvidenceDeclaration() + + def _validate_definition(definition: object) -> tuple[type[object], type[Source]]: definition_type = type(definition) input_class = getattr(definition, "input_class", None) @@ -233,6 +245,12 @@ def _validate_definition(definition: object) -> tuple[type[object], type[Source] if not isinstance(projections, tuple): raise InvalidSourceDefinitionError(definition_type, "projections", "must be a tuple") memory_evidence = getattr(definition, "memory_evidence", None) + if memory_evidence is None: + # An adapter built against the contract before the declaration existed owns + # no evidence metadata, which is exactly the neutral declaration. Tolerating + # the missing attribute keeps that adapter registering, mirroring how the + # manifest and observation paths accept an absent declaration. + memory_evidence = MemoryEvidenceDeclaration() if not isinstance(memory_evidence, MemoryEvidenceDeclaration): raise InvalidSourceDefinitionError( definition_type, diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 0cdc5a9258..f6dd2b10dd 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -38,7 +38,11 @@ InvalidSourceProjectionError, ) from powercontext.limits import MAX_SOURCE_TYPE_LENGTH -from powercontext.sources.definitions import SourceDefinition, SourceDefinitionRegistry +from powercontext.sources.definitions import ( + SourceDefinition, + SourceDefinitionRegistry, + definition_memory_evidence, +) from powercontext.sources.models import MemoryEvidenceDeclaration, Source, SourceMaterialization, SourceProjectionKey _JSON_VALUE = TypeAdapter(JsonValue) @@ -117,10 +121,7 @@ def validate_manifest(self) -> SourceDefinitionManifest: version=self.version, source_schema=self.source_schema, projections=self.projections, - # Accept persisted and remote manifests emitted before the evidence - # declaration existed. They resolve to the neutral declaration, but - # retain their historical content-addressed identity. - memory_evidence=self.memory_evidence if "memory_evidence" in self.__pydantic_fields_set__ else None, + memory_evidence=self.memory_evidence, ) if self.fingerprint != expected: raise ValueError("manifest fingerprint does not match its declaration") # noqa: TRY003 @@ -128,16 +129,16 @@ def validate_manifest(self) -> SourceDefinitionManifest: @model_serializer(mode="wrap") def serialize_manifest(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: - """Preserve the field set the fingerprint actually covers. + """Serialize the manifest in the canonical form its fingerprint covers. - A manifest received without a declaration keeps its historical identity, so - it must also serialize without that field. Writing the model default would - make the stored payload validate against a different fingerprint than the - one it was registered with, which makes registration unreadable. + A manifest whose declaration is absent or neutral carries no ranking + preference, so it must serialize without that field. Writing the neutral + default would make the stored payload validate against a fingerprint that + excludes it, which makes registration unreadable. """ dumped = handler(self) - if "memory_evidence" not in self.__pydantic_fields_set__: + if _declared_evidence(self.memory_evidence) is None: dumped.pop("memory_evidence", None) return dumped @@ -194,6 +195,7 @@ def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> S """Build the immutable declaration transported by a remote worker.""" source_schema = _json_object(definition.source_class.model_json_schema()) + declared = definition_memory_evidence(definition) projections = tuple( SourceProjectionManifest( key=SourceProjectionKey(name=projection.name, version=projection.version), @@ -209,11 +211,11 @@ def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> S version=definition.version, source_schema=source_schema, projections=projections, - memory_evidence=definition.memory_evidence, + memory_evidence=declared, ), source_schema=source_schema, projections=projections, - memory_evidence=definition.memory_evidence, + memory_evidence=declared, ) @@ -248,6 +250,26 @@ def project_source_for_transport( ) +def _declared_evidence( + memory_evidence: MemoryEvidenceDeclaration | None, +) -> dict[str, JsonValue] | None: + """Return the evidence fields a fingerprint covers, or ``None`` when neutral. + + The neutral declaration is Definition-owned metadata that carries no ranking + preference, so it must not contribute to a Definition's content-addressed + identity. Folding it in would change the fingerprint of every Definition that + already registered one, and the immutable ``(name, version)`` registration + would then reject the re-registration every remote worker performs on each + run. A declaration that actually asserts an authority or verification level + still changes the fingerprint, because it does change what the Definition + attests. + """ + + if memory_evidence is None or memory_evidence == MemoryEvidenceDeclaration(): + return None + return {"memory_evidence": memory_evidence.model_dump(mode="json")} + + def _source_definition_fingerprint( *, name: str, @@ -261,7 +283,7 @@ def _source_definition_fingerprint( "version": version, "source_schema": source_schema, "projections": [projection.model_dump(mode="json", by_alias=True) for projection in projections], - **({} if memory_evidence is None else {"memory_evidence": memory_evidence.model_dump(mode="json")}), + **(_declared_evidence(memory_evidence) or {}), } encoded = rfc8785.dumps(_JSON_VALUE.validate_python(declaration)) return f"sha256:{hashlib.sha256(encoded).hexdigest()}" diff --git a/tests/builtin/persistence/test_memory.py b/tests/builtin/persistence/test_memory.py index f478c9dad9..4b2625b75a 100644 --- a/tests/builtin/persistence/test_memory.py +++ b/tests/builtin/persistence/test_memory.py @@ -16,26 +16,38 @@ import asyncio +import pytest +from pydantic import ValidationError from sqlalchemy import BigInteger, DateTime, Integer, String, delete, event, select from sqlalchemy.dialects import mysql +from sqlalchemy.exc import IntegrityError from sqlalchemy.schema import CreateTable, ForeignKeyConstraint, PrimaryKeyConstraint, UniqueConstraint from powercontext.builtin.artifacts.memory import MemoryEntryInput from powercontext.builtin.persistence.database import SELECTION_BATCH_SIZE -from powercontext.builtin.persistence.memory import RelationalMemoryBackend +from powercontext.builtin.persistence.memory import ( + _ENTRY_BIND_RESERVE, + _ENTRY_SELECTION_BATCH_SIZE, + _EVIDENCE_BIND_RESERVE, + _EVIDENCE_SELECTION_BATCH_SIZE, + RelationalMemoryBackend, +) from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.persistence.tables import ( MEMORY_ENTRY_EVIDENCE_TABLE, MEMORY_ENTRY_HEADS_TABLE, MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE, MEMORY_ENTRY_VERSIONS_TABLE, + SOURCES_TABLE, ) from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.builtin.sources import ContentCapture, ContentSource +from powercontext.builtin.tags import TagFilter _INNODB_MAX_INDEX_BYTES = 3072 _EVIDENCE_BUDGET_ENTRIES = SELECTION_BATCH_SIZE +_MAXIMUM_TAG_LABELS = 16 class _UnbudgetedColumnTypeError(TypeError): @@ -329,12 +341,14 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_manifest_evidence_lookups_stay_within_the_shared_bind_budget() -> None: - """A manifest read must batch evidence lookups instead of binding every entry. +def test_manifest_lookups_stay_within_the_shared_bind_budget() -> None: + """Every manifest-sized lookup must chunk instead of binding every entry. - The ceiling that matters depends on the SQLite build (32,766 on most system - builds, 250,000 in CPython's bundled Windows library), so this pins the bind - budget the backend controls rather than one build's variable limit. + The ceiling on bound values depends on the SQLite build, so this pins the bind + budget the backend controls rather than one build's variable limit. The budget + is asserted as the invariant each statement has to satisfy, not as the value + the current constants happen to produce, so tuning a batch size cannot make + the check pass or fail by itself. """ async def scenario() -> None: @@ -351,9 +365,8 @@ def record_statement( ) -> None: if not statement.lstrip().upper().startswith("SELECT"): return - if "pc_memory_entry_evidence" in statement: - assert isinstance(parameters, tuple | list) - bound.append(len(parameters)) + assert isinstance(parameters, tuple | list) + bound.append(len(parameters)) engine = contexts.database.engine.sync_engine event.listen(engine, "before_cursor_execute", record_statement) @@ -373,16 +386,91 @@ def record_statement( mode="append", ) assert appended is not None - assert len(await context.artifacts.memory.entries(appended)) == _EVIDENCE_BUDGET_ENTRIES + 1 + + expected = _EVIDENCE_BUDGET_ENTRIES + 1 + # Every read path that resolves a manifest must stay within budget. + # The lifecycle projection is internal and has no service caller yet, + # so it is exercised on the backend that owns the lookup. + backend = RelationalMemoryBackend( + database=contexts.database, + scope_id="project", + artifacts=contexts.repositories.artifacts, + index=contexts.index, + ) + assert len(await context.artifacts.memory.entries(appended)) == expected + assert len(await backend.lifecycle_projections(appended.as_ref())) == expected + # A tag-filtered read also binds its correlated predicate, so + # exercise the widest filter the contract admits. + widest = TagFilter(tags=tuple(f"label-{index}" for index in range(_MAXIMUM_TAG_LABELS))) + assert await backend.tagged_entry_ids(appended.as_ref(), widest) == frozenset() + assert await backend.tagged_entry_ids(appended.as_ref(), TagFilter(tags=("absent",))) == frozenset() + await context.artifacts.memory.rebuild_projections() finally: event.remove(engine, "before_cursor_execute", record_statement) + # The evidence lookup binds three identity columns per entry; every + # other lookup binds one value per entry. Each chunk plus the statement's + # fixed parameters has to fit the shared budget, so assert that invariant + # rather than the value the current constants happen to produce. assert bound - assert max(bound) <= SELECTION_BATCH_SIZE + limit = max( + 3 * _EVIDENCE_SELECTION_BATCH_SIZE + _EVIDENCE_BIND_RESERVE, + _ENTRY_SELECTION_BATCH_SIZE + _ENTRY_BIND_RESERVE, + ) + assert limit <= SELECTION_BATCH_SIZE + assert max(bound) <= limit asyncio.run(scenario()) +def test_registered_source_cannot_be_deleted_while_memory_evidence_cites_it() -> None: + """The evidence foreign key protects a cited Source from removal. + + Any future retention or erasure implementation has to remove the citing + evidence rows first, so the constraint is pinned here rather than left implicit. + """ + + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + context = await contexts.get("project") + source, _ = await context.sources.capture( + ContentCapture(source_id="turn-1", content="A cited Source cannot simply vanish.") + ) + opened = await context.artifacts.memory.remember( + memory=None, + sources=(source,), + entries=(MemoryEntryInput(kind="fact", text="Cite the Source.", sources=(source,)),), + mode="append", + ) + assert opened is not None + + async with contexts.database.transaction() as connection: + with pytest.raises(IntegrityError): + await connection.execute( + delete(SOURCES_TABLE).where( + SOURCES_TABLE.c.scope_id == "project", + SOURCES_TABLE.c.source_id == "turn-1", + ) + ) + + asyncio.run(scenario()) + + +def test_tag_filter_label_ceiling_matches_the_entry_bind_reserve() -> None: + """The entry bind reserve is sized for the widest tag filter the contract admits. + + A tag-filtered entry lookup binds its correlated predicate on top of the batch, + so the reserve has to cover the maximum label count. If that ceiling ever rises, + the reserve has to rise with it, and this pins the pair together. + """ + + accepted = TagFilter(tags=tuple(f"label-{index}" for index in range(_MAXIMUM_TAG_LABELS))) + assert len(accepted.keys) == _MAXIMUM_TAG_LABELS + with pytest.raises(ValidationError, match="invalid number of labels"): + TagFilter(tags=tuple(f"label-{index}" for index in range(_MAXIMUM_TAG_LABELS + 1))) + assert _ENTRY_BIND_RESERVE >= 2 * _MAXIMUM_TAG_LABELS + 2 + + def test_scope_bound_contexts_do_not_share_rows() -> None: async def scenario() -> None: async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index ed8e387a83..edf9dd60e7 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -19,7 +19,7 @@ import pytest from sqlalchemy import insert -from powercontext import InvalidSourceDefinitionError +from powercontext import InvalidSourceDefinitionError, InvalidSourceObservationError from powercontext.builtin.persistence import ( InvalidRepositoryArgumentError, RepositoryNotFoundError, @@ -32,6 +32,9 @@ from powercontext.builtin.runtime.models import SubmitSourceObservation from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.sources import ( + MemoryEvidenceAuthority, + MemoryEvidenceDeclaration, + MemoryEvidenceVerification, SourceDefinitionManifest, SourceDefinitionRegistry, SourceMaterialization, @@ -40,10 +43,12 @@ manifest_for_definition, project_source_for_transport, ) +from powercontext.sources.definitions import AdapterSourceDefinition from powercontext.sources.observations import _source_definition_fingerprint from tests.builtin.persistence.contract import ( SOURCE_ADAPTERS, CommitSource, + NoteAdapter, NoteInput, NoteSource, repository_profile, @@ -228,6 +233,95 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_neutral_declaration_keeps_a_registered_definition_re_registrable() -> None: + """A neutral declaration must not change a Definition's content-addressed identity. + + A deployed worker re-registers its Definition manifest on every run against an + immutable ``(name, version)`` key. Definitions registered before the evidence + declaration existed carry the same neutral meaning, so the fingerprint the + worker emits after an upgrade has to stay identical. + """ + + async def scenario() -> None: + worker_registry = SourceDefinitionRegistry.from_adapters(SOURCE_ADAPTERS) + definition = worker_registry.definition_for_name("note") + manifest = manifest_for_definition(definition) + + # A worker that omits the declaration must emit a manifest that is + # indistinguishable from the neutral default, and one whose fingerprint + # matches the historical, pre-declaration identity. + assert _legacy_manifest(manifest).fingerprint == manifest.fingerprint + neutral = SourceDefinitionManifest.model_validate( + manifest.model_dump(mode="json") | {"memory_evidence": MemoryEvidenceDeclaration().model_dump(mode="json")} + ) + assert neutral.fingerprint == manifest.fingerprint + + async with repository_profile() as (profile, _repositories): + contexts = RelationalContexts(database=profile.database) + first = await contexts.register_source_definition(manifest) + # The upgraded worker re-registers the same Definition on its next run. + second = await contexts.register_source_definition(manifest_for_definition(definition)) + + assert first == manifest + assert second == first + + asyncio.run(scenario()) + + +def test_source_observation_cannot_contradict_the_registered_declaration() -> None: + """The registered manifest owns the evidence declaration, and a contradiction is rejected. + + The declaration reaches storage only from the Definition-owned manifest. An + observation that omits the field is stamped from the manifest, and one that + asserts a different authority is rejected instead of silently downgraded or + upgraded. + """ + + async def scenario() -> None: + declared = MemoryEvidenceDeclaration( + authority=MemoryEvidenceAuthority.SYSTEM_ATTESTED, + verification=MemoryEvidenceVerification.VERIFIED, + ) + definition = AdapterSourceDefinition(NoteAdapter(), memory_evidence=declared) + worker_registry = SourceDefinitionRegistry((definition,)) + manifest = manifest_for_definition(definition) + assert manifest.memory_evidence == declared + + source = await worker_registry.resolve(NoteInput(note_id="note-1", body="Server owns the declaration.")) + projected = project_source_for_transport(worker_registry, source) + + async with repository_profile() as (profile, repositories): + # The remote Definition must become the active one for this server. + contexts = RelationalContexts( + database=profile.database, + source_registry=SourceDefinitionRegistry(()), + ) + await contexts.register_source_definition(manifest) + + # A worker that omits the declaration is accepted; the manifest supplies it. + omitted = SourceObservation.model_validate(projected.model_dump(mode="json", exclude={"memory_evidence"})) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=omitted) + ) + async with profile.database.transaction() as connection: + stored = await repositories.sources.get(connection, "scope-a", receipt.source_ref) + assert isinstance(stored.value, SourceObservation) + assert stored.value.memory_evidence == declared + + # A worker that asserts a different authority cannot override the Definition. + contradicted = SourceObservation.model_validate( + projected.model_dump(mode="json") + | {"memory_evidence": MemoryEvidenceDeclaration().model_dump(mode="json")} + ) + with pytest.raises(InvalidSourceObservationError) as error: + await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=contradicted) + ) + assert error.value.issue == "memory_evidence" + + asyncio.run(scenario()) + + def test_stored_observation_representation_does_not_depend_on_the_active_registry() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry.from_adapters(SOURCE_ADAPTERS) diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 835420432b..9b0d3d2e8d 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -19,6 +19,7 @@ import json import urllib.request from dataclasses import replace +from typing import Any, cast import pytest import rfc8785 @@ -26,7 +27,7 @@ from referencing.exceptions import Unresolvable from powercontext.builtin.runtime.relational import _json_schema_validator, _validate_source_observation -from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION, ContentCapture +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION, ContentCapture, ContentSource from powercontext.http import SourceDefinitionManifest as HttpSourceDefinitionManifest from powercontext.http import SourceObservation as HttpSourceObservation from powercontext.http import SubmitSourceObservationRequest @@ -38,10 +39,13 @@ MemoryEvidenceVerification, Source, SourceCatalog, + SourceDefinition, SourceDefinitionManifest, SourceDefinitionRegistry, + SourceMaterialization, SourceObservation, TextEvidence, + definition_memory_evidence, manifest_for_definition, project_source_for_transport, ) @@ -73,6 +77,43 @@ def _legacy_manifest(current: SourceDefinitionManifest) -> SourceDefinitionManif ) +def test_definition_without_a_declaration_attribute_still_registers() -> None: + """An adapter written before the declaration existed must keep registering. + + The attribute is part of the Definition contract now, but an out-of-tree + adapter built directly on the protocol does not carry it. Resolving that to the + neutral declaration keeps such an adapter usable instead of failing its + registration. + """ + + class LegacyDefinition: + name = "legacy" + version = "1" + projections: tuple[object, ...] = () + input_class = ContentCapture + source_class = ContentSource + + async def resolve(self, value: ContentCapture, /) -> ContentSource: + return ContentSource( + name=value.source_id, + materialization=SourceMaterialization.CAPTURED, + content=value.content, + ) + + async def read(self, source: ContentSource, /) -> str: + return source.content + + definition = LegacyDefinition() + # Deliberately outside the current protocol: the attribute does not exist yet, + # which is what the runtime has to tolerate. + manifest = manifest_for_definition(cast(SourceDefinition[Any, Any, Any], definition)) + + assert definition_memory_evidence(definition) == MemoryEvidenceDeclaration() + assert manifest.memory_evidence == MemoryEvidenceDeclaration() + # A neutral declaration keeps the manifest at its pre-declaration identity. + assert manifest.fingerprint == _legacy_manifest(manifest).fingerprint + + def test_definition_manifest_has_a_stable_content_addressed_identity() -> None: first = manifest_for_definition(CONTENT_SOURCE_DEFINITION) second = manifest_for_definition(CONTENT_SOURCE_DEFINITION) @@ -154,7 +195,20 @@ def test_legacy_definition_manifest_round_trips_without_gaining_the_evidence_fie assert transported.memory_evidence is None assert "memory_evidence" not in json.loads(legacy.model_dump_json(by_alias=True)) assert SourceDefinitionManifest.model_validate_json(current.model_dump_json(by_alias=True)) == current - assert "memory_evidence" in current.model_dump(mode="json", by_alias=True) + + # A neutral declaration carries no ranking preference, so it does not change + # the manifest a Definition emits and can be omitted from the transport. + assert current == legacy + assert "memory_evidence" not in current.model_dump(mode="json", by_alias=True) + + declared = manifest_for_definition( + replace( + CONTENT_SOURCE_DEFINITION, + memory_evidence=MemoryEvidenceDeclaration(authority=MemoryEvidenceAuthority.SYSTEM_ATTESTED), + ) + ) + assert declared != current + assert "memory_evidence" in declared.model_dump(mode="json", by_alias=True) def test_source_observation_request_mapping_preserves_null_projection_values() -> None: From cc290a5347efba867fa0cd29f8a2027ca3cfe7ed Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Tue, 29 Sep 2026 12:10:36 +0800 Subject: [PATCH 08/30] fix(ci): restore the connector manifest round trip and the commit complexity budget Comparing the worker's manifest against the registration response compared two encodings of the same Definition: the transport model carries the evidence declaration as an optional value, so the response states it explicitly while the worker's manifest omits a neutral declaration. Normalize both sides before comparing, so a neutral declaration does not read as a mismatch. The emitted manifest also carries a neutral declaration by omission, matching the form its fingerprint covers and the form restored from storage. Extracting the lifecycle write and the evidence insert keeps _commit within its complexity budget; the lifecycle block this PR introduced had pushed it over. --- .../builtin/persistence/memory.py | 99 ++++++++++++------- src/powercontext/client/ingestion.py | 22 ++++- src/powercontext/sources/observations.py | 6 +- 3 files changed, 87 insertions(+), 40 deletions(-) diff --git a/src/powercontext/builtin/persistence/memory.py b/src/powercontext/builtin/persistence/memory.py index 10e1011b0c..bdf4b3e3af 100644 --- a/src/powercontext/builtin/persistence/memory.py +++ b/src/powercontext/builtin/persistence/memory.py @@ -723,9 +723,7 @@ async def _commit(self, connection: AsyncConnection, value: MemoryCommit) -> Mem insert(MEMORY_ENTRY_VERSIONS_TABLE), [_entry_values(self._scope_id, entry) for entry in value.entry_versions], ) - evidence_rows = [row for entry in value.entry_versions for row in _evidence_values(self._scope_id, entry)] - if evidence_rows: - await connection.execute(insert(MEMORY_ENTRY_EVIDENCE_TABLE), evidence_rows) + await _insert_source_evidence(connection, self._scope_id, value.entry_versions) # Only entries whose pointer or state changed need projection work; the # rest of the active head stays exactly as the previous revision left it. previous_active = ( @@ -776,41 +774,7 @@ async def _commit(self, connection: AsyncConnection, value: MemoryCommit) -> Mem value.memory.as_ref(), upserts, ) - # Lifecycle rows cover every manifest entry, including inactive ones, and - # follow the same per-entry rule as the active head: only the entries whose - # identity or state changed are rewritten, so repeated appends stay bounded - # by the appended entries instead of the manifest size. - previous_entries = ( - {} - if value.base is None - else {item.entry_id: (item.entry_version_id, item.state) for item in value.base.content.manifest.entries} - ) - current_entries = { - item.entry_id: (item.entry_version_id, item.state) for item in value.memory.content.manifest.entries - } - lifecycle_removed = tuple(sorted(entry_id for entry_id in previous_entries if entry_id not in current_entries)) - lifecycle_changed = tuple( - sorted( - entry_id for entry_id, identity in current_entries.items() if previous_entries.get(entry_id) != identity - ) - ) - lifecycle_drop = tuple(sorted({*lifecycle_removed, *lifecycle_changed})) - if lifecycle_drop: - await connection.execute( - delete(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE).where( - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.scope_id == self._scope_id, - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.memory_artifact_id == value.memory.artifact_id, - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.entry_id.in_(lifecycle_drop), - ) - ) - if lifecycle_changed: - lifecycle = await self._lifecycle_for_entries(connection, committed, lifecycle_changed) - if len(lifecycle) != len(lifecycle_changed): - raise _InvalidMemoryCommitError("lifecycle-projection") - await connection.execute( - insert(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE), - [_lifecycle_values(self._scope_id, projection) for projection in lifecycle], - ) + await _refresh_lifecycle_projections(connection, self, value, committed) return committed @@ -878,6 +842,65 @@ def _entry_values(scope_id: str, value: MemoryEntryVersion) -> dict[str, object] } +async def _refresh_lifecycle_projections( + connection: AsyncConnection, + backend: RelationalMemoryBackend, + value: MemoryCommit, + committed: Memory, +) -> None: + """Rewrite the lifecycle rows whose entry identity or state changed. + + Lifecycle rows cover every manifest entry, including inactive ones, and follow + the same per-entry rule as the active head: only changed entries are rewritten, + so repeated appends stay bounded by the appended entries rather than the + manifest size. + """ + + previous_entries = ( + {} + if value.base is None + else {item.entry_id: (item.entry_version_id, item.state) for item in value.base.content.manifest.entries} + ) + current_entries = { + item.entry_id: (item.entry_version_id, item.state) for item in value.memory.content.manifest.entries + } + removed = tuple(sorted(entry_id for entry_id in previous_entries if entry_id not in current_entries)) + changed = tuple( + sorted(entry_id for entry_id, identity in current_entries.items() if previous_entries.get(entry_id) != identity) + ) + drop = tuple(sorted({*removed, *changed})) + if drop: + await connection.execute( + delete(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE).where( + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.scope_id == backend._scope_id, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.memory_artifact_id == value.memory.artifact_id, + MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.entry_id.in_(drop), + ) + ) + if changed: + lifecycle = await backend._lifecycle_for_entries(connection, committed, changed) + if len(lifecycle) != len(changed): + raise _InvalidMemoryCommitError("lifecycle-projection") + await connection.execute( + insert(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE), + [_lifecycle_values(backend._scope_id, projection) for projection in lifecycle], + ) + + +async def _insert_source_evidence( + connection: AsyncConnection, + scope_id: str, + entry_versions: tuple[MemoryEntryVersion, ...], +) -> None: + """Write the evidence rows for one batch of entry versions, if any carry sources.""" + + rows: list[dict[str, object]] = [] + for entry in entry_versions: + rows.extend(_evidence_values(scope_id, entry)) + if rows: + await connection.execute(insert(MEMORY_ENTRY_EVIDENCE_TABLE), rows) + + def _evidence_values(scope_id: str, value: MemoryEntryVersion) -> tuple[dict[str, object], ...]: return tuple( { diff --git a/src/powercontext/client/ingestion.py b/src/powercontext/client/ingestion.py index 2cd943b4d4..bed1e9a9d6 100644 --- a/src/powercontext/client/ingestion.py +++ b/src/powercontext/client/ingestion.py @@ -41,6 +41,7 @@ Connector, ConnectorBinding, ConnectorRunResult, + MemoryEvidenceDeclaration, SourceDefinitionRegistry, SourceObservation, SourceRef, @@ -132,6 +133,23 @@ async def save( raise InvalidConnectorRunError("checkpoint-mismatch", "Server returned a different Connector checkpoint") +def _without_neutral_evidence(payload: dict[str, JsonValue]) -> dict[str, JsonValue]: + """Drop an absent-or-neutral evidence declaration before comparing a manifest. + + A neutral declaration carries no ranking preference, so a manifest that states + it and a manifest that omits it describe the same Definition. The transport + model carries the field as an optional value, so a server response states it + explicitly while the worker's own manifest omits it. Comparing the canonical + form rather than one encoding keeps that difference from reading as a + mismatch. + """ + + declaration = payload.get("memory_evidence") + if declaration is None or declaration == MemoryEvidenceDeclaration().model_dump(mode="json"): + return {key: value for key, value in payload.items() if key != "memory_evidence"} + return payload + + class RemoteConnectorWorker: """Register worker-owned Definitions and execute one Connector binding.""" @@ -155,7 +173,9 @@ async def run(self, connector: Connector, binding: ConnectorBinding, /) -> Conne ) ) ) - if registered.model_dump(mode="json", by_alias=True) != manifest.model_dump(mode="json", by_alias=True): + if _without_neutral_evidence( + registered.model_dump(mode="json", by_alias=True) + ) != _without_neutral_evidence(manifest.model_dump(mode="json", by_alias=True)): raise InvalidConnectorRunError( "manifest-mismatch", f"Server returned a different manifest for {definition.name!r}", diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index f6dd2b10dd..d3e0609335 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -203,6 +203,10 @@ def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> S ) for projection in definition.projections ) + # A neutral declaration is carried by omission, never as an explicit field. The + # fingerprint excludes it, so setting it would make this manifest serialize + # differently from the same manifest decoded from its stored form, and the + # round trip a remote worker checks would no longer agree. return SourceDefinitionManifest( name=definition.name, version=definition.version, @@ -215,7 +219,7 @@ def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> S ), source_schema=source_schema, projections=projections, - memory_evidence=declared, + **({} if _declared_evidence(declared) is None else {"memory_evidence": declared}), ) From 1b455fbc63d2ebe68a1952be2ab47533afed9b85 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Fri, 2 Oct 2026 11:20:14 +0800 Subject: [PATCH 09/30] fix(memory): keep the declaration out of the Source contract and read the requested revision Adding the evidence declaration to the Source base class also changed every Source class's schema, and the manifest fingerprint covers that schema, so a Definition registered before the field existed no longer matched the manifest a worker emits. The emitted input contract now excludes the declaration and the definitions only it referenced. A lifecycle read also returned the state of whatever revision last wrote a row: deactivating and reactivating an entry keeps its version identity, so an identity-only check accepted a later row under a historical reference. Lifecycle state is now derived from the requested revision's manifest. Version parts are validated at the length the transport and identity columns allow, so a declaration version cannot register locally and then fail conversion. --- .../builtin/persistence/memory.py | 53 ++++++++++++++++--- src/powercontext/limits.py | 1 + src/powercontext/sources/models.py | 18 ++++++- src/powercontext/sources/observations.py | 47 +++++++++++++++- tests/builtin/persistence/test_memory.py | 42 +++++++++++++++ tests/test_source_observations.py | 31 +++++++++++ 6 files changed, 182 insertions(+), 10 deletions(-) diff --git a/src/powercontext/builtin/persistence/memory.py b/src/powercontext/builtin/persistence/memory.py index bdf4b3e3af..d8a93d68b2 100644 --- a/src/powercontext/builtin/persistence/memory.py +++ b/src/powercontext/builtin/persistence/memory.py @@ -261,7 +261,8 @@ async def lifecycle_projections(self, memory: ArtifactRef, /) -> tuple[MemoryLif """ canonical = await self.get(memory) - version_ids = tuple(item.entry_version_id for item in canonical.content.manifest.entries) + manifest = canonical.content.manifest.entries + version_ids = tuple(item.entry_version_id for item in manifest) if not version_ids: return () async with self._database.connection(self._bound_connection) as connection: @@ -277,12 +278,23 @@ async def lifecycle_projections(self, memory: ArtifactRef, /) -> tuple[MemoryLif .order_by(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.entry_id) ) ).mappings() - projections = tuple( - _decode_lifecycle_projection(memory, row, by_version.get(str(row["entry_version_id"]))) for row in rows + by_entry = {str(row["entry_id"]): row for row in rows} + # The manifest owns each entry's state and version, so derive those from the + # requested revision. Deactivating or reactivating an entry keeps its version + # identity, so a row written by a later revision would otherwise pass an + # identity check and report that later state under a historical reference. + projections: list[MemoryLifecycleProjection] = [] + for item in manifest: + row = by_entry.get(item.entry_id) + projections.append( + _lifecycle_projection( + memory, + item.state, + by_version[item.entry_version_id], + row=row, + ) ) - if {item.entry_id for item in projections} != {item.entry_id for item in canonical.content.manifest.entries}: - raise InvalidMemoryCitationError("lifecycle-projection") - return projections + return tuple(projections) async def projections(self, memory: ArtifactRef, /) -> tuple[MemoryProjection, ...]: canonical = await self.get(memory) @@ -989,14 +1001,41 @@ def _lifecycle_projection( memory_ref: ArtifactRef, state: str, entry: MemoryEntryVersion, + *, + row: Mapping[Any, Any] | None = None, ) -> MemoryLifecycleProjection: - validity = "current" if state == "active" else "inactive" + """Derive one lifecycle projection, optionally carrying a persisted row's metadata. + + ``validity``, ``entry_id`` and ``entry_version_id`` always come from the entry, + because the manifest of the requested revision owns them. A persisted row only + contributes the fields the manifest does not carry, and is rejected when it + disagrees about the source count it was written with. + """ + + validity: MemoryLifecycleValidity = "current" if state == "active" else "inactive" + validity_reason: str | None = None + successor_entry_id: str | None = None + rule_version = "memory-lifecycle-v1" + quality_policy: Literal["neutral"] = "neutral" + if row is not None: + if str(row["entry_id"]) != entry.entry_id: + raise InvalidMemoryCitationError("lifecycle-version") + if int(row["source_count"]) != len(entry.source_evidence): + raise InvalidMemoryCitationError("lifecycle-source-count") + validity_reason = None if row["validity_reason"] is None else str(row["validity_reason"]) + successor_entry_id = None if row["successor_entry_id"] is None else str(row["successor_entry_id"]) + rule_version = str(row["rule_version"]) + quality_policy = cast(Literal["neutral"], str(row["quality_policy"])) return MemoryLifecycleProjection( memory_ref=memory_ref, entry_id=entry.entry_id, entry_version_id=entry.entry_version_id, validity=validity, + validity_reason=validity_reason, + successor_entry_id=successor_entry_id, source_evidence=entry.source_evidence, + rule_version=rule_version, + quality_policy=quality_policy, ) diff --git a/src/powercontext/limits.py b/src/powercontext/limits.py index 90d8984ea5..8eea18ec75 100644 --- a/src/powercontext/limits.py +++ b/src/powercontext/limits.py @@ -25,6 +25,7 @@ MAX_SCOPE_BINDING_EXTERNAL_ID_LENGTH = 256 MAX_SOURCE_ID_LENGTH = 256 MAX_SOURCE_TYPE_LENGTH = 128 +MAX_VERSION_LENGTH = 128 MAX_SOURCE_OBSERVATION_BYTES = 4 * 1024 * 1024 MAX_ARTIFACT_FAMILY_LENGTH = 128 MAX_ARTIFACT_ID_LENGTH = 128 diff --git a/src/powercontext/sources/models.py b/src/powercontext/sources/models.py index 3b4f221b30..d1033f5afa 100644 --- a/src/powercontext/sources/models.py +++ b/src/powercontext/sources/models.py @@ -19,7 +19,7 @@ from pydantic import BaseModel, ConfigDict, field_validator from powercontext.errors import InvalidSourceReferenceError -from powercontext.limits import MAX_SOURCE_ID_LENGTH, MAX_SOURCE_TYPE_LENGTH +from powercontext.limits import MAX_SOURCE_ID_LENGTH, MAX_SOURCE_TYPE_LENGTH, MAX_VERSION_LENGTH class SourceMaterialization(StrEnum): @@ -116,6 +116,20 @@ def _validate_reference_part(field: str, value: object) -> None: raise InvalidSourceReferenceError(field, "must be a non-empty string") if value != value.strip(): raise InvalidSourceReferenceError(field, "must not contain leading or trailing whitespace") - maximum = MAX_SOURCE_TYPE_LENGTH if field == "source_type" else MAX_SOURCE_ID_LENGTH + maximum = _reference_part_limit(field) if len(value) > maximum: raise InvalidSourceReferenceError(field, f"must not exceed {maximum} characters") + + +def _reference_part_limit(field: str) -> int: + """Return the length the contract allows for one reference part. + + A version travels through the transport model and a bounded identity column, so + a value accepted here has to stay usable across both. Admitting a longer value + would let a Definition or Source register locally and then fail when it is + converted for transport. + """ + + if field in {"declaration_version", "definition_version"}: + return MAX_VERSION_LENGTH + return MAX_SOURCE_TYPE_LENGTH if field == "source_type" else MAX_SOURCE_ID_LENGTH diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index d3e0609335..d583141509 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -17,6 +17,8 @@ from __future__ import annotations import hashlib +import json +import re from typing import Any, Literal import rfc8785 @@ -46,6 +48,7 @@ from powercontext.sources.models import MemoryEvidenceDeclaration, Source, SourceMaterialization, SourceProjectionKey _JSON_VALUE = TypeAdapter(JsonValue) +_JSON_SCHEMA_REFERENCE = re.compile(r"#/\$defs/([^\"\\]+)") class SourceProjectionManifest(BaseModel): @@ -191,10 +194,52 @@ def projection(self, key: SourceProjectionKey, /) -> JsonValue: raise InvalidSourceProjectionError(key.name, "key", "was not supplied by the worker") +def _input_contract_schema(source_class: type[Source]) -> dict[str, JsonValue]: + """Return a Source class's input contract, without Definition-owned metadata. + + The declaration is stamped from the Definition rather than carried by the + captured value, so it is not part of what a worker's payload has to satisfy. + Leaving it in would change the schema of every Source class that inherits it, + and the manifest fingerprint covers that schema, so a Definition registered + before the field existed would no longer match the one a worker emits. The + orphaned definitions the removed property referenced are dropped with it, + because an unused ``$defs`` entry still changes the hashed document. + """ + + schema = _json_object(source_class.model_json_schema()) + properties = schema.get("properties") + if isinstance(properties, dict): + properties.pop("memory_evidence", None) + required = schema.get("required") + if isinstance(required, list): + schema["required"] = [field for field in required if field != "memory_evidence"] + _drop_unreferenced_definitions(schema) + return schema + + +def _drop_unreferenced_definitions(schema: dict[str, JsonValue]) -> None: + """Remove ``$defs`` entries no longer referenced anywhere in the schema.""" + + definitions = schema.get("$defs") + if not isinstance(definitions, dict): + return + while True: + # Reference discovery has to see the whole document: the properties that + # point at a definition live outside ``$defs``. + referenced = set(_JSON_SCHEMA_REFERENCE.findall(json.dumps(schema))) + unreferenced = [name for name in definitions if name not in referenced] + if not unreferenced: + break + for name in unreferenced: + del definitions[name] + if not definitions: + del schema["$defs"] + + def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> SourceDefinitionManifest: """Build the immutable declaration transported by a remote worker.""" - source_schema = _json_object(definition.source_class.model_json_schema()) + source_schema = _input_contract_schema(definition.source_class) declared = definition_memory_evidence(definition) projections = tuple( SourceProjectionManifest( diff --git a/tests/builtin/persistence/test_memory.py b/tests/builtin/persistence/test_memory.py index 4b2625b75a..f59ea3be7b 100644 --- a/tests/builtin/persistence/test_memory.py +++ b/tests/builtin/persistence/test_memory.py @@ -471,6 +471,48 @@ def test_tag_filter_label_ceiling_matches_the_entry_bind_reserve() -> None: assert _ENTRY_BIND_RESERVE >= 2 * _MAXIMUM_TAG_LABELS + 2 +def test_lifecycle_reads_report_the_state_of_the_requested_revision() -> None: + """A lifecycle read must not answer a historical reference with a later state. + + ``forget`` and ``reactivate`` keep the same entry version, so an identity-only + check accepts a row the requested revision never had. Deriving the state from + the requested manifest keeps a historical reference from reporting current + state. + """ + + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + context = await contexts.get("project") + opened = await context.artifacts.memory.remember( + memory=None, + entries=(MemoryEntryInput(kind="fact", text="Lifecycle follows its revision."),), + mode="append", + ) + assert opened is not None + entry = (await context.artifacts.memory.entries(opened))[0] + forgotten = await context.artifacts.memory.forget(opened, entries=(entry,), reason="forgotten") + assert forgotten is not None + restored = await context.artifacts.memory.reactivate(forgotten, entries=(entry,), reason="restored") + assert restored is not None + + backend = RelationalMemoryBackend( + database=contexts.database, + scope_id="project", + artifacts=contexts.repositories.artifacts, + index=contexts.index, + ) + for revision, expected in ( + (opened.revision, "current"), + (forgotten.revision, "inactive"), + (restored.revision, "current"), + ): + reference = restored.as_ref().model_copy(update={"revision": revision}) + (projection,) = await backend.lifecycle_projections(reference) + assert projection.validity == expected, f"revision {revision} reported {projection.validity}" + + asyncio.run(scenario()) + + def test_scope_bound_contexts_do_not_share_rows() -> None: async def scenario() -> None: async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 9b0d3d2e8d..89c78b550b 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -31,6 +31,7 @@ from powercontext.http import SourceDefinitionManifest as HttpSourceDefinitionManifest from powercontext.http import SourceObservation as HttpSourceObservation from powercontext.http import SubmitSourceObservationRequest +from powercontext.limits import MAX_VERSION_LENGTH from powercontext.server.mapping import runtime_source_definition_manifest, submit_source_observation_request from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, @@ -77,6 +78,36 @@ def _legacy_manifest(current: SourceDefinitionManifest) -> SourceDefinitionManif ) +def test_neutral_declaration_keeps_the_pre_change_definition_identity() -> None: + """A Definition registered before the declaration existed must still re-register. + + The declaration is inherited by every Source class, so adding it would otherwise + change each class's schema, and the fingerprint covers that schema. The + fingerprint below was computed from the same Definition on the revision before + the declaration was introduced, so it pins the identity an upgraded worker + re-registers against a manifest an older worker already registered. + """ + + manifest = manifest_for_definition(CONTENT_SOURCE_DEFINITION) + + assert "memory_evidence" not in manifest.source_schema.get("properties", {}) + assert manifest.fingerprint == ("sha256:0b58627bec9b987d1c3613d98c72f3131ec2a81f95cbc620ef05d923585e0935") + + +def test_version_parts_are_rejected_before_they_cannot_be_transported() -> None: + """A version length must be validated where it is accepted, not on the way out. + + A declaration version travels through the transport model and a bounded + identity column, so a longer value would register locally and then fail when + it is converted for transport. + """ + + accepted = MemoryEvidenceDeclaration(declaration_version="v" * MAX_VERSION_LENGTH) + assert len(accepted.declaration_version) == MAX_VERSION_LENGTH + with pytest.raises(ValueError, match="declaration_version"): + MemoryEvidenceDeclaration(declaration_version="v" * (MAX_VERSION_LENGTH + 1)) + + def test_definition_without_a_declaration_attribute_still_registers() -> None: """An adapter written before the declaration existed must keep registering. From 5628e973d7087adce76910bb566b28bd7ade4f80 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Fri, 2 Oct 2026 11:48:32 +0800 Subject: [PATCH 10/30] test(sources): keep the schema assertion type-checkable Indexing the schema keeps the assertion readable to the type checker, which cannot narrow the union a JsonValue mapping returns from a defaulted lookup. --- tests/test_source_observations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 89c78b550b..0ce1815951 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -90,7 +90,8 @@ def test_neutral_declaration_keeps_the_pre_change_definition_identity() -> None: manifest = manifest_for_definition(CONTENT_SOURCE_DEFINITION) - assert "memory_evidence" not in manifest.source_schema.get("properties", {}) + declared_properties = cast("dict[str, object]", manifest.source_schema["properties"]) + assert "memory_evidence" not in declared_properties assert manifest.fingerprint == ("sha256:0b58627bec9b987d1c3613d98c72f3131ec2a81f95cbc620ef05d923585e0935") From 67df701b5572324128e5d0e3f4924eb32c8cda18 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Fri, 2 Oct 2026 17:31:24 +0800 Subject: [PATCH 11/30] fix(memory): keep declaration metadata out of captured content and historical reads The declaration is Definition-owned metadata, so it must not travel as captured content: a Source class that forbids extra properties rejected its own generated observation, and an unchanged Source gained a payload field, so replaying an observation captured before the upgrade conflicted. Stripping it from the input contract now covers the whole schema rather than the root properties, because a recursive or nested Source has its inherited fields under a local definition, not at the root. A lifecycle read cannot reuse a persisted row's columns either: revising an entry to cite a Source keeps the row while changing what it describes, so a historical reference reported the latest revision's evidence count, or failed outright. The read derives every field from the requested revision. --- .../builtin/persistence/memory.py | 58 +++---------------- src/powercontext/sources/models.py | 7 +++ src/powercontext/sources/observations.py | 47 ++++++++++++--- tests/builtin/persistence/test_memory.py | 53 +++++++++++++++++ tests/test_source_observations.py | 52 ++++++++++++++++- 5 files changed, 159 insertions(+), 58 deletions(-) diff --git a/src/powercontext/builtin/persistence/memory.py b/src/powercontext/builtin/persistence/memory.py index d8a93d68b2..5cb830425f 100644 --- a/src/powercontext/builtin/persistence/memory.py +++ b/src/powercontext/builtin/persistence/memory.py @@ -19,7 +19,7 @@ from collections.abc import AsyncIterator, Iterable, Iterator, Mapping from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass, replace -from typing import Any, ClassVar, Literal, TypeVar, cast +from typing import Any, ClassVar, TypeVar from pydantic import RootModel from sqlalchemy import delete, insert, select, tuple_ @@ -37,7 +37,6 @@ MemoryEvidenceSnapshot, MemoryHit, MemoryLifecycleProjection, - MemoryLifecycleValidity, MemoryProjection, MemoryRevisionChanges, MemorySearchChannels, @@ -1004,62 +1003,23 @@ def _lifecycle_projection( *, row: Mapping[Any, Any] | None = None, ) -> MemoryLifecycleProjection: - """Derive one lifecycle projection, optionally carrying a persisted row's metadata. + """Derive one lifecycle projection for a requested revision's entry. - ``validity``, ``entry_id`` and ``entry_version_id`` always come from the entry, - because the manifest of the requested revision owns them. A persisted row only - contributes the fields the manifest does not carry, and is rejected when it - disagrees about the source count it was written with. + Every field comes from the entry the requested manifest names. A persisted row + is only a cross-check that the entry has one, because its columns describe the + revision that last wrote it: reusing them would report a later revision's state + under a historical reference, which the entry identity alone does not prevent. """ - validity: MemoryLifecycleValidity = "current" if state == "active" else "inactive" - validity_reason: str | None = None - successor_entry_id: str | None = None - rule_version = "memory-lifecycle-v1" - quality_policy: Literal["neutral"] = "neutral" - if row is not None: - if str(row["entry_id"]) != entry.entry_id: - raise InvalidMemoryCitationError("lifecycle-version") - if int(row["source_count"]) != len(entry.source_evidence): - raise InvalidMemoryCitationError("lifecycle-source-count") - validity_reason = None if row["validity_reason"] is None else str(row["validity_reason"]) - successor_entry_id = None if row["successor_entry_id"] is None else str(row["successor_entry_id"]) - rule_version = str(row["rule_version"]) - quality_policy = cast(Literal["neutral"], str(row["quality_policy"])) + if row is not None and str(row["entry_id"]) != entry.entry_id: + raise InvalidMemoryCitationError("lifecycle-version") return MemoryLifecycleProjection( memory_ref=memory_ref, entry_id=entry.entry_id, entry_version_id=entry.entry_version_id, - validity=validity, - validity_reason=validity_reason, - successor_entry_id=successor_entry_id, - source_evidence=entry.source_evidence, - rule_version=rule_version, - quality_policy=quality_policy, - ) - - -def _decode_lifecycle_projection( - memory_ref: ArtifactRef, - row: Mapping[Any, Any], - entry: MemoryEntryVersion | None, -) -> MemoryLifecycleProjection: - if entry is None or entry.entry_id != str(row["entry_id"]): - raise InvalidMemoryCitationError("lifecycle-version") - projection = MemoryLifecycleProjection( - memory_ref=memory_ref, - entry_id=str(row["entry_id"]), - entry_version_id=str(row["entry_version_id"]), - validity=cast(MemoryLifecycleValidity, str(row["validity"])), - validity_reason=(None if row["validity_reason"] is None else str(row["validity_reason"])), - successor_entry_id=(None if row["successor_entry_id"] is None else str(row["successor_entry_id"])), + validity="current" if state == "active" else "inactive", source_evidence=entry.source_evidence, - rule_version=str(row["rule_version"]), - quality_policy=cast(Literal["neutral"], str(row["quality_policy"])), ) - if int(row["source_count"]) != len(projection.source_evidence): - raise InvalidMemoryCitationError("lifecycle-source-count") - return projection def _validate_rebuild_entry( diff --git a/src/powercontext/sources/models.py b/src/powercontext/sources/models.py index d1033f5afa..2169ed559a 100644 --- a/src/powercontext/sources/models.py +++ b/src/powercontext/sources/models.py @@ -95,6 +95,13 @@ def validate_key_part(cls, value: str, info) -> str: return value +# Fields a Source carries for its Definition's benefit rather than as captured content. +# They are stamped from the Definition, so they belong to the observation envelope +# and the manifest, and must stay out of both the payload a worker generates and the +# input contract that payload is validated against. +DECLARATION_OWNED_FIELDS = frozenset({"memory_evidence"}) + + class Source(BaseModel): """Base value for an adapter-owned Source description.""" diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index d583141509..80a10fa72e 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -45,7 +45,13 @@ SourceDefinitionRegistry, definition_memory_evidence, ) -from powercontext.sources.models import MemoryEvidenceDeclaration, Source, SourceMaterialization, SourceProjectionKey +from powercontext.sources.models import ( + DECLARATION_OWNED_FIELDS, + MemoryEvidenceDeclaration, + Source, + SourceMaterialization, + SourceProjectionKey, +) _JSON_VALUE = TypeAdapter(JsonValue) _JSON_SCHEMA_REFERENCE = re.compile(r"#/\$defs/([^\"\\]+)") @@ -207,16 +213,35 @@ def _input_contract_schema(source_class: type[Source]) -> dict[str, JsonValue]: """ schema = _json_object(source_class.model_json_schema()) - properties = schema.get("properties") - if isinstance(properties, dict): - properties.pop("memory_evidence", None) - required = schema.get("required") - if isinstance(required, list): - schema["required"] = [field for field in required if field != "memory_evidence"] + _remove_declaration_owned_fields(schema) _drop_unreferenced_definitions(schema) return schema +def _remove_declaration_owned_fields(node: JsonValue) -> None: + """Strip Definition-owned fields from every object definition in a schema. + + A recursive or nested Source does not list its inherited fields at the root: + Pydantic emits the type under ``$defs`` and points at it with a local + reference. Every such definition carries the declaration, so the walk has to + cover the whole document rather than the root ``properties``. + """ + + if isinstance(node, dict): + properties = node.get("properties") + if isinstance(properties, dict): + for field in DECLARATION_OWNED_FIELDS: + properties.pop(field, None) + required = node.get("required") + if isinstance(required, list): + node["required"] = [field for field in required if field not in DECLARATION_OWNED_FIELDS] + for value in node.values(): + _remove_declaration_owned_fields(value) + elif isinstance(node, list): + for item in node: + _remove_declaration_owned_fields(item) + + def _drop_unreferenced_definitions(schema: dict[str, JsonValue]) -> None: """Remove ``$defs`` entries no longer referenced anywhere in the schema.""" @@ -282,7 +307,13 @@ def project_source_for_transport( "remote Source observations must retain their canonical value", ) manifest = manifest_for_definition(definition) - payload = _json_object(source.model_dump(mode="json")) + # The captured payload has to satisfy the contract the manifest advertises, and + # Definition-owned metadata is not part of it. Carrying it here would make a + # generated observation fail its own schema, and would change the payload of an + # unchanged Source, so replaying a pre-upgrade observation would conflict. + payload = _json_object({ + key: value for key, value in source.model_dump(mode="json").items() if key not in DECLARATION_OWNED_FIELDS + }) projections = tuple( SourceProjectionValue(key=key, value=registry.project(source, key)) for key in registry.projection_keys(source) ) diff --git a/tests/builtin/persistence/test_memory.py b/tests/builtin/persistence/test_memory.py index f59ea3be7b..ba34517a52 100644 --- a/tests/builtin/persistence/test_memory.py +++ b/tests/builtin/persistence/test_memory.py @@ -513,6 +513,59 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_lifecycle_reads_follow_the_evidence_of_the_requested_revision() -> None: + """A lifecycle read must not borrow a persisted row's evidence count. + + Revising an entry to cite a Source keeps its row but changes what the row + describes. A historical reference has to report the evidence that revision + actually carried rather than the latest row's count. + """ + + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + context = await contexts.get("project") + source, _ = await context.sources.capture( + ContentCapture(source_id="turn-1", content="Evidence added by a later revision.") + ) + opened = await context.artifacts.memory.remember( + memory=None, + entries=(MemoryEntryInput(kind="fact", text="Evidence arrives later."),), + mode="append", + ) + assert opened is not None + entry = (await context.artifacts.memory.entries(opened))[0] + assert entry.sources == () + revised = await context.artifacts.memory.remember( + memory=opened, + sources=(source,), + entries=( + MemoryEntryInput( + kind="fact", + text="Evidence arrives later.", + entry=entry, + sources=(source,), + ), + ), + mode="append", + ) + assert revised is not None + + backend = RelationalMemoryBackend( + database=contexts.database, + scope_id="project", + artifacts=contexts.repositories.artifacts, + index=contexts.index, + ) + for revision, expected in ((opened.revision, 0), (revised.revision, 1)): + reference = revised.as_ref().model_copy(update={"revision": revision}) + (projection,) = await backend.lifecycle_projections(reference) + assert len(projection.source_evidence) == expected, ( + f"revision {revision} reported {len(projection.source_evidence)} evidence entries" + ) + + asyncio.run(scenario()) + + def test_scope_bound_contexts_do_not_share_rows() -> None: async def scenario() -> None: async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 0ce1815951..eb90393b1d 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -23,7 +23,7 @@ import pytest import rfc8785 -from pydantic import ValidationError +from pydantic import ConfigDict, ValidationError from referencing.exceptions import Unresolvable from powercontext.builtin.runtime.relational import _json_schema_validator, _validate_source_observation @@ -50,6 +50,7 @@ manifest_for_definition, project_source_for_transport, ) +from powercontext.sources.definitions import AdapterSourceDefinition class EmptySourceBackend: @@ -109,6 +110,55 @@ def test_version_parts_are_rejected_before_they_cannot_be_transported() -> None: MemoryEvidenceDeclaration(declaration_version="v" * (MAX_VERSION_LENGTH + 1)) +def _strict_content_definition() -> AdapterSourceDefinition[ContentCapture, ContentSource, str]: + """Return a Definition whose Source class rejects properties it does not declare.""" + + class StrictContentSource(ContentSource): + model_config = ConfigDict(extra="forbid") + + class StrictContentAdapter: + name = "strict-content" + input_class = ContentCapture + source_class: type[ContentSource] = StrictContentSource + + async def resolve(self, value: ContentCapture, /) -> ContentSource: + return StrictContentSource( + name=value.source_id, + materialization=SourceMaterialization.CAPTURED, + content=value.content, + ) + + async def read(self, source: ContentSource, /) -> str: + return source.content + + return AdapterSourceDefinition(StrictContentAdapter()) + + +def test_generated_payload_satisfies_its_contract_and_excludes_declaration_metadata() -> None: + """A generated payload must satisfy the schema its manifest advertises. + + The declaration is Definition-owned metadata, so a Source class that forbids + extra properties would reject its own generated observation, and an unchanged + Source would carry a different payload after the field was introduced, which + breaks replaying an observation captured before the upgrade. + """ + + async def scenario() -> None: + definition = _strict_content_definition() + registry = SourceDefinitionRegistry((definition,)) + manifest = manifest_for_definition(definition) + source = await registry.resolve(ContentCapture(source_id="turn-1", content="Captured body.")) + observation = project_source_for_transport(registry, source) + + assert "memory_evidence" not in observation.payload + properties = cast("dict[str, object]", manifest.source_schema["properties"]) + assert "memory_evidence" not in properties + # The payload the worker generates must pass the contract it advertises. + _json_schema_validator(manifest.name, manifest.source_schema).validate(observation.payload) + + asyncio.run(scenario()) + + def test_definition_without_a_declaration_attribute_still_registers() -> None: """An adapter written before the declaration existed must keep registering. From fd2efa17bec24b3a2ebcbba35c6915ce3c2b8a80 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Fri, 2 Oct 2026 22:24:06 +0800 Subject: [PATCH 12/30] fix(sources): identify the inherited declaration by its schema, not its name Removing every property called memory_evidence also stripped a field an ordinary captured model declares for itself, which changed that model's contract: the generated payload satisfied the schema no longer, and a Definition registered before the change no longer matched. The removal now matches the schema a Source inherits, so declaration metadata is recognized by ownership and a captured field of the same name keeps its place. --- src/powercontext/sources/observations.py | 36 +++++++++------- tests/test_source_observations.py | 52 +++++++++++++++++++++++- 2 files changed, 72 insertions(+), 16 deletions(-) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 80a10fa72e..eb71a44cd5 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -213,33 +213,39 @@ def _input_contract_schema(source_class: type[Source]) -> dict[str, JsonValue]: """ schema = _json_object(source_class.model_json_schema()) - _remove_declaration_owned_fields(schema) + _remove_inherited_declaration(schema, _declaration_signature()) _drop_unreferenced_definitions(schema) return schema -def _remove_declaration_owned_fields(node: JsonValue) -> None: - """Strip Definition-owned fields from every object definition in a schema. +def _declaration_signature() -> JsonValue: + """Return the schema Pydantic emits for the declaration a Source inherits.""" - A recursive or nested Source does not list its inherited fields at the root: - Pydantic emits the type under ``$defs`` and points at it with a local - reference. Every such definition carries the declaration, so the walk has to - cover the whole document rather than the root ``properties``. + return _json_object(Source.model_json_schema()["properties"])["memory_evidence"] + + +def _remove_inherited_declaration(node: JsonValue, signature: JsonValue) -> None: + """Strip the inherited declaration from every object schema that carries it. + + Recursive and nested Sources keep their fields in a local definition rather + than at the root, so the walk covers the whole document. The field is matched + by the schema a Source inherits rather than by its name alone, because an + ordinary captured model may declare a field of its own with the same name and + a different meaning, and that field belongs to the captured contract. """ if isinstance(node, dict): properties = node.get("properties") - if isinstance(properties, dict): - for field in DECLARATION_OWNED_FIELDS: - properties.pop(field, None) - required = node.get("required") - if isinstance(required, list): - node["required"] = [field for field in required if field not in DECLARATION_OWNED_FIELDS] + if isinstance(properties, dict) and properties.get("memory_evidence") == signature: + del properties["memory_evidence"] + required = node.get("required") + if isinstance(required, list): + node["required"] = [field for field in required if field != "memory_evidence"] for value in node.values(): - _remove_declaration_owned_fields(value) + _remove_inherited_declaration(value, signature) elif isinstance(node, list): for item in node: - _remove_declaration_owned_fields(item) + _remove_inherited_declaration(item, signature) def _drop_unreferenced_definitions(schema: dict[str, JsonValue]) -> None: diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index eb90393b1d..4f18fa8430 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -23,7 +23,7 @@ import pytest import rfc8785 -from pydantic import ConfigDict, ValidationError +from pydantic import BaseModel, ConfigDict, ValidationError from referencing.exceptions import Unresolvable from powercontext.builtin.runtime.relational import _json_schema_validator, _validate_source_observation @@ -159,6 +159,56 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_captured_models_keep_a_field_that_shares_the_declaration_name() -> None: + """Only the declaration a Source inherits is metadata; a captured field is content. + + A captured model may legitimately declare a field of its own called + ``memory_evidence``. Dropping it would change that model's contract, so the + removal identifies the inherited declaration by what it is, not by its name. + """ + + class CapturedReport(BaseModel): + model_config = ConfigDict(extra="forbid") + + memory_evidence: str + summary: str + + class ReportingSource(ContentSource): + report: CapturedReport + + class ReportingAdapter: + name = "reporting-content" + input_class = ContentCapture + source_class: type[ContentSource] = ReportingSource + + async def resolve(self, value: ContentCapture, /) -> ContentSource: + return ReportingSource( + name=value.source_id, + materialization=SourceMaterialization.CAPTURED, + content=value.content, + report=CapturedReport(memory_evidence="captured detail", summary="summary"), + ) + + async def read(self, source: ContentSource, /) -> str: + return source.content + + definition = AdapterSourceDefinition(ReportingAdapter()) + registry = SourceDefinitionRegistry((definition,)) + manifest = manifest_for_definition(definition) + + definitions = cast("dict[str, dict[str, object]]", manifest.source_schema["$defs"]) + captured = cast("dict[str, object]", definitions["CapturedReport"]["properties"]) + assert "memory_evidence" in captured + root = cast("dict[str, object]", manifest.source_schema["properties"]) + assert "memory_evidence" not in root + + source = asyncio.run(registry.resolve(ContentCapture(source_id="turn-1", content="Captured body."))) + observation = project_source_for_transport(registry, source) + report = cast("dict[str, object]", observation.payload["report"]) + assert report["memory_evidence"] == "captured detail" + _json_schema_validator(manifest.name, manifest.source_schema).validate(observation.payload) + + def test_definition_without_a_declaration_attribute_still_registers() -> None: """An adapter written before the declaration existed must keep registering. From 965af82c5ebd16588f02c2e6a7c44a4a6968d216 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Mon, 12 Oct 2026 20:35:44 +0800 Subject: [PATCH 13/30] fix(sources): strip owned metadata from nested source schemas --- src/powercontext/sources/observations.py | 52 +++++++++++++++++------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index eb71a44cd5..db246d0b7f 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -213,7 +213,7 @@ def _input_contract_schema(source_class: type[Source]) -> dict[str, JsonValue]: """ schema = _json_object(source_class.model_json_schema()) - _remove_inherited_declaration(schema, _declaration_signature()) + _remove_inherited_declaration(schema, source_class) _drop_unreferenced_definitions(schema) return schema @@ -224,7 +224,7 @@ def _declaration_signature() -> JsonValue: return _json_object(Source.model_json_schema()["properties"])["memory_evidence"] -def _remove_inherited_declaration(node: JsonValue, signature: JsonValue) -> None: +def _remove_inherited_declaration(node: JsonValue, source_class: type[Source]) -> None: """Strip the inherited declaration from every object schema that carries it. Recursive and nested Sources keep their fields in a local definition rather @@ -234,18 +234,42 @@ def _remove_inherited_declaration(node: JsonValue, signature: JsonValue) -> None a different meaning, and that field belongs to the captured contract. """ - if isinstance(node, dict): - properties = node.get("properties") - if isinstance(properties, dict) and properties.get("memory_evidence") == signature: - del properties["memory_evidence"] - required = node.get("required") - if isinstance(required, list): - node["required"] = [field for field in required if field != "memory_evidence"] - for value in node.values(): - _remove_inherited_declaration(value, signature) - elif isinstance(node, list): - for item in node: - _remove_inherited_declaration(item, signature) + source_titles = _source_schema_titles(source_class) + + def walk(value: JsonValue) -> None: + if isinstance(value, dict): + properties = value.get("properties") + if value.get("title") in source_titles and isinstance(properties, dict): + properties.pop("memory_evidence", None) + required = value.get("required") + if isinstance(required, list): + value["required"] = [field for field in required if field != "memory_evidence"] + for child in value.values(): + walk(child) + elif isinstance(value, list): + for child in value: + walk(child) + + walk(node) + + +def _source_schema_titles(source_class: type[Source]) -> set[str]: + """Return schema titles belonging to Source subclasses in the payload tree.""" + + titles: set[str] = set() + pending: list[type[BaseModel]] = [source_class] + seen: set[type[BaseModel]] = set() + while pending: + model = pending.pop() + if model in seen: + continue + seen.add(model) + titles.add(str(model.model_config.get("title", model.__name__))) + for field in model.model_fields.values(): + annotation = field.annotation + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + pending.append(annotation) + return titles def _drop_unreferenced_definitions(schema: dict[str, JsonValue]) -> None: From 7683e6c65ae4d8ae29c46dce16be2fd342c896e0 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Mon, 12 Oct 2026 20:45:36 +0800 Subject: [PATCH 14/30] refactor: drop obsolete pre-atomic memory lifecycle path --- .../builtin/artifacts/memory/service.py | 2002 ----------------- .../builtin/persistence/memory.py | 1057 --------- src/powercontext/sources/observations.py | 8 +- tests/builtin/persistence/test_memory.py | 571 ----- tests/test_source_observations.py | 5 +- 5 files changed, 5 insertions(+), 3638 deletions(-) delete mode 100644 src/powercontext/builtin/artifacts/memory/service.py delete mode 100644 src/powercontext/builtin/persistence/memory.py delete mode 100644 tests/builtin/persistence/test_memory.py diff --git a/src/powercontext/builtin/artifacts/memory/service.py b/src/powercontext/builtin/artifacts/memory/service.py deleted file mode 100644 index a2995015a7..0000000000 --- a/src/powercontext/builtin/artifacts/memory/service.py +++ /dev/null @@ -1,2002 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Public orchestration for the Memory Artifact Family.""" - -from __future__ import annotations - -import logging -from collections.abc import Callable, Sequence -from contextlib import nullcontext -from dataclasses import dataclass -from hashlib import sha256 -from time import perf_counter -from typing import Literal, Protocol, TypeAlias, TypeVar, overload -from uuid import uuid4 - -from pydantic import BaseModel - -from powercontext._logging import log_safely -from powercontext.artifacts import Artifact, ArtifactLineage, ArtifactRef -from powercontext.builtin.artifacts.memory.canonical import ( - canonical_embedding, - canonical_error_code, - canonical_json, - embedding_content_hash, - entry_content_bytes, - entry_content_hash, - memory_content_bytes, - normalize_kind, - normalize_query, - normalize_reason, - normalize_text, - validate_identifier, -) -from powercontext.builtin.artifacts.memory.errors import ( - CapabilityNotSupportedError, - InvalidEmbeddingError, - InvalidMemoryCandidateError, - InvalidMemoryCitationError, - InvalidMemoryEvidenceError, - MemoryCapacityExceededError, - MemoryEntryInactiveError, - MemoryEntryNotFoundError, - MemoryWriteRejectedError, -) -from powercontext.builtin.artifacts.memory.fusion import ( - admit_fts_candidates, - admit_vector_candidates, - fuse_rankings, -) -from powercontext.builtin.artifacts.memory.models import ( - EmbeddingProfile, - Memory, - MemoryCapabilities, - MemoryCapacity, - MemoryCapacityBudget, - MemoryCapacityDimension, - MemoryChange, - MemoryCitation, - MemoryCompactionPolicy, - MemoryCompactionResult, - MemoryContent, - MemoryEntryInput, - MemoryEntryVersion, - MemoryEvidenceSnapshot, - MemoryHit, - MemoryManifest, - MemoryManifestEntry, - MemoryQueryEmbedding, - MemoryRerankTrace, - MemoryRevisionChanges, - MemorySearchMode, - MemorySearchResult, - MemoryUsedSearchMode, -) -from powercontext.builtin.artifacts.memory.protocols import ( - CandidatePipeline, - MemoryBackend, - MemoryCandidateRequest, - MemoryCommit, - MemoryProjection, - MemorySearchRequest, - MemoryWriteAssessment, - MemoryWriteGate, - MemoryWriteGateRequest, - MemoryWritePlan, - MemoryWriteRejectionCode, - MemoryWriteVerdict, -) -from powercontext.builtin.artifacts.memory.reranking import MemoryReranker -from powercontext.builtin.artifacts.prompt.service import ScopedPrompts, current_prompt, prompt_operation -from powercontext.builtin.artifacts.search import AdmissionCounts, AdmissionFloor, analyze_fts_query, analyze_text -from powercontext.builtin.inference import ( - EmbeddingModel, - EmbeddingVector, - InferenceTimeoutError, - InferenceUnavailableError, - embed_query, -) -from powercontext.builtin.tags import TagFilter -from powercontext.errors import RevisionConflictError -from powercontext.sources import ( - TEXT_EVIDENCE_PROJECTION_KEY, - MemoryEvidenceDeclaration, - Source, - SourceObservation, - SourceProjectionKey, - SourceRef, - TextEvidence, -) - -MemoryRememberMode: TypeAlias = Literal["append", "extract", "auto"] -IdFactory: TypeAlias = Callable[[str], str] -ValueT = TypeVar("ValueT") -_GATE_EVIDENCE_ITEM_LIMIT = 32 -_GATE_EVIDENCE_TEXT_LIMIT = 2000 -logger = logging.getLogger(__name__) - - -class _SourceResolver(Protocol): - async def get(self, source: Source, /) -> Source: ... - - async def get_ref(self, ref: SourceRef, /) -> Source: ... - - def as_ref(self, source: Source, /) -> SourceRef: ... - - def project(self, source: Source, key: SourceProjectionKey, /) -> object: ... - - def memory_evidence(self, source: Source, /) -> MemoryEvidenceDeclaration: ... - - -class _ArtifactResolver(Protocol): - async def get(self, artifact: Artifact[object], /) -> Artifact[object]: ... - - async def get_ref(self, ref: ArtifactRef, /) -> Artifact[object]: ... - - -@dataclass(frozen=True, slots=True) -class _OperationEvidence: - sources: tuple[Source, ...] - artifacts: tuple[Artifact[object], ...] - - -@dataclass(frozen=True, slots=True) -class _EntryMaterial: - kind: str - text: str - sources: tuple[SourceRef, ...] - source_evidence: tuple[MemoryEvidenceSnapshot, ...] - artifacts: tuple[ArtifactRef, ...] - content_bytes: bytes - content_hash: str - - -@dataclass(frozen=True, slots=True) -class _GateEvidenceEntry: - text: str - complete: bool - - -@dataclass(frozen=True, slots=True) -class _GateCandidateEvidence: - index: int - material: _EntryMaterial - - -@dataclass(frozen=True, slots=True) -class _GateEvidenceProjection: - entries: tuple[str, ...] - rejection: MemoryWriteAssessment | None = None - - -@dataclass(slots=True) -class _GateEvidenceBuilder: - policy_id: str - entries: list[str] - - def append(self, entry: _GateEvidenceEntry) -> MemoryWriteAssessment | None: - if not entry.complete: - return self._budget_rejection() - if any(current == entry.text for current in self.entries): - return None - if len(self.entries) >= _GATE_EVIDENCE_ITEM_LIMIT: - return self._budget_rejection() - self.entries.append(entry.text) - return None - - def projection(self) -> _GateEvidenceProjection: - return _GateEvidenceProjection(tuple(self.entries)) - - def _budget_rejection(self) -> MemoryWriteAssessment: - return MemoryWriteAssessment( - verdict=MemoryWriteVerdict.HOLD, - policy_id=self.policy_id, - code=MemoryWriteRejectionCode.EVIDENCE_LIMIT_EXCEEDED, - reason="the cited evidence exceeds the gate evidence budget", - ) - - -class _InvalidMemoryOperationError(ValueError): - def __init__(self, code: str) -> None: - messages = { - "append-entries": "memory append mode requires at least one entry", - "extract-evidence": "memory extract mode requires evidence and does not accept entries", - "no-work": "memory remember call has no executable work", - "duplicate-target": "a memory entry can only be targeted once per operation", - "id-collision": "generated memory identity collides with current content", - "organize-mode": "unsupported memory organize mode", - "since-greater": "since_revision cannot be greater than the target Revision", - "since-negative": "since_revision cannot be negative", - "search-memories": "memory search requires at least one explicit Memory ref", - "search-limit": "memory search limit must be positive", - "search-mode": "unsupported memory search mode", - "search-query": "memory search query must be non-empty text", - "compaction-limit": "memory compaction limit must be positive", - "history-limit": "memory history revision limit must be positive", - "through-range": "through_revision must be between 1 and the current head Revision", - } - super().__init__(messages[code]) - - -def _extraction_prompt_refs() -> tuple[ArtifactRef, ...]: - selection = current_prompt("memory.extract") - return () if selection is None or selection.artifact is None else (selection.artifact,) - - -def _require_tag_filter(capabilities: MemoryCapabilities, tag_filter: TagFilter | None) -> None: - if tag_filter is not None and not capabilities.tag_filter: - raise CapabilityNotSupportedError("tag-filter") - - -def _annotate_reason(reason: str | None, flagged_reason: str | None) -> str | None: - """Fill a missing audit reason from a flagged gate verdict without overwriting a caller's.""" - - normalized = normalize_reason(reason) - if normalized is not None or flagged_reason is None: - return normalized - return normalize_reason(flagged_reason) - - -class MemoryService: - """Validate and orchestrate Memory operations without exposing storage details.""" - - def __init__( - self, - *, - backend: MemoryBackend, - candidate_pipeline: CandidatePipeline | None = None, - embedding_model: EmbeddingModel | None = None, - reranker: MemoryReranker | None = None, - rerank_candidate_limit: int = 30, - source_resolver: _SourceResolver | None = None, - artifact_resolver: _ArtifactResolver | None = None, - id_factory: IdFactory | None = None, - prompt_context: ScopedPrompts | None = None, - write_gate: MemoryWriteGate | None = None, - capacity_budget: MemoryCapacityBudget | None = None, - compaction: MemoryCompactionPolicy | None = None, - max_history_revisions: int = 100, - ) -> None: - self._backend = backend - self._prompt_context = prompt_context - self._candidate_pipeline = candidate_pipeline - self._write_gate = write_gate - self._embedding_model = embedding_model - if rerank_candidate_limit < 1: - raise _InvalidMemoryOperationError("search-limit") - self._reranker = reranker - self._rerank_candidate_limit = rerank_candidate_limit - self._source_resolver = source_resolver - self._artifact_resolver = artifact_resolver - self._id_factory = _default_id if id_factory is None else id_factory - self._capacity_budget = MemoryCapacityBudget() if capacity_budget is None else capacity_budget.model_copy() - self._compaction = MemoryCompactionPolicy() if compaction is None else compaction.model_copy() - if max_history_revisions < 1: - raise _InvalidMemoryOperationError("history-limit") - self._max_history_revisions = max_history_revisions - # One entry, describing the projections of the most recently written Memory - # revision. Revisions are immutable, so a hit is always valid for that exact - # reference; a rebuilt or externally advanced Memory simply misses. - self._previous_projection_cache: tuple[ArtifactRef, dict[str, MemoryProjection]] | None = None - - async def get(self, memory: Memory, /) -> Memory: - """Return the canonical exact Memory Revision matching ``memory``.""" - - return await self._canonical_memory(memory) - - async def latest(self, memory: Memory, /) -> Memory: - """Return the current head of the same Memory identity.""" - - canonical = await self.get(memory) - return await self._backend.latest(canonical.artifact_id) - - async def revisions( - self, memory: Memory, /, *, since_revision: int = 0, through_revision: int | None = None - ) -> tuple[Memory, ...]: - """Return history in ascending order within ``(since_revision, through_revision]``. - - The upper bound defaults to the current head. The history limit applies - to the requested interval, and oversized intervals are never truncated. - """ - - canonical = await self.get(memory) - latest = await self._backend.latest(canonical.artifact_id) - upper = latest.revision if through_revision is None else through_revision - if upper < 1 or upper > latest.revision: - raise _InvalidMemoryOperationError("through-range") - if since_revision < 0: - raise _InvalidMemoryOperationError("since-negative") - if since_revision > upper: - raise _InvalidMemoryOperationError("since-greater") - if upper - since_revision > self._max_history_revisions: - raise CapabilityNotSupportedError("history-window") - history = [] - for revision in range(since_revision + 1, upper + 1): - history.append( - await self._backend.get( - ArtifactRef(family=Memory.family, artifact_id=canonical.artifact_id, revision=revision) - ) - ) - return tuple(history) - - async def head(self, artifact_id: str, /) -> Memory: - """Return the current Memory head by its stable Artifact identity.""" - - return await self._backend.latest(artifact_id) - - async def capacity(self, memory: Memory, /) -> MemoryCapacity: - """Measure an exact Revision, including eligible tombstones even when compaction is disabled. - - Eligibility can load complete manifests across the tombstone recovery - window. Cost scales with their combined size; this is not a cheap counter. - """ - - canonical = await self._canonical_memory(memory) - values = self._capacity_values(canonical.content) - return MemoryCapacity( - memory_ref=canonical.as_ref(), - active_entry_count=values["active_entries"], - manifest_entry_count=values["manifest_entries"], - manifest_bytes=values["manifest_bytes"], - compactable_entry_count=len(await self._compactable_entry_ids(canonical)), - budget=self._capacity_budget.model_copy(), - exceeded=tuple(dimension for dimension, limit in self._capacity_limits() if values[dimension] > limit), - ) - - def _capacity_limits(self) -> tuple[tuple[MemoryCapacityDimension, int], ...]: - budget = self._capacity_budget - return ( - ("manifest_bytes", budget.max_manifest_bytes), - ("manifest_entries", budget.max_manifest_entries), - ("active_entries", budget.max_active_entries), - ) - - @staticmethod - def _capacity_values( - content: MemoryContent, content_bytes: bytes | None = None - ) -> dict[MemoryCapacityDimension, int]: - return { - "manifest_bytes": len(memory_content_bytes(content) if content_bytes is None else content_bytes), - "manifest_entries": len(content.manifest.entries), - "active_entries": sum(item.state == "active" for item in content.manifest.entries), - } - - def _require_capacity( - self, - base: Memory | None, - content: MemoryContent, - *, - growth: frozenset[MemoryCapacityDimension], - content_bytes: bytes, - ) -> None: - if not growth: - return - values = self._capacity_values(content, content_bytes) - previous = None - for dimension, limit in self._capacity_limits(): - observed = values[dimension] - if dimension not in growth or observed <= limit: - continue - if previous is None: - previous = {} if base is None else self._capacity_values(base.content) - if observed > previous.get(dimension, 0): - raise MemoryCapacityExceededError(dimension, limit, observed) - - async def _compactable_entry_ids(self, memory: Memory) -> tuple[str, ...]: - inactive = {item.entry_id for item in memory.content.manifest.entries if item.state == "inactive"} - if not inactive: - return () - # Only the recovery window matters. An inactive entry untouched throughout - # that window was already inactive at its lower bound. - lower = max(0, memory.revision - self._compaction.min_tombstone_revisions) - if lower < 1: - return () - recent = { - change.entry_id - for revision in await self._backend.changes(memory.as_ref(), lower) - for change in revision.changes - if change.op in {"add", "deactivate", "reactivate"} - } - tagged = await self._backend.any_tagged_entry_ids(memory.as_ref()) - return tuple(sorted(inactive - recent - tagged, key=str.encode)) - - async def compact( - self, memory: Memory, *, dry_run: bool = False, limit: int | None = None, reason: str | None = None - ) -> MemoryCompactionResult: - """Drop aged, untagged tombstones; retain every prior Revision and entry body. - - Previews are available while compaction is disabled. Reclaimed bytes are - the signed difference of complete canonical contents, including the audit - changes and reason, which can outweigh a small manifest reduction. - """ - - if limit is not None and limit < 1: - raise _InvalidMemoryOperationError("compaction-limit") - if not dry_run and not self._compaction.enabled: - raise CapabilityNotSupportedError("compaction") - normalized_reason = normalize_reason(reason) - base = await self._canonical_base(memory) - entry_ids = (await self._compactable_entry_ids(base))[:limit] - selected = frozenset(entry_ids) - manifest = {item.entry_id: item for item in base.content.manifest.entries if item.entry_id not in selected} - changes = tuple( - MemoryChange( - op="compact", - entry_id=item.entry_id, - from_entry_version_id=item.entry_version_id, - to_entry_version_id=None, - reason=normalized_reason, - ) - for item in base.content.manifest.entries - if item.entry_id in selected - ) - if not entry_ids: - return MemoryCompactionResult(memory=base, dry_run=dry_run) - content = MemoryContent(manifest=MemoryManifest(entries=tuple(manifest.values())), changes=changes) - reclaimed = len(memory_content_bytes(base.content)) - len(memory_content_bytes(content)) - result = ( - base - if dry_run - else await self._commit_existing_transition( - base=base, - manifest=manifest, - changes=changes, - current_by_entry={}, - entry_versions=(), - ) - ) - return MemoryCompactionResult(memory=result, entry_ids=entry_ids, reclaimed_bytes=reclaimed, dry_run=dry_run) - - async def head_entries(self, artifact_id: str, /) -> tuple[Memory, tuple[MemoryEntryVersion, ...]]: - """Return the current Memory head together with its validated entry objects. - - ``entries`` re-reads the caller's Memory to prove it matches storage, which a head - read straight from the backend already satisfies. Callers that need both the head - and its entries use this instead, so a read-only pass over many Scopes does not - re-fetch every Memory Revision it just loaded. - """ - - memory = await self._backend.latest(artifact_id) - return memory, await self._validated_entries(memory) - - async def revision(self, memory: ArtifactRef, /) -> Memory: - """Return one exact Memory Revision by its stable reference.""" - - return await self._backend.get(memory) - - async def remember( - self, - *, - memory: Memory | None, - sources: Sequence[Source] = (), - artifacts: Sequence[Artifact[object]] = (), - entries: Sequence[MemoryEntryInput] = (), - mode: MemoryRememberMode = "auto", - ) -> Memory | None: - """Append or extract validated entry changes against one exact head.""" - - plan = await self.plan_remember( - memory=memory, - sources=sources, - artifacts=artifacts, - entries=entries, - mode=mode, - ) - _raise_if_write_held(plan) - return await self.apply(plan) - - async def plan_remember( - self, - *, - memory: Memory | None, - sources: Sequence[Source] = (), - artifacts: Sequence[Artifact[object]] = (), - entries: Sequence[MemoryEntryInput] = (), - mode: MemoryRememberMode = "auto", - ) -> MemoryWritePlan: - """Validate and prepare a write without mutating authoritative storage.""" - - selected_mode = _select_remember_mode( - mode, - has_entries=bool(entries), - has_evidence=bool(sources or artifacts), - ) - binding = ( - self._prompt_context.service.bind(self._prompt_context.scope_id, "memory.extract") - if selected_mode == "extract" and self._prompt_context is not None - else nullcontext() - ) - async with binding: - base = await self._canonical_base(memory) - evidence = await self._canonical_operation_evidence(sources, artifacts) - # Only extraction compares against every current entry; append plans carry - # no current entry set and load one lazily if a candidate revises an entry. - current_entries = ( - await self._validated_entries(base) if base is not None and selected_mode == "extract" else None - ) - candidates = await self._candidates( - selected_mode, - tuple(entries), - evidence, - () if current_entries is None else current_entries, - active_version_ids=( - frozenset() - if base is None - else frozenset( - item.entry_version_id for item in base.content.manifest.entries if item.state == "active" - ) - ), - ) - if not candidates: - return MemoryWritePlan(result=base, commit=None) - - assessment = await self._assess_write(base, candidates, evidence, current_entries) - if assessment is not None and assessment.verdict is MemoryWriteVerdict.HOLD: - # A refused write stays visible: the caller reads the structured code and reason - # from the plan. The plan carries no commit, so nothing is written. - return MemoryWritePlan(result=base, commit=None, decision=assessment) - - flagged_reason = ( - assessment.reason if assessment is not None and assessment.verdict is MemoryWriteVerdict.FLAG else None - ) - commit = await self._prepare_commit( - base=base, - candidates=candidates, - evidence=evidence, - current_entries=current_entries, - flagged_reason=flagged_reason, - ) - if commit is None: - return MemoryWritePlan(result=base, commit=None, decision=assessment) - return MemoryWritePlan(result=commit.memory, commit=commit, decision=assessment) - - async def apply(self, plan: MemoryWritePlan, /) -> Memory | None: - """Apply one prepared write through this service's transaction boundary.""" - - if plan.commit is None: - return plan.result - async with self._backend.begin() as unit_of_work: - committed = await unit_of_work.commit(plan.commit) - if plan.result != committed: - raise InvalidMemoryCitationError("memory-mismatch") - self._cache_projections(committed, plan.commit.projections) - return committed - - async def forget( - self, - memory: Memory, - *, - entries: Sequence[MemoryEntryVersion], - reason: str | None = None, - ) -> Memory: - """Deactivate logical entries without deleting immutable content.""" - - return await self._set_entry_state( - memory, - entries=entries, - target_state="inactive", - reason=reason, - ) - - async def reactivate( - self, - memory: Memory, - *, - entries: Sequence[MemoryEntryVersion], - reason: str | None = None, - ) -> Memory: - """Restore inactive logical entries without creating body versions.""" - - return await self._set_entry_state( - memory, - entries=entries, - target_state="active", - reason=reason, - ) - - async def organize( - self, - memory: Memory, - *, - mode: Literal["default", "dedupe", "normalize"] = "default", - ) -> Memory: - """Apply only exact deduplication and canonical normalization.""" - - if mode not in {"default", "dedupe", "normalize"}: - raise _InvalidMemoryOperationError("organize-mode") - base = await self._canonical_base(memory) - current_entries = await self._validated_entries(base) - manifest = {entry.entry_id: entry for entry in base.content.manifest.entries} - current_by_entry = {entry.entry_id: entry for entry in current_entries} - changes: list[MemoryChange] = [] - changed_ids: set[str] = set() - new_versions: list[MemoryEntryVersion] = [] - - if mode in {"default", "dedupe"}: - dedupe_changes, changed_ids = self._deduplicate_manifest(manifest, current_by_entry) - changes.extend(dedupe_changes) - - if mode in {"default", "normalize"}: - normalize_changes, new_versions = self._normalize_manifest_entries( - base, - manifest, - current_by_entry, - skip=changed_ids, - ) - changes.extend(normalize_changes) - - if not changes: - return base - return await self._commit_existing_transition( - base=base, - manifest=manifest, - changes=changes, - current_by_entry=current_by_entry, - entry_versions=tuple(new_versions), - ) - - async def changes( - self, - memory: Memory, - *, - since_revision: int | None = None, - ) -> tuple[MemoryRevisionChanges, ...]: - """Read compact change summaries without expanding entry bodies.""" - - target = await self._canonical_memory(memory) - if since_revision is not None: - if since_revision < 0: - raise _InvalidMemoryOperationError("since-negative") - if since_revision > target.revision: - raise _InvalidMemoryOperationError("since-greater") - if since_revision == target.revision: - return () - if since_revision > 0: - await self._backend.get( - ArtifactRef( - family=Memory.family, - artifact_id=target.artifact_id, - revision=since_revision, - ) - ) - return await self._backend.changes(target.as_ref(), since_revision) - - @prompt_operation("memory.rerank") - async def search( - self, - query: str, - *, - memories: Sequence[Memory], - limit: int = 10, - mode: MemorySearchMode = "auto", - tag_filter: TagFilter | None = None, - admission: AdmissionFloor | None = None, - query_embedding: MemoryQueryEmbedding | None = None, - ) -> MemorySearchResult: - """Search explicit current Memory heads with capability-safe fallback. - - ``admission=None`` applies the historical fusion-time thresholds bit for bit. - - ``query_embedding`` lets a caller reuse a vector it already paid for (RFC 1560's - expansion rounds reuse the round-0 vector). Reuse is best-effort and honest: it is - applied only when the supplied profile equals the one this search resolved, and the - call reports ``embedding_calls = 0`` in that case, ``1`` when it embedded. The - resolved vector is handed back on :attr:`MemorySearchResult.query_embedding` so the - next round can reuse it. When the mode resolved to ``fts`` there is no vector to - report, so the field stays ``None`` and the next round must pay again. - """ - - if not memories: - raise _InvalidMemoryOperationError("search-memories") - if limit < 1: - raise _InvalidMemoryOperationError("search-limit") - if mode not in {"fts", "vector", "hybrid", "auto"}: - raise _InvalidMemoryOperationError("search-mode") - selected_by_ref: dict[tuple[str, str, int], Memory] = {} - for memory in memories: - ref = memory.as_ref() - selected_by_ref.setdefault((ref.family, ref.artifact_id, ref.revision), memory) - selected = tuple(selected_by_ref.values()) - selected_memories = tuple(memory.as_ref() for memory in selected) - await self._validate_search_heads(selected) - capabilities = await self._backend.capabilities() - _require_tag_filter(capabilities, tag_filter) - selected_mode = await self._select_search_mode( - mode, - memories=selected_memories, - capabilities=capabilities, - ) - try: - normalized_query = normalize_query(query) - except (TypeError, ValueError) as error: - raise _InvalidMemoryOperationError("search-query") from error - query_vector = None - profile = None - embedding_calls = 0 - resolved_embedding: MemoryQueryEmbedding | None = None - if selected_mode in {"vector", "hybrid"}: - profile = capabilities.embedding_profile - if profile is None: - raise CapabilityNotSupportedError(selected_mode) - if profile.distance != "l2" or profile.normalization != "unit": - raise CapabilityNotSupportedError( - selected_mode, - "cosine admission requires a unit-normalized L2 embedding profile", - ) - ( - selected_mode, - query_vector, - resolved_embedding, - embedding_calls, - ) = await self._resolve_query_vector( - query=normalized_query, - requested_mode=mode, - selected_mode=selected_mode, - profile=profile, - capabilities=capabilities, - reuse=query_embedding, - ) - if resolved_embedding is None: - profile = None - coarse_limit = limit if self._reranker is None else max(limit, self._rerank_candidate_limit) - request = MemorySearchRequest( - query=normalized_query, - analyzed_query=analyze_fts_query(normalized_query), - memories=selected_memories, - candidate_limit=max(coarse_limit * 4, 32), - mode=selected_mode, - query_vector=query_vector, - embedding_profile=profile, - tag_filter=tag_filter, - ) - channels = await self._backend.search(request) - admitted_fts = admit_fts_candidates(normalized_query, channels.fts, admission=admission) - admitted_vector = admit_vector_candidates(channels.vector, admission=admission) - hits = fuse_rankings( - fts=admitted_fts if selected_mode in {"fts", "hybrid"} else (), - vector=admitted_vector if selected_mode in {"vector", "hybrid"} else (), - limit=coarse_limit, - ) - return await self._reranked_search_result( - mode=selected_mode, - query=normalized_query, - hits=hits, - limit=limit, - admission=AdmissionCounts( - family=Memory.family, - scope_id="", - retrieved=len(channels.fts) + len(channels.vector), - admitted=len(admitted_fts) + len(admitted_vector), - ), - embedding_calls=embedding_calls, - query_embedding=resolved_embedding, - ) - - async def _resolve_query_vector( - self, - *, - query: str, - requested_mode: MemorySearchMode, - selected_mode: MemoryUsedSearchMode, - profile: EmbeddingProfile, - capabilities: MemoryCapabilities, - reuse: MemoryQueryEmbedding | None, - ) -> tuple[MemoryUsedSearchMode, tuple[float, ...] | None, MemoryQueryEmbedding | None, int]: - """Resolve — or reuse — the query vector for a vector or hybrid search. - - Returns ``(selected_mode, query_vector, resolved_embedding, embedding_calls)``. - ``resolved_embedding`` is ``None`` exactly when the search fell back to ``fts``, which - is also when the caller must drop the embedding profile from the backend request. - - Reuse applies only when the supplied profile equals the resolved one; otherwise the - round embeds and reports one call. A failed embedding is still counted as one call, - because the call was issued — the cost is real even though it produced nothing. - """ - - if reuse is not None and reuse.embedding_profile == profile: - return selected_mode, reuse.query_vector, reuse, 0 - try: - query_vector = (await self._embed_texts((query,), profile, query=True))[0] - except (InferenceUnavailableError, InferenceTimeoutError) as error: - if requested_mode == "auto" and capabilities.fts: - return "fts", None, None, 1 - raise CapabilityNotSupportedError( - selected_mode, - "embedding model is temporarily unavailable", - ) from error - return ( - selected_mode, - query_vector, - MemoryQueryEmbedding(query_vector=query_vector, embedding_profile=profile), - 1, - ) - - async def _reranked_search_result( - self, - *, - mode: MemoryUsedSearchMode, - query: str, - hits: tuple[MemoryHit, ...], - limit: int, - admission: AdmissionCounts | None = None, - embedding_calls: int = 0, - query_embedding: MemoryQueryEmbedding | None = None, - ) -> MemorySearchResult: - """Apply the optional reranker and report what this search paid and admitted. - - ``generation_calls`` counts the RFC 0080 rerank call this search actually issued: ``1`` - when the reranker ran over a non-empty pool, ``0`` otherwise. It is never inferred - from configuration — a deployment whose rerank flag is on but whose reranker did not - run reports ``0``, which is the honest number. - """ - - if self._reranker is None or not hits: - return MemorySearchResult( - mode=mode, - hits=hits[:limit], - admission=admission, - embedding_calls=embedding_calls, - query_embedding=query_embedding, - ) - rerank_started = perf_counter() - decision = await self._reranker.rerank( - query, - hits, - min(limit, len(hits)), - ) - rerank_latency_ms = (perf_counter() - rerank_started) * 1_000 - selected = tuple(hits[rank - 1] for rank in decision.selected_ranks) - return MemorySearchResult( - mode=mode, - hits=selected, - rerank=MemoryRerankTrace( - policy_id=self._reranker.policy_id, - candidate_hits=hits, - selected_ranks=decision.selected_ranks, - discarded_rank_count=decision.discarded_rank_count, - used_fallback=decision.used_fallback, - latency_ms=rerank_latency_ms, - usage=decision.usage, - ), - admission=admission, - embedding_calls=embedding_calls, - generation_calls=1, - query_embedding=query_embedding, - ) - - async def expand( - self, - hits: Sequence[MemoryHit], - *, - layer: Literal["full"] = "full", - ) -> tuple[MemoryEntryVersion, ...]: - """Expand exact hit anchors and reject cross-Revision substitutions.""" - - if layer != "full": - raise CapabilityNotSupportedError("expand-layer") - hit_tuple = tuple(hits) - versions = await self._backend.expand(hit_tuple) - if len(versions) != len(hit_tuple): - raise InvalidMemoryCitationError("expand-count") - for hit, version in zip(hit_tuple, versions, strict=True): - await self._validate_anchor( - memory_ref=hit.memory_ref, - entry_id=hit.entry_id, - entry_version_id=hit.entry_version_id, - version=version, - ) - return versions - - async def entries( - self, memory: Memory, /, *, tag_filter: TagFilter | None = None - ) -> tuple[MemoryEntryVersion, ...]: - """Return the entry objects referenced by one exact current Memory head.""" - - canonical = await self._canonical_memory(memory) - entries = await self._validated_entries(canonical) - if tag_filter is None: - return entries - _require_tag_filter(await self._backend.capabilities(), tag_filter) - matching = await self._backend.tagged_entry_ids(canonical.as_ref(), tag_filter) - return tuple(entry for entry in entries if entry.entry_id in matching) - - async def rebuild_projections(self, embedding_model: EmbeddingModel | None = None, /) -> None: - """Rebuild current-head search projections from authoritative Memory revisions.""" - - self._previous_projection_cache = None - await self._backend.rebuild_projections(embedding_model) - - async def validate_citation(self, citation: MemoryCitation) -> MemoryEntryVersion: - """Resolve one exact Handoff citation.""" - - hit = MemoryHit( - memory_ref=citation.memory_ref, - entry_id=citation.entry_id, - entry_version_id=citation.entry_version_id, - text="", - score=0.0, - matched_by=(), - ) - return (await self.expand((hit,)))[0] - - async def _validate_search_heads(self, memories: tuple[Memory, ...]) -> None: - for memory in memories: - exact = await self._backend.get(memory.as_ref()) - latest = await self._backend.latest(memory.artifact_id) - if exact != memory or exact.as_ref() != latest.as_ref(): - raise CapabilityNotSupportedError("head") - - async def _select_search_mode( - self, - requested: MemorySearchMode, - *, - memories: tuple[ArtifactRef, ...], - capabilities: MemoryCapabilities, - ) -> MemoryUsedSearchMode: - if requested == "fts": - if not capabilities.fts: - raise CapabilityNotSupportedError("fts") - return "fts" - - embedding_profile = None if self._embedding_model is None else self._embedding_model.profile - profile = capabilities.embedding_profile - profile_matches = profile is not None and embedding_profile == profile - vector_complete = ( - await self._backend.vector_complete(memories, profile) - if capabilities.vector and profile_matches and profile is not None - else False - ) - vector_available = capabilities.vector and profile_matches and vector_complete - hybrid_available = capabilities.hybrid and vector_available - - if requested == "vector": - if not vector_available: - raise CapabilityNotSupportedError("vector") - return "vector" - if requested == "hybrid": - if not hybrid_available: - raise CapabilityNotSupportedError("hybrid") - return "hybrid" - if requested == "auto": - if hybrid_available: - return "hybrid" - if capabilities.fts: - return "fts" - raise CapabilityNotSupportedError("fts") - raise _InvalidMemoryOperationError("search-mode") - - def _deduplicate_manifest( - self, - manifest: dict[str, MemoryManifestEntry], - current_by_entry: dict[str, MemoryEntryVersion], - ) -> tuple[list[MemoryChange], set[str]]: - groups: dict[bytes, list[str]] = {} - for item in manifest.values(): - if item.state == "active": - material = self._material_from_version(current_by_entry[item.entry_id]) - groups.setdefault(material.content_bytes, []).append(item.entry_id) - - changes: list[MemoryChange] = [] - changed_ids: set[str] = set() - for entry_ids in groups.values(): - ordered = sorted(entry_ids, key=str.encode) - for entry_id in ordered[1:]: - item = manifest[entry_id] - manifest[entry_id] = item.model_copy(update={"state": "inactive"}) - changes.append( - MemoryChange( - op="deactivate", - entry_id=entry_id, - from_entry_version_id=item.entry_version_id, - to_entry_version_id=None, - reason="dedupe", - ) - ) - changed_ids.add(entry_id) - return changes, changed_ids - - def _normalize_manifest_entries( - self, - base: Memory, - manifest: dict[str, MemoryManifestEntry], - current_by_entry: dict[str, MemoryEntryVersion], - *, - skip: set[str], - ) -> tuple[list[MemoryChange], list[MemoryEntryVersion]]: - changes: list[MemoryChange] = [] - new_versions: list[MemoryEntryVersion] = [] - for entry_id, previous in tuple(current_by_entry.items()): - if entry_id in skip: - continue - material = self._material_from_version(previous) - if _is_canonical_version(previous, material): - continue - version = self._new_entry_version( - memory_id=base.artifact_id, - entry_id=entry_id, - previous=previous, - material=material, - created_in_revision=base.revision + 1, - ) - item = manifest[entry_id] - manifest[entry_id] = _manifest_entry(version, state=item.state) - current_by_entry[entry_id] = version - new_versions.append(version) - changes.append( - MemoryChange( - op="revise", - entry_id=entry_id, - from_entry_version_id=previous.entry_version_id, - to_entry_version_id=version.entry_version_id, - reason="normalize", - ) - ) - return changes, new_versions - - async def _set_entry_state( - self, - memory: Memory, - *, - entries: Sequence[MemoryEntryVersion], - target_state: Literal["active", "inactive"], - reason: str | None, - ) -> Memory: - base = await self._canonical_base(memory) - current_entries = await self._validated_entries(base) - manifest = {entry.entry_id: entry for entry in base.content.manifest.entries} - current_by_entry = {entry.entry_id: entry for entry in current_entries} - normalized_reason = normalize_reason(reason) - changes: list[MemoryChange] = [] - - seen: set[tuple[str, str]] = set() - for entry in entries: - identity = (entry.entry_id, entry.entry_version_id) - if identity in seen: - continue - seen.add(identity) - entry_id = validate_identifier(entry.entry_id) - item = manifest.get(entry_id) - if item is None: - raise MemoryEntryNotFoundError(entry_id) - current = current_by_entry.get(entry_id) - if current is None: - raise MemoryEntryNotFoundError(entry_id) - if entry != current: - raise InvalidMemoryCitationError("entry-mismatch") - if item.state == target_state: - continue - manifest[entry_id] = item.model_copy(update={"state": target_state}) - changes.append( - MemoryChange( - op="reactivate" if target_state == "active" else "deactivate", - entry_id=entry_id, - from_entry_version_id=None if target_state == "active" else item.entry_version_id, - to_entry_version_id=item.entry_version_id if target_state == "active" else None, - reason=normalized_reason, - ) - ) - - if not changes: - return base - return await self._commit_existing_transition( - base=base, - manifest=manifest, - changes=changes, - current_by_entry=current_by_entry, - entry_versions=(), - growth=frozenset({"active_entries"}) if target_state == "active" else frozenset(), - ) - - async def _commit_existing_transition( - self, - *, - base: Memory, - manifest: dict[str, MemoryManifestEntry], - changes: Sequence[MemoryChange], - current_by_entry: dict[str, MemoryEntryVersion], - entry_versions: tuple[MemoryEntryVersion, ...], - growth: frozenset[MemoryCapacityDimension] = frozenset(), - ) -> Memory: - sorted_manifest = tuple(sorted(manifest.values(), key=lambda item: item.entry_id.encode("utf-8"))) - sorted_changes = tuple(sorted(changes, key=lambda change: change.entry_id.encode("utf-8"))) - content = MemoryContent(manifest=MemoryManifest(entries=sorted_manifest), changes=sorted_changes) - content_bytes = memory_content_bytes(content) - self._require_capacity(base, content, growth=growth, content_bytes=content_bytes) - memory = Memory( - artifact_id=base.artifact_id, - revision=base.revision + 1, - content=content, - lineage=ArtifactLineage(), - ) - projections = await self._prepare_projections( - base=base, - manifest_entries=sorted_manifest, - versions_by_entry=current_by_entry, - changed_version_ids=frozenset(version.entry_version_id for version in entry_versions), - ) - commit = MemoryCommit( - base=base, - memory=memory, - content_hash=sha256(content_bytes).hexdigest(), - entry_versions=entry_versions, - projections=projections, - ) - async with self._backend.begin() as unit_of_work: - committed = await unit_of_work.commit(commit) - self._cache_projections(committed, commit.projections) - return committed - - async def _validate_anchor( - self, - *, - memory_ref: ArtifactRef, - entry_id: str, - entry_version_id: str, - version: MemoryEntryVersion, - ) -> None: - memory = await self._backend.get(memory_ref) - item = next( - (candidate for candidate in memory.content.manifest.entries if candidate.entry_id == entry_id), - None, - ) - if ( - item is None - or item.entry_version_id != entry_version_id - or version.memory_artifact_id != memory_ref.artifact_id - or version.entry_id != entry_id - or version.entry_version_id != entry_version_id - ): - raise InvalidMemoryCitationError("expand-anchor") - material = self._material_from_version(version) - if material.content_hash != item.entry_content_hash or version.entry_content_hash != item.entry_content_hash: - raise InvalidMemoryCitationError("hash-mismatch") - - async def _prepare_projections( - self, - *, - base: Memory | None, - manifest_entries: tuple[MemoryManifestEntry, ...], - versions_by_entry: dict[str, MemoryEntryVersion], - changed_version_ids: frozenset[str], - ) -> tuple[MemoryProjection, ...]: - """Build active-head projections, reusing unchanged vectors and embedding only changes.""" - - previous = await self._previous_projections(base) - prepared: list[MemoryProjection] = [] - embed_indices: list[int] = [] - for item in manifest_entries: - if item.state != "active": - continue - replayed = previous.get(item.entry_version_id) - if replayed is not None and item.entry_version_id not in changed_version_ids: - # Entry versions are immutable, so the projection recorded for this - # version id is exactly what this revision carries; re-deriving it - # would re-analyze every unchanged entry on every write. - prepared.append(replayed) - continue - version = versions_by_entry[item.entry_id] - if version.entry_version_id != item.entry_version_id: - raise InvalidMemoryCitationError("projection-version") - prepared.append(MemoryProjection(entry_version=version, searchable_text=analyze_text(version.text))) - embed_indices.append(len(prepared) - 1) - return await self._attach_embeddings(tuple(prepared), embed_indices) - - async def _previous_projections(self, base: Memory | None) -> dict[str, MemoryProjection]: - if base is None: - return {} - reference = base.as_ref() - cached = self._previous_projection_cache - if cached is not None and cached[0] == reference: - active = {item.entry_version_id for item in base.content.manifest.entries if item.state == "active"} - if active <= cached[1].keys(): - return cached[1] - # A commit whose outer transaction rolled back leaves this cache behind - # while the stored revision never advanced; another writer can then reuse - # the same reference with a different entry set, so a cached revision - # missing an active version belongs to no stored revision. - projections = { - projection.entry_version.entry_version_id: projection - for projection in await self._backend.projections(reference) - } - self._previous_projection_cache = (reference, projections) - return projections - - def _cache_projections(self, committed: Memory, projections: tuple[MemoryProjection, ...]) -> None: - self._previous_projection_cache = ( - committed.as_ref(), - {projection.entry_version.entry_version_id: projection for projection in projections}, - ) - - async def _attach_embeddings( - self, - projections: tuple[MemoryProjection, ...], - embed_indices: Sequence[int], - ) -> tuple[MemoryProjection, ...]: - if not projections or not embed_indices or self._embedding_model is None: - return projections - capabilities = await self._backend.capabilities() - profile = capabilities.embedding_profile - if not capabilities.vector or profile is None: - return projections - embedding_profile = self._embedding_model.profile - if embedding_profile != profile: - return projections - try: - vectors = await self._embed_texts( - tuple(projections[index].entry_version.text for index in embed_indices), - profile, - ) - except (InferenceUnavailableError, InferenceTimeoutError): - return projections - updated = list(projections) - for index, vector in zip(embed_indices, vectors, strict=True): - projection = updated[index] - updated[index] = projection.model_copy( - update={ - "embedding": vector, - "embedding_content_hash": embedding_content_hash( - profile_id=profile.profile_id, - model=profile.model, - dimension=profile.dimension, - distance=profile.distance, - normalization=profile.normalization, - entry_content_hash=projection.entry_version.entry_content_hash, - ), - } - ) - return tuple(updated) - - async def _embed_texts( - self, - texts: tuple[str, ...], - profile: EmbeddingProfile, - *, - query: bool = False, - ) -> tuple[EmbeddingVector, ...]: - embedding_model = self._embedding_model - if embedding_model is None or embedding_model.profile != profile: - raise CapabilityNotSupportedError("embedding-profile") - result = await embed_query(embedding_model, texts) if query else await embedding_model.embed(texts) - vectors = result.vectors - if len(vectors) != len(texts): - raise InvalidEmbeddingError("count") - return tuple( - canonical_embedding( - vector, - dimension=profile.dimension, - normalization=profile.normalization, - ) - for vector in vectors - ) - - @overload - async def _canonical_base(self, memory: None) -> None: ... - - @overload - async def _canonical_base(self, memory: Memory) -> Memory: ... - - async def _canonical_base(self, memory: Memory | None) -> Memory | None: - if memory is None: - return None - exact = await self._backend.get(memory.as_ref()) - if exact != memory: - raise InvalidMemoryCitationError("base-mismatch") - latest = await self._backend.latest(memory.artifact_id) - if latest != exact: - raise RevisionConflictError(memory, latest) - return exact - - async def _canonical_memory(self, memory: Memory) -> Memory: - exact = await self._backend.get(memory.as_ref()) - if exact != memory: - raise InvalidMemoryCitationError("memory-mismatch") - return exact - - async def _canonical_operation_evidence( - self, - sources: Sequence[Source], - artifacts: Sequence[Artifact[object]], - ) -> _OperationEvidence: - canonical_sources: list[Source] = [] - for source in sources: - if self._source_resolver is None: - raise InvalidMemoryEvidenceError("source-resolver") - _append_unique(canonical_sources, await self._source_resolver.get(source)) - - canonical_artifacts: list[Artifact[object]] = [] - for artifact in artifacts: - if artifact.family == "prompt": - raise InvalidMemoryEvidenceError("prompt-configuration") - if self._artifact_resolver is None: - raise InvalidMemoryEvidenceError("artifact-resolver") - _append_unique(canonical_artifacts, await self._artifact_resolver.get(artifact)) - return _OperationEvidence( - sources=tuple(canonical_sources), - artifacts=tuple(canonical_artifacts), - ) - - async def _validated_entries(self, memory: Memory) -> tuple[MemoryEntryVersion, ...]: - versions = await self._backend.entries(memory.as_ref()) - versions_by_id = {version.entry_version_id: version for version in versions} - if len(versions_by_id) != len(versions): - raise InvalidMemoryCitationError("duplicate-versions") - - ordered: list[MemoryEntryVersion] = [] - for item in memory.content.manifest.entries: - version = versions_by_id.get(item.entry_version_id) - if version is None: - raise InvalidMemoryCitationError("missing-version") - if version.memory_artifact_id != memory.artifact_id or version.entry_id != item.entry_id: - raise InvalidMemoryCitationError("cross-identity") - material = self._material_from_version(version) - if material.content_hash != item.entry_content_hash: - raise InvalidMemoryCitationError("hash-mismatch") - ordered.append(version) - return tuple(ordered) - - async def _candidates( - self, - mode: Literal["append", "extract"], - entries: tuple[MemoryEntryInput, ...], - evidence: _OperationEvidence, - current_entries: tuple[MemoryEntryVersion, ...], - *, - active_version_ids: frozenset[str], - ) -> tuple[MemoryEntryInput, ...]: - if mode == "append": - return entries - if self._candidate_pipeline is None: - raise CapabilityNotSupportedError("extract") - bounded = tuple(entry for entry in current_entries if entry.entry_version_id in active_version_ids) - return await self._candidate_pipeline.extract( - MemoryCandidateRequest( - sources=evidence.sources, - artifacts=evidence.artifacts, - current_entries=bounded, - ) - ) - - async def _assess_write( - self, - base: Memory | None, - candidates: tuple[MemoryEntryInput, ...], - evidence: _OperationEvidence, - current_entries: tuple[MemoryEntryVersion, ...] | None, - ) -> MemoryWriteAssessment | None: - """Ask the configured gate about one candidate set; ``None`` means no gate is active.""" - - if self._write_gate is None: - return None - projection = await self._gate_evidence(base, candidates, evidence, current_entries) - if projection.rejection is not None: - _log_gate_assessment(projection.rejection) - return projection.rejection - try: - return await self._write_gate.assess( - MemoryWriteGateRequest( - candidates=tuple(candidate.text for candidate in candidates), - evidence=projection.entries, - expected_revision=None if base is None else base.revision, - ) - ) - except Exception: - return MemoryWriteAssessment( - verdict=MemoryWriteVerdict.ACCEPT, - policy_id=self._write_gate.policy_id, - used_fallback=True, - ) - - async def _gate_evidence( - self, - base: Memory | None, - candidates: tuple[MemoryEntryInput, ...], - evidence: _OperationEvidence, - current_entries: tuple[MemoryEntryVersion, ...] | None, - ) -> _GateEvidenceProjection: - builder = _GateEvidenceBuilder(self._write_gate_policy_id(), []) - for candidate in await self._gate_candidate_evidence(base, candidates, evidence, current_entries): - for entry in await self._candidate_gate_evidence(candidate): - if rejection := builder.append(entry): - return _GateEvidenceProjection(tuple(builder.entries), rejection) - return builder.projection() - - async def _gate_candidate_evidence( - self, - base: Memory | None, - candidates: tuple[MemoryEntryInput, ...], - evidence: _OperationEvidence, - current_entries: tuple[MemoryEntryVersion, ...] | None, - ) -> tuple[_GateCandidateEvidence, ...]: - current_by_entry = {} if current_entries is None else {entry.entry_id: entry for entry in current_entries} - targeted: set[str] = set() - resolved: list[_GateCandidateEvidence] = [] - for index, candidate in enumerate(candidates, start=1): - previous = None - if candidate.entry is not None: - _, previous = await self._claim_revision_target(candidate, base, current_by_entry, targeted) - material = await self._material_from_candidate( - candidate, - evidence.sources, - evidence.artifacts, - previous=previous, - ) - resolved.append(_GateCandidateEvidence(index=index, material=material)) - return tuple(resolved) - - async def _candidate_gate_evidence(self, candidate: _GateCandidateEvidence) -> tuple[_GateEvidenceEntry, ...]: - entries = [ - await self._source_ref_gate_evidence(candidate.index, source) for source in candidate.material.sources - ] - entries.extend([ - await self._artifact_ref_gate_evidence(candidate.index, artifact) - for artifact in candidate.material.artifacts - ]) - return tuple(entries) - - def _source_gate_evidence(self, candidate_index: int, source: Source) -> _GateEvidenceEntry: - ref = self._source_refs((source,))[0] - identity = _candidate_gate_identity(candidate_index, f"source:{ref.source_type}:{ref.source_id}") - content = _source_gate_content(source, self._source_resolver) - if isinstance(content, str) and content.strip(): - return _bounded_gate_evidence(identity, content) - return _incomplete_gate_evidence(identity) - - @staticmethod - def _artifact_gate_evidence(candidate_index: int, artifact: Artifact[object]) -> _GateEvidenceEntry: - ref = artifact.as_ref() - return _bounded_gate_evidence( - _candidate_gate_identity(candidate_index, f"artifact:{ref.family}:{ref.artifact_id}@{ref.revision}"), - _artifact_gate_content(artifact), - ) - - async def _source_ref_gate_evidence(self, candidate_index: int, ref: SourceRef) -> _GateEvidenceEntry: - identity = _candidate_gate_identity(candidate_index, f"source:{ref.source_type}:{ref.source_id}") - if self._source_resolver is None: - return _incomplete_gate_evidence(identity) - try: - return self._source_gate_evidence(candidate_index, await self._source_resolver.get_ref(ref)) - except Exception: - return _incomplete_gate_evidence(identity) - - async def _artifact_ref_gate_evidence(self, candidate_index: int, ref: ArtifactRef) -> _GateEvidenceEntry: - identity = _candidate_gate_identity(candidate_index, f"artifact:{ref.family}:{ref.artifact_id}@{ref.revision}") - if self._artifact_resolver is None: - return _incomplete_gate_evidence(identity) - try: - return self._artifact_gate_evidence(candidate_index, await self._artifact_resolver.get_ref(ref)) - except Exception: - return _incomplete_gate_evidence(identity) - - def _write_gate_policy_id(self) -> str: - if self._write_gate is not None: - return self._write_gate.policy_id - return "memory.write-gate" - - async def _prepare_commit( - self, - *, - base: Memory | None, - candidates: tuple[MemoryEntryInput, ...], - evidence: _OperationEvidence, - current_entries: tuple[MemoryEntryVersion, ...] | None, - flagged_reason: str | None = None, - ) -> MemoryCommit | None: - memory_id = base.artifact_id if base is not None else self._new_id("memory") - next_revision = 1 if base is None else base.revision + 1 - manifest = {} if base is None else {entry.entry_id: entry for entry in base.content.manifest.entries} - current_by_entry = {} if current_entries is None else {entry.entry_id: entry for entry in current_entries} - new_versions: list[MemoryEntryVersion] = [] - changes: list[MemoryChange] = [] - targeted: set[str] = set() - # The manifest records each active entry's validated content hash, so exact - # duplicate detection compares hashes; rebuilding every body here would cost - # one material per entry to answer the same question. - new_content: set[str] = {item.entry_content_hash for item in manifest.values() if item.state == "active"} - - for candidate in candidates: - if candidate.entry is None: - material = await self._material_from_candidate(candidate, evidence.sources, evidence.artifacts) - if material.content_hash in new_content: - continue - new_content.add(material.content_hash) - entry_id = self._new_id("entry") - if entry_id in manifest: - raise _InvalidMemoryOperationError("id-collision") - version = self._new_entry_version( - memory_id=memory_id, - entry_id=entry_id, - previous=None, - material=material, - created_in_revision=next_revision, - ) - manifest[entry_id] = _manifest_entry(version, state="active") - current_by_entry[entry_id] = version - new_versions.append(version) - changes.append( - MemoryChange( - op="add", - entry_id=entry_id, - from_entry_version_id=None, - to_entry_version_id=version.entry_version_id, - reason=_annotate_reason(candidate.reason, flagged_reason), - ) - ) - continue - - entry_id, previous = await self._claim_revision_target(candidate, base, current_by_entry, targeted) - item = manifest.get(entry_id) - if item is None: - raise MemoryEntryNotFoundError(entry_id) - if item.state == "inactive": - raise MemoryEntryInactiveError(entry_id) - material = await self._material_from_candidate( - candidate, - evidence.sources, - evidence.artifacts, - previous=previous, - ) - previous_material = self._material_from_version(previous) - if ( - material.content_hash == previous_material.content_hash - and material.content_bytes == previous_material.content_bytes - ): - continue - version = self._new_entry_version( - memory_id=memory_id, - entry_id=entry_id, - previous=previous, - material=material, - created_in_revision=next_revision, - ) - manifest[entry_id] = _manifest_entry(version, state="active") - current_by_entry[entry_id] = version - new_versions.append(version) - changes.append( - MemoryChange( - op="revise", - entry_id=entry_id, - from_entry_version_id=previous.entry_version_id, - to_entry_version_id=version.entry_version_id, - reason=_annotate_reason(candidate.reason, flagged_reason), - ) - ) - - if not changes: - return None - - sorted_manifest = tuple(sorted(manifest.values(), key=lambda item: item.entry_id.encode("utf-8"))) - sorted_changes = tuple(sorted(changes, key=lambda change: change.entry_id.encode("utf-8"))) - content = MemoryContent(manifest=MemoryManifest(entries=sorted_manifest), changes=sorted_changes) - content_bytes = memory_content_bytes(content) - self._require_capacity( - base, - content, - growth=frozenset(dimension for dimension, _ in self._capacity_limits()), - content_bytes=content_bytes, - ) - memory = Memory( - artifact_id=memory_id, - revision=next_revision, - content=content, - lineage=ArtifactLineage( - sources=self._source_refs(evidence.sources), - artifacts=tuple(artifact.as_ref() for artifact in evidence.artifacts) + _extraction_prompt_refs(), - ), - ) - projections = await self._prepare_projections( - base=base, - manifest_entries=sorted_manifest, - versions_by_entry=current_by_entry, - changed_version_ids=frozenset(version.entry_version_id for version in new_versions), - ) - return MemoryCommit( - base=base, - memory=memory, - content_hash=sha256(content_bytes).hexdigest(), - entry_versions=tuple(new_versions), - projections=projections, - ) - - async def _claim_revision_target( - self, - candidate: MemoryEntryInput, - base: Memory | None, - current_by_entry: dict[str, MemoryEntryVersion], - targeted: set[str], - ) -> tuple[str, MemoryEntryVersion]: - entry = candidate.entry - if entry is None: - raise InvalidMemoryCitationError("entry-missing") - entry_id = validate_identifier(entry.entry_id) - if entry_id in targeted: - raise _InvalidMemoryOperationError("duplicate-target") - previous = current_by_entry.get(entry_id) - if previous is None and base is not None: - # A revise needs the exact version it targets; the common append path - # never loads the current entry set at all. - current_by_entry.update({value.entry_id: value for value in await self._validated_entries(base)}) - previous = current_by_entry.get(entry_id) - if previous is None: - raise MemoryEntryNotFoundError(entry_id) - if entry != previous: - raise InvalidMemoryCitationError("entry-mismatch") - targeted.add(entry_id) - return entry_id, previous - - async def _material_from_candidate( - self, - candidate: MemoryEntryInput, - allowed_sources: Sequence[Source], - allowed_artifacts: Sequence[Artifact[object]], - *, - previous: MemoryEntryVersion | None = None, - ) -> _EntryMaterial: - previous_artifacts = () if previous is None else previous.artifacts - candidate_sources = await self._canonical_candidate_sources(candidate.sources, allowed_sources) - candidate_source_refs = self._source_refs(candidate_sources) - candidate_source_evidence = self._source_evidence(candidate_sources) - candidate_artifacts = await self._canonical_candidate_artifacts( - candidate.artifacts, - allowed_artifacts, - previous_artifacts, - ) - if previous is None: - sources = candidate_source_refs - source_evidence = candidate_source_evidence - artifacts = candidate_artifacts - else: - # Revises retain predecessor evidence and add current candidate evidence. - sources = (*previous.sources, *candidate_source_refs) - source_evidence = (*previous.source_evidence, *candidate_source_evidence) - artifacts = (*previous.artifacts, *candidate_artifacts) - try: - return self._entry_material( - kind=candidate.kind, - text=candidate.text, - sources=sources, - source_evidence=source_evidence, - artifacts=artifacts, - ) - except (TypeError, ValueError) as error: - raise InvalidMemoryCandidateError( - "canonical", - str(error), - canonical_code=canonical_error_code(error), - ) from error - - async def _canonical_candidate_sources( - self, - values: Sequence[Source], - allowed: Sequence[Source], - ) -> tuple[Source, ...]: - result: list[Source] = [] - for value in values: - canonical = value if self._source_resolver is None else await self._source_resolver.get(value) - match = _equal_member(canonical, allowed) - if match is None: - raise InvalidMemoryEvidenceError("source-outside") - _append_unique(result, match) - return tuple(result) - - async def _canonical_candidate_artifacts( - self, - values: Sequence[Artifact[object]], - allowed: Sequence[Artifact[object]], - previous: Sequence[ArtifactRef], - ) -> tuple[ArtifactRef, ...]: - result: list[ArtifactRef] = [] - allowed_refs = tuple(artifact.as_ref() for artifact in allowed) - for value in values: - canonical = _matching_allowed_artifact(value, allowed) - if canonical is None: - canonical = value if self._artifact_resolver is None else await self._artifact_resolver.get(value) - reference = canonical.as_ref() - if reference not in (*allowed_refs, *previous): - raise InvalidMemoryEvidenceError("artifact-outside") - if reference not in result: - result.append(reference) - return tuple(result) - - def _material_from_version(self, version: MemoryEntryVersion) -> _EntryMaterial: - return self._entry_material( - kind=version.kind, - text=version.text, - sources=version.sources, - source_evidence=version.source_evidence, - artifacts=version.artifacts, - ) - - def _entry_material( - self, - *, - kind: str, - text: str, - sources: Sequence[SourceRef], - source_evidence: Sequence[MemoryEvidenceSnapshot], - artifacts: Sequence[ArtifactRef], - ) -> _EntryMaterial: - normalized_kind = normalize_kind(kind) - normalized_text = normalize_text(text) - canonical_sources = _canonical_source_refs(sources) - canonical_evidence = _canonical_source_evidence(canonical_sources, source_evidence) - canonical_artifacts = _canonical_artifact_refs(artifacts) - source_refs = tuple(source.model_dump(mode="json") for source in canonical_sources) - artifact_refs = tuple(artifact.model_dump(mode="json") for artifact in canonical_artifacts) - content_bytes = entry_content_bytes( - kind=normalized_kind, - text=normalized_text, - source_refs=source_refs, - artifact_refs=artifact_refs, - ) - return _EntryMaterial( - kind=normalized_kind, - text=normalized_text, - sources=canonical_sources, - source_evidence=canonical_evidence, - artifacts=canonical_artifacts, - content_bytes=content_bytes, - content_hash=entry_content_hash( - kind=normalized_kind, - text=normalized_text, - source_refs=source_refs, - artifact_refs=artifact_refs, - ), - ) - - def _source_refs(self, sources: Sequence[Source]) -> tuple[SourceRef, ...]: - if not sources: - return () - if self._source_resolver is None: - raise InvalidMemoryEvidenceError("source-adapter") - keyed: dict[bytes, SourceRef] = {} - for source in sources: - try: - reference = self._source_resolver.as_ref(source) - except (KeyError, LookupError, TypeError, ValueError): - raise InvalidMemoryEvidenceError("source-adapter") from None - keyed.setdefault(canonical_json(reference.model_dump(mode="json")), reference) - return tuple(keyed[key] for key in sorted(keyed)) - - def _source_evidence(self, sources: Sequence[Source]) -> tuple[MemoryEvidenceSnapshot, ...]: - """Snapshot Definition-owned declarations without changing content identity.""" - - if not sources: - return () - if self._source_resolver is None: - raise InvalidMemoryEvidenceError("source-adapter") - resolver = getattr(self._source_resolver, "memory_evidence", None) - keyed: dict[bytes, MemoryEvidenceSnapshot] = {} - for source in sources: - try: - reference = self._source_resolver.as_ref(source) - declaration = MemoryEvidenceDeclaration() if resolver is None else resolver(source) - snapshot = MemoryEvidenceSnapshot(source=reference, declaration=declaration) - except (KeyError, LookupError, TypeError, ValueError): - raise InvalidMemoryEvidenceError("source-adapter") from None - keyed.setdefault(canonical_json(reference.model_dump(mode="json")), snapshot) - return tuple(keyed[key] for key in sorted(keyed)) - - def _new_entry_version( - self, - *, - memory_id: str, - entry_id: str, - previous: MemoryEntryVersion | None, - material: _EntryMaterial, - created_in_revision: int, - ) -> MemoryEntryVersion: - return MemoryEntryVersion( - memory_artifact_id=memory_id, - entry_id=entry_id, - entry_version_id=self._new_id("version"), - version=1 if previous is None else previous.version + 1, - previous_version_id=None if previous is None else previous.entry_version_id, - kind=material.kind, - text=material.text, - sources=material.sources, - source_evidence=material.source_evidence, - artifacts=material.artifacts, - entry_content_hash=material.content_hash, - created_in_revision=created_in_revision, - ) - - def _new_id(self, kind: str) -> str: - return validate_identifier(self._id_factory(kind)) - - -def _select_remember_mode( - mode: MemoryRememberMode, - *, - has_entries: bool, - has_evidence: bool, -) -> Literal["append", "extract"]: - if mode == "append": - if not has_entries: - raise _InvalidMemoryOperationError("append-entries") - return "append" - if mode == "extract": - if has_entries or not has_evidence: - raise _InvalidMemoryOperationError("extract-evidence") - return "extract" - if mode != "auto": - raise InvalidMemoryCandidateError("remember-mode", mode) - if has_entries: - return "append" - if has_evidence: - return "extract" - raise _InvalidMemoryOperationError("no-work") - - -def _manifest_entry(version: MemoryEntryVersion, *, state: Literal["active", "inactive"]) -> MemoryManifestEntry: - return MemoryManifestEntry( - entry_id=version.entry_id, - entry_version_id=version.entry_version_id, - entry_content_hash=version.entry_content_hash, - state=state, - ) - - -def _raise_if_write_held(plan: MemoryWritePlan) -> None: - decision = plan.decision - if decision is None or decision.verdict is not MemoryWriteVerdict.HOLD: - return - code = "unspecified" if decision.code is None else decision.code.value - raise MemoryWriteRejectedError(code, decision.reason) - - -def _bounded_gate_evidence(identity: str, content: str) -> _GateEvidenceEntry: - normalized = normalize_text(content) - return _GateEvidenceEntry( - text=f"{identity}\n{normalized[:_GATE_EVIDENCE_TEXT_LIMIT]}", - complete=len(normalized) <= _GATE_EVIDENCE_TEXT_LIMIT, - ) - - -def _incomplete_gate_evidence(identity: str) -> _GateEvidenceEntry: - return _GateEvidenceEntry(text=identity, complete=False) - - -def _candidate_gate_identity(candidate_index: int, identity: str) -> str: - return f"candidate:{candidate_index} {identity}" - - -def _source_gate_content(source: Source, resolver: _SourceResolver | None) -> str | None: - content = getattr(source, "content", None) - if isinstance(content, str): - return content - if isinstance(source, SourceObservation): - try: - evidence = TextEvidence.model_validate(source.projection(TEXT_EVIDENCE_PROJECTION_KEY)) - except Exception: - return None - return evidence.content - if resolver is not None: - try: - evidence = TextEvidence.model_validate(resolver.project(source, TEXT_EVIDENCE_PROJECTION_KEY)) - except Exception: - return None - return evidence.content - return None - - -def _log_gate_assessment(assessment: MemoryWriteAssessment) -> None: - event = { - MemoryWriteVerdict.HOLD: "memory.write-gate.hold", - MemoryWriteVerdict.FLAG: "memory.write-gate.flag", - }.get(assessment.verdict, "memory.write-gate.assess") - log_safely( - logger, - logging.INFO, - "Memory write gate assessed a pending write", - extra={ - "event": event, - "decision_kind": "memory.write-gate", - "policy_id": assessment.policy_id, - "verdict": assessment.verdict.value, - "code": None if assessment.code is None else assessment.code.value, - "used_fallback": assessment.used_fallback, - }, - ) - - -def _artifact_gate_content(artifact: Artifact[object]) -> str: - content = artifact.content - if isinstance(content, BaseModel): - return content.model_dump_json() - return str(content) - - -def _matching_allowed_artifact( - value: Artifact[object], - allowed: Sequence[Artifact[object]], -) -> Artifact[object] | None: - reference = value.as_ref() - for artifact in allowed: - if artifact.as_ref() == reference: - return artifact - # Pydantic validates MemoryEntryInput artifacts through the generic Artifact[object] - # annotation, which strips the concrete subclass family. Fall back to the operation's - # canonical evidence set when the revision identity and body match exactly. - if value.family != Artifact.family: - return None - matches = [ - artifact - for artifact in allowed - if artifact.artifact_id == value.artifact_id - and artifact.revision == value.revision - and artifact.content == value.content - ] - return matches[0] if len(matches) == 1 else None - - -def _canonical_source_refs(values: Sequence[SourceRef]) -> tuple[SourceRef, ...]: - keyed = {canonical_json(value.model_dump(mode="json")): value for value in values} - return tuple(keyed[key] for key in sorted(keyed)) - - -def _canonical_source_evidence( - sources: Sequence[SourceRef], - values: Sequence[MemoryEvidenceSnapshot], -) -> tuple[MemoryEvidenceSnapshot, ...]: - """Align provenance snapshots with canonical source identity. - - Entries written before the lifecycle projection existed have no snapshots. - They deliberately derive the neutral declaration instead of acquiring a - trust claim from a current adapter configuration. - """ - - declared = {canonical_json(value.source.model_dump(mode="json")): value for value in values} - canonical: list[MemoryEvidenceSnapshot] = [] - for source in sources: - key = canonical_json(source.model_dump(mode="json")) - canonical.append( - declared.get( - key, - MemoryEvidenceSnapshot(source=source, declaration=MemoryEvidenceDeclaration()), - ) - ) - return tuple(canonical) - - -def _canonical_artifact_refs(values: Sequence[ArtifactRef]) -> tuple[ArtifactRef, ...]: - keyed = {canonical_json(value.model_dump(mode="json")): value for value in values} - return tuple(keyed[key] for key in sorted(keyed)) - - -def _is_canonical_version(version: MemoryEntryVersion, material: _EntryMaterial) -> bool: - return ( - version.kind == material.kind - and version.text == material.text - and version.sources == material.sources - and version.artifacts == material.artifacts - ) - - -def _append_unique(values: list[ValueT], value: ValueT) -> None: - if not any(current == value for current in values): - values.append(value) - - -def _equal_member(value: ValueT, values: Sequence[ValueT]) -> ValueT | None: - return next((current for current in values if current == value), None) - - -def _default_id(kind: str) -> str: - prefixes = {"memory": "mem_art", "entry": "mem_ent", "version": "mem_ver"} - try: - prefix = prefixes[kind] - except KeyError: - raise InvalidMemoryCandidateError("identity-kind", kind) from None - return f"{prefix}_{uuid4().hex}" diff --git a/src/powercontext/builtin/persistence/memory.py b/src/powercontext/builtin/persistence/memory.py deleted file mode 100644 index 5cb830425f..0000000000 --- a/src/powercontext/builtin/persistence/memory.py +++ /dev/null @@ -1,1057 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""SQLAlchemy implementation of the built-in Memory backend contract.""" - -from __future__ import annotations - -from collections.abc import AsyncIterator, Iterable, Iterator, Mapping -from contextlib import AbstractAsyncContextManager, asynccontextmanager -from dataclasses import dataclass, replace -from typing import Any, ClassVar, TypeVar - -from pydantic import RootModel -from sqlalchemy import delete, insert, select, tuple_ -from sqlalchemy.ext.asyncio import AsyncConnection - -from powercontext.artifacts import ArtifactDraft, ArtifactRef -from powercontext.builtin.artifacts.memory import ( - EmbeddingProfile, - InvalidEmbeddingError, - Memory, - MemoryCapabilities, - MemoryCommit, - MemoryContent, - MemoryEntryVersion, - MemoryEvidenceSnapshot, - MemoryHit, - MemoryLifecycleProjection, - MemoryProjection, - MemoryRevisionChanges, - MemorySearchChannels, - MemorySearchRequest, - MemoryUnitOfWork, -) -from powercontext.builtin.artifacts.memory.canonical import ( - canonical_embedding, - embedding_content_hash, - entry_content_hash, - memory_content_hash, -) -from powercontext.builtin.artifacts.memory.errors import ( - CapabilityNotSupportedError, - InvalidMemoryCitationError, - MemoryBackendConfigurationError, -) -from powercontext.builtin.artifacts.search import analyze_text -from powercontext.builtin.inference import EmbeddingModel -from powercontext.builtin.persistence.artifacts import ArtifactRepository -from powercontext.builtin.persistence.codec import dump_model, load_model, stored_bytes -from powercontext.builtin.persistence.database import SELECTION_BATCH_SIZE, AsyncDatabase -from powercontext.builtin.persistence.errors import RepositoryNotFoundError -from powercontext.builtin.persistence.memory_index import MemoryIndex, NoMemoryIndex -from powercontext.builtin.persistence.tables import ( - ARTIFACT_HEADS_TABLE, - ARTIFACT_TAGS_TABLE, - MEMORY_ENTRY_EVIDENCE_TABLE, - MEMORY_ENTRY_HEADS_TABLE, - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE, - MEMORY_ENTRY_VERSIONS_TABLE, -) -from powercontext.builtin.persistence.tags import tag_predicate -from powercontext.builtin.tags import TagFilter -from powercontext.errors import ArtifactNotFoundError -from powercontext.sources import ( - MemoryEvidenceAuthority, - MemoryEvidenceDeclaration, - MemoryEvidenceVerification, - SourceRef, -) - - -class _SourceRefs(RootModel[tuple[SourceRef, ...]]): - pass - - -class _ArtifactRefs(RootModel[tuple[ArtifactRef, ...]]): - pass - - -# The evidence lookup binds three identity columns per entry, so its rows chunk at -# the scope's share of the shared bind budget. The single-column entry-version -# lookups bind one value per row and chunk at the row budget less the fixed -# parameters that accompany the batch: the scope, the artifact identity, and the -# correlated tag predicate, which binds about two values per requested label. -# ``TagFilter`` admits at most 16 labels, so the reserve covers that maximum with -# headroom instead of landing exactly on the budget. -_EVIDENCE_BIND_RESERVE = 1 -_ENTRY_BIND_RESERVE = 40 -_ENTRY_SELECTION_BATCH_SIZE = max(1, SELECTION_BATCH_SIZE - _ENTRY_BIND_RESERVE) -_EVIDENCE_SELECTION_BATCH_SIZE = max(1, (SELECTION_BATCH_SIZE - _EVIDENCE_BIND_RESERVE) // 3) - -_BatchItem = TypeVar("_BatchItem") - - -def _batched(items: Iterable[_BatchItem], size: int, /) -> Iterator[tuple[_BatchItem, ...]]: - """Yield disjoint chunks of at most ``size`` items. - - One statement may bind only a bounded number of values, so a lookup over an - unbounded manifest has to be chunked. Sharing this helper keeps every lookup - on the same budget instead of letting them drift apart. - """ - - batch: list[_BatchItem] = [] - for item in items: - batch.append(item) - if len(batch) >= size: - yield tuple(batch) - batch = [] - if batch: - yield tuple(batch) - - -class _MemoryDraft(ArtifactDraft[MemoryContent]): - family: ClassVar[str] = Memory.family - - -@dataclass(frozen=True, slots=True) -class _RebuildSnapshot: - memory_ref: ArtifactRef - projections: tuple[MemoryProjection, ...] - lifecycle: tuple[MemoryLifecycleProjection, ...] - - -class _InvalidMemoryCommitError(MemoryBackendConfigurationError): - def __init__(self, code: str, actual: str | None = None) -> None: - details = { - "artifact-result": "generic Artifact result differs from prepared revision", - "base-identity": "base and revision identities differ", - "complete": "unit of work is already complete", - "content-hash": "content hash does not match canonical content", - "family": "family is not memory", - "memory-type": f"expected Memory, got {actual}", - "projection": "active manifest and projections differ", - "revision": "revision is not the next prepared revision", - } - super().__init__(f"invalid relational Memory commit: {details[code]}") - - -class _MemoryProjectionRebuildError(MemoryBackendConfigurationError): - def __init__(self, code: str) -> None: - messages = { - "snapshot": "authoritative Memory heads changed while projections were rebuilt", - "vector": "the configured Memory index has no vector capability", - "profile": "the rebuild embedding profile does not match the Memory index", - } - super().__init__(messages[code]) - - -class RelationalMemoryBackend: - """Use shared Artifact revisions plus Memory-owned entry projections.""" - - def __init__( - self, - *, - database: AsyncDatabase, - scope_id: str, - artifacts: ArtifactRepository, - index: MemoryIndex | None = None, - connection: AsyncConnection | None = None, - ) -> None: - self._database = database - self._scope_id = scope_id - self._artifacts = artifacts - self._index = NoMemoryIndex() if index is None else index - self._bound_connection = connection - - async def capabilities(self) -> MemoryCapabilities: - return self._index.capabilities - - async def get(self, memory: ArtifactRef, /) -> Memory: - if memory.family != Memory.family: - raise ArtifactNotFoundError(memory) - async with self._database.connection(self._bound_connection) as connection: - try: - artifact = await self._artifacts.get(connection, self._scope_id, memory) - except RepositoryNotFoundError: - raise ArtifactNotFoundError(memory) from None - return _require_memory(artifact) - - async def latest(self, artifact_id: str, /) -> Memory: - async with self._database.connection(self._bound_connection) as connection: - try: - artifact = await self._artifacts.latest( - connection, - self._scope_id, - Memory.family, - artifact_id, - ) - except RepositoryNotFoundError: - raise ArtifactNotFoundError(artifact_id) from None - return _require_memory(artifact) - - async def tagged_entry_ids(self, memory: ArtifactRef, tag_filter: TagFilter) -> frozenset[str]: - canonical = await self.get(memory) - versions = tuple(entry.entry_version_id for entry in canonical.content.manifest.entries) - table = MEMORY_ENTRY_VERSIONS_TABLE - tagged: set[str] = set() - async with self._database.connection(self._bound_connection) as connection: - for batch in _batched(dict.fromkeys(versions), _ENTRY_SELECTION_BATCH_SIZE): - rows = await connection.scalars( - select(table.c.entry_id).where( - table.c.scope_id == self._scope_id, - table.c.memory_artifact_id == memory.artifact_id, - table.c.entry_version_id.in_(batch), - tag_predicate( - self._scope_id, - "memory", - table.c.memory_artifact_id, - "memory_entry", - table.c.entry_id, - tag_filter, - ), - ) - ) - tagged.update(rows) - return frozenset(tagged) - - async def any_tagged_entry_ids(self, memory: ArtifactRef, /) -> frozenset[str]: - await self.get(memory) - async with self._database.connection(self._bound_connection) as connection: - return await self._any_tagged_entry_ids(connection, memory) - - async def _any_tagged_entry_ids( - self, connection: AsyncConnection, memory: ArtifactRef, *, for_update: bool = False - ) -> frozenset[str]: - table = ARTIFACT_TAGS_TABLE - query = select(table.c.target_id).where( - table.c.scope_id == self._scope_id, - table.c.family == Memory.family, - table.c.artifact_id == memory.artifact_id, - table.c.target_type == "memory_entry", - ) - return frozenset(await connection.scalars(query.with_for_update() if for_update else query)) - - async def entries(self, memory: ArtifactRef, /) -> tuple[MemoryEntryVersion, ...]: - canonical = await self.get(memory) - version_ids = tuple(item.entry_version_id for item in canonical.content.manifest.entries) - if not version_ids: - return () - async with self._database.connection(self._bound_connection) as connection: - return await self._hydrate_entry_versions(connection, memory, version_ids) - - async def lifecycle_projections(self, memory: ArtifactRef, /) -> tuple[MemoryLifecycleProjection, ...]: - """Load the internal neutral lifecycle projection for one exact Memory head. - - Rows are keyed by entry and re-stamped only by the revision that changes - them, so the read selects the artifact's rows and proves completeness by - entry identity, exactly as the active-head projection does. - """ - - canonical = await self.get(memory) - manifest = canonical.content.manifest.entries - version_ids = tuple(item.entry_version_id for item in manifest) - if not version_ids: - return () - async with self._database.connection(self._bound_connection) as connection: - versions = await self._hydrate_entry_versions(connection, memory, version_ids) - by_version = {entry.entry_version_id: entry for entry in versions} - rows = ( - await connection.execute( - select(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE) - .where( - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.scope_id == self._scope_id, - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.memory_artifact_id == memory.artifact_id, - ) - .order_by(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.entry_id) - ) - ).mappings() - by_entry = {str(row["entry_id"]): row for row in rows} - # The manifest owns each entry's state and version, so derive those from the - # requested revision. Deactivating or reactivating an entry keeps its version - # identity, so a row written by a later revision would otherwise pass an - # identity check and report that later state under a historical reference. - projections: list[MemoryLifecycleProjection] = [] - for item in manifest: - row = by_entry.get(item.entry_id) - projections.append( - _lifecycle_projection( - memory, - item.state, - by_version[item.entry_version_id], - row=row, - ) - ) - return tuple(projections) - - async def projections(self, memory: ArtifactRef, /) -> tuple[MemoryProjection, ...]: - canonical = await self.get(memory) - async with self._database.connection(self._bound_connection) as connection: - rows = ( - ( - await connection.execute( - select(MEMORY_ENTRY_HEADS_TABLE, MEMORY_ENTRY_VERSIONS_TABLE) - .join( - MEMORY_ENTRY_VERSIONS_TABLE, - (MEMORY_ENTRY_VERSIONS_TABLE.c.scope_id == MEMORY_ENTRY_HEADS_TABLE.c.scope_id) - & ( - MEMORY_ENTRY_VERSIONS_TABLE.c.memory_artifact_id - == MEMORY_ENTRY_HEADS_TABLE.c.memory_artifact_id - ) - & ( - MEMORY_ENTRY_VERSIONS_TABLE.c.entry_version_id - == MEMORY_ENTRY_HEADS_TABLE.c.entry_version_id - ), - ) - .where( - MEMORY_ENTRY_HEADS_TABLE.c.scope_id == self._scope_id, - MEMORY_ENTRY_HEADS_TABLE.c.memory_artifact_id == memory.artifact_id, - ) - .order_by(MEMORY_ENTRY_HEADS_TABLE.c.entry_id) - ) - ) - .mappings() - .all() - ) - entries = await self._hydrate_source_evidence(connection, tuple(_decode_entry(row) for row in rows)) - projections = tuple( - MemoryProjection( - entry_version=entry, - searchable_text=str(row["searchable_text"]), - ) - for row, entry in zip(rows, entries, strict=True) - ) - projections = await self._index.hydrate(connection, self._scope_id, projections) - active_ids = {item.entry_version_id for item in canonical.content.manifest.entries if item.state == "active"} - if {item.entry_version.entry_version_id for item in projections} != active_ids: - raise InvalidMemoryCitationError("projection-version") - return projections - - async def rebuild_projections(self, embedding_model: EmbeddingModel | None = None, /) -> None: - """Rebuild this scope's active-head and search projections from authoritative rows.""" - - async with self._database.transaction() as connection: - baseline = await self._authoritative_projections(connection) - rebuilt = await self._embed_rebuild(baseline, embedding_model) - async with self._database.transaction() as connection: - if await self._authoritative_projections(connection) != baseline: - raise _MemoryProjectionRebuildError("snapshot") - existing_ids = ( - await connection.execute( - select(MEMORY_ENTRY_HEADS_TABLE.c.memory_artifact_id) - .where(MEMORY_ENTRY_HEADS_TABLE.c.scope_id == self._scope_id) - .distinct() - ) - ).scalars() - by_id = {snapshot.memory_ref.artifact_id: snapshot for snapshot in rebuilt} - for artifact_id in {str(value) for value in existing_ids} | set(by_id): - snapshot = by_id.get(artifact_id) - ref = ( - ArtifactRef(family=Memory.family, artifact_id=artifact_id, revision=1) - if snapshot is None - else snapshot.memory_ref - ) - await self._index.replace(connection, self._scope_id, ref, ()) - await connection.execute( - delete(MEMORY_ENTRY_HEADS_TABLE).where(MEMORY_ENTRY_HEADS_TABLE.c.scope_id == self._scope_id) - ) - await connection.execute( - delete(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE).where( - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.scope_id == self._scope_id - ) - ) - for snapshot in rebuilt: - if snapshot.projections: - await connection.execute( - insert(MEMORY_ENTRY_HEADS_TABLE), - [ - _projection_values(self._scope_id, snapshot.memory_ref, projection) - for projection in snapshot.projections - ], - ) - if snapshot.lifecycle: - await connection.execute( - insert(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE), - [_lifecycle_values(self._scope_id, lifecycle) for lifecycle in snapshot.lifecycle], - ) - await self._index.replace( - connection, - self._scope_id, - snapshot.memory_ref, - snapshot.projections, - ) - - def begin(self) -> AbstractAsyncContextManager[MemoryUnitOfWork]: - return _unit_of_work(self) - - async def changes( - self, - memory: ArtifactRef, - since_revision: int | None, - /, - ) -> tuple[MemoryRevisionChanges, ...]: - target = await self.get(memory) - lower = target.revision - 1 if since_revision is None else since_revision - async with self._database.connection(self._bound_connection) as connection: - revisions = await self._artifacts.revisions( - connection, - self._scope_id, - Memory.family, - memory.artifact_id, - since_revision=lower, - through_revision=target.revision, - ) - selected = (_require_memory(value) for value in revisions) - return tuple( - MemoryRevisionChanges(memory_ref=value.as_ref(), changes=value.content.changes) for value in selected - ) - - async def vector_complete(self, memories: tuple[ArtifactRef, ...], profile: EmbeddingProfile, /) -> bool: - async with self._database.connection(self._bound_connection) as connection: - return await self._index.vector_complete(connection, self._scope_id, memories, profile) - - async def search(self, request: MemorySearchRequest, /) -> MemorySearchChannels: - async with self._database.connection(self._bound_connection) as connection: - await self._validate_search_heads(connection, request.memories) - channels = await self._index.search(connection, self._scope_id, request) - await self._validate_search_heads(connection, request.memories) - return channels - - async def _validate_search_heads( - self, - connection: AsyncConnection, - memories: tuple[ArtifactRef, ...], - ) -> None: - """Reject a projection read when any requested head has advanced.""" - - for memory in memories: - try: - exact = _require_memory(await self._artifacts.get(connection, self._scope_id, memory)) - latest = _require_memory( - await self._artifacts.latest( - connection, - self._scope_id, - Memory.family, - memory.artifact_id, - ) - ) - except RepositoryNotFoundError: - raise ArtifactNotFoundError(memory) from None - if exact.as_ref() != latest.as_ref(): - raise InvalidMemoryCitationError("memory-mismatch") - - async def expand(self, hits: tuple[MemoryHit, ...], /) -> tuple[MemoryEntryVersion, ...]: - expanded: list[MemoryEntryVersion] = [] - async with self._database.connection(self._bound_connection) as connection: - for hit in hits: - row = ( - ( - await connection.execute( - select(MEMORY_ENTRY_VERSIONS_TABLE).where( - MEMORY_ENTRY_VERSIONS_TABLE.c.scope_id == self._scope_id, - MEMORY_ENTRY_VERSIONS_TABLE.c.memory_artifact_id == hit.memory_ref.artifact_id, - MEMORY_ENTRY_VERSIONS_TABLE.c.entry_id == hit.entry_id, - MEMORY_ENTRY_VERSIONS_TABLE.c.entry_version_id == hit.entry_version_id, - ) - ) - ) - .mappings() - .one_or_none() - ) - if row is None: - raise InvalidMemoryCitationError("expand-anchor") - expanded.append(_decode_entry(row)) - return await self._hydrate_source_evidence(connection, tuple(expanded)) - - async def _hydrate_entry_versions( - self, - connection: AsyncConnection, - memory_ref: ArtifactRef, - version_ids: tuple[str, ...], - ) -> tuple[MemoryEntryVersion, ...]: - """Load the named entry versions in request order with their evidence. - - A manifest may name more entries than one statement can bind, so the lookup - chunks at the shared budget. Every caller needs the same completeness - guarantee, so it lives here rather than being restated at each call site. - """ - - if not version_ids: - return () - by_version: dict[str, MemoryEntryVersion] = {} - for batch in _batched(dict.fromkeys(version_ids), _ENTRY_SELECTION_BATCH_SIZE): - rows = ( - await connection.execute( - select(MEMORY_ENTRY_VERSIONS_TABLE).where( - MEMORY_ENTRY_VERSIONS_TABLE.c.scope_id == self._scope_id, - MEMORY_ENTRY_VERSIONS_TABLE.c.memory_artifact_id == memory_ref.artifact_id, - MEMORY_ENTRY_VERSIONS_TABLE.c.entry_version_id.in_(batch), - ) - ) - ).mappings() - hydrated = await self._hydrate_source_evidence(connection, tuple(_decode_entry(row) for row in rows)) - by_version.update({entry.entry_version_id: entry for entry in hydrated}) - entries: list[MemoryEntryVersion] = [] - for version_id in version_ids: - entry = by_version.get(version_id) - if entry is None: - raise InvalidMemoryCitationError("missing-version") - entries.append(entry) - return tuple(entries) - - async def _hydrate_source_evidence( - self, - connection: AsyncConnection, - entries: tuple[MemoryEntryVersion, ...], - ) -> tuple[MemoryEntryVersion, ...]: - if not entries: - return () - identities = dict.fromkeys( - (entry.memory_artifact_id, entry.entry_id, entry.entry_version_id) for entry in entries - ) - snapshots: dict[tuple[str, str, str], list[MemoryEvidenceSnapshot]] = {} - for batch in _batched(identities, _EVIDENCE_SELECTION_BATCH_SIZE): - rows = ( - await connection.execute( - select(MEMORY_ENTRY_EVIDENCE_TABLE) - .where( - MEMORY_ENTRY_EVIDENCE_TABLE.c.scope_id == self._scope_id, - tuple_( - MEMORY_ENTRY_EVIDENCE_TABLE.c.memory_artifact_id, - MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_id, - MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_version_id, - ).in_(batch), - ) - .order_by( - MEMORY_ENTRY_EVIDENCE_TABLE.c.memory_artifact_id, - MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_id, - MEMORY_ENTRY_EVIDENCE_TABLE.c.entry_version_id, - MEMORY_ENTRY_EVIDENCE_TABLE.c.ordinal, - ) - ) - ).mappings() - for row in rows: - identity = (str(row["memory_artifact_id"]), str(row["entry_id"]), str(row["entry_version_id"])) - snapshots.setdefault(identity, []).append( - MemoryEvidenceSnapshot( - source=SourceRef(source_type=str(row["source_type"]), source_id=str(row["source_id"])), - declaration=MemoryEvidenceDeclaration( - authority=MemoryEvidenceAuthority(str(row["authority"])), - verification=MemoryEvidenceVerification(str(row["verification"])), - declaration_version=str(row["declaration_version"]), - ), - ) - ) - hydrated: list[MemoryEntryVersion] = [] - for entry in entries: - identity = (entry.memory_artifact_id, entry.entry_id, entry.entry_version_id) - evidence = tuple(snapshots.get(identity, ())) - if evidence and tuple(snapshot.source for snapshot in evidence) != entry.sources: - raise InvalidMemoryCitationError("evidence-source") - hydrated.append( - entry.model_copy(update={"source_evidence": evidence or _neutral_source_evidence(entry.sources)}) - ) - return tuple(hydrated) - - async def _lifecycle_for_memory( - self, - connection: AsyncConnection, - memory: Memory, - ) -> tuple[MemoryLifecycleProjection, ...]: - return await self._lifecycle_for_entries( - connection, - memory, - tuple(item.entry_id for item in memory.content.manifest.entries), - ) - - async def _lifecycle_for_entries( - self, - connection: AsyncConnection, - memory: Memory, - entry_ids: tuple[str, ...], - ) -> tuple[MemoryLifecycleProjection, ...]: - """Derive lifecycle rows for the named manifest entries of one Memory.""" - - wanted = frozenset(entry_ids) - manifest = tuple(item for item in memory.content.manifest.entries if item.entry_id in wanted) - if not manifest: - return () - version_ids = tuple(item.entry_version_id for item in manifest) - versions = await self._hydrate_entry_versions(connection, memory.as_ref(), version_ids) - by_version = {entry.entry_version_id: entry for entry in versions} - lifecycle: list[MemoryLifecycleProjection] = [] - for item in manifest: - entry = by_version.get(item.entry_version_id) - if entry is None: - raise InvalidMemoryCitationError("missing-version") - _validate_rebuild_entry(memory.as_ref(), item.entry_id, item.entry_content_hash, entry) - lifecycle.append(_lifecycle_projection(memory.as_ref(), item.state, entry)) - return tuple(lifecycle) - - async def _authoritative_projections( - self, - connection: AsyncConnection, - ) -> tuple[_RebuildSnapshot, ...]: - heads = ( - await connection.execute( - select( - ARTIFACT_HEADS_TABLE.c.artifact_id, - ARTIFACT_HEADS_TABLE.c.revision, - ) - .where( - ARTIFACT_HEADS_TABLE.c.scope_id == self._scope_id, - ARTIFACT_HEADS_TABLE.c.family == Memory.family, - ) - .order_by(ARTIFACT_HEADS_TABLE.c.artifact_id) - ) - ).all() - snapshots: list[_RebuildSnapshot] = [] - for artifact_id, revision in heads: - ref = ArtifactRef( - family=Memory.family, - artifact_id=str(artifact_id), - revision=int(revision), - ) - memory = _require_memory(await self._artifacts.get(connection, self._scope_id, ref)) - lifecycle = await self._lifecycle_for_memory(connection, memory) - active = tuple(item for item in memory.content.manifest.entries if item.state == "active") - if not active: - snapshots.append(_RebuildSnapshot(memory_ref=ref, projections=(), lifecycle=lifecycle)) - continue - version_ids = tuple(item.entry_version_id for item in active) - hydrated = await self._hydrate_entry_versions(connection, ref, version_ids) - versions = {entry.entry_version_id: entry for entry in hydrated} - projections: list[MemoryProjection] = [] - for item in active: - version = versions.get(item.entry_version_id) - if version is None: - raise InvalidMemoryCitationError("missing-version") - _validate_rebuild_entry(ref, item.entry_id, item.entry_content_hash, version) - projections.append( - MemoryProjection( - entry_version=version, - searchable_text=analyze_text(version.text), - ) - ) - snapshots.append( - _RebuildSnapshot( - memory_ref=ref, - projections=tuple(projections), - lifecycle=lifecycle, - ) - ) - return tuple(snapshots) - - async def _embed_rebuild( - self, - snapshots: tuple[_RebuildSnapshot, ...], - embedding_model: EmbeddingModel | None, - ) -> tuple[_RebuildSnapshot, ...]: - if embedding_model is None: - return snapshots - profile = self._index.capabilities.embedding_profile - if not self._index.capabilities.vector or profile is None: - raise _MemoryProjectionRebuildError("vector") - if embedding_model.profile != profile: - raise _MemoryProjectionRebuildError("profile") - projections = tuple(projection for snapshot in snapshots for projection in snapshot.projections) - if not projections: - return snapshots - vectors = (await embedding_model.embed(tuple(item.entry_version.text for item in projections))).vectors - if len(vectors) != len(projections): - raise InvalidEmbeddingError("count") - embedded = iter( - projection.model_copy( - update={ - "embedding": canonical_embedding( - vector, - dimension=profile.dimension, - normalization=profile.normalization, - ), - "embedding_content_hash": embedding_content_hash( - profile_id=profile.profile_id, - model=profile.model, - dimension=profile.dimension, - distance=profile.distance, - normalization=profile.normalization, - entry_content_hash=projection.entry_version.entry_content_hash, - ), - } - ) - for projection, vector in zip(projections, vectors, strict=True) - ) - return tuple( - replace(snapshot, projections=tuple(next(embedded) for _ in snapshot.projections)) for snapshot in snapshots - ) - - async def _commit(self, connection: AsyncConnection, value: MemoryCommit) -> Memory: - _validate_commit(value) - draft = _MemoryDraft( - content=value.memory.content, - sources=value.memory.lineage.sources, - artifacts=value.memory.lineage.artifacts, - ) - if value.base is None: - artifact = await self._artifacts.create( - connection, - self._scope_id, - value.memory.artifact_id, - draft, - ) - else: - artifact = await self._artifacts.revise( - connection, - self._scope_id, - value.base, - draft, - ) - committed = _require_memory(artifact) - if committed != value.memory: - raise _InvalidMemoryCommitError("artifact-result") - - compacted = {change.entry_id for change in value.memory.content.changes if change.op == "compact"} - if compacted: - # Artifact revision CAS holds the same head lock as tag replacement. - # Recheck with a current read so a newly tagged entry rolls back the - # entire compaction instead of leaving a dangling tag. - tagged = await self._any_tagged_entry_ids(connection, value.memory.as_ref(), for_update=True) - if compacted & tagged: - raise CapabilityNotSupportedError("compaction-tag-conflict") - - if value.entry_versions: - await connection.execute( - insert(MEMORY_ENTRY_VERSIONS_TABLE), - [_entry_values(self._scope_id, entry) for entry in value.entry_versions], - ) - await _insert_source_evidence(connection, self._scope_id, value.entry_versions) - # Only entries whose pointer or state changed need projection work; the - # rest of the active head stays exactly as the previous revision left it. - previous_active = ( - {} - if value.base is None - else { - item.entry_id: item.entry_version_id - for item in value.base.content.manifest.entries - if item.state == "active" - } - ) - current_active = { - item.entry_id: item.entry_version_id - for item in value.memory.content.manifest.entries - if item.state == "active" - } - removed = tuple(sorted(entry_id for entry_id in previous_active if entry_id not in current_active)) - changed = tuple( - sorted( - entry_id - for entry_id, entry_version_id in current_active.items() - if previous_active.get(entry_id) != entry_version_id - ) - ) - drop = tuple(sorted({*removed, *changed})) - # Clear the index rows before the heads go, as rebuild_projections does: - # index metadata may cascade from the heads, and an index can only find its - # rows through that metadata. - await self._index.delete(connection, self._scope_id, value.memory.as_ref(), drop) - if drop: - await connection.execute( - delete(MEMORY_ENTRY_HEADS_TABLE).where( - MEMORY_ENTRY_HEADS_TABLE.c.scope_id == self._scope_id, - MEMORY_ENTRY_HEADS_TABLE.c.memory_artifact_id == value.memory.artifact_id, - MEMORY_ENTRY_HEADS_TABLE.c.entry_id.in_(drop), - ) - ) - if changed: - by_entry = {projection.entry_version.entry_id: projection for projection in value.projections} - upserts = tuple(by_entry[entry_id] for entry_id in changed) - await connection.execute( - insert(MEMORY_ENTRY_HEADS_TABLE), - [_projection_values(self._scope_id, value.memory.as_ref(), projection) for projection in upserts], - ) - await self._index.upsert( - connection, - self._scope_id, - value.memory.as_ref(), - upserts, - ) - await _refresh_lifecycle_projections(connection, self, value, committed) - return committed - - -class _RelationalMemoryUnitOfWork: - def __init__(self, backend: RelationalMemoryBackend, connection: AsyncConnection) -> None: - self._backend = backend - self._connection = connection - self._complete = False - - async def commit(self, value: MemoryCommit, /) -> Memory: - if self._complete: - raise _InvalidMemoryCommitError("complete") - result = await self._backend._commit(self._connection, value) - self._complete = True - return result - - -@asynccontextmanager -async def _unit_of_work( - backend: RelationalMemoryBackend, -) -> AsyncIterator[_RelationalMemoryUnitOfWork]: - async with backend._database.connection(backend._bound_connection) as connection: - yield _RelationalMemoryUnitOfWork(backend, connection) - - -def _validate_commit(value: MemoryCommit) -> None: - if value.memory.family != Memory.family: - raise _InvalidMemoryCommitError("family") - if value.content_hash != memory_content_hash(value.memory.content): - raise _InvalidMemoryCommitError("content-hash") - expected_revision = 1 if value.base is None else value.base.revision + 1 - if value.memory.revision != expected_revision: - raise _InvalidMemoryCommitError("revision") - if value.base is not None and value.base.artifact_id != value.memory.artifact_id: - raise _InvalidMemoryCommitError("base-identity") - active = {item.entry_version_id for item in value.memory.content.manifest.entries if item.state == "active"} - projected = {item.entry_version.entry_version_id for item in value.projections} - if active != projected: - raise _InvalidMemoryCommitError("projection") - - -def _entry_values(scope_id: str, value: MemoryEntryVersion) -> dict[str, object]: - return { - "scope_id": scope_id, - "family": Memory.family, - "memory_artifact_id": value.memory_artifact_id, - "entry_id": value.entry_id, - "entry_version_id": value.entry_version_id, - "version": value.version, - "previous_version_id": value.previous_version_id, - "kind": value.kind, - "text": value.text, - "source_refs": dump_model( - _SourceRefs(value.sources), - kind="memory-entry", - name=value.entry_version_id, - ), - "artifact_refs": dump_model( - _ArtifactRefs(value.artifacts), - kind="memory-entry", - name=value.entry_version_id, - ), - "entry_content_hash": value.entry_content_hash, - "created_in_revision": value.created_in_revision, - } - - -async def _refresh_lifecycle_projections( - connection: AsyncConnection, - backend: RelationalMemoryBackend, - value: MemoryCommit, - committed: Memory, -) -> None: - """Rewrite the lifecycle rows whose entry identity or state changed. - - Lifecycle rows cover every manifest entry, including inactive ones, and follow - the same per-entry rule as the active head: only changed entries are rewritten, - so repeated appends stay bounded by the appended entries rather than the - manifest size. - """ - - previous_entries = ( - {} - if value.base is None - else {item.entry_id: (item.entry_version_id, item.state) for item in value.base.content.manifest.entries} - ) - current_entries = { - item.entry_id: (item.entry_version_id, item.state) for item in value.memory.content.manifest.entries - } - removed = tuple(sorted(entry_id for entry_id in previous_entries if entry_id not in current_entries)) - changed = tuple( - sorted(entry_id for entry_id, identity in current_entries.items() if previous_entries.get(entry_id) != identity) - ) - drop = tuple(sorted({*removed, *changed})) - if drop: - await connection.execute( - delete(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE).where( - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.scope_id == backend._scope_id, - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.memory_artifact_id == value.memory.artifact_id, - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.entry_id.in_(drop), - ) - ) - if changed: - lifecycle = await backend._lifecycle_for_entries(connection, committed, changed) - if len(lifecycle) != len(changed): - raise _InvalidMemoryCommitError("lifecycle-projection") - await connection.execute( - insert(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE), - [_lifecycle_values(backend._scope_id, projection) for projection in lifecycle], - ) - - -async def _insert_source_evidence( - connection: AsyncConnection, - scope_id: str, - entry_versions: tuple[MemoryEntryVersion, ...], -) -> None: - """Write the evidence rows for one batch of entry versions, if any carry sources.""" - - rows: list[dict[str, object]] = [] - for entry in entry_versions: - rows.extend(_evidence_values(scope_id, entry)) - if rows: - await connection.execute(insert(MEMORY_ENTRY_EVIDENCE_TABLE), rows) - - -def _evidence_values(scope_id: str, value: MemoryEntryVersion) -> tuple[dict[str, object], ...]: - return tuple( - { - "scope_id": scope_id, - "memory_artifact_id": value.memory_artifact_id, - "entry_id": value.entry_id, - "entry_version_id": value.entry_version_id, - "ordinal": ordinal, - "source_type": snapshot.source.source_type, - "source_id": snapshot.source.source_id, - "authority": snapshot.declaration.authority.value, - "verification": snapshot.declaration.verification.value, - "declaration_version": snapshot.declaration.declaration_version, - } - for ordinal, snapshot in enumerate(value.source_evidence) - ) - - -def _projection_values( - scope_id: str, - memory_ref: ArtifactRef, - value: MemoryProjection, -) -> dict[str, object]: - entry = value.entry_version - return { - "scope_id": scope_id, - "family": Memory.family, - "memory_artifact_id": memory_ref.artifact_id, - "head_revision": memory_ref.revision, - "entry_id": entry.entry_id, - "entry_version_id": entry.entry_version_id, - "entry_content_hash": entry.entry_content_hash, - "searchable_text": value.searchable_text, - } - - -def _lifecycle_values(scope_id: str, value: MemoryLifecycleProjection) -> dict[str, object]: - return { - "scope_id": scope_id, - "family": Memory.family, - "memory_artifact_id": value.memory_ref.artifact_id, - "head_revision": value.memory_ref.revision, - "entry_id": value.entry_id, - "entry_version_id": value.entry_version_id, - "validity": value.validity, - "validity_reason": value.validity_reason, - "successor_entry_id": value.successor_entry_id, - "source_count": len(value.source_evidence), - "rule_version": value.rule_version, - "quality_policy": value.quality_policy, - } - - -def _decode_entry(row: Mapping[Any, Any]) -> MemoryEntryVersion: - entry_version_id = str(row["entry_version_id"]) - return MemoryEntryVersion( - memory_artifact_id=str(row["memory_artifact_id"]), - entry_id=str(row["entry_id"]), - entry_version_id=entry_version_id, - version=int(row["version"]), - previous_version_id=(None if row["previous_version_id"] is None else str(row["previous_version_id"])), - kind=str(row["kind"]), - text=str(row["text"]), - sources=load_model( - _SourceRefs, - stored_bytes(row["source_refs"], column="memory payload"), - kind="memory-entry", - name=entry_version_id, - ).root, - artifacts=load_model( - _ArtifactRefs, - stored_bytes(row["artifact_refs"], column="memory payload"), - kind="memory-entry", - name=entry_version_id, - ).root, - entry_content_hash=str(row["entry_content_hash"]), - created_in_revision=int(row["created_in_revision"]), - ) - - -def _neutral_source_evidence(sources: tuple[SourceRef, ...]) -> tuple[MemoryEvidenceSnapshot, ...]: - return tuple(MemoryEvidenceSnapshot(source=source, declaration=MemoryEvidenceDeclaration()) for source in sources) - - -def _lifecycle_projection( - memory_ref: ArtifactRef, - state: str, - entry: MemoryEntryVersion, - *, - row: Mapping[Any, Any] | None = None, -) -> MemoryLifecycleProjection: - """Derive one lifecycle projection for a requested revision's entry. - - Every field comes from the entry the requested manifest names. A persisted row - is only a cross-check that the entry has one, because its columns describe the - revision that last wrote it: reusing them would report a later revision's state - under a historical reference, which the entry identity alone does not prevent. - """ - - if row is not None and str(row["entry_id"]) != entry.entry_id: - raise InvalidMemoryCitationError("lifecycle-version") - return MemoryLifecycleProjection( - memory_ref=memory_ref, - entry_id=entry.entry_id, - entry_version_id=entry.entry_version_id, - validity="current" if state == "active" else "inactive", - source_evidence=entry.source_evidence, - ) - - -def _validate_rebuild_entry( - memory_ref: ArtifactRef, - entry_id: str, - content_hash: str, - version: MemoryEntryVersion, -) -> None: - if ( - version.memory_artifact_id != memory_ref.artifact_id - or version.entry_id != entry_id - or version.entry_content_hash != content_hash - ): - raise InvalidMemoryCitationError("cross-identity") - actual_hash = entry_content_hash( - kind=version.kind, - text=version.text, - source_refs=tuple({"source_type": ref.source_type, "source_id": ref.source_id} for ref in version.sources), - artifact_refs=tuple( - { - "family": ref.family, - "artifact_id": ref.artifact_id, - "revision": ref.revision, - } - for ref in version.artifacts - ), - ) - if actual_hash != content_hash: - raise InvalidMemoryCitationError("hash-mismatch") - - -def _require_memory(value: object) -> Memory: - if type(value) is not Memory: - raise _InvalidMemoryCommitError("memory-type", type(value).__name__) - return value diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index db246d0b7f..c20b6a4cda 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -218,12 +218,6 @@ def _input_contract_schema(source_class: type[Source]) -> dict[str, JsonValue]: return schema -def _declaration_signature() -> JsonValue: - """Return the schema Pydantic emits for the declaration a Source inherits.""" - - return _json_object(Source.model_json_schema()["properties"])["memory_evidence"] - - def _remove_inherited_declaration(node: JsonValue, source_class: type[Source]) -> None: """Strip the inherited declaration from every object schema that carries it. @@ -267,7 +261,7 @@ def _source_schema_titles(source_class: type[Source]) -> set[str]: titles.add(str(model.model_config.get("title", model.__name__))) for field in model.model_fields.values(): annotation = field.annotation - if isinstance(annotation, type) and issubclass(annotation, BaseModel): + if isinstance(annotation, type) and issubclass(annotation, Source): pending.append(annotation) return titles diff --git a/tests/builtin/persistence/test_memory.py b/tests/builtin/persistence/test_memory.py deleted file mode 100644 index c74c280ad4..0000000000 --- a/tests/builtin/persistence/test_memory.py +++ /dev/null @@ -1,571 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import asyncio - -import pytest -from pydantic import ValidationError -from sqlalchemy import BigInteger, DateTime, Integer, String, delete, event, select -from sqlalchemy.dialects import mysql -from sqlalchemy.exc import IntegrityError -from sqlalchemy.schema import CreateTable, ForeignKeyConstraint, PrimaryKeyConstraint, UniqueConstraint - -from powercontext.artifacts import ArtifactLineage -from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent -from powercontext.builtin.persistence.sqlite import SQLiteConfig -from powercontext.builtin.persistence.tables import ( - MEMORY_ENTRY_EVIDENCE_TABLE, - MEMORY_ENTRY_HEADS_TABLE, - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE, - MEMORY_ENTRY_VERSIONS_TABLE, - SOURCES_TABLE, -) -from powercontext.builtin.records import ArtifactWrite -from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts -from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection -from powercontext.builtin.sources import ContentCapture, ContentSource -from powercontext.sources import SourceRef - -_INNODB_MAX_INDEX_BYTES = 3072 -_EVIDENCE_BUDGET_ENTRIES = SELECTION_BATCH_SIZE -_MAXIMUM_TAG_LABELS = 16 - - -class _UnbudgetedColumnTypeError(TypeError): - def __init__(self, column_type: object) -> None: - super().__init__(f"unbudgeted indexed column type: {column_type!r}") - - -def _key_budget(constraint: PrimaryKeyConstraint | UniqueConstraint | ForeignKeyConstraint) -> int: - total = 0 - for column in constraint.columns: - if isinstance(column.type, String): - assert column.type.length is not None - total += column.type.length * 4 - elif isinstance(column.type, BigInteger | Integer | DateTime): - total += 8 - else: - raise _UnbudgetedColumnTypeError(column.type) - return total - - -def test_memory_schema_is_mysql_compilable_and_respects_key_and_payload_limits() -> None: - dialect = mysql.dialect() - versions = str(CreateTable(MEMORY_ENTRY_VERSIONS_TABLE).compile(dialect=dialect)) - evidence = str(CreateTable(MEMORY_ENTRY_EVIDENCE_TABLE).compile(dialect=dialect)) - heads = str(CreateTable(MEMORY_ENTRY_HEADS_TABLE).compile(dialect=dialect)) - lifecycle = str(CreateTable(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE).compile(dialect=dialect)) - - assert "scope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL" in versions - assert "text MEDIUMTEXT NOT NULL" in versions - assert "source_refs MEDIUMBLOB NOT NULL" in versions - assert "artifact_refs MEDIUMBLOB NOT NULL" in versions - assert "declaration_version VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL" in evidence - assert "searchable_text MEDIUMTEXT NOT NULL" in heads - assert "quality_policy VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL" in lifecycle - - budgets = [ - _key_budget(constraint) - for table in ( - MEMORY_ENTRY_VERSIONS_TABLE, - MEMORY_ENTRY_EVIDENCE_TABLE, - MEMORY_ENTRY_HEADS_TABLE, - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE, - ) - for constraint in table.constraints - if isinstance(constraint, PrimaryKeyConstraint | UniqueConstraint | ForeignKeyConstraint) - ] - assert budgets - assert max(budgets) <= 2568 - assert all(budget < _INNODB_MAX_INDEX_BYTES for budget in budgets) - - -def test_sqlite_memory_backend_commits_authoritative_history_and_fts() -> None: - async def scenario() -> None: - async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: - context = await contexts.get("project") - source, _ = await context.sources.capture( - ContentCapture( - source_id="turn-1", - content="PowerContext owns the atomic composition boundary.", - ) - ) - application = contexts.atomic_memory - memory = application.for_scope("project") - async with contexts.database.transaction() as connection: - plan = await application.service.inspect_change( - connection, - "project", - "decision", - AtomicMemoryContent(kind="decision", text="Use one atomic composition boundary."), - None, - lineage=ArtifactLineage(sources=(SourceRef(source_type="content", source_id=source.name),)), - ) - prepared = await application.service.prepare_change(plan) - async with contexts.database.transaction() as connection: - first = (await application.service.commit(connection, prepared, None)).primary - second = await contexts.records.replace_artifact( - "project", - "atomic-memory", - first.ref.artifact_id, - '"revision:1"', - ArtifactWrite( - content={"kind": "decision", "text": "Use one atomic composition boundary for providers."} - ), - ) - constraint = await contexts.records.create_artifact( - "project", - "atomic-memory", - ArtifactWrite(content={"kind": "constraint", "text": "Do not split the provider transaction."}), - ) - assert second.revision == 2 - history = await contexts.records.list_artifact_revisions( - "project", - "atomic-memory", - first.ref.artifact_id, - limit=10, - cursor=None, - ) - assert tuple(item.revision for item in history.items) == (2, 1) - assert (await memory.get(first.ref.artifact_id, revision=1)).artifact == first.artifact - assert first.artifact.lineage.sources == (SourceRef(source_type="content", source_id=source.name),) - assert {item.artifact.content.text for item in (await memory.list()).items} == { - second.content["text"], - "Do not split the provider transaction.", - } - assert constraint.artifact_id != first.ref.artifact_id - result = await memory.search("atomic composition", mode="text") - assert result.mode == "text" - assert tuple(hit.text for hit in result.hits) == (second.content["text"],) - assert result.hits[0].hit.artifact_ref == (await memory.get(first.ref.artifact_id)).ref - unrelated = await memory.search("Should we use blue icons in the mobile navigation bar?", mode="text") - assert unrelated.hits == () - - asyncio.run(scenario()) - - -def test_sqlite_memory_backend_rebuilds_head_and_fts_projections() -> None: - async def scenario() -> None: - async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: - await contexts.get("project") - created = await contexts.records.create_artifact( - "project", - "atomic-memory", - ArtifactWrite( - content={"kind": "decision", "text": "Rebuild search projections from authoritative revisions."} - ), - ) - memory = contexts.atomic_memory.for_scope("project") - authoritative = await memory.get(created.artifact_id) - async with contexts.database.transaction() as connection: - await connection.execute(contexts.atomic_memory.index.table.delete()) - assert (await memory.search("authoritative revisions", mode="text")).hits == () - report = await rebuild_atomic_memory_projection( - contexts.database, - contexts.atomic_memory.index, - maintenance_confirmed=True, - ) - assert report.ready - assert await memory.get(created.artifact_id) == authoritative - rebuilt = await memory.search("authoritative revisions", mode="text") - assert tuple(hit.text for hit in rebuilt.hits) == ( - "Rebuild search projections from authoritative revisions.", - ) - - asyncio.run(scenario()) - - -def test_memory_evidence_snapshots_and_neutral_lifecycle_projection_are_rebuildable() -> None: - async def scenario() -> None: - async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: - context = await contexts.get("project") - source, _ = await context.sources.capture( - ContentCapture(source_id="turn-1", content="A durable source declaration must be snapshotted.") - ) - memory = await context.artifacts.memory.remember( - memory=None, - sources=(source,), - entries=( - MemoryEntryInput( - kind="decision", - text="Keep provenance metadata outside the Memory body.", - sources=(source,), - ), - ), - mode="append", - ) - assert memory is not None - (entry,) = await context.artifacts.memory.entries(memory) - assert entry.source_evidence[0].source.source_id == "turn-1" - assert entry.source_evidence[0].declaration.authority == "untrusted" - assert entry.source_evidence[0].declaration.verification == "unknown" - - backend = RelationalMemoryBackend( - database=contexts.database, - scope_id="project", - artifacts=contexts.repositories.artifacts, - index=contexts.index, - ) - current_lifecycle = (await backend.lifecycle_projections(memory.as_ref()))[0] - assert current_lifecycle.validity == "current" - assert current_lifecycle.context_annotation.time_state == "unknown" - assert current_lifecycle.context_annotation.quality_policy == "neutral" - - inactive = await context.artifacts.memory.forget(memory, entries=(entry,), reason="superseded elsewhere") - assert inactive is not None - lifecycle = await backend.lifecycle_projections(inactive.as_ref()) - assert lifecycle[0].validity == "inactive" - assert lifecycle[0].source_evidence == entry.source_evidence - - async with contexts.database.transaction() as connection: - await connection.execute(delete(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE)) - - await context.artifacts.memory.rebuild_projections() - - rebuilt = await backend.lifecycle_projections(inactive.as_ref()) - assert rebuilt == lifecycle - - asyncio.run(scenario()) - - -async def _lifecycle_rows(contexts: RelationalContexts) -> dict[str, tuple[int, str]]: - async with contexts.database.transaction() as connection: - rows = ( - await connection.execute( - select( - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.entry_id, - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.head_revision, - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE.c.validity, - ) - ) - ).all() - return {str(entry_id): (int(revision), str(validity)) for entry_id, revision, validity in rows} - - -def test_lifecycle_projection_rows_are_rewritten_only_for_changed_entries() -> None: - async def scenario() -> None: - async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: - context = await contexts.get("project") - source, _ = await context.sources.capture( - ContentCapture(source_id="turn-1", content="Lifecycle rows must follow their own entry.") - ) - opened = await context.artifacts.memory.remember( - memory=None, - sources=(source,), - entries=( - MemoryEntryInput(kind="decision", text="Keep the first entry.", sources=(source,)), - MemoryEntryInput(kind="constraint", text="Keep the second entry.", sources=(source,)), - ), - mode="append", - ) - assert opened is not None - opened_rows = await _lifecycle_rows(contexts) - assert len(opened_rows) == 2 - assert {revision for revision, _ in opened_rows.values()} == {opened.revision} - - appended = await context.artifacts.memory.remember( - memory=opened, - sources=(source,), - entries=(MemoryEntryInput(kind="agent-note", text="Append exactly one entry.", sources=(source,)),), - mode="append", - ) - assert appended is not None - assert appended.revision == opened.revision + 1 - - backend = RelationalMemoryBackend( - database=contexts.database, - scope_id="project", - artifacts=contexts.repositories.artifacts, - index=contexts.index, - ) - appended_rows = await _lifecycle_rows(contexts) - - # Appending one entry rewrites that entry alone: every row that was - # already current keeps the revision that last wrote it. - assert len(appended_rows) == 3 - assert { - entry_id: value for entry_id, value in appended_rows.items() if entry_id in opened_rows - } == opened_rows - assert {validity for _, validity in appended_rows.values()} == {"current"} - - forgotten = (await context.artifacts.memory.entries(opened))[0] - inactive = await context.artifacts.memory.forget( - appended, - entries=(forgotten,), - reason="superseded elsewhere", - ) - assert inactive is not None - inactive_rows = await _lifecycle_rows(contexts) - - assert inactive_rows[forgotten.entry_id] == (inactive.revision, "inactive") - assert {entry_id: value for entry_id, value in inactive_rows.items() if entry_id != forgotten.entry_id} == { - entry_id: value for entry_id, value in appended_rows.items() if entry_id != forgotten.entry_id - } - - incremental = await backend.lifecycle_projections(inactive.as_ref()) - assert {item.validity for item in incremental if item.entry_id == forgotten.entry_id} == {"inactive"} - assert {item.validity for item in incremental if item.entry_id != forgotten.entry_id} == {"current"} - - async with contexts.database.transaction() as connection: - await connection.execute(delete(MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE)) - await context.artifacts.memory.rebuild_projections() - - assert await backend.lifecycle_projections(inactive.as_ref()) == incremental - - asyncio.run(scenario()) - - -def test_manifest_lookups_stay_within_the_shared_bind_budget() -> None: - """Every manifest-sized lookup must chunk instead of binding every entry. - - The ceiling on bound values depends on the SQLite build, so this pins the bind - budget the backend controls rather than one build's variable limit. The budget - is asserted as the invariant each statement has to satisfy, not as the value - the current constants happen to produce, so tuning a batch size cannot make - the check pass or fail by itself. - """ - - async def scenario() -> None: - async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: - context = await contexts.get("project") - bound: list[int] = [] - - def record_statement( - _connection: object, - _cursor: object, - statement: str, - parameters: object, - *_rest: object, - ) -> None: - if not statement.lstrip().upper().startswith("SELECT"): - return - assert isinstance(parameters, tuple | list) - bound.append(len(parameters)) - - engine = contexts.database.engine.sync_engine - event.listen(engine, "before_cursor_execute", record_statement) - try: - opened = await context.artifacts.memory.remember( - memory=None, - entries=tuple( - MemoryEntryInput(kind="agent-note", text=f"Bounded entry {index}.", sources=()) - for index in range(_EVIDENCE_BUDGET_ENTRIES) - ), - mode="append", - ) - assert opened is not None - appended = await context.artifacts.memory.remember( - memory=opened, - entries=(MemoryEntryInput(kind="agent-note", text="Appended within budget.", sources=()),), - mode="append", - ) - assert appended is not None - - expected = _EVIDENCE_BUDGET_ENTRIES + 1 - # Every read path that resolves a manifest must stay within budget. - # The lifecycle projection is internal and has no service caller yet, - # so it is exercised on the backend that owns the lookup. - backend = RelationalMemoryBackend( - database=contexts.database, - scope_id="project", - artifacts=contexts.repositories.artifacts, - index=contexts.index, - ) - assert len(await context.artifacts.memory.entries(appended)) == expected - assert len(await backend.lifecycle_projections(appended.as_ref())) == expected - # A tag-filtered read also binds its correlated predicate, so - # exercise the widest filter the contract admits. - widest = TagFilter(tags=tuple(f"label-{index}" for index in range(_MAXIMUM_TAG_LABELS))) - assert await backend.tagged_entry_ids(appended.as_ref(), widest) == frozenset() - assert await backend.tagged_entry_ids(appended.as_ref(), TagFilter(tags=("absent",))) == frozenset() - await context.artifacts.memory.rebuild_projections() - finally: - event.remove(engine, "before_cursor_execute", record_statement) - - # The evidence lookup binds three identity columns per entry; every - # other lookup binds one value per entry. Each chunk plus the statement's - # fixed parameters has to fit the shared budget, so assert that invariant - # rather than the value the current constants happen to produce. - assert bound - limit = max( - 3 * _EVIDENCE_SELECTION_BATCH_SIZE + _EVIDENCE_BIND_RESERVE, - _ENTRY_SELECTION_BATCH_SIZE + _ENTRY_BIND_RESERVE, - ) - assert limit <= SELECTION_BATCH_SIZE - assert max(bound) <= limit - - asyncio.run(scenario()) - - -def test_registered_source_cannot_be_deleted_while_memory_evidence_cites_it() -> None: - """The evidence foreign key protects a cited Source from removal. - - Any future retention or erasure implementation has to remove the citing - evidence rows first, so the constraint is pinned here rather than left implicit. - """ - - async def scenario() -> None: - async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: - context = await contexts.get("project") - source, _ = await context.sources.capture( - ContentCapture(source_id="turn-1", content="A cited Source cannot simply vanish.") - ) - opened = await context.artifacts.memory.remember( - memory=None, - sources=(source,), - entries=(MemoryEntryInput(kind="fact", text="Cite the Source.", sources=(source,)),), - mode="append", - ) - assert opened is not None - - async with contexts.database.transaction() as connection: - with pytest.raises(IntegrityError): - await connection.execute( - delete(SOURCES_TABLE).where( - SOURCES_TABLE.c.scope_id == "project", - SOURCES_TABLE.c.source_id == "turn-1", - ) - ) - - asyncio.run(scenario()) - - -def test_tag_filter_label_ceiling_matches_the_entry_bind_reserve() -> None: - """The entry bind reserve is sized for the widest tag filter the contract admits. - - A tag-filtered entry lookup binds its correlated predicate on top of the batch, - so the reserve has to cover the maximum label count. If that ceiling ever rises, - the reserve has to rise with it, and this pins the pair together. - """ - - accepted = TagFilter(tags=tuple(f"label-{index}" for index in range(_MAXIMUM_TAG_LABELS))) - assert len(accepted.keys) == _MAXIMUM_TAG_LABELS - with pytest.raises(ValidationError, match="invalid number of labels"): - TagFilter(tags=tuple(f"label-{index}" for index in range(_MAXIMUM_TAG_LABELS + 1))) - assert _ENTRY_BIND_RESERVE >= 2 * _MAXIMUM_TAG_LABELS + 2 - - -def test_lifecycle_reads_report_the_state_of_the_requested_revision() -> None: - """A lifecycle read must not answer a historical reference with a later state. - - ``forget`` and ``reactivate`` keep the same entry version, so an identity-only - check accepts a row the requested revision never had. Deriving the state from - the requested manifest keeps a historical reference from reporting current - state. - """ - - async def scenario() -> None: - async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: - context = await contexts.get("project") - opened = await context.artifacts.memory.remember( - memory=None, - entries=(MemoryEntryInput(kind="fact", text="Lifecycle follows its revision."),), - mode="append", - ) - assert opened is not None - entry = (await context.artifacts.memory.entries(opened))[0] - forgotten = await context.artifacts.memory.forget(opened, entries=(entry,), reason="forgotten") - assert forgotten is not None - restored = await context.artifacts.memory.reactivate(forgotten, entries=(entry,), reason="restored") - assert restored is not None - - backend = RelationalMemoryBackend( - database=contexts.database, - scope_id="project", - artifacts=contexts.repositories.artifacts, - index=contexts.index, - ) - for revision, expected in ( - (opened.revision, "current"), - (forgotten.revision, "inactive"), - (restored.revision, "current"), - ): - reference = restored.as_ref().model_copy(update={"revision": revision}) - (projection,) = await backend.lifecycle_projections(reference) - assert projection.validity == expected, f"revision {revision} reported {projection.validity}" - - asyncio.run(scenario()) - - -def test_lifecycle_reads_follow_the_evidence_of_the_requested_revision() -> None: - """A lifecycle read must not borrow a persisted row's evidence count. - - Revising an entry to cite a Source keeps its row but changes what the row - describes. A historical reference has to report the evidence that revision - actually carried rather than the latest row's count. - """ - - async def scenario() -> None: - async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: - context = await contexts.get("project") - source, _ = await context.sources.capture( - ContentCapture(source_id="turn-1", content="Evidence added by a later revision.") - ) - opened = await context.artifacts.memory.remember( - memory=None, - entries=(MemoryEntryInput(kind="fact", text="Evidence arrives later."),), - mode="append", - ) - assert opened is not None - entry = (await context.artifacts.memory.entries(opened))[0] - assert entry.sources == () - revised = await context.artifacts.memory.remember( - memory=opened, - sources=(source,), - entries=( - MemoryEntryInput( - kind="fact", - text="Evidence arrives later.", - entry=entry, - sources=(source,), - ), - ), - mode="append", - ) - assert revised is not None - - backend = RelationalMemoryBackend( - database=contexts.database, - scope_id="project", - artifacts=contexts.repositories.artifacts, - index=contexts.index, - ) - for revision, expected in ((opened.revision, 0), (revised.revision, 1)): - reference = revised.as_ref().model_copy(update={"revision": revision}) - (projection,) = await backend.lifecycle_projections(reference) - assert len(projection.source_evidence) == expected, ( - f"revision {revision} reported {len(projection.source_evidence)} evidence entries" - ) - - asyncio.run(scenario()) - - -def test_scope_bound_contexts_do_not_share_rows() -> None: - async def scenario() -> None: - async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: - left = await contexts.get("left") - right = await contexts.get("right") - - await left.sources.capture(ContentCapture(source_id="same", content="left")) - await right.sources.capture(ContentCapture(source_id="same", content="right")) - - left_sources = await left.sources.list() - right_sources = await right.sources.list() - assert all(isinstance(source, ContentSource) for source in (*left_sources, *right_sources)) - assert tuple(source.content for source in left_sources if isinstance(source, ContentSource)) == ("left",) - assert tuple(source.content for source in right_sources if isinstance(source, ContentSource)) == ("right",) - - asyncio.run(scenario()) diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 4f18fa8430..6def3e787a 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -93,7 +93,10 @@ def test_neutral_declaration_keeps_the_pre_change_definition_identity() -> None: declared_properties = cast("dict[str, object]", manifest.source_schema["properties"]) assert "memory_evidence" not in declared_properties - assert manifest.fingerprint == ("sha256:0b58627bec9b987d1c3613d98c72f3131ec2a81f95cbc620ef05d923585e0935") + # The upstream atomic-memory merge expanded ContentSource's captured schema; + # the neutral declaration is still excluded, so this is the new stable + # fingerprint for the merged contract. + assert manifest.fingerprint == ("sha256:02539903e42db50f9626df111df186bcafc195c9a33204c9df1d7ebc0ebebd12") def test_version_parts_are_rejected_before_they_cannot_be_transported() -> None: From 812e6e6207a454e488ca96c34389f59a62e5d500 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Mon, 12 Oct 2026 21:45:38 +0800 Subject: [PATCH 15/30] fix: align atomic memory migration and generated desktop contract --- .../src-tauri/src/transport/operations.json | 2 +- desktop/ui/src/generated/api.d.ts | 11 ++++ desktop/ui/src/generated/operations.ts | 2 +- docs/en/docs/operate/troubleshoot.md | 2 +- docs/zh/docs/operate/troubleshoot.md | 2 +- .../builtin/artifacts/memory/models.py | 51 ----------------- .../builtin/persistence/tables.py | 56 ------------------- .../builtin/runtime/relational.py | 1 - tests/e2e/real_experience_skill/harness.py | 1 - 9 files changed, 15 insertions(+), 113 deletions(-) diff --git a/desktop/src-tauri/src/transport/operations.json b/desktop/src-tauri/src/transport/operations.json index 82cee3a518..170cd1861a 100644 --- a/desktop/src-tauri/src/transport/operations.json +++ b/desktop/src-tauri/src/transport/operations.json @@ -1,5 +1,5 @@ { - "contractSha256": "7aa91ae9af78300e7bdbcc2e8b2135da5749469290a1aee67bc255c3e9a07b0c", + "contractSha256": "ae214d53039109c127449de9d17578459e0cc07e3ba5f4cacef4bc5c31bbcca2", "operations": { "get_liveness": { "method": "GET", diff --git a/desktop/ui/src/generated/api.d.ts b/desktop/ui/src/generated/api.d.ts index 1ed9b6b95a..0f444959b1 100644 --- a/desktop/ui/src/generated/api.d.ts +++ b/desktop/ui/src/generated/api.d.ts @@ -3083,6 +3083,15 @@ export interface components { [key: string]: unknown; }; }; + /** @enum {string} */ + MemoryEvidenceAuthority: "untrusted" | "user_asserted" | "repository_attested" | "system_attested"; + /** @enum {string} */ + MemoryEvidenceVerification: "unknown" | "verified" | "not_verified"; + MemoryEvidenceDeclaration: { + authority: components["schemas"]["MemoryEvidenceAuthority"]; + verification: components["schemas"]["MemoryEvidenceVerification"]; + declaration_version: string; + }; SourceDefinitionManifest: { name: string; version: string; @@ -3091,6 +3100,7 @@ export interface components { [key: string]: unknown; }; projections: components["schemas"]["SourceProjectionManifest"][]; + memory_evidence?: components["schemas"]["MemoryEvidenceDeclaration"]; }; RegisterSourceDefinitionRequest: { manifest: components["schemas"]["SourceDefinitionManifest"]; @@ -3120,6 +3130,7 @@ export interface components { description?: string | null; source_type: string; definition_fingerprint: string; + memory_evidence?: components["schemas"]["MemoryEvidenceDeclaration"]; payload: { [key: string]: unknown; }; diff --git a/desktop/ui/src/generated/operations.ts b/desktop/ui/src/generated/operations.ts index ea299aa267..14290c5760 100644 --- a/desktop/ui/src/generated/operations.ts +++ b/desktop/ui/src/generated/operations.ts @@ -15,7 +15,7 @@ */ // Generated from openapi/powercontext.yaml. Do not edit. -export const contractSha256 = "7aa91ae9af78300e7bdbcc2e8b2135da5749469290a1aee67bc255c3e9a07b0c"; +export const contractSha256 = "ae214d53039109c127449de9d17578459e0cc07e3ba5f4cacef4bc5c31bbcca2"; export const operations = { "get_liveness": { "method": "GET", diff --git a/docs/en/docs/operate/troubleshoot.md b/docs/en/docs/operate/troubleshoot.md index 74a26b86a5..61c03623b3 100644 --- a/docs/en/docs/operate/troubleshoot.md +++ b/docs/en/docs/operate/troubleshoot.md @@ -283,7 +283,7 @@ so the previous database remains available for recovery: ```bash obloader -D --csv \ - --table 'pc_artifact_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_evidence,pc_memory_entry_heads,pc_memory_entry_lifecycle_projections,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ + --table 'pc_artifact_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_evidence,pc_memory_entry_heads,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ -f ``` diff --git a/docs/zh/docs/operate/troubleshoot.md b/docs/zh/docs/operate/troubleshoot.md index 683531619d..4a70480688 100644 --- a/docs/zh/docs/operate/troubleshoot.md +++ b/docs/zh/docs/operate/troubleshoot.md @@ -274,7 +274,7 @@ collation，但不会包含数据库 URL 或凭据。 ```bash obloader -D --csv \ - --table 'pc_artifact_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_evidence,pc_memory_entry_heads,pc_memory_entry_lifecycle_projections,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ + --table 'pc_artifact_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_evidence,pc_memory_entry_heads,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ -f ``` diff --git a/src/powercontext/builtin/artifacts/memory/models.py b/src/powercontext/builtin/artifacts/memory/models.py index 916e698d2f..ad0d166b7e 100644 --- a/src/powercontext/builtin/artifacts/memory/models.py +++ b/src/powercontext/builtin/artifacts/memory/models.py @@ -156,57 +156,6 @@ class MemoryEntryVersion(BaseModel): artifacts: tuple[ArtifactRef, ...] = () -class MemoryLifecycleProjection(BaseModel): - """Rebuildable lifecycle state for one Memory manifest entry. - - Reserved infrastructure for the stage-2 quality policy. The rules are - deterministic so the rows can be rebuilt from the authoritative manifest, but - no production path consumes them yet: the materialized table has no reader - outside the rebuild, and the annotation below is not wired into Context Pack - assembly. - """ - - memory_ref: ArtifactRef - entry_id: str - entry_version_id: str - validity: MemoryLifecycleValidity - validity_reason: str | None = None - successor_entry_id: str | None = None - source_evidence: tuple[MemoryEvidenceSnapshot, ...] = () - rule_version: str = "memory-lifecycle-v1" - quality_policy: Literal["neutral"] = "neutral" - - @property - def context_annotation(self) -> MemoryContextAnnotation: - """Return the internal annotation reserved for Context Pack assembly.""" - - return MemoryContextAnnotation( - validity=self.validity, - validity_reason=self.validity_reason, - successor_entry_id=self.successor_entry_id, - source_evidence=self.source_evidence, - source_count=len(self.source_evidence), - time_state="unknown", - quality_policy=self.quality_policy, - ) - - -class MemoryContextAnnotation(BaseModel): - """Internal lifecycle context reserved for later Context Pack rendering. - - Nothing renders it yet, and it is deliberately absent from the public transport - models. - """ - - validity: MemoryLifecycleValidity - validity_reason: str | None = None - successor_entry_id: str | None = None - source_evidence: tuple[MemoryEvidenceSnapshot, ...] = () - source_count: int = Field(ge=0, default=0) - time_state: MemoryTimeState = "unknown" - quality_policy: Literal["neutral"] = "neutral" - - class MemoryRevisionChanges(BaseModel): """The compact changes stored by one exact Memory Revision.""" diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index 5eb28ad121..51028d7bf2 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -1087,66 +1087,10 @@ def _entry_text_type(): CheckConstraint("head_revision > 0", name="ck_pc_memory_entry_heads_revision_positive"), ) -# Lifecycle rows are maintained per entry and re-stamped only by the revision that -# changes them, like the active-head and vector rows. ``head_revision`` records the -# revision that last wrote a row; a read identifies rows by entry and proves -# completeness against the authoritative manifest rather than by revision. -# -# The CHECK constraints below describe the stage-1 vocabulary, which is why -# ``quality_policy`` admits only ``neutral``. Stage 2 has to widen them in one DDL -# migration alongside the model literals; the columns are narrow enough that an -# allow-list is the right shape, so the migration is a constraint swap rather than -# a rebuild. -MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE = Table( - "pc_memory_entry_lifecycle_projections", - SHARED_METADATA, - Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), - Column("family", identity_string(MAX_ARTIFACT_FAMILY_LENGTH), nullable=False), - Column("memory_artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), - Column("head_revision", Integer, nullable=False), - Column("entry_id", identity_string(MAX_MEMORY_ENTRY_ID_LENGTH), primary_key=True), - Column("entry_version_id", identity_string(MAX_MEMORY_ENTRY_ID_LENGTH), nullable=False), - Column("validity", identity_string(32), nullable=False), - Column("validity_reason", identity_string(256)), - Column("successor_entry_id", identity_string(MAX_MEMORY_ENTRY_ID_LENGTH)), - Column("source_count", Integer, nullable=False), - Column("rule_version", identity_string(64), nullable=False), - Column("quality_policy", identity_string(16), nullable=False), - ForeignKeyConstraint( - ("scope_id", "family", "memory_artifact_id", "head_revision"), - ( - "pc_artifacts.scope_id", - "pc_artifacts.family", - "pc_artifacts.artifact_id", - "pc_artifacts.revision", - ), - ondelete="RESTRICT", - ), - ForeignKeyConstraint( - ("scope_id", "memory_artifact_id", "entry_id", "entry_version_id"), - ( - "pc_memory_entry_versions.scope_id", - "pc_memory_entry_versions.memory_artifact_id", - "pc_memory_entry_versions.entry_id", - "pc_memory_entry_versions.entry_version_id", - ), - ondelete="RESTRICT", - ), - CheckConstraint("head_revision > 0", name="ck_pc_memory_lifecycle_projection_revision_positive"), - CheckConstraint("source_count >= 0", name="ck_pc_memory_lifecycle_projection_sources_nonnegative"), - CheckConstraint( - "validity IN ('current', 'inactive', 'superseded', 'unresolved_conflict')", - name="ck_pc_memory_lifecycle_projection_validity", - ), - CheckConstraint("quality_policy = 'neutral'", name="ck_pc_memory_lifecycle_projection_quality_neutral"), -) - - MEMORY_TABLES = ( MEMORY_ENTRY_VERSIONS_TABLE, MEMORY_ENTRY_EVIDENCE_TABLE, MEMORY_ENTRY_HEADS_TABLE, - MEMORY_ENTRY_LIFECYCLE_PROJECTIONS_TABLE, ) TOPIC_MEMORY_TABLES = ( diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index f21bd5fa14..95d5d281d6 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -239,7 +239,6 @@ from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, ConnectorBinding, - MemoryEvidenceDeclaration, Source, SourceCatalog, SourceDefinitionManifest, diff --git a/tests/e2e/real_experience_skill/harness.py b/tests/e2e/real_experience_skill/harness.py index 8c0c614512..62ade0fe23 100644 --- a/tests/e2e/real_experience_skill/harness.py +++ b/tests/e2e/real_experience_skill/harness.py @@ -104,7 +104,6 @@ "pc_access_owners", "pc_access_relationships", "pc_memory_vector_entries", - "pc_memory_entry_lifecycle_projections", "pc_memory_entry_heads", "pc_memory_entry_evidence", "pc_memory_entry_versions", From df6e4dcdd3469e537181dbffe1214f894c94dbdd Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Mon, 12 Oct 2026 22:47:28 +0800 Subject: [PATCH 16/30] fix: drop evidence dependencies before legacy memory tables --- evaluation/zcode_guidance/skill-lock.json | 11 ++++++----- tests/e2e/test_atomic_memory_migration_evidence.py | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/evaluation/zcode_guidance/skill-lock.json b/evaluation/zcode_guidance/skill-lock.json index 44f8f303f5..91961e9cd5 100644 --- a/evaluation/zcode_guidance/skill-lock.json +++ b/evaluation/zcode_guidance/skill-lock.json @@ -1,11 +1,12 @@ { "schema_version": 1, "source_path": "integrations/zcode/plugins/powercontext/skills/powercontext-project-context", - "source_commit": "20583250a6bdce5bab8716321ee3911844b6de2d", + "source_commit": "812e6e6207a454e488ca96c34389f59a62e5d500", "files": { - "SKILL.md": "88b6eed1a865b395dedc9508ab6cfcb15f2fc800f13bd9d5df02b963234906a1", - "references/review-publication.md": "83f3fa674d83785d06945669cbbe08b6d679665fed7e165d778dbfbcd9d10841", - "references/scope-memory.md": "2863f4ce693408136ede70818e0d7e25d197367e90db14ab6f889488cebc394d", - "references/work-handoff.md": "76dea45b708f2155e395029afe57b31ac112d8b13d614a6ab604aa436e3d2b2e" + ".powercontext-guidance.json": "655cbbb72bd53c08f9e846e315ed64d3a855eb54767e97580b00b88b237f5358", + "SKILL.md": "391cbb19e5dbc6b6b5e65c02753de702f67cead755bde07ba51efa19d01a4b55", + "references/review-publication.md": "844c3b4796d3b4e961bc943785dbe3c71960424644601b661b155fa9a6f8f3fe", + "references/scope-memory.md": "8d389bc708f68fbaadd0bd363980f970e67acdec2f1a97a622f3a8ccacfbb1b9", + "references/work-handoff.md": "248069bc7f69268ba90de38b2537fe9a385d4a220a35c54603d53e16df10988f" } } diff --git a/tests/e2e/test_atomic_memory_migration_evidence.py b/tests/e2e/test_atomic_memory_migration_evidence.py index e64eb10309..a4b577db22 100644 --- a/tests/e2e/test_atomic_memory_migration_evidence.py +++ b/tests/e2e/test_atomic_memory_migration_evidence.py @@ -1230,6 +1230,7 @@ def test_runtime_start_does_not_read_the_archive_or_legacy_entry_tables(migrated async def scenario() -> None: async with migration_profile(config) as profile, profile.database.transaction() as connection: await connection.execute(text(f"DROP TABLE {ARCHIVE_TABLE.name}")) + await connection.execute(text(f"DROP TABLE {MEMORY_ENTRY_EVIDENCE_TABLE.name}")) await connection.execute(text(f"DROP TABLE {MEMORY_ENTRY_HEADS_TABLE.name}")) await connection.execute(text(f"DROP TABLE {MEMORY_ENTRY_VERSIONS_TABLE.name}")) async with open_builtin_contexts(BuiltinConfig(database=config)) as contexts: @@ -1270,6 +1271,7 @@ async def scenario() -> None: await _seed_and_migrate(config, pinned) async with migration_profile(config) as profile, profile.database.transaction() as connection: await connection.execute(text(f"DROP TABLE {ARCHIVE_TABLE.name}")) + await connection.execute(text(f"DROP TABLE {MEMORY_ENTRY_EVIDENCE_TABLE.name}")) await connection.execute(text(f"DROP TABLE {MEMORY_ENTRY_HEADS_TABLE.name}")) await connection.execute(text(f"DROP TABLE {MEMORY_ENTRY_VERSIONS_TABLE.name}")) async with ( From 3eed5f38a35bbc001e7b7f79fcbca8ed751da64c Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Tue, 13 Oct 2026 00:00:48 +0800 Subject: [PATCH 17/30] fix: import legacy evidence table in migration tests --- tests/e2e/test_atomic_memory_migration_evidence.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/test_atomic_memory_migration_evidence.py b/tests/e2e/test_atomic_memory_migration_evidence.py index a4b577db22..d2b2049037 100644 --- a/tests/e2e/test_atomic_memory_migration_evidence.py +++ b/tests/e2e/test_atomic_memory_migration_evidence.py @@ -81,6 +81,7 @@ ARTIFACT_LINEAGE_SOURCES_TABLE, ARTIFACTS_TABLE, DREAM_RUNS_TABLE, + MEMORY_ENTRY_EVIDENCE_TABLE, MEMORY_ENTRY_HEADS_TABLE, MEMORY_ENTRY_VERSIONS_TABLE, SCOPES_TABLE, From ba977912ee6da91454402f3edd3e3c8aab9fd211 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Tue, 13 Oct 2026 00:48:47 +0800 Subject: [PATCH 18/30] fix: remove evidence before rebuilding legacy memory schema --- tests/e2e/test_atomic_memory_migration_evidence.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/e2e/test_atomic_memory_migration_evidence.py b/tests/e2e/test_atomic_memory_migration_evidence.py index d2b2049037..3156bacd96 100644 --- a/tests/e2e/test_atomic_memory_migration_evidence.py +++ b/tests/e2e/test_atomic_memory_migration_evidence.py @@ -1376,6 +1376,10 @@ async def legacy_schema(contexts, connection) -> None: # Restore the historical foreign keys with frozen SQL rows. Current table # metadata intentionally no longer attaches these tables to public Artifacts. versions = [dict(row) for row in (await connection.execute(select(MEMORY_ENTRY_VERSIONS_TABLE))).mappings()] + # OceanBase/SeekDB enforce the evidence-to-version FK while dropping the + # legacy parent tables; remove that child table before rebuilding the + # historical schema. + await connection.execute(text("DROP TABLE pc_memory_entry_evidence")) await connection.execute(text("DROP TABLE pc_memory_entry_heads")) await connection.execute(text("DROP TABLE pc_memory_entry_versions")) metadata = MetaData() From 9f2412484fa72982d11be388e685821fc00dd89d Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Tue, 13 Oct 2026 12:27:44 +0800 Subject: [PATCH 19/30] fix(sources): preserve nested contracts and capture declarations --- .../builtin/persistence/sources.py | 1 + src/powercontext/sources/catalog.py | 5 +- src/powercontext/sources/definitions.py | 9 +- src/powercontext/sources/observations.py | 104 ++-- .../test_atomic_memory_migration_vector.py | 7 +- .../test_atomic_memory_projection_rebuild.py | 2 + tests/e2e/test_source_declaration_contract.py | 135 ++++++ .../fixtures/source_declaration_upgrade.json | 449 ++++++++++++++++++ tests/source_contract_models.py | 118 +++++ tests/test_source_observations.py | 18 + 10 files changed, 774 insertions(+), 74 deletions(-) create mode 100644 tests/e2e/test_source_declaration_contract.py create mode 100644 tests/fixtures/source_declaration_upgrade.json create mode 100644 tests/source_contract_models.py diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index ba3cd56c56..ee9bcf8785 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -112,6 +112,7 @@ async def add_with_status( ref = SourceRef(source_type=source.source_type, source_id=source.name) representation: Literal["native", "observation"] = "observation" else: + source = self._registry.materialize(source) definition = self._registry.definition_for_source(source) ref = SourceRef(source_type=definition.name, source_id=source.name) representation = "native" diff --git a/src/powercontext/sources/catalog.py b/src/powercontext/sources/catalog.py index 8207301d1d..a6b0d46a07 100644 --- a/src/powercontext/sources/catalog.py +++ b/src/powercontext/sources/catalog.py @@ -23,7 +23,7 @@ SourceNotFoundError, ) from powercontext.sources.adapters import SourceAdapter -from powercontext.sources.definitions import SourceDefinitionRegistry, definition_memory_evidence +from powercontext.sources.definitions import SourceDefinitionRegistry from powercontext.sources.models import MemoryEvidenceDeclaration, Source, SourceProjectionKey, SourceRef from powercontext.sources.observations import SourceObservation from powercontext.sources.protocols import SourceCatalogBackend @@ -77,8 +77,7 @@ def materialize(self, source: Source, /) -> Source: if isinstance(source, SourceObservation): return source - definition = self._registry.definition_for_source(source) - return source.model_copy(update={"memory_evidence": definition_memory_evidence(definition)}) + return self._registry.materialize(source) async def resolve(self, value: object, /) -> Source: return await self._registry.resolve(value) diff --git a/src/powercontext/sources/definitions.py b/src/powercontext/sources/definitions.py index dcb732f58a..5b7bfe612d 100644 --- a/src/powercontext/sources/definitions.py +++ b/src/powercontext/sources/definitions.py @@ -172,8 +172,7 @@ async def resolve(self, value: object, /) -> Source: source = await definition.resolve(value) if type(source) is not definition.source_class: raise InvalidSourceResultError(definition.name, "resolve", definition.source_class, type(source)) - self.definition_for_source(source) - return cast(Source, source.model_copy(update={"memory_evidence": definition_memory_evidence(definition)})) + return self.materialize(source) async def read(self, source: Source, /) -> object: definition = self.definition_for_source(source) @@ -214,6 +213,12 @@ def memory_evidence(self, source: Source, /) -> MemoryEvidenceDeclaration: self.definition_for_source(source) return source.memory_evidence + def materialize(self, source: Source, /) -> Source: + """Stamp Definition-owned metadata on every native persistence path.""" + + definition = self.definition_for_source(source) + return source.model_copy(update={"memory_evidence": definition_memory_evidence(definition)}) + def definition_memory_evidence(definition: object, /) -> MemoryEvidenceDeclaration: """Return a Definition's evidence declaration, or the neutral one when absent. diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index c20b6a4cda..aee886cff7 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -17,8 +17,6 @@ from __future__ import annotations import hashlib -import json -import re from typing import Any, Literal import rfc8785 @@ -33,6 +31,9 @@ model_serializer, model_validator, ) +from pydantic.json_schema import GenerateJsonSchema, JsonSchemaValue +from pydantic_core import core_schema +from typing_extensions import override from powercontext.errors import ( InvalidSourceDefinitionError, @@ -54,7 +55,6 @@ ) _JSON_VALUE = TypeAdapter(JsonValue) -_JSON_SCHEMA_REFERENCE = re.compile(r"#/\$defs/([^\"\\]+)") class SourceProjectionManifest(BaseModel): @@ -212,77 +212,47 @@ def _input_contract_schema(source_class: type[Source]) -> dict[str, JsonValue]: because an unused ``$defs`` entry still changes the hashed document. """ - schema = _json_object(source_class.model_json_schema()) - _remove_inherited_declaration(schema, source_class) - _drop_unreferenced_definitions(schema) - return schema + return _json_object(source_class.model_json_schema(schema_generator=_SourceInputSchema)) -def _remove_inherited_declaration(node: JsonValue, source_class: type[Source]) -> None: - """Strip the inherited declaration from every object schema that carries it. +class _SourceInputSchema(GenerateJsonSchema): + """Omit Source metadata before Pydantic assigns captured schema reference names.""" - Recursive and nested Sources keep their fields in a local definition rather - than at the root, so the walk covers the whole document. The field is matched - by the schema a Source inherits rather than by its name alone, because an - ordinary captured model may declare a field of its own with the same name and - a different meaning, and that field belongs to the captured contract. - """ + @override + def model_schema(self, schema: core_schema.ModelSchema) -> JsonSchemaValue: + if issubclass(schema["cls"], Source): + schema = schema.copy() + schema["schema"] = _source_input_fields(schema["schema"]) + return super().model_schema(schema) - source_titles = _source_schema_titles(source_class) - def walk(value: JsonValue) -> None: - if isinstance(value, dict): - properties = value.get("properties") - if value.get("title") in source_titles and isinstance(properties, dict): - properties.pop("memory_evidence", None) - required = value.get("required") - if isinstance(required, list): - value["required"] = [field for field in required if field != "memory_evidence"] - for child in value.values(): - walk(child) - elif isinstance(value, list): - for child in value: - walk(child) +def _source_input_fields(schema: Any) -> Any: + """Copy only the wrappers containing this Source's field definitions.""" - walk(node) + if schema["type"] == "model-fields": + return { + **schema, + "fields": {name: field for name, field in schema["fields"].items() if name not in DECLARATION_OWNED_FIELDS}, + } + if schema["type"] in {"function-before", "function-after", "function-wrap"}: + return {**schema, "schema": _source_input_fields(schema["schema"])} + return schema -def _source_schema_titles(source_class: type[Source]) -> set[str]: - """Return schema titles belonging to Source subclasses in the payload tree.""" +def _payload_exclusions(value: object) -> dict[Any, Any]: + """Exclude metadata by model ownership while retaining captured fields.""" - titles: set[str] = set() - pending: list[type[BaseModel]] = [source_class] - seen: set[type[BaseModel]] = set() - while pending: - model = pending.pop() - if model in seen: - continue - seen.add(model) - titles.add(str(model.model_config.get("title", model.__name__))) - for field in model.model_fields.values(): - annotation = field.annotation - if isinstance(annotation, type) and issubclass(annotation, Source): - pending.append(annotation) - return titles - - -def _drop_unreferenced_definitions(schema: dict[str, JsonValue]) -> None: - """Remove ``$defs`` entries no longer referenced anywhere in the schema.""" - - definitions = schema.get("$defs") - if not isinstance(definitions, dict): - return - while True: - # Reference discovery has to see the whole document: the properties that - # point at a definition live outside ``$defs``. - referenced = set(_JSON_SCHEMA_REFERENCE.findall(json.dumps(schema))) - unreferenced = [name for name in definitions if name not in referenced] - if not unreferenced: - break - for name in unreferenced: - del definitions[name] - if not definitions: - del schema["$defs"] + if isinstance(value, BaseModel): + excluded: dict[Any, Any] = dict.fromkeys(DECLARATION_OWNED_FIELDS, True) if isinstance(value, Source) else {} + for name in type(value).model_fields: + if name not in excluded and (nested := _payload_exclusions(getattr(value, name))): + excluded[name] = nested + return excluded + if isinstance(value, (list, tuple)): + return {index: nested for index, item in enumerate(value) if (nested := _payload_exclusions(item))} + if isinstance(value, dict): + return {key: nested for key, item in value.items() if (nested := _payload_exclusions(item))} + return {} def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> SourceDefinitionManifest: @@ -335,9 +305,7 @@ def project_source_for_transport( # Definition-owned metadata is not part of it. Carrying it here would make a # generated observation fail its own schema, and would change the payload of an # unchanged Source, so replaying a pre-upgrade observation would conflict. - payload = _json_object({ - key: value for key, value in source.model_dump(mode="json").items() if key not in DECLARATION_OWNED_FIELDS - }) + payload = _json_object(source.model_dump(mode="json", exclude=_payload_exclusions(source))) projections = tuple( SourceProjectionValue(key=key, value=registry.project(source, key)) for key in registry.projection_keys(source) ) diff --git a/tests/e2e/test_atomic_memory_migration_vector.py b/tests/e2e/test_atomic_memory_migration_vector.py index 4f5fa1b29a..de8285fda1 100644 --- a/tests/e2e/test_atomic_memory_migration_vector.py +++ b/tests/e2e/test_atomic_memory_migration_vector.py @@ -60,7 +60,12 @@ async def scenario() -> None: historical = await memory.get(_atomic("alpha", 1).artifact_id, revision=1) async with migration_profile(config, load_vector_extension=True) as profile: async with profile.database.transaction() as connection: - for table in ("pc_memory_artifact_archive", "pc_memory_entry_heads", "pc_memory_entry_versions"): + for table in ( + "pc_memory_artifact_archive", + "pc_memory_entry_evidence", + "pc_memory_entry_heads", + "pc_memory_entry_versions", + ): await connection.execute(text(f"DROP TABLE {table}")) await connection.execute(text("DELETE FROM pc_atomic_memory_current")) rebuilt = await rebuild_atomic_memory_projection( diff --git a/tests/e2e/test_atomic_memory_projection_rebuild.py b/tests/e2e/test_atomic_memory_projection_rebuild.py index 3075699659..590ea71e20 100644 --- a/tests/e2e/test_atomic_memory_projection_rebuild.py +++ b/tests/e2e/test_atomic_memory_projection_rebuild.py @@ -48,6 +48,7 @@ async def scenario() -> None: if retention in {"entries_only", "removed"}: await connection.execute(text("DROP TABLE pc_memory_artifact_archive")) if retention in {"archive_only", "removed"}: + await connection.execute(text("DROP TABLE pc_memory_entry_evidence")) await connection.execute(text("DROP TABLE pc_memory_entry_heads")) await connection.execute(text("DROP TABLE pc_memory_entry_versions")) if retention == "unreadable": @@ -126,6 +127,7 @@ async def scenario() -> None: async with SQLiteProfile.open(config, tables=()) as profile: async with profile.database.transaction() as connection: await connection.execute(text("DROP TABLE pc_memory_artifact_archive")) + await connection.execute(text("DROP TABLE pc_memory_entry_evidence")) await connection.execute(text("DROP TABLE pc_memory_entry_heads")) await connection.execute(text("DROP TABLE pc_memory_entry_versions")) identity = {"scope": SCOPE, "id": _atomic("alpha", 2).artifact_id} diff --git a/tests/e2e/test_source_declaration_contract.py b/tests/e2e/test_source_declaration_contract.py new file mode 100644 index 0000000000..58b6235450 --- /dev/null +++ b/tests/e2e/test_source_declaration_contract.py @@ -0,0 +1,135 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public Source ingestion and capture preserve Definition-owned metadata.""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import replace +from pathlib import Path +from typing import cast + +import httpx +import pytest + +from powercontext.builtin.runtime import BuiltinConfig, CaptureSource, open_builtin_contexts, open_builtin_runtime +from powercontext.builtin.scope import ScopeDraft +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION, ContentCapture +from powercontext.server.app import ServerApplication, create_app +from powercontext.sources import ( + MemoryEvidenceAuthority, + MemoryEvidenceDeclaration, + MemoryEvidenceVerification, + SourceDefinitionRegistry, + manifest_for_definition, + project_source_for_transport, +) +from tests.e2e.atomic_memory_migration_backend import MigrationBackend +from tests.source_contract_models import source_contract_case + + +@pytest.mark.parametrize("case", ["nested", "title_collision", "reference_collision"]) +def test_neutral_worker_upgrade_preserves_http_registration_and_replay( + migration_backend: MigrationBackend, + case: str, +) -> None: + baseline = json.loads( + (Path(__file__).parents[1] / "fixtures/source_declaration_upgrade.json").read_text(encoding="utf-8") + )[case] + definition, source = source_contract_case(case) + registry = SourceDefinitionRegistry((definition,)) + config = BuiltinConfig(database=migration_backend.config) + + async def scenario() -> None: + async with open_builtin_runtime(config) as runtime: + assert runtime.scopes is not None + scope = await runtime.scopes.create( + ScopeDraft(title="Sources", summary="Source contract", idempotency_key="source-contract") + ) + app = create_app(application=cast(ServerApplication, runtime)) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://testserver" + ) as client: + registered = await client.post( + "/v1/source-definitions/register", json={"manifest": baseline["manifest"]} + ) + assert registered.status_code == 200, registered.text + accepted = await client.post( + "/v1/source-observations", json={"scope_id": scope.scope_id, "observation": baseline["observation"]} + ) + assert accepted.status_code == 202, accepted.text + receipt = accepted.json() + async with open_builtin_runtime(config) as runtime: + app = create_app(application=cast(ServerApplication, runtime)) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://testserver" + ) as client: + registered = await client.post( + "/v1/source-definitions/register", + json={"manifest": manifest_for_definition(definition).model_dump(mode="json", by_alias=True)}, + ) + assert registered.status_code == 200, registered.text + replayed = await client.post( + "/v1/source-observations", + json={ + "scope_id": scope.scope_id, + "observation": project_source_for_transport(registry, source).model_dump(mode="json"), + }, + ) + assert replayed.status_code == 202, replayed.text + assert replayed.json() == receipt + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("native_first", [True, False]) +def test_record_and_native_capture_stamp_the_same_declaration( + migration_backend: MigrationBackend, + native_first: bool, +) -> None: + declaration = MemoryEvidenceDeclaration( + authority=MemoryEvidenceAuthority.SYSTEM_ATTESTED, verification=MemoryEvidenceVerification.VERIFIED + ) + registry = SourceDefinitionRegistry((replace(CONTENT_SOURCE_DEFINITION, memory_evidence=declaration),)) + config = BuiltinConfig(database=migration_backend.config) + + async def scenario() -> None: + async with open_builtin_runtime(config, source_registry=registry) as runtime: + assert runtime.scopes is not None + scope = await runtime.scopes.create( + ScopeDraft(title="Capture", summary="Capture contract", idempotency_key="capture-contract") + ) + value = ContentCapture(source_id="same-input", content="Identical captured content.") + + async def native() -> None: + async with open_builtin_contexts(config, source_registry=registry) as contexts: + context = await contexts.get(scope.scope_id) + stored, sequence = await context.sources.capture(value) + assert stored.memory_evidence == declaration + assert sequence == 1 + + async def recorded() -> None: + async with open_builtin_runtime(config, source_registry=registry) as runtime: + receipt = await runtime.sources.for_scope(scope.scope_id).capture( + CaptureSource(source_id=value.source_id, content=value.content, metadata={}) + ) + assert receipt.sequence == 1 + + for capture in (native, recorded) if native_first else (recorded, native): + await capture() + await native() + + asyncio.run(scenario()) diff --git a/tests/fixtures/source_declaration_upgrade.json b/tests/fixtures/source_declaration_upgrade.json new file mode 100644 index 0000000000..3a85de30c9 --- /dev/null +++ b/tests/fixtures/source_declaration_upgrade.json @@ -0,0 +1,449 @@ +{ + "nested": { + "manifest": { + "name": "contract-nested", + "version": "1", + "fingerprint": "sha256:388066571c143240354fdab663935a1822388378912b163992227c2f52a8addd", + "source_schema": { + "$defs": { + "CapturedAudit": { + "additionalProperties": false, + "properties": { + "memory_evidence": { + "title": "Memory Evidence", + "type": "string" + }, + "summary": { + "title": "Summary", + "type": "string" + } + }, + "required": [ + "memory_evidence", + "summary" + ], + "title": "AuditSource", + "type": "object" + }, + "CapturedBundle": { + "additionalProperties": false, + "properties": { + "child": { + "$ref": "#/$defs/ChildSource" + }, + "memory_evidence": { + "title": "Memory Evidence", + "type": "string" + } + }, + "required": [ + "child", + "memory_evidence" + ], + "title": "CapturedBundle", + "type": "object" + }, + "ChildSource": { + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "definition_version": { + "default": "1", + "title": "Definition Version", + "type": "string" + }, + "materialization": { + "$ref": "#/$defs/SourceMaterialization" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "content": { + "title": "Content", + "type": "string" + }, + "children": { + "items": { + "$ref": "#/$defs/ChildSource" + }, + "title": "Children", + "type": "array" + } + }, + "required": [ + "name", + "materialization", + "content" + ], + "title": "ChildSource", + "type": "object" + }, + "SourceMaterialization": { + "description": "Describe where the value read for a Source comes from.", + "enum": [ + "captured", + "referenced" + ], + "title": "SourceMaterialization", + "type": "string" + } + }, + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "definition_version": { + "default": "1", + "title": "Definition Version", + "type": "string" + }, + "materialization": { + "$ref": "#/$defs/SourceMaterialization" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "child": { + "$ref": "#/$defs/ChildSource" + }, + "children": { + "items": { + "$ref": "#/$defs/ChildSource" + }, + "title": "Children", + "type": "array" + }, + "by_name": { + "additionalProperties": { + "$ref": "#/$defs/ChildSource" + }, + "title": "By Name", + "type": "object" + }, + "optional": { + "anyOf": [ + { + "$ref": "#/$defs/ChildSource" + }, + { + "type": "null" + } + ] + }, + "choice": { + "anyOf": [ + { + "$ref": "#/$defs/ChildSource" + }, + { + "$ref": "#/$defs/CapturedAudit" + } + ], + "title": "Choice" + }, + "bundle": { + "$ref": "#/$defs/CapturedBundle" + } + }, + "required": [ + "name", + "materialization", + "child", + "children", + "by_name", + "optional", + "choice", + "bundle" + ], + "title": "BundleSource", + "type": "object" + }, + "projections": [] + }, + "observation": { + "name": "bundle", + "definition_version": "1", + "materialization": "captured", + "description": null, + "source_type": "contract-nested", + "definition_fingerprint": "sha256:388066571c143240354fdab663935a1822388378912b163992227c2f52a8addd", + "payload": { + "name": "bundle", + "definition_version": "1", + "materialization": "captured", + "description": null, + "child": { + "name": "child", + "definition_version": "1", + "materialization": "captured", + "description": null, + "content": "Child evidence.", + "children": [ + { + "name": "child", + "definition_version": "1", + "materialization": "captured", + "description": null, + "content": "Child evidence.", + "children": [] + } + ] + }, + "children": [ + { + "name": "child", + "definition_version": "1", + "materialization": "captured", + "description": null, + "content": "Child evidence.", + "children": [] + } + ], + "by_name": { + "child": { + "name": "child", + "definition_version": "1", + "materialization": "captured", + "description": null, + "content": "Child evidence.", + "children": [] + } + }, + "optional": { + "name": "child", + "definition_version": "1", + "materialization": "captured", + "description": null, + "content": "Child evidence.", + "children": [] + }, + "choice": { + "memory_evidence": "captured evidence", + "summary": "An audit report." + }, + "bundle": { + "child": { + "name": "child", + "definition_version": "1", + "materialization": "captured", + "description": null, + "content": "Child evidence.", + "children": [] + }, + "memory_evidence": "captured bundle evidence" + } + }, + "projections": [] + } + }, + "title_collision": { + "manifest": { + "name": "contract-title_collision", + "version": "1", + "fingerprint": "sha256:a8556c2eb9c363f0da45f80fddb25ddec0d4674dba6e8689b5967740ba8f488a", + "source_schema": { + "$defs": { + "CapturedAudit": { + "additionalProperties": false, + "properties": { + "memory_evidence": { + "title": "Memory Evidence", + "type": "string" + }, + "summary": { + "title": "Summary", + "type": "string" + } + }, + "required": [ + "memory_evidence", + "summary" + ], + "title": "AuditSource", + "type": "object" + }, + "SourceMaterialization": { + "description": "Describe where the value read for a Source comes from.", + "enum": [ + "captured", + "referenced" + ], + "title": "SourceMaterialization", + "type": "string" + } + }, + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "definition_version": { + "default": "1", + "title": "Definition Version", + "type": "string" + }, + "materialization": { + "$ref": "#/$defs/SourceMaterialization" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "audit": { + "$ref": "#/$defs/CapturedAudit" + } + }, + "required": [ + "name", + "materialization", + "audit" + ], + "title": "AuditSource", + "type": "object" + }, + "projections": [] + }, + "observation": { + "name": "audit", + "definition_version": "1", + "materialization": "captured", + "description": null, + "source_type": "contract-title_collision", + "definition_fingerprint": "sha256:a8556c2eb9c363f0da45f80fddb25ddec0d4674dba6e8689b5967740ba8f488a", + "payload": { + "name": "audit", + "definition_version": "1", + "materialization": "captured", + "description": null, + "audit": { + "memory_evidence": "captured evidence", + "summary": "An audit report." + } + }, + "projections": [] + } + }, + "reference_collision": { + "manifest": { + "name": "contract-reference_collision", + "version": "1", + "fingerprint": "sha256:73686a0afda11f31bee74ed39008c36c224779539008bb7a77e6ec773e5a208f", + "source_schema": { + "$defs": { + "MemoryEvidenceDeclaration": { + "additionalProperties": false, + "description": "Ordinary captured data sharing the library declaration's class name.", + "properties": { + "detail": { + "title": "Detail", + "type": "string" + } + }, + "required": [ + "detail" + ], + "title": "MemoryEvidenceDeclaration", + "type": "object" + }, + "SourceMaterialization": { + "description": "Describe where the value read for a Source comes from.", + "enum": [ + "captured", + "referenced" + ], + "title": "SourceMaterialization", + "type": "string" + } + }, + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "definition_version": { + "default": "1", + "title": "Definition Version", + "type": "string" + }, + "materialization": { + "$ref": "#/$defs/SourceMaterialization" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "captured": { + "$ref": "#/$defs/MemoryEvidenceDeclaration" + } + }, + "required": [ + "name", + "materialization", + "captured" + ], + "title": "CollisionSource", + "type": "object" + }, + "projections": [] + }, + "observation": { + "name": "collision", + "definition_version": "1", + "materialization": "captured", + "description": null, + "source_type": "contract-reference_collision", + "definition_fingerprint": "sha256:73686a0afda11f31bee74ed39008c36c224779539008bb7a77e6ec773e5a208f", + "payload": { + "name": "collision", + "definition_version": "1", + "materialization": "captured", + "description": null, + "captured": { + "detail": "Captured declaration-shaped data." + } + }, + "projections": [] + } + } +} diff --git a/tests/source_contract_models.py b/tests/source_contract_models.py new file mode 100644 index 0000000000..8254153206 --- /dev/null +++ b/tests/source_contract_models.py @@ -0,0 +1,118 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Captured model graphs shared by worker and neutral-upgrade acceptance tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from pydantic import BaseModel, ConfigDict, Field + +from powercontext.sources import Source, SourceMaterialization +from powercontext.sources.definitions import AdapterSourceDefinition + + +class CapturedAudit(BaseModel): + model_config = ConfigDict(title="AuditSource", extra="forbid") + + memory_evidence: str + summary: str + + +class ChildSource(Source): + model_config = ConfigDict(extra="forbid") + + content: str + children: list[ChildSource] = Field(default_factory=list) + + +class CapturedBundle(BaseModel): + model_config = ConfigDict(extra="forbid") + + child: ChildSource + memory_evidence: str + + +class BundleSource(Source): + model_config = ConfigDict(extra="forbid") + + child: ChildSource + children: list[ChildSource] + by_name: dict[str, ChildSource] + optional: ChildSource | None + choice: ChildSource | CapturedAudit + bundle: CapturedBundle + + +class AuditSource(Source): + model_config = ConfigDict(extra="forbid") + + audit: CapturedAudit + + +class MemoryEvidenceDeclaration(BaseModel): + """Ordinary captured data sharing the library declaration's class name.""" + + model_config = ConfigDict(extra="forbid") + + detail: str + + +class CollisionSource(Source): + model_config = ConfigDict(extra="forbid") + + captured: MemoryEvidenceDeclaration + + +@dataclass(frozen=True) +class _CapturedAdapter: + name: str + input_class: type[Source] + source_class: type[Source] + + async def resolve(self, value: Source, /) -> Source: + return value + + async def read(self, source: Source, /) -> object: + return source.model_dump(mode="json") + + +def source_contract_case(name: str) -> tuple[AdapterSourceDefinition[Source, Source, object], Source]: + audit = CapturedAudit(memory_evidence="captured evidence", summary="An audit report.") + child = ChildSource(name="child", materialization=SourceMaterialization.CAPTURED, content="Child evidence.") + nested = child.model_copy(update={"children": [child]}) + if name == "nested": + source = BundleSource( + name="bundle", + materialization=SourceMaterialization.CAPTURED, + child=nested, + children=[child], + by_name={"child": child}, + optional=child, + choice=audit, + bundle=CapturedBundle(child=child, memory_evidence="captured bundle evidence"), + ) + elif name == "title_collision": + source = AuditSource(name="audit", materialization=SourceMaterialization.CAPTURED, audit=audit) + elif name == "reference_collision": + source = CollisionSource( + name="collision", + materialization=SourceMaterialization.CAPTURED, + captured=MemoryEvidenceDeclaration(detail="Captured declaration-shaped data."), + ) + else: + raise ValueError(name) + adapter = _CapturedAdapter(name=f"contract-{name}", input_class=type(source), source_class=type(source)) + return AdapterSourceDefinition(adapter), source diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 6def3e787a..9a9f17a29b 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -19,6 +19,7 @@ import json import urllib.request from dataclasses import replace +from pathlib import Path from typing import Any, cast import pytest @@ -51,6 +52,7 @@ project_source_for_transport, ) from powercontext.sources.definitions import AdapterSourceDefinition +from tests.source_contract_models import source_contract_case class EmptySourceBackend: @@ -61,6 +63,22 @@ async def get(self, source: Source, /) -> Source: raise AssertionError(source) +@pytest.mark.parametrize("case", ["nested", "title_collision", "reference_collision"]) +def test_source_contract_preserves_pre_declaration_schema_and_payload(case: str) -> None: + # Frozen output from these models against base b5bcfe24bcce9c574ee85175d623bb9963970a41. + baseline = json.loads( + (Path(__file__).parent / "fixtures/source_declaration_upgrade.json").read_text(encoding="utf-8") + ) + definition, source = source_contract_case(case) + registry = SourceDefinitionRegistry((definition,)) + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + + assert manifest.model_dump(mode="json", by_alias=True) == baseline[case]["manifest"] + assert observation.payload == baseline[case]["observation"]["payload"] + _json_schema_validator(manifest.name, manifest.source_schema).validate(observation.payload) + + def _legacy_manifest(current: SourceDefinitionManifest) -> SourceDefinitionManifest: """Return the declaration shape emitted before the evidence field existed.""" From 0dcad004e1c68ed971cf6521e9e6eed45cb0e14f Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Tue, 13 Oct 2026 12:28:33 +0800 Subject: [PATCH 20/30] test(sources): run declaration regressions on real backends --- scripts/run_atomic_memory_clean_migration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/run_atomic_memory_clean_migration.py b/scripts/run_atomic_memory_clean_migration.py index 710fd2ebd1..838e1b0f33 100644 --- a/scripts/run_atomic_memory_clean_migration.py +++ b/scripts/run_atomic_memory_clean_migration.py @@ -26,6 +26,7 @@ "tests/e2e/test_atomic_memory_migration_evidence.py", "tests/e2e/test_atomic_memory_grant_migration.py", "tests/e2e/test_atomic_memory_migration_vector.py", + "tests/e2e/test_source_declaration_contract.py", ) From f87d6fbc5de8d41ad6d6e3cf5c6b28a6876f23a7 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Tue, 13 Oct 2026 18:54:32 +0800 Subject: [PATCH 21/30] style(sources): format payload traversal --- src/powercontext/sources/observations.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 805c9313db..9770b21f96 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -264,7 +264,11 @@ def _payload_exclusions(value: object) -> dict[Any, Any]: from dataclasses import fields, is_dataclass if is_dataclass(value) and not isinstance(value, type): - return {field.name: nested for field in fields(value) if (nested := _payload_exclusions(getattr(value, field.name)))} + return { + field.name: nested + for field in fields(value) + if (nested := _payload_exclusions(getattr(value, field.name))) + } except TypeError: pass if isinstance(value, dict): From 02a780c48547604cfc0e3cf55f9657268430fc6a Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Tue, 13 Oct 2026 19:24:47 +0800 Subject: [PATCH 22/30] fix(sources): type root model payload exclusions --- src/powercontext/sources/observations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 9770b21f96..bfdfb215ab 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -253,7 +253,7 @@ def _payload_exclusions(value: object) -> dict[Any, Any]: if type(value).__name__ == "RootModel" or ( set(type(value).model_fields) == {"root"} and hasattr(value, "root") ): - return _payload_exclusions(value.root) + return _payload_exclusions(getattr(value, "root")) for name in type(value).model_fields: if name not in excluded and (nested := _payload_exclusions(getattr(value, name))): excluded[name] = nested From 23d8dcd4e7d0c6f5825850db2c4e4c74b8d729d4 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Tue, 13 Oct 2026 20:00:24 +0800 Subject: [PATCH 23/30] fix(sources): make root model narrowing explicit --- src/powercontext/sources/observations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index bfdfb215ab..1d09ce5a79 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -17,7 +17,7 @@ from __future__ import annotations import hashlib -from typing import Any, Literal +from typing import Any, Literal, cast import rfc8785 from pydantic import ( @@ -253,7 +253,7 @@ def _payload_exclusions(value: object) -> dict[Any, Any]: if type(value).__name__ == "RootModel" or ( set(type(value).model_fields) == {"root"} and hasattr(value, "root") ): - return _payload_exclusions(getattr(value, "root")) + return _payload_exclusions(cast(Any, value).root) for name in type(value).model_fields: if name not in excluded and (nested := _payload_exclusions(getattr(value, name))): excluded[name] = nested From df3157e1d43a4ca5ba008a4b17d4053dbfb918ef Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Tue, 13 Oct 2026 23:31:37 +0800 Subject: [PATCH 24/30] fix(sources): harden declaration projection paths --- .../builtin/persistence/sources.py | 4 +- .../builtin/scope/subject_sources.py | 2 +- src/powercontext/sources/definitions.py | 7 ++- src/powercontext/sources/observations.py | 48 ++++++++++++++++--- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index cc4da9b64a..100e9c0d24 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -92,10 +92,12 @@ async def add( scope_id: str, source: Source, /, + *, + apply_definition: bool = False, ) -> StoredSource: """Add one stable Source or return an identical existing capture.""" - stored, _created = await self.add_with_status(connection, scope_id, source) + stored, _created = await self.add_with_status(connection, scope_id, source, apply_definition=apply_definition) return stored async def add_with_status( diff --git a/src/powercontext/builtin/scope/subject_sources.py b/src/powercontext/builtin/scope/subject_sources.py index 9b73a0bc33..8a0f664f3e 100644 --- a/src/powercontext/builtin/scope/subject_sources.py +++ b/src/powercontext/builtin/scope/subject_sources.py @@ -128,7 +128,7 @@ async def _write( await self._scopes.ensure_binding(connection, key, target) stored = {} for actual_scope in sorted((scope_id, target)): - stored[actual_scope] = await self._sources.add(connection, actual_scope, source) + stored[actual_scope] = await self._sources.add(connection, actual_scope, source, apply_definition=True) return target, (_source_record(scope_id, stored[scope_id]), _source_record(target, stored[target])) diff --git a/src/powercontext/sources/definitions.py b/src/powercontext/sources/definitions.py index d4ebcbb021..0edc17db52 100644 --- a/src/powercontext/sources/definitions.py +++ b/src/powercontext/sources/definitions.py @@ -217,7 +217,12 @@ def materialize(self, source: Source, /) -> Source: """Stamp Definition-owned metadata on every native persistence path.""" definition = self.definition_for_source(source) - return source.model_copy(update={"memory_evidence": definition_memory_evidence(definition)}) + stamped = source.model_copy(update={"memory_evidence": definition_memory_evidence(definition)}) + try: + type(stamped).model_validate(stamped.model_dump(mode="python")) + except (TypeError, ValueError) as error: + raise InvalidSourceResultError(definition.name, "materialize", type(source), type(source)) from error + return stamped def definition_memory_evidence(definition: object, /) -> MemoryEvidenceDeclaration: diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 1d09ce5a79..49c48ffc4b 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -25,6 +25,7 @@ ConfigDict, Field, JsonValue, + RootModel, SerializerFunctionWrapHandler, TypeAdapter, field_validator, @@ -227,8 +228,8 @@ def model_schema(self, schema: core_schema.ModelSchema) -> JsonSchemaValue: @override def encode_default(self, dft: Any) -> Any: - if isinstance(dft, BaseModel): - dft = dft.model_dump(mode="json", exclude=_payload_exclusions(dft)) + if isinstance(dft, (BaseModel, list, tuple, set, frozenset, dict)): + dft = _normalize_default(dft) return super().encode_default(dft) @@ -250,16 +251,17 @@ def _payload_exclusions(value: object) -> dict[Any, Any]: if isinstance(value, BaseModel): excluded: dict[Any, Any] = dict.fromkeys(DECLARATION_OWNED_FIELDS, True) if isinstance(value, Source) else {} - if type(value).__name__ == "RootModel" or ( - set(type(value).model_fields) == {"root"} and hasattr(value, "root") - ): + if isinstance(value, RootModel): return _payload_exclusions(cast(Any, value).root) for name in type(value).model_fields: if name not in excluded and (nested := _payload_exclusions(getattr(value, name))): excluded[name] = nested return excluded if isinstance(value, (list, tuple, set, frozenset)): - return {index: nested for index, item in enumerate(value) if (nested := _payload_exclusions(item))} + nested = {index: child for index, item in enumerate(value) if (child := _payload_exclusions(item))} + if isinstance(value, (set, frozenset)) and nested: + return {"__all__": next(iter(nested.values()))} + return nested try: from dataclasses import fields, is_dataclass @@ -276,6 +278,38 @@ def _payload_exclusions(value: object) -> dict[Any, Any]: return {} +def _normalize_default(value: Any) -> Any: + """Serialize defaults while removing Source-owned declaration fields.""" + if isinstance(value, BaseModel): + return _strip_payload_metadata(value, value.model_dump(mode="json")) + if isinstance(value, (list, tuple, set, frozenset)): + return [_normalize_default(item) for item in value] + if isinstance(value, dict): + return {key: _normalize_default(item) for key, item in value.items()} + return value + + +def _strip_payload_metadata(original: object, encoded: Any) -> Any: # noqa: C901 + """Remove declaration fields after JSON encoding, including set-like containers.""" + if isinstance(original, RootModel): + return _strip_payload_metadata(cast(Any, original).root, encoded) + if isinstance(original, Source) and isinstance(encoded, dict): + encoded = {key: value for key, value in encoded.items() if key not in DECLARATION_OWNED_FIELDS} + if isinstance(original, BaseModel) and isinstance(encoded, dict): + for name in type(original).model_fields: + if name in encoded: + encoded[name] = _strip_payload_metadata(getattr(original, name), encoded[name]) + elif isinstance(original, (list, tuple, set, frozenset)) and isinstance(encoded, list): + for index, item in enumerate(original): + if index < len(encoded): + encoded[index] = _strip_payload_metadata(item, encoded[index]) + elif isinstance(original, dict) and isinstance(encoded, dict): + for key, item in original.items(): + if key in encoded: + encoded[key] = _strip_payload_metadata(item, encoded[key]) + return encoded + + def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> SourceDefinitionManifest: """Build the immutable declaration transported by a remote worker.""" @@ -326,7 +360,7 @@ def project_source_for_transport( # Definition-owned metadata is not part of it. Carrying it here would make a # generated observation fail its own schema, and would change the payload of an # unchanged Source, so replaying a pre-upgrade observation would conflict. - payload = _json_object(source.model_dump(mode="json", exclude=_payload_exclusions(source))) + payload = _json_object(_strip_payload_metadata(source, source.model_dump(mode="json"))) projections = tuple( SourceProjectionValue(key=key, value=registry.project(source, key)) for key in registry.projection_keys(source) ) From c58aa7f7ce5543eeb540be8c2b9f993f5446f4e5 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Wed, 14 Oct 2026 21:56:19 +0800 Subject: [PATCH 25/30] fix(sources): follow serialized dictionary keys and dataclass aliases --- src/powercontext/sources/observations.py | 12 ++++++--- tests/test_source_observations.py | 31 +++++++++++++++++++++++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 4ae6c86417..db53b28c3c 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -274,16 +274,22 @@ def _strip_payload_metadata(original: object, encoded: Any) -> Any: # noqa: C90 if key in encoded: encoded[key] = _strip_payload_metadata(getattr(original, name), encoded[key]) elif is_dataclass(original) and not isinstance(original, type) and isinstance(encoded, dict): + declared_fields = getattr(type(original), "__pydantic_fields__", {}) + config = getattr(type(original), "__pydantic_config__", {}) for field in fields(original): - if field.name in encoded: - encoded[field.name] = _strip_payload_metadata(getattr(original, field.name), encoded[field.name]) + declaration = declared_fields.get(field.name) + key = field.name + if declaration is not None and config.get("serialize_by_alias"): + key = declaration.serialization_alias or declaration.alias or field.name + if key in encoded: + encoded[key] = _strip_payload_metadata(getattr(original, field.name), encoded[key]) elif isinstance(original, (list, tuple, set, frozenset)) and isinstance(encoded, list): for index, item in enumerate(original): if index < len(encoded): encoded[index] = _strip_payload_metadata(item, encoded[index]) elif isinstance(original, dict) and isinstance(encoded, dict): for key, item in original.items(): - encoded_key = str(key) if not isinstance(key, str) else key + encoded_key = next(iter(TypeAdapter(dict[Any, Any]).dump_python({key: None}, mode="json"))) if encoded_key in encoded: encoded[encoded_key] = _strip_payload_metadata(item, encoded[encoded_key]) return encoded diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 47c599df37..f4850a65de 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -447,7 +447,20 @@ def reject_request(*_args: object, **_kwargs: object) -> None: ) -@pytest.mark.parametrize("shape", ["dataclass", "pydantic_dataclass", "integer_keys", "alias", "set", "frozenset"]) +@pytest.mark.parametrize( + "shape", + [ + "dataclass", + "pydantic_dataclass", + "aliased_dataclass", + "integer_keys", + "boolean_keys", + "enum_keys", + "alias", + "set", + "frozenset", + ], +) def test_serialized_nested_sources_satisfy_manifest(shape: str) -> None: from pydantic.dataclasses import dataclass as pydantic_dataclass @@ -467,10 +480,26 @@ class AliasedWrapper(BaseModel): model_config = ConfigDict(serialize_by_alias=True) child: Child = Field(alias="captured") + from enum import Enum + + class KeyEnum(Enum): + PRIMARY = "primary" + + @pydantic_dataclass(config=ConfigDict(serialize_by_alias=True)) + class AliasedDataclass: + child: Child = Field(alias="item") + + from pydantic.dataclasses import rebuild_dataclass + + rebuild_dataclass(AliasedDataclass, _types_namespace={"Child": Child}) + annotation, value = { "dataclass": (Wrapper, Wrapper), "pydantic_dataclass": (PydanticWrapper, PydanticWrapper), "integer_keys": (dict[int, Child], lambda child: {1: child}), + "boolean_keys": (dict[bool, Child], lambda child: {True: child}), + "enum_keys": (dict[KeyEnum, Child], lambda child: {KeyEnum.PRIMARY: child}), + "aliased_dataclass": (AliasedDataclass, lambda child: AliasedDataclass(item=child)), "alias": (AliasedWrapper, lambda child: AliasedWrapper(captured=child)), "set": (set[Child], lambda child: {child}), "frozenset": (frozenset[Child], lambda child: frozenset({child})), From 9963fd5ad1f64b1599255be761ab2399fa883cf8 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Wed, 14 Oct 2026 21:57:39 +0800 Subject: [PATCH 26/30] chore(desktop): regenerate operations after master merge --- desktop/src-tauri/src/transport/operations.json | 2 +- desktop/ui/src/generated/operations.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/desktop/src-tauri/src/transport/operations.json b/desktop/src-tauri/src/transport/operations.json index 1b09bd3e32..522aad93e2 100644 --- a/desktop/src-tauri/src/transport/operations.json +++ b/desktop/src-tauri/src/transport/operations.json @@ -1,5 +1,5 @@ { - "contractSha256": "26d1140d01210da4ddf0fb2d99981925ec2f3200b609643a67e2bdbfe3e4afd8", + "contractSha256": "271f71c25fb9b808180f38695f9729f4cfa6124758d1eab7ecdc9b2dc98afdf0", "operations": { "get_liveness": { "method": "GET", diff --git a/desktop/ui/src/generated/operations.ts b/desktop/ui/src/generated/operations.ts index a15cfa663b..f59b75cc2d 100644 --- a/desktop/ui/src/generated/operations.ts +++ b/desktop/ui/src/generated/operations.ts @@ -15,7 +15,7 @@ */ // Generated from openapi/powercontext.yaml. Do not edit. -export const contractSha256 = "26d1140d01210da4ddf0fb2d99981925ec2f3200b609643a67e2bdbfe3e4afd8"; +export const contractSha256 = "271f71c25fb9b808180f38695f9729f4cfa6124758d1eab7ecdc9b2dc98afdf0"; export const operations = { "get_liveness": { "method": "GET", From 60d5b39653b5661411aee47ab9b1c031fe21fd77 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Wed, 14 Oct 2026 23:14:27 +0800 Subject: [PATCH 27/30] test(zcode): stabilize pause recovery boundary --- .../plugins/powercontext/tests/boundary.test.mjs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/integrations/zcode/plugins/powercontext/tests/boundary.test.mjs b/integrations/zcode/plugins/powercontext/tests/boundary.test.mjs index 6828fa4261..25e45eae7b 100644 --- a/integrations/zcode/plugins/powercontext/tests/boundary.test.mjs +++ b/integrations/zcode/plugins/powercontext/tests/boundary.test.mjs @@ -164,8 +164,16 @@ test('pause publication survives process interruption and legacy partial pauses assert.equal(flushes, 0, 'No request may precede complete pause publication') await new Promise(resolve => setTimeout(resolve, 5100)) const first = cases.shift() - assert.equal((await flushBoundary(first.input, settings, first.keys, async () => ({ scopeId: 'scope-crash' }), - AbortSignal.timeout(800), first.data)).state, 'cursor_reached') + let firstResult = await flushBoundary(first.input, settings, first.keys, async () => ({ scopeId: 'scope-crash' }), + AbortSignal.timeout(800), first.data) + // A Windows child can finish just across the five-second lease boundary; let + // the expired lease become observable before asserting the actual flush result. + if (firstResult.state === 'skipped') { + await new Promise(resolve => setTimeout(resolve, 150)) + firstResult = await flushBoundary(first.input, settings, first.keys, async () => ({ scopeId: 'scope-crash' }), + AbortSignal.timeout(800), first.data) + } + assert.equal(firstResult.state, 'cursor_reached') assert.equal(flushes, 1) for (const [index, owner] of cases.entries()) { const pause = join(owner.data, 'pending', owner.pauseName) From bd40d9802a398d746164f467197d52b933268fba Mon Sep 17 00:00:00 2001 From: hidb4ai Date: Mon, 19 Oct 2026 11:24:43 +0800 Subject: [PATCH 28/30] refactor(sources): remove legacy entry evidence extensions Keep immutable evidence declarations in Source payloads and use existing Atomic Memory lineage. Remove unused legacy entry snapshot and lifecycle types, the evidence table, and its maintenance and fixture additions. Preserve the existing legacy schemas and migration behavior. Restore the Atomic Memory persistence, search, rebuild, and scope isolation tests. --- docs/en/docs/operate/troubleshoot.md | 2 +- docs/zh/docs/operate/troubleshoot.md | 2 +- .../builtin/artifacts/memory/__init__.py | 2 - .../builtin/artifacts/memory/models.py | 17 +- .../builtin/persistence/tables.py | 46 +---- tests/builtin/persistence/test_memory.py | 189 ++++++++++++++++++ tests/e2e/real_experience_skill/harness.py | 1 - .../test_atomic_memory_migration_evidence.py | 7 - .../test_atomic_memory_migration_vector.py | 7 +- .../test_atomic_memory_projection_rebuild.py | 2 - 10 files changed, 195 insertions(+), 80 deletions(-) create mode 100644 tests/builtin/persistence/test_memory.py diff --git a/docs/en/docs/operate/troubleshoot.md b/docs/en/docs/operate/troubleshoot.md index 7978dc5948..ccf4a9e91d 100644 --- a/docs/en/docs/operate/troubleshoot.md +++ b/docs/en/docs/operate/troubleshoot.md @@ -284,7 +284,7 @@ so the previous database remains available for recovery: ```bash obloader -D --csv \ - --table 'pc_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_evidence,pc_memory_entry_heads,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ + --table 'pc_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_heads,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ -f ``` diff --git a/docs/zh/docs/operate/troubleshoot.md b/docs/zh/docs/operate/troubleshoot.md index 871a0c72dd..57ce9cdb29 100644 --- a/docs/zh/docs/operate/troubleshoot.md +++ b/docs/zh/docs/operate/troubleshoot.md @@ -275,7 +275,7 @@ collation，但不会包含数据库 URL 或凭据。 ```bash obloader -D --csv \ - --table 'pc_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_evidence,pc_memory_entry_heads,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ + --table 'pc_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_heads,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ -f ``` diff --git a/src/powercontext/builtin/artifacts/memory/__init__.py b/src/powercontext/builtin/artifacts/memory/__init__.py index 9af3dcda56..db5b9aa3ff 100644 --- a/src/powercontext/builtin/artifacts/memory/__init__.py +++ b/src/powercontext/builtin/artifacts/memory/__init__.py @@ -51,7 +51,6 @@ MemoryEntryInput, MemoryEntryState, MemoryEntryVersion, - MemoryEvidenceSnapshot, MemoryHit, MemoryMatchedBy, MemoryQueryEmbedding, @@ -125,7 +124,6 @@ "MemoryEntryState", "MemoryEntryVersion", "MemoryEvidenceProjector", - "MemoryEvidenceSnapshot", "MemoryExtractionCandidate", "MemoryExtractionCurrentEntry", "MemoryExtractionEvidence", diff --git a/src/powercontext/builtin/artifacts/memory/models.py b/src/powercontext/builtin/artifacts/memory/models.py index a5d62ffc86..323b529b21 100644 --- a/src/powercontext/builtin/artifacts/memory/models.py +++ b/src/powercontext/builtin/artifacts/memory/models.py @@ -24,7 +24,7 @@ from powercontext.artifacts import Artifact, ArtifactRef from powercontext.builtin.artifacts.search import AdmissionCounts from powercontext.builtin.inference.models import InferenceUsage -from powercontext.sources import MemoryEvidenceDeclaration, Source, SourceRef +from powercontext.sources import Source, SourceRef MemoryEntryState: TypeAlias = Literal["active", "inactive"] MemoryChangeOp: TypeAlias = Literal["add", "revise", "deactivate", "reactivate", "compact"] @@ -32,20 +32,6 @@ MemorySearchMode: TypeAlias = Literal["fts", "vector", "hybrid", "auto"] MemoryUsedSearchMode: TypeAlias = Literal["fts", "vector", "hybrid"] MemoryMatchedBy: TypeAlias = Literal["fts", "vector"] -MemoryLifecycleValidity: TypeAlias = Literal["current", "inactive", "superseded", "unresolved_conflict"] -MemoryTimeState: TypeAlias = Literal["unknown"] - - -class MemoryEvidenceSnapshot(BaseModel): - """The definition-owned evidence declaration attached to an entry version. - - The snapshot is intentionally independent of the entry content hash. It is - immutable provenance metadata that can be replayed into lifecycle - projections without changing the authoritative Memory body or citation. - """ - - source: SourceRef - declaration: MemoryEvidenceDeclaration class EmbeddingProfile(BaseModel): @@ -152,7 +138,6 @@ class MemoryEntryVersion(BaseModel): entry_content_hash: str created_in_revision: int sources: tuple[SourceRef, ...] = () - source_evidence: tuple[MemoryEvidenceSnapshot, ...] = () artifacts: tuple[ArtifactRef, ...] = () diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index 5b1af2bfbd..5578edb9d6 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -1056,45 +1056,6 @@ def _entry_text_type(): ), ) -MEMORY_ENTRY_EVIDENCE_TABLE = Table( - "pc_memory_entry_evidence", - SHARED_METADATA, - Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), - Column("memory_artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), - Column("entry_id", identity_string(MAX_MEMORY_ENTRY_ID_LENGTH), primary_key=True), - Column("entry_version_id", identity_string(MAX_MEMORY_ENTRY_ID_LENGTH), primary_key=True), - Column("ordinal", Integer, primary_key=True), - Column("source_type", identity_string(MAX_SOURCE_TYPE_LENGTH), nullable=False), - Column("source_id", identity_string(MAX_SOURCE_ID_LENGTH), nullable=False), - Column("authority", identity_string(32), nullable=False), - Column("verification", identity_string(32), nullable=False), - Column("declaration_version", identity_string(MAX_SOURCE_TYPE_LENGTH), nullable=False), - ForeignKeyConstraint( - ("scope_id", "memory_artifact_id", "entry_id", "entry_version_id"), - ( - "pc_memory_entry_versions.scope_id", - "pc_memory_entry_versions.memory_artifact_id", - "pc_memory_entry_versions.entry_id", - "pc_memory_entry_versions.entry_version_id", - ), - ondelete="CASCADE", - ), - ForeignKeyConstraint( - ("scope_id", "source_type", "source_id"), - ("pc_sources.scope_id", "pc_sources.source_type", "pc_sources.source_id"), - ondelete="RESTRICT", - ), - CheckConstraint("ordinal >= 0", name="ck_pc_memory_entry_evidence_ordinal_nonnegative"), - CheckConstraint( - "authority IN ('untrusted', 'user_asserted', 'repository_attested', 'system_attested')", - name="ck_pc_memory_entry_evidence_authority", - ), - CheckConstraint( - "verification IN ('unknown', 'verified', 'not_verified')", - name="ck_pc_memory_entry_evidence_verification", - ), -) - MEMORY_ENTRY_HEADS_TABLE = Table( "pc_memory_entry_heads", SHARED_METADATA, @@ -1119,11 +1080,8 @@ def _entry_text_type(): CheckConstraint("head_revision > 0", name="ck_pc_memory_entry_heads_revision_positive"), ) -MEMORY_TABLES = ( - MEMORY_ENTRY_VERSIONS_TABLE, - MEMORY_ENTRY_EVIDENCE_TABLE, - MEMORY_ENTRY_HEADS_TABLE, -) + +MEMORY_TABLES = (MEMORY_ENTRY_VERSIONS_TABLE, MEMORY_ENTRY_HEADS_TABLE) TOPIC_MEMORY_TABLES = ( TOPIC_MEMORY_RETRIEVAL_SHAPE_TABLE, diff --git a/tests/builtin/persistence/test_memory.py b/tests/builtin/persistence/test_memory.py new file mode 100644 index 0000000000..b2817817c9 --- /dev/null +++ b/tests/builtin/persistence/test_memory.py @@ -0,0 +1,189 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio + +from sqlalchemy import BigInteger, DateTime, Integer, String +from sqlalchemy.dialects import mysql +from sqlalchemy.schema import CreateTable, ForeignKeyConstraint, PrimaryKeyConstraint, UniqueConstraint + +from powercontext.artifacts import ArtifactLineage +from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.tables import ( + MEMORY_ENTRY_HEADS_TABLE, + MEMORY_ENTRY_VERSIONS_TABLE, +) +from powercontext.builtin.records import ArtifactWrite +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts +from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection +from powercontext.builtin.sources import ContentCapture, ContentSource +from powercontext.sources import SourceRef + +_INNODB_MAX_INDEX_BYTES = 3072 + + +class _UnbudgetedColumnTypeError(TypeError): + def __init__(self, column_type: object) -> None: + super().__init__(f"unbudgeted indexed column type: {column_type!r}") + + +def _key_budget(constraint: PrimaryKeyConstraint | UniqueConstraint | ForeignKeyConstraint) -> int: + total = 0 + for column in constraint.columns: + if isinstance(column.type, String): + assert column.type.length is not None + total += column.type.length * 4 + elif isinstance(column.type, BigInteger | Integer | DateTime): + total += 8 + else: + raise _UnbudgetedColumnTypeError(column.type) + return total + + +def test_memory_schema_is_mysql_compilable_and_respects_key_and_payload_limits() -> None: + dialect = mysql.dialect() + versions = str(CreateTable(MEMORY_ENTRY_VERSIONS_TABLE).compile(dialect=dialect)) + heads = str(CreateTable(MEMORY_ENTRY_HEADS_TABLE).compile(dialect=dialect)) + + assert "scope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL" in versions + assert "text MEDIUMTEXT NOT NULL" in versions + assert "source_refs MEDIUMBLOB NOT NULL" in versions + assert "artifact_refs MEDIUMBLOB NOT NULL" in versions + assert "searchable_text MEDIUMTEXT NOT NULL" in heads + + budgets = [ + _key_budget(constraint) + for table in (MEMORY_ENTRY_VERSIONS_TABLE, MEMORY_ENTRY_HEADS_TABLE) + for constraint in table.constraints + if isinstance(constraint, PrimaryKeyConstraint | UniqueConstraint | ForeignKeyConstraint) + ] + assert budgets + assert max(budgets) == 2560 + assert all(budget < _INNODB_MAX_INDEX_BYTES for budget in budgets) + + +def test_sqlite_memory_backend_commits_authoritative_history_and_fts() -> None: + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + context = await contexts.get("project") + source, _ = await context.sources.capture( + ContentCapture( + source_id="turn-1", + content="PowerContext owns the atomic composition boundary.", + ) + ) + application = contexts.atomic_memory + memory = application.for_scope("project") + async with contexts.database.transaction() as connection: + plan = await application.service.inspect_change( + connection, + "project", + "decision", + AtomicMemoryContent(kind="decision", text="Use one atomic composition boundary."), + None, + lineage=ArtifactLineage(sources=(SourceRef(source_type="content", source_id=source.name),)), + ) + prepared = await application.service.prepare_change(plan) + async with contexts.database.transaction() as connection: + first = (await application.service.commit(connection, prepared, None)).primary + second = await contexts.records.replace_artifact( + "project", + "atomic-memory", + first.ref.artifact_id, + '"revision:1"', + ArtifactWrite( + content={"kind": "decision", "text": "Use one atomic composition boundary for providers."} + ), + ) + constraint = await contexts.records.create_artifact( + "project", + "atomic-memory", + ArtifactWrite(content={"kind": "constraint", "text": "Do not split the provider transaction."}), + ) + assert second.revision == 2 + history = await contexts.records.list_artifact_revisions( + "project", + "atomic-memory", + first.ref.artifact_id, + limit=10, + cursor=None, + ) + assert tuple(item.revision for item in history.items) == (2, 1) + assert (await memory.get(first.ref.artifact_id, revision=1)).artifact == first.artifact + assert first.artifact.lineage.sources == (SourceRef(source_type="content", source_id=source.name),) + assert {item.artifact.content.text for item in (await memory.list()).items} == { + second.content["text"], + "Do not split the provider transaction.", + } + assert constraint.artifact_id != first.ref.artifact_id + result = await memory.search("atomic composition", mode="text") + assert result.mode == "text" + assert tuple(hit.text for hit in result.hits) == (second.content["text"],) + assert result.hits[0].hit.artifact_ref == (await memory.get(first.ref.artifact_id)).ref + unrelated = await memory.search("Should we use blue icons in the mobile navigation bar?", mode="text") + assert unrelated.hits == () + + asyncio.run(scenario()) + + +def test_sqlite_memory_backend_rebuilds_head_and_fts_projections() -> None: + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + await contexts.get("project") + created = await contexts.records.create_artifact( + "project", + "atomic-memory", + ArtifactWrite( + content={"kind": "decision", "text": "Rebuild search projections from authoritative revisions."} + ), + ) + memory = contexts.atomic_memory.for_scope("project") + authoritative = await memory.get(created.artifact_id) + async with contexts.database.transaction() as connection: + await connection.execute(contexts.atomic_memory.index.table.delete()) + assert (await memory.search("authoritative revisions", mode="text")).hits == () + report = await rebuild_atomic_memory_projection( + contexts.database, + contexts.atomic_memory.index, + maintenance_confirmed=True, + ) + assert report.ready + assert await memory.get(created.artifact_id) == authoritative + rebuilt = await memory.search("authoritative revisions", mode="text") + assert tuple(hit.text for hit in rebuilt.hits) == ( + "Rebuild search projections from authoritative revisions.", + ) + + asyncio.run(scenario()) + + +def test_scope_bound_contexts_do_not_share_rows() -> None: + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + left = await contexts.get("left") + right = await contexts.get("right") + + await left.sources.capture(ContentCapture(source_id="same", content="left")) + await right.sources.capture(ContentCapture(source_id="same", content="right")) + + left_sources = await left.sources.list() + right_sources = await right.sources.list() + assert all(isinstance(source, ContentSource) for source in (*left_sources, *right_sources)) + assert tuple(source.content for source in left_sources if isinstance(source, ContentSource)) == ("left",) + assert tuple(source.content for source in right_sources if isinstance(source, ContentSource)) == ("right",) + + asyncio.run(scenario()) diff --git a/tests/e2e/real_experience_skill/harness.py b/tests/e2e/real_experience_skill/harness.py index 6efefe9af3..c4aaae3e91 100644 --- a/tests/e2e/real_experience_skill/harness.py +++ b/tests/e2e/real_experience_skill/harness.py @@ -105,7 +105,6 @@ "pc_access_relationships", "pc_memory_vector_entries", "pc_memory_entry_heads", - "pc_memory_entry_evidence", "pc_memory_entry_versions", "pc_skill_publications", "pc_candidate_heads", diff --git a/tests/e2e/test_atomic_memory_migration_evidence.py b/tests/e2e/test_atomic_memory_migration_evidence.py index 050633541f..337b40c02a 100644 --- a/tests/e2e/test_atomic_memory_migration_evidence.py +++ b/tests/e2e/test_atomic_memory_migration_evidence.py @@ -82,7 +82,6 @@ CANDIDATE_HEADS_TABLE, CANDIDATE_VERSIONS_TABLE, DREAM_RUNS_TABLE, - MEMORY_ENTRY_EVIDENCE_TABLE, MEMORY_ENTRY_HEADS_TABLE, MEMORY_ENTRY_VERSIONS_TABLE, SCOPES_TABLE, @@ -1249,7 +1248,6 @@ def test_runtime_start_does_not_read_the_archive_or_legacy_entry_tables(migrated async def scenario() -> None: async with migration_profile(config) as profile, profile.database.transaction() as connection: await connection.execute(text(f"DROP TABLE {ARCHIVE_TABLE.name}")) - await connection.execute(text(f"DROP TABLE {MEMORY_ENTRY_EVIDENCE_TABLE.name}")) await connection.execute(text(f"DROP TABLE {MEMORY_ENTRY_HEADS_TABLE.name}")) await connection.execute(text(f"DROP TABLE {MEMORY_ENTRY_VERSIONS_TABLE.name}")) async with open_builtin_contexts(BuiltinConfig(database=config)) as contexts: @@ -1290,7 +1288,6 @@ async def scenario() -> None: await _seed_and_migrate(config, pinned) async with migration_profile(config) as profile, profile.database.transaction() as connection: await connection.execute(text(f"DROP TABLE {ARCHIVE_TABLE.name}")) - await connection.execute(text(f"DROP TABLE {MEMORY_ENTRY_EVIDENCE_TABLE.name}")) await connection.execute(text(f"DROP TABLE {MEMORY_ENTRY_HEADS_TABLE.name}")) await connection.execute(text(f"DROP TABLE {MEMORY_ENTRY_VERSIONS_TABLE.name}")) async with ( @@ -1394,10 +1391,6 @@ async def legacy_schema(contexts, connection) -> None: # Restore the historical foreign keys with frozen SQL rows. Current table # metadata intentionally no longer attaches these tables to public Artifacts. versions = [dict(row) for row in (await connection.execute(select(MEMORY_ENTRY_VERSIONS_TABLE))).mappings()] - # OceanBase/SeekDB enforce the evidence-to-version FK while dropping the - # legacy parent tables; remove that child table before rebuilding the - # historical schema. - await connection.execute(text("DROP TABLE pc_memory_entry_evidence")) await connection.execute(text("DROP TABLE pc_memory_entry_heads")) await connection.execute(text("DROP TABLE pc_memory_entry_versions")) metadata = MetaData() diff --git a/tests/e2e/test_atomic_memory_migration_vector.py b/tests/e2e/test_atomic_memory_migration_vector.py index de8285fda1..4f5fa1b29a 100644 --- a/tests/e2e/test_atomic_memory_migration_vector.py +++ b/tests/e2e/test_atomic_memory_migration_vector.py @@ -60,12 +60,7 @@ async def scenario() -> None: historical = await memory.get(_atomic("alpha", 1).artifact_id, revision=1) async with migration_profile(config, load_vector_extension=True) as profile: async with profile.database.transaction() as connection: - for table in ( - "pc_memory_artifact_archive", - "pc_memory_entry_evidence", - "pc_memory_entry_heads", - "pc_memory_entry_versions", - ): + for table in ("pc_memory_artifact_archive", "pc_memory_entry_heads", "pc_memory_entry_versions"): await connection.execute(text(f"DROP TABLE {table}")) await connection.execute(text("DELETE FROM pc_atomic_memory_current")) rebuilt = await rebuild_atomic_memory_projection( diff --git a/tests/e2e/test_atomic_memory_projection_rebuild.py b/tests/e2e/test_atomic_memory_projection_rebuild.py index 590ea71e20..3075699659 100644 --- a/tests/e2e/test_atomic_memory_projection_rebuild.py +++ b/tests/e2e/test_atomic_memory_projection_rebuild.py @@ -48,7 +48,6 @@ async def scenario() -> None: if retention in {"entries_only", "removed"}: await connection.execute(text("DROP TABLE pc_memory_artifact_archive")) if retention in {"archive_only", "removed"}: - await connection.execute(text("DROP TABLE pc_memory_entry_evidence")) await connection.execute(text("DROP TABLE pc_memory_entry_heads")) await connection.execute(text("DROP TABLE pc_memory_entry_versions")) if retention == "unreadable": @@ -127,7 +126,6 @@ async def scenario() -> None: async with SQLiteProfile.open(config, tables=()) as profile: async with profile.database.transaction() as connection: await connection.execute(text("DROP TABLE pc_memory_artifact_archive")) - await connection.execute(text("DROP TABLE pc_memory_entry_evidence")) await connection.execute(text("DROP TABLE pc_memory_entry_heads")) await connection.execute(text("DROP TABLE pc_memory_entry_versions")) identity = {"scope": SCOPE, "id": _atomic("alpha", 2).artifact_id} From 2cd004f85712ff55add3f995e0bb2bcee71c5d7d Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Thu, 22 Oct 2026 12:51:37 +0800 Subject: [PATCH 29/30] fix(http): preserve legacy source manifest responses --- src/powercontext/server/mapping.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/powercontext/server/mapping.py b/src/powercontext/server/mapping.py index 77f09c2eb7..49ae864940 100644 --- a/src/powercontext/server/mapping.py +++ b/src/powercontext/server/mapping.py @@ -528,7 +528,10 @@ def runtime_source_definition_manifest(value: SourceDefinitionManifest) -> Runti def source_definition_manifest_response(value: RuntimeSourceDefinitionManifest) -> SourceDefinitionManifest: - return SourceDefinitionManifest.model_validate(value.model_dump(mode="json", by_alias=True)) + payload = value.model_dump(mode="json", by_alias=True) + # Keep the pre-declaration wire shape for neutral manifests. Older workers + # reject the new field, and null is not valid for the generated schema. + return SourceDefinitionManifest.model_validate(_without_absent_evidence(payload)) def runtime_connector_binding(value: HttpConnectorBinding) -> RuntimeConnectorBinding: From 5d765ebd7551e7fa85018aa3e39e273403347c06 Mon Sep 17 00:00:00 2001 From: jiannnnyyyyy <2529204828@qq.com> Date: Thu, 22 Oct 2026 15:08:40 +0800 Subject: [PATCH 30/30] test: tolerate backend artifact timestamp precision --- tests/e2e/test_source_declaration_contract.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/e2e/test_source_declaration_contract.py b/tests/e2e/test_source_declaration_contract.py index 3c378f0138..5e23db3654 100644 --- a/tests/e2e/test_source_declaration_contract.py +++ b/tests/e2e/test_source_declaration_contract.py @@ -258,7 +258,17 @@ async def scenario() -> None: async with open_builtin_contexts(config) as contexts: context = await contexts.get(scope.scope_id) - assert await contexts.records.get_artifact(scope.scope_id, "profile", "profile") == restored + reread = await contexts.records.get_artifact(scope.scope_id, "profile", "profile") + assert reread.scope_id == restored.scope_id + assert reread.family == restored.family + assert reread.artifact_id == restored.artifact_id + assert reread.revision == restored.revision + assert reread.content == restored.content + assert reread.sources == restored.sources + assert reread.content_digest == restored.content_digest + assert reread.created_by == restored.created_by + assert reread.restored_from_revision == restored.restored_from_revision + assert reread.reason == restored.reason sources = {source.name: source for source in await context.sources.list()} assert all(sources[ref.source_id].memory_evidence == declaration for ref in restored.sources)