diff --git a/docs/en/docs/operate/portable-archive.md b/docs/en/docs/operate/portable-archive.md index d53274f328..e80626aaa7 100644 --- a/docs/en/docs/operate/portable-archive.md +++ b/docs/en/docs/operate/portable-archive.md @@ -131,6 +131,7 @@ Format version 1 carries the portable relational representation of these support | --- | --- | | Scope identity, hierarchy, context/external references, and creation identity | Scope access bindings and host-local default selection | | Source journal heads, Source records, and referenced remote Source Definition manifests | Search projections and indexes | +| Accepted Memory evidence: per-Source declaration bindings, the Definition manifests they reference, and exact-revision evidence snapshots | Today's registry declarations, which never replace a saved snapshot | | Artifact Revisions, their timestamps/actors/rollback metadata, stable handles, lineage, cross-Scope publication provenance, and heads | Source cursors and scheduler state | | Atomic Memory merge relationships and marked merge-input lineage | External Skill registrations and host-local installation state | | Candidate versions, decision heads, and their evidence references | Audit events, usage facts, evaluation receipts, and restore receipts | diff --git a/docs/zh/docs/operate/portable-archive.md b/docs/zh/docs/operate/portable-archive.md index 77eeecf320..5bf0cd05d7 100644 --- a/docs/zh/docs/operate/portable-archive.md +++ b/docs/zh/docs/operate/portable-archive.md @@ -111,6 +111,7 @@ powercontext archive restore ./backups/payments.pcb --yes | --- | --- | | Scope identity、层级、context/external reference 和创建 identity | Scope access binding 和 host-local default selection | | Source journal head、Source record 和引用的远程 Source Definition manifest | 搜索 projection 和 index | +| 已接受的 Memory evidence：逐 Source 的 declaration binding、其引用的 Definition manifest 和精确 revision 的 evidence snapshot | 当前 registry 的 declaration，它不会覆盖已保存的快照 | | Artifact Revision 及其时间戳、actor、回滚元数据；稳定句柄、lineage、跨 Scope 发布来源和 head | Source cursor 和 scheduler state | | Atomic Memory 合并关系和带合并输入标记的 lineage | External Skill registration 和 host-local installation state | | Candidate version、decision head 及其 evidence reference | Audit event、usage fact、evaluation receipt 和 restore receipt | diff --git a/integrations/opendal/src/powercontext_connector_opendal/source.py b/integrations/opendal/src/powercontext_connector_opendal/source.py index 5ab48cf7d6..c089367cba 100644 --- a/integrations/opendal/src/powercontext_connector_opendal/source.py +++ b/integrations/opendal/src/powercontext_connector_opendal/source.py @@ -23,6 +23,7 @@ from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, + MemoryEvidenceDeclaration, Source, SourceMaterialization, SourceProjection, @@ -92,6 +93,7 @@ class TextFileSnapshotSourceDefinition: name = TEXT_FILE_SNAPSHOT_SOURCE_NAME version = "1" source_class = TextFileSnapshotSource + memory_evidence: MemoryEvidenceDeclaration | None = None def __init__(self) -> None: projection = cast(SourceProjection[TextFileSnapshotSource], TextFileEvidenceProjection()) diff --git a/src/powercontext/__init__.py b/src/powercontext/__init__.py index 7448a04133..f7589beabc 100644 --- a/src/powercontext/__init__.py +++ b/src/powercontext/__init__.py @@ -51,12 +51,14 @@ ) from powercontext.sources import ( AdapterSourceDefinition, + MemoryEvidenceDeclaration, Source, SourceAdapter, SourceCatalog, SourceCatalogBackend, SourceDefinition, SourceDefinitionRegistry, + SourceEvidence, SourceMaterialization, SourceProjection, SourceProjectionKey, @@ -90,6 +92,7 @@ "InvalidSourceProjectionError", "InvalidSourceReferenceError", "InvalidSourceResultError", + "MemoryEvidenceDeclaration", "PolicyTransition", "PowerContext", "PowerContextError", @@ -104,6 +107,7 @@ "SourceDefinitionNotFoundError", "SourceDefinitionRegistry", "SourceError", + "SourceEvidence", "SourceMaterialization", "SourceNotFoundError", "SourceProjection", diff --git a/src/powercontext/builtin/artifacts/atomic_memory/service.py b/src/powercontext/builtin/artifacts/atomic_memory/service.py index 5c6f1c177a..40bcf70f3e 100644 --- a/src/powercontext/builtin/artifacts/atomic_memory/service.py +++ b/src/powercontext/builtin/artifacts/atomic_memory/service.py @@ -21,6 +21,7 @@ from typing import TYPE_CHECKING, Any, Protocol, cast from sqlalchemy.ext.asyncio import AsyncConnection +from typing_extensions import override from powercontext.artifacts import ArtifactLineage, ArtifactRef from powercontext.builtin.artifacts.atomic_memory.errors import ( @@ -53,6 +54,7 @@ from powercontext.builtin.artifacts.merge import ArtifactMergeSecurity, ArtifactMergeService, MergeTags from powercontext.builtin.artifacts.merge_models import ( ArtifactMergeErrors, + ArtifactMergeMutationResult, ArtifactMergePlan, ArtifactMergeRead, ArtifactMergeRecord, @@ -94,6 +96,9 @@ async def publish_many( records: Sequence[tuple[AtomicMemoryRecord, PreparedAtomicMemoryProjection]], ) -> None: ... async def remove(self, connection: AsyncConnection, scope_id: str, artifact_id: str) -> None: ... + async def capture_evidence( + self, connection: AsyncConnection, scope_id: str, record: AtomicMemoryRecord + ) -> None: ... ATOMIC_MERGE_ERRORS = ArtifactMergeErrors( @@ -300,6 +305,33 @@ async def remove(self, connection: AsyncConnection, scope_id: str, artifact_id: await self.projections.remove(connection, scope_id, artifact_id) +class _AtomicMemoryMergeService(ArtifactMergeService): + """Accept evidence with every new body, including retired restoration results. + + Shared merges, the Atomic facade and processing all use this commit seam. + State-only writes must not attest old bodies. Capture joins the caller's + transaction so its failure rolls back authority, governance and projections. + """ + + @override + async def _commit_locked( + self, + connection: AsyncConnection, + prepared: Sequence[tuple[PreparedArtifactMerge, SourceRef | None]], + current: dict[str, ArtifactMergeRecord], + context: Any, + source_read_set: _SourceReadSet | None, + ) -> tuple[ArtifactMergeMutationResult, ...]: + results = await super()._commit_locked(connection, prepared, current, context, source_read_set) + projections = cast(AtomicMemoryMergeAdapter, self.adapter).projections + for (item, _), result in zip(prepared, results, strict=True): + revisions = {write.artifact_id for write in item.plan.writes if write.draft is not None} + for record in result.records: + if record.ref.artifact_id in revisions: + await projections.capture_evidence(connection, item.plan.scope_id, _atomic_record(record)) + return results + + class _AtomicPreviewAdapter: def __init__(self, signer): self.signer = signer @@ -326,7 +358,7 @@ def __init__( ) -> None: self.artifacts, self.states, self.security = artifacts, states, security self.projections, self.merge_tags, self.preview_signer = projections, merge_tags, preview_signer - self.shared = ArtifactMergeService( + self.shared = _AtomicMemoryMergeService( artifacts=artifacts, adapter=AtomicMemoryMergeAdapter(projections), security=security, diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/atomic_memory_evidence.py new file mode 100644 index 0000000000..99d5c38311 --- /dev/null +++ b/src/powercontext/builtin/persistence/atomic_memory_evidence.py @@ -0,0 +1,211 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Immutable per-revision evidence inputs and their neutral internal annotation.""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from typing import Literal + +import rfc8785 +from aiosqlite import Connection as SQLiteConnection +from pydantic import BaseModel, ConfigDict, ValidationError +from sqlalchemy import insert, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.artifacts import ArtifactRef +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE +from powercontext.builtin.persistence.sources import SourceRepository, _require_identity +from powercontext.limits import MAX_SCOPE_ID_LENGTH +from powercontext.sources import SourceRef +from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence + + +class AtomicMemoryEvidenceAnnotation(BaseModel): + """Exact revision facts, not a trust score or recursively inferred evidence.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + scope_id: str + artifact_ref: ArtifactRef + sources: tuple[SourceEvidence, ...] + format: Literal["powercontext.atomic-memory.evidence.v1"] = "powercontext.atomic-memory.evidence.v1" + + +class AtomicMemoryEvidenceRepository: + """Join caller transactions; retained inputs are never deleted with the index. + + The annotation is derived from retained inputs at read time. There is no + separate cache to confuse with accepted history or join into search. + """ + + async def save( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + sources: Sequence[SourceEvidence], + ) -> AtomicMemoryEvidenceAnnotation: + artifact_ref = self._reference(scope_id, artifact_ref) + payload = await self._payload(connection, scope_id, artifact_ref) + if payload is not None: + return self._decode(scope_id, artifact_ref, payload) or self._neutral(scope_id, artifact_ref, sources) + accepted = [] + for source in sources: + value = source.model_dump(mode="json") + try: + declaration = MemoryEvidenceDeclaration.model_validate(value.get("declaration")) + except ValidationError: + declaration = MemoryEvidenceDeclaration() + accepted.append(SourceEvidence.model_validate({**value, "declaration": declaration})) + annotation = AtomicMemoryEvidenceAnnotation( + scope_id=scope_id, artifact_ref=artifact_ref, sources=tuple(accepted) + ) + if connection.dialect.name == "sqlite": + driver = (await connection.get_raw_connection()).driver_connection + if isinstance(driver, SQLiteConnection) and not driver.in_transaction: + # SQLite legacy transaction mode does not BEGIN on SELECT; + # SAVEPOINT must not accidentally own and commit the write. + await connection.exec_driver_sql("BEGIN") + try: + async with connection.begin_nested(): + await connection.execute( + insert(ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE).values( + scope_id=scope_id, + artifact_id=artifact_ref.artifact_id, + revision=artifact_ref.revision, + sources=rfc8785.dumps([source.model_dump(mode="json") for source in annotation.sources]), + ) + ) + except IntegrityError: + payload = await self._payload(connection, scope_id, artifact_ref) + if payload is None: + raise + return self._decode(scope_id, artifact_ref, payload) or self._neutral(scope_id, artifact_ref, sources) + return annotation + + async def read( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef + ) -> AtomicMemoryEvidenceAnnotation | None: + artifact_ref = self._reference(scope_id, artifact_ref) + payload = await self._payload(connection, scope_id, artifact_ref) + return None if payload is None else self._decode(scope_id, artifact_ref, payload) + + async def verify( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + source_refs: Sequence[SourceRef], + *, + sources: SourceRepository | None = None, + ) -> AtomicMemoryEvidenceAnnotation: + """Derive exact facts from retained inputs, never today's declarations. + + Missing or damaged declaration records are neutral. Missing Sources, + corrupt authoritative payloads and unequal exact materials remain errors. + No accepted snapshot is saved or repaired by this operation. + """ + + from powercontext.builtin.persistence.atomic_memory_index import AtomicMemoryIndexError + from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY + + artifact_ref = self._reference(scope_id, artifact_ref) + repository = SourceRepository(BUILTIN_SOURCE_REGISTRY) if sources is None else sources + materials = await repository.historical_evidence(connection, scope_id, source_refs) + annotation = await self.read(connection, scope_id, artifact_ref) + if annotation is not None: + by_ref = {(item.source_ref.source_type, item.source_ref.source_id): item for item in materials} + for accepted in annotation.sources: + material = by_ref.get((accepted.source_ref.source_type, accepted.source_ref.source_id)) + if material is not None and accepted.material_digest != material.material_digest: + raise AtomicMemoryIndexError( + "evidence-material", "Historical evidence does not match the exact Source material" + ) + if annotation is None or tuple(item.source_ref for item in annotation.sources) != tuple( + item.source_ref for item in materials + ): + return self._neutral(scope_id, artifact_ref, materials) + verified = tuple( + accepted + if ( + accepted.materialization == material.materialization + and accepted.definition_name == material.definition_name + and accepted.definition_version == material.definition_version + and accepted.manifest_fingerprint == material.manifest_fingerprint + and accepted.declaration == material.declaration + ) + else accepted.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) + for accepted, material in zip(annotation.sources, materials, strict=True) + ) + return annotation.model_copy(update={"sources": verified}) + + @staticmethod + def _reference(scope_id: str, artifact_ref: ArtifactRef) -> ArtifactRef: + _require_identity("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) + reference = ArtifactRef.model_validate(artifact_ref.model_dump()) + if reference.family != "atomic-memory": + from powercontext.builtin.persistence.errors import InvalidRepositoryArgumentError + + raise InvalidRepositoryArgumentError("artifact_ref", "must reference an Atomic Memory revision") + return reference + + @staticmethod + async def _payload(connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef) -> object | None: + table = ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE + return await connection.scalar( + select(table.c.sources).where( + table.c.scope_id == scope_id, + table.c.artifact_id == artifact_ref.artifact_id, + table.c.revision == artifact_ref.revision, + ) + ) + + @staticmethod + def _decode(scope_id: str, artifact_ref: ArtifactRef, payload: object) -> AtomicMemoryEvidenceAnnotation | None: + try: + if not isinstance(payload, (bytes, bytearray, memoryview)): + return None + values = json.loads(bytes(payload)) + if not isinstance(values, list): + return None + sources = [] + for value in values: + if not isinstance(value, dict): + return None + try: + declaration = MemoryEvidenceDeclaration.model_validate(value.get("declaration")) + except ValidationError: + declaration = MemoryEvidenceDeclaration() + sources.append(SourceEvidence.model_validate({**value, "declaration": declaration})) + return AtomicMemoryEvidenceAnnotation(scope_id=scope_id, artifact_ref=artifact_ref, sources=tuple(sources)) + except (ValueError, TypeError): + # An unreadable declaration snapshot conveys no historical claim. + # This does not catch SQL errors or actual Source resolution errors. + return None + + @staticmethod + def _neutral( + scope_id: str, artifact_ref: ArtifactRef, sources: Sequence[SourceEvidence] + ) -> AtomicMemoryEvidenceAnnotation: + # Preserve a damaged row exactly, without replacing it with today's + # stronger configuration. A caller can only receive neutral facts. + return AtomicMemoryEvidenceAnnotation( + scope_id=scope_id, + artifact_ref=artifact_ref, + sources=tuple(source.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) for source in sources), + ) diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py b/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py new file mode 100644 index 0000000000..22846565c1 --- /dev/null +++ b/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Retained exact-version evidence, independent of disposable search projections.""" + +from sqlalchemy import BigInteger, CheckConstraint, Column, LargeBinary, Table, Text +from sqlalchemy.dialects.mysql import LONGBLOB, MEDIUMTEXT + +from powercontext.builtin.persistence.tables import SHARED_METADATA, identity_string +from powercontext.limits import ( + MAX_ARTIFACT_ID_LENGTH, + MAX_SCOPE_ID_LENGTH, + MAX_SOURCE_ID_LENGTH, + MAX_SOURCE_TYPE_LENGTH, +) + +ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION = "p0005" +ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE = Table( + "pc_atomic_memory_evidence_schema", + SHARED_METADATA, + Column("schema_version", identity_string(32), primary_key=True), +) +ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE = Table( + "pc_atomic_memory_evidence_history", + SHARED_METADATA, + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), + Column("revision", BigInteger, primary_key=True), + Column("sources", LargeBinary().with_variant(LONGBLOB(), "mysql"), nullable=False), + CheckConstraint("revision > 0", name="ck_pc_atomic_memory_evidence_history_revision"), +) +SOURCE_EVIDENCE_BINDINGS_TABLE = Table( + "pc_source_evidence_bindings", + SHARED_METADATA, + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("source_type", identity_string(MAX_SOURCE_TYPE_LENGTH), primary_key=True), + Column("source_id", identity_string(MAX_SOURCE_ID_LENGTH), primary_key=True), + Column("material_digest", identity_string(71), nullable=False), + Column("definition_name", Text().with_variant(MEDIUMTEXT(), "mysql"), nullable=False), + Column("definition_version", Text().with_variant(MEDIUMTEXT(), "mysql"), nullable=False), + Column("manifest_fingerprint", identity_string(71)), + Column("declaration", LargeBinary().with_variant(LONGBLOB(), "mysql"), nullable=False), +) +ATOMIC_MEMORY_EVIDENCE_TABLES = ( + ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE, + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + SOURCE_EVIDENCE_BINDINGS_TABLE, +) diff --git a/src/powercontext/builtin/persistence/atomic_memory_index.py b/src/powercontext/builtin/persistence/atomic_memory_index.py index effe642f6c..8ce323ea19 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/atomic_memory_index.py @@ -43,9 +43,16 @@ lexical_search_score, ) from powercontext.builtin.inference import EmbeddingModel +from powercontext.builtin.persistence.atomic_memory_evidence import ( + AtomicMemoryEvidenceAnnotation, + AtomicMemoryEvidenceRepository, +) +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_PROJECTION_FORMAT +from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.tables import ARTIFACT_TAGS_TABLE from powercontext.builtin.tags import TagFilter +from powercontext.sources.evidence import SourceEvidence AtomicMemorySearchMode = Literal["fts", "vector", "hybrid"] ATOMIC_MEMORY_RRF_CONSTANT = 60 @@ -189,6 +196,19 @@ async def replace_many( async def delete(self, connection: AsyncConnection, scope_id: str, artifact_id: str, /) -> None: ... + async def save_evidence( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + sources: Sequence[SourceEvidence], + /, + ) -> AtomicMemoryEvidenceAnnotation: ... + + async def read_evidence( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / + ) -> AtomicMemoryEvidenceAnnotation | None: ... + async def search( self, connection: AsyncConnection, scope_id: str, request: AtomicMemorySearchRequest, / ) -> AtomicMemorySearchChannels: ... @@ -214,11 +234,13 @@ def __init__( index: AtomicMemoryIndex, *, embedding_model: EmbeddingModel | None = None, + sources: SourceRepository | None = None, ) -> None: if not callable(getattr(index, "replace_many", None)): raise AtomicMemoryIndexError("publication", "Atomic Memory index must publish complete projection sets") self.index = index self._embedding_model = embedding_model + self._sources = sources @property def profile_fingerprint(self) -> str | None: @@ -268,6 +290,17 @@ async def prepare(self, content: BaseModel) -> PreparedAtomicMemoryProjection: embedding_input_hash=None if vector is None else atomic_memory_embedding_input_hash(kind, body), ) + async def capture_evidence(self, connection: AsyncConnection, scope_id: str, record: Any) -> None: + """Accept history only for a committed body write, never state-only republishing.""" + + artifact = record.artifact + if ( + self._sources is not None + and await self.index.read_evidence(connection, scope_id, artifact.as_ref()) is None + ): + evidence = await self._sources.evidence(connection, scope_id, artifact.lineage.sources) + await self.index.save_evidence(connection, scope_id, artifact.as_ref(), evidence) + async def publish( self, connection: AsyncConnection, @@ -561,12 +594,28 @@ def __init__(self, table: Table, profile: EmbeddingProfile | None) -> None: if profile is not None and (profile.dimension < 1 or profile.distance != "l2"): raise AtomicMemoryIndexError("embedding-profile", "Atomic Memory requires a positive L2 embedding profile") self.table = table - self.tables: tuple[Table, ...] = (table,) + self.tables: tuple[Table, ...] = (table, *ATOMIC_MEMORY_EVIDENCE_TABLES) + self._evidence = AtomicMemoryEvidenceRepository() self.profile = profile self.capabilities = AtomicMemoryIndexCapabilities( vector=profile is not None, hybrid=profile is not None, embedding_profile=profile ) + async def save_evidence( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + sources: Sequence[SourceEvidence], + /, + ) -> AtomicMemoryEvidenceAnnotation: + return await self._evidence.save(connection, scope_id, artifact_ref, sources) + + async def read_evidence( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / + ) -> AtomicMemoryEvidenceAnnotation | None: + return await self._evidence.read(connection, scope_id, artifact_ref) + async def validate_current(self, connection: AsyncConnection, /) -> None: """Audit vector deployment readiness at startup, outside ordinary bounded search.""" profile = self.capabilities.embedding_profile diff --git a/src/powercontext/builtin/persistence/atomic_memory_readiness.py b/src/powercontext/builtin/persistence/atomic_memory_readiness.py index ff12a47d4b..5dfc3356c5 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_readiness.py +++ b/src/powercontext/builtin/persistence/atomic_memory_readiness.py @@ -25,7 +25,12 @@ from sqlalchemy import inspect, text from sqlalchemy.ext.asyncio import AsyncConnection +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES from powercontext.builtin.persistence.errors import PersistenceError +from powercontext.builtin.persistence.migrations.atomic_memory_evidence import ( + AtomicMemoryEvidenceSchemaError, + require_atomic_memory_evidence_schema, +) LEGACY_CITATION_COLUMN = "memory_citations" _LEGACY_CITATION_TABLES = ("pc_artifacts", "pc_artifact_candidate_versions", "pc_candidate_versions") @@ -90,11 +95,16 @@ async def legacy_foreign_keys(connection: AsyncConnection, tables: set[str]) -> async def atomic_memory_readiness_issues( connection: AsyncConnection, tables: set[str] | None = None, *, collections_removed: bool = True ) -> list[str]: - """Check public migration completion, allowing collections only during offline pre-removal acceptance.""" + """Check versioned schema and public migration completion without reading retained history.""" if tables is None: tables = set(await connection.run_sync(lambda value: inspect(value).get_table_names())) issues: list[str] = [] + if "pc_artifacts" in tables or any(table.name in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): + try: + await require_atomic_memory_evidence_schema(connection, tables) + except AtomicMemoryEvidenceSchemaError as error: + issues.append(str(error)) if ( collections_removed and "pc_artifacts" in tables diff --git a/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json b/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json index 9d6c8f65cc..f7159e2595 100644 --- a/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json +++ b/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json @@ -101,12 +101,30 @@ "pc_atomic_memory_current_fts_idx", "CREATE TABLE 'pc_atomic_memory_current_fts_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID" ], + [ + "table", + "pc_atomic_memory_evidence_history", + "pc_atomic_memory_evidence_history", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)" + ], + [ + "table", + "pc_atomic_memory_evidence_schema", + "pc_atomic_memory_evidence_schema", + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (schema_version)\n)" + ], [ "table", "pc_memory_artifact_archive", "pc_memory_artifact_archive", "CREATE TABLE pc_memory_artifact_archive (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent BLOB NOT NULL, \n\tmemory_citations BLOB, \n\tmetadata BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)" ], + [ + "table", + "pc_source_evidence_bindings", + "pc_source_evidence_bindings", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tmaterial_digest VARCHAR(71) NOT NULL, \n\tdefinition_name TEXT NOT NULL, \n\tdefinition_version TEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71), \n\tdeclaration BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)" + ], [ "trigger", "pc_atomic_memory_fts_delete", diff --git a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py new file mode 100644 index 0000000000..d23f81fd2e --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py @@ -0,0 +1,130 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Apply the frozen additive p0005 revision to runtime-owned evidence tables. + +The production bundle covers a historical subset of business tables. Runtime +profiles therefore apply this same frozen additive revision independently of +that bundle's whole-database inventory, without stamping unrelated schema or +backfilling declarations for old Source/Artifact rows. SQLite DDL joins the +caller's transaction; MySQL DDL may commit and is resumed by exact postconditions. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from aiosqlite import Connection as SQLiteConnection +from sqlalchemy import inspect, select +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ( + ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE, + ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION, + ATOMIC_MEMORY_EVIDENCE_TABLES, +) +from powercontext.builtin.persistence.errors import PersistenceError + + +class AtomicMemoryEvidenceSchemaError(PersistenceError): + """An evidence schema is not the registered versioned layout.""" + + def __init__(self, detail: str) -> None: + message = { + "missing": "Atomic Memory evidence schema is incomplete; apply the registered p0005 migration", + "version": "Atomic Memory evidence schema version is unsupported", + "shape": "Atomic Memory evidence table has an incompatible shape", + }[detail] + super().__init__(message) + + +async def apply_atomic_memory_evidence_migration(connection: AsyncConnection) -> None: + """Add only missing p0005 tables, validate exact shapes, then mark ready. + + No reads or writes of Source/Artifact history are needed, and no current + declaration is used to invent accepted historical inputs. + """ + + if connection.dialect.name == "sqlite": + driver = (await connection.get_raw_connection()).driver_connection + if isinstance(driver, SQLiteConnection) and not driver.in_transaction: + await connection.exec_driver_sql("BEGIN IMMEDIATE") + dialect = "sqlite" if connection.dialect.name == "sqlite" else "mysql" + resources = Path(__file__).parent / "resources/schemas/atomic_memory_evidence_v1.json" + statements = json.loads(resources.read_text())[dialect] + tables = set(await connection.run_sync(lambda sync: inspect(sync).get_table_names())) + for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): + if table.name in tables: + await _require_shape(connection, table.name, statement, dialect) + marker = ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE + versions = ( + tuple((await connection.execute(select(marker.c.schema_version))).scalars()) if marker.name in tables else () + ) + if versions and versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): + raise AtomicMemoryEvidenceSchemaError("version") + if versions and any(table.name not in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): + # A ready marker cannot authorize recreating missing retained history. + raise AtomicMemoryEvidenceSchemaError("missing") + for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): + if table.name not in tables: + await connection.exec_driver_sql(statement) + await _require_shape(connection, table.name, statement, dialect) + if not versions: + await connection.exec_driver_sql(statements[-1]) + + +async def require_atomic_memory_evidence_schema(connection: AsyncConnection, tables: set[str] | None = None) -> None: + """Check retained evidence readiness without DDL or historical payload reads.""" + + if tables is None: + tables = set(await connection.run_sync(lambda sync: inspect(sync).get_table_names())) + if any(table.name not in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): + raise AtomicMemoryEvidenceSchemaError("missing") + dialect = "sqlite" if connection.dialect.name == "sqlite" else "mysql" + resources = Path(__file__).parent / "resources/schemas/atomic_memory_evidence_v1.json" + statements = json.loads(resources.read_text())[dialect] + for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): + await _require_shape(connection, table.name, statement, dialect) + versions = tuple((await connection.execute(select(ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE.c.schema_version))).scalars()) + if versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): + raise AtomicMemoryEvidenceSchemaError("version") + + +async def _require_shape(connection: AsyncConnection, name: str, statement: str, dialect: str) -> None: + if dialect == "sqlite": + actual = await connection.exec_driver_sql( + "SELECT type, name, sql FROM sqlite_master WHERE (name = ? OR tbl_name = ?) " + "AND substr(name, 1, 7) != 'sqlite_' ORDER BY type, name", + (name, name), + ) + if actual.all() != [("table", name, statement)]: + raise AtomicMemoryEvidenceSchemaError("shape") + return + # Reuse the frozen MySQL inventory parser so physical formatting accepted + # by the shared backend does not become a second schema convention. + from powercontext.builtin.persistence.migrations.models import MigrationError + from powercontext.builtin.persistence.migrations.mysql_schema import _matches, _parse_table + + actual = (await connection.exec_driver_sql(f"SHOW CREATE TABLE `{name}`")).one()[1] + try: + actual_name, actual_table = _parse_table(actual) + expected_name, expected_table = _parse_table(statement) + except MigrationError as error: + raise AtomicMemoryEvidenceSchemaError("shape") from error + if not _matches( + {"tables": {actual_name: actual_table}, "views": [], "triggers": []}, + {"tables": {expected_name: expected_table}, "views": [], "triggers": []}, + ): + raise AtomicMemoryEvidenceSchemaError("shape") diff --git a/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0005_atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0005_atomic_memory_evidence.py new file mode 100644 index 0000000000..2550f917ed --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0005_atomic_memory_evidence.py @@ -0,0 +1,25 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create the frozen additive evidence layout without backfilling trust.""" + +from alembic import context, op + +revision = "p0005" +down_revision = "p0004" + + +def upgrade() -> None: + context.config.attributes["execute_revision"](revision) + op.execute("INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')") diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py index 95c5792974..a3112f44ac 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql_schema.py +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -14,7 +14,7 @@ """Recognize the frozen MySQL layouts without importing application metadata. -Only the reviewed linear p0001--p0004 history is supported. DDL recovery accepts +Only the reviewed linear p0001--p0005 history is supported. DDL recovery accepts an exact prefix of the next revision, never an arbitrary collection of objects. The caller owns maintenance evidence, backup, the pinned database lock and the version row; this module never writes or stamps a revision. @@ -37,7 +37,7 @@ _LEXEME = re.compile( r"\s+|`(?:``|[^`])*`|'(?:''|\\.|[^'\\])*'|[A-Za-z_][A-Za-z_0-9$]*|[0-9]+|>=|<=|<>|!=|[(),.=<>+*/%\^&|~!?:@-]" ) -_REVISIONS = ("p0001", "p0002", "p0003", "p0004") +_REVISIONS = ("p0001", "p0002", "p0003", "p0004", "p0005") _CITATION_TABLES = ("pc_artifacts", "pc_artifact_candidate_versions") _FAMILY_CHECK = "ck_pc_artifact_tags_family" _CONTROL_TABLE = "pc_schema_revision" @@ -157,11 +157,15 @@ def __init__(self, bundle: MigrationBundle) -> None: self.bundle = bundle baseline = json.loads((bundle.snapshot_directory / "schemas/pre_dream.json").read_text())["mysql"] current = json.loads((bundle.snapshot_directory / "schemas/v1_1_0.json").read_text())["mysql"] + evidence = json.loads((bundle.snapshot_directory / "schemas/atomic_memory_evidence_v1.json").read_text())[ + "mysql" + ] self._statements = { "p0001": tuple(baseline), "p0002": tuple(f"ALTER TABLE {table} ADD COLUMN memory_citations MEDIUMBLOB" for table in _CITATION_TABLES), "p0003": (f"ALTER TABLE pc_artifact_tags DROP CHECK {_FAMILY_CHECK}",), "p0004": (_ARTIFACT_HANDLES_DDL,), + "p0005": tuple(evidence[:-1]), } inventory = _empty_inventory() self._prefixes: dict[str, list[dict[str, Any]]] = {} diff --git a/src/powercontext/builtin/persistence/migrations/resources/manifest.json b/src/powercontext/builtin/persistence/migrations/resources/manifest.json index 16364daa4f..dc2b373564 100644 --- a/src/powercontext/builtin/persistence/migrations/resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/resources/manifest.json @@ -5,12 +5,16 @@ ], "p0002": [], "p0003": [], - "p0004": [] + "p0004": [], + "p0005": [ + "schemas/atomic_memory_evidence_v1.json" + ] }, "sqlite_baselines": { "e1cae089a8a565291a7a1f782cea1f6c6e1a6cdd9eab75ef0c4ca61dfda453cb": "p0001", "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d": "p0003", - "8cb2d41e53cde751c8d0ef4023c344ec5ec78d9e579366c1c93d17de30aa5ea4": "p0004" + "8cb2d41e53cde751c8d0ef4023c344ec5ec78d9e579366c1c93d17de30aa5ea4": "p0004", + "5d2a458f6cb278e4fbff5aa7cef9429bcb940be5fb0a29649be50c30482e00e1": "p0005" }, "sqlite_fingerprints": { "p0001": [ @@ -26,6 +30,10 @@ "p0004": [ "f65244be963a382a62565118ff874cd775ab1ea33cf51a49881b40391e47c051", "8cb2d41e53cde751c8d0ef4023c344ec5ec78d9e579366c1c93d17de30aa5ea4" + ], + "p0005": [ + "10b8f62310f90e4e267e795efef0aa6e6716c1ecad515ab745e783507f6496f3", + "5d2a458f6cb278e4fbff5aa7cef9429bcb940be5fb0a29649be50c30482e00e1" ] }, "affected_objects": [ @@ -33,7 +41,10 @@ "pc_artifact_heads", "pc_artifact_candidate_versions", "pc_artifact_tags", - "pc_artifact_handles" + "pc_artifact_handles", + "pc_atomic_memory_evidence_schema", + "pc_atomic_memory_evidence_history", + "pc_source_evidence_bindings" ], "retained_objects": [ "pc_retained_p0002_artifact_tags" diff --git a/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json b/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json new file mode 100644 index 0000000000..e4af1dc164 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json @@ -0,0 +1,14 @@ +{ + "sqlite": [ + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tmaterial_digest VARCHAR(71) NOT NULL, \n\tdefinition_name TEXT NOT NULL, \n\tdefinition_version TEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71), \n\tdeclaration BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", + "INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')" + ], + "mysql": [ + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_type VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tmaterial_digest VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tdefinition_name MEDIUMTEXT NOT NULL, \n\tdefinition_version MEDIUMTEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tdeclaration LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", + "INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')" + ] +} diff --git a/src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_memory_evidence.py new file mode 100644 index 0000000000..9190834ba0 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_memory_evidence.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Frozen additive evidence revision; never synthesize historical declarations.""" + +import json +from pathlib import Path + +from alembic import op + +revision = "p0005" +down_revision = "p0004" + + +def upgrade() -> None: + statements = json.loads((Path(__file__).parents[1] / "schemas/atomic_memory_evidence_v1.json").read_text()) + connection = op.get_bind() + for statement in statements["sqlite" if connection.dialect.name == "sqlite" else "mysql"]: + connection.exec_driver_sql(statement) diff --git a/src/powercontext/builtin/persistence/schema.py b/src/powercontext/builtin/persistence/schema.py index 57c0f97112..1059c85678 100644 --- a/src/powercontext/builtin/persistence/schema.py +++ b/src/powercontext/builtin/persistence/schema.py @@ -21,12 +21,20 @@ from sqlalchemy import MetaData, Table, text from sqlalchemy.ext.asyncio import AsyncConnection +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES +from powercontext.builtin.persistence.migrations.atomic_memory_evidence import apply_atomic_memory_evidence_migration + async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) -> None: - """Create missing tables and indexes using SQLAlchemy's native metadata support.""" + """Create selected tables, applying retained evidence through its frozen revision.""" groups: dict[MetaData, list[Table]] = {} + evidence_names = {table.name for table in ATOMIC_MEMORY_EVIDENCE_TABLES} + if any(table.name in evidence_names or table.name == "pc_sources" for table in tables): + await apply_atomic_memory_evidence_migration(connection) for table in tables: + if table.name in evidence_names: + continue groups.setdefault(table.metadata, []).append(table) for metadata, selected in groups.items(): diff --git a/src/powercontext/builtin/persistence/source_definitions.py b/src/powercontext/builtin/persistence/source_definitions.py index 9e5d4a8640..beffb3f87f 100644 --- a/src/powercontext/builtin/persistence/source_definitions.py +++ b/src/powercontext/builtin/persistence/source_definitions.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Persistence for worker-owned Source Definition manifests.""" +"""Immutable historical manifests owned by the Source registration boundary.""" from __future__ import annotations @@ -26,15 +26,18 @@ from powercontext.builtin.persistence.codec import dump_model, load_model, stored_bytes from powercontext.builtin.persistence.errors import ( IdentityMismatchError, + InvalidStoredColumnError, + InvalidStoredPayloadError, RepositoryNotFoundError, StoredPayloadConflictError, ) from powercontext.builtin.persistence.tables import SOURCE_DEFINITION_MANIFESTS_TABLE -from powercontext.sources import SourceDefinitionManifest +from powercontext.sources import SourceDefinition, SourceDefinitionManifest +from powercontext.sources.observations import _manifest_for_definition class SourceDefinitionManifestRepository: - """Register immutable worker-owned Definition manifests by name and version.""" + """Retain native and worker manifests without changing their identities.""" async def register( self, @@ -59,7 +62,9 @@ async def register( ) except IntegrityError: existing = await self.find(connection, manifest.name, manifest.version) - if existing is None or existing != manifest: + if existing is None: + raise + if existing != manifest: raise StoredPayloadConflictError( "source-definition-manifest", (manifest.name, manifest.version), @@ -67,6 +72,29 @@ async def register( return existing return manifest + async def register_native( + self, + connection: AsyncConnection, + definition: SourceDefinition[Any, Any, Any], + /, + ) -> SourceDefinitionManifest | None: + """Retain the existing manifest representation for trusted native material. + + This internal registration cannot replace an older manifest. A missing, + invalid or conflicting ancillary manifest leaves the Source declaration + neutral; database failures still abort the caller's transaction. External + registration keeps its existing active-Definition ownership guard. + """ + + try: + manifest = _manifest_for_definition(definition, transport=False) + except Exception: # Native schema providers may not expose a valid manifest. + return None + try: + return await self.register(connection, manifest) + except (IdentityMismatchError, InvalidStoredColumnError, InvalidStoredPayloadError, StoredPayloadConflictError): + return None + async def get( self, connection: AsyncConnection, diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 9b57917df0..5da26bfdca 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -19,10 +19,16 @@ from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence from contextlib import suppress from dataclasses import dataclass +from hashlib import sha256 from typing import Any, Literal, cast import rfc8785 -from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError +from jsonschema.exceptions import ValidationError as JsonSchemaValidationError +from pydantic import AliasChoices, AliasGenerator, BaseModel, ConfigDict, JsonValue, TypeAdapter +from referencing import Registry +from referencing.exceptions import Unresolvable from sqlalchemy import func, insert, select, tuple_, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection, AsyncTransaction @@ -30,6 +36,7 @@ from powercontext.builtin.artifacts.experience import EXPERIENCE_INCUBATION_CURSOR_NAME from powercontext.builtin.artifacts.profile.models import PROFILE_SOURCE_WINDOW_BINDING from powercontext.builtin.artifacts.topic_memory import TOPIC_MEMORY_SOURCE_WINDOW_BINDING +from powercontext.builtin.persistence.atomic_memory_evidence_schema import SOURCE_EVIDENCE_BINDINGS_TABLE from powercontext.builtin.persistence.codec import load_model, stored_bytes from powercontext.builtin.persistence.errors import ( IdentityMismatchError, @@ -40,12 +47,22 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository +from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository from powercontext.builtin.persistence.tables import SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE -from powercontext.builtin.sources.content import ContentSource +from powercontext.builtin.sources.content import CONTENT_SOURCE_NAME, ContentSource from powercontext.builtin.triggers import SOURCE_WINDOW_TRIGGER_NAME from powercontext.errors import SourceDefinitionNotFoundError from powercontext.limits import MAX_SCOPE_ID_LENGTH -from powercontext.sources import Source, SourceAdapter, SourceDefinitionRegistry, SourceObservation, SourceRef +from powercontext.sources import ( + MemoryEvidenceDeclaration, + Source, + SourceAdapter, + SourceDefinitionManifest, + SourceDefinitionRegistry, + SourceEvidence, + SourceObservation, + SourceRef, +) _AnySourceAdapter = SourceAdapter[Any, Any, Any] _JSON_OBJECT = TypeAdapter(dict[str, JsonValue]) @@ -64,6 +81,9 @@ class _StoredSourcePayload(BaseModel): encoding: Literal["powercontext-source-v1"] representation: Literal["native", "observation"] value: dict[str, JsonValue] + # Server-owned header permits historical reads without adapter code. Never hash it. + source: Source | None = None + source_keys: dict[str, str] | None = None class StoredSource(BaseModel): @@ -149,6 +169,7 @@ async def add_many_with_status( existing = await self._existing_many(connection, scope_id, tuple(ref for _, ref, _ in prepared)) results: list[tuple[StoredSource, bool]] = [] new_rows = [] + new_sources: dict[tuple[str, str], tuple[SourceRef, Source]] = {} upgrades: dict[tuple[str, str], bytes] = {} for source, ref, payload in prepared: key = (ref.source_type, ref.source_id) @@ -166,11 +187,14 @@ async def add_many_with_status( upgrades[key] = payload elif stored.value != source: raise StoredPayloadConflictError("source", (scope_id, ref)) + else: + await self._retain_native_header(connection, scope_id, ref, source) results.append((stored, False)) continue position += 1 stored = StoredSource(ref=ref, value=source, journal_position=position) existing[key] = stored + new_sources[key] = (ref, source) new_rows.append({ "scope_id": scope_id, "source_type": ref.source_type, @@ -187,6 +211,8 @@ async def add_many_with_status( .values(position=position) ) await connection.execute(insert(SOURCES_TABLE), new_rows) + for ref, source in new_sources.values(): + await self._bind_memory_evidence(connection, scope_id, ref, source) for (source_type, source_id), payload in upgrades.items(): await connection.execute( update(SOURCES_TABLE) @@ -303,6 +329,425 @@ async def preload( connection, connection.get_transaction(), connection.get_nested_transaction(), scope_id, rows ) + async def evidence( + self, + connection: AsyncConnection, + scope_id: str, + refs: Sequence[SourceRef], + /, + ) -> tuple[SourceEvidence, ...]: + """Read accepted declarations against exact stored Source material. + + Source reads retain their existing errors and never become generation + eligibility checks. Missing or invalid declaration bindings are neutral; + database failures propagate. No current registry declaration or provider + value is substituted for historical registration facts. + """ + + return await self.historical_evidence(connection, scope_id, refs) + + async def historical_evidence( + self, connection: AsyncConnection, scope_id: str, refs: Sequence[SourceRef], / + ) -> tuple[SourceEvidence, ...]: + """Check retained material without loading today's native Definition. + + Only the stored representation selects decoding. Canonical JSON retains + all native fields, rather than round-tripping through a newer model. + This does not supply a decoder or provider value for ordinary reads. + """ + + _require_identity("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) + identities = tuple(dict.fromkeys((ref.source_type, ref.source_id) for ref in refs)) + if not identities: + return () + rows = ( + ( + await connection.execute( + select(SOURCES_TABLE).where( + SOURCES_TABLE.c.scope_id == scope_id, + tuple_(SOURCES_TABLE.c.source_type, SOURCES_TABLE.c.source_id).in_(identities), + ) + ) + ) + .mappings() + .all() + ) + by_ref = {(row["source_type"], row["source_id"]): row for row in rows} + materials: list[SourceEvidence] = [] + fingerprints: dict[tuple[str, str], str] = {} + for ref in identities: + reference = SourceRef(source_type=ref[0], source_id=ref[1]) + row = by_ref.get(ref) + if row is None: + raise RepositoryNotFoundError("source", (scope_id, reference)) + source, material, verifiable = await self._historical_material(connection, reference, row) + if not verifiable: + fingerprints[ref] = "" # Never authorize a retained declaration for this capture. + if isinstance(source, SourceObservation): + fingerprints[ref] = source.definition_fingerprint + elif ref[0] == CONTENT_SOURCE_NAME: + # The server-owned Content witness is the sole permitted material + # mutation; this exclusion is the same one used at acceptance. + internal = material.get("internal") + if ( + isinstance(internal, dict) + and internal.get("role") == "lineage_only" + and internal.get("operation") in {"artifact_create", "artifact_replace"} + ): + continue + material = {key: value for key, value in material.items() if key != "handoff_receipt"} + materials.append( + SourceEvidence( + source_ref=reference, + materialization=source.materialization, + material_digest=_source_material_digest_material(material), + definition_name=ref[0], + definition_version=source.definition_version, + ) + ) + return await self._bound_evidence(connection, scope_id, tuple(materials), fingerprints) + + async def _historical_material( + self, connection: AsyncConnection, reference: SourceRef, row: Mapping[Any, Any] + ) -> tuple[Source, dict[str, JsonValue], bool]: + """Validate the stored representation and its retained schema, not a live model.""" + + payload = stored_bytes(row["payload"], column="payload") + try: + envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=reference.source_type) + except InvalidStoredPayloadError: + return await self._historical_legacy_material(connection, reference, row, payload) + material = envelope.value + source = ( + await self._historical_canonical_source(connection, reference, row, material) + if envelope.representation == "native" and envelope.source is None + else None + ) + if source is None: + source = self._historical_source(reference, envelope) + decoded = SourceRef( + source_type=source.source_type if isinstance(source, SourceObservation) else reference.source_type, + source_id=source.name, + ) + if reference != decoded: + raise IdentityMismatchError("source", reference, decoded) + table = SOURCE_EVIDENCE_BINDINGS_TABLE + binding = ( + await connection.execute( + select(table.c.manifest_fingerprint, table.c.material_digest).where( + table.c.scope_id == row["scope_id"], + table.c.source_type == reference.source_type, + table.c.source_id == reference.source_id, + table.c.definition_name == reference.source_type, + table.c.definition_version == source.definition_version, + ) + ) + ).one_or_none() + manifest = await _find_evidence_manifest(connection, reference.source_type, source.definition_version) + if manifest is not None and binding is not None and manifest.fingerprint == binding.manifest_fingerprint: + try: + Draft202012Validator.check_schema(manifest.source_schema) + value = source.payload if isinstance(source, SourceObservation) else material + Draft202012Validator(manifest.source_schema, registry=Registry()).validate(value) + except JsonSchemaValidationError as error: + if envelope.representation == "native" and binding.material_digest == _source_material_digest_material( + material + ): + # Older releases bound declarations without checking serialization + # compatibility. Exact accepted bytes remain readable, but neutral. + return source, material, False + raise InvalidStoredPayloadError( + "source", reference.source_type, "payload does not match the historical schema" + ) from error + except (SchemaError, Unresolvable): + self._validate_unbound_material(reference, envelope, source) + else: + self._validate_unbound_material(reference, envelope, source) + return source, material, True + + async def _historical_legacy_material( + self, + connection: AsyncConnection, + reference: SourceRef, + row: Mapping[Any, Any], + payload: bytes, + ) -> tuple[Source, dict[str, JsonValue], bool]: + """Read bare legacy material without guessing absent identity fields.""" + try: + material = _JSON_OBJECT.validate_json(payload, strict=True) + except ValueError as error: + raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error + source = await self._historical_canonical_source(connection, reference, row, material) + if source is not None: + return source, material, False + try: + self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + source = load_model(Source, payload, kind="source", name=reference.source_type) + if not {"name", "definition_version", "materialization"} <= source.model_fields_set: + raise InvalidStoredPayloadError( + "source", reference.source_type, "legacy native identity requires its registered decoder" + ) from None + if source.name != reference.source_id: + raise IdentityMismatchError("source", reference.source_id, source.name) from None + else: + source = self.decode_source(row).value + return source, material, True + + async def _historical_canonical_source( + self, + connection: AsyncConnection, + reference: SourceRef, + row: Mapping[Any, Any], + material: dict[str, JsonValue], + ) -> Source | None: + """Recover evidence identity from canonical retained material, never a live alias.""" + + if not {"name", "definition_version", "materialization"} <= material.keys(): + return None + if material["name"] != reference.source_id: + return None + digest_material = material + if reference.source_type == CONTENT_SOURCE_NAME: + digest_material = {key: value for key, value in material.items() if key != "handoff_receipt"} + material_digest = _source_material_digest_material(digest_material) + table = SOURCE_EVIDENCE_BINDINGS_TABLE + binding = ( + await connection.execute( + select(table.c.definition_name, table.c.definition_version, table.c.material_digest).where( + table.c.scope_id == row["scope_id"], + table.c.source_type == reference.source_type, + table.c.source_id == reference.source_id, + ) + ) + ).one_or_none() + if binding is not None: + if ( + binding.definition_name != reference.source_type + or binding.material_digest != material_digest + or binding.definition_version != material["definition_version"] + ): + return None + version = binding.definition_version + elif isinstance(material["definition_version"], str): + # Canonical retained identity is explicit even when its historical + # evidence binding predates the evidence tables. It stays neutral. + version = material["definition_version"] + else: + return None + try: + definition = self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + definition = None + if ( + definition is not None + and definition.version == version + and any( + ( + ( + definition.source_class.model_fields[key].alias is not None + and definition.source_class.model_fields[key].alias != key + ) + or ( + definition.source_class.model_fields[key].serialization_alias is not None + and definition.source_class.model_fields[key].serialization_alias != key + ) + or ( + definition.source_class.model_fields[key].validation_alias is not None + and definition.source_class.model_fields[key].validation_alias != key + ) + ) + for key in ("definition_version", "materialization") + ) + ): + # The current exact-version codec is required to disambiguate retained identity keys. + return None + try: + return Source.model_validate({ + "name": reference.source_id, + "definition_version": version, + "materialization": material["materialization"], + "description": material.get("description"), + }) + except (TypeError, ValueError): + return None + + def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayload) -> Source: + """Decode identity independently of retained material and today's business defaults.""" + + material = envelope.value + if envelope.representation == "observation": + return load_model(SourceObservation, rfc8785.dumps(material), kind="source", name=reference.source_type) + if envelope.source is not None: + keys = envelope.source_keys or {key: key for key in ("name", "definition_version", "materialization")} + for key, alias in keys.items(): + value = getattr(envelope.source, key) + if alias not in material or material[alias] != value: + raise IdentityMismatchError("source-envelope", value, material.get(alias)) + return envelope.source + # Legacy aliased rows need their registered decoder once; canonical rows do not. + if {"name", "definition_version", "materialization"} <= material.keys() and material[ + "name" + ] == reference.source_id: + definition = None + with suppress(SourceDefinitionNotFoundError): + definition = self._registry.definition_for_name(reference.source_type) + if definition is None or not _claims_identity_keys(definition.source_class): + return load_model(Source, rfc8785.dumps(material), kind="source", name=reference.source_type) + with suppress(SourceDefinitionNotFoundError): + definition = self._registry.definition_for_name(reference.source_type) + if any( + (definition.source_class.model_fields[key].serialization_alias or key) != key + for key in ("name", "definition_version", "materialization") + ): + source, _ = self._decode_envelope(reference.source_type, envelope) + return source + header = material + with suppress(SourceDefinitionNotFoundError): + definition = self._registry.definition_for_name(reference.source_type) + header = _normalize_native_material(definition.source_class, material) + if not {"name", "definition_version", "materialization"} <= header.keys(): + raise InvalidStoredPayloadError( + "source", reference.source_type, "legacy native identity requires its registered decoder" + ) + try: + return load_model(Source, rfc8785.dumps(header), kind="source", name=reference.source_type) + except InvalidStoredPayloadError: + source, _ = self._decode_envelope(reference.source_type, envelope) + return source + + def _validate_unbound_material(self, reference: SourceRef, envelope: _StoredSourcePayload, source: Source) -> None: + """Keep supported exact-read errors when no historical schema is available. + + A live decoder validates only material of its own version; it never + contributes a declaration or reinterprets a valid historical binding. + """ + + if envelope.representation == "observation": + return # SourceObservation already validated its envelope and payload. + try: + definition = self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + return + if _claims_identity_keys(definition.source_class): + # The retained canonical identity/header is authoritative; a live business alias may claim it. + return + if envelope.source is None and any( + (definition.source_class.model_fields[key].serialization_alias or key) != key + for key in ("name", "definition_version", "materialization") + ): + # Today's identity aliases cannot validate an older unbound representation. + return + if definition.version == source.definition_version: + self._decode_envelope(reference.source_type, envelope) + + @staticmethod + async def _bound_evidence( + connection: AsyncConnection, + scope_id: str, + materials: tuple[SourceEvidence, ...], + fingerprints: Mapping[tuple[str, str], str], + ) -> tuple[SourceEvidence, ...]: + if not materials: + return () + table = SOURCE_EVIDENCE_BINDINGS_TABLE + rows = ( + ( + await connection.execute( + select(table).where( + table.c.scope_id == scope_id, + tuple_(table.c.source_type, table.c.source_id).in_( + tuple((value.source_ref.source_type, value.source_ref.source_id) for value in materials) + ), + ) + ) + ) + .mappings() + .all() + ) + bindings = {(row["source_type"], row["source_id"]): row for row in rows} + manifests: dict[tuple[str, str], SourceDefinitionManifest | None] = {} + evidence: list[SourceEvidence] = [] + for value in materials: + identity = (value.source_ref.source_type, value.source_ref.source_id) + binding = bindings.get(identity) + if binding is None or ( + binding["material_digest"] != value.material_digest + or binding["definition_name"] != value.definition_name + or binding["definition_version"] != value.definition_version + or not isinstance(binding["manifest_fingerprint"], str) + ): + evidence.append(value) + continue + definition_identity = (value.definition_name, value.definition_version) + if definition_identity not in manifests: + manifests[definition_identity] = await _find_evidence_manifest(connection, *definition_identity) + manifest = manifests[definition_identity] + if ( + manifest is None + or manifest.fingerprint != binding["manifest_fingerprint"] + or (identity in fingerprints and fingerprints[identity] != manifest.fingerprint) + ): + evidence.append(value) + continue + try: + declaration = MemoryEvidenceDeclaration.model_validate_json( + stored_bytes(binding["declaration"], column="declaration") + ) + except (InvalidStoredColumnError, TypeError, ValueError): + declaration = MemoryEvidenceDeclaration() + evidence.append( + value.model_copy( + update={ + "manifest_fingerprint": manifest.fingerprint, + "declaration": declaration, + } + ) + ) + return tuple(evidence) + + async def _bind_memory_evidence( + self, + connection: AsyncConnection, + scope_id: str, + ref: SourceRef, + source: Source, + ) -> None: + """Freeze registration evidence only for a newly accepted Source row.""" + + if isinstance(source, SourceObservation): + manifest = await _find_evidence_manifest(connection, ref.source_type, source.definition_version) + if manifest is not None and manifest.fingerprint != source.definition_fingerprint: + manifest = None + else: + definition = self._registry.definition_for_source(source) + manifest = await SourceDefinitionManifestRepository().register_native(connection, definition) + if manifest is not None: + # A native decoder can accept representations its generated schema + # cannot describe. Such captures are valid, but cannot attest evidence. + try: + Draft202012Validator.check_schema(manifest.source_schema) + Draft202012Validator(manifest.source_schema, registry=Registry()).validate( + source.model_dump(mode="json", by_alias=True) + ) + except (JsonSchemaValidationError, SchemaError, Unresolvable): + manifest = None + declaration = ( + MemoryEvidenceDeclaration() if manifest is None else self._registry.memory_evidence_for_source(source) + ) + await connection.execute( + insert(SOURCE_EVIDENCE_BINDINGS_TABLE).values( + scope_id=scope_id, + source_type=ref.source_type, + source_id=ref.source_id, + material_digest=_source_material_digest(source), + definition_name=ref.source_type, + definition_version=source.definition_version, + manifest_fingerprint=None if manifest is None else manifest.fingerprint, + declaration=declaration.model_dump_json().encode("utf-8"), + ) + ) + async def list( self, connection: AsyncConnection, @@ -466,6 +911,47 @@ def decode_source(self, row: Mapping[Any, Any], /) -> StoredSource: journal_position=int(row["journal_position"]), ) + async def _retain_native_header( + self, connection: AsyncConnection, scope_id: str, ref: SourceRef, source: Source + ) -> None: + """Enrich only an identical replay's unreconstructible legacy header. + + The retained value, binding, manifest, digest and journal position never change. + This grants no authority and does not publish another processing intent. + """ + + if isinstance(source, SourceObservation): + return + row = await self._find_row(connection, scope_id, ref) + if row is None: + # Same-batch duplicates have no retained row until the bulk insert. + return + payload = stored_bytes(row["payload"], column="payload") + try: + envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=ref.source_type) + except InvalidStoredPayloadError: + envelope = _StoredSourcePayload( + encoding="powercontext-source-v1", + representation="native", + value=_JSON_OBJECT.validate_json(payload, strict=True), + ) + if envelope.source is not None: + return + header = _source_header(source) + with suppress(InvalidStoredPayloadError): + if load_model(Source, rfc8785.dumps(envelope.value), kind="source", name=ref.source_type) == header: + return + enriched = envelope.model_copy(update={"source": header, "source_keys": _source_keys(source, envelope.value)}) + await connection.execute( + update(SOURCES_TABLE) + .where( + SOURCES_TABLE.c.scope_id == scope_id, + SOURCES_TABLE.c.source_type == ref.source_type, + SOURCES_TABLE.c.source_id == ref.source_id, + ) + .values(payload=rfc8785.dumps(enriched.model_dump(mode="json", exclude_none=True))) + ) + def _decode_envelope( self, source_type: str, @@ -484,11 +970,15 @@ def _decode_envelope( definition = self._definition_by_name(source_type) source = load_model( definition.source_class, - payload, + rfc8785.dumps(_normalize_native_material(definition.source_class, envelope.value)), kind="source", name=source_type, ) self._registry.definition_for_source(source) + if envelope.source is not None: + for key in ("name", "definition_version", "materialization"): + if getattr(envelope.source, key) != getattr(source, key): + raise IdentityMismatchError("source-envelope", envelope.source, _source_header(source)) return source, SourceRef(source_type=definition.name, source_id=source.name) def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Source, SourceRef]: @@ -503,9 +993,13 @@ def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Sour ) decoded = SourceRef(source_type=source.source_type, source_id=source.name) else: + try: + material = _JSON_OBJECT.validate_json(payload, strict=True) + except ValueError as error: + raise InvalidStoredPayloadError("source", source_type, "payload is not JSON") from error source = load_model( definition.source_class, - payload, + rfc8785.dumps(_normalize_native_material(definition.source_class, material)), kind="source", name=source_type, ) @@ -514,6 +1008,113 @@ def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Sour return cast(Source, source), decoded +def _source_material_digest(source: Source) -> str: + # Preserve every material field except the narrow server witness upgrade. + material = source.model_dump(mode="json", by_alias=True) + if isinstance(source, ContentSource): + material.pop("handoff_receipt", None) + return f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}" + + +def _management_write_source(source: Source) -> bool: + return ( + isinstance(source, ContentSource) + and source.internal is not None + and source.internal.role == "lineage_only" + and source.internal.operation in {"artifact_create", "artifact_replace"} + ) + + +async def _find_evidence_manifest( + connection: AsyncConnection, name: str, version: str +) -> SourceDefinitionManifest | None: + try: + return await SourceDefinitionManifestRepository().find(connection, name, version) + except (IdentityMismatchError, InvalidStoredColumnError, InvalidStoredPayloadError): + return None + + +def _source_material_digest_material(material: Mapping[str, JsonValue]) -> str: + return f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}" + + +def _source_header(source: Source) -> Source: + return Source(**{key: getattr(source, key) for key in Source.model_fields}) + + +def _source_keys(source: Source, material: dict[str, JsonValue] | None = None) -> dict[str, str]: + """Retain the exact serialized spelling used for each native identity.""" + keys: dict[str, str] = {} + for key in ("name", "definition_version", "materialization"): + field = type(source).model_fields[key] + preferred = field.serialization_alias or key + candidates = [preferred] + if isinstance(field.validation_alias, str): + candidates.append(field.validation_alias) + elif isinstance(field.validation_alias, AliasChoices): + candidates.extend(choice for choice in field.validation_alias.choices if isinstance(choice, str)) + if field.alias is not None: + candidates.append(field.alias) + generator = type(source).model_config.get("alias_generator") + if isinstance(generator, AliasGenerator): + generator = generator.serialization_alias or generator.alias + if generator is not None: + candidates.append(generator(key)) + if material is not None: + keys[key] = next( + (alias for alias in dict.fromkeys(candidates) if material.get(alias) == getattr(source, key)), + preferred, + ) + else: + keys[key] = preferred + return keys + + +def _claims_identity_keys(source_class: type[Source]) -> bool: + """Report whether a business field serializes to a canonical identity spelling. + + A headerless envelope cannot distinguish an identity value from a business + value with that name. When the registered model claims the spelling for a + different field, only its declared aliases are authoritative. + """ + + return any( + name not in Source.model_fields and key in (field.alias, field.validation_alias, field.serialization_alias) + for name, field in source_class.model_fields.items() + for key in Source.model_fields + ) + + +def _normalize_native_material(source_class: type[Source], material: dict[str, JsonValue]) -> dict[str, JsonValue]: + """Read pre-canonical inherited envelope aliases, never business-field aliases. + + Derive spellings from the registered generator, not a version/key allowlist. + Explicit adapter aliases and conflicting identities retain strict validation. + """ + + generator = source_class.model_config.get("alias_generator") + if isinstance(generator, AliasGenerator): + generator = generator.serialization_alias or generator.alias + if generator is None: + return material + normalized = dict(material) + for key in Source.model_fields: + field = source_class.model_fields[key] + if field.validation_alias != key and field.alias_priority != 1: + continue + alias = generator(key) + if any( + field_name != key and alias in (field.alias, field.validation_alias, field.serialization_alias) + for field_name, field in source_class.model_fields.items() + ): + continue + if alias != key and alias in normalized: + if key in normalized and normalized[key] != normalized[alias]: + raise IdentityMismatchError("source", normalized[key], normalized[alias]) + normalized[key] = normalized.pop(alias) + return normalized + + def _dump_source( source: Source, *, @@ -525,8 +1126,16 @@ def _dump_source( encoding="powercontext-source-v1", representation=representation, value=_JSON_OBJECT.validate_python(source.model_dump(mode="json", by_alias=True)), + source=None if representation == "observation" else _source_header(source), + source_keys=None if representation == "observation" else _source_keys(source), ) - return rfc8785.dumps(envelope.model_dump(mode="json")) + if envelope.source_keys is not None: + for key, alias in envelope.source_keys.items(): + if alias not in envelope.value: + raise InvalidStoredPayloadError("source", name, f"serialized native identity omits {key!r}") + if envelope.value[alias] != getattr(source, key): + raise InvalidStoredPayloadError("source", name, f"serialized native identity changes {key!r}") + return rfc8785.dumps(envelope.model_dump(mode="json", exclude_none=True)) except (TypeError, ValueError) as error: raise InvalidStoredPayloadError("source", name, "value is not JSON serializable") from error diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index 54bb08b822..892de7b955 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -74,6 +74,10 @@ verdict_key, ) from powercontext.builtin.artifacts.skill.package import SkillPackageError, capture_skill_archive +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ( + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + SOURCE_EVIDENCE_BINDINGS_TABLE, +) from powercontext.builtin.persistence.candidates import decode_candidate_proposal_envelope from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.errors import ( @@ -116,6 +120,7 @@ from powercontext.builtin.tags import normalize_tags from powercontext.errors import SourceError from powercontext.sources import SourceDefinitionManifest, SourceObservation +from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence FORMAT_VERSION = 1 _RECORDS_NAME = "records.ndjson" @@ -164,6 +169,7 @@ "source_definition", "source_journal_head", "source", + "source_evidence_binding", "skill_package", "artifact_revision", "artifact_handle", @@ -173,6 +179,7 @@ "artifact_head", "memory_entry_version", "memory_entry_head", + "atomic_memory_evidence_history", "candidate_version", "candidate_head", "topic_memory_publication", @@ -299,6 +306,11 @@ class _RecordSpec: ("scope_id", "source_type", "source_id"), ("payload", "journal_position"), ), + "source_evidence_binding": _RecordSpec( + SOURCE_EVIDENCE_BINDINGS_TABLE, + ("scope_id", "source_type", "source_id"), + ("material_digest", "definition_name", "definition_version", "manifest_fingerprint", "declaration"), + ), "skill_package": _RecordSpec( SKILL_PACKAGES_TABLE, ("scope_id", "tree_digest"), @@ -373,6 +385,11 @@ class _RecordSpec: ("scope_id", "memory_artifact_id", "entry_id"), ("family", "head_revision", "entry_version_id", "entry_content_hash"), ), + "atomic_memory_evidence_history": _RecordSpec( + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + ("scope_id", "artifact_id", "revision"), + ("sources",), + ), "candidate_version": _RecordSpec( CANDIDATE_VERSIONS_TABLE, ("scope_id", "candidate_id", "version"), @@ -961,14 +978,9 @@ async def _export_rows( return # Definition manifests are global, but export authority is scope-bound. - # Include only immutable declarations referenced by selected observations. - definitions: set[tuple[str, str]] = set() - async for row in _export_rows(connection, "source", scopes): - observation = _source_observation(_record_from_row("source", row)) - if observation is not None: - definitions.add((observation.source_type, observation.definition_version)) + # Include immutable manifests referenced by observations or retained bindings. table = SOURCE_DEFINITION_MANIFESTS_TABLE - for name, definition_version in sorted(definitions): + for name, definition_version in sorted(await _referenced_definitions(connection, scopes)): row = ( ( await connection.execute( @@ -981,10 +993,31 @@ async def _export_rows( .one_or_none() ) if row is None: - raise BundleFormatError("observation references missing Source definition") + raise BundleFormatError("evidence or observation references missing Source definition") yield cast(Mapping[str, Any], row) +async def _referenced_definitions(connection: AsyncConnection, scopes: tuple[str, ...]) -> set[tuple[str, str]]: + """Collect Definition identities whose immutable manifest must travel with the scope.""" + + definitions: set[tuple[str, str]] = set() + async for row in _export_rows(connection, "source", scopes): + observation = _source_observation(_record_from_row("source", row)) + if observation is not None: + definitions.add((observation.source_type, observation.definition_version)) + async for binding in _export_rows(connection, "source_evidence_binding", scopes): + if binding["manifest_fingerprint"] is not None: + definitions.add((str(binding["definition_name"]), str(binding["definition_version"]))) + async for row in _export_rows(connection, "atomic_memory_evidence_history", scopes): + record = _record_from_row("atomic_memory_evidence_history", row) + definitions.update( + (source.definition_name, source.definition_version) + for source in _evidence_history_value(record) + if source.manifest_fingerprint is not None + ) + return definitions + + def _source_observation(record: _Record) -> SourceObservation | None: try: value = json.loads(cast(bytes, _database_value(record.payload["payload"]))) @@ -1429,6 +1462,30 @@ def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: strings("fingerprint") bytes_values("manifest") _source_definition(record) + elif kind == "source_evidence_binding": + strings("material_digest", "definition_name", "definition_version") + strings("manifest_fingerprint", nullable=("manifest_fingerprint",)) + bytes_values("declaration") + try: + MemoryEvidenceDeclaration.model_validate_json(cast(bytes, _database_value(payload["declaration"]))) + except (ValidationError, ValueError, TypeError) as error: + raise BundleFormatError("Source evidence binding has an invalid declaration") from error + if ( + payload["definition_name"] != identity["source_type"] + or not _valid_digest(payload["material_digest"]) + or (payload["manifest_fingerprint"] is not None and not _valid_digest(payload["manifest_fingerprint"])) + ): + raise BundleFormatError("Source evidence binding identity or digest is invalid") + elif kind == "atomic_memory_evidence_history": + bytes_values("sources") + sources = _evidence_history_value(record) + if any( + not _valid_digest(source.material_digest) + or source.definition_name != source.source_ref.source_type + or (source.manifest_fingerprint is not None and not _valid_digest(source.manifest_fingerprint)) + for source in sources + ): + raise BundleFormatError("Atomic Memory evidence has invalid bindings") elif kind == "source": bytes_values("payload") integers("journal_position", positive=True) @@ -1640,6 +1697,19 @@ def _recurrence_value(record: _Record, /) -> RecurrenceMatch | RecurrenceObserva return value +def _evidence_history_value(record: _Record, /) -> tuple[SourceEvidence, ...]: + try: + value = json.loads(cast(bytes, _database_value(record.payload["sources"]))) + except (ValueError, UnicodeDecodeError, TypeError) as error: + raise BundleFormatError("Atomic Memory evidence payload is not valid JSON") from error + if not isinstance(value, list): + raise BundleFormatError("Atomic Memory evidence payload is not a list") + try: + return tuple(SourceEvidence.model_validate(item) for item in value) + except ValidationError as error: + raise BundleFormatError("Atomic Memory evidence payload has an invalid Source entry") from error + + def _validate_datetime_text(value: object, /) -> None: try: parsed = datetime.fromisoformat(cast(str, value)) @@ -1972,6 +2042,46 @@ def _validate_record_dependencies( # noqa: C901 - logical record kinds have dis }, ): raise BundleFormatError("observation references missing or mismatched Source definition") + elif record.record_type == "source_evidence_binding": + if not contains( + "source", + {"scope_id": scope, "source_type": identity["source_type"], "source_id": identity["source_id"]}, + ): + raise BundleFormatError("Source evidence binding references missing Source") + fingerprint = payload["manifest_fingerprint"] + if fingerprint is not None and not contains( + "source_definition_fingerprint", + { + "definition_name": payload["definition_name"], + "definition_version": payload["definition_version"], + "fingerprint": fingerprint, + }, + ): + raise BundleFormatError("Source evidence binding references missing Source definition") + elif record.record_type == "atomic_memory_evidence_history": + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": "atomic-memory", + "artifact_id": identity["artifact_id"], + "revision": identity["revision"], + }, + ): + raise BundleFormatError("Atomic Memory evidence references missing exact Artifact revision") + for source in _evidence_history_value(record): + if not contains("source", {"scope_id": scope, **source.source_ref.model_dump()}): + raise BundleFormatError("Atomic Memory evidence references missing Source") + fingerprint = source.manifest_fingerprint + if fingerprint is not None and not contains( + "source_definition_fingerprint", + { + "definition_name": source.definition_name, + "definition_version": source.definition_version, + "fingerprint": fingerprint, + }, + ): + raise BundleFormatError("Atomic Memory evidence references missing Source definition") elif record.record_type in {"recurrence_match", "recurrence_observation"}: value = _recurrence_value(record) if not contains( diff --git a/src/powercontext/builtin/runtime/atomic_memory.py b/src/powercontext/builtin/runtime/atomic_memory.py index 0ab5824c3b..8bc55e6407 100644 --- a/src/powercontext/builtin/runtime/atomic_memory.py +++ b/src/powercontext/builtin/runtime/atomic_memory.py @@ -69,6 +69,10 @@ ) from powercontext.builtin.persistence.artifacts import ArtifactRepository, with_artifact_content from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository +from powercontext.builtin.persistence.atomic_memory_evidence import ( + AtomicMemoryEvidenceAnnotation, + AtomicMemoryEvidenceRepository, +) from powercontext.builtin.persistence.atomic_memory_index import ( ATOMIC_MEMORY_RRF_CONSTANT, AtomicMemoryIndex, @@ -273,6 +277,7 @@ def __init__( self.publisher = AtomicMemoryProjectionPublisher( index, embedding_model=embedding_model, + sources=artifacts.source_repository, ) self.service = AtomicMemoryService( artifacts=artifacts, @@ -323,6 +328,23 @@ async def get(self, artifact_id: str, *, revision: int | None = None, context=No connection, self.scope_id, artifact_id, context, revision=revision ) + async def evidence(self, artifact_id: str, *, revision: int, context=None) -> AtomicMemoryEvidenceAnnotation: + """Inspect neutral evidence facts for one authorized exact body revision. + + A missing historical snapshot remains unknown; current registration + cannot attest a revision that did not accept a declaration. + """ + application = self.application + async with self._read(context) as connection: + record = await application.service.get(connection, self.scope_id, artifact_id, context, revision=revision) + return await AtomicMemoryEvidenceRepository().verify( + connection, + self.scope_id, + record.ref, + record.artifact.lineage.sources, + sources=application.artifacts.source_repository, + ) + async def get_many(self, artifact_ids: Sequence[str], *, context=None) -> tuple[AtomicMemoryRecord, ...]: """Read a caller's exact results in one snapshot after checking each resource.""" diff --git a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py index 716480b6e3..1dfe6e6e3f 100644 --- a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py +++ b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Explicit offline repair of only the current Atomic Memory projection.""" +"""Offline current-projection repair and verification of retained Atomic history.""" from __future__ import annotations @@ -25,12 +25,14 @@ from sqlalchemy.engine import RowMapping from sqlalchemy.ext.asyncio import AsyncConnection +from powercontext.artifacts import ArtifactRef from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemory, AtomicMemoryRecord, AtomicMemoryStateValue from powercontext.builtin.artifacts.memory.canonical import canonical_json from powercontext.builtin.artifacts.search import analyze_text from powercontext.builtin.inference import EmbeddingModel from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository +from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceRepository from powercontext.builtin.persistence.atomic_memory_index import ( AtomicMemoryIndex, AtomicMemoryIndexError, @@ -49,6 +51,7 @@ from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex from powercontext.builtin.persistence.revision_metadata_schema import ensure_artifact_revision_metadata_schema +from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE @@ -115,6 +118,42 @@ async def _identity_batch( return tuple((str(row.scope_id), str(row.artifact_id)) for row in rows) +async def _verify_history( + connection: AsyncConnection, artifacts: ArtifactRepository, batch_size: int, sources: SourceRepository | None +) -> dict[str, int]: + table = ARTIFACTS_TABLE + evidence = AtomicMemoryEvidenceRepository() + after: tuple[str, str, int] | None = None + revisions = neutral_sources = 0 + while True: + statement = select(table.c.scope_id, table.c.artifact_id, table.c.revision).where( + table.c.family == AtomicMemory.family + ) + if after is not None: + statement = statement.where(tuple_(table.c.scope_id, table.c.artifact_id, table.c.revision) > after) + rows = ( + await connection.execute( + statement.order_by(table.c.scope_id, table.c.artifact_id, table.c.revision).limit(batch_size) + ) + ).all() + if not rows: + break + for row in rows: + scope_id, artifact_id, revision = str(row.scope_id), str(row.artifact_id), int(row.revision) + reference = ArtifactRef(family=AtomicMemory.family, artifact_id=artifact_id, revision=revision) + artifact = await artifacts.get(connection, scope_id, reference) + annotation = await evidence.verify( + connection, scope_id, reference, artifact.lineage.sources, sources=sources + ) + revisions += 1 + neutral_sources += sum( + source.declaration.authority == "untrusted" and source.declaration.verification == "unknown" + for source in annotation.sources + ) + after = scope_id, artifact_id, revision + return {"historical_revisions": revisions, "neutral_sources": neutral_sources} + + async def _record( connection: AsyncConnection, artifacts: ArtifactRepository, scope_id: str, artifact_id: str ) -> AtomicMemoryRecord: @@ -176,12 +215,20 @@ async def _projection_differences( async def verify_atomic_memory_current( - connection: AsyncConnection, index: AtomicMemoryIndex, /, *, batch_size: int = 100 + connection: AsyncConnection, + index: AtomicMemoryIndex, + /, + *, + batch_size: int = 100, + sources: SourceRepository | None = None, ) -> AtomicMemoryProjectionRebuildReport: - """Check every current identity against authority, including post-migration creations.""" + """Check current projections and every exact revision's retained evidence.""" + if isinstance(batch_size, bool) or not 1 <= batch_size <= 1000: + raise AtomicMemoryIndexError("batch-size", "Projection rebuild batch size must be between 1 and 1000") await _require_authority(connection) artifacts = ArtifactRepository((AtomicMemory,)) + history_counts = await _verify_history(connection, artifacts, batch_size, sources) profile = index.capabilities.embedding_profile fingerprint = None if profile is None else atomic_memory_profile_fingerprint(profile) errors: list[str] = [] @@ -219,7 +266,7 @@ async def verify_atomic_memory_current( return AtomicMemoryProjectionRebuildReport( ready=not errors, profile_fingerprint=fingerprint, - counts={"active_rows": active, "obsolete_rows": obsolete}, + counts={"active_rows": active, "obsolete_rows": obsolete, **history_counts}, errors=tuple(errors), ) @@ -243,6 +290,7 @@ async def rebuild_atomic_memory_projection( maintenance_confirmed: bool, embedding_model: EmbeddingModel | None = None, batch_size: int = 100, + sources: SourceRepository | None = None, ) -> AtomicMemoryProjectionRebuildReport: """Repair current without changing revisions, Family state or retained history. @@ -311,7 +359,7 @@ async def rebuild_atomic_memory_projection( ) await connection.execute(index.table.delete().where(_obsolete_current(index))) removed += obsolete - report = await verify_atomic_memory_current(connection, index, batch_size=batch_size) + report = await verify_atomic_memory_current(connection, index, batch_size=batch_size, sources=sources) return report.model_copy( update={ "counts": { diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index 6862718bd6..5efe33949a 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -215,7 +215,7 @@ ) from powercontext.builtin.statistics import ModelUsagePurpose from powercontext.errors import InvalidSourceProjectionError, SourceProjectionNotFoundError -from powercontext.sources import Source, SourceDefinitionRegistry, SourceProjectionKey +from powercontext.sources import AdapterSourceDefinition, Source, SourceDefinitionRegistry, SourceProjectionKey if TYPE_CHECKING: from pydantic_ai.models import Model @@ -919,7 +919,17 @@ def _validate_processing_source_registry( type(source_registry) is not SourceDefinitionRegistry or len(definitions) != len(builtin_definitions) or any( - definition is not expected for definition, expected in zip(definitions, builtin_definitions, strict=True) + not ( + definition is expected + or ( + type(definition) is AdapterSourceDefinition + and type(expected) is AdapterSourceDefinition + and definition.adapter is expected.adapter + and definition.version == expected.version + and definition.projections is expected.projections + ) + ) + for definition, expected in zip(definitions, builtin_definitions, strict=True) ) ): raise BuiltinConfigurationError("artifact-processing-source-registry") diff --git a/src/powercontext/sources/__init__.py b/src/powercontext/sources/__init__.py index 91e1a27b08..5469244670 100644 --- a/src/powercontext/sources/__init__.py +++ b/src/powercontext/sources/__init__.py @@ -30,6 +30,7 @@ SourceDefinitionRegistry, SourceProjection, ) +from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence from powercontext.sources.models import Source, SourceMaterialization, SourceProjectionKey, SourceRef from powercontext.sources.observations import ( SourceDefinitionManifest, @@ -53,6 +54,7 @@ "ConnectorRunSession", "ConnectorRunStatus", "ConnectorSubmissionStatus", + "MemoryEvidenceDeclaration", "Source", "SourceAdapter", "SourceCatalog", @@ -60,6 +62,7 @@ "SourceDefinition", "SourceDefinitionManifest", "SourceDefinitionRegistry", + "SourceEvidence", "SourceMaterialization", "SourceObservation", "SourceProjection", diff --git a/src/powercontext/sources/definitions.py b/src/powercontext/sources/definitions.py index d9c54572dc..18f6bd5162 100644 --- a/src/powercontext/sources/definitions.py +++ b/src/powercontext/sources/definitions.py @@ -18,6 +18,7 @@ from collections.abc import Iterable, Mapping from dataclasses import dataclass +from types import MappingProxyType from typing import Any, Generic, Protocol, TypeVar, cast from pydantic import BaseModel, JsonValue, TypeAdapter @@ -34,6 +35,7 @@ SourceProjectionNotFoundError, ) from powercontext.sources.adapters import SourceAdapter +from powercontext.sources.evidence import MemoryEvidenceDeclaration from powercontext.sources.models import Source, SourceProjectionKey InputT = TypeVar("InputT") @@ -42,6 +44,7 @@ _JSON_VALUE = TypeAdapter(JsonValue) _AnySourceAdapter = SourceAdapter[Any, Any, Any] +_NEUTRAL_MEMORY_EVIDENCE = MemoryEvidenceDeclaration() class SourceProjection(Protocol[SourceT]): @@ -60,6 +63,7 @@ class SourceDefinition(SourceAdapter[InputT, SourceT, ValueT_co], Protocol[Input version: str projections: tuple[SourceProjection[SourceT], ...] + memory_evidence: MemoryEvidenceDeclaration | None @dataclass(frozen=True, slots=True) @@ -69,6 +73,7 @@ class AdapterSourceDefinition(Generic[InputT, SourceT, ValueT_co]): adapter: SourceAdapter[InputT, SourceT, ValueT_co] version: str = "1" projections: tuple[SourceProjection[SourceT], ...] = () + memory_evidence: MemoryEvidenceDeclaration | None = None @property def input_class(self) -> type[InputT]: @@ -93,14 +98,28 @@ async def read(self, source: SourceT, /) -> ValueT_co: class SourceDefinitionRegistry: - """Provide one immutable routing view for Source persistence and consumers.""" - - def __init__(self, definitions: Iterable[_AnySourceDefinition], /) -> None: + """Provide one immutable routing and trusted declaration view. + + Native Definitions optionally supply ``memory_evidence``. Deployment-owned + ``remote_declarations`` keys are exact ``(name, version, fingerprint)`` + manifest identities, never worker payload or projection values. Declarations + are validated and copied at construction; invalid or unavailable values are + neutral and never prevent registration of an otherwise valid Definition. + """ + + def __init__( + self, + definitions: Iterable[_AnySourceDefinition], + /, + *, + remote_declarations: Mapping[tuple[str, str, str], MemoryEvidenceDeclaration] | None = None, + ) -> None: by_input: dict[type[object], _AnySourceDefinition] = {} by_source: dict[type[Source], _AnySourceDefinition] = {} by_name: dict[str, _AnySourceDefinition] = {} projections: dict[type[Source], Mapping[SourceProjectionKey, SourceProjection[Any]]] = {} registered: list[_AnySourceDefinition] = [] + memory_evidence: dict[type[Source], MemoryEvidenceDeclaration] = {} for definition in definitions: input_class, source_class = _validate_definition(definition) @@ -122,6 +141,11 @@ def __init__(self, definitions: Iterable[_AnySourceDefinition], /) -> None: by_source[source_class] = definition by_name[definition.name] = definition projections[source_class] = projection_routes + try: + declaration = getattr(definition, "memory_evidence", None) + except Exception: # Declaration providers may be unavailable; routing remains valid. + declaration = None + memory_evidence[source_class] = _snapshot_memory_evidence(declaration) registered.append(definition) self._definitions = tuple(registered) @@ -129,6 +153,10 @@ def __init__(self, definitions: Iterable[_AnySourceDefinition], /) -> None: self._by_source = by_source self._by_name = by_name self._projections = projections + self._memory_evidence = MappingProxyType(memory_evidence) + self._remote_declarations = MappingProxyType({ + key: _snapshot_memory_evidence(value) for key, value in (remote_declarations or {}).items() + }) @classmethod def from_adapters(cls, adapters: Iterable[_AnySourceAdapter], /) -> SourceDefinitionRegistry: @@ -161,6 +189,23 @@ def definition_for_source(self, source: object, /) -> _AnySourceDefinition: ) return definition + def memory_evidence_for_source(self, source: Source, /) -> MemoryEvidenceDeclaration: + """Read the frozen trusted declaration using the exact registered route.""" + + from powercontext.sources.observations import SourceObservation + + if isinstance(source, SourceObservation): + return self.memory_evidence_for_manifest( + source.source_type, source.definition_version, source.definition_fingerprint + ) + self.definition_for_source(source) + return self._memory_evidence[type(source)] + + def memory_evidence_for_manifest(self, name: str, version: str, fingerprint: str, /) -> MemoryEvidenceDeclaration: + """Return only deployment-configured evidence for an exact manifest.""" + + return self._remote_declarations.get((name, version, fingerprint), _NEUTRAL_MEMORY_EVIDENCE) + async def resolve(self, value: object, /) -> Source: input_class = type(value) try: @@ -203,6 +248,18 @@ def project(self, source: Source, key: SourceProjectionKey, /) -> JsonValue: raise InvalidSourceProjectionError(key.name, "result", "must match the declared output schema") from error +def _snapshot_memory_evidence(value: object) -> MemoryEvidenceDeclaration: + if value is None: + return _NEUTRAL_MEMORY_EVIDENCE + try: + # Revalidate even constructed models, and never retain caller-owned data. + if isinstance(value, MemoryEvidenceDeclaration): + value = value.model_dump(mode="python") + return MemoryEvidenceDeclaration.model_validate(value) + except (TypeError, ValueError): + return _NEUTRAL_MEMORY_EVIDENCE + + def _validate_definition(definition: object) -> tuple[type[object], type[Source]]: definition_type = type(definition) input_class = getattr(definition, "input_class", None) diff --git a/src/powercontext/sources/evidence.py b/src/powercontext/sources/evidence.py new file mode 100644 index 0000000000..33eebfed9f --- /dev/null +++ b/src/powercontext/sources/evidence.py @@ -0,0 +1,60 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Typed declarations and exact material facts accepted at Source registration.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from powercontext.sources.models import SourceMaterialization, SourceRef + + +class MemoryEvidenceDeclaration(BaseModel): + """A trusted adapter's claim, not a score, permission or verification engine. + + ``declaration_version`` identifies this contract, independently of the + Definition version and Artifact content revision. Missing or unsupported + declarations resolve to ``untrusted`` / ``unknown``. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + authority: Literal["untrusted", "user_asserted", "repository_attested", "system_attested"] = "untrusted" + verification: Literal["unknown", "verified", "not_verified"] = "unknown" + declaration_version: Literal["1"] = "1" + + +class SourceEvidence(BaseModel): + """Retain one exact Source material binding without duplicating its body. + + The digest checks equality of canonical Source JSON, excluding only the + server-owned ContentSource handoff receipt witness. It authenticates neither + the Source nor its declaration. Each Source retains its own declaration. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + source_ref: SourceRef + materialization: SourceMaterialization + material_digest: str + definition_name: str + definition_version: str + manifest_fingerprint: str | None = None + declaration: MemoryEvidenceDeclaration = Field(default_factory=MemoryEvidenceDeclaration) + + +__all__ = ["MemoryEvidenceDeclaration", "SourceEvidence"] diff --git a/src/powercontext/sources/models.py b/src/powercontext/sources/models.py index 6e2cd48df6..0f1a4a26fe 100644 --- a/src/powercontext/sources/models.py +++ b/src/powercontext/sources/models.py @@ -60,6 +60,8 @@ def validate_key_part(cls, value: str, info) -> str: class Source(BaseModel): """Base value for an adapter-owned Source description.""" + model_config = ConfigDict(populate_by_name=True) + name: str definition_version: str = "1" materialization: SourceMaterialization diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 576e29fb79..843196a3c6 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -20,7 +20,17 @@ from typing import Any, Literal import rfc8785 -from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, field_validator, model_validator +from pydantic import ( + AliasChoices, + AliasPath, + BaseModel, + ConfigDict, + Field, + JsonValue, + TypeAdapter, + field_validator, + model_validator, +) from powercontext.errors import ( InvalidSourceDefinitionError, @@ -161,9 +171,28 @@ def projection(self, key: SourceProjectionKey, /) -> JsonValue: def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> SourceDefinitionManifest: - """Build the immutable declaration transported by a remote worker.""" + """Build a declaration matching the canonical remote payload representation.""" + + return _manifest_for_definition(definition, transport=True) - source_schema = _json_object(definition.source_class.model_json_schema()) + +def _manifest_for_definition( + definition: SourceDefinition[Any, Any, Any], + /, + *, + transport: bool, +) -> SourceDefinitionManifest: + if transport and _validation_schema_matches_payload(definition.source_class): + source_schema = _json_object(definition.source_class.model_json_schema()) + elif transport: + source_schema = _json_object( + definition.source_class.model_json_schema( + mode="serialization", + by_alias=bool(definition.source_class.model_config.get("serialize_by_alias")), + ) + ) + else: + source_schema = _json_object(definition.source_class.model_json_schema(mode="serialization")) projections = tuple( SourceProjectionManifest( key=SourceProjectionKey(name=projection.name, version=projection.version), @@ -215,6 +244,47 @@ def project_source_for_transport( ) +def _validation_schema_matches_payload( + source_class: type[BaseModel], + seen: set[type[BaseModel]] | None = None, +) -> bool: + """Return whether validation-mode keys already describe the serialized payload.""" + + visited = set() if seen is None else seen + if source_class in visited: + return True + visited.add(source_class) + serialize_by_alias = bool(source_class.model_config.get("serialize_by_alias")) + for name, field in source_class.model_fields.items(): + if field.exclude: + continue + serialization_key = (field.serialization_alias or field.alias or name) if serialize_by_alias else name + if field.is_required() and _validation_schema_key(name, field) != serialization_key: + return False + annotation = field.annotation + if ( + field.is_required() + and isinstance(annotation, type) + and issubclass(annotation, BaseModel) + and not _validation_schema_matches_payload(annotation, visited) + ): + return False + return True + + +def _validation_schema_key(name: str, field: Any) -> str | None: + alias = field.validation_alias + if isinstance(alias, str): + return alias + if isinstance(alias, AliasChoices): + return next((choice for choice in alias.choices if isinstance(choice, str)), None) + if isinstance(alias, AliasPath): + return name # Pydantic keeps the field name for path-based validation. + if alias is not None: + return None + return field.alias if field.alias is not None else name + + def _source_definition_fingerprint( *, name: str, diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index 945c6b78ce..71022d4a3c 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -299,10 +299,10 @@ def test_mysql_initialization_and_shared_evidence_noop(backend: _BackendTarget) assert result.state == "ready" assert result.changed assert result.backup_state == "not_required" - assert result.revision == "p0004" + assert result.revision == "p0005" replacement = backend.runner() assert replacement.plan().state == "ready" - assert replacement.verify().revision == "p0004" + assert replacement.verify().revision == "p0005" assert not replacement.apply().changed @@ -322,7 +322,7 @@ def test_mysql_operator_backup_choices_preserve_baseline_data_without_fork( result = _apply(runner, policy=policy) assert result.backup_state == ("user_confirmed" if policy == "manual" else "skipped") state = _read_state(backend) - assert state["revision"] == "p0004" + assert state["revision"] == "p0005" assert state["content"] == [(b'{"content":"preserved"}',)] assert state["tags"] == [("Keep",)] assert "memory_citations" in state["columns"]["pc_artifacts"] @@ -390,7 +390,7 @@ def test_mysql_committed_partial_ddl_resumes_from_shared_original_evidence(backe replacement = _acknowledge_known_completed_interruption(backend, recovery) assert _apply(replacement, policy="manual").state == "ready" state = _read_state(backend) - assert state["revision"] == "p0004" + assert state["revision"] == "p0005" assert state["content"] == [(b'{"content":"preserved"}',)] assert state["tags"] == [("Keep",)] assert replacement.verify().state == "ready" @@ -436,7 +436,7 @@ def test_mysql_empty_source_recovers_baseline_and_citation_ddl_without_probing_b ) result = _apply(replacement) assert result.backup_state == "not_required" - assert replacement.verify().revision == "p0004" + assert replacement.verify().revision == "p0005" assert _read_state(backend)["content"] == [] @@ -453,7 +453,7 @@ def test_mysql_baseline_adoption_recovers_an_empty_committed_version_table(backe assert recovered.adopt_baseline replacement = _acknowledge_known_completed_interruption(backend, recovered) assert _apply(replacement, policy="manual").state == "ready" - assert replacement.verify().revision == "p0004" + assert replacement.verify().revision == "p0005" assert _read_state(backend)["tags"] == [("Keep",)] @@ -479,7 +479,7 @@ def interrupt(journal, evidence) -> None: assert recovery.source_revision == "p0002" replacement = _acknowledge_known_completed_interruption(backend, recovery) assert _apply(replacement, policy="manual").state == "ready" - assert replacement.verify().revision == "p0004" + assert replacement.verify().revision == "p0005" assert _read_state(backend)["content"] == [(b'{"content":"preserved"}',)] @@ -488,7 +488,7 @@ def test_mysql_completed_head_with_an_interrupted_run_requires_exact_acknowledge pytest.skip("embedded seekdb closes its engine and has no remote in-flight run receipt") _load_baseline(backend) assert _apply(backend.runner(), policy="manual").state == "ready" - assert backend.runner().verify().revision == "p0004" + assert backend.runner().verify().revision == "p0005" def committed_write_then_interrupt( connection: Connection | None, _identity: BackendIdentity, verify: Callable[[], None] @@ -502,12 +502,12 @@ def committed_write_then_interrupt( with pytest.raises(_InterruptedDDL): backend.connections().run(committed_write_then_interrupt, writable=True) before = _read_state(backend) - assert before["revision"] == "p0004" + assert before["revision"] == "p0005" assert before["tags"] == [("Confirmed",)] replacement = backend.runner(backup_factory=_forbidden_automatic_backup) recovery = replacement.plan() assert recovery.state == "recovery_required" - assert recovery.source_revision == "p0004" + assert recovery.source_revision == "p0005" assert recovery.revisions == () pending = (recovery.coordination or {}).get("pending_run_id") assert pending @@ -526,6 +526,6 @@ def committed_write_then_interrupt( assert result.state == "ready" assert not result.changed assert _read_state(backend) == before - assert backend.runner().verify().revision == "p0004" + assert backend.runner().verify().revision == "p0005" assert not backend.runner().apply().changed assert "pending_run_id" not in (backend.runner().plan().coordination or {}) diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 48bd9b8a51..66e9b3022b 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -15,23 +15,33 @@ from __future__ import annotations import asyncio +import json +from urllib.parse import quote, unquote import pytest -from sqlalchemy import insert +from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator +from pydantic.alias_generators import to_camel +from sqlalchemy import delete, insert, select, update from powercontext import InvalidSourceDefinitionError from powercontext.builtin.persistence import ( InvalidRepositoryArgumentError, + InvalidStoredPayloadError, RepositoryNotFoundError, StoredPayloadConflictError, ) +from powercontext.builtin.persistence.atomic_memory_evidence_schema import SOURCE_EVIDENCE_BINDINGS_TABLE from powercontext.builtin.persistence.codec import dump_model from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import SHARED_TABLES, SOURCES_TABLE +from powercontext.builtin.runtime import SubmitSourceObservation from powercontext.builtin.runtime.relational import RelationalContexts -from powercontext.builtin.sources import CONTENT_SOURCE_ADAPTER, ContentCapture +from powercontext.builtin.sources import CONTENT_SOURCE_ADAPTER, ContentCapture, ContentSource from powercontext.sources import ( + AdapterSourceDefinition, + MemoryEvidenceDeclaration, + Source, SourceDefinitionRegistry, SourceMaterialization, SourceObservation, @@ -47,6 +57,49 @@ ) +class SerializationAliasInput(BaseModel): + source_id: str + body: str + + +class SerializationAliasSource(Source): + body: str = Field(serialization_alias="wire_text") + + +class CamelCaseSource(SerializationAliasSource): + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + +class UpperCaseSource(SerializationAliasSource): + model_config = ConfigDict(alias_generator=str.upper, populate_by_name=True) + + +class NativeCamelCaseSource(SerializationAliasSource): + model_config = ConfigDict(alias_generator=to_camel) + + body: str + + +class SerializationAliasAdapter: + name = "serialization-alias" + input_class = SerializationAliasInput + + def __init__(self, source_class: type[SerializationAliasSource]) -> None: + self.source_class = source_class + + async def resolve(self, value: SerializationAliasInput, /) -> SerializationAliasSource: + return self.source_class( + name=value.source_id, + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + description="Remote alias description.", + body=value.body, + ) + + async def read(self, source: SerializationAliasSource, /) -> str: + return source.body + + def test_two_source_adapters_share_one_repository_and_journal() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): @@ -100,6 +153,43 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_source_batch_binds_each_new_identity_and_preserves_replayed_evidence() -> None: + class BatchSource(SerializationAliasSource): + body: str + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(BatchSource), version="2", memory_evidence=declaration), + )) + repository = SourceRepository(registry) + first = await registry.resolve(SerializationAliasInput(source_id="first", body="First retained text.")) + second = await registry.resolve(SerializationAliasInput(source_id="second", body="Second retained text.")) + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + async with profile.database.transaction() as connection: + results = await repository.add_many_with_status(connection, "scope", (first, first, second)) + assert [created for _, created in results] == [True, False, True] + assert [stored.journal_position for stored, _ in results] == [1, 1, 2] + refs = (results[0][0].ref, results[2][0].ref) + evidence = await repository.historical_evidence(connection, "scope", refs) + assert [item.declaration for item in evidence] == [declaration, declaration] + assert await repository.journal_position(connection, "scope") == 2 + + neutral = SourceRepository( + SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(BatchSource), version="2"), + )) + ) + async with profile.database.transaction() as connection: + replayed = await neutral.add_many_with_status(connection, "scope", (second, first, second)) + assert [created for _, created in replayed] == [False, False, False] + assert [stored.journal_position for stored, _ in replayed] == [2, 1, 2] + assert await neutral.historical_evidence(connection, "scope", refs) == evidence + assert await neutral.journal_position(connection, "scope") == 2 + + asyncio.run(scenario()) + + def test_reusing_a_stable_source_identity_with_different_payload_is_a_conflict() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): @@ -171,6 +261,473 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("source_class", (SerializationAliasSource, CamelCaseSource, UpperCaseSource)) +def test_remote_observation_ingestion_accepts_serialization_aliases( + source_class: type[SerializationAliasSource], +) -> None: + async def scenario() -> None: + worker = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(source_class), version="2"), + )) + source = await worker.resolve(SerializationAliasInput(source_id="remote-1", body="Remote note text.")) + manifest = manifest_for_definition(worker.definition_for_name("serialization-alias")) + observation = project_source_for_transport(worker, source) + + async with repository_profile() as (profile, _repositories): + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + context = await contexts.get("scope-a") + stored = await context.sources.get(observation) + + assert receipt.source_ref == SourceRef(source_type="serialization-alias", source_id="remote-1") + assert isinstance(stored, SourceObservation) + assert stored.payload == { + "name": "remote-1", + "definition_version": "2", + "materialization": "captured", + "description": "Remote alias description.", + "body": "Remote note text.", + } + + asyncio.run(scenario()) + + +def test_native_generated_alias_source_survives_read_replay_and_restart(tmp_path) -> None: + async def scenario() -> None: + adapter = SerializationAliasAdapter(NativeCamelCaseSource) + registry = SourceDefinitionRegistry((AdapterSourceDefinition(adapter, version="2"),)) + value = SerializationAliasInput(source_id="native-2", body="Keep the native version and body.") + source = await adapter.resolve(value) + # Keep the persisted version nondefault regardless of the constructor's alias rules. + source.definition_version = "2" + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'native-alias.db'}") + + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=registry) + context = await contexts.get("scope-a") + assert await context.sources.add(source) == source + loaded = await context.sources.get(source) + assert loaded == source + assert await context.sources.read(loaded) == value.body + assert await context.sources.add(source) == source + assert await context.sources.journal.position(source) == 1 + + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=registry) + context = await contexts.get("scope-a") + loaded = await context.sources.get(source) + assert loaded == source + assert loaded.definition_version == "2" + resolved = await context.sources.resolve(value) + assert resolved == source + assert await context.sources.add(resolved) == source + assert await context.sources.journal.position(resolved) == 1 + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("current_version", ["1", "2"]) +def test_canonical_history_survives_current_identity_aliases(current_version) -> None: + class CurrentSource(SerializationAliasSource): + name: str = Field(alias="source_name") + label: str = Field(alias="name") + + async def scenario() -> None: + original = SourceRepository((SerializationAliasAdapter(SerializationAliasSource),)) + source = SerializationAliasSource( + name="canonical", materialization=SourceMaterialization.CAPTURED, body="Retained." + ) + material = source.model_dump(mode="json", by_alias=True) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await original.add(connection, "scope-a", source) + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + await connection.execute(delete(SOURCE_EVIDENCE_BINDINGS_TABLE)) + expected = await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) + current = SourceRepository( + SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version=current_version), + )) + ) + assert await current.historical_evidence(connection, "scope-a", (stored.ref,)) == expected + assert expected[0].definition_version == "1" + assert expected[0].declaration == MemoryEvidenceDeclaration() + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("has_header", [False, True]) +def test_business_name_alias_does_not_reinterpret_canonical_history(has_header: bool) -> None: + class CurrentSource(SerializationAliasSource): + label: str = Field(alias="name") + + async def scenario() -> None: + original = SourceRepository((SerializationAliasAdapter(SerializationAliasSource),)) + current = SourceRepository( + SourceDefinitionRegistry((AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version="1"),)) + ) + source = SerializationAliasSource( + name="canonical-business-alias", + materialization=SourceMaterialization.CAPTURED, + body="Retained.", + ) + material = source.model_dump(mode="json", by_alias=True) + value = {"encoding": "powercontext-source-v1", "representation": "native", "value": material} + if has_header: + value.update( + source=Source( + name=source.name, + definition_version=source.definition_version, + materialization=source.materialization, + ).model_dump(mode="json", exclude_none=True), + source_keys={ + "name": "name", + "definition_version": "definition_version", + "materialization": "materialization", + }, + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await original.add(connection, "scope-a", source) + await connection.execute( + update(SOURCES_TABLE) + .where(SOURCES_TABLE.c.source_id == source.name) + .values(payload=json.dumps(value).encode()) + ) + await connection.execute(delete(SOURCE_EVIDENCE_BINDINGS_TABLE)) + expected = await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) + assert await current.historical_evidence(connection, "scope-a", (stored.ref,)) == expected + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("business_version", ["1", "2"]) +def test_bound_aliased_history_does_not_treat_business_fields_as_identity(business_version) -> None: + class AliasedSource(NativeCamelCaseSource): + definition_version: str = Field(alias="source_version") + materialization: SourceMaterialization = Field(alias="source_materialization") + business_version: str = Field(alias="definition_version") + business_kind: str = Field(alias="materialization") + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(AliasedSource), version="2", memory_evidence=declaration), + )) + repository = SourceRepository(registry) + source = AliasedSource( + name="aliased-history", + source_version="2", + source_materialization=SourceMaterialization.CAPTURED, + definition_version=business_version, + materialization="referenced", + body="Retained.", + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await repository.add(connection, "scope-a", source) + expected = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": source.model_dump(mode="json", by_alias=True), + }).encode() + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + assert await repository.historical_evidence(connection, "scope-a", (stored.ref,)) == expected + assert expected[0].definition_version == "2" + assert expected[0].materialization is SourceMaterialization.CAPTURED + assert expected[0].declaration == declaration + + asyncio.run(scenario()) + + +def test_unbound_versioned_canonical_history_ignores_new_definition_aliases() -> None: + class CurrentSource(SerializationAliasSource): + model_config = ConfigDict(populate_by_name=True) + name: str = Field(alias="source_name") + label: str = Field(alias="name") + + async def scenario() -> None: + old_source = SerializationAliasSource( + name="old-version-two", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Retained without an evidence binding.", + ) + old_repository = SourceRepository(()) + + class CurrentAdapter: + name = "serialization-alias" + input_class = SerializationAliasInput + source_class = CurrentSource + + async def resolve(self, value: SerializationAliasInput, /) -> CurrentSource: + raise AssertionError # Historical evidence must not invoke the current decoder. + + async def read(self, source: CurrentSource, /) -> str: + return source.body + + registry = SourceDefinitionRegistry((AdapterSourceDefinition(CurrentAdapter(), version="3"),)) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + # Model a canonical Source accepted before evidence bindings existed. + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="scope-a", + source_type="serialization-alias", + source_id=old_source.name, + payload=json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": old_source.model_dump(mode="json", by_alias=False), + }).encode(), + journal_position=1, + ) + ) + ref = SourceRef(source_type="serialization-alias", source_id=old_source.name) + baseline = await old_repository.historical_evidence(connection, "scope-a", (ref,)) + upgraded = await SourceRepository(registry).historical_evidence(connection, "scope-a", (ref,)) + assert upgraded == baseline + assert upgraded[0].definition_version == "2" + assert upgraded[0].declaration == MemoryEvidenceDeclaration() + + asyncio.run(scenario()) + + +def test_headerless_content_receipt_preserves_accepted_material_digest() -> None: + async def scenario() -> None: + repository = SourceRepository((CONTENT_SOURCE_ADAPTER,)) + source = ContentSource( + name="receipt-digest", + materialization=SourceMaterialization.CAPTURED, + content="Receipt content.", + handoff_receipt=True, + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await repository.add(connection, "scope-a", source) + original = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) + material = source.model_dump(mode="json", by_alias=True) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + restored = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) + assert restored == original + + asyncio.run(scenario()) + + +def test_precanonical_generated_alias_source_rows_remain_readable_and_replayable() -> None: + async def scenario() -> None: + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(NativeCamelCaseSource), version="2"), + )) + repository = SourceRepository(registry) + source = NativeCamelCaseSource( + name="legacy-2", definition_version="2", materialization=SourceMaterialization.CAPTURED, body="Retained." + ) + material = source.model_dump(mode="json", by_alias=True) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + async with ( + repository_profile() as (profile, _repositories), + profile.database.transaction() as connection, + ): + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="scope-a", + source_type="serialization-alias", + source_id=source.name, + payload=payload, + journal_position=1, + ) + ) + ref = SourceRef(source_type="serialization-alias", source_id=source.name) + with pytest.raises(InvalidStoredPayloadError): + await SourceRepository(()).historical_evidence(connection, "scope-a", (ref,)) + loaded = await repository.get(connection, "scope-a", ref) + assert loaded.value == source + assert await repository.add(connection, "scope-a", source) == loaded + assert await repository.journal_position(connection, "scope-a") == 1 + + asyncio.run(scenario()) + + +def test_excluded_native_identity_is_rejected_before_capture() -> None: + class ExcludedVersionSource(NativeCamelCaseSource): + definition_version: str = Field(default="1", exclude=True) + + async def scenario() -> None: + repository = SourceRepository((SerializationAliasAdapter(ExcludedVersionSource),)) + source = ExcludedVersionSource( + name="excluded", materialization=SourceMaterialization.CAPTURED, body="Retained." + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + with pytest.raises(InvalidStoredPayloadError): + await repository.add(connection, "scope-a", source) + ref = SourceRef(source_type="serialization-alias", source_id=source.name) + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, "scope-a", ref) + assert await repository.journal_position(connection, "scope-a") == 0 + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("identity_field", ["name", "definition_version", "materialization"]) +def test_encoded_native_identity_is_rejected_before_capture(identity_field) -> None: + class EncodedSource(NativeCamelCaseSource): + @field_serializer("name", "definition_version", "materialization") + def encode_identity(self, value, info): + if info.field_name == identity_field: + return quote(f"wire:{value}", safe="") + return value + + @field_validator("name", "definition_version", "materialization", mode="before") + @classmethod + def decode_identity(cls, value, info): + if info.field_name == identity_field: + return unquote(value).removeprefix("wire:") + return value + + async def scenario() -> None: + repository = SourceRepository((SerializationAliasAdapter(EncodedSource),)) + source = EncodedSource(name="note/42", materialization=SourceMaterialization.CAPTURED, body="Retained.") + assert EncodedSource.model_validate_json(source.model_dump_json(by_alias=True)) == source + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + with pytest.raises(InvalidStoredPayloadError): + await repository.add(connection, "scope-a", source) + ref = SourceRef(source_type="serialization-alias", source_id=source.name) + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, "scope-a", ref) + assert await repository.journal_position(connection, "scope-a") == 0 + + asyncio.run(scenario()) + + +def test_explicit_name_alias_evidence_survives_restart_without_the_native_definition(tmp_path) -> None: + class AliasedSource(NativeCamelCaseSource): + name: str = Field(alias="source_name") + business_name: str = Field(alias="name") + description: str | None = Field(default="captured description", exclude=True) + + class UpdatedSource(AliasedSource): + description: str | None = Field(default="current description", exclude=True) + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition( + SerializationAliasAdapter(AliasedSource), + version="2", + memory_evidence=declaration, + ), + )) + source = AliasedSource( + source_name="native-alias", + name="business label", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Retained.", + ) + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'explicit-alias.db'}") + repository = SourceRepository(registry) + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + stored = await repository.add(connection, "scope-a", source) + assert (await repository.get(connection, "scope-a", stored.ref)).value == source + assert await repository.add(connection, "scope-a", source) == stored + evidence = await repository.evidence(connection, "scope-a", (stored.ref,)) + assert evidence[0].declaration == declaration + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + updated = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(UpdatedSource), version="2"), + )) + loaded = await SourceRepository(updated).get(connection, "scope-a", stored.ref) + assert loaded.value.description == "current description" + assert await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) == evidence + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("representation", ["enveloped", "bare"]) +def test_replay_retains_historical_native_identity_spelling(representation) -> None: + from pydantic import AliasChoices + + class OldSource(NativeCamelCaseSource): + name: str = Field(alias="legacy_name") + + class CurrentSource(NativeCamelCaseSource): + name: str = Field( + validation_alias=AliasChoices("source_name", "legacy_name"), + serialization_alias="source_name", + ) + + async def scenario() -> None: + original_registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(OldSource), version="2"), + )) + current_registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version="2"), + )) + original = SourceRepository(original_registry) + current = SourceRepository(current_registry) + source = OldSource( + legacy_name="legacy-replay", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Retained exactly.", + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await original.add(connection, "scope-a", source) + material = source.model_dump(mode="json", by_alias=True) + if representation == "bare": + payload = json.dumps(material).encode() + else: + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + loaded = await current.get(connection, "scope-a", stored.ref) + assert loaded.value.name == source.name + replay = await current.add(connection, "scope-a", loaded.value) + assert replay.ref == stored.ref + assert await current.get(connection, "scope-a", stored.ref) == loaded + evidence = await current.historical_evidence(connection, "scope-a", (stored.ref,)) + assert evidence[0].source_ref == stored.ref + raw = await connection.scalar( + select(SOURCES_TABLE.c.payload).where(SOURCES_TABLE.c.source_id == source.name) + ) + assert isinstance(raw, bytes) + envelope = json.loads(raw) + assert envelope["source_keys"]["name"] == "legacy_name" + + asyncio.run(scenario()) + + def test_repository_rejects_a_scope_outside_the_relational_identity_baseline() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): @@ -333,3 +890,45 @@ async def add(scope_id: str, source: NoteSource): assert following.journal_position == 2 asyncio.run(scenario()) + + +def test_nested_uppercase_source_aliases_survive_read_replay_and_restart(tmp_path) -> None: + class UpperNested(Source): + model_config = ConfigDict(alias_generator=str.upper) + body: str + + class NestedNativeSource(NativeCamelCaseSource): + nested: UpperNested + + async def scenario() -> None: + adapter = SerializationAliasAdapter(NestedNativeSource) + registry = SourceDefinitionRegistry((AdapterSourceDefinition(adapter, version="2"),)) + repository = SourceRepository(registry) + source = NestedNativeSource( + name="nested-legacy", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Outer.", + nested=UpperNested( + name="inner", + materialization=SourceMaterialization.REFERENCED, + body="Nested.", + ), + ) + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'nested-alias.db'}") + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + original = await repository.add(connection, "scope-a", source) + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + loaded = await repository.get(connection, "scope-a", original.ref) + assert loaded.value == source + assert await repository.add(connection, "scope-a", source) == original + evidence = await repository.historical_evidence(connection, "scope-a", (original.ref,)) + assert evidence[0].source_ref == original.ref + + asyncio.run(scenario()) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 5fe8846acf..4825a1e2e2 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import json import os from collections.abc import Iterator from pathlib import Path @@ -26,6 +27,7 @@ from pydantic import SecretStr from sqlalchemy.engine import make_url +from powercontext.builtin.persistence.migrations.backup import identify_fork_server from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.seekdb import SeekDBConfig from powercontext.builtin.persistence.sqlite import SQLiteConfig @@ -73,6 +75,17 @@ async def manage(*, create: bool) -> None: nonlocal owned async with migration_profile(config) as profile, profile.database.transaction() as connection: assert (await connection.exec_driver_sql("SELECT DATABASE()")).scalar_one() == url.database + version = str((await connection.exec_driver_sql("SELECT VERSION()")).scalar_one()) + comment = str((await connection.exec_driver_sql("SELECT @@version_comment")).scalar_one()) + product, engine_version = identify_fork_server(version, comment) + if product != "oceanbase_ai" or engine_version is None: + pytest.fail( + "Acceptance requires an identified real OceanBase version, not MySQL or SeekDB", pytrace=False + ) + if create: + evidence = {"backend": "oceanbase", "version": version, "engine_version": engine_version} + request.node.user_properties.append(("backend_identity", evidence)) + print(json.dumps(evidence, ensure_ascii=False)) await connection.exec_driver_sql( f"CREATE DATABASE `{database}`" if create else f"DROP DATABASE `{database}`" ) diff --git a/tests/e2e/test_atomic_memory_evidence_neutral_recall.py b/tests/e2e/test_atomic_memory_evidence_neutral_recall.py new file mode 100644 index 0000000000..c3ad6ee50b --- /dev/null +++ b/tests/e2e/test_atomic_memory_evidence_neutral_recall.py @@ -0,0 +1,131 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Accepted Source declarations do not alter recall or consumer revision citations.""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import replace +from typing import cast + +import httpx +import pytest + +from powercontext.builtin.artifacts.handoff import PrepareHandoff +from powercontext.builtin.runtime import ( + BuiltinConfig, + HandoffArtifactCitation, + PrepareContextRequest, + RuntimeConfig, + open_builtin_contexts, + open_builtin_runtime, +) +from powercontext.builtin.scope import ScopeDraft +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION +from powercontext.server.app import ServerApplication, create_app +from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceRef +from powercontext.sources.evidence import MemoryEvidenceDeclaration +from tests.builtin.persistence.contract import NoteAdapter, NoteInput +from tests.e2e.atomic_memory_migration_backend import MigrationBackend +from tests.e2e.test_atomic_memory_source_evidence import _commit +from tests.e2e.test_handoff_runtime import _EchoHandoffPipeline + +pytestmark = pytest.mark.parametrize("migration_backend", ["sqlite", "oceanbase"], indirect=True) + + +def test_source_declarations_preserve_recall_and_consumers(migration_backend: MigrationBackend) -> None: + async def scenario() -> None: + config = BuiltinConfig( + database=migration_backend.config, + runtime=RuntimeConfig(artifact_processing_families=(), atomic_memory_related_mode="fts"), + ) + arms = [] + neutral = MemoryEvidenceDeclaration() + candidates = ( + (SourceRef(source_type="note", source_id="note"), "memory-a", "alpha acceptance"), + (SourceRef(source_type="content", source_id="source"), "memory-b", "alpha guidance"), + ) + for declaration in ( + neutral, + MemoryEvidenceDeclaration(authority="system_attested", verification="verified"), + ): + registry = SourceDefinitionRegistry(( + replace(CONTENT_SOURCE_DEFINITION, memory_evidence=declaration), + AdapterSourceDefinition(NoteAdapter()), + )) + async with open_builtin_runtime( + config, + scheduler_path=migration_backend.directory / "consumer-scheduler.db", + source_registry=registry, + handoff_pipeline=_EchoHandoffPipeline(), + ) as runtime: + assert runtime.scopes is not None and runtime.atomic_memory is not None + scope = await runtime.scopes.create( + ScopeDraft( + title="Recall", + summary="Evidence neutrality.", + idempotency_key=declaration.authority, + ) + ) + scope_id = scope.scope_id + async with open_builtin_contexts(config, source_registry=registry) as contexts: + await contexts.records.capture_source(scope_id, "content", "source", "alpha guidance", {}) + context = await contexts.get(scope_id) + note = await context.sources.resolve(NoteInput(note_id="note", body="alpha acceptance")) + await context.sources.add(note) + memories = [] + for source, artifact_id, text in candidates: + memory = await _commit(contexts, scope_id, artifact_id, text, sources=(source,)) + annotation = await contexts.atomic_memory.for_scope(scope_id).evidence(artifact_id, revision=1) + assert annotation.artifact_ref == memory.ref + expected = declaration if source.source_type == "content" else neutral + assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( + (source, expected), + ) + memories.append(memory) + page = await runtime.atomic_memory.for_scope(scope_id).search("alpha", mode="text", limit=2) + assert tuple((hit.hit.artifact_ref, hit.text, hit.matched_by) for hit in page.hits) == tuple( + (memory.ref, text, ("text",)) for memory, (_, _, text) in zip(memories, candidates, strict=True) + ) + recalled = tuple((hit.hit.artifact_ref.model_dump(mode="json"), hit.text) for hit in page.hits) + app = create_app(application=cast(ServerApplication, runtime)) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://testserver" + ) as client: + response = await client.post("/v1/memory/search", json={"scope_id": scope_id, "query": "alpha"}) + assert response.status_code == 200, response.text + public = response.json() + assert ( + tuple((item["memory"]["artifact"], item["memory"]["text"]) for item in public["hits"]) == recalled + ) + prepared = await runtime.context.for_scope(scope_id).prepare(PrepareContextRequest(query="alpha")) + assert prepared.status == "ready" and prepared.content is not None + items = json.loads(prepared.content.splitlines()[-2])["items"] + assert tuple((item["citation"]["artifact_ref"], item["content"]) for item in items) == recalled + citation = HandoffArtifactCitation(artifact_ref=page.hits[-1].hit.artifact_ref) + handoffs = runtime.handoff.for_scope(scope_id) + finalized = await handoffs.finalize( + await handoffs.prepare( + PrepareHandoff(objective="Keep the recalled revision.", evidence=(citation,)) + ) + ) + assert finalized.content.state[0].citations == (citation,) + assert finalized.content.next_action is not None + assert finalized.content.next_action.citations == (citation,) + arms.append((page.hits, public, prepared.content.replace(scope_id, "scope"), finalized.content)) + assert arms[0] == arms[1] + + asyncio.run(scenario()) diff --git a/tests/e2e/test_atomic_memory_projection_rebuild.py b/tests/e2e/test_atomic_memory_projection_rebuild.py index 3075699659..5263ff1a0c 100644 --- a/tests/e2e/test_atomic_memory_projection_rebuild.py +++ b/tests/e2e/test_atomic_memory_projection_rebuild.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +from dataclasses import replace from pathlib import Path import pytest @@ -29,6 +30,9 @@ from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION +from powercontext.sources import SourceDefinitionRegistry +from powercontext.sources.evidence import MemoryEvidenceDeclaration from tests.e2e.atomic_memory_migration_fixture import SCOPE, _atomic, _seed_and_migrate from tests.legacy_memory import add_legacy_citation_columns @@ -36,6 +40,12 @@ @pytest.mark.parametrize("retention", ["complete", "archive_only", "entries_only", "removed", "unreadable"]) def test_rebuild_uses_current_authority_independently_of_legacy_retention(tmp_path: Path, retention: str) -> None: config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'projection.db'}") + strong_registry = SourceDefinitionRegistry(( + replace( + CONTENT_SOURCE_DEFINITION, + memory_evidence=MemoryEvidenceDeclaration(authority="system_attested", verification="verified"), + ), + )) async def scenario() -> None: await _seed_and_migrate(config) @@ -43,6 +53,8 @@ async def scenario() -> None: memory = contexts.atomic_memory.for_scope(SCOPE) current = await memory.get(_atomic("alpha", 2).artifact_id) historical = await memory.get(_atomic("alpha", 1).artifact_id, revision=1) + annotation = await memory.evidence(historical.ref.artifact_id, revision=1) + assert all(source.declaration == MemoryEvidenceDeclaration() for source in annotation.sources) async with SQLiteProfile.open(config, tables=()) as profile: async with profile.database.transaction() as connection: if retention in {"entries_only", "removed"}: @@ -63,10 +75,13 @@ async def scenario() -> None: ) assert report.ready, report.errors assert report.counts["rebuilt_rows"] == 2 - async with open_builtin_contexts(BuiltinConfig(database=config)) as contexts: + assert report.counts["historical_revisions"] == 3 + assert report.counts["neutral_sources"] == 4 + async with open_builtin_contexts(BuiltinConfig(database=config), source_registry=strong_registry) as contexts: memory = contexts.atomic_memory.for_scope(SCOPE) assert await memory.get(current.ref.artifact_id) == current assert await memory.get(historical.ref.artifact_id, revision=1) == historical + assert await memory.evidence(historical.ref.artifact_id, revision=1) == annotation result = await memory.search("Tasks A and C", mode="text") assert current.ref in tuple(hit.hit.artifact_ref for hit in result.hits) diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py new file mode 100644 index 0000000000..5d674241d1 --- /dev/null +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -0,0 +1,416 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Versioned source evidence acceptance on disposable SQLite and OceanBase.""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import replace +from typing import cast + +import pytest +from pydantic import Field, field_serializer, field_validator +from sqlalchemy import event, text, update +from sqlalchemy.exc import SQLAlchemyError + +from powercontext.artifacts import ArtifactLineage +from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent +from powercontext.builtin.artifacts.atomic_memory.extraction import ( + AtomicMemoryCandidate, + AtomicMemoryExtractionOutput, +) +from powercontext.builtin.inference import GenerationResult +from powercontext.builtin.persistence.tables import SOURCES_TABLE +from powercontext.builtin.runtime import ( + BuiltinConfig, + SubmitSourceObservation, + open_builtin_contexts, + open_builtin_runtime, +) +from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection +from powercontext.builtin.runtime.relational import RelationalContexts +from powercontext.builtin.scope import ScopeDraft +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION +from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceMaterialization, SourceRef +from powercontext.sources.evidence import MemoryEvidenceDeclaration +from powercontext.sources.observations import manifest_for_definition, project_source_for_transport +from tests.builtin.persistence.contract import NoteAdapter, NoteInput, NoteSource +from tests.e2e.atomic_memory_migration_backend import MigrationBackend +from tests.e2e.dream_support import atomic_memory_pipeline + +pytestmark = pytest.mark.parametrize("migration_backend", ["sqlite", "oceanbase"], indirect=True) + + +async def _commit(contexts, scope_id, artifact_id, text_value, *, sources=(), artifacts=(), expected_revision=None): + application = contexts.atomic_memory + async with contexts.database.transaction() as connection: + plan = await application.service.inspect_change( + connection, + scope_id, + artifact_id, + AtomicMemoryContent(kind="fact", text=text_value), + None, + expected_revision=expected_revision, + lineage=ArtifactLineage(sources=sources, artifacts=artifacts), + ) + prepared = await application.service.prepare_change(plan) + async with contexts.database.transaction() as connection: + result = await application.service.commit(connection, prepared, None) + return result.primary + + +@pytest.mark.parametrize("declared", [False, True]) +def test_unverifiable_native_serializer_keeps_memory_evidence_neutral(migration_backend, declared) -> None: + class SerializedNote(NoteSource): + count: int = 12 + + @field_serializer("count") + def serialize_count(self, value): + return str(value) + + @field_validator("count", mode="before") + @classmethod + def parse_count(cls, value): + return int(value) + + class SerializedAdapter(NoteAdapter): + source_class = SerializedNote + + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializedAdapter(), memory_evidence=declaration if declared else None), + )) + config = BuiltinConfig(database=migration_backend.config) + + async def scenario(): + async with open_builtin_contexts(config, source_registry=registry) as contexts: + context = await contexts.get("scope") + source = SerializedNote(name="serialized", materialization=SourceMaterialization.CAPTURED, body="Retained.") + await context.sources.add(source) + assert await context.sources.get(source) == source + assert await context.sources.add(source) == source + ref = SourceRef(source_type="note", source_id=source.name) + record = await _commit(contexts, "scope", "serialized", "Retained fact.", sources=(ref,)) + evidence = await contexts.atomic_memory.for_scope("scope").evidence("serialized", revision=1) + assert evidence.artifact_ref == record.ref + assert evidence.sources[0].source_ref == ref + assert evidence.sources[0].declaration == MemoryEvidenceDeclaration() + assert evidence.sources[0].manifest_fingerprint is None + if declared: + # Retain the binding shape accepted before schema compatibility checks. + async with contexts.database.transaction() as connection: + await connection.execute( + text( + "UPDATE pc_source_evidence_bindings SET manifest_fingerprint = " + "(SELECT fingerprint FROM pc_source_definition_manifests WHERE definition_name = 'note'), " + "declaration = :declaration WHERE scope_id = 'scope' AND source_id = 'serialized'" + ), + {"declaration": declaration.model_dump_json().encode()}, + ) + await _commit(contexts, "scope", "legacy", "Previously accepted fact.", sources=(ref,)) + legacy = await contexts.atomic_memory.for_scope("scope").evidence("legacy", revision=1) + assert legacy.sources == evidence.sources + async with open_builtin_contexts(config, source_registry=SourceDefinitionRegistry(())) as contexts: + report = await rebuild_atomic_memory_projection( + contexts.database, + contexts.atomic_memory.index, + maintenance_confirmed=True, + sources=contexts.repositories.sources, + ) + assert report.ready, report.errors + assert await contexts.atomic_memory.for_scope("scope").evidence("serialized", revision=1) == evidence + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("has_header", [False, True]) +@pytest.mark.parametrize("historical_version", ["1", "2"]) +def test_canonical_historical_evidence_survives_business_alias_rebuild( + migration_backend: MigrationBackend, has_header: bool, historical_version: str +) -> None: + class CurrentNote(NoteSource): + name: str = Field(alias="source_name") + label: str = Field(alias="name") + + class CurrentAdapter(NoteAdapter): + source_class = CurrentNote + + config = BuiltinConfig(database=migration_backend.config) + old_registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter(), version=historical_version),)) + current_registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(CurrentAdapter(), version=str(int(historical_version) + 1)), + )) + + async def scenario(): + source = NoteSource( + name=f"canonical-v{historical_version}", + definition_version=historical_version, + materialization=SourceMaterialization.CAPTURED, + body="Retained.", + ) + ref = SourceRef(source_type="note", source_id=source.name) + async with open_builtin_contexts(config, source_registry=old_registry) as contexts: + context = await contexts.get("scope") + await context.sources.add(source) + await _commit(contexts, "scope", "canonical", "Canonical historical fact.", sources=(ref,)) + material = source.model_dump(mode="json", by_alias=True) + payload_value = { + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + } + if has_header: + payload_value.update( + source=source.model_dump( + mode="json", + by_alias=False, + include={"name", "definition_version", "materialization"}, + ), + source_keys={ + "name": "name", + "definition_version": "definition_version", + "materialization": "materialization", + }, + ) + payload = json.dumps(payload_value).encode() + async with contexts.database.transaction() as connection: + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + + async with open_builtin_contexts(config, source_registry=current_registry) as contexts: + before = await contexts.atomic_memory.for_scope("scope").evidence("canonical", revision=1) + assert before.sources[0].source_ref == ref + assert before.sources[0].declaration == MemoryEvidenceDeclaration() + report = await rebuild_atomic_memory_projection( + contexts.database, + contexts.atomic_memory.index, + maintenance_confirmed=True, + sources=contexts.repositories.sources, + ) + assert report.ready, report.errors + after = await contexts.atomic_memory.for_scope("scope").evidence("canonical", revision=1) + assert after == before + + asyncio.run(scenario()) + + +def test_mixed_evidence_survives_failed_revision_restart_and_rebuild(migration_backend: MigrationBackend) -> None: + strong = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + asserted = MemoryEvidenceDeclaration(authority="user_asserted", verification="not_verified") + remote = MemoryEvidenceDeclaration(authority="system_attested", verification="verified") + + class RemoteNoteAdapter(NoteAdapter): + name = "remote-note" + + worker = SourceDefinitionRegistry((AdapterSourceDefinition(RemoteNoteAdapter(), memory_evidence=remote),)) + manifest = manifest_for_definition(worker.definition_for_name("remote-note")) + registry = SourceDefinitionRegistry( + ( + replace(CONTENT_SOURCE_DEFINITION, memory_evidence=strong), + AdapterSourceDefinition(NoteAdapter(), memory_evidence=asserted), + ), + remote_declarations={(manifest.name, manifest.version, manifest.fingerprint): remote}, + ) + config = BuiltinConfig(database=migration_backend.config) + refs = ( + SourceRef(source_type="content", source_id="cafe\u0301"), + SourceRef(source_type="note", source_id="native"), + SourceRef(source_type="remote-note", source_id="remote"), + ) + + async def scenario() -> None: + async with open_builtin_contexts(config, source_registry=registry) as contexts: + context = await contexts.get("scope") + await contexts.records.capture_source( + "scope", "content", "cafe\u0301", "Release received.", {"authority": "untrusted"} + ) + native = await context.sources.resolve(NoteInput(note_id="native", body="User correction.")) + await context.sources.add(native) + source = await worker.resolve(NoteInput(note_id="remote", body="Remote release evidence.")) + await contexts.register_source_definition(manifest) + await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope", observation=project_source_for_transport(worker, source)) + ) + original = await _commit(contexts, "scope", "rule", "Release rule one.", sources=refs) + memory = contexts.atomic_memory.for_scope("scope") + annotation = await memory.evidence("rule", revision=1) + assert annotation.artifact_ref == original.ref + assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( + (refs[0], strong), + (refs[1], asserted), + (refs[2], remote), + ) + + def reject_snapshot(connection, cursor, statement, *args): + if statement.lstrip().upper().startswith("INSERT INTO PC_ATOMIC_MEMORY_EVIDENCE_HISTORY"): + raise SQLAlchemyError(statement) + + engine = contexts.database.engine.sync_engine + event.listen(engine, "before_cursor_execute", reject_snapshot) + try: + with pytest.raises(SQLAlchemyError): + await _commit(contexts, "scope", "rule", "Failed revision.", expected_revision=1) + finally: + event.remove(engine, "before_cursor_execute", reject_snapshot) + assert await memory.get("rule") == original + revised = await _commit(contexts, "scope", "rule", "Rule two.", sources=(refs[2],), expected_revision=1) + current = await memory.evidence("rule", revision=2) + assert current.sources == (annotation.sources[2],) + neutral_registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION, AdapterSourceDefinition(NoteAdapter()))) + async with open_builtin_contexts(config, source_registry=neutral_registry) as contexts: + await contexts.records.capture_source( + "other", refs[0].source_type, refs[0].source_id, "Other Scope release.", {} + ) + other = await _commit(contexts, "other", "rule", "Other Scope rule.", sources=(refs[0],)) + other_annotation = await contexts.atomic_memory.for_scope("other").evidence("rule", revision=1) + assert other_annotation.scope_id == "other" + assert tuple((item.source_ref, item.declaration) for item in other_annotation.sources) == ( + (refs[0], MemoryEvidenceDeclaration()), + ) + assert other_annotation.sources[0].material_digest != annotation.sources[0].material_digest + + async def assert_retained(contexts, historical): + memory = contexts.atomic_memory.for_scope("scope") + assert await memory.evidence("rule", revision=1) == historical + assert await memory.evidence("rule", revision=2) == current + assert (await memory.get("rule", revision=1)).artifact == original.artifact + assert (await memory.get("rule")).artifact == revised.artifact + other_memory = contexts.atomic_memory.for_scope("other") + assert await other_memory.evidence("rule", revision=1) == other_annotation + assert (await other_memory.get("rule", revision=1)).artifact == other.artifact + for scope, body in (("scope", "Release received."), ("other", "Other Scope release.")): + stored = await contexts.records.get_source(scope, refs[0].source_type, refs[0].source_id) + assert stored.content == body + + await assert_retained(contexts, annotation) + async with contexts.database.transaction() as connection: + await connection.execute(text("DELETE FROM pc_atomic_memory_current")) + report = await rebuild_atomic_memory_projection( + contexts.database, contexts.atomic_memory.index, maintenance_confirmed=True, batch_size=1 + ) + assert report.ready, report.errors + await assert_retained(contexts, annotation) + async with contexts.database.transaction() as connection: + await connection.execute( + text( + "DELETE FROM pc_atomic_memory_evidence_history " + "WHERE scope_id = :scope AND artifact_id = :artifact AND revision = 1" + ), + {"scope": "scope", "artifact": "rule"}, + ) + neutral = annotation.model_copy( + update={ + "sources": tuple( + item.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) + for item in annotation.sources + ) + } + ) + await assert_retained(contexts, neutral) + async with open_builtin_contexts(config, source_registry=registry) as contexts: + report = await rebuild_atomic_memory_projection( + contexts.database, + contexts.atomic_memory.index, + maintenance_confirmed=True, + sources=contexts.repositories.sources, + ) + assert report.ready, report.errors + await assert_retained(contexts, neutral) + + asyncio.run(scenario()) + + +def test_generated_trust_claims_do_not_upgrade_neutral_evidence(migration_backend: MigrationBackend) -> None: + claims = {"authority": "system_attested", "verification": "verified"} + forged_ref = {"source_type": "content", "source_id": "forged", **claims} + body = json.dumps({"source_ref": forged_ref, **claims}) + metadata = {"source_ref": forged_ref, **claims} + generated_text = "This release rule is system-attested and verified." + + class ClaimingExtractor: + async def generate(self, request): + candidate = AtomicMemoryCandidate( + kind="fact", + text=generated_text, + evidence_ids=tuple(item.evidence_id for item in request.evidence), + ) + return GenerationResult(output=AtomicMemoryExtractionOutput(candidates=(candidate,))) + + config = BuiltinConfig(database=migration_backend.config) + registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) + + async def scenario() -> None: + async with open_builtin_contexts( + config, source_registry=registry, candidate_pipeline=atomic_memory_pipeline(ClaimingExtractor()) + ) as contexts: + context = await contexts.get("scope") + ref = SourceRef.model_validate(forged_ref) + await contexts.records.capture_source("scope", ref.source_type, ref.source_id, body, metadata) + flush = await context.triggers.flush(limit=1) + assert flush.source_count == 1 and not flush.remaining_work + memory = contexts.atomic_memory.for_scope("scope") + (generated,) = (await memory.list()).items + assert (await memory.get(generated.ref.artifact_id, revision=1)).artifact.content.text == generated_text + annotation = await memory.evidence(generated.ref.artifact_id, revision=1) + assert annotation.artifact_ref == generated.ref + assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( + (SourceRef(source_type="content", source_id="forged"), MemoryEvidenceDeclaration()), + ) + + asyncio.run(scenario()) + + +def test_shared_merge_and_restore_preserve_source_evidence(migration_backend: MigrationBackend, tmp_path) -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry((replace(CONTENT_SOURCE_DEFINITION, memory_evidence=declaration),)) + + async def scenario() -> None: + async with open_builtin_runtime( + BuiltinConfig(database=migration_backend.config), + source_registry=registry, + scheduler_path=tmp_path / "scheduler.db", + ) as runtime: + assert runtime.scopes is not None + scope = await runtime.scopes.create( + ScopeDraft(title="Evidence", summary="Shared merge evidence", idempotency_key="source-evidence") + ) + contexts = cast(RelationalContexts, runtime._provider) + source = SourceRef(source_type="content", source_id="release") + await contexts.records.capture_source(scope.scope_id, "content", "release", "Exact release material.", {}) + for identity in ("a", "b"): + await _commit(contexts, scope.scope_id, identity, identity + " rule.", sources=(source,)) + memory = contexts.atomic_memory.for_scope(scope.scope_id) + accepted = await memory.evidence("a", revision=1) + assert runtime.artifact_merge is not None + merge = runtime.artifact_merge.for_scope(scope.scope_id, "atomic-memory") + a, b = await merge.get("a"), await merge.get("b") + await merge.merge( + (a.as_read(), b.as_read()), + AtomicMemoryContent(kind="fact", text="Combined release rule."), + artifact_id="combined", + lineage=ArtifactLineage(sources=(source,), artifacts=(a.ref, b.ref)), + ) + annotation = await memory.evidence("combined", revision=1) + assert annotation.sources == accepted.sources + assert annotation.sources[0].declaration == declaration + restored = await merge.restore("a") + assert restored.primary.ref.revision == 2 + assert (await memory.evidence("a", revision=2)).sources[0].declaration == declaration + assert await memory.evidence("combined", revision=1) == annotation + assert await memory.evidence("a", revision=1) == accepted + + asyncio.run(scenario()) diff --git a/tests/e2e/test_observability.py b/tests/e2e/test_observability.py index 94105b532d..2f78ef22c8 100644 --- a/tests/e2e/test_observability.py +++ b/tests/e2e/test_observability.py @@ -50,6 +50,7 @@ ) from powercontext.builtin.inference import GenerationResult from powercontext.builtin.inference.pydantic_ai import PydanticAIEmbeddingModel +from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceRepository from powercontext.builtin.persistence.atomic_memory_index import ( AtomicMemoryIndexCapabilities, AtomicMemorySearchChannels, @@ -239,11 +240,17 @@ class _VectorMemoryIndex: tables = () def __init__(self, profile=None): - pass + self._evidence = AtomicMemoryEvidenceRepository() async def initialize(self, connection: AsyncConnection, /): pass + async def save_evidence(self, connection, scope_id, artifact_ref, sources, /): + return await self._evidence.save(connection, scope_id, artifact_ref, sources) + + async def read_evidence(self, connection, scope_id, artifact_ref, /): + return await self._evidence.read(connection, scope_id, artifact_ref) + async def validate_current(self, connection: AsyncConnection, /): pass diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index a4101aace3..a26f66eb65 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -34,7 +34,7 @@ from sqlalchemy import create_engine, event, insert, select, update from sqlalchemy.engine import make_url -from powercontext.artifacts import ArtifactAddress, ArtifactRef, StableHandle +from powercontext.artifacts import ArtifactAddress, ArtifactLineage, ArtifactRef, StableHandle from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent from powercontext.builtin.artifacts.atomic_memory.errors import InvalidAtomicMemoryStateError from powercontext.builtin.artifacts.experience import ExperienceContent @@ -83,6 +83,7 @@ from powercontext.limits import MAX_ARTIFACT_ID_LENGTH from powercontext.sources import ( AdapterSourceDefinition, + MemoryEvidenceDeclaration, SourceDefinitionRegistry, SourceRef, manifest_for_definition, @@ -93,6 +94,54 @@ from tests.e2e.atomic_memory_migration_fixture import _seed_and_migrate +def test_archive_preserves_accepted_native_evidence(tmp_path: Path) -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter(), memory_evidence=declaration),)) + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "evidence.pcb" + source_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'source.db'}")) + target_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + async with open_builtin_contexts(source_config, source_registry=registry) as source: + scope = await source.scopes.create( + ScopeDraft(title="Evidence", summary="Portable evidence", idempotency_key="evidence") + ) + context = await source.get(scope.scope_id) + native = await context.sources.resolve(NoteInput(note_id="native", body="Retained evidence.")) + await context.sources.add(native) + ref = SourceRef(source_type="note", source_id="native") + service = source.atomic_memory.service + async with source.database.transaction() as connection: + plan = await service.inspect_change( + connection, + scope.scope_id, + "rule", + AtomicMemoryContent(kind="fact", text="Preserve evidence."), + None, + lineage=ArtifactLineage(sources=(ref,)), + ) + prepared = await service.prepare_change(plan) + async with source.database.transaction() as connection: + await service.commit(connection, prepared, None) + expected = await source.atomic_memory.for_scope(scope.scope_id).evidence("rule", revision=1) + assert expected.sources[0].declaration == declaration + await source.portability.export([scope.scope_id], archive, authorize=authorize) + inspection = await source.portability.inspect(archive) + assert inspection.records_by_type["source_evidence_binding"] == 1 + assert inspection.records_by_type["atomic_memory_evidence_history"] == 1 + assert inspection.records_by_type["source_definition"] == 1 + async with open_builtin_contexts(target_config, source_registry=registry) as target: + await target.portability.restore(archive) + async with open_builtin_contexts(target_config, source_registry=registry) as target: + actual = await target.atomic_memory.for_scope(scope.scope_id).evidence("rule", revision=1) + assert actual == expected + + asyncio.run(scenario()) + + def test_topic_restore_can_retry_after_projection_failure_and_restart(tmp_path: Path) -> None: async def authorize(_scopes: tuple[str, ...]) -> None: pass diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index b4933a7934..12e80e8101 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -15,20 +15,32 @@ from __future__ import annotations import asyncio +import hashlib import urllib.request +from decimal import Decimal +from typing import Self import pytest -from pydantic import ValidationError +import rfc8785 +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_serializer +from pydantic.alias_generators import to_camel from referencing.exceptions import Unresolvable +from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import SHARED_TABLES +from powercontext.builtin.runtime import SubmitSourceObservation +from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION, ContentCapture from powercontext.builtin.sources.observation_validation import json_schema_validator from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, + AdapterSourceDefinition, Source, SourceCatalog, SourceDefinitionManifest, SourceDefinitionRegistry, + SourceMaterialization, SourceObservation, TextEvidence, manifest_for_definition, @@ -75,6 +87,366 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize( + ("name_alias", "body_alias"), + [("source_name", "wire_text"), ("body", "wire_text"), ("source_name", "name"), ("wire_text", "wire_text")], +) +def test_serialization_aliases_preserve_the_worker_envelope_during_ingestion( + name_alias: str, body_alias: str, tmp_path +) -> None: + class AliasedSource(Source): + name: str = Field(serialization_alias=name_alias) + definition_version: str = Field(default="1", serialization_alias="wire_version") + materialization: SourceMaterialization = Field(serialization_alias="wire_materialization") + description: str | None = Field(default=None, serialization_alias="wire_description") + body: str = Field(serialization_alias=body_alias) + + class AliasedAdapter: + name = "aliased" + input_class = str + source_class = AliasedSource + + async def resolve(self, value: str, /) -> AliasedSource: + return AliasedSource( + name="remote-aliased", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + description="Aliased description.", + body=value, + ) + + async def read(self, source: AliasedSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(AliasedAdapter(), version="2") + worker = SourceDefinitionRegistry((definition,)) + source = await worker.resolve("Retain the aliased business value.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(worker, source) + expected = { + "name": "remote-aliased", + "definition_version": "2", + "materialization": "captured", + "description": "Aliased description.", + "body": "Retain the aliased business value.", + } + assert manifest.source_schema == AliasedSource.model_json_schema() + baseline_declaration = { + "name": definition.name, + "version": definition.version, + "source_schema": AliasedSource.model_json_schema(), + "projections": [], + } + baseline_fingerprint = f"sha256:{hashlib.sha256(rfc8785.dumps(baseline_declaration)).hexdigest()}" + assert manifest.fingerprint == baseline_fingerprint + assert observation.payload == expected + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'remote-manifest.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + + current_manifest = manifest_for_definition(definition) + assert current_manifest == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(current_manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + assert receipt.source_ref.source_id == "remote-aliased" + assert isinstance(stored, SourceObservation) + assert stored.payload == expected + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as native_profile: + async with native_profile.database.transaction() as connection: + native_manifest = await SourceDefinitionManifestRepository().register_native(connection, definition) + assert native_manifest is not None + assert native_manifest.source_schema == AliasedSource.model_json_schema(mode="serialization") + assert native_manifest.fingerprint != manifest.fingerprint + + asyncio.run(scenario()) + + +def test_alias_serialization_remote_manifest_and_payload_remain_upgrade_stable(tmp_path) -> None: + class AliasedSource(Source): + model_config = ConfigDict(serialize_by_alias=True, populate_by_name=True) + body: str = Field(alias="wire_text") + + class AliasedAdapter: + name = "aliased-upgrade" + input_class = str + source_class = AliasedSource + + async def resolve(self, value: str, /) -> AliasedSource: + return AliasedSource( + name="remote-aliased", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + wire_text=value, + ) + + async def read(self, source: AliasedSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(AliasedAdapter(), version="2") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retained alias.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + expected = AliasedSource.model_json_schema() + expected_fingerprint = hashlib.sha256( + rfc8785.dumps({ + "name": definition.name, + "version": definition.version, + "source_schema": expected, + "projections": [], + }) + ).hexdigest() + assert manifest.source_schema == expected + assert manifest.fingerprint == f"sha256:{expected_fingerprint}" + assert observation.payload["wire_text"] == "Retained alias." + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'remote-alias-upgrade.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "remote-aliased" + + asyncio.run(scenario()) + + +def test_optional_alias_keeps_the_preexisting_manifest_fingerprint(tmp_path) -> None: + class VersionedSource(Source): + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + body: str + + class VersionedAdapter: + name = "versioned-source" + input_class = str + source_class = VersionedSource + + async def resolve(self, value: str, /) -> VersionedSource: + return VersionedSource( + name="versioned", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body=value, + ) + + async def read(self, source: VersionedSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(VersionedAdapter(), version="2") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retained.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + baseline = { + "name": definition.name, + "version": definition.version, + "source_schema": VersionedSource.model_json_schema(), + "projections": [], + } + assert manifest.source_schema == VersionedSource.model_json_schema() + assert manifest.fingerprint == f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" + assert observation.payload["definition_version"] == "2" + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'optional-alias.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "versioned" + + asyncio.run(scenario()) + + +def test_nullable_nested_alias_keeps_the_preexisting_manifest_fingerprint(tmp_path) -> None: + class Detail(BaseModel): + label: str = Field(alias="wire_label") + + class NullableSource(Source): + detail: Detail | None = None + + class NullableAdapter: + name = "nullable-nested" + input_class = str + source_class = NullableSource + + async def resolve(self, value: str, /) -> NullableSource: + return NullableSource( + name="nullable", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + detail=None, + ) + + async def read(self, source: NullableSource, /) -> str: + return source.name + + async def scenario() -> None: + definition = AdapterSourceDefinition(NullableAdapter(), version="2") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retained.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + baseline = { + "name": definition.name, + "version": definition.version, + "source_schema": NullableSource.model_json_schema(), + "projections": [], + } + assert manifest.source_schema == NullableSource.model_json_schema() + assert manifest.fingerprint == f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" + assert observation.payload["detail"] is None + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'nullable-nested.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "nullable" + + asyncio.run(scenario()) + + +def test_decimal_source_manifest_fingerprint_survives_upgrade(tmp_path) -> None: + class AmountSource(Source): + amount: Decimal + + class AmountAdapter: + name = "amount-source" + input_class = str + source_class = AmountSource + + async def resolve(self, value: str, /) -> AmountSource: + return AmountSource( + name="amount", + definition_version="1", + materialization=SourceMaterialization.CAPTURED, + amount=Decimal(value), + ) + + async def read(self, source: AmountSource, /) -> str: + return str(source.amount) + + async def scenario() -> None: + definition = AdapterSourceDefinition(AmountAdapter(), version="1") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("1.25") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + baseline = { + "name": definition.name, + "version": definition.version, + "source_schema": AmountSource.model_json_schema(), + "projections": [], + } + baseline_fingerprint = f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" + assert manifest.source_schema == AmountSource.model_json_schema() + assert manifest.fingerprint == baseline_fingerprint + assert observation.payload["amount"] == "1.25" + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'decimal-upgrade.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "amount" + + asyncio.run(scenario()) + + +def test_recursive_source_transport_preserves_canonical_nested_payload() -> None: + class RecursiveSource(Source): + name: str = Field(serialization_alias="source_name") + definition_version: str = Field(default="1", serialization_alias="wire_version") + body: str = Field(serialization_alias="wire_text") + children: list[Self] = Field(default_factory=list) + + class RecursiveAdapter: + name = "recursive" + input_class = str + source_class = RecursiveSource + + async def resolve(self, value: str, /) -> RecursiveSource: + leaf = RecursiveSource(name="leaf", materialization=SourceMaterialization.CAPTURED, body="Nested text.") + return RecursiveSource( + name="root", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body=value, + children=[leaf], + ) + + async def read(self, source: RecursiveSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(RecursiveAdapter(), version="2") + worker = SourceDefinitionRegistry((definition,)) + source = await worker.resolve("Root text.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(worker, source) + expected = { + "name": "root", + "definition_version": "2", + "materialization": "captured", + "description": None, + "body": "Root text.", + "children": [ + { + "name": "leaf", + "definition_version": "1", + "materialization": "captured", + "description": None, + "body": "Nested text.", + "children": [], + } + ], + } + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + assert receipt.source_ref.source_id == "root" + assert isinstance(stored, SourceObservation) + assert stored.payload == expected + + asyncio.run(scenario()) + + def test_remote_source_observation_requires_captured_materialization() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) @@ -87,6 +459,40 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_remote_identity_exclusions_and_serializers_fail_observation_validation() -> None: + class ExcludedIdentitySource(Source): + definition_version: str = Field(default="1", exclude=True) + + class TransformedIdentitySource(Source): + @field_serializer("name") + def encode_name(self, value: str) -> str: + return f"wire:{value}" + + class Adapter: + input_class = str + name = "remote-identity" + + def __init__(self, source_class: type[Source]) -> None: + self.source_class = source_class + + async def resolve(self, value: str, /) -> Source: + return self.source_class(name=value, materialization=SourceMaterialization.CAPTURED) + + async def read(self, source: Source, /) -> str: + return source.name + + async def scenario() -> None: + for source_class in (ExcludedIdentitySource, TransformedIdentitySource): + definition = AdapterSourceDefinition(Adapter(source_class), version="1") + registry = SourceDefinitionRegistry((definition,)) + assert manifest_for_definition(definition).source_schema == source_class.model_json_schema() + source = await registry.resolve("identity") + with pytest.raises(ValidationError): + project_source_for_transport(registry, source) + + asyncio.run(scenario()) + + def test_remote_schema_references_are_rejected_without_network_access(monkeypatch: pytest.MonkeyPatch) -> None: attempted = False