From 8439875a32a7de9b54b5a5dc4cfdc0fb85f98a33 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 14 Oct 2026 14:35:27 +0800 Subject: [PATCH 01/94] feat(memory): persist versioned source evidence Implement issue01 of the Source evidence declarations specification. Bind trusted native and remote declarations to exact Source material and retain per-revision Atomic Memory snapshots in the existing transaction. Preserve neutral fallback, exact identities, scope authorization and public retrieval contracts. Add versioned p0004 migrations and SQLite persistence regressions. Refs #1975. Full suite: 5255 passed, 431 skipped, 7 failed; introduced observability adapter failures fixed and its file passes 18 tests. Topic Memory vector failure also reproduces at the unchanged baseline. --- .../powercontext_connector_opendal/source.py | 2 + src/powercontext/__init__.py | 4 + .../artifacts/atomic_memory/service.py | 8 + .../persistence/atomic_memory_evidence.py | 161 ++++++++++++++++++ .../atomic_memory_evidence_schema.py | 59 +++++++ .../persistence/atomic_memory_index.py | 52 +++++- .../migrations/atomic_memory_evidence.py | 100 +++++++++++ .../versions/p0004_atomic_memory_evidence.py | 25 +++ .../persistence/migrations/mysql_schema.py | 6 +- .../migrations/resources/manifest.json | 17 +- .../schemas/atomic_memory_evidence_v1.json | 14 ++ .../versions/p0004_atomic_memory_evidence.py | 30 ++++ .../builtin/persistence/schema.py | 10 +- .../builtin/persistence/source_definitions.py | 35 +++- .../builtin/persistence/sources.py | 158 ++++++++++++++++- .../builtin/runtime/atomic_memory.py | 55 ++++++ .../builtin/runtime/composition.py | 14 +- src/powercontext/sources/__init__.py | 3 + src/powercontext/sources/definitions.py | 63 ++++++- src/powercontext/sources/evidence.py | 60 +++++++ 20 files changed, 860 insertions(+), 16 deletions(-) create mode 100644 src/powercontext/builtin/persistence/atomic_memory_evidence.py create mode 100644 src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py create mode 100644 src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py create mode 100644 src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0004_atomic_memory_evidence.py create mode 100644 src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json create mode 100644 src/powercontext/builtin/persistence/migrations/resources/versions/p0004_atomic_memory_evidence.py create mode 100644 src/powercontext/sources/evidence.py diff --git a/integrations/opendal/src/powercontext_connector_opendal/source.py b/integrations/opendal/src/powercontext_connector_opendal/source.py index 5ab48cf7d6..c089367cba 100644 --- a/integrations/opendal/src/powercontext_connector_opendal/source.py +++ b/integrations/opendal/src/powercontext_connector_opendal/source.py @@ -23,6 +23,7 @@ from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, + MemoryEvidenceDeclaration, Source, SourceMaterialization, SourceProjection, @@ -92,6 +93,7 @@ class TextFileSnapshotSourceDefinition: name = TEXT_FILE_SNAPSHOT_SOURCE_NAME version = "1" source_class = TextFileSnapshotSource + memory_evidence: MemoryEvidenceDeclaration | None = None def __init__(self) -> None: projection = cast(SourceProjection[TextFileSnapshotSource], TextFileEvidenceProjection()) diff --git a/src/powercontext/__init__.py b/src/powercontext/__init__.py index 7448a04133..f7589beabc 100644 --- a/src/powercontext/__init__.py +++ b/src/powercontext/__init__.py @@ -51,12 +51,14 @@ ) from powercontext.sources import ( AdapterSourceDefinition, + MemoryEvidenceDeclaration, Source, SourceAdapter, SourceCatalog, SourceCatalogBackend, SourceDefinition, SourceDefinitionRegistry, + SourceEvidence, SourceMaterialization, SourceProjection, SourceProjectionKey, @@ -90,6 +92,7 @@ "InvalidSourceProjectionError", "InvalidSourceReferenceError", "InvalidSourceResultError", + "MemoryEvidenceDeclaration", "PolicyTransition", "PowerContext", "PowerContextError", @@ -104,6 +107,7 @@ "SourceDefinitionNotFoundError", "SourceDefinitionRegistry", "SourceError", + "SourceEvidence", "SourceMaterialization", "SourceNotFoundError", "SourceProjection", diff --git a/src/powercontext/builtin/artifacts/atomic_memory/service.py b/src/powercontext/builtin/artifacts/atomic_memory/service.py index 0947035cd5..90fb984ef4 100644 --- a/src/powercontext/builtin/artifacts/atomic_memory/service.py +++ b/src/powercontext/builtin/artifacts/atomic_memory/service.py @@ -80,6 +80,9 @@ async def publish( prepared: PreparedAtomicMemoryProjection, ) -> None: ... async def remove(self, connection: AsyncConnection, scope_id: str, artifact_id: str) -> None: ... + async def capture_evidence( + self, connection: AsyncConnection, scope_id: str, record: AtomicMemoryRecord + ) -> None: ... ATOMIC_MERGE_ERRORS = ArtifactMergeErrors( @@ -478,6 +481,11 @@ async def commit_window( read_set=tuple(_common_read(read) for read in read_set), direct_source=direct_source, ) + for item, result in zip(prepared, results, strict=True): + revisions = {write.artifact_id for write in item.plan.writes if write.draft is not None} + for record in result.records: + if record.artifact.artifact_id in revisions: + await self.projections.capture_evidence(connection, scope_id, _atomic_record(record)) return tuple( AtomicMemoryMutationResult( changed=result.changed, diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/atomic_memory_evidence.py new file mode 100644 index 0000000000..cb054ff704 --- /dev/null +++ b/src/powercontext/builtin/persistence/atomic_memory_evidence.py @@ -0,0 +1,161 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Immutable per-revision evidence inputs and their neutral internal annotation.""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from typing import Literal + +import rfc8785 +from aiosqlite import Connection as SQLiteConnection +from pydantic import BaseModel, ConfigDict, ValidationError +from sqlalchemy import insert, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.artifacts import ArtifactRef +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE +from powercontext.builtin.persistence.sources import _require_identity +from powercontext.limits import MAX_SCOPE_ID_LENGTH +from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence + + +class AtomicMemoryEvidenceAnnotation(BaseModel): + """Exact revision facts, not a trust score or recursively inferred evidence.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + scope_id: str + artifact_ref: ArtifactRef + sources: tuple[SourceEvidence, ...] + format: Literal["powercontext.atomic-memory.evidence.v1"] = "powercontext.atomic-memory.evidence.v1" + + +class AtomicMemoryEvidenceRepository: + """Join caller transactions; retained inputs are never deleted with the index. + + The annotation is derived from retained inputs at read time. There is no + separate cache to confuse with accepted history or join into search. + """ + + async def save( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + sources: Sequence[SourceEvidence], + ) -> AtomicMemoryEvidenceAnnotation: + artifact_ref = self._reference(scope_id, artifact_ref) + payload = await self._payload(connection, scope_id, artifact_ref) + if payload is not None: + return self._decode(scope_id, artifact_ref, payload) or self._neutral(scope_id, artifact_ref, sources) + accepted = [] + for source in sources: + value = source.model_dump(mode="json") + try: + declaration = MemoryEvidenceDeclaration.model_validate(value.get("declaration")) + except ValidationError: + declaration = MemoryEvidenceDeclaration() + accepted.append(SourceEvidence.model_validate({**value, "declaration": declaration})) + annotation = AtomicMemoryEvidenceAnnotation( + scope_id=scope_id, artifact_ref=artifact_ref, sources=tuple(accepted) + ) + if connection.dialect.name == "sqlite": + driver = (await connection.get_raw_connection()).driver_connection + if isinstance(driver, SQLiteConnection) and not driver.in_transaction: + # SQLite legacy transaction mode does not BEGIN on SELECT; + # SAVEPOINT must not accidentally own and commit the write. + await connection.exec_driver_sql("BEGIN") + try: + async with connection.begin_nested(): + await connection.execute( + insert(ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE).values( + scope_id=scope_id, + artifact_id=artifact_ref.artifact_id, + revision=artifact_ref.revision, + sources=rfc8785.dumps([source.model_dump(mode="json") for source in annotation.sources]), + ) + ) + except IntegrityError: + payload = await self._payload(connection, scope_id, artifact_ref) + if payload is None: + raise + return self._decode(scope_id, artifact_ref, payload) or self._neutral(scope_id, artifact_ref, sources) + return annotation + + async def read( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef + ) -> AtomicMemoryEvidenceAnnotation | None: + artifact_ref = self._reference(scope_id, artifact_ref) + payload = await self._payload(connection, scope_id, artifact_ref) + return None if payload is None else self._decode(scope_id, artifact_ref, payload) + + @staticmethod + def _reference(scope_id: str, artifact_ref: ArtifactRef) -> ArtifactRef: + _require_identity("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) + reference = ArtifactRef.model_validate(artifact_ref.model_dump()) + if reference.family != "atomic-memory": + from powercontext.builtin.persistence.errors import InvalidRepositoryArgumentError + + raise InvalidRepositoryArgumentError("artifact_ref", "must reference an Atomic Memory revision") + return reference + + @staticmethod + async def _payload(connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef) -> object | None: + table = ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE + return await connection.scalar( + select(table.c.sources).where( + table.c.scope_id == scope_id, + table.c.artifact_id == artifact_ref.artifact_id, + table.c.revision == artifact_ref.revision, + ) + ) + + @staticmethod + def _decode(scope_id: str, artifact_ref: ArtifactRef, payload: object) -> AtomicMemoryEvidenceAnnotation | None: + try: + if not isinstance(payload, (bytes, bytearray, memoryview)): + return None + values = json.loads(bytes(payload)) + if not isinstance(values, list): + return None + sources = [] + for value in values: + if not isinstance(value, dict): + return None + try: + declaration = MemoryEvidenceDeclaration.model_validate(value.get("declaration")) + except ValidationError: + declaration = MemoryEvidenceDeclaration() + sources.append(SourceEvidence.model_validate({**value, "declaration": declaration})) + return AtomicMemoryEvidenceAnnotation(scope_id=scope_id, artifact_ref=artifact_ref, sources=tuple(sources)) + except (ValueError, TypeError): + # An unreadable declaration snapshot conveys no historical claim. + # This does not catch SQL errors or actual Source resolution errors. + return None + + @staticmethod + def _neutral( + scope_id: str, artifact_ref: ArtifactRef, sources: Sequence[SourceEvidence] + ) -> AtomicMemoryEvidenceAnnotation: + # Preserve a damaged row exactly, without replacing it with today's + # stronger configuration. A caller can only receive neutral facts. + return AtomicMemoryEvidenceAnnotation( + scope_id=scope_id, + artifact_ref=artifact_ref, + sources=tuple(source.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) for source in sources), + ) diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py b/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py new file mode 100644 index 0000000000..b4b446a20a --- /dev/null +++ b/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Retained exact-version evidence, independent of disposable search projections.""" + +from sqlalchemy import BigInteger, CheckConstraint, Column, LargeBinary, Table, Text +from sqlalchemy.dialects.mysql import LONGBLOB, MEDIUMTEXT + +from powercontext.builtin.persistence.tables import SHARED_METADATA, identity_string +from powercontext.limits import ( + MAX_ARTIFACT_ID_LENGTH, + MAX_SCOPE_ID_LENGTH, + MAX_SOURCE_ID_LENGTH, + MAX_SOURCE_TYPE_LENGTH, +) + +ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION = "p0004" +ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE = Table( + "pc_atomic_memory_evidence_schema", + SHARED_METADATA, + Column("schema_version", identity_string(32), primary_key=True), +) +ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE = Table( + "pc_atomic_memory_evidence_history", + SHARED_METADATA, + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), + Column("revision", BigInteger, primary_key=True), + Column("sources", LargeBinary().with_variant(LONGBLOB(), "mysql"), nullable=False), + CheckConstraint("revision > 0", name="ck_pc_atomic_memory_evidence_history_revision"), +) +SOURCE_EVIDENCE_BINDINGS_TABLE = Table( + "pc_source_evidence_bindings", + SHARED_METADATA, + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("source_type", identity_string(MAX_SOURCE_TYPE_LENGTH), primary_key=True), + Column("source_id", identity_string(MAX_SOURCE_ID_LENGTH), primary_key=True), + Column("material_digest", identity_string(71), nullable=False), + Column("definition_name", Text().with_variant(MEDIUMTEXT(), "mysql"), nullable=False), + Column("definition_version", Text().with_variant(MEDIUMTEXT(), "mysql"), nullable=False), + Column("manifest_fingerprint", identity_string(71)), + Column("declaration", LargeBinary().with_variant(LONGBLOB(), "mysql"), nullable=False), +) +ATOMIC_MEMORY_EVIDENCE_TABLES = ( + ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE, + ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, + SOURCE_EVIDENCE_BINDINGS_TABLE, +) diff --git a/src/powercontext/builtin/persistence/atomic_memory_index.py b/src/powercontext/builtin/persistence/atomic_memory_index.py index 4cb2cbb00a..d125a46861 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/atomic_memory_index.py @@ -18,6 +18,7 @@ import json import math +from collections.abc import Sequence from dataclasses import dataclass, replace from hashlib import sha256 from typing import Any, Literal, Protocol @@ -37,9 +38,16 @@ fts_query_requirements, ) from powercontext.builtin.inference import EmbeddingModel +from powercontext.builtin.persistence.atomic_memory_evidence import ( + AtomicMemoryEvidenceAnnotation, + AtomicMemoryEvidenceRepository, +) +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_PROJECTION_FORMAT +from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.tables import ARTIFACT_TAGS_TABLE from powercontext.builtin.tags import TagFilter +from powercontext.sources.evidence import SourceEvidence AtomicMemorySearchMode = Literal["fts", "vector", "hybrid"] ATOMIC_MEMORY_RRF_CONSTANT = 60 @@ -166,6 +174,19 @@ async def replace( async def delete(self, connection: AsyncConnection, scope_id: str, artifact_id: str, /) -> None: ... + async def save_evidence( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + sources: Sequence[SourceEvidence], + /, + ) -> AtomicMemoryEvidenceAnnotation: ... + + async def read_evidence( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / + ) -> AtomicMemoryEvidenceAnnotation | None: ... + async def search( self, connection: AsyncConnection, scope_id: str, request: AtomicMemorySearchRequest, / ) -> AtomicMemorySearchChannels: ... @@ -191,9 +212,11 @@ def __init__( index: AtomicMemoryIndex, *, embedding_model: EmbeddingModel | None = None, + sources: SourceRepository | None = None, ) -> None: self.index = index self._embedding_model = embedding_model + self._sources = sources @property def profile_fingerprint(self) -> str | None: @@ -243,6 +266,17 @@ async def prepare(self, content: BaseModel) -> PreparedAtomicMemoryProjection: embedding_input_hash=None if vector is None else atomic_memory_embedding_input_hash(kind, body), ) + async def capture_evidence(self, connection: AsyncConnection, scope_id: str, record: Any) -> None: + """Accept history only for a committed body write, never state-only republishing.""" + + artifact = record.artifact + if ( + self._sources is not None + and await self.index.read_evidence(connection, scope_id, artifact.as_ref()) is None + ): + evidence = await self._sources.evidence(connection, scope_id, artifact.lineage.sources) + await self.index.save_evidence(connection, scope_id, artifact.as_ref(), evidence) + async def publish( self, connection: AsyncConnection, @@ -450,12 +484,28 @@ def __init__(self, table: Table, profile: EmbeddingProfile | None) -> None: if profile is not None and (profile.dimension < 1 or profile.distance != "l2"): raise AtomicMemoryIndexError("embedding-profile", "Atomic Memory requires a positive L2 embedding profile") self.table = table - self.tables: tuple[Table, ...] = (table,) + self.tables: tuple[Table, ...] = (table, *ATOMIC_MEMORY_EVIDENCE_TABLES) + self._evidence = AtomicMemoryEvidenceRepository() self.profile = profile self.capabilities = AtomicMemoryIndexCapabilities( vector=profile is not None, hybrid=profile is not None, embedding_profile=profile ) + async def save_evidence( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + sources: Sequence[SourceEvidence], + /, + ) -> AtomicMemoryEvidenceAnnotation: + return await self._evidence.save(connection, scope_id, artifact_ref, sources) + + async def read_evidence( + self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / + ) -> AtomicMemoryEvidenceAnnotation | None: + return await self._evidence.read(connection, scope_id, artifact_ref) + def _encode_embedding(self, vector: tuple[float, ...]) -> object: return vector diff --git a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py new file mode 100644 index 0000000000..cafc2cdfe9 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py @@ -0,0 +1,100 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Apply the frozen additive p0004 revision to runtime-owned evidence tables. + +The production bundle covers a historical subset of business tables. Runtime +profiles therefore apply this same frozen additive revision independently of +that bundle's whole-database inventory, without stamping unrelated schema or +backfilling declarations for old Source/Artifact rows. SQLite DDL joins the +caller's transaction; MySQL DDL may commit and is resumed by exact postconditions. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from aiosqlite import Connection as SQLiteConnection +from sqlalchemy import inspect, select +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ( + ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE, + ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION, + ATOMIC_MEMORY_EVIDENCE_TABLES, +) +from powercontext.builtin.persistence.errors import PersistenceError + + +class AtomicMemoryEvidenceSchemaError(PersistenceError): + """An evidence schema is not the registered versioned layout.""" + + def __init__(self, detail: str) -> None: + message = { + "version": "Atomic Memory evidence schema version is unsupported", + "shape": "Atomic Memory evidence table has an incompatible shape", + }[detail] + super().__init__(message) + + +async def apply_atomic_memory_evidence_migration(connection: AsyncConnection) -> None: + """Add only missing p0004 tables, validate exact shapes, then mark ready. + + No reads or writes of Source/Artifact history are needed, and no current + declaration is used to invent accepted historical inputs. + """ + + if connection.dialect.name == "sqlite": + driver = (await connection.get_raw_connection()).driver_connection + if isinstance(driver, SQLiteConnection) and not driver.in_transaction: + await connection.exec_driver_sql("BEGIN IMMEDIATE") + dialect = "sqlite" if connection.dialect.name == "sqlite" else "mysql" + resources = Path(__file__).parent / "resources/schemas/atomic_memory_evidence_v1.json" + statements = json.loads(resources.read_text())[dialect] + for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): + exists = await connection.run_sync(lambda sync, name=table.name: inspect(sync).has_table(name)) + if not exists: + await connection.exec_driver_sql(statement) + await _require_shape(connection, table.name, statement, dialect) + marker = ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE + versions = tuple((await connection.execute(select(marker.c.schema_version))).scalars()) + if not versions: + await connection.exec_driver_sql(statements[-1]) + elif versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): + raise AtomicMemoryEvidenceSchemaError("version") + + +async def _require_shape(connection: AsyncConnection, name: str, statement: str, dialect: str) -> None: + if dialect == "sqlite": + actual = await connection.exec_driver_sql( + "SELECT type, name, sql FROM sqlite_schema WHERE (name = ? OR tbl_name = ?) " + "AND substr(name, 1, 7) != 'sqlite_' ORDER BY type, name", + (name, name), + ) + if actual.all() != [("table", name, statement)]: + raise AtomicMemoryEvidenceSchemaError("shape") + return + # Reuse the frozen MySQL inventory parser so physical formatting accepted + # by the shared backend does not become a second schema convention. + from powercontext.builtin.persistence.migrations.mysql_schema import _matches, _parse_table + + actual = (await connection.exec_driver_sql(f"SHOW CREATE TABLE `{name}`")).one()[1] + actual_name, actual_table = _parse_table(actual) + expected_name, expected_table = _parse_table(statement) + if not _matches( + {"tables": {actual_name: actual_table}, "views": [], "triggers": []}, + {"tables": {expected_name: expected_table}, "views": [], "triggers": []}, + ): + raise AtomicMemoryEvidenceSchemaError("shape") diff --git a/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0004_atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0004_atomic_memory_evidence.py new file mode 100644 index 0000000000..e83df8afbf --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0004_atomic_memory_evidence.py @@ -0,0 +1,25 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create the frozen additive evidence layout without backfilling trust.""" + +from alembic import context, op + +revision = "p0004" +down_revision = "p0003" + + +def upgrade() -> None: + context.config.attributes["execute_revision"](revision) + op.execute("INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0004')") diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py index b3502db1bc..01cf62dc69 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql_schema.py +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -35,7 +35,7 @@ from .models import MigrationError, digest _LEXEME = re.compile(r"\s+|`(?:``|[^`])*`|'(?:''|\\.|[^'\\])*'|[A-Za-z_][A-Za-z_0-9$]*|[0-9]+|>=|<=|<>|!=|[(),.=<>+-]") -_REVISIONS = ("p0001", "p0002", "p0003") +_REVISIONS = ("p0001", "p0002", "p0003", "p0004") _CITATION_TABLES = ("pc_artifacts", "pc_artifact_candidate_versions") _FAMILY_CHECK = "ck_pc_artifact_tags_family" _CONTROL_TABLE = "pc_schema_revision" @@ -141,10 +141,14 @@ def __init__(self, bundle: MigrationBundle) -> None: self.bundle = bundle baseline = json.loads((bundle.snapshot_directory / "schemas/pre_dream.json").read_text())["mysql"] current = json.loads((bundle.snapshot_directory / "schemas/v1_1_0.json").read_text())["mysql"] + evidence = json.loads((bundle.snapshot_directory / "schemas/atomic_memory_evidence_v1.json").read_text())[ + "mysql" + ] self._statements = { "p0001": tuple(baseline), "p0002": tuple(f"ALTER TABLE {table} ADD COLUMN memory_citations MEDIUMBLOB" for table in _CITATION_TABLES), "p0003": (f"ALTER TABLE pc_artifact_tags DROP CHECK {_FAMILY_CHECK}",), + "p0004": tuple(evidence[:-1]), } inventory = _empty_inventory() self._prefixes: dict[str, list[dict[str, Any]]] = {} diff --git a/src/powercontext/builtin/persistence/migrations/resources/manifest.json b/src/powercontext/builtin/persistence/migrations/resources/manifest.json index 65c798ea8b..73dbd2a358 100644 --- a/src/powercontext/builtin/persistence/migrations/resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/resources/manifest.json @@ -4,11 +4,15 @@ "schemas/pre_dream.json" ], "p0002": [], - "p0003": [] + "p0003": [], + "p0004": [ + "schemas/atomic_memory_evidence_v1.json" + ] }, "sqlite_baselines": { "e1cae089a8a565291a7a1f782cea1f6c6e1a6cdd9eab75ef0c4ca61dfda453cb": "p0001", - "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d": "p0003" + "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d": "p0003", + "93c1c20ae9bba2114158658e7852324af2bbaef12d28fa298cdbe3cae4847a84": "p0004" }, "sqlite_fingerprints": { "p0001": [ @@ -20,13 +24,20 @@ "p0003": [ "9958fa0413ed3c16442df27dc48057d438e2cf3f7e4312c5a914cccd9d3acaab", "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d" + ], + "p0004": [ + "4f21a735d9ff2e53b70599f658d762ee30a14604e4fe96c97826215433dcd509", + "93c1c20ae9bba2114158658e7852324af2bbaef12d28fa298cdbe3cae4847a84" ] }, "affected_objects": [ "pc_artifacts", "pc_artifact_heads", "pc_artifact_candidate_versions", - "pc_artifact_tags" + "pc_artifact_tags", + "pc_atomic_memory_evidence_schema", + "pc_atomic_memory_evidence_history", + "pc_source_evidence_bindings" ], "retained_objects": [ "pc_retained_p0002_artifact_tags" diff --git a/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json b/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json new file mode 100644 index 0000000000..fdd79d655e --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json @@ -0,0 +1,14 @@ +{ + "sqlite": [ + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tmaterial_digest VARCHAR(71) NOT NULL, \n\tdefinition_name TEXT NOT NULL, \n\tdefinition_version TEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71), \n\tdeclaration BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", + "INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0004')" + ], + "mysql": [ + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_type VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tmaterial_digest VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tdefinition_name MEDIUMTEXT NOT NULL, \n\tdefinition_version MEDIUMTEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tdeclaration LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", + "INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0004')" + ] +} diff --git a/src/powercontext/builtin/persistence/migrations/resources/versions/p0004_atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/resources/versions/p0004_atomic_memory_evidence.py new file mode 100644 index 0000000000..32648cfd55 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/versions/p0004_atomic_memory_evidence.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Frozen additive evidence revision; never synthesize historical declarations.""" + +import json +from pathlib import Path + +from alembic import op + +revision = "p0004" +down_revision = "p0003" + + +def upgrade() -> None: + statements = json.loads((Path(__file__).parents[1] / "schemas/atomic_memory_evidence_v1.json").read_text()) + connection = op.get_bind() + for statement in statements["sqlite" if connection.dialect.name == "sqlite" else "mysql"]: + connection.exec_driver_sql(statement) diff --git a/src/powercontext/builtin/persistence/schema.py b/src/powercontext/builtin/persistence/schema.py index d1e4170bc3..3fff568777 100644 --- a/src/powercontext/builtin/persistence/schema.py +++ b/src/powercontext/builtin/persistence/schema.py @@ -21,12 +21,20 @@ from sqlalchemy import MetaData, Table from sqlalchemy.ext.asyncio import AsyncConnection +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES +from powercontext.builtin.persistence.migrations.atomic_memory_evidence import apply_atomic_memory_evidence_migration + async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) -> None: - """Create missing tables and indexes using SQLAlchemy's native metadata support.""" + """Create selected tables, applying retained evidence through its frozen revision.""" groups: dict[MetaData, list[Table]] = {} + evidence_names = {table.name for table in ATOMIC_MEMORY_EVIDENCE_TABLES} + if any(table.name in evidence_names or table.name == "pc_sources" for table in tables): + await apply_atomic_memory_evidence_migration(connection) for table in tables: + if table.name in evidence_names: + continue groups.setdefault(table.metadata, []).append(table) for metadata, selected in groups.items(): diff --git a/src/powercontext/builtin/persistence/source_definitions.py b/src/powercontext/builtin/persistence/source_definitions.py index 9e5d4a8640..3705e9b566 100644 --- a/src/powercontext/builtin/persistence/source_definitions.py +++ b/src/powercontext/builtin/persistence/source_definitions.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Persistence for worker-owned Source Definition manifests.""" +"""Immutable historical manifests owned by the Source registration boundary.""" from __future__ import annotations @@ -26,15 +26,17 @@ from powercontext.builtin.persistence.codec import dump_model, load_model, stored_bytes from powercontext.builtin.persistence.errors import ( IdentityMismatchError, + InvalidStoredColumnError, + InvalidStoredPayloadError, RepositoryNotFoundError, StoredPayloadConflictError, ) from powercontext.builtin.persistence.tables import SOURCE_DEFINITION_MANIFESTS_TABLE -from powercontext.sources import SourceDefinitionManifest +from powercontext.sources import SourceDefinition, SourceDefinitionManifest, manifest_for_definition class SourceDefinitionManifestRepository: - """Register immutable worker-owned Definition manifests by name and version.""" + """Retain native and worker manifests without changing their identities.""" async def register( self, @@ -59,7 +61,9 @@ async def register( ) except IntegrityError: existing = await self.find(connection, manifest.name, manifest.version) - if existing is None or existing != manifest: + if existing is None: + raise + if existing != manifest: raise StoredPayloadConflictError( "source-definition-manifest", (manifest.name, manifest.version), @@ -67,6 +71,29 @@ async def register( return existing return manifest + async def register_native( + self, + connection: AsyncConnection, + definition: SourceDefinition[Any, Any, Any], + /, + ) -> SourceDefinitionManifest | None: + """Retain the existing manifest representation for trusted native material. + + This internal registration cannot replace an older manifest. A missing, + invalid or conflicting ancillary manifest leaves the Source declaration + neutral; database failures still abort the caller's transaction. External + registration keeps its existing active-Definition ownership guard. + """ + + try: + manifest = manifest_for_definition(definition) + except Exception: # Native schema providers may not expose a valid manifest. + return None + try: + return await self.register(connection, manifest) + except (IdentityMismatchError, InvalidStoredColumnError, InvalidStoredPayloadError, StoredPayloadConflictError): + return None + async def get( self, connection: AsyncConnection, diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index ba3cd56c56..5b38249f1f 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -18,6 +18,7 @@ from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence from contextlib import suppress +from hashlib import sha256 from typing import Any, Literal, cast import rfc8785 @@ -29,6 +30,7 @@ from powercontext.builtin.artifacts.experience import EXPERIENCE_INCUBATION_CURSOR_NAME from powercontext.builtin.artifacts.profile.models import PROFILE_SOURCE_WINDOW_BINDING from powercontext.builtin.artifacts.topic_memory import TOPIC_MEMORY_SOURCE_WINDOW_BINDING +from powercontext.builtin.persistence.atomic_memory_evidence_schema import SOURCE_EVIDENCE_BINDINGS_TABLE from powercontext.builtin.persistence.codec import load_model, stored_bytes from powercontext.builtin.persistence.errors import ( IdentityMismatchError, @@ -39,12 +41,22 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository +from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository from powercontext.builtin.persistence.tables import SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE from powercontext.builtin.sources.content import ContentSource from powercontext.builtin.triggers import SOURCE_WINDOW_TRIGGER_NAME from powercontext.errors import SourceDefinitionNotFoundError from powercontext.limits import MAX_SCOPE_ID_LENGTH -from powercontext.sources import Source, SourceAdapter, SourceDefinitionRegistry, SourceObservation, SourceRef +from powercontext.sources import ( + MemoryEvidenceDeclaration, + Source, + SourceAdapter, + SourceDefinitionManifest, + SourceDefinitionRegistry, + SourceEvidence, + SourceObservation, + SourceRef, +) _AnySourceAdapter = SourceAdapter[Any, Any, Any] _JSON_OBJECT = TypeAdapter(dict[str, JsonValue]) @@ -165,6 +177,7 @@ async def add_with_status( if stored.value != source: raise StoredPayloadConflictError("source", (scope_id, ref)) from None return stored, False + await self._bind_memory_evidence(connection, scope_id, ref, source) # Centralized here so capture, record projection and import all publish # discoverable input atomically. Replayed Source identities do not dirty # a binding again. Disabled processors simply retain ordinary dirty. @@ -233,6 +246,123 @@ async def get_many( raise RepositoryNotFoundError("source", (scope_id, ref)) return tuple(decoded[(ref.source_type, ref.source_id)] for ref in ordered) + async def evidence( + self, + connection: AsyncConnection, + scope_id: str, + refs: Sequence[SourceRef], + /, + ) -> tuple[SourceEvidence, ...]: + """Read accepted declarations against exact stored Source material. + + Source reads retain their existing errors and never become generation + eligibility checks. Missing or invalid declaration bindings are neutral; + database failures propagate. No current registry declaration or provider + value is substituted for historical registration facts. + """ + + stored_sources = await self.get_many(connection, scope_id, refs) + # These server-owned records describe a management write, not evidence + # supplied for that write. Other lineage-only Sources remain represented. + sources = tuple(stored for stored in stored_sources if not _management_write_source(stored.value)) + if not sources: + return () + table = SOURCE_EVIDENCE_BINDINGS_TABLE + rows = ( + ( + await connection.execute( + select(table).where( + table.c.scope_id == scope_id, + tuple_(table.c.source_type, table.c.source_id).in_( + tuple((stored.ref.source_type, stored.ref.source_id) for stored in sources) + ), + ) + ) + ) + .mappings() + .all() + ) + bindings = {(row["source_type"], row["source_id"]): row for row in rows} + manifests: dict[tuple[str, str], SourceDefinitionManifest | None] = {} + evidence: list[SourceEvidence] = [] + for stored in sources: + source = stored.value + digest = _source_material_digest(source) + value = SourceEvidence( + source_ref=stored.ref, + materialization=source.materialization, + material_digest=digest, + definition_name=stored.ref.source_type, + definition_version=source.definition_version, + ) + binding = bindings.get((stored.ref.source_type, stored.ref.source_id)) + if binding is None or ( + binding["material_digest"] != digest + or binding["definition_name"] != value.definition_name + or binding["definition_version"] != value.definition_version + or not isinstance(binding["manifest_fingerprint"], str) + ): + evidence.append(value) + continue + identity = (value.definition_name, value.definition_version) + if identity not in manifests: + manifests[identity] = await _find_evidence_manifest(connection, *identity) + manifest = manifests[identity] + if ( + manifest is None + or manifest.fingerprint != binding["manifest_fingerprint"] + or (isinstance(source, SourceObservation) and source.definition_fingerprint != manifest.fingerprint) + ): + evidence.append(value) + continue + try: + declaration = MemoryEvidenceDeclaration.model_validate_json( + stored_bytes(binding["declaration"], column="declaration") + ) + except (InvalidStoredColumnError, TypeError, ValueError): + declaration = MemoryEvidenceDeclaration() + evidence.append( + value.model_copy( + update={ + "manifest_fingerprint": manifest.fingerprint, + "declaration": declaration, + } + ) + ) + return tuple(evidence) + + async def _bind_memory_evidence( + self, + connection: AsyncConnection, + scope_id: str, + ref: SourceRef, + source: Source, + ) -> None: + """Freeze registration evidence only for a newly accepted Source row.""" + + if isinstance(source, SourceObservation): + manifest = await _find_evidence_manifest(connection, ref.source_type, source.definition_version) + if manifest is not None and manifest.fingerprint != source.definition_fingerprint: + manifest = None + else: + definition = self._registry.definition_for_source(source) + manifest = await SourceDefinitionManifestRepository().register_native(connection, definition) + declaration = ( + MemoryEvidenceDeclaration() if manifest is None else self._registry.memory_evidence_for_source(source) + ) + await connection.execute( + insert(SOURCE_EVIDENCE_BINDINGS_TABLE).values( + scope_id=scope_id, + source_type=ref.source_type, + source_id=ref.source_id, + material_digest=_source_material_digest(source), + definition_name=ref.source_type, + definition_version=source.definition_version, + manifest_fingerprint=None if manifest is None else manifest.fingerprint, + declaration=declaration.model_dump_json().encode("utf-8"), + ) + ) + async def list( self, connection: AsyncConnection, @@ -442,6 +572,32 @@ def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Sour return cast(Source, source), decoded +def _source_material_digest(source: Source) -> str: + # Preserve every material field except the narrow server witness upgrade. + material = source.model_dump(mode="json", by_alias=True) + if isinstance(source, ContentSource): + material.pop("handoff_receipt", None) + return f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}" + + +def _management_write_source(source: Source) -> bool: + return ( + isinstance(source, ContentSource) + and source.internal is not None + and source.internal.role == "lineage_only" + and source.internal.operation in {"artifact_create", "artifact_replace"} + ) + + +async def _find_evidence_manifest( + connection: AsyncConnection, name: str, version: str +) -> SourceDefinitionManifest | None: + try: + return await SourceDefinitionManifestRepository().find(connection, name, version) + except (IdentityMismatchError, InvalidStoredColumnError, InvalidStoredPayloadError): + return None + + def _dump_source( source: Source, *, diff --git a/src/powercontext/builtin/runtime/atomic_memory.py b/src/powercontext/builtin/runtime/atomic_memory.py index f79cb78fcb..f2a2a14ed9 100644 --- a/src/powercontext/builtin/runtime/atomic_memory.py +++ b/src/powercontext/builtin/runtime/atomic_memory.py @@ -58,6 +58,7 @@ ) from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository, atomic_memory_state_expression +from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceAnnotation from powercontext.builtin.persistence.atomic_memory_index import ( ATOMIC_MEMORY_RRF_CONSTANT, AtomicMemoryIndex, @@ -79,6 +80,7 @@ from powercontext.builtin.records import BaseOperationNotSupportedError, InvalidBaseAccessRequestError from powercontext.builtin.runtime.artifact_merge import ArtifactMergeAccess, merge_artifact_tags from powercontext.builtin.tags import TagFilter +from powercontext.sources.evidence import MemoryEvidenceDeclaration class AtomicMemoryAccess(ArtifactMergeAccess): @@ -186,6 +188,7 @@ def __init__( self.publisher = AtomicMemoryProjectionPublisher( index, embedding_model=embedding_model, + sources=artifacts.source_repository, ) self.service = AtomicMemoryService( artifacts=artifacts, @@ -222,6 +225,58 @@ async def get(self, artifact_id: str, *, revision: int | None = None, context=No connection, self.scope_id, artifact_id, context, revision=revision ) + async def evidence(self, artifact_id: str, *, revision: int, context=None) -> AtomicMemoryEvidenceAnnotation: + """Inspect neutral evidence facts for one authorized exact body revision. + + A missing historical snapshot remains unknown; current registration + cannot attest a revision that did not accept a declaration. + """ + application = self.application + async with ( + deferred_decision_audit(context), + application.database.transaction(consistent_snapshot=True) as connection, + ): + record = await application.service.get(connection, self.scope_id, artifact_id, context, revision=revision) + sources = application.artifacts.source_repository + materials = ( + () + if sources is None + else await sources.evidence(connection, self.scope_id, record.artifact.lineage.sources) + ) + annotation = await application.index.read_evidence(connection, self.scope_id, record.ref) + if annotation is not None: + by_ref = {(item.source_ref.source_type, item.source_ref.source_id): item for item in materials} + for accepted in annotation.sources: + material = by_ref.get((accepted.source_ref.source_type, accepted.source_ref.source_id)) + if material is not None and accepted.material_digest != material.material_digest: + raise AtomicMemoryIndexError( + "evidence-material", "Historical evidence does not match the exact Source material" + ) + if annotation is None or tuple(item.source_ref for item in annotation.sources) != tuple( + item.source_ref for item in materials + ): + return AtomicMemoryEvidenceAnnotation( + scope_id=self.scope_id, + artifact_ref=record.ref, + sources=tuple( + material.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) + for material in materials + ), + ) + verified = tuple( + accepted + if ( + accepted.materialization == material.materialization + and accepted.definition_name == material.definition_name + and accepted.definition_version == material.definition_version + and accepted.manifest_fingerprint == material.manifest_fingerprint + and accepted.declaration == material.declaration + ) + else accepted.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) + for accepted, material in zip(annotation.sources, materials, strict=True) + ) + return annotation.model_copy(update={"sources": verified}) + async def list( self, *, diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index b8015b896c..cd2e4b9e40 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -196,7 +196,7 @@ ) from powercontext.builtin.statistics import ModelUsagePurpose from powercontext.errors import InvalidSourceProjectionError, SourceProjectionNotFoundError -from powercontext.sources import Source, SourceDefinitionRegistry, SourceProjectionKey +from powercontext.sources import AdapterSourceDefinition, Source, SourceDefinitionRegistry, SourceProjectionKey if TYPE_CHECKING: from pydantic_ai.models import Model @@ -796,7 +796,17 @@ def _validate_processing_source_registry( type(source_registry) is not SourceDefinitionRegistry or len(definitions) != len(builtin_definitions) or any( - definition is not expected for definition, expected in zip(definitions, builtin_definitions, strict=True) + not ( + definition is expected + or ( + type(definition) is AdapterSourceDefinition + and type(expected) is AdapterSourceDefinition + and definition.adapter is expected.adapter + and definition.version == expected.version + and definition.projections is expected.projections + ) + ) + for definition, expected in zip(definitions, builtin_definitions, strict=True) ) ): raise BuiltinConfigurationError("artifact-processing-source-registry") diff --git a/src/powercontext/sources/__init__.py b/src/powercontext/sources/__init__.py index 91e1a27b08..5469244670 100644 --- a/src/powercontext/sources/__init__.py +++ b/src/powercontext/sources/__init__.py @@ -30,6 +30,7 @@ SourceDefinitionRegistry, SourceProjection, ) +from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence from powercontext.sources.models import Source, SourceMaterialization, SourceProjectionKey, SourceRef from powercontext.sources.observations import ( SourceDefinitionManifest, @@ -53,6 +54,7 @@ "ConnectorRunSession", "ConnectorRunStatus", "ConnectorSubmissionStatus", + "MemoryEvidenceDeclaration", "Source", "SourceAdapter", "SourceCatalog", @@ -60,6 +62,7 @@ "SourceDefinition", "SourceDefinitionManifest", "SourceDefinitionRegistry", + "SourceEvidence", "SourceMaterialization", "SourceObservation", "SourceProjection", diff --git a/src/powercontext/sources/definitions.py b/src/powercontext/sources/definitions.py index d9c54572dc..18f6bd5162 100644 --- a/src/powercontext/sources/definitions.py +++ b/src/powercontext/sources/definitions.py @@ -18,6 +18,7 @@ from collections.abc import Iterable, Mapping from dataclasses import dataclass +from types import MappingProxyType from typing import Any, Generic, Protocol, TypeVar, cast from pydantic import BaseModel, JsonValue, TypeAdapter @@ -34,6 +35,7 @@ SourceProjectionNotFoundError, ) from powercontext.sources.adapters import SourceAdapter +from powercontext.sources.evidence import MemoryEvidenceDeclaration from powercontext.sources.models import Source, SourceProjectionKey InputT = TypeVar("InputT") @@ -42,6 +44,7 @@ _JSON_VALUE = TypeAdapter(JsonValue) _AnySourceAdapter = SourceAdapter[Any, Any, Any] +_NEUTRAL_MEMORY_EVIDENCE = MemoryEvidenceDeclaration() class SourceProjection(Protocol[SourceT]): @@ -60,6 +63,7 @@ class SourceDefinition(SourceAdapter[InputT, SourceT, ValueT_co], Protocol[Input version: str projections: tuple[SourceProjection[SourceT], ...] + memory_evidence: MemoryEvidenceDeclaration | None @dataclass(frozen=True, slots=True) @@ -69,6 +73,7 @@ class AdapterSourceDefinition(Generic[InputT, SourceT, ValueT_co]): adapter: SourceAdapter[InputT, SourceT, ValueT_co] version: str = "1" projections: tuple[SourceProjection[SourceT], ...] = () + memory_evidence: MemoryEvidenceDeclaration | None = None @property def input_class(self) -> type[InputT]: @@ -93,14 +98,28 @@ async def read(self, source: SourceT, /) -> ValueT_co: class SourceDefinitionRegistry: - """Provide one immutable routing view for Source persistence and consumers.""" - - def __init__(self, definitions: Iterable[_AnySourceDefinition], /) -> None: + """Provide one immutable routing and trusted declaration view. + + Native Definitions optionally supply ``memory_evidence``. Deployment-owned + ``remote_declarations`` keys are exact ``(name, version, fingerprint)`` + manifest identities, never worker payload or projection values. Declarations + are validated and copied at construction; invalid or unavailable values are + neutral and never prevent registration of an otherwise valid Definition. + """ + + def __init__( + self, + definitions: Iterable[_AnySourceDefinition], + /, + *, + remote_declarations: Mapping[tuple[str, str, str], MemoryEvidenceDeclaration] | None = None, + ) -> None: by_input: dict[type[object], _AnySourceDefinition] = {} by_source: dict[type[Source], _AnySourceDefinition] = {} by_name: dict[str, _AnySourceDefinition] = {} projections: dict[type[Source], Mapping[SourceProjectionKey, SourceProjection[Any]]] = {} registered: list[_AnySourceDefinition] = [] + memory_evidence: dict[type[Source], MemoryEvidenceDeclaration] = {} for definition in definitions: input_class, source_class = _validate_definition(definition) @@ -122,6 +141,11 @@ def __init__(self, definitions: Iterable[_AnySourceDefinition], /) -> None: by_source[source_class] = definition by_name[definition.name] = definition projections[source_class] = projection_routes + try: + declaration = getattr(definition, "memory_evidence", None) + except Exception: # Declaration providers may be unavailable; routing remains valid. + declaration = None + memory_evidence[source_class] = _snapshot_memory_evidence(declaration) registered.append(definition) self._definitions = tuple(registered) @@ -129,6 +153,10 @@ def __init__(self, definitions: Iterable[_AnySourceDefinition], /) -> None: self._by_source = by_source self._by_name = by_name self._projections = projections + self._memory_evidence = MappingProxyType(memory_evidence) + self._remote_declarations = MappingProxyType({ + key: _snapshot_memory_evidence(value) for key, value in (remote_declarations or {}).items() + }) @classmethod def from_adapters(cls, adapters: Iterable[_AnySourceAdapter], /) -> SourceDefinitionRegistry: @@ -161,6 +189,23 @@ def definition_for_source(self, source: object, /) -> _AnySourceDefinition: ) return definition + def memory_evidence_for_source(self, source: Source, /) -> MemoryEvidenceDeclaration: + """Read the frozen trusted declaration using the exact registered route.""" + + from powercontext.sources.observations import SourceObservation + + if isinstance(source, SourceObservation): + return self.memory_evidence_for_manifest( + source.source_type, source.definition_version, source.definition_fingerprint + ) + self.definition_for_source(source) + return self._memory_evidence[type(source)] + + def memory_evidence_for_manifest(self, name: str, version: str, fingerprint: str, /) -> MemoryEvidenceDeclaration: + """Return only deployment-configured evidence for an exact manifest.""" + + return self._remote_declarations.get((name, version, fingerprint), _NEUTRAL_MEMORY_EVIDENCE) + async def resolve(self, value: object, /) -> Source: input_class = type(value) try: @@ -203,6 +248,18 @@ def project(self, source: Source, key: SourceProjectionKey, /) -> JsonValue: raise InvalidSourceProjectionError(key.name, "result", "must match the declared output schema") from error +def _snapshot_memory_evidence(value: object) -> MemoryEvidenceDeclaration: + if value is None: + return _NEUTRAL_MEMORY_EVIDENCE + try: + # Revalidate even constructed models, and never retain caller-owned data. + if isinstance(value, MemoryEvidenceDeclaration): + value = value.model_dump(mode="python") + return MemoryEvidenceDeclaration.model_validate(value) + except (TypeError, ValueError): + return _NEUTRAL_MEMORY_EVIDENCE + + def _validate_definition(definition: object) -> tuple[type[object], type[Source]]: definition_type = type(definition) input_class = getattr(definition, "input_class", None) diff --git a/src/powercontext/sources/evidence.py b/src/powercontext/sources/evidence.py new file mode 100644 index 0000000000..33eebfed9f --- /dev/null +++ b/src/powercontext/sources/evidence.py @@ -0,0 +1,60 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Typed declarations and exact material facts accepted at Source registration.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from powercontext.sources.models import SourceMaterialization, SourceRef + + +class MemoryEvidenceDeclaration(BaseModel): + """A trusted adapter's claim, not a score, permission or verification engine. + + ``declaration_version`` identifies this contract, independently of the + Definition version and Artifact content revision. Missing or unsupported + declarations resolve to ``untrusted`` / ``unknown``. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + authority: Literal["untrusted", "user_asserted", "repository_attested", "system_attested"] = "untrusted" + verification: Literal["unknown", "verified", "not_verified"] = "unknown" + declaration_version: Literal["1"] = "1" + + +class SourceEvidence(BaseModel): + """Retain one exact Source material binding without duplicating its body. + + The digest checks equality of canonical Source JSON, excluding only the + server-owned ContentSource handoff receipt witness. It authenticates neither + the Source nor its declaration. Each Source retains its own declaration. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + source_ref: SourceRef + materialization: SourceMaterialization + material_digest: str + definition_name: str + definition_version: str + manifest_fingerprint: str | None = None + declaration: MemoryEvidenceDeclaration = Field(default_factory=MemoryEvidenceDeclaration) + + +__all__ = ["MemoryEvidenceDeclaration", "SourceEvidence"] From 5dfa2ad62bc8ba1515de43c3181a5c1e3013e54f Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 14 Oct 2026 15:41:42 +0800 Subject: [PATCH 02/94] feat(memory): verify historical evidence during projection rebuild --- .../persistence/atomic_memory_evidence.py | 52 ++++- .../builtin/persistence/sources.py | 196 ++++++++++++++++-- .../builtin/runtime/atomic_memory.py | 50 +---- .../builtin/runtime/atomic_memory_rebuild.py | 58 +++++- 4 files changed, 291 insertions(+), 65 deletions(-) diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/atomic_memory_evidence.py index cb054ff704..99d5c38311 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_evidence.py +++ b/src/powercontext/builtin/persistence/atomic_memory_evidence.py @@ -29,8 +29,9 @@ from powercontext.artifacts import ArtifactRef from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE -from powercontext.builtin.persistence.sources import _require_identity +from powercontext.builtin.persistence.sources import SourceRepository, _require_identity from powercontext.limits import MAX_SCOPE_ID_LENGTH +from powercontext.sources import SourceRef from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence @@ -104,6 +105,55 @@ async def read( payload = await self._payload(connection, scope_id, artifact_ref) return None if payload is None else self._decode(scope_id, artifact_ref, payload) + async def verify( + self, + connection: AsyncConnection, + scope_id: str, + artifact_ref: ArtifactRef, + source_refs: Sequence[SourceRef], + *, + sources: SourceRepository | None = None, + ) -> AtomicMemoryEvidenceAnnotation: + """Derive exact facts from retained inputs, never today's declarations. + + Missing or damaged declaration records are neutral. Missing Sources, + corrupt authoritative payloads and unequal exact materials remain errors. + No accepted snapshot is saved or repaired by this operation. + """ + + from powercontext.builtin.persistence.atomic_memory_index import AtomicMemoryIndexError + from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY + + artifact_ref = self._reference(scope_id, artifact_ref) + repository = SourceRepository(BUILTIN_SOURCE_REGISTRY) if sources is None else sources + materials = await repository.historical_evidence(connection, scope_id, source_refs) + annotation = await self.read(connection, scope_id, artifact_ref) + if annotation is not None: + by_ref = {(item.source_ref.source_type, item.source_ref.source_id): item for item in materials} + for accepted in annotation.sources: + material = by_ref.get((accepted.source_ref.source_type, accepted.source_ref.source_id)) + if material is not None and accepted.material_digest != material.material_digest: + raise AtomicMemoryIndexError( + "evidence-material", "Historical evidence does not match the exact Source material" + ) + if annotation is None or tuple(item.source_ref for item in annotation.sources) != tuple( + item.source_ref for item in materials + ): + return self._neutral(scope_id, artifact_ref, materials) + verified = tuple( + accepted + if ( + accepted.materialization == material.materialization + and accepted.definition_name == material.definition_name + and accepted.definition_version == material.definition_version + and accepted.manifest_fingerprint == material.manifest_fingerprint + and accepted.declaration == material.declaration + ) + else accepted.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) + for accepted, material in zip(annotation.sources, materials, strict=True) + ) + return annotation.model_copy(update={"sources": verified}) + @staticmethod def _reference(scope_id: str, artifact_ref: ArtifactRef) -> ArtifactRef: _require_identity("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 5b38249f1f..597c89a842 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -22,7 +22,12 @@ from typing import Any, Literal, cast import rfc8785 +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError +from jsonschema.exceptions import ValidationError as JsonSchemaValidationError from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter +from referencing import Registry +from referencing.exceptions import Unresolvable from sqlalchemy import func, insert, select, tuple_, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection @@ -43,7 +48,7 @@ from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository from powercontext.builtin.persistence.tables import SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE -from powercontext.builtin.sources.content import ContentSource +from powercontext.builtin.sources.content import CONTENT_SOURCE_NAME, ContentSource from powercontext.builtin.triggers import SOURCE_WINDOW_TRIGGER_NAME from powercontext.errors import SourceDefinitionNotFoundError from powercontext.limits import MAX_SCOPE_ID_LENGTH @@ -267,6 +272,167 @@ async def evidence( sources = tuple(stored for stored in stored_sources if not _management_write_source(stored.value)) if not sources: return () + materials = tuple( + SourceEvidence( + source_ref=stored.ref, + materialization=stored.value.materialization, + material_digest=_source_material_digest(stored.value), + definition_name=stored.ref.source_type, + definition_version=stored.value.definition_version, + ) + for stored in sources + ) + fingerprints = { + (stored.ref.source_type, stored.ref.source_id): stored.value.definition_fingerprint + for stored in sources + if isinstance(stored.value, SourceObservation) + } + return await self._bound_evidence(connection, scope_id, materials, fingerprints) + + async def historical_evidence( + self, connection: AsyncConnection, scope_id: str, refs: Sequence[SourceRef], / + ) -> tuple[SourceEvidence, ...]: + """Check retained material without loading today's native Definition. + + Only the stored representation selects decoding. Canonical JSON retains + all native fields, rather than round-tripping through a newer model. + This does not supply a decoder or provider value for ordinary reads. + """ + + _require_identity("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) + identities = tuple(dict.fromkeys((ref.source_type, ref.source_id) for ref in refs)) + if not identities: + return () + rows = ( + ( + await connection.execute( + select(SOURCES_TABLE).where( + SOURCES_TABLE.c.scope_id == scope_id, + tuple_(SOURCES_TABLE.c.source_type, SOURCES_TABLE.c.source_id).in_(identities), + ) + ) + ) + .mappings() + .all() + ) + by_ref = {(row["source_type"], row["source_id"]): row for row in rows} + materials: list[SourceEvidence] = [] + fingerprints: dict[tuple[str, str], str] = {} + for ref in identities: + reference = SourceRef(source_type=ref[0], source_id=ref[1]) + row = by_ref.get(ref) + if row is None: + raise RepositoryNotFoundError("source", (scope_id, reference)) + source, material = await self._historical_material(connection, reference, row) + if isinstance(source, SourceObservation): + fingerprints[ref] = source.definition_fingerprint + elif ref[0] == CONTENT_SOURCE_NAME: + # The server-owned Content witness is the sole permitted material + # mutation; this exclusion is the same one used at acceptance. + internal = material.get("internal") + if ( + isinstance(internal, dict) + and internal.get("role") == "lineage_only" + and internal.get("operation") in {"artifact_create", "artifact_replace"} + ): + continue + material = {key: value for key, value in material.items() if key != "handoff_receipt"} + materials.append( + SourceEvidence( + source_ref=reference, + materialization=source.materialization, + material_digest=f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}", + definition_name=ref[0], + definition_version=source.definition_version, + ) + ) + return await self._bound_evidence(connection, scope_id, tuple(materials), fingerprints) + + async def _historical_material( + self, connection: AsyncConnection, reference: SourceRef, row: Mapping[Any, Any] + ) -> tuple[Source, dict[str, JsonValue]]: + """Validate the stored representation and its retained schema, not a live model.""" + + payload = stored_bytes(row["payload"], column="payload") + try: + envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=reference.source_type) + except InvalidStoredPayloadError: + # Bare legacy material has no representation discriminator. Use + # its registered decoder when available; otherwise retain opaque + # canonical fields for neutral maintenance, not a guessed schema. + try: + self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + source = load_model(Source, payload, kind="source", name=reference.source_type) + if source.name != reference.source_id: + raise IdentityMismatchError("source", reference.source_id, source.name) from None + try: + material = _JSON_OBJECT.validate_json(payload, strict=True) + except ValueError as error: + raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error + return source, material + source = self._decode_row(row).value + return source, source.model_dump(mode="json", by_alias=True) + material = envelope.value + model = SourceObservation if envelope.representation == "observation" else Source + source = load_model(model, rfc8785.dumps(material), kind="source", name=reference.source_type) + decoded = SourceRef( + source_type=source.source_type if isinstance(source, SourceObservation) else reference.source_type, + source_id=source.name, + ) + if reference != decoded: + raise IdentityMismatchError("source", reference, decoded) + table = SOURCE_EVIDENCE_BINDINGS_TABLE + fingerprint = await connection.scalar( + select(table.c.manifest_fingerprint).where( + table.c.scope_id == row["scope_id"], + table.c.source_type == reference.source_type, + table.c.source_id == reference.source_id, + table.c.definition_name == reference.source_type, + table.c.definition_version == source.definition_version, + ) + ) + manifest = await _find_evidence_manifest(connection, reference.source_type, source.definition_version) + if manifest is not None and manifest.fingerprint == fingerprint: + try: + Draft202012Validator.check_schema(manifest.source_schema) + value = source.payload if isinstance(source, SourceObservation) else material + Draft202012Validator(manifest.source_schema, registry=Registry()).validate(value) + except JsonSchemaValidationError as error: + raise InvalidStoredPayloadError( + "source", reference.source_type, "payload does not match the historical schema" + ) from error + except (SchemaError, Unresolvable): + self._validate_unbound_material(reference, envelope, source) + else: + self._validate_unbound_material(reference, envelope, source) + return source, material + + def _validate_unbound_material(self, reference: SourceRef, envelope: _StoredSourcePayload, source: Source) -> None: + """Keep supported exact-read errors when no historical schema is available. + + A live decoder validates only material of its own version; it never + contributes a declaration or reinterprets a valid historical binding. + """ + + if envelope.representation == "observation": + return # SourceObservation already validated its envelope and payload. + try: + definition = self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + return + if definition.version == source.definition_version: + self._decode_envelope(reference.source_type, envelope) + + @staticmethod + async def _bound_evidence( + connection: AsyncConnection, + scope_id: str, + materials: tuple[SourceEvidence, ...], + fingerprints: Mapping[tuple[str, str], str], + ) -> tuple[SourceEvidence, ...]: + if not materials: + return () table = SOURCE_EVIDENCE_BINDINGS_TABLE rows = ( ( @@ -274,7 +440,7 @@ async def evidence( select(table).where( table.c.scope_id == scope_id, tuple_(table.c.source_type, table.c.source_id).in_( - tuple((stored.ref.source_type, stored.ref.source_id) for stored in sources) + tuple((value.source_ref.source_type, value.source_ref.source_id) for value in materials) ), ) ) @@ -285,33 +451,25 @@ async def evidence( bindings = {(row["source_type"], row["source_id"]): row for row in rows} manifests: dict[tuple[str, str], SourceDefinitionManifest | None] = {} evidence: list[SourceEvidence] = [] - for stored in sources: - source = stored.value - digest = _source_material_digest(source) - value = SourceEvidence( - source_ref=stored.ref, - materialization=source.materialization, - material_digest=digest, - definition_name=stored.ref.source_type, - definition_version=source.definition_version, - ) - binding = bindings.get((stored.ref.source_type, stored.ref.source_id)) + for value in materials: + identity = (value.source_ref.source_type, value.source_ref.source_id) + binding = bindings.get(identity) if binding is None or ( - binding["material_digest"] != digest + binding["material_digest"] != value.material_digest or binding["definition_name"] != value.definition_name or binding["definition_version"] != value.definition_version or not isinstance(binding["manifest_fingerprint"], str) ): evidence.append(value) continue - identity = (value.definition_name, value.definition_version) - if identity not in manifests: - manifests[identity] = await _find_evidence_manifest(connection, *identity) - manifest = manifests[identity] + definition_identity = (value.definition_name, value.definition_version) + if definition_identity not in manifests: + manifests[definition_identity] = await _find_evidence_manifest(connection, *definition_identity) + manifest = manifests[definition_identity] if ( manifest is None or manifest.fingerprint != binding["manifest_fingerprint"] - or (isinstance(source, SourceObservation) and source.definition_fingerprint != manifest.fingerprint) + or (identity in fingerprints and fingerprints[identity] != manifest.fingerprint) ): evidence.append(value) continue diff --git a/src/powercontext/builtin/runtime/atomic_memory.py b/src/powercontext/builtin/runtime/atomic_memory.py index f2a2a14ed9..70688c79e7 100644 --- a/src/powercontext/builtin/runtime/atomic_memory.py +++ b/src/powercontext/builtin/runtime/atomic_memory.py @@ -58,7 +58,10 @@ ) from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository, atomic_memory_state_expression -from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceAnnotation +from powercontext.builtin.persistence.atomic_memory_evidence import ( + AtomicMemoryEvidenceAnnotation, + AtomicMemoryEvidenceRepository, +) from powercontext.builtin.persistence.atomic_memory_index import ( ATOMIC_MEMORY_RRF_CONSTANT, AtomicMemoryIndex, @@ -80,7 +83,6 @@ from powercontext.builtin.records import BaseOperationNotSupportedError, InvalidBaseAccessRequestError from powercontext.builtin.runtime.artifact_merge import ArtifactMergeAccess, merge_artifact_tags from powercontext.builtin.tags import TagFilter -from powercontext.sources.evidence import MemoryEvidenceDeclaration class AtomicMemoryAccess(ArtifactMergeAccess): @@ -237,45 +239,13 @@ async def evidence(self, artifact_id: str, *, revision: int, context=None) -> At application.database.transaction(consistent_snapshot=True) as connection, ): record = await application.service.get(connection, self.scope_id, artifact_id, context, revision=revision) - sources = application.artifacts.source_repository - materials = ( - () - if sources is None - else await sources.evidence(connection, self.scope_id, record.artifact.lineage.sources) - ) - annotation = await application.index.read_evidence(connection, self.scope_id, record.ref) - if annotation is not None: - by_ref = {(item.source_ref.source_type, item.source_ref.source_id): item for item in materials} - for accepted in annotation.sources: - material = by_ref.get((accepted.source_ref.source_type, accepted.source_ref.source_id)) - if material is not None and accepted.material_digest != material.material_digest: - raise AtomicMemoryIndexError( - "evidence-material", "Historical evidence does not match the exact Source material" - ) - if annotation is None or tuple(item.source_ref for item in annotation.sources) != tuple( - item.source_ref for item in materials - ): - return AtomicMemoryEvidenceAnnotation( - scope_id=self.scope_id, - artifact_ref=record.ref, - sources=tuple( - material.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) - for material in materials - ), - ) - verified = tuple( - accepted - if ( - accepted.materialization == material.materialization - and accepted.definition_name == material.definition_name - and accepted.definition_version == material.definition_version - and accepted.manifest_fingerprint == material.manifest_fingerprint - and accepted.declaration == material.declaration - ) - else accepted.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) - for accepted, material in zip(annotation.sources, materials, strict=True) + return await AtomicMemoryEvidenceRepository().verify( + connection, + self.scope_id, + record.ref, + record.artifact.lineage.sources, + sources=application.artifacts.source_repository, ) - return annotation.model_copy(update={"sources": verified}) async def list( self, diff --git a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py index 90757129b0..67f429a9ca 100644 --- a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py +++ b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Explicit offline repair of only the current Atomic Memory projection.""" +"""Offline current-projection repair and verification of retained Atomic history.""" from __future__ import annotations @@ -25,12 +25,14 @@ from sqlalchemy.engine import RowMapping from sqlalchemy.ext.asyncio import AsyncConnection +from powercontext.artifacts import ArtifactRef from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemory, AtomicMemoryRecord, AtomicMemoryStateValue from powercontext.builtin.artifacts.memory.canonical import canonical_json from powercontext.builtin.artifacts.search import analyze_text from powercontext.builtin.inference import EmbeddingModel from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository +from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceRepository from powercontext.builtin.persistence.atomic_memory_index import ( AtomicMemoryIndex, AtomicMemoryIndexError, @@ -48,6 +50,7 @@ ) from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex +from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE @@ -114,6 +117,42 @@ async def _identity_batch( return tuple((str(row.scope_id), str(row.artifact_id)) for row in rows) +async def _verify_history( + connection: AsyncConnection, artifacts: ArtifactRepository, batch_size: int, sources: SourceRepository | None +) -> dict[str, int]: + table = ARTIFACTS_TABLE + evidence = AtomicMemoryEvidenceRepository() + after: tuple[str, str, int] | None = None + revisions = neutral_sources = 0 + while True: + statement = select(table.c.scope_id, table.c.artifact_id, table.c.revision).where( + table.c.family == AtomicMemory.family + ) + if after is not None: + statement = statement.where(tuple_(table.c.scope_id, table.c.artifact_id, table.c.revision) > after) + rows = ( + await connection.execute( + statement.order_by(table.c.scope_id, table.c.artifact_id, table.c.revision).limit(batch_size) + ) + ).all() + if not rows: + break + for row in rows: + scope_id, artifact_id, revision = str(row.scope_id), str(row.artifact_id), int(row.revision) + reference = ArtifactRef(family=AtomicMemory.family, artifact_id=artifact_id, revision=revision) + artifact = await artifacts.get(connection, scope_id, reference) + annotation = await evidence.verify( + connection, scope_id, reference, artifact.lineage.sources, sources=sources + ) + revisions += 1 + neutral_sources += sum( + source.declaration.authority == "untrusted" and source.declaration.verification == "unknown" + for source in annotation.sources + ) + after = scope_id, artifact_id, revision + return {"historical_revisions": revisions, "neutral_sources": neutral_sources} + + async def _record( connection: AsyncConnection, artifacts: ArtifactRepository, scope_id: str, artifact_id: str ) -> AtomicMemoryRecord: @@ -175,12 +214,20 @@ async def _projection_differences( async def verify_atomic_memory_current( - connection: AsyncConnection, index: AtomicMemoryIndex, /, *, batch_size: int = 100 + connection: AsyncConnection, + index: AtomicMemoryIndex, + /, + *, + batch_size: int = 100, + sources: SourceRepository | None = None, ) -> AtomicMemoryProjectionRebuildReport: - """Check every current identity against authority, including post-migration creations.""" + """Check current projections and every exact revision's retained evidence.""" + if isinstance(batch_size, bool) or not 1 <= batch_size <= 1000: + raise AtomicMemoryIndexError("batch-size", "Projection rebuild batch size must be between 1 and 1000") await _require_authority(connection) artifacts = ArtifactRepository((AtomicMemory,)) + history_counts = await _verify_history(connection, artifacts, batch_size, sources) profile = index.capabilities.embedding_profile fingerprint = None if profile is None else atomic_memory_profile_fingerprint(profile) errors: list[str] = [] @@ -218,7 +265,7 @@ async def verify_atomic_memory_current( return AtomicMemoryProjectionRebuildReport( ready=not errors, profile_fingerprint=fingerprint, - counts={"active_rows": active, "obsolete_rows": obsolete}, + counts={"active_rows": active, "obsolete_rows": obsolete, **history_counts}, errors=tuple(errors), ) @@ -241,6 +288,7 @@ async def rebuild_atomic_memory_projection( maintenance_confirmed: bool, embedding_model: EmbeddingModel | None = None, batch_size: int = 100, + sources: SourceRepository | None = None, ) -> AtomicMemoryProjectionRebuildReport: """Repair current without changing revisions, Family state or retained history. @@ -309,7 +357,7 @@ async def rebuild_atomic_memory_projection( ) await connection.execute(index.table.delete().where(_obsolete_current(index))) removed += obsolete - report = await verify_atomic_memory_current(connection, index, batch_size=batch_size) + report = await verify_atomic_memory_current(connection, index, batch_size=batch_size, sources=sources) return report.model_copy( update={ "counts": { From 034e0cc5a8d661f928485fb243000763c612bb0c Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 14 Oct 2026 16:34:18 +0800 Subject: [PATCH 03/94] fix(memory): enforce evidence schema safety and backend conformance --- .../persistence/atomic_memory_readiness.py | 10 +++- .../migrations/atomic_memory_evidence.py | 46 +++++++++++++++---- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/src/powercontext/builtin/persistence/atomic_memory_readiness.py b/src/powercontext/builtin/persistence/atomic_memory_readiness.py index fdd46b5b74..455af19b99 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_readiness.py +++ b/src/powercontext/builtin/persistence/atomic_memory_readiness.py @@ -26,6 +26,10 @@ from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.builtin.persistence.errors import PersistenceError +from powercontext.builtin.persistence.migrations.atomic_memory_evidence import ( + AtomicMemoryEvidenceSchemaError, + require_atomic_memory_evidence_schema, +) LEGACY_CITATION_COLUMN = "memory_citations" _LEGACY_CITATION_TABLES = ("pc_artifacts", "pc_artifact_candidate_versions") @@ -90,11 +94,15 @@ async def legacy_foreign_keys(connection: AsyncConnection, tables: set[str]) -> async def atomic_memory_readiness_issues( connection: AsyncConnection, tables: set[str] | None = None, *, collections_removed: bool = True ) -> list[str]: - """Check public migration completion, allowing collections only during offline pre-removal acceptance.""" + """Check versioned schema and public migration completion without reading retained history.""" if tables is None: tables = set(await connection.run_sync(lambda value: inspect(value).get_table_names())) issues: list[str] = [] + try: + await require_atomic_memory_evidence_schema(connection, tables) + except AtomicMemoryEvidenceSchemaError as error: + issues.append(str(error)) if ( collections_removed and "pc_artifacts" in tables diff --git a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py index cafc2cdfe9..72ef0935db 100644 --- a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py +++ b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py @@ -43,6 +43,7 @@ class AtomicMemoryEvidenceSchemaError(PersistenceError): def __init__(self, detail: str) -> None: message = { + "missing": "Atomic Memory evidence schema is incomplete; apply the registered p0004 migration", "version": "Atomic Memory evidence schema version is unsupported", "shape": "Atomic Memory evidence table has an incompatible shape", }[detail] @@ -63,16 +64,41 @@ async def apply_atomic_memory_evidence_migration(connection: AsyncConnection) -> dialect = "sqlite" if connection.dialect.name == "sqlite" else "mysql" resources = Path(__file__).parent / "resources/schemas/atomic_memory_evidence_v1.json" statements = json.loads(resources.read_text())[dialect] + tables = set(await connection.run_sync(lambda sync: inspect(sync).get_table_names())) for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): - exists = await connection.run_sync(lambda sync, name=table.name: inspect(sync).has_table(name)) - if not exists: - await connection.exec_driver_sql(statement) - await _require_shape(connection, table.name, statement, dialect) + if table.name in tables: + await _require_shape(connection, table.name, statement, dialect) marker = ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE - versions = tuple((await connection.execute(select(marker.c.schema_version))).scalars()) + versions = ( + tuple((await connection.execute(select(marker.c.schema_version))).scalars()) if marker.name in tables else () + ) + if versions and versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): + raise AtomicMemoryEvidenceSchemaError("version") + if versions and any(table.name not in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): + # A ready marker cannot authorize recreating missing retained history. + raise AtomicMemoryEvidenceSchemaError("missing") + for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): + if table.name not in tables: + await connection.exec_driver_sql(statement) + await _require_shape(connection, table.name, statement, dialect) if not versions: await connection.exec_driver_sql(statements[-1]) - elif versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): + + +async def require_atomic_memory_evidence_schema(connection: AsyncConnection, tables: set[str] | None = None) -> None: + """Check retained evidence readiness without DDL or historical payload reads.""" + + if tables is None: + tables = set(await connection.run_sync(lambda sync: inspect(sync).get_table_names())) + if any(table.name not in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): + raise AtomicMemoryEvidenceSchemaError("missing") + dialect = "sqlite" if connection.dialect.name == "sqlite" else "mysql" + resources = Path(__file__).parent / "resources/schemas/atomic_memory_evidence_v1.json" + statements = json.loads(resources.read_text())[dialect] + for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): + await _require_shape(connection, table.name, statement, dialect) + versions = tuple((await connection.execute(select(ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE.c.schema_version))).scalars()) + if versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): raise AtomicMemoryEvidenceSchemaError("version") @@ -88,11 +114,15 @@ async def _require_shape(connection: AsyncConnection, name: str, statement: str, return # Reuse the frozen MySQL inventory parser so physical formatting accepted # by the shared backend does not become a second schema convention. + from powercontext.builtin.persistence.migrations.models import MigrationError from powercontext.builtin.persistence.migrations.mysql_schema import _matches, _parse_table actual = (await connection.exec_driver_sql(f"SHOW CREATE TABLE `{name}`")).one()[1] - actual_name, actual_table = _parse_table(actual) - expected_name, expected_table = _parse_table(statement) + try: + actual_name, actual_table = _parse_table(actual) + expected_name, expected_table = _parse_table(statement) + except MigrationError as error: + raise AtomicMemoryEvidenceSchemaError("shape") from error if not _matches( {"tables": {actual_name: actual_table}, "views": [], "triggers": []}, {"tables": {expected_name: expected_table}, "views": [], "triggers": []}, From 4d1bf17abba42774d189d28eb46d768100ffa497 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 14 Oct 2026 16:38:37 +0800 Subject: [PATCH 04/94] fix(memory): preserve unrelated database inspection --- .../builtin/persistence/atomic_memory_readiness.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/powercontext/builtin/persistence/atomic_memory_readiness.py b/src/powercontext/builtin/persistence/atomic_memory_readiness.py index 455af19b99..52df67b4a5 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_readiness.py +++ b/src/powercontext/builtin/persistence/atomic_memory_readiness.py @@ -25,6 +25,7 @@ from sqlalchemy import inspect, text from sqlalchemy.ext.asyncio import AsyncConnection +from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES from powercontext.builtin.persistence.errors import PersistenceError from powercontext.builtin.persistence.migrations.atomic_memory_evidence import ( AtomicMemoryEvidenceSchemaError, @@ -99,10 +100,11 @@ async def atomic_memory_readiness_issues( if tables is None: tables = set(await connection.run_sync(lambda value: inspect(value).get_table_names())) issues: list[str] = [] - try: - await require_atomic_memory_evidence_schema(connection, tables) - except AtomicMemoryEvidenceSchemaError as error: - issues.append(str(error)) + if "pc_artifacts" in tables or any(table.name in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): + try: + await require_atomic_memory_evidence_schema(connection, tables) + except AtomicMemoryEvidenceSchemaError as error: + issues.append(str(error)) if ( collections_removed and "pc_artifacts" in tables From cf8068a0f883e73ca78f2b7e4d5d794612b74584 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Thu, 15 Oct 2026 01:02:50 +0800 Subject: [PATCH 05/94] test(memory): consolidate source evidence E2E acceptance Cover persisted declarations, scope isolation, historical neutrality, synthetic model trust claims, and unchanged recall consumers on SQLite and OceanBase. --- tests/e2e/conftest.py | 13 + ...t_atomic_memory_evidence_neutral_recall.py | 131 ++++++++++ .../test_atomic_memory_projection_rebuild.py | 17 +- .../e2e/test_atomic_memory_source_evidence.py | 228 ++++++++++++++++++ tests/e2e/test_observability.py | 9 +- 5 files changed, 396 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/test_atomic_memory_evidence_neutral_recall.py create mode 100644 tests/e2e/test_atomic_memory_source_evidence.py diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index bdb820e071..a1f2778fa8 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import json import os from collections.abc import Iterator from pathlib import Path @@ -26,6 +27,7 @@ from pydantic import SecretStr from sqlalchemy.engine import make_url +from powercontext.builtin.persistence.migrations.backup import identify_fork_server from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.seekdb import SeekDBConfig from powercontext.builtin.persistence.sqlite import SQLiteConfig @@ -63,6 +65,17 @@ async def manage(*, create: bool) -> None: nonlocal owned async with migration_profile(config) as profile, profile.database.transaction() as connection: assert (await connection.exec_driver_sql("SELECT DATABASE()")).scalar_one() == url.database + version = str((await connection.exec_driver_sql("SELECT VERSION()")).scalar_one()) + comment = str((await connection.exec_driver_sql("SELECT @@version_comment")).scalar_one()) + product, engine_version = identify_fork_server(version, comment) + if product != "oceanbase_ai" or engine_version is None: + pytest.fail( + "Acceptance requires an identified real OceanBase version, not MySQL or SeekDB", pytrace=False + ) + if create: + evidence = {"backend": "oceanbase", "version": version, "engine_version": engine_version} + request.node.user_properties.append(("backend_identity", evidence)) + print(json.dumps(evidence, ensure_ascii=False)) await connection.exec_driver_sql( f"CREATE DATABASE `{database}`" if create else f"DROP DATABASE `{database}`" ) diff --git a/tests/e2e/test_atomic_memory_evidence_neutral_recall.py b/tests/e2e/test_atomic_memory_evidence_neutral_recall.py new file mode 100644 index 0000000000..c3ad6ee50b --- /dev/null +++ b/tests/e2e/test_atomic_memory_evidence_neutral_recall.py @@ -0,0 +1,131 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Accepted Source declarations do not alter recall or consumer revision citations.""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import replace +from typing import cast + +import httpx +import pytest + +from powercontext.builtin.artifacts.handoff import PrepareHandoff +from powercontext.builtin.runtime import ( + BuiltinConfig, + HandoffArtifactCitation, + PrepareContextRequest, + RuntimeConfig, + open_builtin_contexts, + open_builtin_runtime, +) +from powercontext.builtin.scope import ScopeDraft +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION +from powercontext.server.app import ServerApplication, create_app +from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceRef +from powercontext.sources.evidence import MemoryEvidenceDeclaration +from tests.builtin.persistence.contract import NoteAdapter, NoteInput +from tests.e2e.atomic_memory_migration_backend import MigrationBackend +from tests.e2e.test_atomic_memory_source_evidence import _commit +from tests.e2e.test_handoff_runtime import _EchoHandoffPipeline + +pytestmark = pytest.mark.parametrize("migration_backend", ["sqlite", "oceanbase"], indirect=True) + + +def test_source_declarations_preserve_recall_and_consumers(migration_backend: MigrationBackend) -> None: + async def scenario() -> None: + config = BuiltinConfig( + database=migration_backend.config, + runtime=RuntimeConfig(artifact_processing_families=(), atomic_memory_related_mode="fts"), + ) + arms = [] + neutral = MemoryEvidenceDeclaration() + candidates = ( + (SourceRef(source_type="note", source_id="note"), "memory-a", "alpha acceptance"), + (SourceRef(source_type="content", source_id="source"), "memory-b", "alpha guidance"), + ) + for declaration in ( + neutral, + MemoryEvidenceDeclaration(authority="system_attested", verification="verified"), + ): + registry = SourceDefinitionRegistry(( + replace(CONTENT_SOURCE_DEFINITION, memory_evidence=declaration), + AdapterSourceDefinition(NoteAdapter()), + )) + async with open_builtin_runtime( + config, + scheduler_path=migration_backend.directory / "consumer-scheduler.db", + source_registry=registry, + handoff_pipeline=_EchoHandoffPipeline(), + ) as runtime: + assert runtime.scopes is not None and runtime.atomic_memory is not None + scope = await runtime.scopes.create( + ScopeDraft( + title="Recall", + summary="Evidence neutrality.", + idempotency_key=declaration.authority, + ) + ) + scope_id = scope.scope_id + async with open_builtin_contexts(config, source_registry=registry) as contexts: + await contexts.records.capture_source(scope_id, "content", "source", "alpha guidance", {}) + context = await contexts.get(scope_id) + note = await context.sources.resolve(NoteInput(note_id="note", body="alpha acceptance")) + await context.sources.add(note) + memories = [] + for source, artifact_id, text in candidates: + memory = await _commit(contexts, scope_id, artifact_id, text, sources=(source,)) + annotation = await contexts.atomic_memory.for_scope(scope_id).evidence(artifact_id, revision=1) + assert annotation.artifact_ref == memory.ref + expected = declaration if source.source_type == "content" else neutral + assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( + (source, expected), + ) + memories.append(memory) + page = await runtime.atomic_memory.for_scope(scope_id).search("alpha", mode="text", limit=2) + assert tuple((hit.hit.artifact_ref, hit.text, hit.matched_by) for hit in page.hits) == tuple( + (memory.ref, text, ("text",)) for memory, (_, _, text) in zip(memories, candidates, strict=True) + ) + recalled = tuple((hit.hit.artifact_ref.model_dump(mode="json"), hit.text) for hit in page.hits) + app = create_app(application=cast(ServerApplication, runtime)) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://testserver" + ) as client: + response = await client.post("/v1/memory/search", json={"scope_id": scope_id, "query": "alpha"}) + assert response.status_code == 200, response.text + public = response.json() + assert ( + tuple((item["memory"]["artifact"], item["memory"]["text"]) for item in public["hits"]) == recalled + ) + prepared = await runtime.context.for_scope(scope_id).prepare(PrepareContextRequest(query="alpha")) + assert prepared.status == "ready" and prepared.content is not None + items = json.loads(prepared.content.splitlines()[-2])["items"] + assert tuple((item["citation"]["artifact_ref"], item["content"]) for item in items) == recalled + citation = HandoffArtifactCitation(artifact_ref=page.hits[-1].hit.artifact_ref) + handoffs = runtime.handoff.for_scope(scope_id) + finalized = await handoffs.finalize( + await handoffs.prepare( + PrepareHandoff(objective="Keep the recalled revision.", evidence=(citation,)) + ) + ) + assert finalized.content.state[0].citations == (citation,) + assert finalized.content.next_action is not None + assert finalized.content.next_action.citations == (citation,) + arms.append((page.hits, public, prepared.content.replace(scope_id, "scope"), finalized.content)) + assert arms[0] == arms[1] + + asyncio.run(scenario()) diff --git a/tests/e2e/test_atomic_memory_projection_rebuild.py b/tests/e2e/test_atomic_memory_projection_rebuild.py index 3075699659..5263ff1a0c 100644 --- a/tests/e2e/test_atomic_memory_projection_rebuild.py +++ b/tests/e2e/test_atomic_memory_projection_rebuild.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +from dataclasses import replace from pathlib import Path import pytest @@ -29,6 +30,9 @@ from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION +from powercontext.sources import SourceDefinitionRegistry +from powercontext.sources.evidence import MemoryEvidenceDeclaration from tests.e2e.atomic_memory_migration_fixture import SCOPE, _atomic, _seed_and_migrate from tests.legacy_memory import add_legacy_citation_columns @@ -36,6 +40,12 @@ @pytest.mark.parametrize("retention", ["complete", "archive_only", "entries_only", "removed", "unreadable"]) def test_rebuild_uses_current_authority_independently_of_legacy_retention(tmp_path: Path, retention: str) -> None: config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'projection.db'}") + strong_registry = SourceDefinitionRegistry(( + replace( + CONTENT_SOURCE_DEFINITION, + memory_evidence=MemoryEvidenceDeclaration(authority="system_attested", verification="verified"), + ), + )) async def scenario() -> None: await _seed_and_migrate(config) @@ -43,6 +53,8 @@ async def scenario() -> None: memory = contexts.atomic_memory.for_scope(SCOPE) current = await memory.get(_atomic("alpha", 2).artifact_id) historical = await memory.get(_atomic("alpha", 1).artifact_id, revision=1) + annotation = await memory.evidence(historical.ref.artifact_id, revision=1) + assert all(source.declaration == MemoryEvidenceDeclaration() for source in annotation.sources) async with SQLiteProfile.open(config, tables=()) as profile: async with profile.database.transaction() as connection: if retention in {"entries_only", "removed"}: @@ -63,10 +75,13 @@ async def scenario() -> None: ) assert report.ready, report.errors assert report.counts["rebuilt_rows"] == 2 - async with open_builtin_contexts(BuiltinConfig(database=config)) as contexts: + assert report.counts["historical_revisions"] == 3 + assert report.counts["neutral_sources"] == 4 + async with open_builtin_contexts(BuiltinConfig(database=config), source_registry=strong_registry) as contexts: memory = contexts.atomic_memory.for_scope(SCOPE) assert await memory.get(current.ref.artifact_id) == current assert await memory.get(historical.ref.artifact_id, revision=1) == historical + assert await memory.evidence(historical.ref.artifact_id, revision=1) == annotation result = await memory.search("Tasks A and C", mode="text") assert current.ref in tuple(hit.hit.artifact_ref for hit in result.hits) diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py new file mode 100644 index 0000000000..5fe44a12e3 --- /dev/null +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -0,0 +1,228 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Versioned source evidence acceptance on disposable SQLite and OceanBase.""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import replace + +import pytest +from sqlalchemy import event, text +from sqlalchemy.exc import SQLAlchemyError + +from powercontext.artifacts import ArtifactLineage +from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent +from powercontext.builtin.artifacts.atomic_memory.extraction import ( + AtomicMemoryCandidate, + AtomicMemoryExtractionOutput, +) +from powercontext.builtin.inference import GenerationResult +from powercontext.builtin.runtime import BuiltinConfig, SubmitSourceObservation, open_builtin_contexts +from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection +from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION +from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceRef +from powercontext.sources.evidence import MemoryEvidenceDeclaration +from powercontext.sources.observations import manifest_for_definition, project_source_for_transport +from tests.builtin.persistence.contract import NoteAdapter, NoteInput +from tests.e2e.atomic_memory_migration_backend import MigrationBackend +from tests.e2e.dream_support import atomic_memory_pipeline + +pytestmark = pytest.mark.parametrize("migration_backend", ["sqlite", "oceanbase"], indirect=True) + + +async def _commit(contexts, scope_id, artifact_id, text_value, *, sources=(), artifacts=(), expected_revision=None): + application = contexts.atomic_memory + async with contexts.database.transaction() as connection: + plan = await application.service.inspect_change( + connection, + scope_id, + artifact_id, + AtomicMemoryContent(kind="fact", text=text_value), + None, + expected_revision=expected_revision, + lineage=ArtifactLineage(sources=sources, artifacts=artifacts), + ) + prepared = await application.service.prepare_change(plan) + async with contexts.database.transaction() as connection: + result = await application.service.commit(connection, prepared, None) + return result.primary + + +def test_mixed_evidence_survives_failed_revision_restart_and_rebuild(migration_backend: MigrationBackend) -> None: + strong = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + asserted = MemoryEvidenceDeclaration(authority="user_asserted", verification="not_verified") + remote = MemoryEvidenceDeclaration(authority="system_attested", verification="verified") + + class RemoteNoteAdapter(NoteAdapter): + name = "remote-note" + + worker = SourceDefinitionRegistry((AdapterSourceDefinition(RemoteNoteAdapter(), memory_evidence=remote),)) + manifest = manifest_for_definition(worker.definition_for_name("remote-note")) + registry = SourceDefinitionRegistry( + ( + replace(CONTENT_SOURCE_DEFINITION, memory_evidence=strong), + AdapterSourceDefinition(NoteAdapter(), memory_evidence=asserted), + ), + remote_declarations={(manifest.name, manifest.version, manifest.fingerprint): remote}, + ) + config = BuiltinConfig(database=migration_backend.config) + refs = ( + SourceRef(source_type="content", source_id="cafe\u0301"), + SourceRef(source_type="note", source_id="native"), + SourceRef(source_type="remote-note", source_id="remote"), + ) + + async def scenario() -> None: + async with open_builtin_contexts(config, source_registry=registry) as contexts: + context = await contexts.get("scope") + await contexts.records.capture_source( + "scope", "content", "cafe\u0301", "Release received.", {"authority": "untrusted"} + ) + native = await context.sources.resolve(NoteInput(note_id="native", body="User correction.")) + await context.sources.add(native) + source = await worker.resolve(NoteInput(note_id="remote", body="Remote release evidence.")) + await contexts.register_source_definition(manifest) + await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope", observation=project_source_for_transport(worker, source)) + ) + original = await _commit(contexts, "scope", "rule", "Release rule one.", sources=refs) + memory = contexts.atomic_memory.for_scope("scope") + annotation = await memory.evidence("rule", revision=1) + assert annotation.artifact_ref == original.ref + assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( + (refs[0], strong), + (refs[1], asserted), + (refs[2], remote), + ) + + def reject_snapshot(connection, cursor, statement, *args): + if statement.lstrip().upper().startswith("INSERT INTO PC_ATOMIC_MEMORY_EVIDENCE_HISTORY"): + raise SQLAlchemyError(statement) + + engine = contexts.database.engine.sync_engine + event.listen(engine, "before_cursor_execute", reject_snapshot) + try: + with pytest.raises(SQLAlchemyError): + await _commit(contexts, "scope", "rule", "Failed revision.", expected_revision=1) + finally: + event.remove(engine, "before_cursor_execute", reject_snapshot) + assert await memory.get("rule") == original + revised = await _commit(contexts, "scope", "rule", "Rule two.", sources=(refs[2],), expected_revision=1) + current = await memory.evidence("rule", revision=2) + assert current.sources == (annotation.sources[2],) + neutral_registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION, AdapterSourceDefinition(NoteAdapter()))) + async with open_builtin_contexts(config, source_registry=neutral_registry) as contexts: + await contexts.records.capture_source( + "other", refs[0].source_type, refs[0].source_id, "Other Scope release.", {} + ) + other = await _commit(contexts, "other", "rule", "Other Scope rule.", sources=(refs[0],)) + other_annotation = await contexts.atomic_memory.for_scope("other").evidence("rule", revision=1) + assert other_annotation.scope_id == "other" + assert tuple((item.source_ref, item.declaration) for item in other_annotation.sources) == ( + (refs[0], MemoryEvidenceDeclaration()), + ) + assert other_annotation.sources[0].material_digest != annotation.sources[0].material_digest + + async def assert_retained(contexts, historical): + memory = contexts.atomic_memory.for_scope("scope") + assert await memory.evidence("rule", revision=1) == historical + assert await memory.evidence("rule", revision=2) == current + assert (await memory.get("rule", revision=1)).artifact == original.artifact + assert (await memory.get("rule")).artifact == revised.artifact + other_memory = contexts.atomic_memory.for_scope("other") + assert await other_memory.evidence("rule", revision=1) == other_annotation + assert (await other_memory.get("rule", revision=1)).artifact == other.artifact + for scope, body in (("scope", "Release received."), ("other", "Other Scope release.")): + stored = await contexts.records.get_source(scope, refs[0].source_type, refs[0].source_id) + assert stored.content == body + + await assert_retained(contexts, annotation) + async with contexts.database.transaction() as connection: + await connection.execute(text("DELETE FROM pc_atomic_memory_current")) + report = await rebuild_atomic_memory_projection( + contexts.database, contexts.atomic_memory.index, maintenance_confirmed=True, batch_size=1 + ) + assert report.ready, report.errors + await assert_retained(contexts, annotation) + async with contexts.database.transaction() as connection: + await connection.execute( + text( + "DELETE FROM pc_atomic_memory_evidence_history " + "WHERE scope_id = :scope AND artifact_id = :artifact AND revision = 1" + ), + {"scope": "scope", "artifact": "rule"}, + ) + neutral = annotation.model_copy( + update={ + "sources": tuple( + item.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) + for item in annotation.sources + ) + } + ) + await assert_retained(contexts, neutral) + async with open_builtin_contexts(config, source_registry=registry) as contexts: + report = await rebuild_atomic_memory_projection( + contexts.database, + contexts.atomic_memory.index, + maintenance_confirmed=True, + sources=contexts.repositories.sources, + ) + assert report.ready, report.errors + await assert_retained(contexts, neutral) + + asyncio.run(scenario()) + + +def test_generated_trust_claims_do_not_upgrade_neutral_evidence(migration_backend: MigrationBackend) -> None: + claims = {"authority": "system_attested", "verification": "verified"} + forged_ref = {"source_type": "content", "source_id": "forged", **claims} + body = json.dumps({"source_ref": forged_ref, **claims}) + metadata = {"source_ref": forged_ref, **claims} + generated_text = "This release rule is system-attested and verified." + + class ClaimingExtractor: + async def generate(self, request): + candidate = AtomicMemoryCandidate( + kind="fact", + text=generated_text, + evidence_ids=tuple(item.evidence_id for item in request.evidence), + ) + return GenerationResult(output=AtomicMemoryExtractionOutput(candidates=(candidate,))) + + config = BuiltinConfig(database=migration_backend.config) + registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) + + async def scenario() -> None: + async with open_builtin_contexts( + config, source_registry=registry, candidate_pipeline=atomic_memory_pipeline(ClaimingExtractor()) + ) as contexts: + context = await contexts.get("scope") + ref = SourceRef.model_validate(forged_ref) + await contexts.records.capture_source("scope", ref.source_type, ref.source_id, body, metadata) + flush = await context.triggers.flush(limit=1) + assert flush.source_count == 1 and not flush.remaining_work + memory = contexts.atomic_memory.for_scope("scope") + (generated,) = (await memory.list()).items + assert (await memory.get(generated.ref.artifact_id, revision=1)).artifact.content.text == generated_text + annotation = await memory.evidence(generated.ref.artifact_id, revision=1) + assert annotation.artifact_ref == generated.ref + assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( + (SourceRef(source_type="content", source_id="forged"), MemoryEvidenceDeclaration()), + ) + + asyncio.run(scenario()) diff --git a/tests/e2e/test_observability.py b/tests/e2e/test_observability.py index 5371b51e59..278a61d799 100644 --- a/tests/e2e/test_observability.py +++ b/tests/e2e/test_observability.py @@ -50,6 +50,7 @@ ) from powercontext.builtin.inference import GenerationResult from powercontext.builtin.inference.pydantic_ai import PydanticAIEmbeddingModel +from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceRepository from powercontext.builtin.persistence.atomic_memory_index import ( AtomicMemoryIndexCapabilities, AtomicMemoryProjection, @@ -240,11 +241,17 @@ class _VectorMemoryIndex: tables = () def __init__(self, profile=None): - pass + self._evidence = AtomicMemoryEvidenceRepository() async def initialize(self, connection: AsyncConnection, /): pass + async def save_evidence(self, connection, scope_id, artifact_ref, sources, /): + return await self._evidence.save(connection, scope_id, artifact_ref, sources) + + async def read_evidence(self, connection, scope_id, artifact_ref, /): + return await self._evidence.read(connection, scope_id, artifact_ref) + async def replace(self, connection: AsyncConnection, scope_id: str, projection: AtomicMemoryProjection, /): pass From 0d70cbaf2798a3cb1f453527f67670a0bd86d002 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Thu, 15 Oct 2026 18:20:15 +0800 Subject: [PATCH 06/94] fix(memory): verify evidence against retained source material --- .../builtin/persistence/sources.py | 35 ++++++------------- src/powercontext/sources/observations.py | 2 +- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 597c89a842..0b58316aa7 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -266,28 +266,7 @@ async def evidence( value is substituted for historical registration facts. """ - stored_sources = await self.get_many(connection, scope_id, refs) - # These server-owned records describe a management write, not evidence - # supplied for that write. Other lineage-only Sources remain represented. - sources = tuple(stored for stored in stored_sources if not _management_write_source(stored.value)) - if not sources: - return () - materials = tuple( - SourceEvidence( - source_ref=stored.ref, - materialization=stored.value.materialization, - material_digest=_source_material_digest(stored.value), - definition_name=stored.ref.source_type, - definition_version=stored.value.definition_version, - ) - for stored in sources - ) - fingerprints = { - (stored.ref.source_type, stored.ref.source_id): stored.value.definition_fingerprint - for stored in sources - if isinstance(stored.value, SourceObservation) - } - return await self._bound_evidence(connection, scope_id, materials, fingerprints) + return await self.historical_evidence(connection, scope_id, refs) async def historical_evidence( self, connection: AsyncConnection, scope_id: str, refs: Sequence[SourceRef], / @@ -341,7 +320,7 @@ async def historical_evidence( SourceEvidence( source_ref=reference, materialization=source.materialization, - material_digest=f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}", + material_digest=_source_material_digest_material(material), definition_name=ref[0], definition_version=source.definition_version, ) @@ -372,7 +351,11 @@ async def _historical_material( raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error return source, material source = self._decode_row(row).value - return source, source.model_dump(mode="json", by_alias=True) + try: + material = _JSON_OBJECT.validate_json(payload, strict=True) + except ValueError as error: + raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error + return source, material material = envelope.value model = SourceObservation if envelope.representation == "observation" else Source source = load_model(model, rfc8785.dumps(material), kind="source", name=reference.source_type) @@ -756,6 +739,10 @@ async def _find_evidence_manifest( return None +def _source_material_digest_material(material: Mapping[str, JsonValue]) -> str: + return f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}" + + def _dump_source( source: Source, *, diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 576e29fb79..a7b1858c03 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -163,7 +163,7 @@ def projection(self, key: SourceProjectionKey, /) -> JsonValue: def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> SourceDefinitionManifest: """Build the immutable declaration transported by a remote worker.""" - source_schema = _json_object(definition.source_class.model_json_schema()) + source_schema = _json_object(definition.source_class.model_json_schema(mode="serialization")) projections = tuple( SourceProjectionManifest( key=SourceProjectionKey(name=projection.name, version=projection.version), From 3305b6bb1edb9b1c0b86b3ec4955239dad9d36d8 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Thu, 15 Oct 2026 21:17:41 +0800 Subject: [PATCH 07/94] fix(sources): align remote payload aliases with manifests --- src/powercontext/sources/observations.py | 2 +- tests/builtin/persistence/test_sources.py | 53 +++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index a7b1858c03..c9bb8a802f 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -199,7 +199,7 @@ def project_source_for_transport( "remote Source observations must retain their canonical value", ) manifest = manifest_for_definition(definition) - payload = _json_object(source.model_dump(mode="json")) + payload = _json_object(source.model_dump(mode="json", by_alias=True)) projections = tuple( SourceProjectionValue(key=key, value=registry.project(source, key)) for key in registry.projection_keys(source) ) diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 22e080499f..1e88d9bb96 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -17,6 +17,7 @@ import asyncio import pytest +from pydantic import BaseModel, Field from sqlalchemy import insert from powercontext import InvalidSourceDefinitionError @@ -29,8 +30,11 @@ from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import SHARED_TABLES, SOURCES_TABLE +from powercontext.builtin.runtime import SubmitSourceObservation from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.sources import ( + AdapterSourceDefinition, + Source, SourceDefinitionRegistry, SourceMaterialization, SourceObservation, @@ -46,6 +50,31 @@ ) +class SerializationAliasInput(BaseModel): + source_id: str + body: str + + +class SerializationAliasSource(Source): + body: str = Field(serialization_alias="wire_text") + + +class SerializationAliasAdapter: + name = "serialization-alias" + input_class = SerializationAliasInput + source_class = SerializationAliasSource + + async def resolve(self, value: SerializationAliasInput, /) -> SerializationAliasSource: + return SerializationAliasSource( + name=value.source_id, + materialization=SourceMaterialization.CAPTURED, + body=value.body, + ) + + async def read(self, source: SerializationAliasSource, /) -> str: + return source.body + + def test_two_source_adapters_share_one_repository_and_journal() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): @@ -149,6 +178,30 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_remote_observation_ingestion_accepts_serialization_aliases() -> None: + async def scenario() -> None: + worker = SourceDefinitionRegistry((AdapterSourceDefinition(SerializationAliasAdapter()),)) + source = await worker.resolve(SerializationAliasInput(source_id="remote-1", body="Remote note text.")) + manifest = manifest_for_definition(worker.definition_for_name("serialization-alias")) + observation = project_source_for_transport(worker, source) + + async with repository_profile() as (profile, _repositories): + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + context = await contexts.get("scope-a") + stored = await context.sources.get(observation) + + assert receipt.source_ref == SourceRef(source_type="serialization-alias", source_id="remote-1") + assert isinstance(stored, SourceObservation) + assert stored.payload["wire_text"] == "Remote note text." + assert "body" not in stored.payload + + asyncio.run(scenario()) + + def test_repository_rejects_a_scope_outside_the_relational_identity_baseline() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): From c9ee82d5dc9cf75f2e7c8800959a611c507a7d03 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Fri, 16 Oct 2026 17:16:32 +0800 Subject: [PATCH 08/94] fix(sources): preserve envelope keys with alias generators --- src/powercontext/sources/models.py | 11 ++++--- tests/builtin/persistence/test_sources.py | 37 ++++++++++++++++++----- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/powercontext/sources/models.py b/src/powercontext/sources/models.py index 6e2cd48df6..6235d78f9e 100644 --- a/src/powercontext/sources/models.py +++ b/src/powercontext/sources/models.py @@ -16,7 +16,7 @@ from enum import StrEnum -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from powercontext.errors import InvalidSourceReferenceError from powercontext.limits import MAX_SOURCE_ID_LENGTH, MAX_SOURCE_TYPE_LENGTH @@ -60,10 +60,11 @@ def validate_key_part(cls, value: str, info) -> str: class Source(BaseModel): """Base value for an adapter-owned Source description.""" - name: str - definition_version: str = "1" - materialization: SourceMaterialization - description: str | None = None + # Envelope keys remain canonical when a Definition generates aliases for its own fields. + name: str = Field(serialization_alias="name") + definition_version: str = Field(default="1", serialization_alias="definition_version") + materialization: SourceMaterialization = Field(serialization_alias="materialization") + description: str | None = Field(default=None, serialization_alias="description") @field_validator("definition_version") @classmethod diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 1e88d9bb96..b74a798a7e 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -17,7 +17,8 @@ import asyncio import pytest -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel from sqlalchemy import insert from powercontext import InvalidSourceDefinitionError @@ -59,15 +60,27 @@ class SerializationAliasSource(Source): body: str = Field(serialization_alias="wire_text") +class CamelCaseSource(SerializationAliasSource): + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + +class UpperCaseSource(SerializationAliasSource): + model_config = ConfigDict(alias_generator=str.upper, populate_by_name=True) + + class SerializationAliasAdapter: name = "serialization-alias" input_class = SerializationAliasInput - source_class = SerializationAliasSource + + def __init__(self, source_class: type[SerializationAliasSource]) -> None: + self.source_class = source_class async def resolve(self, value: SerializationAliasInput, /) -> SerializationAliasSource: - return SerializationAliasSource( + return self.source_class( name=value.source_id, + definition_version="2", materialization=SourceMaterialization.CAPTURED, + description="Remote alias description.", body=value.body, ) @@ -178,9 +191,14 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_remote_observation_ingestion_accepts_serialization_aliases() -> None: +@pytest.mark.parametrize("source_class", (SerializationAliasSource, CamelCaseSource, UpperCaseSource)) +def test_remote_observation_ingestion_accepts_serialization_aliases( + source_class: type[SerializationAliasSource], +) -> None: async def scenario() -> None: - worker = SourceDefinitionRegistry((AdapterSourceDefinition(SerializationAliasAdapter()),)) + worker = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(source_class), version="2"), + )) source = await worker.resolve(SerializationAliasInput(source_id="remote-1", body="Remote note text.")) manifest = manifest_for_definition(worker.definition_for_name("serialization-alias")) observation = project_source_for_transport(worker, source) @@ -196,8 +214,13 @@ async def scenario() -> None: assert receipt.source_ref == SourceRef(source_type="serialization-alias", source_id="remote-1") assert isinstance(stored, SourceObservation) - assert stored.payload["wire_text"] == "Remote note text." - assert "body" not in stored.payload + assert stored.payload == { + "name": "remote-1", + "definition_version": "2", + "materialization": "captured", + "description": "Remote alias description.", + "wire_text": "Remote note text.", + } asyncio.run(scenario()) From dc9b6a175154be0a2c334e7c02c99c2df0da3651 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Sat, 17 Oct 2026 13:28:08 +0800 Subject: [PATCH 09/94] fix(sources): align envelope validation with serialization --- src/powercontext/sources/models.py | 10 +++--- tests/builtin/persistence/test_sources.py | 40 +++++++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/powercontext/sources/models.py b/src/powercontext/sources/models.py index 6235d78f9e..0c31b8a387 100644 --- a/src/powercontext/sources/models.py +++ b/src/powercontext/sources/models.py @@ -60,11 +60,11 @@ def validate_key_part(cls, value: str, info) -> str: class Source(BaseModel): """Base value for an adapter-owned Source description.""" - # Envelope keys remain canonical when a Definition generates aliases for its own fields. - name: str = Field(serialization_alias="name") - definition_version: str = Field(default="1", serialization_alias="definition_version") - materialization: SourceMaterialization = Field(serialization_alias="materialization") - description: str | None = Field(default=None, serialization_alias="description") + # Envelope keys stay canonical for validation and serialization, independently of adapter aliases. + name: str = Field(alias="name") + definition_version: str = Field(default="1", alias="definition_version") + materialization: SourceMaterialization = Field(alias="materialization") + description: str | None = Field(default=None, alias="description") @field_validator("definition_version") @classmethod diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index b74a798a7e..0f66d408f5 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -68,6 +68,12 @@ class UpperCaseSource(SerializationAliasSource): model_config = ConfigDict(alias_generator=str.upper, populate_by_name=True) +class NativeCamelCaseSource(SerializationAliasSource): + model_config = ConfigDict(alias_generator=to_camel) + + body: str + + class SerializationAliasAdapter: name = "serialization-alias" input_class = SerializationAliasInput @@ -225,6 +231,40 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_native_generated_alias_source_survives_read_replay_and_restart(tmp_path) -> None: + async def scenario() -> None: + adapter = SerializationAliasAdapter(NativeCamelCaseSource) + registry = SourceDefinitionRegistry((AdapterSourceDefinition(adapter, version="2"),)) + value = SerializationAliasInput(source_id="native-2", body="Keep the native version and body.") + source = await adapter.resolve(value) + # Keep the persisted version nondefault regardless of the constructor's alias rules. + source.definition_version = "2" + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'native-alias.db'}") + + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=registry) + context = await contexts.get("scope-a") + assert await context.sources.add(source) == source + loaded = await context.sources.get(source) + assert loaded == source + assert await context.sources.read(loaded) == value.body + assert await context.sources.add(source) == source + assert await context.sources.journal.position(source) == 1 + + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=registry) + context = await contexts.get("scope-a") + loaded = await context.sources.get(source) + assert loaded == source + assert loaded.definition_version == "2" + resolved = await context.sources.resolve(value) + assert resolved == source + assert await context.sources.add(resolved) == source + assert await context.sources.journal.position(resolved) == 1 + + asyncio.run(scenario()) + + def test_repository_rejects_a_scope_outside_the_relational_identity_baseline() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): From 8ee3aba7e99f34c1f296678a3fd3e3ff2f602b45 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Sun, 18 Oct 2026 15:17:15 +0800 Subject: [PATCH 10/94] fix(memory): preserve source identity and shared evidence Retain native Source identity independently of adapter aliases and mutable description defaults, preserve legacy read/replay behavior, and keep serialized business fields intact. Capture evidence at the shared Atomic Memory body-commit boundary, including restoration revisions, without retroactively attesting state-only writes. Refs #1975. --- .../artifacts/atomic_memory/service.py | 35 ++++- .../builtin/persistence/sources.py | 144 +++++++++++++++++- tests/builtin/persistence/test_sources.py | 90 +++++++++++ .../e2e/test_atomic_memory_source_evidence.py | 52 ++++++- 4 files changed, 308 insertions(+), 13 deletions(-) diff --git a/src/powercontext/builtin/artifacts/atomic_memory/service.py b/src/powercontext/builtin/artifacts/atomic_memory/service.py index 90fb984ef4..94c049493f 100644 --- a/src/powercontext/builtin/artifacts/atomic_memory/service.py +++ b/src/powercontext/builtin/artifacts/atomic_memory/service.py @@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Any, Protocol, cast from sqlalchemy.ext.asyncio import AsyncConnection +from typing_extensions import override from powercontext.artifacts import ArtifactLineage, ArtifactRef from powercontext.builtin.artifacts.atomic_memory.errors import ( @@ -52,6 +53,7 @@ from powercontext.builtin.artifacts.merge import ArtifactMergeSecurity, ArtifactMergeService, MergeTags from powercontext.builtin.artifacts.merge_models import ( ArtifactMergeErrors, + ArtifactMergeMutationResult, ArtifactMergePlan, ArtifactMergeRead, ArtifactMergeRecord, @@ -278,6 +280,32 @@ async def remove(self, connection: AsyncConnection, scope_id: str, artifact_id: await self.projections.remove(connection, scope_id, artifact_id) +class _AtomicMemoryMergeService(ArtifactMergeService): + """Accept evidence with every new body, including retired restoration results. + + Shared merges, the Atomic facade and processing all use this commit seam. + State-only writes must not attest old bodies. Capture joins the caller's + transaction so its failure rolls back authority, governance and projections. + """ + + @override + async def _commit_locked( + self, + connection: AsyncConnection, + prepared: PreparedArtifactMerge, + current: dict[str, ArtifactMergeRecord], + context: Any, + direct_source: SourceRef | None, + ) -> ArtifactMergeMutationResult: + result = await super()._commit_locked(connection, prepared, current, context, direct_source) + revisions = {write.artifact_id for write in prepared.plan.writes if write.draft is not None} + projections = cast(AtomicMemoryMergeAdapter, self.adapter).projections + for record in result.records: + if record.ref.artifact_id in revisions: + await projections.capture_evidence(connection, prepared.plan.scope_id, _atomic_record(record)) + return result + + class _AtomicPreviewAdapter: def __init__(self, signer): self.signer = signer @@ -304,7 +332,7 @@ def __init__( ) -> None: self.artifacts, self.states, self.security = artifacts, states, security self.projections, self.merge_tags, self.preview_signer = projections, merge_tags, preview_signer - self.shared = ArtifactMergeService( + self.shared = _AtomicMemoryMergeService( artifacts=artifacts, adapter=AtomicMemoryMergeAdapter(projections), security=security, @@ -481,11 +509,6 @@ async def commit_window( read_set=tuple(_common_read(read) for read in read_set), direct_source=direct_source, ) - for item, result in zip(prepared, results, strict=True): - revisions = {write.artifact_id for write in item.plan.writes if write.draft is not None} - for record in result.records: - if record.artifact.artifact_id in revisions: - await self.projections.capture_evidence(connection, scope_id, _atomic_record(record)) return tuple( AtomicMemoryMutationResult( changed=result.changed, diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 0b58316aa7..24cff31caf 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -25,7 +25,7 @@ from jsonschema import Draft202012Validator from jsonschema.exceptions import SchemaError from jsonschema.exceptions import ValidationError as JsonSchemaValidationError -from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter +from pydantic import AliasGenerator, BaseModel, ConfigDict, JsonValue, TypeAdapter from referencing import Registry from referencing.exceptions import Unresolvable from sqlalchemy import func, insert, select, tuple_, update @@ -79,6 +79,9 @@ class _StoredSourcePayload(BaseModel): encoding: Literal["powercontext-source-v1"] representation: Literal["native", "observation"] value: dict[str, JsonValue] + # Server-owned header permits historical reads without adapter code. Never hash it. + source: Source | None = None + source_keys: dict[str, str] | None = None class StoredSource(BaseModel): @@ -158,6 +161,8 @@ async def add_with_status( return StoredSource(ref=ref, value=source, journal_position=stored.journal_position), False if stored.value != source: raise StoredPayloadConflictError("source", (scope_id, ref)) + if representation == "native": + await self._retain_native_header(connection, scope_id, ref, existing, source) return stored, False position = await _next_journal_position(connection, scope_id) @@ -357,8 +362,7 @@ async def _historical_material( raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error return source, material material = envelope.value - model = SourceObservation if envelope.representation == "observation" else Source - source = load_model(model, rfc8785.dumps(material), kind="source", name=reference.source_type) + source = self._historical_source(reference, envelope) decoded = SourceRef( source_type=source.source_type if isinstance(source, SourceObservation) else reference.source_type, source_id=source.name, @@ -391,6 +395,40 @@ async def _historical_material( self._validate_unbound_material(reference, envelope, source) return source, material + def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayload) -> Source: + """Decode identity independently of retained material and today's business defaults.""" + + material = envelope.value + if envelope.representation == "observation": + return load_model(SourceObservation, rfc8785.dumps(material), kind="source", name=reference.source_type) + if envelope.source is not None: + keys = envelope.source_keys or {key: key for key in ("name", "definition_version", "materialization")} + for key, alias in keys.items(): + value = getattr(envelope.source, key) + if alias not in material or material[alias] != value: + raise IdentityMismatchError("source-envelope", value, material.get(alias)) + return envelope.source + # Legacy aliased rows need their registered decoder once; canonical rows do not. + with suppress(SourceDefinitionNotFoundError): + definition = self._registry.definition_for_name(reference.source_type) + if any( + (definition.source_class.model_fields[key].serialization_alias or key) != key + for key in ("name", "definition_version", "materialization") + ): + source, _ = self._decode_envelope(reference.source_type, envelope) + return source + if {"name", "definition_version", "materialization"} <= material.keys(): + return load_model(Source, rfc8785.dumps(material), kind="source", name=reference.source_type) + header = material + with suppress(SourceDefinitionNotFoundError): + definition = self._registry.definition_for_name(reference.source_type) + header = _normalize_native_material(definition.source_class, material) + try: + return load_model(Source, rfc8785.dumps(header), kind="source", name=reference.source_type) + except InvalidStoredPayloadError: + source, _ = self._decode_envelope(reference.source_type, envelope) + return source + def _validate_unbound_material(self, reference: SourceRef, envelope: _StoredSourcePayload, source: Source) -> None: """Keep supported exact-read errors when no historical schema is available. @@ -665,6 +703,41 @@ def _decode_row(self, row: Mapping[Any, Any]) -> StoredSource: journal_position=int(row["journal_position"]), ) + async def _retain_native_header( + self, connection: AsyncConnection, scope_id: str, ref: SourceRef, row: Mapping[Any, Any], source: Source + ) -> None: + """Enrich only an identical replay's unreconstructible legacy header. + + The retained value, binding, manifest, digest and journal position never change. + This grants no authority and does not publish another processing intent. + """ + + payload = stored_bytes(row["payload"], column="payload") + try: + envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=ref.source_type) + except InvalidStoredPayloadError: + envelope = _StoredSourcePayload( + encoding="powercontext-source-v1", + representation="native", + value=_JSON_OBJECT.validate_json(payload, strict=True), + ) + if envelope.source is not None: + return + header = _source_header(source) + with suppress(InvalidStoredPayloadError): + if load_model(Source, rfc8785.dumps(envelope.value), kind="source", name=ref.source_type) == header: + return + enriched = envelope.model_copy(update={"source": header, "source_keys": _source_keys(source, envelope.value)}) + await connection.execute( + update(SOURCES_TABLE) + .where( + SOURCES_TABLE.c.scope_id == scope_id, + SOURCES_TABLE.c.source_type == ref.source_type, + SOURCES_TABLE.c.source_id == ref.source_id, + ) + .values(payload=rfc8785.dumps(enriched.model_dump(mode="json", exclude_none=True))) + ) + def _decode_envelope( self, source_type: str, @@ -683,11 +756,15 @@ def _decode_envelope( definition = self._definition_by_name(source_type) source = load_model( definition.source_class, - payload, + rfc8785.dumps(_normalize_native_material(definition.source_class, envelope.value)), kind="source", name=source_type, ) self._registry.definition_for_source(source) + if envelope.source is not None: + for key in ("name", "definition_version", "materialization"): + if getattr(envelope.source, key) != getattr(source, key): + raise IdentityMismatchError("source-envelope", envelope.source, _source_header(source)) return source, SourceRef(source_type=definition.name, source_id=source.name) def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Source, SourceRef]: @@ -702,9 +779,13 @@ def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Sour ) decoded = SourceRef(source_type=source.source_type, source_id=source.name) else: + try: + material = _JSON_OBJECT.validate_json(payload, strict=True) + except ValueError as error: + raise InvalidStoredPayloadError("source", source_type, "payload is not JSON") from error source = load_model( definition.source_class, - payload, + rfc8785.dumps(_normalize_native_material(definition.source_class, material)), kind="source", name=source_type, ) @@ -743,6 +824,55 @@ def _source_material_digest_material(material: Mapping[str, JsonValue]) -> str: return f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}" +def _source_header(source: Source) -> Source: + return Source(**{key: getattr(source, key) for key in Source.model_fields}) + + +def _source_keys(source: Source, material: dict[str, JsonValue] | None = None) -> dict[str, str]: + """Retain declared identity spellings, including pre-canonical generated keys.""" + keys = { + key: type(source).model_fields[key].serialization_alias or key + for key in ("name", "definition_version", "materialization") + } + generator = type(source).model_config.get("alias_generator") + if isinstance(generator, AliasGenerator): + generator = generator.serialization_alias or generator.alias + if material is not None and generator is not None: + for key, alias in keys.items(): + if alias not in material: + keys[key] = generator(key) + return keys + + +def _normalize_native_material(source_class: type[Source], material: dict[str, JsonValue]) -> dict[str, JsonValue]: + """Read pre-canonical inherited envelope aliases, never business-field aliases. + + Derive spellings from the registered generator, not a version/key allowlist. + Explicit adapter aliases and conflicting identities retain strict validation. + """ + + generator = source_class.model_config.get("alias_generator") + if isinstance(generator, AliasGenerator): + generator = generator.serialization_alias or generator.alias + if generator is None: + return material + normalized = dict(material) + for key in Source.model_fields: + if source_class.model_fields[key].validation_alias != key: + continue + alias = generator(key) + if any( + field_name != key and alias in (field.alias, field.validation_alias, field.serialization_alias) + for field_name, field in source_class.model_fields.items() + ): + continue + if alias != key and alias in normalized: + if key in normalized and normalized[key] != normalized[alias]: + raise IdentityMismatchError("source", normalized[key], normalized[alias]) + normalized[key] = normalized.pop(alias) + return normalized + + def _dump_source( source: Source, *, @@ -754,8 +884,10 @@ def _dump_source( encoding="powercontext-source-v1", representation=representation, value=_JSON_OBJECT.validate_python(source.model_dump(mode="json", by_alias=True)), + source=None if representation == "observation" else _source_header(source), + source_keys=None if representation == "observation" else _source_keys(source), ) - return rfc8785.dumps(envelope.model_dump(mode="json")) + return rfc8785.dumps(envelope.model_dump(mode="json", exclude_none=True)) except (TypeError, ValueError) as error: raise InvalidStoredPayloadError("source", name, "value is not JSON serializable") from error diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 0f66d408f5..cb1e0a5521 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +import json import pytest from pydantic import BaseModel, ConfigDict, Field @@ -35,6 +36,7 @@ from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.sources import ( AdapterSourceDefinition, + MemoryEvidenceDeclaration, Source, SourceDefinitionRegistry, SourceMaterialization, @@ -265,6 +267,94 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_precanonical_generated_alias_source_rows_remain_readable_and_replayable() -> None: + async def scenario() -> None: + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(NativeCamelCaseSource), version="2"), + )) + repository = SourceRepository(registry) + source = NativeCamelCaseSource( + name="legacy-2", definition_version="2", materialization=SourceMaterialization.CAPTURED, body="Retained." + ) + material = source.model_dump(mode="json", by_alias=True) + material["definitionVersion"] = material.pop("definition_version") + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + async with ( + repository_profile() as (profile, _repositories), + profile.database.transaction() as connection, + ): + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="scope-a", + source_type="serialization-alias", + source_id=source.name, + payload=payload, + journal_position=1, + ) + ) + ref = SourceRef(source_type="serialization-alias", source_id=source.name) + loaded = await repository.get(connection, "scope-a", ref) + assert loaded.value == source + assert await repository.add(connection, "scope-a", source) == loaded + assert await repository.journal_position(connection, "scope-a") == 1 + + asyncio.run(scenario()) + + +def test_explicit_name_alias_evidence_survives_restart_without_the_native_definition(tmp_path) -> None: + class AliasedSource(NativeCamelCaseSource): + name: str = Field(alias="source_name") + business_name: str = Field(alias="name") + description: str | None = Field(default="captured description", exclude=True) + + class UpdatedSource(AliasedSource): + description: str | None = Field(default="current description", exclude=True) + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition( + SerializationAliasAdapter(AliasedSource), + version="2", + memory_evidence=declaration, + ), + )) + source = AliasedSource( + source_name="native-alias", + name="business label", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Retained.", + ) + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'explicit-alias.db'}") + repository = SourceRepository(registry) + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + stored = await repository.add(connection, "scope-a", source) + assert (await repository.get(connection, "scope-a", stored.ref)).value == source + assert await repository.add(connection, "scope-a", source) == stored + evidence = await repository.evidence(connection, "scope-a", (stored.ref,)) + assert evidence[0].declaration == declaration + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + updated = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(UpdatedSource), version="2"), + )) + loaded = await SourceRepository(updated).get(connection, "scope-a", stored.ref) + assert loaded.value.description == "current description" + assert await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) == evidence + + asyncio.run(scenario()) + + def test_repository_rejects_a_scope_outside_the_relational_identity_baseline() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py index 5fe44a12e3..7e0178c5df 100644 --- a/tests/e2e/test_atomic_memory_source_evidence.py +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -19,6 +19,7 @@ import asyncio import json from dataclasses import replace +from typing import cast import pytest from sqlalchemy import event, text @@ -31,8 +32,15 @@ AtomicMemoryExtractionOutput, ) from powercontext.builtin.inference import GenerationResult -from powercontext.builtin.runtime import BuiltinConfig, SubmitSourceObservation, open_builtin_contexts +from powercontext.builtin.runtime import ( + BuiltinConfig, + SubmitSourceObservation, + open_builtin_contexts, + open_builtin_runtime, +) from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection +from powercontext.builtin.runtime.relational import RelationalContexts +from powercontext.builtin.scope import ScopeDraft from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceRef from powercontext.sources.evidence import MemoryEvidenceDeclaration @@ -226,3 +234,45 @@ async def scenario() -> None: ) asyncio.run(scenario()) + + +def test_shared_merge_and_restore_preserve_source_evidence(migration_backend: MigrationBackend, tmp_path) -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry((replace(CONTENT_SOURCE_DEFINITION, memory_evidence=declaration),)) + + async def scenario() -> None: + async with open_builtin_runtime( + BuiltinConfig(database=migration_backend.config), + source_registry=registry, + scheduler_path=tmp_path / "scheduler.db", + ) as runtime: + assert runtime.scopes is not None + scope = await runtime.scopes.create( + ScopeDraft(title="Evidence", summary="Shared merge evidence", idempotency_key="source-evidence") + ) + contexts = cast(RelationalContexts, runtime._provider) + source = SourceRef(source_type="content", source_id="release") + await contexts.records.capture_source(scope.scope_id, "content", "release", "Exact release material.", {}) + for identity in ("a", "b"): + await _commit(contexts, scope.scope_id, identity, identity + " rule.", sources=(source,)) + memory = contexts.atomic_memory.for_scope(scope.scope_id) + accepted = await memory.evidence("a", revision=1) + assert runtime.artifact_merge is not None + merge = runtime.artifact_merge.for_scope(scope.scope_id, "atomic-memory") + a, b = await merge.get("a"), await merge.get("b") + await merge.merge( + (a.as_read(), b.as_read()), + AtomicMemoryContent(kind="fact", text="Combined release rule."), + artifact_id="combined", + lineage=ArtifactLineage(sources=(source,), artifacts=(a.ref, b.ref)), + ) + annotation = await memory.evidence("combined", revision=1) + assert annotation.sources == accepted.sources + assert annotation.sources[0].declaration == declaration + restored = await merge.restore("a") + assert restored.primary.ref.revision == 2 + assert (await memory.evidence("a", revision=2)).sources[0].declaration == declaration + assert await memory.evidence("combined", revision=1) == annotation + assert await memory.evidence("a", revision=1) == accepted + + asyncio.run(scenario()) From 6a836b1106e1056cee5c00f85b27c89939b4c4d8 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Mon, 19 Oct 2026 12:05:05 +0800 Subject: [PATCH 11/94] fix(sqlite): preserve startup catalog compatibility --- .../builtin/persistence/migrations/atomic_memory_evidence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py index 14e2e2809f..d23f81fd2e 100644 --- a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py +++ b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py @@ -105,7 +105,7 @@ async def require_atomic_memory_evidence_schema(connection: AsyncConnection, tab async def _require_shape(connection: AsyncConnection, name: str, statement: str, dialect: str) -> None: if dialect == "sqlite": actual = await connection.exec_driver_sql( - "SELECT type, name, sql FROM sqlite_schema WHERE (name = ? OR tbl_name = ?) " + "SELECT type, name, sql FROM sqlite_master WHERE (name = ? OR tbl_name = ?) " "AND substr(name, 1, 7) != 'sqlite_' ORDER BY type, name", (name, name), ) From b5a4c1e38534f8ecfe4842de945d9331932867f3 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Mon, 19 Oct 2026 12:05:05 +0800 Subject: [PATCH 12/94] fix(sources): keep unverifiable evidence neutral Check native serialization compatibility at acceptance without blocking valid captures or Memory writes. Keep previously accepted incompatible declarations neutral while preserving exact material and digest checks. Reject legacy identities that cannot be decoded exactly instead of supplying a default Definition version. --- .../builtin/persistence/sources.py | 91 ++++++++++++------- tests/builtin/persistence/test_sources.py | 3 + .../e2e/test_atomic_memory_source_evidence.py | 69 +++++++++++++- 3 files changed, 129 insertions(+), 34 deletions(-) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 24cff31caf..e2c465ad2e 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -307,7 +307,9 @@ async def historical_evidence( row = by_ref.get(ref) if row is None: raise RepositoryNotFoundError("source", (scope_id, reference)) - source, material = await self._historical_material(connection, reference, row) + source, material, verifiable = await self._historical_material(connection, reference, row) + if not verifiable: + fingerprints[ref] = "" # Never authorize a retained declaration for this capture. if isinstance(source, SourceObservation): fingerprints[ref] = source.definition_fingerprint elif ref[0] == CONTENT_SOURCE_NAME: @@ -334,33 +336,14 @@ async def historical_evidence( async def _historical_material( self, connection: AsyncConnection, reference: SourceRef, row: Mapping[Any, Any] - ) -> tuple[Source, dict[str, JsonValue]]: + ) -> tuple[Source, dict[str, JsonValue], bool]: """Validate the stored representation and its retained schema, not a live model.""" payload = stored_bytes(row["payload"], column="payload") try: envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=reference.source_type) except InvalidStoredPayloadError: - # Bare legacy material has no representation discriminator. Use - # its registered decoder when available; otherwise retain opaque - # canonical fields for neutral maintenance, not a guessed schema. - try: - self._registry.definition_for_name(reference.source_type) - except SourceDefinitionNotFoundError: - source = load_model(Source, payload, kind="source", name=reference.source_type) - if source.name != reference.source_id: - raise IdentityMismatchError("source", reference.source_id, source.name) from None - try: - material = _JSON_OBJECT.validate_json(payload, strict=True) - except ValueError as error: - raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error - return source, material - source = self._decode_row(row).value - try: - material = _JSON_OBJECT.validate_json(payload, strict=True) - except ValueError as error: - raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error - return source, material + return self._historical_legacy_material(reference, row, payload) material = envelope.value source = self._historical_source(reference, envelope) decoded = SourceRef( @@ -370,22 +353,30 @@ async def _historical_material( if reference != decoded: raise IdentityMismatchError("source", reference, decoded) table = SOURCE_EVIDENCE_BINDINGS_TABLE - fingerprint = await connection.scalar( - select(table.c.manifest_fingerprint).where( - table.c.scope_id == row["scope_id"], - table.c.source_type == reference.source_type, - table.c.source_id == reference.source_id, - table.c.definition_name == reference.source_type, - table.c.definition_version == source.definition_version, + binding = ( + await connection.execute( + select(table.c.manifest_fingerprint, table.c.material_digest).where( + table.c.scope_id == row["scope_id"], + table.c.source_type == reference.source_type, + table.c.source_id == reference.source_id, + table.c.definition_name == reference.source_type, + table.c.definition_version == source.definition_version, + ) ) - ) + ).one_or_none() manifest = await _find_evidence_manifest(connection, reference.source_type, source.definition_version) - if manifest is not None and manifest.fingerprint == fingerprint: + if manifest is not None and binding is not None and manifest.fingerprint == binding.manifest_fingerprint: try: Draft202012Validator.check_schema(manifest.source_schema) value = source.payload if isinstance(source, SourceObservation) else material Draft202012Validator(manifest.source_schema, registry=Registry()).validate(value) except JsonSchemaValidationError as error: + if envelope.representation == "native" and binding.material_digest == _source_material_digest_material( + material + ): + # Older releases bound declarations without checking serialization + # compatibility. Exact accepted bytes remain readable, but neutral. + return source, material, False raise InvalidStoredPayloadError( "source", reference.source_type, "payload does not match the historical schema" ) from error @@ -393,7 +384,29 @@ async def _historical_material( self._validate_unbound_material(reference, envelope, source) else: self._validate_unbound_material(reference, envelope, source) - return source, material + return source, material, True + + def _historical_legacy_material( + self, reference: SourceRef, row: Mapping[Any, Any], payload: bytes + ) -> tuple[Source, dict[str, JsonValue], bool]: + """Read bare legacy material without guessing absent identity fields.""" + try: + material = _JSON_OBJECT.validate_json(payload, strict=True) + except ValueError as error: + raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error + try: + self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + source = load_model(Source, payload, kind="source", name=reference.source_type) + if not {"name", "definition_version", "materialization"} <= source.model_fields_set: + raise InvalidStoredPayloadError( + "source", reference.source_type, "legacy native identity requires its registered decoder" + ) from None + if source.name != reference.source_id: + raise IdentityMismatchError("source", reference.source_id, source.name) from None + else: + source = self._decode_row(row).value + return source, material, True def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayload) -> Source: """Decode identity independently of retained material and today's business defaults.""" @@ -423,6 +436,10 @@ def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayloa with suppress(SourceDefinitionNotFoundError): definition = self._registry.definition_for_name(reference.source_type) header = _normalize_native_material(definition.source_class, material) + if not {"name", "definition_version", "materialization"} <= header.keys(): + raise InvalidStoredPayloadError( + "source", reference.source_type, "legacy native identity requires its registered decoder" + ) try: return load_model(Source, rfc8785.dumps(header), kind="source", name=reference.source_type) except InvalidStoredPayloadError: @@ -526,6 +543,16 @@ async def _bind_memory_evidence( else: definition = self._registry.definition_for_source(source) manifest = await SourceDefinitionManifestRepository().register_native(connection, definition) + if manifest is not None: + # A native decoder can accept representations its generated schema + # cannot describe. Such captures are valid, but cannot attest evidence. + try: + Draft202012Validator.check_schema(manifest.source_schema) + Draft202012Validator(manifest.source_schema, registry=Registry()).validate( + source.model_dump(mode="json", by_alias=True) + ) + except (JsonSchemaValidationError, SchemaError, Unresolvable): + manifest = None declaration = ( MemoryEvidenceDeclaration() if manifest is None else self._registry.memory_evidence_for_source(source) ) diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index cb1e0a5521..9df98d87d2 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -25,6 +25,7 @@ from powercontext import InvalidSourceDefinitionError from powercontext.builtin.persistence import ( InvalidRepositoryArgumentError, + InvalidStoredPayloadError, RepositoryNotFoundError, StoredPayloadConflictError, ) @@ -297,6 +298,8 @@ async def scenario() -> None: ) ) ref = SourceRef(source_type="serialization-alias", source_id=source.name) + with pytest.raises(InvalidStoredPayloadError): + await SourceRepository(()).historical_evidence(connection, "scope-a", (ref,)) loaded = await repository.get(connection, "scope-a", ref) assert loaded.value == source assert await repository.add(connection, "scope-a", source) == loaded diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py index 7e0178c5df..551175e8c3 100644 --- a/tests/e2e/test_atomic_memory_source_evidence.py +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -22,6 +22,7 @@ from typing import cast import pytest +from pydantic import field_serializer, field_validator from sqlalchemy import event, text from sqlalchemy.exc import SQLAlchemyError @@ -42,10 +43,10 @@ from powercontext.builtin.runtime.relational import RelationalContexts from powercontext.builtin.scope import ScopeDraft from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION -from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceRef +from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceMaterialization, SourceRef from powercontext.sources.evidence import MemoryEvidenceDeclaration from powercontext.sources.observations import manifest_for_definition, project_source_for_transport -from tests.builtin.persistence.contract import NoteAdapter, NoteInput +from tests.builtin.persistence.contract import NoteAdapter, NoteInput, NoteSource from tests.e2e.atomic_memory_migration_backend import MigrationBackend from tests.e2e.dream_support import atomic_memory_pipeline @@ -70,6 +71,70 @@ async def _commit(contexts, scope_id, artifact_id, text_value, *, sources=(), ar return result.primary +@pytest.mark.parametrize("declared", [False, True]) +def test_unverifiable_native_serializer_keeps_memory_evidence_neutral(migration_backend, declared) -> None: + class SerializedNote(NoteSource): + count: int = 12 + + @field_serializer("count") + def serialize_count(self, value): + return str(value) + + @field_validator("count", mode="before") + @classmethod + def parse_count(cls, value): + return int(value) + + class SerializedAdapter(NoteAdapter): + source_class = SerializedNote + + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializedAdapter(), memory_evidence=declaration if declared else None), + )) + config = BuiltinConfig(database=migration_backend.config) + + async def scenario(): + async with open_builtin_contexts(config, source_registry=registry) as contexts: + context = await contexts.get("scope") + source = SerializedNote(name="serialized", materialization=SourceMaterialization.CAPTURED, body="Retained.") + await context.sources.add(source) + assert await context.sources.get(source) == source + assert await context.sources.add(source) == source + ref = SourceRef(source_type="note", source_id=source.name) + record = await _commit(contexts, "scope", "serialized", "Retained fact.", sources=(ref,)) + evidence = await contexts.atomic_memory.for_scope("scope").evidence("serialized", revision=1) + assert evidence.artifact_ref == record.ref + assert evidence.sources[0].source_ref == ref + assert evidence.sources[0].declaration == MemoryEvidenceDeclaration() + assert evidence.sources[0].manifest_fingerprint is None + if declared: + # Retain the binding shape accepted before schema compatibility checks. + async with contexts.database.transaction() as connection: + await connection.execute( + text( + "UPDATE pc_source_evidence_bindings SET manifest_fingerprint = " + "(SELECT fingerprint FROM pc_source_definition_manifests WHERE definition_name = 'note'), " + "declaration = :declaration WHERE scope_id = 'scope' AND source_id = 'serialized'" + ), + {"declaration": declaration.model_dump_json().encode()}, + ) + await _commit(contexts, "scope", "legacy", "Previously accepted fact.", sources=(ref,)) + legacy = await contexts.atomic_memory.for_scope("scope").evidence("legacy", revision=1) + assert legacy.sources == evidence.sources + async with open_builtin_contexts(config, source_registry=SourceDefinitionRegistry(())) as contexts: + report = await rebuild_atomic_memory_projection( + contexts.database, + contexts.atomic_memory.index, + maintenance_confirmed=True, + sources=contexts.repositories.sources, + ) + assert report.ready, report.errors + assert await contexts.atomic_memory.for_scope("scope").evidence("serialized", revision=1) == evidence + + asyncio.run(scenario()) + + def test_mixed_evidence_survives_failed_revision_restart_and_rebuild(migration_backend: MigrationBackend) -> None: strong = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") asserted = MemoryEvidenceDeclaration(authority="user_asserted", verification="not_verified") From 2bce54e6fb33a18dccae4825f0218e887bff20a3 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Mon, 19 Oct 2026 13:39:43 +0800 Subject: [PATCH 13/94] fix(sources): reject omitted identity before capture Reject native serialized material that omits an identity field before journal allocation or persistence, rather than accepting Sources that later fail evidence capture. Preserve strict retained identity checks. --- .../builtin/persistence/sources.py | 4 ++++ tests/builtin/persistence/test_sources.py | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index e2c465ad2e..92e0a543a4 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -914,6 +914,10 @@ def _dump_source( source=None if representation == "observation" else _source_header(source), source_keys=None if representation == "observation" else _source_keys(source), ) + if envelope.source_keys is not None: + for key, alias in envelope.source_keys.items(): + if alias not in envelope.value: + raise InvalidStoredPayloadError("source", name, f"serialized native identity omits {key!r}") return rfc8785.dumps(envelope.model_dump(mode="json", exclude_none=True)) except (TypeError, ValueError) as error: raise InvalidStoredPayloadError("source", name, "value is not JSON serializable") from error diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 9df98d87d2..dcfe399aaf 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -308,6 +308,26 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_excluded_native_identity_is_rejected_before_capture() -> None: + class ExcludedVersionSource(NativeCamelCaseSource): + definition_version: str = Field(default="1", exclude=True) + + async def scenario() -> None: + repository = SourceRepository((SerializationAliasAdapter(ExcludedVersionSource),)) + source = ExcludedVersionSource( + name="excluded", materialization=SourceMaterialization.CAPTURED, body="Retained." + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + with pytest.raises(InvalidStoredPayloadError): + await repository.add(connection, "scope-a", source) + ref = SourceRef(source_type="serialization-alias", source_id=source.name) + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, "scope-a", ref) + assert await repository.journal_position(connection, "scope-a") == 0 + + asyncio.run(scenario()) + + def test_explicit_name_alias_evidence_survives_restart_without_the_native_definition(tmp_path) -> None: class AliasedSource(NativeCamelCaseSource): name: str = Field(alias="source_name") From bb36c937d43f99011b516fb74e4d984e1562c783 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Mon, 19 Oct 2026 17:36:26 +0800 Subject: [PATCH 14/94] fix(sources): reject transformed identity before capture Validate serialized identity values as well as key presence before persistence, matching historical evidence validation. Cover round-trippable codecs for all three identity fields without weakening retained-material checks. --- .../builtin/persistence/sources.py | 2 ++ tests/builtin/persistence/test_sources.py | 34 ++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 92e0a543a4..726e249e29 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -918,6 +918,8 @@ def _dump_source( for key, alias in envelope.source_keys.items(): if alias not in envelope.value: raise InvalidStoredPayloadError("source", name, f"serialized native identity omits {key!r}") + if envelope.value[alias] != getattr(source, key): + raise InvalidStoredPayloadError("source", name, f"serialized native identity changes {key!r}") return rfc8785.dumps(envelope.model_dump(mode="json", exclude_none=True)) except (TypeError, ValueError) as error: raise InvalidStoredPayloadError("source", name, "value is not JSON serializable") from error diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index dcfe399aaf..6c4802af34 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -16,9 +16,10 @@ import asyncio import json +from urllib.parse import quote, unquote import pytest -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator from pydantic.alias_generators import to_camel from sqlalchemy import insert @@ -328,6 +329,37 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("identity_field", ["name", "definition_version", "materialization"]) +def test_encoded_native_identity_is_rejected_before_capture(identity_field) -> None: + class EncodedSource(NativeCamelCaseSource): + @field_serializer("name", "definition_version", "materialization") + def encode_identity(self, value, info): + if info.field_name == identity_field: + return quote(f"wire:{value}", safe="") + return value + + @field_validator("name", "definition_version", "materialization", mode="before") + @classmethod + def decode_identity(cls, value, info): + if info.field_name == identity_field: + return unquote(value).removeprefix("wire:") + return value + + async def scenario() -> None: + repository = SourceRepository((SerializationAliasAdapter(EncodedSource),)) + source = EncodedSource(name="note/42", materialization=SourceMaterialization.CAPTURED, body="Retained.") + assert EncodedSource.model_validate_json(source.model_dump_json(by_alias=True)) == source + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + with pytest.raises(InvalidStoredPayloadError): + await repository.add(connection, "scope-a", source) + ref = SourceRef(source_type="serialization-alias", source_id=source.name) + with pytest.raises(RepositoryNotFoundError): + await repository.get(connection, "scope-a", ref) + assert await repository.journal_position(connection, "scope-a") == 0 + + asyncio.run(scenario()) + + def test_explicit_name_alias_evidence_survives_restart_without_the_native_definition(tmp_path) -> None: class AliasedSource(NativeCamelCaseSource): name: str = Field(alias="source_name") From 73ec5aa8348ac642da87711207aa39ac9dd5a6c8 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Tue, 20 Oct 2026 13:08:43 +0800 Subject: [PATCH 15/94] fix(sources): separate worker and historical identity decoding Canonicalize worker envelope keys and their manifest without changing native persisted schemas. Read retained canonical identities before consulting current adapter aliases, preserving neutral historical evidence. --- .../builtin/persistence/source_definitions.py | 5 +- .../builtin/persistence/sources.py | 12 ++- src/powercontext/sources/observations.py | 48 +++++++++- tests/builtin/persistence/test_sources.py | 74 ++++++++++++++- tests/test_source_observations.py | 90 ++++++++++++++++++- 5 files changed, 221 insertions(+), 8 deletions(-) diff --git a/src/powercontext/builtin/persistence/source_definitions.py b/src/powercontext/builtin/persistence/source_definitions.py index 3705e9b566..beffb3f87f 100644 --- a/src/powercontext/builtin/persistence/source_definitions.py +++ b/src/powercontext/builtin/persistence/source_definitions.py @@ -32,7 +32,8 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.tables import SOURCE_DEFINITION_MANIFESTS_TABLE -from powercontext.sources import SourceDefinition, SourceDefinitionManifest, manifest_for_definition +from powercontext.sources import SourceDefinition, SourceDefinitionManifest +from powercontext.sources.observations import _manifest_for_definition class SourceDefinitionManifestRepository: @@ -86,7 +87,7 @@ async def register_native( """ try: - manifest = manifest_for_definition(definition) + manifest = _manifest_for_definition(definition, transport=False) except Exception: # Native schema providers may not expose a valid manifest. return None try: diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 726e249e29..d896200386 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -422,6 +422,10 @@ def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayloa raise IdentityMismatchError("source-envelope", value, material.get(alias)) return envelope.source # Legacy aliased rows need their registered decoder once; canonical rows do not. + if {"name", "definition_version", "materialization"} <= material.keys() and material[ + "name" + ] == reference.source_id: + return load_model(Source, rfc8785.dumps(material), kind="source", name=reference.source_type) with suppress(SourceDefinitionNotFoundError): definition = self._registry.definition_for_name(reference.source_type) if any( @@ -430,8 +434,6 @@ def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayloa ): source, _ = self._decode_envelope(reference.source_type, envelope) return source - if {"name", "definition_version", "materialization"} <= material.keys(): - return load_model(Source, rfc8785.dumps(material), kind="source", name=reference.source_type) header = material with suppress(SourceDefinitionNotFoundError): definition = self._registry.definition_for_name(reference.source_type) @@ -459,6 +461,12 @@ def _validate_unbound_material(self, reference: SourceRef, envelope: _StoredSour definition = self._registry.definition_for_name(reference.source_type) except SourceDefinitionNotFoundError: return + if envelope.source is None and any( + (definition.source_class.model_fields[key].serialization_alias or key) != key + for key in ("name", "definition_version", "materialization") + ): + # Today's identity aliases cannot validate an older unbound representation. + return if definition.version == source.definition_version: self._decode_envelope(reference.source_type, envelope) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index c9bb8a802f..f3e11e676c 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -161,9 +161,26 @@ def projection(self, key: SourceProjectionKey, /) -> JsonValue: def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> SourceDefinitionManifest: - """Build the immutable declaration transported by a remote worker.""" + """Build a worker declaration with canonical envelope keys and aliased business fields.""" + return _manifest_for_definition(definition, transport=True) + + +def _manifest_for_definition( + definition: SourceDefinition[Any, Any, Any], + /, + *, + transport: bool, +) -> SourceDefinitionManifest: + aliases = _transport_identity_aliases(definition.source_class) if transport else {} source_schema = _json_object(definition.source_class.model_json_schema(mode="serialization")) + if aliases: + properties = source_schema.get("properties") + if isinstance(properties, dict): + source_schema["properties"] = {aliases.get(key, key): value for key, value in properties.items()} + required = source_schema.get("required") + if isinstance(required, list): + source_schema["required"] = [aliases.get(key, key) if isinstance(key, str) else key for key in required] projections = tuple( SourceProjectionManifest( key=SourceProjectionKey(name=projection.name, version=projection.version), @@ -200,6 +217,8 @@ def project_source_for_transport( ) manifest = manifest_for_definition(definition) payload = _json_object(source.model_dump(mode="json", by_alias=True)) + aliases = _transport_identity_aliases(type(source)) + payload = {aliases.get(key, key): value for key, value in payload.items()} projections = tuple( SourceProjectionValue(key=key, value=registry.project(source, key)) for key in registry.projection_keys(source) ) @@ -215,6 +234,33 @@ def project_source_for_transport( ) +def _transport_identity_aliases(source_class: type[Source], /) -> dict[str, str]: + """Reject ambiguous aliases before moving envelope keys in schemas or values.""" + + identity = Source.model_fields + fields = source_class.model_fields + serialized: dict[str, str] = {} + projected: set[str] = set() + for name, field in fields.items(): + if field.exclude: + continue + alias = field.serialization_alias if field.serialization_alias is not None else name + if alias in serialized: + raise InvalidSourceDefinitionError( + source_class, + "serialization_alias", + f"{alias!r} is shared by {serialized[alias]!r} and {name!r}", + ) + serialized[alias] = name + target = name if name in identity else alias + if target in projected: + raise InvalidSourceDefinitionError( + source_class, "serialization_alias", f"{target!r} collides after envelope normalization" + ) + projected.add(target) + return {alias: name for alias, name in serialized.items() if name in identity} + + def _source_definition_fingerprint( *, name: str, diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 6c4802af34..6bdd8bc39a 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -21,7 +21,7 @@ import pytest from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator from pydantic.alias_generators import to_camel -from sqlalchemy import insert +from sqlalchemy import delete, insert, update from powercontext import InvalidSourceDefinitionError from powercontext.builtin.persistence import ( @@ -30,6 +30,7 @@ RepositoryNotFoundError, StoredPayloadConflictError, ) +from powercontext.builtin.persistence.atomic_memory_evidence_schema import SOURCE_EVIDENCE_BINDINGS_TABLE from powercontext.builtin.persistence.codec import dump_model from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile @@ -269,6 +270,77 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("current_version", ["1", "2"]) +def test_canonical_history_survives_current_identity_aliases(current_version) -> None: + class CurrentSource(NativeCamelCaseSource): + name: str = Field(alias="source_name") + + async def scenario() -> None: + original = SourceRepository((SerializationAliasAdapter(NativeCamelCaseSource),)) + source = NativeCamelCaseSource( + name="canonical", materialization=SourceMaterialization.CAPTURED, body="Retained." + ) + material = source.model_dump(mode="json", by_alias=True) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await original.add(connection, "scope-a", source) + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + await connection.execute(delete(SOURCE_EVIDENCE_BINDINGS_TABLE)) + expected = await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) + current = SourceRepository( + SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version=current_version), + )) + ) + assert await current.historical_evidence(connection, "scope-a", (stored.ref,)) == expected + assert expected[0].definition_version == "1" + assert expected[0].declaration == MemoryEvidenceDeclaration() + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("business_version", ["1", "2"]) +def test_bound_aliased_history_does_not_treat_business_fields_as_identity(business_version) -> None: + class AliasedSource(NativeCamelCaseSource): + definition_version: str = Field(alias="source_version") + materialization: SourceMaterialization = Field(alias="source_materialization") + business_version: str = Field(alias="definition_version") + business_kind: str = Field(alias="materialization") + + async def scenario() -> None: + declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") + registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(AliasedSource), version="2", memory_evidence=declaration), + )) + repository = SourceRepository(registry) + source = AliasedSource( + name="aliased-history", source_version="2", source_materialization=SourceMaterialization.CAPTURED, + definition_version=business_version, materialization="referenced", body="Retained.", + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await repository.add(connection, "scope-a", source) + expected = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) + payload = json.dumps({ + "encoding": "powercontext-source-v1", "representation": "native", + "value": source.model_dump(mode="json", by_alias=True), + }).encode() + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + assert await repository.historical_evidence(connection, "scope-a", (stored.ref,)) == expected + assert expected[0].definition_version == "2" + assert expected[0].materialization is SourceMaterialization.CAPTURED + assert expected[0].declaration == declaration + + asyncio.run(scenario()) + + def test_precanonical_generated_alias_source_rows_remain_readable_and_replayable() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry(( diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index c313ae7962..cf27fe086a 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -18,17 +18,24 @@ import urllib.request import pytest -from pydantic import ValidationError +from pydantic import Field, ValidationError from referencing.exceptions import Unresolvable -from powercontext.builtin.runtime.relational import _json_schema_validator +from powercontext import InvalidSourceDefinitionError +from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import SHARED_TABLES +from powercontext.builtin.runtime import SubmitSourceObservation +from powercontext.builtin.runtime.relational import RelationalContexts, _json_schema_validator from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION, ContentCapture from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, + AdapterSourceDefinition, Source, SourceCatalog, SourceDefinitionManifest, SourceDefinitionRegistry, + SourceMaterialization, SourceObservation, TextEvidence, manifest_for_definition, @@ -75,6 +82,85 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize( + ("name_alias", "body_alias"), + [("source_name", "wire_text"), ("body", "wire_text"), ("source_name", "name"), ("wire_text", "wire_text")], +) +def test_serialization_aliases_preserve_the_worker_envelope_during_ingestion(name_alias: str, body_alias: str) -> None: + class AliasedSource(Source): + name: str = Field(serialization_alias=name_alias) + definition_version: str = Field(default="1", serialization_alias="wire_version") + materialization: SourceMaterialization = Field(serialization_alias="wire_materialization") + description: str | None = Field(default=None, serialization_alias="wire_description") + body: str = Field(serialization_alias=body_alias) + + class AliasedAdapter: + name = "aliased" + input_class = str + source_class = AliasedSource + + async def resolve(self, value: str, /) -> AliasedSource: + return AliasedSource( + name="remote-aliased", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + description="Aliased description.", + body=value, + ) + + async def read(self, source: AliasedSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(AliasedAdapter(), version="2") + worker = SourceDefinitionRegistry((definition,)) + source = await worker.resolve("Retain the aliased business value.") + if body_alias == "name" or name_alias == body_alias: + with pytest.raises(InvalidSourceDefinitionError): + manifest_for_definition(definition) + with pytest.raises(InvalidSourceDefinitionError): + project_source_for_transport(worker, source) + return + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(worker, source) + expected = { + "name": "remote-aliased", + "definition_version": "2", + "materialization": "captured", + "description": "Aliased description.", + "wire_text": "Retain the aliased business value.", + } + assert observation.payload == expected + assert source.model_dump(mode="json", by_alias=True) == { + name_alias: "remote-aliased", + "wire_version": "2", + "wire_materialization": "captured", + "wire_description": "Aliased description.", + "wire_text": "Retain the aliased business value.", + } + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + assert receipt.source_ref.source_id == "remote-aliased" + assert isinstance(stored, SourceObservation) + assert stored.payload == expected + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as native_profile: + async with native_profile.database.transaction() as connection: + native_manifest = await SourceDefinitionManifestRepository().register_native(connection, definition) + assert native_manifest is not None + assert native_manifest.source_schema == AliasedSource.model_json_schema(mode="serialization") + assert native_manifest.fingerprint != manifest.fingerprint + + asyncio.run(scenario()) + + def test_remote_source_observation_requires_captured_materialization() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) From 25d9e7a7d3514a7110761c472f0efb947b2e5826 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Tue, 20 Oct 2026 13:22:07 +0800 Subject: [PATCH 16/94] fix(sources): disambiguate canonical history from business aliases Only trust retained canonical identity keys when the registered model does not claim those spellings for business fields. Keep headerless business-aliased material on the declared decoder, and format the regression file with the pinned hook version. --- .../builtin/persistence/sources.py | 21 ++++++++++++++++++- tests/builtin/persistence/test_sources.py | 11 +++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index d896200386..fa4813985c 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -425,7 +425,11 @@ def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayloa if {"name", "definition_version", "materialization"} <= material.keys() and material[ "name" ] == reference.source_id: - return load_model(Source, rfc8785.dumps(material), kind="source", name=reference.source_type) + definition = None + with suppress(SourceDefinitionNotFoundError): + definition = self._registry.definition_for_name(reference.source_type) + if definition is None or not _claims_identity_keys(definition.source_class): + return load_model(Source, rfc8785.dumps(material), kind="source", name=reference.source_type) with suppress(SourceDefinitionNotFoundError): definition = self._registry.definition_for_name(reference.source_type) if any( @@ -879,6 +883,21 @@ def _source_keys(source: Source, material: dict[str, JsonValue] | None = None) - return keys +def _claims_identity_keys(source_class: type[Source]) -> bool: + """Report whether a business field serializes to a canonical identity spelling. + + A headerless envelope cannot distinguish an identity value from a business + value with that name. When the registered model claims the spelling for a + different field, only its declared aliases are authoritative. + """ + + return any( + name not in Source.model_fields and key in (field.alias, field.validation_alias, field.serialization_alias) + for name, field in source_class.model_fields.items() + for key in Source.model_fields + ) + + def _normalize_native_material(source_class: type[Source], material: dict[str, JsonValue]) -> dict[str, JsonValue]: """Read pre-canonical inherited envelope aliases, never business-field aliases. diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 6bdd8bc39a..849ef7fcdb 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -320,14 +320,19 @@ async def scenario() -> None: )) repository = SourceRepository(registry) source = AliasedSource( - name="aliased-history", source_version="2", source_materialization=SourceMaterialization.CAPTURED, - definition_version=business_version, materialization="referenced", body="Retained.", + name="aliased-history", + source_version="2", + source_materialization=SourceMaterialization.CAPTURED, + definition_version=business_version, + materialization="referenced", + body="Retained.", ) async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: stored = await repository.add(connection, "scope-a", source) expected = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) payload = json.dumps({ - "encoding": "powercontext-source-v1", "representation": "native", + "encoding": "powercontext-source-v1", + "representation": "native", "value": source.model_dump(mode="json", by_alias=True), }).encode() await connection.execute( From 5ceed572d1e938cc7f4251cf90060a32fbc555d5 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Tue, 20 Oct 2026 21:00:11 +0800 Subject: [PATCH 17/94] fix(sources): normalize recursive transport root schemas Resolve referenced root schemas before canonicalizing transport identity keys while preserving nested serialization aliases. Cover manifest registration, observation submission, and SQLite readback. --- src/powercontext/sources/observations.py | 9 ++++ tests/test_source_observations.py | 65 ++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index f3e11e676c..981f3d258c 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -175,6 +175,15 @@ def _manifest_for_definition( aliases = _transport_identity_aliases(definition.source_class) if transport else {} source_schema = _json_object(definition.source_class.model_json_schema(mode="serialization")) if aliases: + reference = source_schema.get("$ref") + definitions = source_schema.get("$defs") + if isinstance(reference, str) and reference.startswith("#/$defs/") and isinstance(definitions, dict): + key = reference.removeprefix("#/$defs/").replace("~1", "/").replace("~0", "~") + root = definitions.get(key) + if isinstance(root, dict): + # Inline only the root: recursive children retain their serialized aliases in $defs. + source_schema = {**source_schema, **root} + source_schema.pop("$ref") properties = source_schema.get("properties") if isinstance(properties, dict): source_schema["properties"] = {aliases.get(key, key): value for key, value in properties.items()} diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index cf27fe086a..a152d352a5 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -16,6 +16,7 @@ import asyncio import urllib.request +from typing import Self import pytest from pydantic import Field, ValidationError @@ -161,6 +162,70 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_recursive_source_transport_preserves_nested_serialization_aliases() -> None: + class RecursiveSource(Source): + name: str = Field(serialization_alias="source_name") + definition_version: str = Field(default="1", serialization_alias="wire_version") + body: str = Field(serialization_alias="wire_text") + children: list[Self] = Field(default_factory=list) + + class RecursiveAdapter: + name = "recursive" + input_class = str + source_class = RecursiveSource + + async def resolve(self, value: str, /) -> RecursiveSource: + leaf = RecursiveSource(name="leaf", materialization=SourceMaterialization.CAPTURED, body="Nested text.") + return RecursiveSource( + name="root", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body=value, + children=[leaf], + ) + + async def read(self, source: RecursiveSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(RecursiveAdapter(), version="2") + worker = SourceDefinitionRegistry((definition,)) + source = await worker.resolve("Root text.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(worker, source) + expected = { + "name": "root", + "definition_version": "2", + "materialization": "captured", + "description": None, + "wire_text": "Root text.", + "children": [ + { + "source_name": "leaf", + "wire_version": "1", + "materialization": "captured", + "description": None, + "wire_text": "Nested text.", + "children": [], + } + ], + } + + async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + await contexts.register_source_definition(manifest) + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + stored = await (await contexts.get("scope-a")).sources.get(observation) + + assert receipt.source_ref.source_id == "root" + assert isinstance(stored, SourceObservation) + assert stored.payload == expected + + asyncio.run(scenario()) + + def test_remote_source_observation_requires_captured_materialization() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) From 2b95df67139460e76a0aaf35162d3a0ff97dfc53 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 21 Oct 2026 11:16:59 +0800 Subject: [PATCH 18/94] fix(sources): preserve versioned alias compatibility Keep remote manifests and payloads on the canonical field-name contract so existing Definition fingerprints remain stable. Preserve native historical identity and retained alias spellings across upgrades, replay, and rebuild. --- .../builtin/persistence/sources.py | 137 +++++++++++-- src/powercontext/sources/models.py | 13 +- src/powercontext/sources/observations.py | 56 +----- tests/builtin/persistence/test_sources.py | 185 +++++++++++++++++- .../e2e/test_atomic_memory_source_evidence.py | 69 ++++++- tests/test_source_observations.py | 87 +++++--- 6 files changed, 442 insertions(+), 105 deletions(-) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index fa4813985c..b32990e757 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -25,7 +25,7 @@ from jsonschema import Draft202012Validator from jsonschema.exceptions import SchemaError from jsonschema.exceptions import ValidationError as JsonSchemaValidationError -from pydantic import AliasGenerator, BaseModel, ConfigDict, JsonValue, TypeAdapter +from pydantic import AliasChoices, AliasGenerator, BaseModel, ConfigDict, JsonValue, TypeAdapter from referencing import Registry from referencing.exceptions import Unresolvable from sqlalchemy import func, insert, select, tuple_, update @@ -343,9 +343,15 @@ async def _historical_material( try: envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=reference.source_type) except InvalidStoredPayloadError: - return self._historical_legacy_material(reference, row, payload) + return await self._historical_legacy_material(connection, reference, row, payload) material = envelope.value - source = self._historical_source(reference, envelope) + source = ( + await self._historical_canonical_source(connection, reference, row, material) + if envelope.representation == "native" and envelope.source is None + else None + ) + if source is None: + source = self._historical_source(reference, envelope) decoded = SourceRef( source_type=source.source_type if isinstance(source, SourceObservation) else reference.source_type, source_id=source.name, @@ -386,14 +392,21 @@ async def _historical_material( self._validate_unbound_material(reference, envelope, source) return source, material, True - def _historical_legacy_material( - self, reference: SourceRef, row: Mapping[Any, Any], payload: bytes + async def _historical_legacy_material( + self, + connection: AsyncConnection, + reference: SourceRef, + row: Mapping[Any, Any], + payload: bytes, ) -> tuple[Source, dict[str, JsonValue], bool]: """Read bare legacy material without guessing absent identity fields.""" try: material = _JSON_OBJECT.validate_json(payload, strict=True) except ValueError as error: raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error + source = await self._historical_canonical_source(connection, reference, row, material) + if source is not None: + return source, material, False try: self._registry.definition_for_name(reference.source_type) except SourceDefinitionNotFoundError: @@ -408,6 +421,78 @@ def _historical_legacy_material( source = self._decode_row(row).value return source, material, True + async def _historical_canonical_source( + self, + connection: AsyncConnection, + reference: SourceRef, + row: Mapping[Any, Any], + material: dict[str, JsonValue], + ) -> Source | None: + """Recover evidence identity from canonical retained material, never a live alias.""" + + if not {"name", "definition_version", "materialization"} <= material.keys(): + return None + if material["name"] != reference.source_id: + return None + digest_material = material + if reference.source_type == CONTENT_SOURCE_NAME: + digest_material = {key: value for key, value in material.items() if key != "handoff_receipt"} + material_digest = _source_material_digest_material(digest_material) + table = SOURCE_EVIDENCE_BINDINGS_TABLE + binding = ( + await connection.execute( + select(table.c.definition_name, table.c.definition_version, table.c.material_digest).where( + table.c.scope_id == row["scope_id"], + table.c.source_type == reference.source_type, + table.c.source_id == reference.source_id, + ) + ) + ).one_or_none() + if binding is not None: + if binding.definition_name != reference.source_type or binding.material_digest != material_digest: + return None + version = binding.definition_version + elif material["definition_version"] == "1": + version = "1" + else: + return None + try: + definition = self._registry.definition_for_name(reference.source_type) + except SourceDefinitionNotFoundError: + definition = None + if ( + definition is not None + and definition.version == version + and any( + ( + ( + definition.source_class.model_fields[key].alias is not None + and definition.source_class.model_fields[key].alias != key + ) + or ( + definition.source_class.model_fields[key].serialization_alias is not None + and definition.source_class.model_fields[key].serialization_alias != key + ) + or ( + definition.source_class.model_fields[key].validation_alias is not None + and definition.source_class.model_fields[key].validation_alias != key + ) + ) + for key in ("definition_version", "materialization") + ) + ): + # The current exact-version codec is required to disambiguate retained identity keys. + return None + try: + return Source.model_validate({ + "name": reference.source_id, + "definition_version": version, + "materialization": material["materialization"], + "description": material.get("description"), + }) + except (TypeError, ValueError): + return None + def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayload) -> Source: """Decode identity independently of retained material and today's business defaults.""" @@ -465,6 +550,9 @@ def _validate_unbound_material(self, reference: SourceRef, envelope: _StoredSour definition = self._registry.definition_for_name(reference.source_type) except SourceDefinitionNotFoundError: return + if _claims_identity_keys(definition.source_class): + # The retained canonical identity/header is authoritative; a live business alias may claim it. + return if envelope.source is None and any( (definition.source_class.model_fields[key].serialization_alias or key) != key for key in ("name", "definition_version", "materialization") @@ -868,18 +956,30 @@ def _source_header(source: Source) -> Source: def _source_keys(source: Source, material: dict[str, JsonValue] | None = None) -> dict[str, str]: - """Retain declared identity spellings, including pre-canonical generated keys.""" - keys = { - key: type(source).model_fields[key].serialization_alias or key - for key in ("name", "definition_version", "materialization") - } - generator = type(source).model_config.get("alias_generator") - if isinstance(generator, AliasGenerator): - generator = generator.serialization_alias or generator.alias - if material is not None and generator is not None: - for key, alias in keys.items(): - if alias not in material: - keys[key] = generator(key) + """Retain the exact serialized spelling used for each native identity.""" + keys: dict[str, str] = {} + for key in ("name", "definition_version", "materialization"): + field = type(source).model_fields[key] + preferred = field.serialization_alias or key + candidates = [preferred] + if isinstance(field.validation_alias, str): + candidates.append(field.validation_alias) + elif isinstance(field.validation_alias, AliasChoices): + candidates.extend(choice for choice in field.validation_alias.choices if isinstance(choice, str)) + if field.alias is not None: + candidates.append(field.alias) + generator = type(source).model_config.get("alias_generator") + if isinstance(generator, AliasGenerator): + generator = generator.serialization_alias or generator.alias + if generator is not None: + candidates.append(generator(key)) + if material is not None: + keys[key] = next( + (alias for alias in dict.fromkeys(candidates) if material.get(alias) == getattr(source, key)), + preferred, + ) + else: + keys[key] = preferred return keys @@ -912,7 +1012,8 @@ def _normalize_native_material(source_class: type[Source], material: dict[str, J return material normalized = dict(material) for key in Source.model_fields: - if source_class.model_fields[key].validation_alias != key: + field = source_class.model_fields[key] + if field.validation_alias != key and field.alias_priority != 1: continue alias = generator(key) if any( diff --git a/src/powercontext/sources/models.py b/src/powercontext/sources/models.py index 0c31b8a387..0f1a4a26fe 100644 --- a/src/powercontext/sources/models.py +++ b/src/powercontext/sources/models.py @@ -16,7 +16,7 @@ from enum import StrEnum -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, field_validator from powercontext.errors import InvalidSourceReferenceError from powercontext.limits import MAX_SOURCE_ID_LENGTH, MAX_SOURCE_TYPE_LENGTH @@ -60,11 +60,12 @@ def validate_key_part(cls, value: str, info) -> str: class Source(BaseModel): """Base value for an adapter-owned Source description.""" - # Envelope keys stay canonical for validation and serialization, independently of adapter aliases. - name: str = Field(alias="name") - definition_version: str = Field(default="1", alias="definition_version") - materialization: SourceMaterialization = Field(alias="materialization") - description: str | None = Field(default=None, alias="description") + model_config = ConfigDict(populate_by_name=True) + + name: str + definition_version: str = "1" + materialization: SourceMaterialization + description: str | None = None @field_validator("definition_version") @classmethod diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 981f3d258c..8c1b0c630e 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -161,7 +161,7 @@ def projection(self, key: SourceProjectionKey, /) -> JsonValue: def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> SourceDefinitionManifest: - """Build a worker declaration with canonical envelope keys and aliased business fields.""" + """Build a declaration matching the canonical remote payload representation.""" return _manifest_for_definition(definition, transport=True) @@ -172,24 +172,11 @@ def _manifest_for_definition( *, transport: bool, ) -> SourceDefinitionManifest: - aliases = _transport_identity_aliases(definition.source_class) if transport else {} - source_schema = _json_object(definition.source_class.model_json_schema(mode="serialization")) - if aliases: - reference = source_schema.get("$ref") - definitions = source_schema.get("$defs") - if isinstance(reference, str) and reference.startswith("#/$defs/") and isinstance(definitions, dict): - key = reference.removeprefix("#/$defs/").replace("~1", "/").replace("~0", "~") - root = definitions.get(key) - if isinstance(root, dict): - # Inline only the root: recursive children retain their serialized aliases in $defs. - source_schema = {**source_schema, **root} - source_schema.pop("$ref") - properties = source_schema.get("properties") - if isinstance(properties, dict): - source_schema["properties"] = {aliases.get(key, key): value for key, value in properties.items()} - required = source_schema.get("required") - if isinstance(required, list): - source_schema["required"] = [aliases.get(key, key) if isinstance(key, str) else key for key in required] + source_schema = _json_object( + definition.source_class.model_json_schema(mode="validation", by_alias=False) + if transport + else definition.source_class.model_json_schema(mode="serialization") + ) projections = tuple( SourceProjectionManifest( key=SourceProjectionKey(name=projection.name, version=projection.version), @@ -225,9 +212,7 @@ def project_source_for_transport( "remote Source observations must retain their canonical value", ) manifest = manifest_for_definition(definition) - payload = _json_object(source.model_dump(mode="json", by_alias=True)) - aliases = _transport_identity_aliases(type(source)) - payload = {aliases.get(key, key): value for key, value in payload.items()} + payload = _json_object(source.model_dump(mode="json", by_alias=False)) projections = tuple( SourceProjectionValue(key=key, value=registry.project(source, key)) for key in registry.projection_keys(source) ) @@ -243,33 +228,6 @@ def project_source_for_transport( ) -def _transport_identity_aliases(source_class: type[Source], /) -> dict[str, str]: - """Reject ambiguous aliases before moving envelope keys in schemas or values.""" - - identity = Source.model_fields - fields = source_class.model_fields - serialized: dict[str, str] = {} - projected: set[str] = set() - for name, field in fields.items(): - if field.exclude: - continue - alias = field.serialization_alias if field.serialization_alias is not None else name - if alias in serialized: - raise InvalidSourceDefinitionError( - source_class, - "serialization_alias", - f"{alias!r} is shared by {serialized[alias]!r} and {name!r}", - ) - serialized[alias] = name - target = name if name in identity else alias - if target in projected: - raise InvalidSourceDefinitionError( - source_class, "serialization_alias", f"{target!r} collides after envelope normalization" - ) - projected.add(target) - return {alias: name for alias, name in serialized.items() if name in identity} - - def _source_definition_fingerprint( *, name: str, diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 849ef7fcdb..6c901725a3 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -21,7 +21,7 @@ import pytest from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator from pydantic.alias_generators import to_camel -from sqlalchemy import delete, insert, update +from sqlalchemy import delete, insert, select, update from powercontext import InvalidSourceDefinitionError from powercontext.builtin.persistence import ( @@ -37,6 +37,7 @@ from powercontext.builtin.persistence.tables import SHARED_TABLES, SOURCES_TABLE from powercontext.builtin.runtime import SubmitSourceObservation from powercontext.builtin.runtime.relational import RelationalContexts +from powercontext.builtin.sources.content import CONTENT_SOURCE_ADAPTER, ContentSource from powercontext.sources import ( AdapterSourceDefinition, MemoryEvidenceDeclaration, @@ -230,7 +231,7 @@ async def scenario() -> None: "definition_version": "2", "materialization": "captured", "description": "Remote alias description.", - "wire_text": "Remote note text.", + "body": "Remote note text.", } asyncio.run(scenario()) @@ -272,12 +273,13 @@ async def scenario() -> None: @pytest.mark.parametrize("current_version", ["1", "2"]) def test_canonical_history_survives_current_identity_aliases(current_version) -> None: - class CurrentSource(NativeCamelCaseSource): + class CurrentSource(SerializationAliasSource): name: str = Field(alias="source_name") + label: str = Field(alias="name") async def scenario() -> None: - original = SourceRepository((SerializationAliasAdapter(NativeCamelCaseSource),)) - source = NativeCamelCaseSource( + original = SourceRepository((SerializationAliasAdapter(SerializationAliasSource),)) + source = SerializationAliasSource( name="canonical", materialization=SourceMaterialization.CAPTURED, body="Retained." ) material = source.model_dump(mode="json", by_alias=True) @@ -305,6 +307,50 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("has_header", [False, True]) +def test_business_name_alias_does_not_reinterpret_canonical_history(has_header: bool) -> None: + class CurrentSource(SerializationAliasSource): + label: str = Field(alias="name") + + async def scenario() -> None: + original = SourceRepository((SerializationAliasAdapter(SerializationAliasSource),)) + current = SourceRepository( + SourceDefinitionRegistry((AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version="1"),)) + ) + source = SerializationAliasSource( + name="canonical-business-alias", + materialization=SourceMaterialization.CAPTURED, + body="Retained.", + ) + material = source.model_dump(mode="json", by_alias=True) + value = {"encoding": "powercontext-source-v1", "representation": "native", "value": material} + if has_header: + value.update( + source=Source( + name=source.name, + definition_version=source.definition_version, + materialization=source.materialization, + ).model_dump(mode="json", exclude_none=True), + source_keys={ + "name": "name", + "definition_version": "definition_version", + "materialization": "materialization", + }, + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await original.add(connection, "scope-a", source) + await connection.execute( + update(SOURCES_TABLE) + .where(SOURCES_TABLE.c.source_id == source.name) + .values(payload=json.dumps(value).encode()) + ) + await connection.execute(delete(SOURCE_EVIDENCE_BINDINGS_TABLE)) + expected = await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) + assert await current.historical_evidence(connection, "scope-a", (stored.ref,)) == expected + + asyncio.run(scenario()) + + @pytest.mark.parametrize("business_version", ["1", "2"]) def test_bound_aliased_history_does_not_treat_business_fields_as_identity(business_version) -> None: class AliasedSource(NativeCamelCaseSource): @@ -346,6 +392,33 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_headerless_content_receipt_preserves_accepted_material_digest() -> None: + async def scenario() -> None: + repository = SourceRepository((CONTENT_SOURCE_ADAPTER,)) + source = ContentSource( + name="receipt-digest", + materialization=SourceMaterialization.CAPTURED, + content="Receipt content.", + handoff_receipt=True, + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await repository.add(connection, "scope-a", source) + original = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) + material = source.model_dump(mode="json", by_alias=True) + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + restored = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) + assert restored == original + + asyncio.run(scenario()) + + def test_precanonical_generated_alias_source_rows_remain_readable_and_replayable() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry(( @@ -356,7 +429,6 @@ async def scenario() -> None: name="legacy-2", definition_version="2", materialization=SourceMaterialization.CAPTURED, body="Retained." ) material = source.model_dump(mode="json", by_alias=True) - material["definitionVersion"] = material.pop("definition_version") payload = json.dumps({ "encoding": "powercontext-source-v1", "representation": "native", @@ -487,6 +559,65 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("representation", ["enveloped", "bare"]) +def test_replay_retains_historical_native_identity_spelling(representation) -> None: + from pydantic import AliasChoices + + class OldSource(NativeCamelCaseSource): + name: str = Field(alias="legacy_name") + + class CurrentSource(NativeCamelCaseSource): + name: str = Field( + validation_alias=AliasChoices("source_name", "legacy_name"), + serialization_alias="source_name", + ) + + async def scenario() -> None: + original_registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(OldSource), version="2"), + )) + current_registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version="2"), + )) + original = SourceRepository(original_registry) + current = SourceRepository(current_registry) + source = OldSource( + legacy_name="legacy-replay", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Retained exactly.", + ) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + stored = await original.add(connection, "scope-a", source) + material = source.model_dump(mode="json", by_alias=True) + if representation == "bare": + payload = json.dumps(material).encode() + else: + payload = json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + }).encode() + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + loaded = await current.get(connection, "scope-a", stored.ref) + assert loaded.value.name == source.name + replay = await current.add(connection, "scope-a", loaded.value) + assert replay.ref == stored.ref + assert await current.get(connection, "scope-a", stored.ref) == loaded + evidence = await current.historical_evidence(connection, "scope-a", (stored.ref,)) + assert evidence[0].source_ref == stored.ref + raw = await connection.scalar( + select(SOURCES_TABLE.c.payload).where(SOURCES_TABLE.c.source_id == source.name) + ) + assert isinstance(raw, bytes) + envelope = json.loads(raw) + assert envelope["source_keys"]["name"] == "legacy_name" + + asyncio.run(scenario()) + + def test_repository_rejects_a_scope_outside_the_relational_identity_baseline() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): @@ -649,3 +780,45 @@ async def add(scope_id: str, source: NoteSource): assert following.journal_position == 2 asyncio.run(scenario()) + + +def test_nested_uppercase_source_aliases_survive_read_replay_and_restart(tmp_path) -> None: + class UpperNested(Source): + model_config = ConfigDict(alias_generator=str.upper) + body: str + + class NestedNativeSource(NativeCamelCaseSource): + nested: UpperNested + + async def scenario() -> None: + adapter = SerializationAliasAdapter(NestedNativeSource) + registry = SourceDefinitionRegistry((AdapterSourceDefinition(adapter, version="2"),)) + repository = SourceRepository(registry) + source = NestedNativeSource( + name="nested-legacy", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Outer.", + nested=UpperNested( + name="inner", + materialization=SourceMaterialization.REFERENCED, + body="Nested.", + ), + ) + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'nested-alias.db'}") + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + original = await repository.add(connection, "scope-a", source) + async with ( + SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, + profile.database.transaction() as connection, + ): + loaded = await repository.get(connection, "scope-a", original.ref) + assert loaded.value == source + assert await repository.add(connection, "scope-a", source) == original + evidence = await repository.historical_evidence(connection, "scope-a", (original.ref,)) + assert evidence[0].source_ref == original.ref + + asyncio.run(scenario()) diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py index 551175e8c3..928fca8780 100644 --- a/tests/e2e/test_atomic_memory_source_evidence.py +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -22,8 +22,8 @@ from typing import cast import pytest -from pydantic import field_serializer, field_validator -from sqlalchemy import event, text +from pydantic import Field, field_serializer, field_validator +from sqlalchemy import event, text, update from sqlalchemy.exc import SQLAlchemyError from powercontext.artifacts import ArtifactLineage @@ -33,6 +33,7 @@ AtomicMemoryExtractionOutput, ) from powercontext.builtin.inference import GenerationResult +from powercontext.builtin.persistence.tables import SOURCES_TABLE from powercontext.builtin.runtime import ( BuiltinConfig, SubmitSourceObservation, @@ -135,6 +136,70 @@ async def scenario(): asyncio.run(scenario()) +@pytest.mark.parametrize("has_header", [False, True]) +def test_headerless_canonical_v1_evidence_survives_business_alias_rebuild( + migration_backend: MigrationBackend, has_header: bool +) -> None: + class CurrentNote(NoteSource): + name: str = Field(alias="source_name") + label: str = Field(alias="name") + + class CurrentAdapter(NoteAdapter): + source_class = CurrentNote + + config = BuiltinConfig(database=migration_backend.config) + old_registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter()),)) + current_registry = SourceDefinitionRegistry((AdapterSourceDefinition(CurrentAdapter(), version="2"),)) + + async def scenario(): + source = NoteSource(name="canonical-v1", materialization=SourceMaterialization.CAPTURED, body="Retained.") + ref = SourceRef(source_type="note", source_id=source.name) + async with open_builtin_contexts(config, source_registry=old_registry) as contexts: + context = await contexts.get("scope") + await context.sources.add(source) + await _commit(contexts, "scope", "canonical", "Canonical historical fact.", sources=(ref,)) + material = source.model_dump(mode="json", by_alias=True) + payload_value = { + "encoding": "powercontext-source-v1", + "representation": "native", + "value": material, + } + if has_header: + payload_value.update( + source=source.model_dump( + mode="json", + by_alias=False, + include={"name", "definition_version", "materialization"}, + ), + source_keys={ + "name": "name", + "definition_version": "definition_version", + "materialization": "materialization", + }, + ) + payload = json.dumps(payload_value).encode() + async with contexts.database.transaction() as connection: + await connection.execute( + update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) + ) + + async with open_builtin_contexts(config, source_registry=current_registry) as contexts: + before = await contexts.atomic_memory.for_scope("scope").evidence("canonical", revision=1) + assert before.sources[0].source_ref == ref + assert before.sources[0].declaration == MemoryEvidenceDeclaration() + report = await rebuild_atomic_memory_projection( + contexts.database, + contexts.atomic_memory.index, + maintenance_confirmed=True, + sources=contexts.repositories.sources, + ) + assert report.ready, report.errors + after = await contexts.atomic_memory.for_scope("scope").evidence("canonical", revision=1) + assert after == before + + asyncio.run(scenario()) + + def test_mixed_evidence_survives_failed_revision_restart_and_rebuild(migration_backend: MigrationBackend) -> None: strong = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") asserted = MemoryEvidenceDeclaration(authority="user_asserted", verification="not_verified") diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index a152d352a5..0fab312e44 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -15,14 +15,15 @@ from __future__ import annotations import asyncio +import hashlib import urllib.request from typing import Self import pytest -from pydantic import Field, ValidationError +import rfc8785 +from pydantic import Field, ValidationError, field_serializer from referencing.exceptions import Unresolvable -from powercontext import InvalidSourceDefinitionError from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import SHARED_TABLES @@ -87,7 +88,9 @@ async def scenario() -> None: ("name_alias", "body_alias"), [("source_name", "wire_text"), ("body", "wire_text"), ("source_name", "name"), ("wire_text", "wire_text")], ) -def test_serialization_aliases_preserve_the_worker_envelope_during_ingestion(name_alias: str, body_alias: str) -> None: +def test_serialization_aliases_preserve_the_worker_envelope_during_ingestion( + name_alias: str, body_alias: str, tmp_path +) -> None: class AliasedSource(Source): name: str = Field(serialization_alias=name_alias) definition_version: str = Field(default="1", serialization_alias="wire_version") @@ -116,12 +119,6 @@ async def scenario() -> None: definition = AdapterSourceDefinition(AliasedAdapter(), version="2") worker = SourceDefinitionRegistry((definition,)) source = await worker.resolve("Retain the aliased business value.") - if body_alias == "name" or name_alias == body_alias: - with pytest.raises(InvalidSourceDefinitionError): - manifest_for_definition(definition) - with pytest.raises(InvalidSourceDefinitionError): - project_source_for_transport(worker, source) - return manifest = manifest_for_definition(definition) observation = project_source_for_transport(worker, source) expected = { @@ -129,20 +126,28 @@ async def scenario() -> None: "definition_version": "2", "materialization": "captured", "description": "Aliased description.", - "wire_text": "Retain the aliased business value.", + "body": "Retain the aliased business value.", } - assert observation.payload == expected - assert source.model_dump(mode="json", by_alias=True) == { - name_alias: "remote-aliased", - "wire_version": "2", - "wire_materialization": "captured", - "wire_description": "Aliased description.", - "wire_text": "Retain the aliased business value.", + assert manifest.source_schema == AliasedSource.model_json_schema() + baseline_declaration = { + "name": definition.name, + "version": definition.version, + "source_schema": AliasedSource.model_json_schema(), + "projections": [], } + baseline_fingerprint = f"sha256:{hashlib.sha256(rfc8785.dumps(baseline_declaration)).hexdigest()}" + assert manifest.fingerprint == baseline_fingerprint + assert observation.payload == expected + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'remote-manifest.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest - async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: + current_manifest = manifest_for_definition(definition) + assert current_manifest == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) - await contexts.register_source_definition(manifest) + assert await contexts.register_source_definition(current_manifest) == manifest receipt = await contexts.submit_source_observation( SubmitSourceObservation(scope_id="scope-a", observation=observation) ) @@ -162,7 +167,7 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_recursive_source_transport_preserves_nested_serialization_aliases() -> None: +def test_recursive_source_transport_preserves_canonical_nested_payload() -> None: class RecursiveSource(Source): name: str = Field(serialization_alias="source_name") definition_version: str = Field(default="1", serialization_alias="wire_version") @@ -198,14 +203,14 @@ async def scenario() -> None: "definition_version": "2", "materialization": "captured", "description": None, - "wire_text": "Root text.", + "body": "Root text.", "children": [ { - "source_name": "leaf", - "wire_version": "1", + "name": "leaf", + "definition_version": "1", "materialization": "captured", "description": None, - "wire_text": "Nested text.", + "body": "Nested text.", "children": [], } ], @@ -238,6 +243,40 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_remote_identity_exclusions_and_serializers_fail_observation_validation() -> None: + class ExcludedIdentitySource(Source): + definition_version: str = Field(default="1", exclude=True) + + class TransformedIdentitySource(Source): + @field_serializer("name") + def encode_name(self, value: str) -> str: + return f"wire:{value}" + + class Adapter: + input_class = str + name = "remote-identity" + + def __init__(self, source_class: type[Source]) -> None: + self.source_class = source_class + + async def resolve(self, value: str, /) -> Source: + return self.source_class(name=value, materialization=SourceMaterialization.CAPTURED) + + async def read(self, source: Source, /) -> str: + return source.name + + async def scenario() -> None: + for source_class in (ExcludedIdentitySource, TransformedIdentitySource): + definition = AdapterSourceDefinition(Adapter(source_class), version="1") + registry = SourceDefinitionRegistry((definition,)) + assert manifest_for_definition(definition).source_schema == source_class.model_json_schema(by_alias=False) + source = await registry.resolve("identity") + with pytest.raises(ValidationError): + project_source_for_transport(registry, source) + + asyncio.run(scenario()) + + def test_remote_schema_references_are_rejected_without_network_access(monkeypatch: pytest.MonkeyPatch) -> None: attempted = False From 1e67e5d980d20a4ce05b80f42847634a2466b345 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Wed, 21 Oct 2026 18:03:07 +0800 Subject: [PATCH 19/94] chore(migrations): initialize issue 2154 implementation branch From 13517fdd9c6a4a67ff870a333e161d878460db85 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Wed, 21 Oct 2026 20:34:46 +0800 Subject: [PATCH 20/94] fix(sources): keep upgraded alias definitions compatible Generate remote transport schemas with the same alias policy as their payloads so existing serialize_by_alias Definitions keep their fingerprint and observations after upgrade. Recover explicit canonical historical versions beyond version 1 instead of handing accepted material to a newer aliased Definition. --- .../builtin/persistence/sources.py | 12 +++- src/powercontext/sources/observations.py | 16 +++-- tests/builtin/persistence/test_sources.py | 52 ++++++++++++++++ .../e2e/test_atomic_memory_source_evidence.py | 18 ++++-- tests/test_source_observations.py | 61 ++++++++++++++++++- 5 files changed, 143 insertions(+), 16 deletions(-) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index b32990e757..88ff8e331e 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -449,11 +449,17 @@ async def _historical_canonical_source( ) ).one_or_none() if binding is not None: - if binding.definition_name != reference.source_type or binding.material_digest != material_digest: + if ( + binding.definition_name != reference.source_type + or binding.material_digest != material_digest + or binding.definition_version != material["definition_version"] + ): return None version = binding.definition_version - elif material["definition_version"] == "1": - version = "1" + elif isinstance(material["definition_version"], str): + # Canonical retained identity is explicit even when its historical + # evidence binding predates the evidence tables. It stays neutral. + version = material["definition_version"] else: return None try: diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 8c1b0c630e..3531a6ab81 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -172,11 +172,15 @@ def _manifest_for_definition( *, transport: bool, ) -> SourceDefinitionManifest: - source_schema = _json_object( - definition.source_class.model_json_schema(mode="validation", by_alias=False) - if transport - else definition.source_class.model_json_schema(mode="serialization") - ) + if transport: + source_schema = _json_object( + definition.source_class.model_json_schema( + mode="serialization", + by_alias=bool(definition.source_class.model_config.get("serialize_by_alias")), + ) + ) + else: + source_schema = _json_object(definition.source_class.model_json_schema(mode="serialization")) projections = tuple( SourceProjectionManifest( key=SourceProjectionKey(name=projection.name, version=projection.version), @@ -212,7 +216,7 @@ def project_source_for_transport( "remote Source observations must retain their canonical value", ) manifest = manifest_for_definition(definition) - payload = _json_object(source.model_dump(mode="json", by_alias=False)) + payload = _json_object(source.model_dump(mode="json")) projections = tuple( SourceProjectionValue(key=key, value=registry.project(source, key)) for key in registry.projection_keys(source) ) diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 6c901725a3..86f86f4f64 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -392,6 +392,58 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_unbound_versioned_canonical_history_ignores_new_definition_aliases() -> None: + class CurrentSource(SerializationAliasSource): + model_config = ConfigDict(populate_by_name=True) + name: str = Field(alias="source_name") + label: str = Field(alias="name") + + async def scenario() -> None: + old_source = SerializationAliasSource( + name="old-version-two", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body="Retained without an evidence binding.", + ) + old_repository = SourceRepository(()) + + class CurrentAdapter: + name = "serialization-alias" + input_class = SerializationAliasInput + source_class = CurrentSource + + async def resolve(self, value: SerializationAliasInput, /) -> CurrentSource: + raise AssertionError # Historical evidence must not invoke the current decoder. + + async def read(self, source: CurrentSource, /) -> str: + return source.body + + registry = SourceDefinitionRegistry((AdapterSourceDefinition(CurrentAdapter(), version="3"),)) + async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: + # Model a canonical Source accepted before evidence bindings existed. + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="scope-a", + source_type="serialization-alias", + source_id=old_source.name, + payload=json.dumps({ + "encoding": "powercontext-source-v1", + "representation": "native", + "value": old_source.model_dump(mode="json", by_alias=False), + }).encode(), + journal_position=1, + ) + ) + ref = SourceRef(source_type="serialization-alias", source_id=old_source.name) + baseline = await old_repository.historical_evidence(connection, "scope-a", (ref,)) + upgraded = await SourceRepository(registry).historical_evidence(connection, "scope-a", (ref,)) + assert upgraded == baseline + assert upgraded[0].definition_version == "2" + assert upgraded[0].declaration == MemoryEvidenceDeclaration() + + asyncio.run(scenario()) + + def test_headerless_content_receipt_preserves_accepted_material_digest() -> None: async def scenario() -> None: repository = SourceRepository((CONTENT_SOURCE_ADAPTER,)) diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py index 928fca8780..5d674241d1 100644 --- a/tests/e2e/test_atomic_memory_source_evidence.py +++ b/tests/e2e/test_atomic_memory_source_evidence.py @@ -137,8 +137,9 @@ async def scenario(): @pytest.mark.parametrize("has_header", [False, True]) -def test_headerless_canonical_v1_evidence_survives_business_alias_rebuild( - migration_backend: MigrationBackend, has_header: bool +@pytest.mark.parametrize("historical_version", ["1", "2"]) +def test_canonical_historical_evidence_survives_business_alias_rebuild( + migration_backend: MigrationBackend, has_header: bool, historical_version: str ) -> None: class CurrentNote(NoteSource): name: str = Field(alias="source_name") @@ -148,11 +149,18 @@ class CurrentAdapter(NoteAdapter): source_class = CurrentNote config = BuiltinConfig(database=migration_backend.config) - old_registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter()),)) - current_registry = SourceDefinitionRegistry((AdapterSourceDefinition(CurrentAdapter(), version="2"),)) + old_registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter(), version=historical_version),)) + current_registry = SourceDefinitionRegistry(( + AdapterSourceDefinition(CurrentAdapter(), version=str(int(historical_version) + 1)), + )) async def scenario(): - source = NoteSource(name="canonical-v1", materialization=SourceMaterialization.CAPTURED, body="Retained.") + source = NoteSource( + name=f"canonical-v{historical_version}", + definition_version=historical_version, + materialization=SourceMaterialization.CAPTURED, + body="Retained.", + ) ref = SourceRef(source_type="note", source_id=source.name) async with open_builtin_contexts(config, source_registry=old_registry) as contexts: context = await contexts.get("scope") diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 0fab312e44..93cb1b7f52 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -21,7 +21,7 @@ import pytest import rfc8785 -from pydantic import Field, ValidationError, field_serializer +from pydantic import ConfigDict, Field, ValidationError, field_serializer from referencing.exceptions import Unresolvable from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository @@ -167,6 +167,61 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_alias_serialization_remote_manifest_and_payload_remain_upgrade_stable(tmp_path) -> None: + class AliasedSource(Source): + model_config = ConfigDict(serialize_by_alias=True, populate_by_name=True) + body: str = Field(alias="wire_text") + + class AliasedAdapter: + name = "aliased-upgrade" + input_class = str + source_class = AliasedSource + + async def resolve(self, value: str, /) -> AliasedSource: + return AliasedSource( + name="remote-aliased", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + wire_text=value, + ) + + async def read(self, source: AliasedSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(AliasedAdapter(), version="2") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retained alias.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + expected = AliasedSource.model_json_schema() + expected_fingerprint = hashlib.sha256( + rfc8785.dumps({ + "name": definition.name, + "version": definition.version, + "source_schema": expected, + "projections": [], + }) + ).hexdigest() + assert manifest.source_schema == expected + assert manifest.fingerprint == f"sha256:{expected_fingerprint}" + assert observation.payload["wire_text"] == "Retained alias." + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'remote-alias-upgrade.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "remote-aliased" + + asyncio.run(scenario()) + + def test_recursive_source_transport_preserves_canonical_nested_payload() -> None: class RecursiveSource(Source): name: str = Field(serialization_alias="source_name") @@ -269,7 +324,9 @@ async def scenario() -> None: for source_class in (ExcludedIdentitySource, TransformedIdentitySource): definition = AdapterSourceDefinition(Adapter(source_class), version="1") registry = SourceDefinitionRegistry((definition,)) - assert manifest_for_definition(definition).source_schema == source_class.model_json_schema(by_alias=False) + assert manifest_for_definition(definition).source_schema == source_class.model_json_schema( + mode="serialization", by_alias=False + ) source = await registry.resolve("identity") with pytest.raises(ValidationError): project_source_for_transport(registry, source) From 579b67e2edfa56df103b5f141b7ad7511cf60f6e Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Wed, 21 Oct 2026 21:26:06 +0800 Subject: [PATCH 21/94] fix(migrations): parse arithmetic CHECK constraints --- docs/en/docs/operate/database-migrations.md | 6 + docs/zh/docs/operate/database-migrations.md | 2 + .../persistence/migrations/mysql_schema.py | 69 +++++-- .../test_mysql_migration_checks.py | 170 ++++++++++++++++++ tests/fixtures/database_migrations/README.md | 24 +++ .../schemas/seekdb_v1_3_0_checks.json | 25 +++ 6 files changed, 282 insertions(+), 14 deletions(-) create mode 100644 tests/builtin/persistence/test_mysql_migration_checks.py create mode 100644 tests/fixtures/database_migrations/schemas/seekdb_v1_3_0_checks.json diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index dc4239a61b..5e40364468 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -27,6 +27,12 @@ through an ODP endpoint on OceanBase AI Database 4.6.3, covering coordinator exc committed DDL, data preservation, and repeat execution. This does not establish complete Server readiness, automatic detection of unfinished remote DDL after a crash, or full Fork restoration acceptance. +MySQL-compatible schema reflection supports arithmetic CHECK expressions with binary `+`, `-`, `*`, `/`, `%`, +unary signs, and nested grouping, preserving arithmetic, comparison, and boolean precedence. Unsupported CHECK +operators/functions remain rejected; their errors identify the reflected table, named constraint, and unsupported +construct instead of misreporting balanced parentheses. Parsing an expression does not register a historical layout +or bypass migration/data-integrity checks. + The existing `server processing-migrate` remains available. Follow [Migrate Artifact processing state](artifact-processing-migration.md); it does not yet forward to the unified entry point. See [Deploy the Server](deploy-server.md) for service deployment. diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 1aa59a040f..bed350885c 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -14,6 +14,8 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 输出中的 `ready` 仅表示当前迁移 bundle 已通过结构和数据验证；`readiness_scope=registered_bundle`、`server_ready=false` 明确区分它与完整 Server、索引、旧任务及集群可用。不要以此替代生产升级或完整业务库初始化。普通业务启动的全面版本门禁和完整历史基线尚未纳管。已通过 OceanBase AI 4.6.3 的 ODP 入口完成隔离四表验收，覆盖固定维护主机互斥、DDL 提交后中断的显式确认恢复、数据保留与重复执行。这不代表完整 Server 升级、崩溃后远端 DDL 自动判定或完整 Fork 恢复已通过验收。 +MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、`*`、`/`、`%`、一元正负号及嵌套括号，并保留算术、比较和布尔运算的优先级。未支持的 CHECK 运算符或函数仍被拒绝，错误会标明反射的表名、约束名及未支持的语法，避免将合法分组误报为括号不匹配。表达式解析成功不代表历史结构已登记，也不会绕过迁移或数据完整性检查。 + 原有 `server processing-migrate` 保留，请按[迁移 Artifact 处理状态](artifact-processing-migration.md)使用。它尚未自动转发到统一入口。服务部署参考[部署 Server](deploy-server.md)。 ## Topic Memory 恢复 bundle diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py index 1af7804cc9..3449fb7395 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql_schema.py +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -34,7 +34,9 @@ from .bundle import MigrationBundle from .models import MigrationError, digest -_LEXEME = re.compile(r"\s+|`(?:``|[^`])*`|'(?:''|\\.|[^'\\])*'|[A-Za-z_][A-Za-z_0-9$]*|[0-9]+|>=|<=|<>|!=|[(),.=<>+-]") +_LEXEME = re.compile( + r"\s+|`(?:``|[^`])*`|'(?:''|\\.|[^'\\])*'|[A-Za-z_][A-Za-z_0-9$]*|[0-9]+|>=|<=|<>|!=|[(),.=<>+*/%\^&|~!?:@-]" +) _REVISIONS = ("p0001", "p0002", "p0003", "p0004") _CITATION_TABLES = ("pc_artifacts", "pc_artifact_candidate_versions") _FAMILY_CHECK = "ck_pc_artifact_tags_family" @@ -125,7 +127,11 @@ def _reflect_tables(connection: Connection, database: str, exclude: frozenset[st row = connection.exec_driver_sql(f"SHOW CREATE TABLE {_quote(name)}").first() if row is None or not isinstance(row[1], str): raise MigrationError("incompatible_schema", "A table definition could not be reflected.") - reflected_name, table = _parse_table(row[1], database=database) + try: + reflected_name, table = _parse_table(row[1], database=database) + except MigrationError as error: + detail = str(error).removeprefix(f"{error.code}: ") + raise MigrationError(error.code, f"Table {name!r}: {detail}") from error if reflected_name != name: raise MigrationError("incompatible_schema", "A reflected table identity is inconsistent.") inventory["tables"][name] = table @@ -526,10 +532,17 @@ def _put_named(objects: dict[str, Any], name: str, definition: dict[str, Any]) - def _check_definition(part: list[str], constraint: str | None) -> tuple[str, dict[str, Any]]: - expression, end = _group(part, 1) - if constraint is None or part[end:] not in ([], ["enforced"], ["not", "enforced"]): - raise MigrationError("incompatible_schema", "An unsupported CHECK definition was discovered.") - return constraint, {"expression": _Expression(expression).parse(), "enforced": part[end:] != ["not", "enforced"]} + try: + expression, end = _group(part, 1) + if constraint is None or part[end:] not in ([], ["enforced"], ["not", "enforced"]): + raise MigrationError("incompatible_schema", "An unsupported CHECK definition was discovered.") # noqa: TRY301 + return constraint, { + "expression": _Expression(expression).parse(), + "enforced": part[end:] != ["not", "enforced"], + } + except MigrationError as error: + detail = str(error).removeprefix(f"{error.code}: ") + raise MigrationError(error.code, f"CHECK constraint {constraint!r}: {detail}") from error def _index_definition(part: list[str], constraint: str | None = None) -> tuple[str, dict[str, Any]]: @@ -642,12 +655,37 @@ def __init__(self, tokens: list[str]) -> None: def parse(self) -> list[Any]: value = self._expression(0) if self.position != len(self.tokens): - raise MigrationError("incompatible_schema", "An unsupported CHECK expression was discovered.") + lexeme = self.tokens[self.position] + construct = "string literal" if lexeme.startswith("'") else repr(lexeme) + if ( + lexeme == "(" + and self.position > 0 + and re.fullmatch(r"[a-z_][a-z_0-9$]*", self.tokens[self.position - 1]) + ): + construct = f"function {self.tokens[self.position - 1]!r}" + raise MigrationError("incompatible_schema", f"Unsupported CHECK expression token {construct}.") return value def _expression(self, precedence: int) -> list[Any]: left = self._value() - priorities = {"or": 1, "and": 2, "=": 3, ">": 3, "<": 3, ">=": 3, "<=": 3, "!=": 3, "<>": 3, "is": 3, "in": 3} + priorities = { + "or": 1, + "and": 2, + "=": 3, + ">": 3, + "<": 3, + ">=": 3, + "<=": 3, + "!=": 3, + "<>": 3, + "is": 3, + "in": 3, + "+": 4, + "-": 4, + "*": 5, + "/": 5, + "%": 5, + } while self.position < len(self.tokens): operator = self.tokens[self.position] priority = priorities.get(operator, 0) @@ -674,11 +712,12 @@ def _value(self) -> list[Any]: lexeme = self.tokens[self.position] self.position += 1 if lexeme == "(": - value = self._expression(0) - if self.tokens[self.position : self.position + 1] != [")"]: - raise MigrationError("incompatible_schema", "A CHECK expression has unbalanced parentheses.") - self.position += 1 - return value + # Parse a balanced group independently so unsupported operators are + # not mistaken for a missing closing parenthesis. + group, self.position = _group(self.tokens, self.position - 1) + return _Expression(group).parse() + if lexeme in {"+", "-"}: + return ["unary", lexeme, self._value()] if lexeme == "_utf8mb4": if self.position >= len(self.tokens) or not self.tokens[self.position].startswith("'"): raise MigrationError("incompatible_schema", "A CHECK expression has an unsupported literal introducer.") @@ -691,7 +730,7 @@ def _value(self) -> list[Any]: if lexeme == "null": return ["null"] if not re.fullmatch(r"[a-z_][a-z_0-9$]*", lexeme): - raise MigrationError("incompatible_schema", "An unsupported CHECK value was discovered.") + raise MigrationError("incompatible_schema", f"Unsupported CHECK value token {lexeme!r}.") return ["column", lexeme] @@ -709,6 +748,8 @@ def _expression_sql(expression: list[Any]) -> str: return f"({_expression_sql(expression[1])} IS {'NOT ' if kind == 'is_not_null' else ''}NULL)" if kind == "in": return f"({_expression_sql(expression[1])} IN ({', '.join(_expression_sql(value) for value in expression[2])}))" + if kind == "unary": + return f"({expression[1]}{_expression_sql(expression[2])})" if kind == "binary": return f"({_expression_sql(expression[2])} {expression[1].upper()} {_expression_sql(expression[3])})" raise MigrationError("invalid_bundle", "An unsupported frozen CHECK expression was discovered.") diff --git a/tests/builtin/persistence/test_mysql_migration_checks.py b/tests/builtin/persistence/test_mysql_migration_checks.py new file mode 100644 index 0000000000..080dd7229d --- /dev/null +++ b/tests/builtin/persistence/test_mysql_migration_checks.py @@ -0,0 +1,170 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CHECK reflection preserves SQL semantics without expanding migration eligibility.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest +from sqlalchemy import create_engine + +from powercontext.builtin.persistence.migrations.bundle import MigrationBundle +from powercontext.builtin.persistence.migrations.connections import MaintenanceConnections +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.mysql_schema import MySQLSchema, _expression_sql, mysql_inventory +from powercontext.builtin.persistence.seekdb import SeekDBConfig +from tests.builtin.persistence.test_mysql_migration_schema import _inventory, _ReflectedDatabase + +_FIXTURE = Path(__file__).parents[2] / "fixtures/database_migrations/schemas/seekdb_v1_3_0_checks.json" +_RESOURCES = Path(__file__).parents[3] / "src/powercontext/builtin/persistence/migrations/resources" + + +def _check_inventory(expression: str): + return _inventory( + _ReflectedDatabase({ + "check_fixture": "CREATE TABLE check_fixture (a INTEGER, b INTEGER, c INTEGER, " + f"CONSTRAINT ck_arithmetic CHECK ({expression}))" + }) + ) + + +@pytest.mark.parametrize( + ("expression", "values", "expected"), + [ + ("a = (b + c)", (5, 2, 3), 1), + ("a = (b + c)", (6, 2, 3), 0), + ("a + b * c = 14", (2, 3, 4), 1), + ("(a + b) * c = 20", (2, 3, 4), 1), + ("a - b - c = 5", (10, 3, 2), 1), + ("a - (b - c) = 9", (10, 3, 2), 1), + ("a / b / c = 2", (16.0, 4, 2), 1), + ("a % b + c = 3", (10, 4, 1), 1), + ("-a + +b = -(c + 1)", (5, 2, 2), 1), + ("a IS NULL OR a = b + c AND b > 0", (None, -2, 3), 1), + ("a IS NULL OR a = b + c AND b > 0", (1, -2, 3), 0), + ("(a IS NULL OR a = b + c) AND b > 0", (None, -2, 3), 0), + ("a + b IN (3, 4) AND c IS NOT NULL", (1, 2, 0), 1), + ("a = b + c", (None, 2, 3), None), + ], +) +def test_reflected_arithmetic_renders_with_equivalent_semantics(expression, values, expected): + inventory = _check_inventory(expression) + rendered = _expression_sql(inventory["tables"]["check_fixture"]["checks"]["ck_arithmetic"]["expression"]) + # SQLite executes the portable expressions; this does not claim MySQL division semantics. + with create_engine("sqlite://").connect() as connection: + result = connection.exec_driver_sql( + f"SELECT {rendered} FROM (SELECT ? AS a, ? AS b, ? AS c)", # noqa: S608 -- test expressions + values, + ).scalar_one() + assert result == expected + assert _check_inventory(rendered) == inventory + + +def test_historical_seekdb_checks_reflect_without_accepting_an_unknown_layout(): + fixture = json.loads(_FIXTURE.read_text()) + inventory = _inventory( + _ReflectedDatabase({name: value["show_create_table"] for name, value in fixture["tables"].items()}) + ) + compiled = _inventory( + _ReflectedDatabase({name: value["create_table"] for name, value in fixture["tables"].items()}) + ) + for name in fixture["tables"]: + # seekdb promotes inline column CHECKs to table constraints in SHOW CREATE. + # Compare the historical table-level checks and round-trip every reflected check. + for constraint, check in compiled["tables"][name]["checks"].items(): + assert inventory["tables"][name]["checks"][constraint] == check + for check in inventory["tables"][name]["checks"].values(): + roundtrip = _check_inventory(_expression_sql(check["expression"])) + assert roundtrip["tables"]["check_fixture"]["checks"]["ck_arithmetic"] == check + checks = inventory["tables"]["pc_recall_effort_daily"]["checks"] + rendered = _expression_sql(checks["ck_pc_recall_effort_daily_drop_total"]["expression"]) + with create_engine("sqlite://").connect() as connection: + for total, expected in [(5, 1), (6, 0)]: + assert ( + connection.exec_driver_sql( + f"SELECT {rendered} FROM (SELECT ? AS dropped_items, " # noqa: S608 -- frozen expression + "2 AS dropped_below_min_bytes, 3 AS dropped_no_fitting_truncation)", + (total,), + ).scalar_one() + == expected + ) + schema = MySQLSchema(MigrationBundle(_RESOURCES)) + with pytest.raises(MigrationError, match="registered complete layout"): + schema.identify(inventory) + with pytest.raises(MigrationError, match="recorded revision"): + schema.verify(inventory, "p0004") + with pytest.raises(MigrationError, match="controlled next-revision"): + schema.recovery(inventory, "p0003") + + +@pytest.mark.parametrize( + ("expression", "construct"), + [ + ("a = (b ^ c)", "'^'"), + ("a = (b DIV c)", "'div'"), + ("a = abs(b)", "'abs'"), + ("a LIKE 'private literal'", "'like'"), + ("a = 'private literal' (b)", "'('"), + ("a = @b", "'@'"), + ], +) +def test_unsupported_checks_report_table_constraint_and_construct(expression, construct): + with pytest.raises(MigrationError) as caught: + _check_inventory(expression) + message = str(caught.value) + assert caught.value.code == "incompatible_schema" + assert "check_fixture" in message + assert "ck_arithmetic" in message + assert construct in message + assert "Unsupported CHECK" in message + assert "unbalanced" not in message + assert "private literal" not in message + + +@pytest.mark.parametrize("expression", ["a = (b + c", "a = b + c)", "a = (b +)", "a = ()", "a = -"]) +def test_malformed_checks_remain_rejected(expression): + with pytest.raises(MigrationError, match="check_fixture"): + _check_inventory(expression) + + +@pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") +def test_real_seekdb_historical_check_reflection_and_rendering(short_tmp_path): + pytest.importorskip("pylibseekdb") + fixture = json.loads(_FIXTURE.read_text()) + + def reflect(connection, _identity, guard): + assert connection is not None + for value in fixture["tables"].values(): + guard() + connection.exec_driver_sql(value["create_table"]) + connection.commit() + inventory = mysql_inventory(connection) + checks = inventory["tables"]["pc_recall_effort_daily"]["checks"] + rendered = _expression_sql(checks["ck_pc_recall_effort_daily_drop_total"]["expression"]) + for total, expected in [(5, 1), (6, 0)]: + assert ( + connection.exec_driver_sql( + f"SELECT {rendered} FROM (SELECT {total} AS dropped_items, " # noqa: S608 -- test constants + "2 AS dropped_below_min_bytes, 3 AS dropped_no_fitting_truncation) AS sample" + ).scalar_one() + == expected + ) + with pytest.raises(MigrationError, match="registered complete layout"): + MySQLSchema(MigrationBundle(_RESOURCES)).identify(inventory) + + MaintenanceConnections(SeekDBConfig(path=short_tmp_path / "seekdb")).run(reflect, writable=True) diff --git a/tests/fixtures/database_migrations/README.md b/tests/fixtures/database_migrations/README.md index a8b4494757..888508ca8e 100644 --- a/tests/fixtures/database_migrations/README.md +++ b/tests/fixtures/database_migrations/README.md @@ -25,6 +25,30 @@ The manifest registers exact business fingerprints, alternate released shapes, r Plans bind the canonical target, actual schema, immutable resource snapshot, configuration digest and maintenance choices. Alembic executes the reviewed snapshot even if installed resources change; changes detected during backup stop before migration DDL. +## Historical CHECK reflection fixture + +`schemas/seekdb_v1_3_0_checks.json` records actual `SHOW CREATE TABLE` output for +`pc_recall_effort_daily`, `pc_recall_token_daily`, and `pc_model_usage_daily`. The input DDL was compiled +from `powercontext-v1.3.0` (`95bfbc9e0b59f58aee119c750c69a448b7666d0c`), using that tag's +`tables.py` and `limits.py`, then executed in a fresh disposable embedded seekdb database on macOS. +The fixture records both source blob IDs, the compiler/binding versions, and the engine's `SELECT VERSION()` result. +It was captured with `pylibseekdb 1.4.0.post1` on engine `seekdb-v1.4.0.0`; it is not a copy of a customer's database +or a claim that the original 1.3.0 deployment used this engine version. + +To recapture, load both historical Python sources from the tag, compile each selected table with SQLAlchemy's +MySQL `CreateTable` (and its named indexes with `CreateIndex`), execute them in a new embedded instance, and save +`SHOW CREATE TABLE` without rewriting its expressions. In particular, retain seekdb's nested parentheses, +backticks, literal introducers, and promotion of inline column CHECKs to named table constraints. + +The fixture reproduces #2160 on the original parser. It exercises arithmetic, comparisons, `IN`, and backend formatting; +it is a parser regression input, not a registered migration baseline. Successful reflection must still reject these +unregistered tables during layout identification, verification, and recovery. The opt-in real backend test re-creates +the historical tables and executes the rendered drop-total CHECK against matching and mismatching values: + +```bash +POWERCONTEXT_TEST_MIGRATION_SEEKDB=1 uv run --locked --extra seekdb pytest -q tests/builtin/persistence/test_mysql_migration_checks.py +``` + ## Explicit maintenance Repository-only scratch commands: diff --git a/tests/fixtures/database_migrations/schemas/seekdb_v1_3_0_checks.json b/tests/fixtures/database_migrations/schemas/seekdb_v1_3_0_checks.json new file mode 100644 index 0000000000..b41613f461 --- /dev/null +++ b/tests/fixtures/database_migrations/schemas/seekdb_v1_3_0_checks.json @@ -0,0 +1,25 @@ +{ + "source_tag": "powercontext-v1.3.0", + "source_commit": "95bfbc9e0b59f58aee119c750c69a448b7666d0c", + "source_blobs": { + "src/powercontext/limits.py": "90d8984ea5ed4e35554b7279b7ac8b166d93bc34", + "src/powercontext/builtin/persistence/tables.py": "69c5191fa41c6c7ac9c63ab98e0bb485f539004c" + }, + "pylibseekdb_version": "1.4.0.post1", + "sqlalchemy_version": "2.0.54", + "tables": { + "pc_recall_effort_daily": { + "create_table": "CREATE TABLE pc_recall_effort_daily (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tusage_date DATE NOT NULL, \n\tpolicy_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tassessment VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tpreparations BIGINT NOT NULL CONSTRAINT ck_recall_effort_preparations_nonnegative CHECK (preparations >= 0), \n\trounds BIGINT NOT NULL CONSTRAINT ck_recall_effort_rounds_nonnegative CHECK (rounds >= 0), \n\texpanded_preparations BIGINT NOT NULL CONSTRAINT ck_recall_effort_expanded_preparations_nonnegative CHECK (expanded_preparations >= 0), \n\tadmission_expansions BIGINT NOT NULL CONSTRAINT ck_recall_effort_admission_expansions_nonnegative CHECK (admission_expansions >= 0), \n\tpolicy_floor_expansions BIGINT NOT NULL CONSTRAINT ck_recall_effort_policy_floor_expansions_nonnegative CHECK (policy_floor_expansions >= 0), \n\tcandidate_round_samples BIGINT NOT NULL CONSTRAINT ck_recall_effort_candidate_round_samples_nonnegative CHECK (candidate_round_samples >= 0), \n\tcandidates_assessed BIGINT NOT NULL CONSTRAINT ck_recall_effort_candidates_assessed_nonnegative CHECK (candidates_assessed >= 0), \n\tfinal_candidate_pool BIGINT NOT NULL CONSTRAINT ck_recall_effort_final_candidate_pool_nonnegative CHECK (final_candidate_pool >= 0), \n\tadded_embeddings BIGINT NOT NULL CONSTRAINT ck_recall_effort_added_embeddings_nonnegative CHECK (added_embeddings >= 0), \n\tadded_generation_calls BIGINT NOT NULL CONSTRAINT ck_recall_effort_added_generation_calls_nonnegative CHECK (added_generation_calls >= 0), \n\ttruncated_items BIGINT NOT NULL CONSTRAINT ck_recall_effort_truncated_items_nonnegative CHECK (truncated_items >= 0), \n\tdropped_items BIGINT NOT NULL CONSTRAINT ck_recall_effort_dropped_items_nonnegative CHECK (dropped_items >= 0), \n\tdropped_below_min_bytes BIGINT NOT NULL CONSTRAINT ck_recall_effort_dropped_below_min_bytes_nonnegative CHECK (dropped_below_min_bytes >= 0), \n\tdropped_no_fitting_truncation BIGINT NOT NULL CONSTRAINT ck_recall_effort_dropped_no_fitting_truncation_nonnegative CHECK (dropped_no_fitting_truncation >= 0), \n\tPRIMARY KEY (scope_id, usage_date, policy_id, assessment), \n\tCONSTRAINT ck_pc_recall_effort_daily_drop_total CHECK (dropped_items = dropped_below_min_bytes + dropped_no_fitting_truncation)\n)", + "show_create_table": "CREATE TABLE `pc_recall_effort_daily` (\n `scope_id` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `usage_date` date NOT NULL,\n `policy_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `assessment` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `preparations` bigint(20) NOT NULL,\n `rounds` bigint(20) NOT NULL,\n `expanded_preparations` bigint(20) NOT NULL,\n `admission_expansions` bigint(20) NOT NULL,\n `policy_floor_expansions` bigint(20) NOT NULL,\n `candidate_round_samples` bigint(20) NOT NULL,\n `candidates_assessed` bigint(20) NOT NULL,\n `final_candidate_pool` bigint(20) NOT NULL,\n `added_embeddings` bigint(20) NOT NULL,\n `added_generation_calls` bigint(20) NOT NULL,\n `truncated_items` bigint(20) NOT NULL,\n `dropped_items` bigint(20) NOT NULL,\n `dropped_below_min_bytes` bigint(20) NOT NULL,\n `dropped_no_fitting_truncation` bigint(20) NOT NULL,\n PRIMARY KEY (`scope_id`, `usage_date`, `policy_id`, `assessment`),\n CONSTRAINT `ck_recall_effort_preparations_nonnegative` CHECK ((`preparations` >= 0)),\n CONSTRAINT `ck_recall_effort_rounds_nonnegative` CHECK ((`rounds` >= 0)),\n CONSTRAINT `ck_recall_effort_expanded_preparations_nonnegative` CHECK ((`expanded_preparations` >= 0)),\n CONSTRAINT `ck_recall_effort_admission_expansions_nonnegative` CHECK ((`admission_expansions` >= 0)),\n CONSTRAINT `ck_recall_effort_policy_floor_expansions_nonnegative` CHECK ((`policy_floor_expansions` >= 0)),\n CONSTRAINT `ck_recall_effort_candidate_round_samples_nonnegative` CHECK ((`candidate_round_samples` >= 0)),\n CONSTRAINT `ck_recall_effort_candidates_assessed_nonnegative` CHECK ((`candidates_assessed` >= 0)),\n CONSTRAINT `ck_recall_effort_final_candidate_pool_nonnegative` CHECK ((`final_candidate_pool` >= 0)),\n CONSTRAINT `ck_recall_effort_added_embeddings_nonnegative` CHECK ((`added_embeddings` >= 0)),\n CONSTRAINT `ck_recall_effort_added_generation_calls_nonnegative` CHECK ((`added_generation_calls` >= 0)),\n CONSTRAINT `ck_recall_effort_truncated_items_nonnegative` CHECK ((`truncated_items` >= 0)),\n CONSTRAINT `ck_recall_effort_dropped_items_nonnegative` CHECK ((`dropped_items` >= 0)),\n CONSTRAINT `ck_recall_effort_dropped_below_min_bytes_nonnegative` CHECK ((`dropped_below_min_bytes` >= 0)),\n CONSTRAINT `ck_recall_effort_dropped_no_fitting_truncation_nonnegative` CHECK ((`dropped_no_fitting_truncation` >= 0)),\n CONSTRAINT `ck_pc_recall_effort_daily_drop_total` CHECK ((`dropped_items` = (`dropped_below_min_bytes` + `dropped_no_fitting_truncation`)))\n) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0" + }, + "pc_recall_token_daily": { + "create_table": "CREATE TABLE pc_recall_token_daily (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tusage_date DATE NOT NULL, \n\testimator_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\testimator_version VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tpreparations BIGINT NOT NULL, \n\tready_preparations BIGINT NOT NULL, \n\tcomparable_preparations BIGINT NOT NULL, \n\tbaseline_tokens BIGINT NOT NULL, \n\trecalled_tokens BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, usage_date, estimator_id, estimator_version), \n\tCONSTRAINT ck_pc_recall_token_daily_preparations_nonnegative CHECK (preparations >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_ready_nonnegative CHECK (ready_preparations >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_comparable_nonnegative CHECK (comparable_preparations >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_comparable_ready CHECK (comparable_preparations <= ready_preparations), \n\tCONSTRAINT ck_pc_recall_token_daily_ready_total CHECK (ready_preparations <= preparations), \n\tCONSTRAINT ck_pc_recall_token_daily_baseline_nonnegative CHECK (baseline_tokens >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_recalled_nonnegative CHECK (recalled_tokens >= 0)\n)", + "show_create_table": "CREATE TABLE `pc_recall_token_daily` (\n `scope_id` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `usage_date` date NOT NULL,\n `estimator_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `estimator_version` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `preparations` bigint(20) NOT NULL,\n `ready_preparations` bigint(20) NOT NULL,\n `comparable_preparations` bigint(20) NOT NULL,\n `baseline_tokens` bigint(20) NOT NULL,\n `recalled_tokens` bigint(20) NOT NULL,\n PRIMARY KEY (`scope_id`, `usage_date`, `estimator_id`, `estimator_version`),\n CONSTRAINT `ck_pc_recall_token_daily_preparations_nonnegative` CHECK ((`preparations` >= 0)),\n CONSTRAINT `ck_pc_recall_token_daily_ready_nonnegative` CHECK ((`ready_preparations` >= 0)),\n CONSTRAINT `ck_pc_recall_token_daily_comparable_nonnegative` CHECK ((`comparable_preparations` >= 0)),\n CONSTRAINT `ck_pc_recall_token_daily_comparable_ready` CHECK ((`comparable_preparations` <= `ready_preparations`)),\n CONSTRAINT `ck_pc_recall_token_daily_ready_total` CHECK ((`ready_preparations` <= `preparations`)),\n CONSTRAINT `ck_pc_recall_token_daily_baseline_nonnegative` CHECK ((`baseline_tokens` >= 0)),\n CONSTRAINT `ck_pc_recall_token_daily_recalled_nonnegative` CHECK ((`recalled_tokens` >= 0))\n) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0" + }, + "pc_model_usage_daily": { + "create_table": "CREATE TABLE pc_model_usage_daily (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tusage_date DATE NOT NULL, \n\tpurpose VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\toperation VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trequests BIGINT NOT NULL, \n\tinput_tokens BIGINT NOT NULL, \n\toutput_tokens BIGINT NOT NULL, \n\tinput_complete BOOL NOT NULL, \n\toutput_complete BOOL NOT NULL, \n\tPRIMARY KEY (scope_id, usage_date, purpose, operation), \n\tCONSTRAINT ck_pc_model_usage_daily_operation CHECK (operation IN ('generation', 'embedding')), \n\tCONSTRAINT ck_pc_model_usage_daily_requests_nonnegative CHECK (requests >= 0), \n\tCONSTRAINT ck_pc_model_usage_daily_input_nonnegative CHECK (input_tokens >= 0), \n\tCONSTRAINT ck_pc_model_usage_daily_output_nonnegative CHECK (output_tokens >= 0)\n)", + "show_create_table": "CREATE TABLE `pc_model_usage_daily` (\n `scope_id` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `usage_date` date NOT NULL,\n `purpose` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `operation` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `requests` bigint(20) NOT NULL,\n `input_tokens` bigint(20) NOT NULL,\n `output_tokens` bigint(20) NOT NULL,\n `input_complete` tinyint(1) NOT NULL,\n `output_complete` tinyint(1) NOT NULL,\n PRIMARY KEY (`scope_id`, `usage_date`, `purpose`, `operation`),\n CONSTRAINT `ck_pc_model_usage_daily_operation` CHECK ((`operation` in ('generation','embedding'))),\n CONSTRAINT `ck_pc_model_usage_daily_requests_nonnegative` CHECK ((`requests` >= 0)),\n CONSTRAINT `ck_pc_model_usage_daily_input_nonnegative` CHECK ((`input_tokens` >= 0)),\n CONSTRAINT `ck_pc_model_usage_daily_output_nonnegative` CHECK ((`output_tokens` >= 0))\n) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0" + } + }, + "backend_version": "5.7.25-OceanBase seekdb-v1.4.0.0" +} From d7f2d897e14e50243a7e2596b0223539edd74816 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Wed, 21 Oct 2026 23:05:29 +0800 Subject: [PATCH 22/94] ci(release): gate database upgrade support on installed artifacts --- .github/workflows/master.yml | 36 ++ .github/workflows/release.yml | 17 +- Makefile | 15 +- docs/en/development/releasing.md | 34 ++ docs/en/docs/get-started/install-and-run.md | 38 ++- docs/zh/development/releasing.md | 28 ++ docs/zh/docs/get-started/install-and-run.md | 31 +- release/upgrade-support.json | 10 + scripts/ci_legacy_upgrade_rejection.py | 203 ++++++++++++ scripts/release_version.py | 5 +- scripts/run_upgrade_release_gate.py | 54 +++ scripts/upgrade_release_gate.py | 349 ++++++++++++++++++++ tests/release_upgrades/README.md | 79 +++++ tests/test_release_version.py | 11 +- tests/test_upgrade_release_gate.py | 210 ++++++++++++ 15 files changed, 1076 insertions(+), 44 deletions(-) create mode 100644 release/upgrade-support.json create mode 100644 scripts/ci_legacy_upgrade_rejection.py create mode 100644 scripts/run_upgrade_release_gate.py create mode 100644 scripts/upgrade_release_gate.py create mode 100644 tests/release_upgrades/README.md create mode 100644 tests/test_upgrade_release_gate.py diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 01ff65b202..d9c4d2ad9c 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -19,6 +19,42 @@ jobs: changes: uses: ./.github/workflows/ci-changes.yml + upgrade-release-gate: + name: Upgrade release gate + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + persist-credentials: false + - name: Set up the environment + uses: ./.github/actions/setup-python-env + - name: Verify upgrade policy and gate regressions + run: | + make upgrade-policy-check + uv run pytest -q tests/test_upgrade_release_gate.py + - name: Build the target artifact + run: uv build --wheel --out-dir dist + - name: Run declared installed-release acceptance + run: >- + uv run python scripts/run_upgrade_release_gate.py + --output "$RUNNER_TEMP/upgrade-release-evidence" + - name: Reproduce unsupported full 1.3.0 database upgrade + run: | + wheel=$(find dist -name '*.whl' -print -quit) + uv run python scripts/ci_legacy_upgrade_rejection.py \ + --wheel "$wheel" --output "$RUNNER_TEMP/upgrade-release-evidence/legacy-rejection" + - name: Preserve upgrade acceptance evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: upgrade-release-evidence-${{ github.sha }} + path: ${{ runner.temp }}/upgrade-release-evidence/ + if-no-files-found: error + retention-days: 30 + quality: timeout-minutes: 20 runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dff86dbc30..d21ccd47ea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ concurrency: jobs: release-build: runs-on: ubuntu-24.04 - timeout-minutes: 20 + timeout-minutes: 90 steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -74,6 +74,21 @@ jobs: uv build --out-dir dist uvx --from twine twine check dist/* + - name: Gate upgrade claims on installed release acceptance + run: >- + uv run python scripts/run_upgrade_release_gate.py + --release --release-event "$GITHUB_EVENT_PATH" + --output "$RUNNER_TEMP/upgrade-release-evidence" + + - name: Preserve upgrade acceptance evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: upgrade-release-evidence + path: ${{ runner.temp }}/upgrade-release-evidence/ + if-no-files-found: error + retention-days: 30 + - name: Upload release distributions uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: diff --git a/Makefile b/Makefile index 60b09a699b..6f968cea9d 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ notebooks-test: ## Execute provider-free tutorials in fresh kernels; use ARGS fo @uv run --locked --group notebooks python examples/jupyter/run.py $(ARGS) .PHONY: check -check: version-check workflow-actions-check integration-manifest-check integration-guidance-check ## Run code quality tools. +check: version-check upgrade-policy-check workflow-actions-check integration-manifest-check integration-guidance-check ## Run code quality tools. .PHONY: integration-guidance integration-guidance-check integration-guidance: ## Refresh the managed regions of packaged integration guidance. @@ -235,16 +235,25 @@ version-check: ## Check release references and generated API code; optionally as @$(MAKE) api-generate-check js-api-generate-check .PHONY: release-check -release-check: version-check docs-install ## Check version references and require matching bilingual notes for a stable release. +release-check: version-check upgrade-policy-check docs-install ## Check version references and require matching bilingual notes for a stable release. @cd website && node --import tsx --test tests/releases.test.ts +.PHONY: upgrade-policy-check +upgrade-policy-check: ## Verify the declared upgrade matrix, persistence review, and installation guidance. + @uv run python scripts/upgrade_release_gate.py + +.PHONY: upgrade-release-test +upgrade-release-test: ## Run installed-release acceptance for the single wheel in dist (fresh-install-only makes no upgrade claim). + @uv run python scripts/run_upgrade_release_gate.py + .PHONY: clean-build clean-build: ## Clean build artifacts @echo "🚀 Removing build artifacts" @uv run python -c "import shutil; import os; shutil.rmtree('dist') if os.path.exists('dist') else None" .PHONY: publish -publish: ## Publish a release to PyPI. +publish: ## Publish after release upgrade acceptance; UPGRADE_RELEASE_NOTES must name the reviewed release notes file. + @uv run python scripts/run_upgrade_release_gate.py --release --release-notes "$$UPGRADE_RELEASE_NOTES" @echo "🚀 Publishing." @uv publish dist/* diff --git a/docs/en/development/releasing.md b/docs/en/development/releasing.md index 0300298b75..e95ae9c0b6 100644 --- a/docs/en/development/releasing.md +++ b/docs/en/development/releasing.md @@ -56,6 +56,7 @@ the website lists stable releases only. `make docs-test` also runs this check as | Source | Managed value | | --- | --- | | `openapi/powercontext.yaml` | `info.version`, the prepared release version. | +| `release/upgrade-support.json` | Target release version; source versions and the support policy require compatibility review. | | `src/powercontext/http/_generated/` | `API_VERSION` in `operations.py` and `info.version` in `schema.py`, regenerated through `make api-generate`. | | `README.md`, `README_CN.md`, `README_JP.md` | Current release prose, pinned package installation commands, and matching Agent integration tag. | | `docs/en/docs/get-started/quickstart.md`, `docs/zh/docs/get-started/quickstart.md` | Current release and matching installation/tag instructions. | @@ -126,6 +127,39 @@ installation pins are synchronized. ## Verify before publication +### Database upgrade support + +`release/upgrade-support.json` declares the target version and every supported source version, backend, and +configuration. The current policy is `fresh-install-only`; existing Server database upgrades are unsupported. +Do not advertise a supported upgrade until its installed-release acceptance suite is implemented and passes. +The suite contract and evidence requirements are in `tests/release_upgrades/README.md` in the repository. + +`make check` and `make release-check` validate this policy and the bilingual installation notices. Changes to +managed persistence inputs require a compatibility review with an updated digest and rationale. A digest is a +review trigger, not evidence that a migration is correct. `make version-bump` also updates the policy target; +regenerate the support notices with `uv run python scripts/upgrade_release_gate.py --write-docs` after matrix changes. + +After building exactly one wheel into `dist/`, execute the declared upgrade cases: + +```bash +make upgrade-release-test +uv run python scripts/run_upgrade_release_gate.py --release --release-notes /path/to/release-notes.md +``` + +The release check requires the actual wheel version to match the policy and the release notes to include the exact +English support declaration from the installation guide. It installs each exact source release and the target wheel +in separate environments. Missing tests, skipped backends, failures, and missing evidence block the check. A passing +`fresh-install-only` policy check does not establish an upgrade path. Ordinary PR builds may have a development +version; the `--release` check must run against the final tagged wheel without overriding its version. + +The **Upgrade release gate** CI job also reproduces the known unsupported upgrade rejections using a real 1.3.0 +SQLite Server database. Replace that negative fixture with supported-path acceptance when implementing the full +migration chain. Repository administrators must make this job required in branch protection. +The release workflow gates distribution upload, PyPI publication, and release assets on the acceptance result and +archives its evidence. It runs after `release.published`, so it cannot prevent the earlier manual publication of +the GitHub Release page: run the command above before that action. `make publish` requires +`UPGRADE_RELEASE_NOTES=/path/to/release-notes.md` and runs the same gate before uploading packages. + Run the checks for the complete release changes, including generated contracts and documentation: ```bash diff --git a/docs/en/docs/get-started/install-and-run.md b/docs/en/docs/get-started/install-and-run.md index 2a06f4215a..f6021a91bd 100644 --- a/docs/en/docs/get-started/install-and-run.md +++ b/docs/en/docs/get-started/install-and-run.md @@ -174,17 +174,18 @@ For a long-running process, Docker, authentication, or remote access, continue w ## Update or replace an installation -Before upgrading an existing deployment, back up its database and configuration. Version 1.1.0 upgrades legacy -tag-table constraints during Server startup to support Topic Memory tags. Stop the old Server instances and start -one upgraded instance first so the schema upgrade completes before other instances connect. Databases from before -1.0.0 also need the [Artifact processing migration](../operate/artifact-processing-migration.md) if it has not already -been completed. Existing collection-based Memory databases also require the [offline Atomic Memory migration](../operate/atomic-memory-migration.md) -for existing collection databases. Complete any required Artifact processing migration first, stop all writers, then -apply and verify the Atomic Memory migration. Start the new service only after verification returns `ready: true`. -Upgrade the Server, clients, and Agent integrations together. -The Dashboard must be explicitly enabled; static Bearer authentication is optional for local use. See -[Deploy the Server](../operate/deploy-server.md). Remote plaintext HTTP connections require explicit client consent; -see [Connect to a remote Server](../operate/connect-remote-server.md). + +This release supports fresh installations only; upgrades of existing Server databases are not supported. + + +Keep existing deployments on their working version until their source version and backend have an accepted upgrade +path. To evaluate a fresh installation, use a separate program environment, database location, and port. Replacing +the installed package does not migrate its database. Backups do not make an unsupported migration supported. + +The support declaration is checked against the release matrix and installed-artifact acceptance; see +[Preparing a release](../../development/releasing.md). Domain-specific +[Processing](../operate/artifact-processing-migration.md) and +[Atomic Memory](../operate/atomic-memory-migration.md) maintenance guides do not establish full Server readiness. Review the [Dream coordinated upgrade requirements](../workflows/review-dream-revisions.md#coordinated-upgrade) before upgrading an existing database. The unified Candidate routes replace `/v1/artifact-candidates/*` with @@ -193,21 +194,22 @@ migration after unfinished runs have drained. The current unified migration bund it does not provide a complete production Server upgrade path. Atomic Memory verification alone does not establish that readiness. Do not bypass `migration_required` or `migration_framework_not_ready`. -To upgrade to 1.4.0: +To install 1.4.0 in a separate environment for a fresh deployment: ```bash -uv tool install --force "powercontext[cli,server]==1.4.0" +uv venv /path/to/powercontext-preview +uv pip install --python /path/to/powercontext-preview/bin/python "powercontext[cli,server]==1.4.0" ``` -To replace the installed tool with another Git ref: +To install another Git ref in the separate environment: ```bash -uv tool install --force "powercontext[cli,server] @ git+https://github.com/oceanbase/powercontext.git@" +uv pip install --python /path/to/powercontext-preview/bin/python "powercontext[cli,server] @ git+https://github.com/oceanbase/powercontext.git@" ``` -Update each installed host using its [integration guide](../integrations/index.md) and the same ref. Restart the Server and open a new host session -after updating. Existing SQLite data remains in the user data directory unless `POWERCONTEXT_HOME` or the database URL -changes. +Use the matching ref for clients and Agent integrations connected to the fresh deployment. Preserve the original +deployment's database location and configuration; do not point the new Server at it as an installation check. +On Windows, the virtual environment interpreter is at `Scripts/python.exe`. ## Install a Python role diff --git a/docs/zh/development/releasing.md b/docs/zh/development/releasing.md index f53da6fb39..bfc18fc73c 100644 --- a/docs/zh/development/releasing.md +++ b/docs/zh/development/releasing.md @@ -53,6 +53,7 @@ beta 和 RC 跳过网站条目检查。`make docs-test` 也会通过网站测试 | 来源 | 自动管理的内容 | | --- | --- | | `openapi/powercontext.yaml` | `info.version`，作为待发布版本。 | +| `release/upgrade-support.json` | 目标发布版本；来源版本和支持策略需要单独进行兼容性审查。 | | `src/powercontext/http/_generated/` | `operations.py` 的 `API_VERSION` 和 `schema.py` 的 `info.version`，通过 `make api-generate` 重新生成。 | | `README.md`、`README_CN.md`、`README_JP.md` | 当前版本介绍、固定包版本的安装命令及配套 Agent 集成 tag。 | | `docs/en/docs/get-started/quickstart.md`、`docs/zh/docs/get-started/quickstart.md` | 当前版本介绍及匹配的安装命令、tag。 | @@ -119,6 +120,33 @@ DSH README 中说明某项能力已包含在 `1.1.0` 的文字同样属于历史 ## 发布前验证 +### 数据库升级支持范围 + +`release/upgrade-support.json` 声明目标版本，以及每个受支持的来源版本、后端和配置组合。当前策略为 +`fresh-install-only`，不支持已有 Server 数据库升级。只有实现并通过已安装发行包的完整升级验收，才能声明支持 +对应组合。验收契约和证据要求见仓库中的 `tests/release_upgrades/README.md`。 + +`make check` 和 `make release-check` 会核对策略及双语安装说明。受管持久化文件变化时，必须审查兼容性影响， +更新摘要和理由。摘要用于触发审查，本身不证明迁移正确。`make version-bump` 同时更新策略中的目标版本； +支持矩阵变化后，运行 `uv run python scripts/upgrade_release_gate.py --write-docs` 更新安装说明中的支持范围。 + +将唯一的目标 wheel 构建到 `dist/` 后，执行声明的升级组合验收： + +```bash +make upgrade-release-test +uv run python scripts/run_upgrade_release_gate.py --release --release-notes /path/to/release-notes.md +``` + +发布检查要求实际 wheel 版本与策略一致，发布说明包含安装指南中的完整英文支持声明。每个组合分别安装准确的 +旧版发行包和目标 wheel。测试缺失、后端跳过、执行失败或证据缺失都会阻断检查。`fresh-install-only` 策略检查 +通过不表示存在升级路径。普通 PR 构建允许使用开发版本；`--release` 必须验证最终 tag 构建的 wheel，不能覆盖版本号。 + +CI 中的 **Upgrade release gate** 还会使用真实 1.3.0 SQLite Server 数据库复现已知的不支持升级拒绝行为。 +完整迁移链实现后，应将这一负向场景替换为支持路径的验收。仓库管理员需要在分支保护中将该检查设为必需。 +发布工作流只有验收通过才会上传发行包、发布 PyPI 和上传 Release 附件，并归档验收证据。工作流在 +`release.published` 后运行，无法阻止之前手动发布 GitHub Release 页面，因此必须先运行上述命令再发布页面。 +`make publish` 要求提供 `UPGRADE_RELEASE_NOTES=/path/to/release-notes.md`，并在上传软件包之前执行相同门禁。 + 针对本次发布的完整变化运行检查，包括生成的契约和文档： ```bash diff --git a/docs/zh/docs/get-started/install-and-run.md b/docs/zh/docs/get-started/install-and-run.md index 1af6e476ab..5e8360b924 100644 --- a/docs/zh/docs/get-started/install-and-run.md +++ b/docs/zh/docs/get-started/install-and-run.md @@ -160,14 +160,16 @@ Agent 诊断见[各自的集成文档](../integrations/index.md)；Server 状态 ## 更新或替换安装 -升级已有部署前，先备份数据库和配置。1.1.0 会在 Server 启动时升级旧标签表约束，以支持 Topic Memory 标签。 -先停止旧 Server 实例，再启动一个升级后的实例，待结构升级完成后再启动其他实例。对于 1.0.0 之前的数据库， -如果尚未完成[Artifact 处理迁移](../operate/artifact-processing-migration.md)，还需要先执行该迁移。 -升级到 1.4.0 时，已有集合式 Memory 数据库还需完成[Atomic Memory 离线迁移](../operate/atomic-memory-migration.md)。 -先完成所需的 Artifact 处理迁移，再停止所有写入进程，执行 Atomic Memory 迁移并验证 `ready: true` 后启动新服务。 -Server、客户端和 Agent 集成需一起升级。Dashboard 需要显式启用，本地使用可选择静态 Bearer 认证， -见[部署 Server](../operate/deploy-server.md)；远程明文 HTTP 连接需要客户端明确同意， -见[连接远程 Server](../operate/connect-remote-server.md)。 + +此版本仅支持新建部署，尚不支持已有 Server 数据库升级。 + + +已有部署应保持当前可用版本，等待来源版本及后端具备已验收的升级路径。试用新建部署时，请使用独立程序环境、 +数据库位置和端口。替换程序包不会完成数据库迁移；备份也不能使不受支持的迁移变为可用。 + +支持声明由发布矩阵和安装包验收共同检查，详见[准备发布](../../development/releasing.md)。 +[Artifact 处理迁移](../operate/artifact-processing-migration.md)和 +[Atomic Memory 迁移](../operate/atomic-memory-migration.md)各自的验证结果不代表完整 Server 就绪。 升级已有数据库前，请先检查 [Dream 协调升级要求](../workflows/review-dream-revisions.md#协调升级)。 统一 Candidate 路由将 `/v1/artifact-candidates/*` 替换为 `/v1/candidates/*`，不提供别名，Dream 客户端要求契约 2。 @@ -175,20 +177,21 @@ Server、客户端和 Agent 集成需一起升级。Dashboard 需要显式启用 Server 升级路径；Atomic Memory 验证通过也不代表整体迁移就绪。不要绕过 `migration_required` 或 `migration_framework_not_ready`。 -升级到 1.4.0： +安装 1.4.0 到独立环境以试用新建部署： ```bash -uv tool install --force "powercontext[cli,server]==1.4.0" +uv venv /path/to/powercontext-preview +uv pip install --python /path/to/powercontext-preview/bin/python "powercontext[cli,server]==1.4.0" ``` -使用其他 Git ref 替换现有工具： +在独立环境安装其他 Git ref： ```bash -uv tool install --force "powercontext[cli,server] @ git+https://github.com/oceanbase/powercontext.git@" +uv pip install --python /path/to/powercontext-preview/bin/python "powercontext[cli,server] @ git+https://github.com/oceanbase/powercontext.git@" ``` -按[各自的集成文档](../integrations/index.md)更新已安装宿主，并使用同一个 ref。更新后重启 Server，再开启新的宿主会话。只要没有修改 -`POWERCONTEXT_HOME` 或数据库 URL，现有 SQLite 数据会继续保留。 +连接新建部署的客户端及 Agent 集成应使用匹配的 ref。保留原部署的数据库位置和配置，不要用原数据库测试新版启动。 +Windows 的虚拟环境解释器位于 `Scripts/python.exe`。 ## 为 Python 项目安装角色 diff --git a/release/upgrade-support.json b/release/upgrade-support.json new file mode 100644 index 0000000000..a1a0dff078 --- /dev/null +++ b/release/upgrade-support.json @@ -0,0 +1,10 @@ +{ + "format": 1, + "target_version": "1.4.0", + "mode": "fresh-install-only", + "schema_review": { + "persistence_sha256": "ca3b475f2d0c883e19291a8d8b1dadbe50f93ebd93aad550a9112a3497737927", + "rationale": "The full Server upgrade framework is incomplete (#2154). Existing databases are unsupported; no production upgrade combination is advertised until installed-release acceptance passes." + }, + "cases": [] +} diff --git a/scripts/ci_legacy_upgrade_rejection.py b/scripts/ci_legacy_upgrade_rejection.py new file mode 100644 index 0000000000..cefdf44def --- /dev/null +++ b/scripts/ci_legacy_upgrade_rejection.py @@ -0,0 +1,203 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reproduce unsupported 1.3.0 upgrade rejection on an actual old SQLite Server. + +This is a negative regression, never positive upgrade acceptance. Once the path +is supported, replace this fixture with the full matrix suite before advertising it. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sqlite3 +import subprocess +import tempfile +from pathlib import Path + +import httpx +from ci_release_smoke import _available_port, _stop_process, _wait_until_ready +from upgrade_release_gate import installed, require, run, sha256, wheel_version + + +def console(python: Path) -> Path: + return python.with_name("powercontext.exe" if os.name == "nt" else "powercontext") + + +def source_database(python: Path, directory: Path, environment: dict[str, str], log: Path) -> dict[str, str | bool]: + port = _available_port() + base_url = f"http://127.0.0.1:{port}" + with log.open("w") as stream: + process = subprocess.Popen( # noqa: S603 -- installed source CLI against a disposable database + [str(console(python)), "server", "run", "--no-env-file", "--host", "127.0.0.1", "--port", str(port)], + cwd=directory, + env=environment, + stdout=stream, + stderr=subprocess.STDOUT, + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0, + start_new_session=os.name != "nt", + text=True, + ) + try: + _wait_until_ready(base_url, process, 90) + with httpx.Client(base_url=base_url, trust_env=False, timeout=30) as client: + response = client.post( + "/v1/scopes", + json={ + "title": "Upgrade fixture", + "summary": "Disposable old-release database for upgrade validation.", + "idempotency_key": "upgrade-fixture", + }, + ) + response.raise_for_status() + scope_id = response.json()["scope_id"] + response = client.post( + "/v1/memory/remember", + json={"scope_id": scope_id, "kind": "fact", "text": "Historical release migration fixture."}, + ) + response.raise_for_status() + require(response.json().get("records"), "Old Server did not persist the fixture") + response = client.post( + "/v1/memory/search", json={"scope_id": scope_id, "query": "historical migration", "mode": "fts"} + ) + response.raise_for_status() + require(response.json().get("hits"), "Old Server cannot retrieve its fixture") + return {"scope_id": scope_id, "memory_readback": True} + finally: + _stop_process(process) + + +def rejected(command: list[str], environment: dict[str, str], directory: Path, log: Path, reason: str) -> None: + with log.open("w", encoding="utf-8") as stream: + process = subprocess.Popen( # noqa: S603 -- installed target CLI, isolated data only + command, + env=environment, + cwd=directory, + stdout=stream, + stderr=subprocess.STDOUT, + text=True, + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0, + start_new_session=os.name != "nt", + ) + try: + process.wait(timeout=90) + finally: + _stop_process(process) + output = log.read_text(encoding="utf-8") + require(process.returncode != 0 and reason in output, f"Expected rejection {reason}; inspect {log.name}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheel", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + wheel = args.wheel.resolve() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + environment = { + key: value + for key, value in os.environ.items() + if not key.startswith(("POWERCONTEXT_", "PYTEST_")) and key not in {"PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV"} + } + environment["NO_PROXY"] = environment["no_proxy"] = "localhost,127.0.0.1" + with tempfile.TemporaryDirectory(prefix="pc-old-release-") as temporary: + directory = Path(temporary) + interpreters = {} + for role, package in {"source": "powercontext[cli,server]==1.3.0", "target": f"{wheel}[cli,server]"}.items(): + venv = directory / role + run( + ["uv", "venv", "--python", "3.12", str(venv)], + environment=environment, + cwd=directory, + log=output / f"{role}-venv.log", + ) + python = venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + run( + ["uv", "pip", "install", "--python", str(python), package], + environment=environment, + cwd=directory, + log=output / f"{role}-install.log", + ) + interpreters[role] = python + metadata = {role: installed(python, environment, directory) for role, python in interpreters.items()} + require(metadata["source"]["powercontext"] == "1.3.0", "Wrong source package") + require(metadata["target"]["powercontext"] == wheel_version(wheel), "Wrong target package") + database = directory / "source.db" + environment.update({ + "POWERCONTEXT_HOME": str(directory / "home"), + "POWERCONTEXT_SERVER_DATABASE_URL": f"sqlite+aiosqlite:///{database}", + "POWERCONTEXT_SERVER_ACCESS_MODE": "disabled", + }) + seed = source_database(interpreters["source"], directory, environment, output / "source.log") + with sqlite3.connect(database) as connection: + schema = connection.execute( + "SELECT type, name, tbl_name, sql FROM sqlite_schema ORDER BY type, name" + ).fetchall() + tables = {row[1] for row in schema if row[0] == "table"} + require( + {"pc_artifact_candidate_heads", "pc_artifact_candidate_versions", "pc_scopes", "pc_artifacts"} <= tables, + "Source fixture lacks historical Server objects", + ) + (output / "source-schema.json").write_text(json.dumps(schema, indent=2) + "\n") + env_file = directory / "deployment.env" + env_file.write_text(f"POWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite:///{database}\n") + target = str(console(interpreters["target"])) + rejected( + [target, "server", "db-migrate", "status", "--env-file", str(env_file)], + environment, + directory, + output / "status.log", + "unknown_baseline", + ) + rejected( + [target, "server", "run", "--no-env-file", "--host", "127.0.0.1", "--port", str(_available_port())], + environment, + directory, + output / "legacy-startup.log", + "migration_required", + ) + # This second negative fixture intentionally adds only the prototype control + # table to a copy. It must never be mistaken for a completed migration. + versioned = directory / "versioned.db" + shutil.copyfile(database, versioned) + with sqlite3.connect(versioned) as connection: + connection.execute("CREATE TABLE pc_schema_revision (version_num VARCHAR(32) PRIMARY KEY)") + environment["POWERCONTEXT_SERVER_DATABASE_URL"] = f"sqlite+aiosqlite:///{versioned}" + rejected( + [target, "server", "run", "--no-env-file", "--host", "127.0.0.1", "--port", str(_available_port())], + environment, + directory, + output / "versioned-startup.log", + "migration_framework_not_ready", + ) + evidence = { + "kind": "unsupported-upgrade-regression", + "backend": "sqlite", + "engine_version": sqlite3.sqlite_version, + "installed": metadata, + "wheel_sha256": sha256(wheel), + "seed": seed, + "table_count": len(tables), + "status": "passed", + } + (output / "result.json").write_text(json.dumps(evidence, indent=2) + "\n") + print("Actual 1.3.0 full database: all three unsupported-upgrade regressions reproduced.") + + +if __name__ == "__main__": + main() diff --git a/scripts/release_version.py b/scripts/release_version.py index 57bf563e31..538f094d95 100644 --- a/scripts/release_version.py +++ b/scripts/release_version.py @@ -33,6 +33,7 @@ # Match current-release contexts only: migration thresholds, release history, # independent plugin versions and third-party dependencies must retain their meaning. REFERENCE_PREFIXES = { + "release/upgrade-support.json": (r'^ "target_version": "',), "openapi/powercontext.yaml": (r"^ version: ",), "README.md": (*INSTALL_REFERENCES, r"PowerContext "), "README_CN.md": (*INSTALL_REFERENCES, r"PowerContext "), @@ -45,13 +46,13 @@ *INSTALL_REFERENCES, r"PowerContext ", r"package `", - r"^To upgrade to ", + r"^To install ", ), "docs/zh/docs/get-started/install-and-run.md": ( *INSTALL_REFERENCES, r"PowerContext ", r"Python 包版本为 `", - r"^升级到 ", + r"^安装 ", ), "integrations/dsh/plugins/powercontext/README.md": INSTALL_REFERENCES, } diff --git a/scripts/run_upgrade_release_gate.py b/scripts/run_upgrade_release_gate.py new file mode 100644 index 0000000000..89ddb8f0d6 --- /dev/null +++ b/scripts/run_upgrade_release_gate.py @@ -0,0 +1,54 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Select the built wheel and run the same upgrade gate locally and in CI.""" + +import argparse +import json +import tempfile +from pathlib import Path + +from upgrade_release_gate import ROOT, GateError, gate, require + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dist", type=Path, default=Path("dist")) + parser.add_argument("--output", type=Path) + parser.add_argument("--release", action="store_true") + notes = parser.add_mutually_exclusive_group() + notes.add_argument("--release-event", type=Path) + notes.add_argument("--release-notes", type=Path) + args = parser.parse_args() + try: + wheels = list(args.dist.glob("*.whl")) + require(len(wheels) == 1, "Build exactly one release wheel before running upgrade acceptance") + release_notes = None + if args.release_event: + release_notes = json.loads(args.release_event.read_text(encoding="utf-8"))["release"]["body"] or "" + elif args.release_notes: + release_notes = args.release_notes.read_text(encoding="utf-8") + output = args.output or Path(tempfile.mkdtemp(prefix="pc-upgrade-evidence-")) / "report" + gate(ROOT, wheels[0].resolve(), output.resolve(), release=args.release, release_notes=release_notes) + result = json.loads((output / "result.json").read_text(encoding="utf-8")) + print( + f"Upgrade gate passed: {result['mode']}; {len(result['cases'])} accepted upgrade combinations. " + f"Evidence: {output / 'result.json'}" + ) + except (GateError, OSError, ValueError) as error: + raise SystemExit(f"Upgrade gate failed: {error}") from error # noqa: TRY003 -- operator diagnostic + + +if __name__ == "__main__": + main() diff --git a/scripts/upgrade_release_gate.py b/scripts/upgrade_release_gate.py new file mode 100644 index 0000000000..62bdbddce9 --- /dev/null +++ b/scripts/upgrade_release_gate.py @@ -0,0 +1,349 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gate upgrade claims on this build's complete, installed-release acceptance.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +import tempfile +import zipfile +from email.parser import Parser +from pathlib import Path +from typing import Any +from xml.etree import ElementTree + +ROOT = Path(__file__).resolve().parents[1] +MANIFEST = Path("release/upgrade-support.json") +MANAGED_INPUTS = ( + "src/powercontext/builtin/persistence", + "src/powercontext/builtin/code", + "src/powercontext/server/identity.py", + "src/powercontext/server/authz", + "src/powercontext/limits.py", +) +PHASES = frozenset({ + "test_source_release_database", + "test_public_migration_commands", + "test_target_startup", + "test_read_write_and_retrieval", + "test_authorization", + "test_candidates_and_processing", + "test_preserved_history_and_cursors", + "test_fresh_upgrade_equivalence", +}) +VERSION = r"\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?" +NOTICE = { + "en": "This release supports fresh installations only; upgrades of existing Server databases are not supported.", + "zh": "此版本仅支持新建部署，尚不支持已有 Server 数据库升级。", # noqa: RUF001 -- Chinese prose +} + + +class GateError(ValueError): + """An upgrade claim has no valid declaration or passing acceptance.""" + + +def require(condition: Any, message: str) -> None: + if not condition: + raise GateError(message) + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def managed_digest(root: Path) -> str: + """Bind the review declaration to all persistence inputs, not a new-file count. + + This deliberately over-approximates schema changes. It is a review trigger, + not a claim to prove SQL or data-format compatibility from Python syntax. + """ + inputs = [root / relative for relative in MANAGED_INPUTS] + files = { + path.relative_to(root).as_posix(): sha256(path) + for location in inputs + for path in (location.rglob("*") if location.is_dir() else [location]) + if path.is_file() and path.suffix in {".py", ".sql", ".json"} + } + require(files, "No managed persistence inputs found") + return hashlib.sha256(json.dumps(files, sort_keys=True).encode()).hexdigest() + + +def load_policy(root: Path) -> dict[str, Any]: + policy = json.loads((root / MANIFEST).read_text(encoding="utf-8")) + require(set(policy) == {"format", "target_version", "mode", "schema_review", "cases"}, "Invalid policy fields") + require(policy["format"] == 1, "Unsupported upgrade policy format") + contract = (root / "openapi/powercontext.yaml").read_text(encoding="utf-8") + match = re.search(rf"^ version: ({VERSION})$", contract, re.MULTILINE) + require( + match is not None and match[1] == policy["target_version"], "Upgrade policy target differs from API release" + ) + require(policy["mode"] in {"fresh-install-only", "supported-upgrades"}, "Unknown upgrade policy mode") + review = policy["schema_review"] + require(set(review) == {"persistence_sha256", "rationale"}, "Invalid schema review fields") + require( + isinstance(review["rationale"], str) and review["rationale"].strip(), "Schema compatibility rationale required" + ) + require( + review["persistence_sha256"] == managed_digest(root), "Persistence changed: update the compatibility review" + ) + cases = policy["cases"] + require(isinstance(cases, list), "Upgrade cases must be a list") + require(bool(cases) == (policy["mode"] == "supported-upgrades"), "Upgrade mode and support matrix disagree") + identities: set[tuple[str, str, str]] = set() + for case in cases: + require(set(case) == {"source_version", "backend", "configuration", "suite"}, "Invalid upgrade case fields") + require( + isinstance(case["source_version"], str) and re.fullmatch(VERSION, case["source_version"]), + "Pin a source release", + ) + require(case["backend"] in {"sqlite", "seekdb", "oceanbase"}, "Unknown backend") + require(re.fullmatch(r"[a-z][a-z0-9-]*", case["configuration"]), "Configuration needs a stable identifier") + identity = (case["source_version"], case["backend"], case["configuration"]) + require(identity not in identities, "Duplicate upgrade combination") + identities.add(identity) + suite = (root / case["suite"]).resolve() + require( + suite.is_relative_to((root / "tests/release_upgrades").resolve()), + "Suite must be under tests/release_upgrades", + ) + require( + suite.is_file() and suite.name.startswith("test_") and suite.suffix == ".py", "Missing acceptance suite" + ) + return policy + + +def support_text(policy: dict[str, Any], language: str) -> str: + if policy["mode"] == "fresh-install-only": + return NOTICE[language] + title = ( + "Supported database upgrades (other combinations are unsupported):" + if language == "en" + else "支持的数据库升级组合（其他组合不受支持）：" # noqa: RUF001 -- Chinese prose + ) + lines = [title, "", "| Source | Target | Backend | Configuration |", "| --- | --- | --- | --- |"] + lines.extend( + f"| {case['source_version']} | {policy['target_version']} | {case['backend']} | {case['configuration']} |" + for case in policy["cases"] + ) + return "\n".join(lines) + + +def check_docs(root: Path, policy: dict[str, Any], *, write: bool = False) -> None: + for language in ("en", "zh"): + path = root / f"docs/{language}/docs/get-started/install-and-run.md" + content = path.read_text(encoding="utf-8") + pattern = r"(?<=\n).*?(?=\n)" + matches = list(re.finditer(pattern, content, re.DOTALL)) + require(len(matches) == 1, f"{path.name}: exactly one upgrade-support region required") + expected = support_text(policy, language) + if write: + match = matches[0] + path.write_text(content[: match.start()] + expected + content[match.end() :], encoding="utf-8") + else: + require(matches[0][0] == expected, f"{language} upgrade support instructions differ from the policy") + + +def validate_junit(report: Path) -> None: + tree = ElementTree.parse(report) # noqa: S314 -- locally produced acceptance report + cases = list(tree.iter("testcase")) + require(cases, "No acceptance tests executed") + require(not any(list(tree.iter(tag)) for tag in ("failure", "error", "skipped")), "Acceptance failed or skipped") + names = {case.attrib.get("name", "").split("[", 1)[0] for case in cases} + require(names >= PHASES, f"Missing full Server acceptance phases: {sorted(PHASES - names)}") + + +def validate_backend_evidence(path: Path, backend: str) -> dict[str, Any]: + evidence = json.loads(path.read_text(encoding="utf-8")) + require(evidence.get("backend") == backend, "Backend evidence does not match the declared case") + for key in ("engine_version", "driver_version", "source_schema", "target_schema", "fresh_schema"): + require(isinstance(evidence.get(key), str) and evidence[key].strip(), f"Missing backend evidence: {key}") + if backend == "oceanbase": + require(evidence.get("tenant_mode") == "mysql", "OceanBase MySQL tenant evidence required") + require( + isinstance(evidence.get("proxy_version"), str) and evidence["proxy_version"], + "Record proxy version or direct", + ) + return evidence + + +def run(command: list[str], *, environment: dict[str, str], cwd: Path, log: Path) -> None: + with log.open("w", encoding="utf-8") as output: + result = subprocess.run( # noqa: S603 -- fixed tools and reviewed repository test paths, no shell + command, env=environment, cwd=cwd, stdout=output, stderr=subprocess.STDOUT, timeout=1800, check=False + ) + require(result.returncode == 0, f"Command failed; inspect {log.name}") + + +def installed(python: Path, environment: dict[str, str], cwd: Path) -> dict[str, str]: + result = subprocess.run( # noqa: S603 -- isolated environment interpreter + [ + str(python), + "-I", + "-c", + "import json,platform; from importlib.metadata import version; " + "print(json.dumps({'powercontext': version('powercontext'), 'python': platform.python_version()}))", + ], + env=environment, + cwd=cwd, + capture_output=True, + text=True, + check=True, + timeout=30, + ) + return json.loads(result.stdout) + + +def wheel_version(wheel: Path) -> str: + with zipfile.ZipFile(wheel) as archive: + names = [name for name in archive.namelist() if name.endswith(".dist-info/METADATA")] + require(len(names) == 1, "Expected exactly one wheel distribution") + metadata = Parser().parsestr(archive.read(names[0]).decode()) + require(metadata["Name"] == "powercontext", "Expected a PowerContext wheel") + return str(metadata["Version"]) + + +def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dict[str, Any]: + output.mkdir(parents=True, exist_ok=False) + # Never inherit a developer's live deployment or pytest selection overrides. + environment = { + key: value + for key, value in os.environ.items() + if not key.startswith(("POWERCONTEXT_", "PYTEST_")) and key not in {"PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV"} + } + with tempfile.TemporaryDirectory(prefix="pc-upgrade-") as temporary: + directory = Path(temporary) + extras = "cli,server,seekdb" if case["backend"] == "seekdb" else "cli,server" + interpreters = {} + packages = {"source": f"powercontext[{extras}]=={case['source_version']}", "target": f"{wheel}[{extras}]"} + for role, package in packages.items(): + venv = directory / role + run( + ["uv", "venv", "--python", sys.executable, str(venv)], + environment=environment, + cwd=directory, + log=output / f"{role}-venv.log", + ) + python = venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + run( + ["uv", "pip", "install", "--python", str(python), package], + environment=environment, + cwd=directory, + log=output / f"{role}-install.log", + ) + interpreters[role] = python + metadata = {role: installed(python, environment, directory) for role, python in interpreters.items()} + require(metadata["source"]["powercontext"] == case["source_version"], "Source release mismatch") + require(metadata["target"]["powercontext"] == wheel_version(wheel), "Target wheel version mismatch") + environment.update({ + "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", + "PC_UPGRADE_SOURCE_PYTHON": str(interpreters["source"]), + "PC_UPGRADE_TARGET_PYTHON": str(interpreters["target"]), + "PC_UPGRADE_BACKEND": case["backend"], + "PC_UPGRADE_CONFIGURATION": case["configuration"], + "PC_UPGRADE_WORK": str(directory), + "PC_UPGRADE_EVIDENCE": str(output / "backend.json"), + }) + report = output / "junit.xml" + run( + [ + sys.executable, + "-m", + "pytest", + "-c", + "/dev/null" if os.name != "nt" else "NUL", + "--confcutdir", + str(root / "tests/release_upgrades"), + "--strict-markers", + "-o", + "xfail_strict=true", + "-ra", + f"--junitxml={report}", + str(root / case["suite"]), + ], + environment=environment, + cwd=directory, + log=output / "acceptance.log", + ) + validate_junit(report) + backend = validate_backend_evidence(output / "backend.json", case["backend"]) + return {"case": case, "installed": metadata, "database": backend, "status": "passed"} + + +def gate(root: Path, wheel: Path, output: Path, *, release: bool = False, release_notes: str | None = None) -> None: + policy = load_policy(root) + check_docs(root, policy) + actual_version = wheel_version(wheel) + if release: + require(actual_version == policy["target_version"], "Wheel version differs from upgrade policy target") + require(release_notes is not None, "Published release notes must declare upgrade support") + require( + support_text(policy, "en") in (release_notes or ""), + "GitHub Release notes must include the exact upgrade support declaration", + ) + # A fresh output directory prevents accepting stale reports from another build. + output.mkdir(parents=True, exist_ok=False) + evidence: dict[str, Any] = { + "policy_sha256": sha256(root / MANIFEST), + "target_version": actual_version, + "target_wheel_sha256": sha256(wheel), + "mode": policy["mode"], + "cases": [], + "status": "failed", + } + try: + for index, case in enumerate(policy["cases"]): + evidence["cases"].append(run_case(root, wheel, case, output / f"case-{index}")) + evidence["status"] = "passed" + finally: + (output / "result.json").write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheel", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--release", action="store_true") + parser.add_argument("--release-notes", type=Path) + parser.add_argument("--write-docs", action="store_true") + args = parser.parse_args() + try: + policy = load_policy(ROOT) + check_docs(ROOT, policy, write=args.write_docs) + if args.wheel: + require(args.output is not None, "--output is required with --wheel") + gate( + ROOT, + args.wheel.resolve(), + args.output.resolve(), + release=args.release, + release_notes=args.release_notes.read_text(encoding="utf-8") if args.release_notes else None, + ) + else: + require(not args.release, "--release requires the actual release wheel") + print(f"Upgrade policy: {policy['mode']}; {len(policy['cases'])} supported combinations") + except (GateError, OSError, ValueError, subprocess.SubprocessError, ElementTree.ParseError) as error: + print(f"Upgrade release gate failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/release_upgrades/README.md b/tests/release_upgrades/README.md new file mode 100644 index 0000000000..65ebec7430 --- /dev/null +++ b/tests/release_upgrades/README.md @@ -0,0 +1,79 @@ +# Installed-release upgrade acceptance + +`release/upgrade-support.json` is the explicit upgrade support policy. Currently it declares +`fresh-install-only`: **no production database upgrade combination is supported**. An empty matrix in this mode +does not prove a migration works. Full migration integration is tracked in #2154. + +To advertise an upgrade, add a reviewed case with `source_version` (an exact published package version), `backend` +(`sqlite`, `seekdb`, or `oceanbase`), a stable `configuration` identifier, and a `suite` path under this directory. +Set `mode` to `supported-upgrades`. Optional features that change persistent objects need explicit cases/configurations. +No ranges, moving refs, silent backend fallback, or implicit combinations are supported. + +The runner creates separate source/target environments, installs the exact published source package and the built +target wheel, verifies their versions, then executes the reviewed suite in an isolated working directory. Repository +pytest configuration and deployment environment variables are excluded. The suite must exercise those installed +processes, not import the current checkout's runtime in place of either version. Use `-I` when invoking their Python. + +The suite receives: + +- `PC_UPGRADE_SOURCE_PYTHON`, `PC_UPGRADE_TARGET_PYTHON`: isolated installed interpreters. +- `PC_UPGRADE_BACKEND`, `PC_UPGRADE_CONFIGURATION`: the declared case. +- `PC_UPGRADE_WORK`: disposable workspace for both databases, controlled inference fixtures, and environment files. +- `PC_UPGRADE_EVIDENCE`: destination for backend evidence JSON. + +Provide these behavioral tests (additional tests are allowed): + +| Test | Required behavior | +| --- | --- | +| `test_source_release_database` | Start the actual source Server to create its complete database. Populate representative Memory history, references, permissions, Candidates, tasks and cursors through that version's public interfaces; verify the data before shutdown. Do not create a minimal schema by hand. | +| `test_public_migration_commands` | Stop writers, run the target artifact's public maintenance commands, and validate full readiness. A limited bundle verification does not pass. | +| `test_target_startup` | Start the target Server against the migrated database and verify readiness. | +| `test_read_write_and_retrieval` | Read the old records, write new ones, and retrieve both through public interfaces. | +| `test_authorization` | Verify permitted and denied access and preservation of relevant grants. | +| `test_candidates_and_processing` | Verify retained Candidates and task consumption, including idempotency. | +| `test_preserved_history_and_cursors` | Verify exact history/references and unchanged or declared-transformed cursor and receipt semantics. | +| `test_fresh_upgrade_equivalence` | Initialize a fresh target database; compare schema/backend objects and behavior with the upgraded database, explicitly allowing only registered retained/archive objects. | + +Use a session fixture for the shared lifecycle; do not depend on pytest's test execution order. All collected tests +must execute and pass. Empty collection, missing phases, skips (including missing backends), xfail, failure, command +failure, and missing/malformed reports fail the gate. A real OceanBase suite must provision an isolated database and +fail if its required environment is unavailable; it cannot replace execution with mocks. Controlled model responses +may test domain behavior, but do not establish model quality. + +Write backend evidence containing `backend`, nonempty `engine_version`, `driver_version`, and recorded schema +inventories or their digests in `source_schema`, `target_schema`, and `fresh_schema`. OceanBase additionally requires +`tenant_mode: "mysql"` and `proxy_version` (or `direct`). Do not record credentials or business payloads. The controller +records installed Python/package versions, target wheel SHA-256, policy SHA-256, and the individual case result. + +Evidence is created during the gate run in a new directory. There is no option to import old successful reports. +Test names and metadata do not themselves prove correctness: reviewers must inspect the assertions and confirm +they follow the public operations and compare actual persisted data. This contract is intended to host the real +upgrade acceptance added with #2154, without claiming that it already exists. + +For example, after implementing and reviewing an actual suite, a case would have this shape: + +```json +{ + "source_version": "1.3.0", + "backend": "sqlite", + "configuration": "full-server-no-vector", + "suite": "tests/release_upgrades/test_sqlite_130.py" +} +``` + +This example is not an existing supported case. Regenerate installation support regions with +`uv run python scripts/upgrade_release_gate.py --write-docs` and run `make upgrade-policy-check`. When persistence +inputs change, calculate `managed_digest(ROOT)` from `upgrade_release_gate`, review the compatibility impact, and +update both the digest and rationale in `schema_review`. The inputs include persistence, optional code storage, +Server identity and authorization, and shared limits used in table definitions. Add new schema owners to +`MANAGED_INPUTS` when introducing them. The digest intentionally includes non-schema code: it forces a review of +changes, but does not infer semantic compatibility or replace real upgrade tests. + +Build exactly one wheel into `dist/`, then run `make upgrade-release-test`. Before release, additionally run +`uv run python scripts/run_upgrade_release_gate.py --release --release-notes /path/to/notes.md`. The notes must contain +the exact English support declaration generated in the installation guide. The release workflow performs this check +against the GitHub event's notes and gates both PyPI publication and release assets on the same tested wheel. + +Repository administrators must require **Upgrade release gate** in branch protection. Workflow code cannot configure +that setting. Because the release workflow starts on `release.published`, it can block package/asset publication, +not the earlier manual creation of a GitHub Release; run the prepublication command before publishing that page. diff --git a/tests/test_release_version.py b/tests/test_release_version.py index 9629fd7c07..74a5bca627 100644 --- a/tests/test_release_version.py +++ b/tests/test_release_version.py @@ -49,6 +49,7 @@ def release_repo(tmp_path: Path) -> Path: source = Path(__file__).parents[1] for relative in ( "scripts/release_version.py", + "release/upgrade-support.json", "openapi/powercontext.yaml", *INSTALL_PAGES, *HISTORICAL_AND_INDEPENDENT_FILES, @@ -98,13 +99,11 @@ def test_release_updates_installation_references_and_preserves_history(release_r english = (release_repo / "docs/en/docs/get-started/install-and-run.md").read_text(encoding="utf-8") chinese = (release_repo / "docs/zh/docs/get-started/install-and-run.md").read_text(encoding="utf-8") assert f"package `{version}`" in english - assert f"To upgrade to {version}:" in english + assert f"To install {version} in a separate environment" in english assert f"Python 包版本为 `{version}`" in chinese - assert f"升级到 {version}" in chinese - assert "Version 1.1.0 upgrades legacy" in english - assert "1.0.0 also need" in english - assert "1.1.0 会在 Server 启动时升级旧标签表约束" in chinese - assert "对于 1.0.0 之前的数据库" in chinese + assert f"安装 {version} 到独立环境" in chinese + assert "upgrades of existing Server databases are not supported" in english + assert "尚不支持已有 Server 数据库升级" in chinese dsh = (release_repo / "integrations/dsh/plugins/powercontext/README.md").read_text(encoding="utf-8") assert "Release 1.1.0 includes direct-operation Scope failure handling" in dsh for relative, original in preserved.items(): diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py new file mode 100644 index 0000000000..2909bdbcba --- /dev/null +++ b/tests/test_upgrade_release_gate.py @@ -0,0 +1,210 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Release-blocking regressions, not evidence that any database upgrade works.""" + +import json +import zipfile +from pathlib import Path +from xml.etree import ElementTree + +import pytest +import upgrade_release_gate as gate +import yaml + + +@pytest.fixture +def release(tmp_path): + persistence = tmp_path / "src/powercontext/builtin/persistence/tables.py" + persistence.parent.mkdir(parents=True) + persistence.write_text("# disposable persistence input\n") + contract = tmp_path / "openapi/powercontext.yaml" + contract.parent.mkdir() + contract.write_text("info:\n version: 2.0.0\n") + (tmp_path / "release").mkdir() + policy = { + "format": 1, + "target_version": "2.0.0", + "mode": "fresh-install-only", + "schema_review": {"persistence_sha256": gate.managed_digest(tmp_path), "rationale": "No upgrades supported"}, + "cases": [], + } + (tmp_path / gate.MANIFEST).write_text(json.dumps(policy)) + for language in ("en", "zh"): + page = tmp_path / f"docs/{language}/docs/get-started/install-and-run.md" + page.parent.mkdir(parents=True) + page.write_text("\nplaceholder\n\n") + gate.check_docs(tmp_path, policy, write=True) + wheel = tmp_path / "powercontext-2.0.0-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("powercontext-2.0.0.dist-info/METADATA", "Name: powercontext\nVersion: 2.0.0\n") + return tmp_path, wheel, policy + + +def test_fresh_install_policy_does_not_claim_upgrade_acceptance(release): + root, wheel, policy = release + gate.gate(root, wheel, root / "evidence", release=True, release_notes=gate.support_text(policy, "en")) + result = json.loads((root / "evidence/result.json").read_text()) + assert result["mode"] == "fresh-install-only" and result["cases"] == [] + assert result["target_wheel_sha256"] == gate.sha256(wheel) + + +def test_release_notes_cannot_omit_the_support_limit(release): + root, wheel, _ = release + with pytest.raises(gate.GateError, match="notes"): + gate.gate(root, wheel, root / "evidence", release=True, release_notes="Upgrade your deployment now.") + + +def test_release_rejects_wrong_artifact_version(release): + root, wheel, _ = release + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("powercontext-9.0.0.dist-info/METADATA", "Name: powercontext\nVersion: 9.0.0\n") + with pytest.raises(gate.GateError, match="Wheel version"): + gate.gate(root, wheel, root / "evidence", release=True) + + +def test_schema_change_needs_a_new_compatibility_review(release): + root, _, _ = release + (root / "src/powercontext/builtin/persistence/tables.py").write_text("# different schema\n") + with pytest.raises(gate.GateError, match="Persistence changed"): + gate.load_policy(root) + + +def test_supported_mode_cannot_succeed_with_an_empty_matrix(release): + root, _, policy = release + policy["mode"] = "supported-upgrades" + (root / gate.MANIFEST).write_text(json.dumps(policy)) + with pytest.raises(gate.GateError, match="matrix disagree"): + gate.load_policy(root) + + +@pytest.mark.parametrize( + "relative", + [ + "src/powercontext/server/identity.py", + "src/powercontext/server/authz/repository.py", + "src/powercontext/builtin/code/seekdb.py", + "src/powercontext/limits.py", + ], +) +def test_schema_owners_outside_persistence_need_compatibility_review(release, relative): + root, _, _ = release + owner = root / relative + owner.parent.mkdir(parents=True, exist_ok=True) + owner.write_text("# changed managed schema input\n") + with pytest.raises(gate.GateError, match="Persistence changed"): + gate.load_policy(root) + + +def supported_case(root, policy): + suite = root / "tests/release_upgrades/test_example.py" + suite.parent.mkdir(parents=True) + suite.write_text("# This fixture is only used to exercise gate failures.\n") + policy["mode"] = "supported-upgrades" + policy["cases"] = [ + { + "source_version": "1.3.0", + "backend": "sqlite", + "configuration": "full-server", + "suite": suite.relative_to(root).as_posix(), + } + ] + (root / gate.MANIFEST).write_text(json.dumps(policy)) + gate.check_docs(root, policy, write=True) + + +@pytest.mark.parametrize( + "blocker", ["migration_required", "unmanaged full Server layout", "migration_framework_not_ready"] +) +def test_any_upgrade_stage_failure_blocks_release_and_records_failure(release, monkeypatch, blocker): + root, wheel, policy = release + supported_case(root, policy) + + def fail(*_args): + raise gate.GateError(blocker) + + monkeypatch.setattr(gate, "run_case", fail) + with pytest.raises(gate.GateError, match=blocker): + gate.gate(root, wheel, root / "evidence") + assert json.loads((root / "evidence/result.json").read_text())["status"] == "failed" + + +def test_missing_suite_cannot_be_advertised(release): + root, _, policy = release + supported_case(root, policy) + (root / policy["cases"][0]["suite"]).unlink() + with pytest.raises(gate.GateError, match="Missing acceptance suite"): + gate.load_policy(root) + + +def test_installation_guidance_cannot_drift_from_matrix(release): + root, _, policy = release + page = root / "docs/en/docs/get-started/install-and-run.md" + page.write_text(page.read_text().replace(gate.NOTICE["en"], "All upgrades are supported.")) + with pytest.raises(gate.GateError, match="instructions differ"): + gate.check_docs(root, policy) + + +@pytest.mark.parametrize("invalid", ["empty", "missing-startup", "skipped", "failure", "error", "truncated"]) +def test_partial_or_skipped_acceptance_is_not_passing_evidence(tmp_path, invalid): + report = tmp_path / "junit.xml" + suite = ElementTree.Element("testsuite") + for phase in sorted(gate.PHASES): + if invalid == "empty" or (invalid == "missing-startup" and phase == "test_target_startup"): + continue + case = ElementTree.SubElement(suite, "testcase", name=phase) + if phase == "test_target_startup" and invalid in {"skipped", "failure", "error"}: + ElementTree.SubElement(case, invalid) + ElementTree.ElementTree(suite).write(report) + if invalid == "truncated": + report.write_text("") + with pytest.raises((gate.GateError, ElementTree.ParseError)): + gate.validate_junit(report) + + +def test_complete_passing_report_is_accepted(tmp_path): + report = tmp_path / "junit.xml" + suite = ElementTree.Element("testsuite") + for phase in gate.PHASES: + ElementTree.SubElement(suite, "testcase", name=phase) + ElementTree.ElementTree(suite).write(report) + gate.validate_junit(report) + + +def test_missing_backend_evidence_fails_even_with_a_passing_report(tmp_path): + report = tmp_path / "backend.json" + report.write_text(json.dumps({"backend": "oceanbase", "engine_version": "4.6.3"})) + with pytest.raises(gate.GateError, match="driver_version"): + gate.validate_backend_evidence(report, "oceanbase") + + +def test_old_evidence_directory_cannot_be_reused(release): + root, wheel, _ = release + gate.gate(root, wheel, root / "evidence") + with pytest.raises(FileExistsError): + gate.gate(root, wheel, root / "evidence") + + +def test_publishing_jobs_depend_on_nonoptional_preupload_upgrade_gate(): + root = Path(__file__).parents[1] + workflow = yaml.safe_load((root / ".github/workflows/release.yml").read_text()) + build = workflow["jobs"]["release-build"] + steps = build["steps"] + index = next(i for i, step in enumerate(steps) if "run_upgrade_release_gate.py" in step.get("run", "")) + assert "if" not in steps[index] and not steps[index].get("continue-on-error") + assert "--release --release-event" in steps[index]["run"] + upload = next(i for i, step in enumerate(steps) if step.get("with", {}).get("name") == "release-dists") + assert index < upload + for name in ("pypi-publish", "release-assets"): + assert workflow["jobs"][name]["needs"] == "release-build" From c4fc8ab7c3e8eb669c23dee6b4d0820583493620 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 22 Oct 2026 10:32:14 +0800 Subject: [PATCH 23/94] fix(migrations): preserve literal percent in CHECK validation --- .github/workflows/master.yml | 1 + .../persistence/migrations/mysql_schema.py | 4 +- .../test_mysql_migration_checks.py | 56 +++++++++++++++++++ tests/fixtures/database_migrations/README.md | 5 ++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 01ff65b202..500fa104e2 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -298,6 +298,7 @@ jobs: tests/builtin/persistence/test_database_backup_providers.py \ tests/builtin/persistence/test_database_migration_connections.py \ tests/e2e/test_topic_memory_recovery_migration.py \ + tests/builtin/persistence/test_mysql_migration_checks.py \ tests/builtin/persistence/test_mysql_migration_schema.py -k seekdb \ 2>&1 | tee seekdb-migrations.log uv run --locked --extra seekdb pytest --collect-only -q --color=no \ diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py index 3449fb7395..95c5792974 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql_schema.py +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -231,8 +231,10 @@ def validate_data(self, connection: Connection, revision: str) -> None: for name, table in expected["tables"].items(): for check in table["checks"].values(): expression = _expression_sql(check["expression"]) + # No bind parameters: keep literal percent signs out of DBAPI interpolation. violation = connection.exec_driver_sql( - f"SELECT 1 FROM {_quote(name)} WHERE NOT ({expression}) LIMIT 1" # noqa: S608 -- frozen expressions + f"SELECT 1 FROM {_quote(name)} WHERE NOT ({expression}) LIMIT 1", # noqa: S608 -- frozen expressions + execution_options={"no_parameters": True}, ).first() if violation is not None: raise MigrationError("data_integrity_error", "Existing rows violate a registered CHECK constraint.") diff --git a/tests/builtin/persistence/test_mysql_migration_checks.py b/tests/builtin/persistence/test_mysql_migration_checks.py index 080dd7229d..fc779a9d93 100644 --- a/tests/builtin/persistence/test_mysql_migration_checks.py +++ b/tests/builtin/persistence/test_mysql_migration_checks.py @@ -18,6 +18,7 @@ import json import os +import shutil from pathlib import Path import pytest @@ -26,8 +27,10 @@ from powercontext.builtin.persistence.migrations.bundle import MigrationBundle from powercontext.builtin.persistence.migrations.connections import MaintenanceConnections from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.mysql import MySQLMigrationRunner from powercontext.builtin.persistence.migrations.mysql_schema import MySQLSchema, _expression_sql, mysql_inventory from powercontext.builtin.persistence.seekdb import SeekDBConfig +from tests.builtin.persistence.test_mysql_migration_runner import _seekdb_scenario_lifetime from tests.builtin.persistence.test_mysql_migration_schema import _inventory, _ReflectedDatabase _FIXTURE = Path(__file__).parents[2] / "fixtures/database_migrations/schemas/seekdb_v1_3_0_checks.json" @@ -168,3 +171,56 @@ def reflect(connection, _identity, guard): MySQLSchema(MigrationBundle(_RESOURCES)).identify(inventory) MaintenanceConnections(SeekDBConfig(path=short_tmp_path / "seekdb")).run(reflect, writable=True) + + +@pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") +@pytest.mark.parametrize( + ("expression", "column_type", "valid", "invalid"), + [("a % 2 = 0", "INTEGER", 2, 3), ("a = '100%s%%'", "VARCHAR(32)", "100%s%%", "different")], + ids=["remainder", "percent_literal"], +) +def test_real_seekdb_percent_checks_validate_data_and_plan( + short_tmp_path, monkeypatch, expression, column_type, valid, invalid +): + pytest.importorskip("pylibseekdb") + # Extend an isolated bundle, preserving the shipped layouts and their eligibility. + directory = short_tmp_path / "bundle" + shutil.copytree(_RESOURCES, directory) + ddl = f"CREATE TABLE pc_check_fixture (a {column_type}, CONSTRAINT ck_percent CHECK ({expression}) NOT ENFORCED)" + for name in ("pre_dream", "v1_1_0"): + path = directory / "schemas" / f"{name}.json" + layout = json.loads(path.read_text()) + layout["mysql"].append(ddl) + path.write_text(json.dumps(layout, ensure_ascii=False)) + bundle = MigrationBundle(directory) + schema = MySQLSchema(bundle) + connections = MaintenanceConnections(SeekDBConfig(path=short_tmp_path / "seekdb")) + runner = MySQLMigrationRunner(connections, bundle, evidence_directory=short_tmp_path / "evidence") + + def prepare(connection, _identity, guard): + assert connection is not None + baseline = json.loads((directory / "schemas/pre_dream.json").read_text())["mysql"] + for statement in baseline: + guard() + connection.exec_driver_sql(statement, execution_options={"no_parameters": True}) + connection.exec_driver_sql("INSERT INTO pc_check_fixture (a) VALUES (%s), (NULL)", (valid,)) + connection.commit() + inventory = mysql_inventory(connection) + assert schema.identify(inventory) == "p0001" + schema.verify(inventory, "p0001") + schema.validate_data(connection, "p0001") + + def invalidate(connection, _identity, guard): + assert connection is not None + guard() + connection.exec_driver_sql("UPDATE pc_check_fixture SET a = %s WHERE a IS NOT NULL", (invalid,)) + connection.commit() + with pytest.raises(MigrationError, match=r"data_integrity_error.*CHECK"): + schema.validate_data(connection, "p0001") + + with _seekdb_scenario_lifetime(monkeypatch): + connections.run(prepare, writable=True) + assert runner.plan().state == "migration_required" + connections.run(invalidate, writable=True) + with pytest.raises(MigrationError, match=r"data_integrity_error.*CHECK"): + runner.plan() diff --git a/tests/fixtures/database_migrations/README.md b/tests/fixtures/database_migrations/README.md index 888508ca8e..07ebef7631 100644 --- a/tests/fixtures/database_migrations/README.md +++ b/tests/fixtures/database_migrations/README.md @@ -49,6 +49,11 @@ the historical tables and executes the rendered drop-total CHECK against matchin POWERCONTEXT_TEST_MIGRATION_SEEKDB=1 uv run --locked --extra seekdb pytest -q tests/builtin/persistence/test_mysql_migration_checks.py ``` +The same test module extends a disposable bundle with remainder and percent-literal CHECKs and exercises +`MySQLSchema.validate_data()` and `MySQLMigrationRunner.plan()` through the official aiomysql bridge. +Valid and NULL rows pass; stored violations of the fixture's `NOT ENFORCED` constraints fail with +`data_integrity_error`. These synthetic constraints are not added to the shipped migration layouts. + ## Explicit maintenance Repository-only scratch commands: From eff8bacc92be8a4ba7f307434b7db812cebff465 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 22 Oct 2026 10:42:51 +0800 Subject: [PATCH 24/94] ci(release): discover upgrade sources and exercise embedded seekdb --- .github/workflows/master.yml | 9 +- docs/en/development/releasing.md | 15 ++- docs/zh/development/releasing.md | 12 +- release/upgrade-support.json | 20 +++- scripts/ci_legacy_upgrade_rejection.py | 113 ++++++++++++------ scripts/run_upgrade_release_gate.py | 4 +- scripts/upgrade_database_probe.py | 85 +++++++++++++ scripts/upgrade_release_gate.py | 49 ++++++-- scripts/upgrade_release_matrix.py | 105 ++++++++++++++++ tests/release_upgrades/README.md | 70 +++++++++-- tests/test_upgrade_release_gate.py | 158 ++++++++++++++++++++++++- 11 files changed, 570 insertions(+), 70 deletions(-) create mode 100644 scripts/upgrade_database_probe.py create mode 100644 scripts/upgrade_release_matrix.py diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index d9c4d2ad9c..c6489517a7 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -41,11 +41,14 @@ jobs: run: >- uv run python scripts/run_upgrade_release_gate.py --output "$RUNNER_TEMP/upgrade-release-evidence" - - name: Reproduce unsupported full 1.3.0 database upgrade + - name: Reproduce unsupported SQLite and embedded seekDB upgrades run: | wheel=$(find dist -name '*.whl' -print -quit) - uv run python scripts/ci_legacy_upgrade_rejection.py \ - --wheel "$wheel" --output "$RUNNER_TEMP/upgrade-release-evidence/legacy-rejection" + for backend in sqlite seekdb; do + uv run python scripts/ci_legacy_upgrade_rejection.py \ + --backend "$backend" --respect-policy --wheel "$wheel" \ + --output "$RUNNER_TEMP/upgrade-release-evidence/legacy-$backend" + done - name: Preserve upgrade acceptance evidence if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 diff --git a/docs/en/development/releasing.md b/docs/en/development/releasing.md index e95ae9c0b6..878945cf10 100644 --- a/docs/en/development/releasing.md +++ b/docs/en/development/releasing.md @@ -129,8 +129,12 @@ installation pins are synchronized. ### Database upgrade support -`release/upgrade-support.json` declares the target version and every supported source version, backend, and -configuration. The current policy is `fresh-install-only`; existing Server database upgrades are unsupported. +`release/upgrade-support.json` declares the target version, source-version selection policy, backends, and +configurations. `automatic_sources` selects every stable PyPI release from `minimum_version` up to, but excluding, +the actual target wheel. Newly published stable versions become source cases on subsequent runs without adding +per-version tests. The runner expands every source across all backend profiles and records source wheel SHA-256s. +The current policy is `fresh-install-only`; existing Server database upgrades are unsupported. Its null suite paths +record candidates only, not accepted upgrades. Supported mode requires implemented suites for every backend profile. Do not advertise a supported upgrade until its installed-release acceptance suite is implemented and passes. The suite contract and evidence requirements are in `tests/release_upgrades/README.md` in the repository. @@ -152,9 +156,10 @@ in separate environments. Missing tests, skipped backends, failures, and missing `fresh-install-only` policy check does not establish an upgrade path. Ordinary PR builds may have a development version; the `--release` check must run against the final tagged wheel without overriding its version. -The **Upgrade release gate** CI job also reproduces the known unsupported upgrade rejections using a real 1.3.0 -SQLite Server database. Replace that negative fixture with supported-path acceptance when implementing the full -migration chain. Repository administrators must make this job required in branch protection. +The **Upgrade release gate** CI job also reproduces the known unsupported upgrade rejections using real 1.3.0 +SQLite and embedded seekDB Server databases. It installs and starts seekDB inside the runner with a disposable +directory; no external service is needed. Once supported mode is activated, these negative assertions retire and +the full discovered upgrade matrix must pass. Repository administrators must make this job required in branch protection. The release workflow gates distribution upload, PyPI publication, and release assets on the acceptance result and archives its evidence. It runs after `release.published`, so it cannot prevent the earlier manual publication of the GitHub Release page: run the command above before that action. `make publish` requires diff --git a/docs/zh/development/releasing.md b/docs/zh/development/releasing.md index bfc18fc73c..760ea31da7 100644 --- a/docs/zh/development/releasing.md +++ b/docs/zh/development/releasing.md @@ -122,8 +122,11 @@ DSH README 中说明某项能力已包含在 `1.1.0` 的文字同样属于历史 ### 数据库升级支持范围 -`release/upgrade-support.json` 声明目标版本，以及每个受支持的来源版本、后端和配置组合。当前策略为 -`fresh-install-only`，不支持已有 Server 数据库升级。只有实现并通过已安装发行包的完整升级验收，才能声明支持 +`release/upgrade-support.json` 声明目标版本、来源版本选择策略、后端和配置。`automatic_sources` 从 PyPI 自动选择 +大于等于 `minimum_version`、小于实际目标 wheel 的全部稳定版本，再与每个后端配置展开组合。新稳定版发布后， +后续运行会自动将其加入来源，不必逐版本新增测试。证据会记录具体来源 wheel 的 SHA-256。 +当前策略为 `fresh-install-only`，不支持已有 Server 数据库升级；配置中的空套件路径只生成候选组合，不算验收通过。 +启用支持升级模式时，每个后端配置都必须有真实套件。只有实现并通过已安装发行包的完整升级验收，才能声明支持 对应组合。验收契约和证据要求见仓库中的 `tests/release_upgrades/README.md`。 `make check` 和 `make release-check` 会核对策略及双语安装说明。受管持久化文件变化时，必须审查兼容性影响， @@ -141,8 +144,9 @@ uv run python scripts/run_upgrade_release_gate.py --release --release-notes /pat 旧版发行包和目标 wheel。测试缺失、后端跳过、执行失败或证据缺失都会阻断检查。`fresh-install-only` 策略检查 通过不表示存在升级路径。普通 PR 构建允许使用开发版本；`--release` 必须验证最终 tag 构建的 wheel，不能覆盖版本号。 -CI 中的 **Upgrade release gate** 还会使用真实 1.3.0 SQLite Server 数据库复现已知的不支持升级拒绝行为。 -完整迁移链实现后，应将这一负向场景替换为支持路径的验收。仓库管理员需要在分支保护中将该检查设为必需。 +CI 中的 **Upgrade release gate** 还会使用真实 1.3.0 SQLite 和嵌入式 seekDB Server 数据库复现已知的升级拒绝行为。 +seekDB 直接在 runner 的临时目录中启动，不需要外部服务。启用支持升级模式后，固定的拒绝断言退出，自动展开的 +全部成功升级组合必须通过验收。仓库管理员需要在分支保护中将该检查设为必需。 发布工作流只有验收通过才会上传发行包、发布 PyPI 和上传 Release 附件，并归档验收证据。工作流在 `release.published` 后运行，无法阻止之前手动发布 GitHub Release 页面，因此必须先运行上述命令再发布页面。 `make publish` 要求提供 `UPGRADE_RELEASE_NOTES=/path/to/release-notes.md`，并在上传软件包之前执行相同门禁。 diff --git a/release/upgrade-support.json b/release/upgrade-support.json index a1a0dff078..4e61ff6624 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -1,10 +1,26 @@ { - "format": 1, + "format": 2, "target_version": "1.4.0", "mode": "fresh-install-only", "schema_review": { "persistence_sha256": "ca3b475f2d0c883e19291a8d8b1dadbe50f93ebd93aad550a9112a3497737927", "rationale": "The full Server upgrade framework is incomplete (#2154). Existing databases are unsupported; no production upgrade combination is advertised until installed-release acceptance passes." }, - "cases": [] + "cases": [], + "automatic_sources": { + "strategy": "all-stable-since", + "minimum_version": "1.3.0", + "profiles": [ + { + "backend": "sqlite", + "configuration": "full-server-no-vector", + "suite": null + }, + { + "backend": "seekdb", + "configuration": "full-server-no-vector", + "suite": null + } + ] + } } diff --git a/scripts/ci_legacy_upgrade_rejection.py b/scripts/ci_legacy_upgrade_rejection.py index cefdf44def..7bedf3a5f2 100644 --- a/scripts/ci_legacy_upgrade_rejection.py +++ b/scripts/ci_legacy_upgrade_rejection.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Reproduce unsupported 1.3.0 upgrade rejection on an actual old SQLite Server. +"""Reproduce unsupported 1.3.0 upgrades on an actual SQLite or embedded seekDB Server. -This is a negative regression, never positive upgrade acceptance. Once the path -is supported, replace this fixture with the full matrix suite before advertising it. +This is a negative regression, never positive upgrade acceptance. CI retires +these assertions once the supported-path matrix becomes mandatory. """ from __future__ import annotations @@ -23,15 +23,13 @@ import argparse import json import os -import shutil -import sqlite3 import subprocess import tempfile from pathlib import Path import httpx from ci_release_smoke import _available_port, _stop_process, _wait_until_ready -from upgrade_release_gate import installed, require, run, sha256, wheel_version +from upgrade_release_gate import ROOT, installed, load_policy, require, run, sha256, wheel_version def console(python: Path) -> Path: @@ -53,7 +51,7 @@ def source_database(python: Path, directory: Path, environment: dict[str, str], text=True, ) try: - _wait_until_ready(base_url, process, 90) + _wait_until_ready(base_url, process, 180) with httpx.Client(base_url=base_url, trust_env=False, timeout=30) as client: response = client.post( "/v1/scopes", @@ -81,7 +79,9 @@ def source_database(python: Path, directory: Path, environment: dict[str, str], _stop_process(process) -def rejected(command: list[str], environment: dict[str, str], directory: Path, log: Path, reason: str) -> None: +def rejected( + command: list[str], environment: dict[str, str], directory: Path, log: Path, reasons: tuple[str, ...] +) -> None: with log.open("w", encoding="utf-8") as stream: process = subprocess.Popen( # noqa: S603 -- installed target CLI, isolated data only command, @@ -94,31 +94,57 @@ def rejected(command: list[str], environment: dict[str, str], directory: Path, l start_new_session=os.name != "nt", ) try: - process.wait(timeout=90) + process.wait(timeout=180) finally: _stop_process(process) output = log.read_text(encoding="utf-8") - require(process.returncode != 0 and reason in output, f"Expected rejection {reason}; inspect {log.name}") + require( + process.returncode != 0 and any(reason in output for reason in reasons), + f"Expected rejection {reasons}; inspect {log.name}", + ) def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--wheel", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--backend", choices=("sqlite", "seekdb"), default="sqlite") + parser.add_argument( + "--respect-policy", + action="store_true", + help="Retire rejection assertions once supported-path acceptance is required", + ) args = parser.parse_args() wheel = args.wheel.resolve() output = args.output.resolve() output.mkdir(parents=True, exist_ok=False) + if args.respect_policy and load_policy(ROOT)["mode"] == "supported-upgrades": + (output / "result.json").write_text( + json.dumps( + { + "kind": "unsupported-upgrade-regression", + "backend": args.backend, + "status": "not-applicable", + "reason": "Supported upgrades are evaluated by the required acceptance gate.", + }, + indent=2, + ) + + "\n" + ) + print("Historical rejection assertions retired; the supported upgrade matrix must pass instead.") + return environment = { key: value for key, value in os.environ.items() if not key.startswith(("POWERCONTEXT_", "PYTEST_")) and key not in {"PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV"} } environment["NO_PROXY"] = environment["no_proxy"] = "localhost,127.0.0.1" - with tempfile.TemporaryDirectory(prefix="pc-old-release-") as temporary: + # A short pathname avoids the embedded engine's Unix socket length limit. + with tempfile.TemporaryDirectory(prefix="pc-up-", dir="/tmp" if os.name != "nt" else None) as temporary: # noqa: S108 -- mkdtemp creates a private random directory directory = Path(temporary) interpreters = {} - for role, package in {"source": "powercontext[cli,server]==1.3.0", "target": f"{wheel}[cli,server]"}.items(): + extras = "cli,server,seekdb" if args.backend == "seekdb" else "cli,server" + for role, package in {"source": f"powercontext[{extras}]==1.3.0", "target": f"{wheel}[{extras}]"}.items(): venv = directory / role run( ["uv", "venv", "--python", "3.12", str(venv)], @@ -137,58 +163,77 @@ def main() -> None: metadata = {role: installed(python, environment, directory) for role, python in interpreters.items()} require(metadata["source"]["powercontext"] == "1.3.0", "Wrong source package") require(metadata["target"]["powercontext"] == wheel_version(wheel), "Wrong target package") - database = directory / "source.db" + database = directory / "database" + database_environment = ( + {"POWERCONTEXT_SERVER_DATABASE_KIND": "seekdb", "POWERCONTEXT_SERVER_DATABASE_PATH": str(database)} + if args.backend == "seekdb" + else {"POWERCONTEXT_SERVER_DATABASE_URL": f"sqlite+aiosqlite:///{database}"} + ) environment.update({ "POWERCONTEXT_HOME": str(directory / "home"), - "POWERCONTEXT_SERVER_DATABASE_URL": f"sqlite+aiosqlite:///{database}", + **database_environment, "POWERCONTEXT_SERVER_ACCESS_MODE": "disabled", }) seed = source_database(interpreters["source"], directory, environment, output / "source.log") - with sqlite3.connect(database) as connection: - schema = connection.execute( - "SELECT type, name, tbl_name, sql FROM sqlite_schema ORDER BY type, name" - ).fetchall() - tables = {row[1] for row in schema if row[0] == "table"} + probe = [ + "-I", + str(ROOT / "scripts/upgrade_database_probe.py"), + "--backend", + args.backend, + "--database", + str(database), + ] + for role, python in interpreters.items(): + run( + [str(python), *probe, "--output", str(output / f"{role}-schema.json")], + environment=environment, + cwd=directory, + log=output / f"{role}-probe.log", + ) + schema = json.loads((output / "source-schema.json").read_text()) + target_schema = json.loads((output / "target-schema.json").read_text()) + tables = set(schema["tables"]) require( {"pc_artifact_candidate_heads", "pc_artifact_candidate_versions", "pc_scopes", "pc_artifacts"} <= tables, "Source fixture lacks historical Server objects", ) - (output / "source-schema.json").write_text(json.dumps(schema, indent=2) + "\n") env_file = directory / "deployment.env" - env_file.write_text(f"POWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite:///{database}\n") + env_file.write_text("".join(f"{key}={value}\n" for key, value in database_environment.items())) target = str(console(interpreters["target"])) rejected( [target, "server", "db-migrate", "status", "--env-file", str(env_file)], environment, directory, output / "status.log", - "unknown_baseline", + ("unknown_baseline", "incompatible_schema") if args.backend == "seekdb" else ("unknown_baseline",), ) rejected( [target, "server", "run", "--no-env-file", "--host", "127.0.0.1", "--port", str(_available_port())], environment, directory, output / "legacy-startup.log", - "migration_required", + ("migration_required",), + ) + # Only after both historical checks, mutate this disposable fixture into + # a deliberately incomplete versioned state. This is not a migration. + run( + [str(interpreters["target"]), *probe, "--add-revision", "--output", str(output / "versioned-schema.json")], + environment=environment, + cwd=directory, + log=output / "versioned-probe.log", ) - # This second negative fixture intentionally adds only the prototype control - # table to a copy. It must never be mistaken for a completed migration. - versioned = directory / "versioned.db" - shutil.copyfile(database, versioned) - with sqlite3.connect(versioned) as connection: - connection.execute("CREATE TABLE pc_schema_revision (version_num VARCHAR(32) PRIMARY KEY)") - environment["POWERCONTEXT_SERVER_DATABASE_URL"] = f"sqlite+aiosqlite:///{versioned}" rejected( [target, "server", "run", "--no-env-file", "--host", "127.0.0.1", "--port", str(_available_port())], environment, directory, output / "versioned-startup.log", - "migration_framework_not_ready", + ("migration_framework_not_ready",), ) evidence = { "kind": "unsupported-upgrade-regression", - "backend": "sqlite", - "engine_version": sqlite3.sqlite_version, + "backend": args.backend, + "source_database": {key: value for key, value in schema.items() if key != "tables"}, + "target_database": {key: value for key, value in target_schema.items() if key != "tables"}, "installed": metadata, "wheel_sha256": sha256(wheel), "seed": seed, @@ -196,7 +241,7 @@ def main() -> None: "status": "passed", } (output / "result.json").write_text(json.dumps(evidence, indent=2) + "\n") - print("Actual 1.3.0 full database: all three unsupported-upgrade regressions reproduced.") + print(f"Actual 1.3.0 {args.backend} database: all three unsupported-upgrade regressions reproduced.") if __name__ == "__main__": diff --git a/scripts/run_upgrade_release_gate.py b/scripts/run_upgrade_release_gate.py index 89ddb8f0d6..b585530f15 100644 --- a/scripts/run_upgrade_release_gate.py +++ b/scripts/run_upgrade_release_gate.py @@ -43,7 +43,9 @@ def main() -> None: gate(ROOT, wheels[0].resolve(), output.resolve(), release=args.release, release_notes=release_notes) result = json.loads((output / "result.json").read_text(encoding="utf-8")) print( - f"Upgrade gate passed: {result['mode']}; {len(result['cases'])} accepted upgrade combinations. " + f"Upgrade gate passed: {result['mode']}; " + f"{len(result['resolved_matrix']['candidate_cases'])} discovered candidates; " + f"{len(result['cases'])} accepted upgrade combinations. " f"Evidence: {output / 'result.json'}" ) except (GateError, OSError, ValueError) as error: diff --git a/scripts/upgrade_database_probe.py b/scripts/upgrade_database_probe.py new file mode 100644 index 0000000000..8b8822979d --- /dev/null +++ b/scripts/upgrade_database_probe.py @@ -0,0 +1,85 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Inspect a disposable database using the installed release's own driver. + +Run only after the Server has exited. --add-revision creates a deliberately +incomplete control table for a negative fixture; it never represents a migration. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sqlite3 +from importlib import import_module +from importlib.metadata import version +from pathlib import Path +from typing import Any + +REVISION_DDL = "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) PRIMARY KEY)" + + +async def inspect(backend: str, database: Path, *, add_revision: bool) -> dict[str, Any]: + if backend == "sqlite": + with sqlite3.connect(database) as connection: + if add_revision: + connection.execute(REVISION_DDL) + rows = connection.execute("SELECT name, sql FROM sqlite_schema WHERE type='table' ORDER BY name").fetchall() + return {"engine_version": sqlite3.sqlite_version, "driver_version": version("aiosqlite"), "tables": dict(rows)} + + # Use the driver's public API, not a current-checkout Profile that could run + # startup guards or mutate an old schema before the target CLI is tested. + binding = import_module("pylibseekdb") + pymysql = import_module("pymysql") + instance = await binding.aopen(str(database)) + try: + with ( + pymysql.connect(**instance.connection_options(), database="test", autocommit=True) as connection, + connection.cursor() as cursor, + ): + if add_revision: + cursor.execute(REVISION_DDL) + cursor.execute("SELECT VERSION()") + engine = str(cursor.fetchone()[0]) + cursor.execute("SHOW FULL TABLES WHERE Table_type = 'BASE TABLE'") + names = sorted(row[0] for row in cursor.fetchall()) + tables = {} + for name in names: + cursor.execute("SHOW CREATE TABLE `" + name.replace("`", "``") + "`") + tables[name] = cursor.fetchone()[1] + return { + "engine_version": engine, + "driver_version": version("pymysql"), + "embedded_binding_version": version("pylibseekdb"), + "tables": tables, + } + finally: + instance.close() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--backend", choices=("sqlite", "seekdb"), required=True) + parser.add_argument("--database", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--add-revision", action="store_true") + args = parser.parse_args() + result = asyncio.run(inspect(args.backend, args.database, add_revision=args.add_revision)) + args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/scripts/upgrade_release_gate.py b/scripts/upgrade_release_gate.py index 62bdbddce9..ba0178a88d 100644 --- a/scripts/upgrade_release_gate.py +++ b/scripts/upgrade_release_gate.py @@ -30,6 +30,8 @@ from typing import Any from xml.etree import ElementTree +from upgrade_release_matrix import STABLE, resolve_matrix + ROOT = Path(__file__).resolve().parents[1] MANIFEST = Path("release/upgrade-support.json") MANAGED_INPUTS = ( @@ -88,8 +90,11 @@ def managed_digest(root: Path) -> str: def load_policy(root: Path) -> dict[str, Any]: policy = json.loads((root / MANIFEST).read_text(encoding="utf-8")) - require(set(policy) == {"format", "target_version", "mode", "schema_review", "cases"}, "Invalid policy fields") - require(policy["format"] == 1, "Unsupported upgrade policy format") + require( + set(policy) == {"format", "target_version", "mode", "schema_review", "cases", "automatic_sources"}, + "Invalid policy fields", + ) + require(policy["format"] == 2, "Unsupported upgrade policy format") contract = (root / "openapi/powercontext.yaml").read_text(encoding="utf-8") match = re.search(rf"^ version: ({VERSION})$", contract, re.MULTILINE) require( @@ -106,7 +111,21 @@ def load_policy(root: Path) -> dict[str, Any]: ) cases = policy["cases"] require(isinstance(cases, list), "Upgrade cases must be a list") - require(bool(cases) == (policy["mode"] == "supported-upgrades"), "Upgrade mode and support matrix disagree") + automatic = policy["automatic_sources"] + profiles = [] + if automatic is not None: + require(set(automatic) == {"strategy", "minimum_version", "profiles"}, "Invalid automatic source policy") + require(automatic["strategy"] == "all-stable-since", "Unknown source selection strategy") + require( + isinstance(automatic["minimum_version"], str) and re.fullmatch(STABLE, automatic["minimum_version"]), + "Minimum source must be a stable version", + ) + profiles = automatic["profiles"] + require(isinstance(profiles, list) and profiles, "Automatic sources require backend profiles") + for profile in profiles: + require(set(profile) == {"backend", "configuration", "suite"}, "Invalid automatic backend profile") + supported = policy["mode"] == "supported-upgrades" + require(bool(cases or profiles) if supported else not cases, "Upgrade mode and support matrix disagree") identities: set[tuple[str, str, str]] = set() for case in cases: require(set(case) == {"source_version", "backend", "configuration", "suite"}, "Invalid upgrade case fields") @@ -114,11 +133,15 @@ def load_policy(root: Path) -> dict[str, Any]: isinstance(case["source_version"], str) and re.fullmatch(VERSION, case["source_version"]), "Pin a source release", ) + for case in [*cases, *profiles]: require(case["backend"] in {"sqlite", "seekdb", "oceanbase"}, "Unknown backend") require(re.fullmatch(r"[a-z][a-z0-9-]*", case["configuration"]), "Configuration needs a stable identifier") - identity = (case["source_version"], case["backend"], case["configuration"]) + identity = (case.get("source_version", "automatic"), case["backend"], case["configuration"]) require(identity not in identities, "Duplicate upgrade combination") identities.add(identity) + if case["suite"] is None: + require(not supported, "Supported upgrades require a real acceptance suite for every backend profile") + continue suite = (root / case["suite"]).resolve() require( suite.is_relative_to((root / "tests/release_upgrades").resolve()), @@ -143,6 +166,11 @@ def support_text(policy: dict[str, Any], language: str) -> str: f"| {case['source_version']} | {policy['target_version']} | {case['backend']} | {case['configuration']} |" for case in policy["cases"] ) + if automatic := policy["automatic_sources"]: + lines.extend( + f"| stable >= {automatic['minimum_version']}, < target | {policy['target_version']} | {profile['backend']} | {profile['configuration']} |" + for profile in automatic["profiles"] + ) return "\n".join(lines) @@ -232,7 +260,11 @@ def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dic directory = Path(temporary) extras = "cli,server,seekdb" if case["backend"] == "seekdb" else "cli,server" interpreters = {} - packages = {"source": f"powercontext[{extras}]=={case['source_version']}", "target": f"{wheel}[{extras}]"} + artifact = case["source_artifact"] + packages = { + "source": f"powercontext[{extras}] @ {artifact['url']}#sha256={artifact['sha256']}", + "target": f"{wheel}[{extras}]", + } for role, package in packages.items(): venv = directory / role run( @@ -255,6 +287,7 @@ def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dic environment.update({ "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", "PC_UPGRADE_SOURCE_PYTHON": str(interpreters["source"]), + "PC_UPGRADE_SOURCE_VERSION": case["source_version"], "PC_UPGRADE_TARGET_PYTHON": str(interpreters["target"]), "PC_UPGRADE_BACKEND": case["backend"], "PC_UPGRADE_CONFIGURATION": case["configuration"], @@ -309,7 +342,9 @@ def gate(root: Path, wheel: Path, output: Path, *, release: bool = False, releas "status": "failed", } try: - for index, case in enumerate(policy["cases"]): + matrix = resolve_matrix(policy, actual_version) + evidence["resolved_matrix"] = matrix + for index, case in enumerate(matrix["required_cases"]): evidence["cases"].append(run_case(root, wheel, case, output / f"case-{index}")) evidence["status"] = "passed" finally: @@ -338,7 +373,7 @@ def main() -> int: ) else: require(not args.release, "--release requires the actual release wheel") - print(f"Upgrade policy: {policy['mode']}; {len(policy['cases'])} supported combinations") + print(f"Upgrade policy: {policy['mode']}; source/backend combinations are resolved against the target wheel") except (GateError, OSError, ValueError, subprocess.SubprocessError, ElementTree.ParseError) as error: print(f"Upgrade release gate failed: {error}", file=sys.stderr) return 1 diff --git a/scripts/upgrade_release_matrix.py b/scripts/upgrade_release_matrix.py new file mode 100644 index 0000000000..371535049c --- /dev/null +++ b/scripts/upgrade_release_matrix.py @@ -0,0 +1,105 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Resolve a release support window into immutable source wheels and backend cases.""" + +from __future__ import annotations + +import hashlib +import json +import re +import urllib.request +from typing import Any +from urllib.parse import urlsplit + +from packaging.version import Version + +CATALOG_URL = "https://pypi.org/pypi/powercontext/json" +STABLE = r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)" + + +def fetch_catalog() -> dict[str, Any]: + # A discovery outage must fail closed, never reduce the matrix to zero cases. + with urllib.request.urlopen(CATALOG_URL, timeout=30) as response: # noqa: S310 -- fixed public PyPI endpoint + return json.load(response) + + +def source_artifact(catalog: dict[str, Any], version: str) -> dict[str, str]: + files = catalog["releases"].get(version, []) + wheels = [ + item + for item in files + if item.get("filename") == f"powercontext-{version}-py3-none-any.whl" + and item.get("packagetype") == "bdist_wheel" + and not item.get("yanked", False) + ] + if len(wheels) != 1: + raise ValueError(f"Required source {version} has no unique non-yanked universal wheel") # noqa: TRY003 + wheel = wheels[0] + url = urlsplit(wheel["url"]) + digest = wheel["digests"]["sha256"] + if ( + url.scheme != "https" + or url.netloc != "files.pythonhosted.org" + or url.fragment + or not re.fullmatch(r"[0-9a-f]{64}", digest) + ): + raise ValueError(f"Invalid source artifact metadata for {version}") # noqa: TRY003 + return {"filename": wheel["filename"], "url": wheel["url"], "sha256": digest} + + +def resolve_matrix(policy: dict[str, Any], target: str) -> dict[str, Any]: + automatic = policy["automatic_sources"] + explicit = policy["cases"] + if automatic is None and not explicit: + return {"candidate_cases": [], "required_cases": []} + catalog = fetch_catalog() + if not isinstance(catalog.get("releases"), dict) or not catalog["releases"]: + raise ValueError("Empty or malformed PyPI release catalog") # noqa: TRY003 + cases = [dict(case) for case in explicit] + if automatic: + minimum = Version(automatic["minimum_version"]) + versions = sorted( + ( + version + for version in catalog["releases"] + if re.fullmatch(STABLE, version) and minimum <= Version(version) < Version(target) + ), + key=Version, + ) + # Requiring the lower bound also detects a truncated catalog instead of + # silently dropping an older version that remains in the support promise. + if automatic["minimum_version"] not in versions: + raise ValueError("Automatic source window has no published minimum version below the target") # noqa: TRY003 + cases.extend( + {"source_version": version, **profile} for version in versions for profile in automatic["profiles"] + ) + identities = set() + for case in cases: + version = case["source_version"] + if Version(version) >= Version(target): + raise ValueError(f"Source {version} must precede target {target}") # noqa: TRY003 + identity = (version, case["backend"], case["configuration"]) + if identity in identities: + raise ValueError(f"Duplicate resolved upgrade case: {identity}") # noqa: TRY003 + identities.add(identity) + case["source_artifact"] = source_artifact(catalog, version) + return { + "catalog_url": CATALOG_URL, + "catalog_sha256": hashlib.sha256(json.dumps(catalog, sort_keys=True).encode()).hexdigest(), + "candidate_cases": cases, + # Fresh-install-only records discovery, never reports unexecuted upgrades + # as accepted. Switching mode requires real suites for every profile. + "required_cases": cases if policy["mode"] == "supported-upgrades" else [], + } diff --git a/tests/release_upgrades/README.md b/tests/release_upgrades/README.md index 65ebec7430..16266ab711 100644 --- a/tests/release_upgrades/README.md +++ b/tests/release_upgrades/README.md @@ -1,13 +1,39 @@ # Installed-release upgrade acceptance `release/upgrade-support.json` is the explicit upgrade support policy. Currently it declares -`fresh-install-only`: **no production database upgrade combination is supported**. An empty matrix in this mode -does not prove a migration works. Full migration integration is tracked in #2154. +`fresh-install-only`: **no production database upgrade combination is supported**. Discovered candidates in this mode +are not accepted upgrades. Full migration integration is tracked in #2154. -To advertise an upgrade, add a reviewed case with `source_version` (an exact published package version), `backend` -(`sqlite`, `seekdb`, or `oceanbase`), a stable `configuration` identifier, and a `suite` path under this directory. -Set `mode` to `supported-upgrades`. Optional features that change persistent objects need explicit cases/configurations. -No ranges, moving refs, silent backend fallback, or implicit combinations are supported. +## Automatic source versions + +The default `automatic_sources` policy uses `all-stable-since`, with `minimum_version: "1.3.0"` and SQLite/seekDB +backend profiles. Every gate run reads the public PowerContext PyPI release catalog and selects all stable `X.Y.Z` +versions at or above the minimum and strictly below the **actual target wheel version**. It excludes prereleases, +the target itself, and newer versions. The next run automatically includes newly published stable source versions; +no new test file or per-version matrix entry is needed. The target is the local build, so it need not be on PyPI yet. + +Each selected source is expanded across every configured backend/profile. A missing or yanked required wheel, +discovery outage, missing minimum version, or empty source window fails the gate instead of reducing coverage. +Source wheels are installed by their discovered public URL and SHA-256. The evidence records the catalog digest, +exact selected artifacts, and expanded matrix. Unrelated package indexes cannot substitute a source wheel. + +For example, with a minimum of `1.4.0`, a target of `1.4.1` tests `1.4.0`; after `1.4.1` is published, target `1.5.0` +tests both `1.4.0` and `1.4.1` on every backend. Other stable patches in the same interval are included too. This is +a promise to test **all** versions in the configured window, not just the latest patch. Change the minimum only +through a reviewed change to the support policy. The resolver currently requires universal `py3-none-any` wheels; +a future distribution format change must extend it explicitly, not silently omit a release. + +Currently the profiles have `suite: null` and the mode remains `fresh-install-only`. Discovery records candidate +cases, but executes and accepts **zero** positive upgrades. To activate supported upgrades, implement the complete +suite contract below for every profile, replace every null with a real suite path, set `mode` to `supported-upgrades`, +and regenerate the installation notices. The gate rejects supported mode with a missing suite. Future source releases +then use those same suites, with version adapters added where historical public interfaces differ. Tests must fail +when migration support is missing; the runner never changes expected success into a rejection automatically. + +`cases` remains available for exact source versions/configurations outside the automatic profiles. Each case needs +`source_version`, `backend`, `configuration`, and `suite`. Overlapping resolved cases fail rather than running twice. +Optional features that create persistent objects require additional backend profiles. OceanBase can be included by +adding its real environment/suite; embedded seekDB evidence does not establish OceanBase compatibility. The runner creates separate source/target environments, installs the exact published source package and the built target wheel, verifies their versions, then executes the reviewed suite in an isolated working directory. Repository @@ -17,6 +43,7 @@ processes, not import the current checkout's runtime in place of either version. The suite receives: - `PC_UPGRADE_SOURCE_PYTHON`, `PC_UPGRADE_TARGET_PYTHON`: isolated installed interpreters. +- `PC_UPGRADE_SOURCE_VERSION`: the exact source release selected for this case. - `PC_UPGRADE_BACKEND`, `PC_UPGRADE_CONFIGURATION`: the declared case. - `PC_UPGRADE_WORK`: disposable workspace for both databases, controlled inference fixtures, and environment files. - `PC_UPGRADE_EVIDENCE`: destination for backend evidence JSON. @@ -50,18 +77,20 @@ Test names and metadata do not themselves prove correctness: reviewers must insp they follow the public operations and compare actual persisted data. This contract is intended to host the real upgrade acceptance added with #2154, without claiming that it already exists. -For example, after implementing and reviewing an actual suite, a case would have this shape: +For example, after implementing and reviewing actual suites, automatic selection would have this shape: ```json { - "source_version": "1.3.0", - "backend": "sqlite", - "configuration": "full-server-no-vector", - "suite": "tests/release_upgrades/test_sqlite_130.py" + "strategy": "all-stable-since", + "minimum_version": "1.3.0", + "profiles": [ + {"backend": "sqlite", "configuration": "full-server-no-vector", "suite": "tests/release_upgrades/test_sqlite.py"}, + {"backend": "seekdb", "configuration": "full-server-no-vector", "suite": "tests/release_upgrades/test_seekdb.py"} + ] } ``` -This example is not an existing supported case. Regenerate installation support regions with +These suite paths illustrate the contract; they do not exist yet. Regenerate installation support regions with `uv run python scripts/upgrade_release_gate.py --write-docs` and run `make upgrade-policy-check`. When persistence inputs change, calculate `managed_digest(ROOT)` from `upgrade_release_gate`, review the compatibility impact, and update both the digest and rationale in `schema_review`. The inputs include persistence, optional code storage, @@ -77,3 +106,20 @@ against the GitHub event's notes and gates both PyPI publication and release ass Repository administrators must require **Upgrade release gate** in branch protection. Workflow code cannot configure that setting. Because the release workflow starts on `release.published`, it can block package/asset publication, not the earlier manual creation of a GitHub Release; run the prepublication command before publishing that page. + +## Actual historical rejection regressions + +The PR/master job runs `ci_legacy_upgrade_rejection.py` for **both SQLite and embedded seekDB** against a database +created and populated by installed PowerContext 1.3.0. It uses separate source/target environments, one disposable +database directory, sequential process ownership, and public HTTP writes/readback before testing target CLI behavior. +The installed drivers inventory the actual tables and record each environment's engine, driver, and binding versions. +No external seekDB service or credentials are needed. A missing embedded binding fails the job; it is never skipped. + +The regressions require historical startup rejection, full-baseline migration rejection, and rejection after adding +only the prototype revision table to the disposable fixture. The last operation is deliberately **not a migration**. +SeekDB permits either the known CHECK-reflection rejection or unknown-baseline rejection, so fixing #2160 does not +require retaining the parser defect. These tests do not certify that an upgrade succeeds. + +CI invokes them with `--respect-policy` after the main gate. Once supported-path acceptance is activated, these +historical rejection assertions are recorded as `not-applicable` instead of failing because migration was fixed. +The preceding positive gate must still execute all required version/backend cases and accept no skips. diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py index 2909bdbcba..80bc9b4931 100644 --- a/tests/test_upgrade_release_gate.py +++ b/tests/test_upgrade_release_gate.py @@ -21,11 +21,12 @@ import pytest import upgrade_release_gate as gate +import upgrade_release_matrix as matrix import yaml @pytest.fixture -def release(tmp_path): +def release(tmp_path, monkeypatch): persistence = tmp_path / "src/powercontext/builtin/persistence/tables.py" persistence.parent.mkdir(parents=True) persistence.write_text("# disposable persistence input\n") @@ -34,13 +35,30 @@ def release(tmp_path): contract.write_text("info:\n version: 2.0.0\n") (tmp_path / "release").mkdir() policy = { - "format": 1, + "format": 2, "target_version": "2.0.0", "mode": "fresh-install-only", "schema_review": {"persistence_sha256": gate.managed_digest(tmp_path), "rationale": "No upgrades supported"}, "cases": [], + "automatic_sources": None, } (tmp_path / gate.MANIFEST).write_text(json.dumps(policy)) + monkeypatch.setattr( + matrix, + "fetch_catalog", + lambda: { + "releases": { + "1.3.0": [ + { + "filename": "powercontext-1.3.0-py3-none-any.whl", + "packagetype": "bdist_wheel", + "url": "https://files.pythonhosted.org/packages/test.whl", + "digests": {"sha256": "a" * 64}, + } + ] + } + }, + ) for language in ("en", "zh"): page = tmp_path / f"docs/{language}/docs/get-started/install-and-run.md" page.parent.mkdir(parents=True) @@ -196,6 +214,142 @@ def test_old_evidence_directory_cannot_be_reused(release): gate.gate(root, wheel, root / "evidence") +def catalog_for(*versions): + return { + "releases": { + version: [ + { + "filename": f"powercontext-{version}-py3-none-any.whl", + "packagetype": "bdist_wheel", + "yanked": False, + "url": f"https://files.pythonhosted.org/packages/powercontext-{version}-py3-none-any.whl", + "digests": {"sha256": "a" * 64}, + } + ] + for version in versions + } + } + + +def automatic_policy(policy, *, minimum="1.4.0"): + policy["automatic_sources"] = { + "strategy": "all-stable-since", + "minimum_version": minimum, + "profiles": [ + {"backend": backend, "configuration": "full-server-no-vector", "suite": None} + for backend in ("sqlite", "seekdb") + ], + } + + +@pytest.mark.parametrize( + "target,expected", + [ + ("1.4.1", ["1.4.0"]), + ("1.5.0", ["1.4.0", "1.4.1"]), + ("1.5.1", ["1.4.0", "1.4.1", "1.5.0"]), + ("1.5.1.dev2", ["1.4.0", "1.4.1", "1.5.0"]), + ], +) +def test_release_window_expands_for_each_target_without_new_test_scripts(release, monkeypatch, target, expected): + _, _, policy = release + automatic_policy(policy) + monkeypatch.setattr( + matrix, "fetch_catalog", lambda: catalog_for("1.3.0", "1.4.0", "1.4.1", "1.5.0rc1", "1.5.0", "1.5.1", "1.6.0") + ) + resolved = matrix.resolve_matrix(policy, target) + assert [(case["source_version"], case["backend"]) for case in resolved["candidate_cases"]] == [ + (version, backend) for version in expected for backend in ("sqlite", "seekdb") + ] + assert resolved["required_cases"] == [] # Discovery does not establish upgrade support. + assert all(case["source_artifact"]["sha256"] == "a" * 64 for case in resolved["candidate_cases"]) + + +def test_newly_published_source_is_automatically_added_on_next_run(release, monkeypatch): + _, _, policy = release + automatic_policy(policy) + catalog = catalog_for("1.4.0") + monkeypatch.setattr(matrix, "fetch_catalog", lambda: catalog) + first = matrix.resolve_matrix(policy, "1.5.0") + catalog["releases"].update(catalog_for("1.4.1")["releases"]) + second = matrix.resolve_matrix(policy, "1.5.0") + assert len(first["candidate_cases"]) == 2 + assert len(second["candidate_cases"]) == 4 + assert first["catalog_sha256"] != second["catalog_sha256"] + + +@pytest.mark.parametrize( + "broken", ["empty", "missing-minimum", "yanked", "missing-wheel", "bad-hash", "foreign-origin"] +) +def test_unavailable_source_cannot_silently_shrink_matrix(release, monkeypatch, broken): + _, _, policy = release + automatic_policy(policy) + catalog = catalog_for("1.4.0", "1.4.1") + wheel = catalog["releases"]["1.4.0"][0] + if broken == "empty": + catalog["releases"] = {} + elif broken == "missing-minimum": + del catalog["releases"]["1.4.0"] + elif broken == "yanked": + wheel["yanked"] = True + elif broken == "missing-wheel": + catalog["releases"]["1.4.1"] = [] + elif broken == "bad-hash": + wheel["digests"]["sha256"] = "invalid" + else: + wheel["url"] = "https://untrusted.invalid/package.whl" + monkeypatch.setattr(matrix, "fetch_catalog", lambda: catalog) + with pytest.raises(ValueError): + matrix.resolve_matrix(policy, "1.5.0") + + +def test_catalog_outage_records_a_failed_gate(release, monkeypatch): + root, wheel, policy = release + automatic_policy(policy) + (root / gate.MANIFEST).write_text(json.dumps(policy)) + + def unavailable(): + raise OSError("Catalog unavailable") # noqa: TRY003 -- exercise a distinct discovery outage + + monkeypatch.setattr(matrix, "fetch_catalog", unavailable) + with pytest.raises(OSError, match="Catalog unavailable"): + gate.gate(root, wheel, root / "evidence") + assert json.loads((root / "evidence/result.json").read_text())["status"] == "failed" + + +def test_switching_to_supported_upgrades_requires_all_backend_suites(release): + root, _, policy = release + automatic_policy(policy) + policy["mode"] = "supported-upgrades" + (root / gate.MANIFEST).write_text(json.dumps(policy)) + with pytest.raises(gate.GateError, match="every backend profile"): + gate.load_policy(root) + + +def test_supported_window_executes_every_discovered_version_on_every_backend(release, monkeypatch): + root, wheel, policy = release + supported_case(root, policy) + suite = policy["cases"][0]["suite"] + policy["cases"] = [] + automatic_policy(policy) + for profile in policy["automatic_sources"]["profiles"]: + profile["suite"] = suite + (root / gate.MANIFEST).write_text(json.dumps(policy)) + gate.check_docs(root, policy, write=True) + monkeypatch.setattr(matrix, "fetch_catalog", lambda: catalog_for("1.4.0", "1.4.1")) + executed = [] + + def execute(_root, _wheel, case, _output): + executed.append((case["source_version"], case["backend"])) + return {"case": case, "status": "passed"} + + monkeypatch.setattr(gate, "run_case", execute) + gate.gate(root, wheel, root / "evidence") + assert executed == [(version, backend) for version in ("1.4.0", "1.4.1") for backend in ("sqlite", "seekdb")] + result = json.loads((root / "evidence/result.json").read_text()) + assert len(result["cases"]) == len(result["resolved_matrix"]["required_cases"]) == 4 + + def test_publishing_jobs_depend_on_nonoptional_preupload_upgrade_gate(): root = Path(__file__).parents[1] workflow = yaml.safe_load((root / ".github/workflows/release.yml").read_text()) From 2be07394bac23f132c290ffb8bde5aeea1d98da1 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 22 Oct 2026 11:26:31 +0800 Subject: [PATCH 25/94] ci(release): pin upgrade baseline and add OceanBase coverage --- .../setup-upgrade-oceanbase/action.yml | 62 ++++++++++++ .github/workflows/master.yml | 20 +++- .github/workflows/release.yml | 25 +++++ docs/en/development/releasing.md | 15 +-- docs/zh/development/releasing.md | 16 +-- release/upgrade-support.json | 9 +- scripts/ci_legacy_upgrade_rejection.py | 41 ++++++-- scripts/upgrade_database_probe.py | 38 ++++++-- scripts/upgrade_release_gate.py | 10 +- scripts/upgrade_release_matrix.py | 19 +--- tests/release_upgrades/README.md | 81 ++++++++-------- tests/test_upgrade_release_gate.py | 97 ++++++++++++------- 12 files changed, 305 insertions(+), 128 deletions(-) create mode 100644 .github/actions/setup-upgrade-oceanbase/action.yml diff --git a/.github/actions/setup-upgrade-oceanbase/action.yml b/.github/actions/setup-upgrade-oceanbase/action.yml new file mode 100644 index 0000000000..949b649610 --- /dev/null +++ b/.github/actions/setup-upgrade-oceanbase/action.yml @@ -0,0 +1,62 @@ +name: Setup upgrade OceanBase +description: Start the disposable MySQL tenant used by installed-release upgrade checks +runs: + using: composite + steps: + - name: Start and verify OceanBase + shell: bash + env: + OCEANBASE_IMAGE: ghcr.io/oceanbase/oceanbase-ce:4.3.5.6-106000012026040916 + run: | + docker run --detach --name upgrade-oceanbase \ + --publish 127.0.0.1:2881:2881 \ + --ulimit nofile=65536:65536 \ + --env MODE=slim \ + --env OB_DATABASE=pc_upgrade_test \ + --env OB_TENANT_PASSWORD=powercontext-e2e \ + "$OCEANBASE_IMAGE" + for attempt in $(seq 1 120); do + if docker exec upgrade-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ + -Dpc_upgrade_test -e 'SELECT 1' >/dev/null 2>&1; then + # The slim image's default 1 GB maximum is too small for the + # indexes created across the clean migration acceptance cases. + docker exec upgrade-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@sys \ + -e "ALTER SYSTEM SET datafile_maxsize='8G'; ALTER SYSTEM SET datafile_next='512M'; ALTER SYSTEM SET datafile_size='4G';" + datafile_ready=false + for config_attempt in $(seq 1 30); do + docker exec upgrade-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@sys --batch --skip-column-names \ + -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next');" \ + > "$RUNNER_TEMP/upgrade-datafile.tsv" + if awk ' + $5 == "datafile_size" && $7 == "4G" { size = 1 } + $5 == "datafile_maxsize" && $7 == "8G" { maximum = 1 } + $5 == "datafile_next" && $7 == "512M" { increment = 1 } + END { exit !(size && maximum && increment) } + ' "$RUNNER_TEMP/upgrade-datafile.tsv"; then + datafile_ready=true + break + fi + sleep 2 + done + cat "$RUNNER_TEMP/upgrade-datafile.tsv" + if [ "$datafile_ready" != true ]; then + echo 'OceanBase datafile configuration did not become effective within one minute.' >&2 + exit 1 + fi + docker exec upgrade-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ + -Dpc_upgrade_test \ + -e "CREATE DATABASE pc_upgrade_fresh; SET GLOBAL recyclebin=OFF; SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode'; SHOW VARIABLES LIKE 'recyclebin';" + exit 0 + fi + if [ "$(docker inspect --format '{{.State.Running}}' upgrade-oceanbase)" != true ]; then + echo 'OceanBase exited before becoming ready.' >&2 + exit 1 + fi + sleep 5 + done + echo 'OceanBase did not become ready within 10 minutes.' >&2 + exit 1 diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index c6489517a7..d45eb86a9a 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -23,6 +23,9 @@ jobs: name: Upgrade release gate runs-on: ubuntu-24.04 timeout-minutes: 90 + env: + PC_UPGRADE_OCEANBASE_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_upgrade_test?charset=utf8mb4 + PC_UPGRADE_OCEANBASE_FRESH_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_upgrade_fresh?charset=utf8mb4 steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -37,18 +40,31 @@ jobs: uv run pytest -q tests/test_upgrade_release_gate.py - name: Build the target artifact run: uv build --wheel --out-dir dist + - name: Start the isolated upgrade OceanBase environment + uses: ./.github/actions/setup-upgrade-oceanbase + - name: Run declared installed-release acceptance run: >- uv run python scripts/run_upgrade_release_gate.py --output "$RUNNER_TEMP/upgrade-release-evidence" - - name: Reproduce unsupported SQLite and embedded seekDB upgrades + - name: Reproduce unsupported upgrades on all three backends run: | wheel=$(find dist -name '*.whl' -print -quit) - for backend in sqlite seekdb; do + for backend in sqlite seekdb oceanbase; do uv run python scripts/ci_legacy_upgrade_rejection.py \ --backend "$backend" --respect-policy --wheel "$wheel" \ --output "$RUNNER_TEMP/upgrade-release-evidence/legacy-$backend" done + - name: Preserve OceanBase diagnostics + if: always() + run: | + mkdir -p "$RUNNER_TEMP/upgrade-release-evidence" + docker logs upgrade-oceanbase > "$RUNNER_TEMP/upgrade-release-evidence/oceanbase.log" 2>&1 || true + docker inspect --format '{{json .State}}' upgrade-oceanbase \ + > "$RUNNER_TEMP/upgrade-release-evidence/oceanbase-state.json" 2>&1 || true + - name: Stop the isolated upgrade OceanBase environment + if: always() + run: docker rm --force upgrade-oceanbase - name: Preserve upgrade acceptance evidence if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d21ccd47ea..2dd1952db7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,9 @@ jobs: release-build: runs-on: ubuntu-24.04 timeout-minutes: 90 + env: + PC_UPGRADE_OCEANBASE_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_upgrade_test?charset=utf8mb4 + PC_UPGRADE_OCEANBASE_FRESH_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_upgrade_fresh?charset=utf8mb4 steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -74,12 +77,34 @@ jobs: uv build --out-dir dist uvx --from twine twine check dist/* + - name: Start the isolated upgrade OceanBase environment + uses: ./.github/actions/setup-upgrade-oceanbase + - name: Gate upgrade claims on installed release acceptance run: >- uv run python scripts/run_upgrade_release_gate.py --release --release-event "$GITHUB_EVENT_PATH" --output "$RUNNER_TEMP/upgrade-release-evidence" + - name: Reproduce unsupported upgrades on all three backends + run: | + wheel=$(find dist -name '*.whl' -print -quit) + for backend in sqlite seekdb oceanbase; do + uv run python scripts/ci_legacy_upgrade_rejection.py \ + --backend "$backend" --respect-policy --wheel "$wheel" \ + --output "$RUNNER_TEMP/upgrade-release-evidence/legacy-$backend" + done + - name: Preserve OceanBase diagnostics + if: always() + run: | + mkdir -p "$RUNNER_TEMP/upgrade-release-evidence" + docker logs upgrade-oceanbase > "$RUNNER_TEMP/upgrade-release-evidence/oceanbase.log" 2>&1 || true + docker inspect --format '{{json .State}}' upgrade-oceanbase \ + > "$RUNNER_TEMP/upgrade-release-evidence/oceanbase-state.json" 2>&1 || true + - name: Stop the isolated upgrade OceanBase environment + if: always() + run: docker rm --force upgrade-oceanbase + - name: Preserve upgrade acceptance evidence if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 diff --git a/docs/en/development/releasing.md b/docs/en/development/releasing.md index 878945cf10..4dff970068 100644 --- a/docs/en/development/releasing.md +++ b/docs/en/development/releasing.md @@ -130,9 +130,11 @@ installation pins are synchronized. ### Database upgrade support `release/upgrade-support.json` declares the target version, source-version selection policy, backends, and -configurations. `automatic_sources` selects every stable PyPI release from `minimum_version` up to, but excluding, -the actual target wheel. Newly published stable versions become source cases on subsequent runs without adding -per-version tests. The runner expands every source across all backend profiles and records source wheel SHA-256s. +configurations. `automatic_sources` pins `source_version: "1.3.0"` with the `fixed-version` strategy and expands it +across SQLite, embedded seekDB, and OceanBase. The target is the actual wheel built by CI. Future releases such as +1.4.1 and 1.5.0 continue testing from 1.3.0 automatically; intermediate releases are not added as sources. Source +wheel URLs and SHA-256s are resolved from public PyPI and recorded with the target artifact. This fixed baseline +alone does not certify databases created/populated by other releases; use explicit cases for additional claims. The current policy is `fresh-install-only`; existing Server database upgrades are unsupported. Its null suite paths record candidates only, not accepted upgrades. Supported mode requires implemented suites for every backend profile. Do not advertise a supported upgrade until its installed-release acceptance suite is implemented and passes. @@ -157,9 +159,10 @@ in separate environments. Missing tests, skipped backends, failures, and missing version; the `--release` check must run against the final tagged wheel without overriding its version. The **Upgrade release gate** CI job also reproduces the known unsupported upgrade rejections using real 1.3.0 -SQLite and embedded seekDB Server databases. It installs and starts seekDB inside the runner with a disposable -directory; no external service is needed. Once supported mode is activated, these negative assertions retire and -the full discovered upgrade matrix must pass. Repository administrators must make this job required in branch protection. +SQLite, embedded seekDB, and OceanBase Server databases. seekDB runs in a temporary runner directory. Both PR +and publication jobs start a disposable OceanBase CE container, wait for its MySQL tenant, archive diagnostics, and +remove it even on failure. No external database credentials are needed. Once supported mode is activated, these negative assertions retire and +the fixed-source three-backend upgrade matrix must pass. Repository administrators must make this job required in branch protection. The release workflow gates distribution upload, PyPI publication, and release assets on the acceptance result and archives its evidence. It runs after `release.published`, so it cannot prevent the earlier manual publication of the GitHub Release page: run the command above before that action. `make publish` requires diff --git a/docs/zh/development/releasing.md b/docs/zh/development/releasing.md index 760ea31da7..b538eabe88 100644 --- a/docs/zh/development/releasing.md +++ b/docs/zh/development/releasing.md @@ -122,9 +122,11 @@ DSH README 中说明某项能力已包含在 `1.1.0` 的文字同样属于历史 ### 数据库升级支持范围 -`release/upgrade-support.json` 声明目标版本、来源版本选择策略、后端和配置。`automatic_sources` 从 PyPI 自动选择 -大于等于 `minimum_version`、小于实际目标 wheel 的全部稳定版本，再与每个后端配置展开组合。新稳定版发布后， -后续运行会自动将其加入来源，不必逐版本新增测试。证据会记录具体来源 wheel 的 SHA-256。 +`release/upgrade-support.json` 声明目标版本、来源版本选择策略、后端和配置。`automatic_sources` 使用 +`fixed-version` 策略，将 `source_version` 固定为 `1.3.0`，展开 SQLite、嵌入式 seekDB 和 OceanBase 三种后端。 +目标始终是本次 CI 构建的实际 wheel；后续发布 1.4.1、1.5.0 时，自动测试 1.3.0 到本次版本，不增加中间版本来源。 +来源 wheel 的公开 PyPI 地址和 SHA-256 与目标产物一起记录。固定起点不能证明其他版本实际生成的数据同样兼容； +如需额外声明这些路径，使用显式 `cases` 补充验收。 当前策略为 `fresh-install-only`，不支持已有 Server 数据库升级；配置中的空套件路径只生成候选组合，不算验收通过。 启用支持升级模式时，每个后端配置都必须有真实套件。只有实现并通过已安装发行包的完整升级验收，才能声明支持 对应组合。验收契约和证据要求见仓库中的 `tests/release_upgrades/README.md`。 @@ -144,9 +146,11 @@ uv run python scripts/run_upgrade_release_gate.py --release --release-notes /pat 旧版发行包和目标 wheel。测试缺失、后端跳过、执行失败或证据缺失都会阻断检查。`fresh-install-only` 策略检查 通过不表示存在升级路径。普通 PR 构建允许使用开发版本；`--release` 必须验证最终 tag 构建的 wheel，不能覆盖版本号。 -CI 中的 **Upgrade release gate** 还会使用真实 1.3.0 SQLite 和嵌入式 seekDB Server 数据库复现已知的升级拒绝行为。 -seekDB 直接在 runner 的临时目录中启动，不需要外部服务。启用支持升级模式后，固定的拒绝断言退出，自动展开的 -全部成功升级组合必须通过验收。仓库管理员需要在分支保护中将该检查设为必需。 +CI 中的 **Upgrade release gate** 使用真实 1.3.0 SQLite、嵌入式 seekDB 和 OceanBase Server 数据库复现升级拒绝行为。 +seekDB 直接在 runner 临时目录启动；PR 和发布工作流均启动隔离的 OceanBase CE 容器，等待 MySQL 租户就绪， +归档诊断并在测试失败时也清理容器，不依赖外部数据库凭据。任一后端无法启动都使检查失败，不能跳过。 +启用支持升级模式后，固定的拒绝断言退出，1.3.0 到目标版本的三种后端完整升级验收必须通过。 +仓库管理员需要在分支保护中将该检查设为必需。 发布工作流只有验收通过才会上传发行包、发布 PyPI 和上传 Release 附件，并归档验收证据。工作流在 `release.published` 后运行，无法阻止之前手动发布 GitHub Release 页面，因此必须先运行上述命令再发布页面。 `make publish` 要求提供 `UPGRADE_RELEASE_NOTES=/path/to/release-notes.md`，并在上传软件包之前执行相同门禁。 diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 4e61ff6624..77d31c67eb 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -8,8 +8,8 @@ }, "cases": [], "automatic_sources": { - "strategy": "all-stable-since", - "minimum_version": "1.3.0", + "strategy": "fixed-version", + "source_version": "1.3.0", "profiles": [ { "backend": "sqlite", @@ -20,6 +20,11 @@ "backend": "seekdb", "configuration": "full-server-no-vector", "suite": null + }, + { + "backend": "oceanbase", + "configuration": "full-server-no-vector", + "suite": null } ] } diff --git a/scripts/ci_legacy_upgrade_rejection.py b/scripts/ci_legacy_upgrade_rejection.py index 7bedf3a5f2..ef9f9a46d1 100644 --- a/scripts/ci_legacy_upgrade_rejection.py +++ b/scripts/ci_legacy_upgrade_rejection.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Reproduce unsupported 1.3.0 upgrades on an actual SQLite or embedded seekDB Server. +"""Reproduce unsupported 1.3.0 upgrades on actual SQLite, embedded seekDB or OceanBase Servers. This is a negative regression, never positive upgrade acceptance. CI retires these assertions once the supported-path matrix becomes mandatory. @@ -26,6 +26,7 @@ import subprocess import tempfile from pathlib import Path +from urllib.parse import urlsplit import httpx from ci_release_smoke import _available_port, _stop_process, _wait_until_ready @@ -108,7 +109,7 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--wheel", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--backend", choices=("sqlite", "seekdb"), default="sqlite") + parser.add_argument("--backend", choices=("sqlite", "seekdb", "oceanbase"), default="sqlite") parser.add_argument( "--respect-policy", action="store_true", @@ -133,6 +134,14 @@ def main() -> None: ) print("Historical rejection assertions retired; the supported upgrade matrix must pass instead.") return + if args.backend == "oceanbase": + url = urlsplit(os.environ.get("PC_UPGRADE_OCEANBASE_URL", "")) + require( + url.scheme == "mysql+aoceanbase" + and url.hostname in {"localhost", "127.0.0.1"} + and url.path == "/pc_upgrade_test", + "OceanBase regression requires the dedicated local pc_upgrade_test database", + ) environment = { key: value for key, value in os.environ.items() @@ -164,17 +173,20 @@ def main() -> None: require(metadata["source"]["powercontext"] == "1.3.0", "Wrong source package") require(metadata["target"]["powercontext"] == wheel_version(wheel), "Wrong target package") database = directory / "database" - database_environment = ( - {"POWERCONTEXT_SERVER_DATABASE_KIND": "seekdb", "POWERCONTEXT_SERVER_DATABASE_PATH": str(database)} - if args.backend == "seekdb" - else {"POWERCONTEXT_SERVER_DATABASE_URL": f"sqlite+aiosqlite:///{database}"} - ) + if args.backend == "seekdb": + database_environment = { + "POWERCONTEXT_SERVER_DATABASE_KIND": "seekdb", + "POWERCONTEXT_SERVER_DATABASE_PATH": str(database), + } + elif args.backend == "oceanbase": + database_environment = {"POWERCONTEXT_SERVER_DATABASE_URL": environment["PC_UPGRADE_OCEANBASE_URL"]} + else: + database_environment = {"POWERCONTEXT_SERVER_DATABASE_URL": f"sqlite+aiosqlite:///{database}"} environment.update({ "POWERCONTEXT_HOME": str(directory / "home"), **database_environment, "POWERCONTEXT_SERVER_ACCESS_MODE": "disabled", }) - seed = source_database(interpreters["source"], directory, environment, output / "source.log") probe = [ "-I", str(ROOT / "scripts/upgrade_database_probe.py"), @@ -183,6 +195,17 @@ def main() -> None: "--database", str(database), ] + if args.backend == "oceanbase": + run( + [str(interpreters["source"]), *probe, "--output", str(output / "empty-schema.json")], + environment=environment, + cwd=directory, + log=output / "empty-probe.log", + ) + empty_schema = json.loads((output / "empty-schema.json").read_text()) + require(not empty_schema["tables"], "OceanBase test database must be empty; refusing to overwrite data") + require(empty_schema["tenant_mode"] == "mysql", "OceanBase MySQL tenant required") + seed = source_database(interpreters["source"], directory, environment, output / "source.log") for role, python in interpreters.items(): run( [str(python), *probe, "--output", str(output / f"{role}-schema.json")], @@ -205,7 +228,7 @@ def main() -> None: environment, directory, output / "status.log", - ("unknown_baseline", "incompatible_schema") if args.backend == "seekdb" else ("unknown_baseline",), + ("unknown_baseline", "incompatible_schema") if args.backend != "sqlite" else ("unknown_baseline",), ) rejected( [target, "server", "run", "--no-env-file", "--host", "127.0.0.1", "--port", str(_available_port())], diff --git a/scripts/upgrade_database_probe.py b/scripts/upgrade_database_probe.py index 8b8822979d..0188b77823 100644 --- a/scripts/upgrade_database_probe.py +++ b/scripts/upgrade_database_probe.py @@ -23,11 +23,13 @@ import argparse import asyncio import json +import os import sqlite3 from importlib import import_module from importlib.metadata import version from pathlib import Path from typing import Any +from urllib.parse import unquote, urlsplit REVISION_DDL = "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) PRIMARY KEY)" @@ -42,18 +44,35 @@ async def inspect(backend: str, database: Path, *, add_revision: bool) -> dict[s # Use the driver's public API, not a current-checkout Profile that could run # startup guards or mutate an old schema before the target CLI is tested. - binding = import_module("pylibseekdb") pymysql = import_module("pymysql") - instance = await binding.aopen(str(database)) + instance = None + details = {} + if backend == "seekdb": + binding = import_module("pylibseekdb") + instance = await binding.aopen(str(database)) + options = {**instance.connection_options(), "database": "test"} + details["embedded_binding_version"] = version("pylibseekdb") + else: + url = urlsplit(os.environ["PC_UPGRADE_OCEANBASE_URL"]) + options = { + "host": url.hostname, + "port": url.port or 2881, + "user": unquote(url.username or ""), + "password": unquote(url.password or ""), + "database": url.path.removeprefix("/"), + "charset": "utf8mb4", + "connect_timeout": 15, + } try: - with ( - pymysql.connect(**instance.connection_options(), database="test", autocommit=True) as connection, - connection.cursor() as cursor, - ): + with pymysql.connect(**options, autocommit=True) as connection, connection.cursor() as cursor: if add_revision: cursor.execute(REVISION_DDL) cursor.execute("SELECT VERSION()") engine = str(cursor.fetchone()[0]) + if backend == "oceanbase": + cursor.execute("SHOW VARIABLES LIKE 'ob_compatibility_mode'") + details["tenant_mode"] = str(cursor.fetchone()[1]).lower() + details["proxy_version"] = "direct" cursor.execute("SHOW FULL TABLES WHERE Table_type = 'BASE TABLE'") names = sorted(row[0] for row in cursor.fetchall()) tables = {} @@ -63,16 +82,17 @@ async def inspect(backend: str, database: Path, *, add_revision: bool) -> dict[s return { "engine_version": engine, "driver_version": version("pymysql"), - "embedded_binding_version": version("pylibseekdb"), + **details, "tables": tables, } finally: - instance.close() + if instance is not None: + instance.close() def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--backend", choices=("sqlite", "seekdb"), required=True) + parser.add_argument("--backend", choices=("sqlite", "seekdb", "oceanbase"), required=True) parser.add_argument("--database", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--add-revision", action="store_true") diff --git a/scripts/upgrade_release_gate.py b/scripts/upgrade_release_gate.py index ba0178a88d..cf0a3cbc4d 100644 --- a/scripts/upgrade_release_gate.py +++ b/scripts/upgrade_release_gate.py @@ -114,11 +114,11 @@ def load_policy(root: Path) -> dict[str, Any]: automatic = policy["automatic_sources"] profiles = [] if automatic is not None: - require(set(automatic) == {"strategy", "minimum_version", "profiles"}, "Invalid automatic source policy") - require(automatic["strategy"] == "all-stable-since", "Unknown source selection strategy") + require(set(automatic) == {"strategy", "source_version", "profiles"}, "Invalid automatic source policy") + require(automatic["strategy"] == "fixed-version", "Unknown source selection strategy") require( - isinstance(automatic["minimum_version"], str) and re.fullmatch(STABLE, automatic["minimum_version"]), - "Minimum source must be a stable version", + isinstance(automatic["source_version"], str) and re.fullmatch(STABLE, automatic["source_version"]), + "Fixed source must be a stable version", ) profiles = automatic["profiles"] require(isinstance(profiles, list) and profiles, "Automatic sources require backend profiles") @@ -168,7 +168,7 @@ def support_text(policy: dict[str, Any], language: str) -> str: ) if automatic := policy["automatic_sources"]: lines.extend( - f"| stable >= {automatic['minimum_version']}, < target | {policy['target_version']} | {profile['backend']} | {profile['configuration']} |" + f"| {automatic['source_version']} | {policy['target_version']} | {profile['backend']} | {profile['configuration']} |" for profile in automatic["profiles"] ) return "\n".join(lines) diff --git a/scripts/upgrade_release_matrix.py b/scripts/upgrade_release_matrix.py index 371535049c..dbb56b33ce 100644 --- a/scripts/upgrade_release_matrix.py +++ b/scripts/upgrade_release_matrix.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Resolve a release support window into immutable source wheels and backend cases.""" +"""Resolve a fixed source release into immutable source wheels and backend cases.""" from __future__ import annotations @@ -69,22 +69,7 @@ def resolve_matrix(policy: dict[str, Any], target: str) -> dict[str, Any]: raise ValueError("Empty or malformed PyPI release catalog") # noqa: TRY003 cases = [dict(case) for case in explicit] if automatic: - minimum = Version(automatic["minimum_version"]) - versions = sorted( - ( - version - for version in catalog["releases"] - if re.fullmatch(STABLE, version) and minimum <= Version(version) < Version(target) - ), - key=Version, - ) - # Requiring the lower bound also detects a truncated catalog instead of - # silently dropping an older version that remains in the support promise. - if automatic["minimum_version"] not in versions: - raise ValueError("Automatic source window has no published minimum version below the target") # noqa: TRY003 - cases.extend( - {"source_version": version, **profile} for version in versions for profile in automatic["profiles"] - ) + cases.extend({"source_version": automatic["source_version"], **profile} for profile in automatic["profiles"]) identities = set() for case in cases: version = case["source_version"] diff --git a/tests/release_upgrades/README.md b/tests/release_upgrades/README.md index 16266ab711..80c785b266 100644 --- a/tests/release_upgrades/README.md +++ b/tests/release_upgrades/README.md @@ -4,36 +4,34 @@ `fresh-install-only`: **no production database upgrade combination is supported**. Discovered candidates in this mode are not accepted upgrades. Full migration integration is tracked in #2154. -## Automatic source versions - -The default `automatic_sources` policy uses `all-stable-since`, with `minimum_version: "1.3.0"` and SQLite/seekDB -backend profiles. Every gate run reads the public PowerContext PyPI release catalog and selects all stable `X.Y.Z` -versions at or above the minimum and strictly below the **actual target wheel version**. It excludes prereleases, -the target itself, and newer versions. The next run automatically includes newly published stable source versions; -no new test file or per-version matrix entry is needed. The target is the local build, so it need not be on PyPI yet. - -Each selected source is expanded across every configured backend/profile. A missing or yanked required wheel, -discovery outage, missing minimum version, or empty source window fails the gate instead of reducing coverage. -Source wheels are installed by their discovered public URL and SHA-256. The evidence records the catalog digest, -exact selected artifacts, and expanded matrix. Unrelated package indexes cannot substitute a source wheel. - -For example, with a minimum of `1.4.0`, a target of `1.4.1` tests `1.4.0`; after `1.4.1` is published, target `1.5.0` -tests both `1.4.0` and `1.4.1` on every backend. Other stable patches in the same interval are included too. This is -a promise to test **all** versions in the configured window, not just the latest patch. Change the minimum only -through a reviewed change to the support policy. The resolver currently requires universal `py3-none-any` wheels; -a future distribution format change must extend it explicitly, not silently omit a release. - -Currently the profiles have `suite: null` and the mode remains `fresh-install-only`. Discovery records candidate -cases, but executes and accepts **zero** positive upgrades. To activate supported upgrades, implement the complete -suite contract below for every profile, replace every null with a real suite path, set `mode` to `supported-upgrades`, -and regenerate the installation notices. The gate rejects supported mode with a missing suite. Future source releases -then use those same suites, with version adapters added where historical public interfaces differ. Tests must fail -when migration support is missing; the runner never changes expected success into a rejection automatically. - -`cases` remains available for exact source versions/configurations outside the automatic profiles. Each case needs -`source_version`, `backend`, `configuration`, and `suite`. Overlapping resolved cases fail rather than running twice. -Optional features that create persistent objects require additional backend profiles. OceanBase can be included by -adding its real environment/suite; embedded seekDB evidence does not establish OceanBase compatibility. +## Fixed source and current target + +The `automatic_sources` policy uses `strategy: "fixed-version"` and `source_version: "1.3.0"`, with SQLite, +embedded seekDB, and OceanBase profiles. Every run expands that exact source across the three backends. The target +is the **actual wheel built by this job**, not the latest published PyPI package. Thus releases 1.4.1 and 1.5.0 +exercise 1.3.0 → 1.4.1 and 1.3.0 → 1.5.0 without adding test scripts or changing the source version. Newly published +intermediate versions do not become additional sources. This fixed baseline does not establish support for every +intermediate release's database or data shapes; add explicit `cases` if those need separate acceptance. + +The runner resolves the exact source wheel from public PyPI and installs its URL with a SHA-256. Missing/yanked +source wheels, invalid artifact metadata, catalog outages, and a target at or below the source version fail the gate. +Evidence records the catalog digest, exact source artifact, target artifact digest, and expanded matrix. The resolver +requires universal `py3-none-any` wheels; a distribution format change must extend it explicitly. + +Currently the three profiles have `suite: null` and the mode remains `fresh-install-only`. Resolving candidates +executes and accepts **zero** positive upgrades. To activate supported upgrades, implement the complete suite +contract below for each backend, replace the null paths, set `mode` to `supported-upgrades`, and regenerate the +installation notices. A missing suite fails supported mode. Tests must fail when migration support is missing; +the runner never changes expected success into a rejection automatically. + +`cases` remains available for additional exact source versions/configurations. Each case needs `source_version`, +`backend`, `configuration`, and `suite`; overlapping cases fail. Optional features that create persistent objects +require additional profiles. The fixed CI baseline and advertised upgrade support must stay consistent. + +PR/master and publication CI start an isolated OceanBase CE container using the image and MySQL tenant setup from +the existing E2E workflow. They wait for SQL readiness and adequate datafile capacity before testing, save diagnostics, +and remove the container even when a check fails. No external database credentials are needed. Embedded seekDB +runs directly in a temporary runner directory. A backend startup failure fails the job instead of skipping it. The runner creates separate source/target environments, installs the exact published source package and the built target wheel, verifies their versions, then executes the reviewed suite in an isolated working directory. Repository @@ -47,6 +45,9 @@ The suite receives: - `PC_UPGRADE_BACKEND`, `PC_UPGRADE_CONFIGURATION`: the declared case. - `PC_UPGRADE_WORK`: disposable workspace for both databases, controlled inference fixtures, and environment files. - `PC_UPGRADE_EVIDENCE`: destination for backend evidence JSON. +- `PC_UPGRADE_OCEANBASE_URL`, `PC_UPGRADE_OCEANBASE_FRESH_URL`: dedicated disposable MySQL-tenant databases for + the upgraded and fresh-install comparisons. CI supplies both; local positive OceanBase suites must supply their + own isolated databases. Each suite must isolate/reset its own fixtures when multiple configurations run. Provide these behavioral tests (additional tests are allowed): @@ -81,11 +82,12 @@ For example, after implementing and reviewing actual suites, automatic selection ```json { - "strategy": "all-stable-since", - "minimum_version": "1.3.0", + "strategy": "fixed-version", + "source_version": "1.3.0", "profiles": [ {"backend": "sqlite", "configuration": "full-server-no-vector", "suite": "tests/release_upgrades/test_sqlite.py"}, - {"backend": "seekdb", "configuration": "full-server-no-vector", "suite": "tests/release_upgrades/test_seekdb.py"} + {"backend": "seekdb", "configuration": "full-server-no-vector", "suite": "tests/release_upgrades/test_seekdb.py"}, + {"backend": "oceanbase", "configuration": "full-server-no-vector", "suite": "tests/release_upgrades/test_oceanbase.py"} ] } ``` @@ -109,15 +111,18 @@ not the earlier manual creation of a GitHub Release; run the prepublication comm ## Actual historical rejection regressions -The PR/master job runs `ci_legacy_upgrade_rejection.py` for **both SQLite and embedded seekDB** against a database -created and populated by installed PowerContext 1.3.0. It uses separate source/target environments, one disposable -database directory, sequential process ownership, and public HTTP writes/readback before testing target CLI behavior. +The PR/master and publication jobs run `ci_legacy_upgrade_rejection.py` for **SQLite, embedded seekDB, and OceanBase** +against a database created and populated by installed PowerContext 1.3.0. It uses separate source/target environments, one disposable +database per backend, sequential process ownership, and public HTTP writes/readback before testing target CLI behavior. The installed drivers inventory the actual tables and record each environment's engine, driver, and binding versions. -No external seekDB service or credentials are needed. A missing embedded binding fails the job; it is never skipped. +OceanBase uses the container's dedicated `pc_upgrade_test` database and records its MySQL tenant mode and direct +connection. The negative runner only accepts the local test database and verifies it is empty before starting the +old Server; it refuses to overwrite existing data. No external service or credentials are needed. A missing embedded +binding or unavailable OceanBase instance fails the job; neither backend is skipped. The regressions require historical startup rejection, full-baseline migration rejection, and rejection after adding only the prototype revision table to the disposable fixture. The last operation is deliberately **not a migration**. -SeekDB permits either the known CHECK-reflection rejection or unknown-baseline rejection, so fixing #2160 does not +SeekDB and OceanBase permit either the known CHECK-reflection rejection or unknown-baseline rejection, so fixing #2160 does not require retaining the parser defect. These tests do not certify that an upgrade succeeds. CI invokes them with `--respect-policy` after the main gate. Once supported-path acceptance is activated, these diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py index 80bc9b4931..e012b0a75d 100644 --- a/tests/test_upgrade_release_gate.py +++ b/tests/test_upgrade_release_gate.py @@ -231,69 +231,66 @@ def catalog_for(*versions): } -def automatic_policy(policy, *, minimum="1.4.0"): +def automatic_policy(policy): policy["automatic_sources"] = { - "strategy": "all-stable-since", - "minimum_version": minimum, + "strategy": "fixed-version", + "source_version": "1.3.0", "profiles": [ {"backend": backend, "configuration": "full-server-no-vector", "suite": None} - for backend in ("sqlite", "seekdb") + for backend in ("sqlite", "seekdb", "oceanbase") ], } -@pytest.mark.parametrize( - "target,expected", - [ - ("1.4.1", ["1.4.0"]), - ("1.5.0", ["1.4.0", "1.4.1"]), - ("1.5.1", ["1.4.0", "1.4.1", "1.5.0"]), - ("1.5.1.dev2", ["1.4.0", "1.4.1", "1.5.0"]), - ], -) -def test_release_window_expands_for_each_target_without_new_test_scripts(release, monkeypatch, target, expected): +@pytest.mark.parametrize("target", ["1.4.0", "1.4.1", "1.5.0", "1.5.1.dev2"]) +def test_fixed_source_tracks_each_target_on_all_three_backends(release, monkeypatch, target): _, _, policy = release automatic_policy(policy) monkeypatch.setattr( - matrix, "fetch_catalog", lambda: catalog_for("1.3.0", "1.4.0", "1.4.1", "1.5.0rc1", "1.5.0", "1.5.1", "1.6.0") + matrix, "fetch_catalog", lambda: catalog_for("1.3.0", "1.4.0", "1.4.1", "1.5.0rc1", "1.5.0", "1.6.0") ) resolved = matrix.resolve_matrix(policy, target) assert [(case["source_version"], case["backend"]) for case in resolved["candidate_cases"]] == [ - (version, backend) for version in expected for backend in ("sqlite", "seekdb") + ("1.3.0", backend) for backend in ("sqlite", "seekdb", "oceanbase") ] - assert resolved["required_cases"] == [] # Discovery does not establish upgrade support. + assert resolved["required_cases"] == [] # Candidate selection does not establish upgrade support. assert all(case["source_artifact"]["sha256"] == "a" * 64 for case in resolved["candidate_cases"]) -def test_newly_published_source_is_automatically_added_on_next_run(release, monkeypatch): +def test_newly_published_versions_do_not_change_the_fixed_source(release, monkeypatch): _, _, policy = release automatic_policy(policy) - catalog = catalog_for("1.4.0") + catalog = catalog_for("1.3.0") monkeypatch.setattr(matrix, "fetch_catalog", lambda: catalog) first = matrix.resolve_matrix(policy, "1.5.0") catalog["releases"].update(catalog_for("1.4.1")["releases"]) second = matrix.resolve_matrix(policy, "1.5.0") - assert len(first["candidate_cases"]) == 2 - assert len(second["candidate_cases"]) == 4 - assert first["catalog_sha256"] != second["catalog_sha256"] + assert first["candidate_cases"] == second["candidate_cases"] + assert len(second["candidate_cases"]) == 3 -@pytest.mark.parametrize( - "broken", ["empty", "missing-minimum", "yanked", "missing-wheel", "bad-hash", "foreign-origin"] -) +@pytest.mark.parametrize("target", ["1.2.0", "1.3.0", "1.3.0rc1"]) +def test_fixed_source_must_precede_target(release, target): + _, _, policy = release + automatic_policy(policy) + with pytest.raises(ValueError, match="must precede target"): + matrix.resolve_matrix(policy, target) + + +@pytest.mark.parametrize("broken", ["empty", "missing-source", "yanked", "missing-wheel", "bad-hash", "foreign-origin"]) def test_unavailable_source_cannot_silently_shrink_matrix(release, monkeypatch, broken): _, _, policy = release automatic_policy(policy) - catalog = catalog_for("1.4.0", "1.4.1") - wheel = catalog["releases"]["1.4.0"][0] + catalog = catalog_for("1.3.0", "1.4.1") + wheel = catalog["releases"]["1.3.0"][0] if broken == "empty": catalog["releases"] = {} - elif broken == "missing-minimum": - del catalog["releases"]["1.4.0"] + elif broken == "missing-source": + del catalog["releases"]["1.3.0"] elif broken == "yanked": wheel["yanked"] = True elif broken == "missing-wheel": - catalog["releases"]["1.4.1"] = [] + catalog["releases"]["1.3.0"] = [] elif broken == "bad-hash": wheel["digests"]["sha256"] = "invalid" else: @@ -326,7 +323,7 @@ def test_switching_to_supported_upgrades_requires_all_backend_suites(release): gate.load_policy(root) -def test_supported_window_executes_every_discovered_version_on_every_backend(release, monkeypatch): +def test_supported_fixed_source_executes_every_backend(release, monkeypatch): root, wheel, policy = release supported_case(root, policy) suite = policy["cases"][0]["suite"] @@ -336,7 +333,7 @@ def test_supported_window_executes_every_discovered_version_on_every_backend(rel profile["suite"] = suite (root / gate.MANIFEST).write_text(json.dumps(policy)) gate.check_docs(root, policy, write=True) - monkeypatch.setattr(matrix, "fetch_catalog", lambda: catalog_for("1.4.0", "1.4.1")) + monkeypatch.setattr(matrix, "fetch_catalog", lambda: catalog_for("1.3.0", "1.4.1")) executed = [] def execute(_root, _wheel, case, _output): @@ -345,9 +342,9 @@ def execute(_root, _wheel, case, _output): monkeypatch.setattr(gate, "run_case", execute) gate.gate(root, wheel, root / "evidence") - assert executed == [(version, backend) for version in ("1.4.0", "1.4.1") for backend in ("sqlite", "seekdb")] + assert executed == [(version, backend) for version in ("1.3.0",) for backend in ("sqlite", "seekdb", "oceanbase")] result = json.loads((root / "evidence/result.json").read_text()) - assert len(result["cases"]) == len(result["resolved_matrix"]["required_cases"]) == 4 + assert len(result["cases"]) == len(result["resolved_matrix"]["required_cases"]) == 3 def test_publishing_jobs_depend_on_nonoptional_preupload_upgrade_gate(): @@ -362,3 +359,35 @@ def test_publishing_jobs_depend_on_nonoptional_preupload_upgrade_gate(): assert index < upload for name in ("pypi-publish", "release-assets"): assert workflow["jobs"][name]["needs"] == "release-build" + + +def test_repository_baseline_covers_all_three_databases(): + policy = gate.load_policy(Path(__file__).parents[1]) + selection = policy["automatic_sources"] + assert selection["strategy"] == "fixed-version" + assert selection["source_version"] == "1.3.0" + assert {profile["backend"] for profile in selection["profiles"]} == {"sqlite", "seekdb", "oceanbase"} + + +@pytest.mark.parametrize("workflow_name,job_name", [("master", "upgrade-release-gate"), ("release", "release-build")]) +def test_upgrade_jobs_require_real_backends_and_always_clean_up(workflow_name, job_name): + root = Path(__file__).parents[1] + workflow = yaml.safe_load((root / f".github/workflows/{workflow_name}.yml").read_text()) + job = workflow["jobs"][job_name] + steps = job["steps"] + setup_index = next( + i for i, step in enumerate(steps) if step.get("uses") == "./.github/actions/setup-upgrade-oceanbase" + ) + gate_index = next(i for i, step in enumerate(steps) if "run_upgrade_release_gate.py" in step.get("run", "")) + regression_index = next( + i for i, step in enumerate(steps) if "ci_legacy_upgrade_rejection.py" in step.get("run", "") + ) + assert setup_index < gate_index < regression_index + for index in (setup_index, gate_index, regression_index): + assert "if" not in steps[index] and not steps[index].get("continue-on-error") + assert "for backend in sqlite seekdb oceanbase; do" in steps[regression_index]["run"] + assert "--respect-policy" in steps[regression_index]["run"] + assert "/pc_upgrade_test?" in job["env"]["PC_UPGRADE_OCEANBASE_URL"] + assert "/pc_upgrade_fresh?" in job["env"]["PC_UPGRADE_OCEANBASE_FRESH_URL"] + cleanup = next(step for step in steps if "docker rm --force upgrade-oceanbase" in step.get("run", "")) + assert cleanup["if"] == "always()" From 76d4863b4e3d4ec1b629460db842fd7fc960982b Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 22 Oct 2026 11:36:07 +0800 Subject: [PATCH 26/94] fix(ci): select OceanBase for the historical upgrade fixture --- scripts/ci_legacy_upgrade_rejection.py | 25 ++++++++++++++----------- tests/test_upgrade_release_gate.py | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/scripts/ci_legacy_upgrade_rejection.py b/scripts/ci_legacy_upgrade_rejection.py index ef9f9a46d1..4cbabf5332 100644 --- a/scripts/ci_legacy_upgrade_rejection.py +++ b/scripts/ci_legacy_upgrade_rejection.py @@ -37,6 +37,17 @@ def console(python: Path) -> Path: return python.with_name("powercontext.exe" if os.name == "nt" else "powercontext") +def database_environment(backend: str, database: Path, environment: dict[str, str]) -> dict[str, str]: + if backend == "seekdb": + return {"POWERCONTEXT_SERVER_DATABASE_KIND": "seekdb", "POWERCONTEXT_SERVER_DATABASE_PATH": str(database)} + if backend == "oceanbase": + return { + "POWERCONTEXT_SERVER_DATABASE_KIND": "oceanbase", + "POWERCONTEXT_SERVER_DATABASE_URL": environment["PC_UPGRADE_OCEANBASE_URL"], + } + return {"POWERCONTEXT_SERVER_DATABASE_URL": f"sqlite+aiosqlite:///{database}"} + + def source_database(python: Path, directory: Path, environment: dict[str, str], log: Path) -> dict[str, str | bool]: port = _available_port() base_url = f"http://127.0.0.1:{port}" @@ -173,18 +184,10 @@ def main() -> None: require(metadata["source"]["powercontext"] == "1.3.0", "Wrong source package") require(metadata["target"]["powercontext"] == wheel_version(wheel), "Wrong target package") database = directory / "database" - if args.backend == "seekdb": - database_environment = { - "POWERCONTEXT_SERVER_DATABASE_KIND": "seekdb", - "POWERCONTEXT_SERVER_DATABASE_PATH": str(database), - } - elif args.backend == "oceanbase": - database_environment = {"POWERCONTEXT_SERVER_DATABASE_URL": environment["PC_UPGRADE_OCEANBASE_URL"]} - else: - database_environment = {"POWERCONTEXT_SERVER_DATABASE_URL": f"sqlite+aiosqlite:///{database}"} + database_configuration = database_environment(args.backend, database, environment) environment.update({ "POWERCONTEXT_HOME": str(directory / "home"), - **database_environment, + **database_configuration, "POWERCONTEXT_SERVER_ACCESS_MODE": "disabled", }) probe = [ @@ -221,7 +224,7 @@ def main() -> None: "Source fixture lacks historical Server objects", ) env_file = directory / "deployment.env" - env_file.write_text("".join(f"{key}={value}\n" for key, value in database_environment.items())) + env_file.write_text("".join(f"{key}={value}\n" for key, value in database_configuration.items())) target = str(console(interpreters["target"])) rejected( [target, "server", "db-migrate", "status", "--env-file", str(env_file)], diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py index e012b0a75d..28525694aa 100644 --- a/tests/test_upgrade_release_gate.py +++ b/tests/test_upgrade_release_gate.py @@ -15,6 +15,7 @@ """Release-blocking regressions, not evidence that any database upgrade works.""" import json +import os import zipfile from pathlib import Path from xml.etree import ElementTree @@ -23,6 +24,9 @@ import upgrade_release_gate as gate import upgrade_release_matrix as matrix import yaml +from ci_legacy_upgrade_rejection import database_environment + +from powercontext.server.settings import ServerSettings @pytest.fixture @@ -391,3 +395,15 @@ def test_upgrade_jobs_require_real_backends_and_always_clean_up(workflow_name, j assert "/pc_upgrade_fresh?" in job["env"]["PC_UPGRADE_OCEANBASE_FRESH_URL"] cleanup = next(step for step in steps if "docker rm --force upgrade-oceanbase" in step.get("run", "")) assert cleanup["if"] == "always()" + + +def test_oceanbase_fixture_selects_the_oceanbase_server_profile(tmp_path, monkeypatch): + for key in os.environ: + if key.startswith("POWERCONTEXT_"): + monkeypatch.delenv(key) + url = "mysql+aoceanbase://root%40test:test@127.0.0.1:2881/pc_upgrade_test?charset=utf8mb4" + for key, value in database_environment("oceanbase", tmp_path / "unused", {"PC_UPGRADE_OCEANBASE_URL": url}).items(): + monkeypatch.setenv(key, value) + settings = ServerSettings() + assert settings.database.kind == "oceanbase" + assert settings.database.url.get_secret_value() == url From 8391cf98e90e298c774afc6f679918197b4ba7c0 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 22 Oct 2026 11:42:39 +0800 Subject: [PATCH 27/94] fix(ci): provide OceanBase maintenance coordination options --- scripts/ci_legacy_upgrade_rejection.py | 5 ++++- tests/release_upgrades/README.md | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/ci_legacy_upgrade_rejection.py b/scripts/ci_legacy_upgrade_rejection.py index 4cbabf5332..d1b712ec00 100644 --- a/scripts/ci_legacy_upgrade_rejection.py +++ b/scripts/ci_legacy_upgrade_rejection.py @@ -226,8 +226,11 @@ def main() -> None: env_file = directory / "deployment.env" env_file.write_text("".join(f"{key}={value}\n" for key, value in database_configuration.items())) target = str(console(interpreters["target"])) + status_command = [target, "server", "db-migrate", "status", "--env-file", str(env_file)] + if args.backend == "oceanbase": + status_command.extend(["--evidence-dir", str(output / "maintenance"), "--lock-coordination", "single-host"]) rejected( - [target, "server", "db-migrate", "status", "--env-file", str(env_file)], + status_command, environment, directory, output / "status.log", diff --git a/tests/release_upgrades/README.md b/tests/release_upgrades/README.md index 80c785b266..4e7eddff93 100644 --- a/tests/release_upgrades/README.md +++ b/tests/release_upgrades/README.md @@ -116,7 +116,8 @@ against a database created and populated by installed PowerContext 1.3.0. It use database per backend, sequential process ownership, and public HTTP writes/readback before testing target CLI behavior. The installed drivers inventory the actual tables and record each environment's engine, driver, and binding versions. OceanBase uses the container's dedicated `pc_upgrade_test` database and records its MySQL tenant mode and direct -connection. The negative runner only accepts the local test database and verifies it is empty before starting the +connection. Its public migration status command supplies `--evidence-dir` and `--lock-coordination single-host`, +as required by OceanBase maintenance. The negative runner only accepts the local test database and verifies it is empty before starting the old Server; it refuses to overwrite existing data. No external service or credentials are needed. A missing embedded binding or unavailable OceanBase instance fails the job; neither backend is skipped. From fd4173d9fbd6e845f4ba50620266ec5ba560b6dd Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 22 Oct 2026 15:26:07 +0800 Subject: [PATCH 28/94] ci(release): require successful full database upgrades --- .github/workflows/master.yml | 8 - .github/workflows/release.yml | 8 - docs/en/development/releasing.md | 24 +- docs/en/docs/get-started/install-and-run.md | 2 +- docs/zh/development/releasing.md | 23 +- docs/zh/docs/get-started/install-and-run.md | 2 +- release/upgrade-support.json | 10 +- scripts/upgrade_database_probe.py | 20 +- scripts/upgrade_release_gate.py | 43 +- scripts/upgrade_release_matrix.py | 6 +- tests/conftest.py | 4 + tests/release_upgrades/README.md | 75 ++- tests/release_upgrades/processing_fixture.py | 60 +++ tests/release_upgrades/test_full_server.py | 465 +++++++++++++++++++ tests/test_release_version.py | 4 +- tests/test_upgrade_release_gate.py | 92 +++- 16 files changed, 714 insertions(+), 132 deletions(-) create mode 100644 tests/release_upgrades/processing_fixture.py create mode 100644 tests/release_upgrades/test_full_server.py diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index d45eb86a9a..eb4ac2e4a6 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -47,14 +47,6 @@ jobs: run: >- uv run python scripts/run_upgrade_release_gate.py --output "$RUNNER_TEMP/upgrade-release-evidence" - - name: Reproduce unsupported upgrades on all three backends - run: | - wheel=$(find dist -name '*.whl' -print -quit) - for backend in sqlite seekdb oceanbase; do - uv run python scripts/ci_legacy_upgrade_rejection.py \ - --backend "$backend" --respect-policy --wheel "$wheel" \ - --output "$RUNNER_TEMP/upgrade-release-evidence/legacy-$backend" - done - name: Preserve OceanBase diagnostics if: always() run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2dd1952db7..2dddbb90d0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -86,14 +86,6 @@ jobs: --release --release-event "$GITHUB_EVENT_PATH" --output "$RUNNER_TEMP/upgrade-release-evidence" - - name: Reproduce unsupported upgrades on all three backends - run: | - wheel=$(find dist -name '*.whl' -print -quit) - for backend in sqlite seekdb oceanbase; do - uv run python scripts/ci_legacy_upgrade_rejection.py \ - --backend "$backend" --respect-policy --wheel "$wheel" \ - --output "$RUNNER_TEMP/upgrade-release-evidence/legacy-$backend" - done - name: Preserve OceanBase diagnostics if: always() run: | diff --git a/docs/en/development/releasing.md b/docs/en/development/releasing.md index 4dff970068..db765663cc 100644 --- a/docs/en/development/releasing.md +++ b/docs/en/development/releasing.md @@ -135,9 +135,11 @@ across SQLite, embedded seekDB, and OceanBase. The target is the actual wheel bu 1.4.1 and 1.5.0 continue testing from 1.3.0 automatically; intermediate releases are not added as sources. Source wheel URLs and SHA-256s are resolved from public PyPI and recorded with the target artifact. This fixed baseline alone does not certify databases created/populated by other releases; use explicit cases for additional claims. -The current policy is `fresh-install-only`; existing Server database upgrades are unsupported. Its null suite paths -record candidates only, not accepted upgrades. Supported mode requires implemented suites for every backend profile. -Do not advertise a supported upgrade until its installed-release acceptance suite is implemented and passes. +The current `upgrade-required` policy mandates positive acceptance on all three backends while keeping upgrade +support unverified. Every profile points to `tests/release_upgrades/test_full_server.py`; missing migration support +must fail. Neither a fresh-install-only declaration nor correct rejection of an old database satisfies this gate. +After the full migration implementation and acceptance pass, review the declaration, set `supported-upgrades`, and +regenerate the installation notices before publication. Both modes run the same mandatory acceptance matrix. The suite contract and evidence requirements are in `tests/release_upgrades/README.md` in the repository. `make check` and `make release-check` validate this policy and the bilingual installation notices. Changes to @@ -154,15 +156,17 @@ uv run python scripts/run_upgrade_release_gate.py --release --release-notes /pat The release check requires the actual wheel version to match the policy and the release notes to include the exact English support declaration from the installation guide. It installs each exact source release and the target wheel -in separate environments. Missing tests, skipped backends, failures, and missing evidence block the check. A passing -`fresh-install-only` policy check does not establish an upgrade path. Ordinary PR builds may have a development +in separate environments. Missing tests, skipped backends, failures, and missing evidence block the check. Static +policy validation alone does not establish an upgrade path. Ordinary PR builds may have a development version; the `--release` check must run against the final tagged wheel without overriding its version. -The **Upgrade release gate** CI job also reproduces the known unsupported upgrade rejections using real 1.3.0 -SQLite, embedded seekDB, and OceanBase Server databases. seekDB runs in a temporary runner directory. Both PR -and publication jobs start a disposable OceanBase CE container, wait for its MySQL tenant, archive diagnostics, and -remove it even on failure. No external database credentials are needed. Once supported mode is activated, these negative assertions retire and -the fixed-source three-backend upgrade matrix must pass. Repository administrators must make this job required in branch protection. +The **Upgrade release gate** CI job requires successful public migration commands, full Server readiness, target +startup, preserved records and new writes on real 1.3.0 databases. Limited bundle verification with +`server_ready=false` fails. Every backend runs even if another backend fails, with separate logs and results. +seekDB runs embedded in a temporary runner directory. PR and publication jobs start a disposable OceanBase CE +container, wait for its MySQL tenant, archive diagnostics, and remove it even on failure. No external database +credentials are needed. Unavailable backends fail instead of being skipped. Historical rejection regressions are +not part of the positive acceptance gate. Administrators must make this job required in branch protection. The release workflow gates distribution upload, PyPI publication, and release assets on the acceptance result and archives its evidence. It runs after `release.published`, so it cannot prevent the earlier manual publication of the GitHub Release page: run the command above before that action. `make publish` requires diff --git a/docs/en/docs/get-started/install-and-run.md b/docs/en/docs/get-started/install-and-run.md index f6021a91bd..b103a4c7cc 100644 --- a/docs/en/docs/get-started/install-and-run.md +++ b/docs/en/docs/get-started/install-and-run.md @@ -175,7 +175,7 @@ For a long-running process, Docker, authentication, or remote access, continue w ## Update or replace an installation -This release supports fresh installations only; upgrades of existing Server databases are not supported. +Upgrades from 1.3.0 require passing full Server acceptance on SQLite, seekDB and OceanBase. This migration path is not yet verified; do not upgrade existing databases. Keep existing deployments on their working version until their source version and backend have an accepted upgrade diff --git a/docs/zh/development/releasing.md b/docs/zh/development/releasing.md index b538eabe88..6c5af20b7d 100644 --- a/docs/zh/development/releasing.md +++ b/docs/zh/development/releasing.md @@ -127,9 +127,11 @@ DSH README 中说明某项能力已包含在 `1.1.0` 的文字同样属于历史 目标始终是本次 CI 构建的实际 wheel；后续发布 1.4.1、1.5.0 时，自动测试 1.3.0 到本次版本，不增加中间版本来源。 来源 wheel 的公开 PyPI 地址和 SHA-256 与目标产物一起记录。固定起点不能证明其他版本实际生成的数据同样兼容； 如需额外声明这些路径，使用显式 `cases` 补充验收。 -当前策略为 `fresh-install-only`，不支持已有 Server 数据库升级；配置中的空套件路径只生成候选组合，不算验收通过。 -启用支持升级模式时，每个后端配置都必须有真实套件。只有实现并通过已安装发行包的完整升级验收，才能声明支持 -对应组合。验收契约和证据要求见仓库中的 `tests/release_upgrades/README.md`。 +当前 `upgrade-required` 策略强制执行三种后端的正向升级验收，同时保持“升级支持尚未验证”的声明。 +所有配置均使用 `tests/release_upgrades/test_full_server.py`，迁移能力缺失必须失败。仅支持新建数据库的声明或 +正确拒绝旧库，都不能满足门禁。完整迁移实现和验收通过后，审查支持范围，将模式改为 `supported-upgrades`， +并在发布前重新生成安装说明；两种模式都运行相同的必测矩阵。 +验收契约和证据要求见仓库中的 `tests/release_upgrades/README.md`。 `make check` 和 `make release-check` 会核对策略及双语安装说明。受管持久化文件变化时，必须审查兼容性影响， 更新摘要和理由。摘要用于触发审查，本身不证明迁移正确。`make version-bump` 同时更新策略中的目标版本； @@ -143,13 +145,14 @@ uv run python scripts/run_upgrade_release_gate.py --release --release-notes /pat ``` 发布检查要求实际 wheel 版本与策略一致，发布说明包含安装指南中的完整英文支持声明。每个组合分别安装准确的 -旧版发行包和目标 wheel。测试缺失、后端跳过、执行失败或证据缺失都会阻断检查。`fresh-install-only` 策略检查 -通过不表示存在升级路径。普通 PR 构建允许使用开发版本；`--release` 必须验证最终 tag 构建的 wheel，不能覆盖版本号。 - -CI 中的 **Upgrade release gate** 使用真实 1.3.0 SQLite、嵌入式 seekDB 和 OceanBase Server 数据库复现升级拒绝行为。 -seekDB 直接在 runner 临时目录启动；PR 和发布工作流均启动隔离的 OceanBase CE 容器，等待 MySQL 租户就绪， -归档诊断并在测试失败时也清理容器，不依赖外部数据库凭据。任一后端无法启动都使检查失败，不能跳过。 -启用支持升级模式后，固定的拒绝断言退出，1.3.0 到目标版本的三种后端完整升级验收必须通过。 +旧版发行包和目标 wheel。测试缺失、后端跳过、执行失败或证据缺失都会阻断检查。仅通过静态策略检查不表示 +存在升级路径。普通 PR 构建允许使用开发版本；`--release` 必须验证最终 tag 构建的 wheel，不能覆盖版本号。 + +CI 中的 **Upgrade release gate** 要求真实 1.3.0 数据库通过公开命令迁移、完整 Server 就绪验证、目标版本启动、 +历史数据读取和新增写入。局部迁移验证返回 `server_ready=false` 也会失败。一个后端失败后，仍继续执行其他后端， +分别保存结果和日志。seekDB 直接在 runner 临时目录中以嵌入式启动；PR 和发布工作流均启动隔离的 OceanBase CE +容器，等待 MySQL 租户就绪，归档诊断并在测试失败时也清理容器，不依赖外部数据库凭据。 +任一后端无法启动都使检查失败，不能跳过。历史拒绝行为回归不参与正向升级门禁。 仓库管理员需要在分支保护中将该检查设为必需。 发布工作流只有验收通过才会上传发行包、发布 PyPI 和上传 Release 附件，并归档验收证据。工作流在 `release.published` 后运行，无法阻止之前手动发布 GitHub Release 页面，因此必须先运行上述命令再发布页面。 diff --git a/docs/zh/docs/get-started/install-and-run.md b/docs/zh/docs/get-started/install-and-run.md index 5e8360b924..becc71943d 100644 --- a/docs/zh/docs/get-started/install-and-run.md +++ b/docs/zh/docs/get-started/install-and-run.md @@ -161,7 +161,7 @@ Agent 诊断见[各自的集成文档](../integrations/index.md)；Server 状态 ## 更新或替换安装 -此版本仅支持新建部署，尚不支持已有 Server 数据库升级。 +1.3.0 升级必须通过 SQLite、seekDB 和 OceanBase 的完整 Server 验收。当前迁移路径尚未验证，请勿升级已有数据库。 已有部署应保持当前可用版本，等待来源版本及后端具备已验收的升级路径。试用新建部署时，请使用独立程序环境、 diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 77d31c67eb..7718d463eb 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -1,10 +1,10 @@ { "format": 2, "target_version": "1.4.0", - "mode": "fresh-install-only", + "mode": "upgrade-required", "schema_review": { "persistence_sha256": "ca3b475f2d0c883e19291a8d8b1dadbe50f93ebd93aad550a9112a3497737927", - "rationale": "The full Server upgrade framework is incomplete (#2154). Existing databases are unsupported; no production upgrade combination is advertised until installed-release acceptance passes." + "rationale": "Full Server upgrades from 1.3.0 are mandatory on SQLite, embedded seekDB and OceanBase. Acceptance must fail until the migration integration tracked in #2154 is complete." }, "cases": [], "automatic_sources": { @@ -14,17 +14,17 @@ { "backend": "sqlite", "configuration": "full-server-no-vector", - "suite": null + "suite": "tests/release_upgrades/test_full_server.py" }, { "backend": "seekdb", "configuration": "full-server-no-vector", - "suite": null + "suite": "tests/release_upgrades/test_full_server.py" }, { "backend": "oceanbase", "configuration": "full-server-no-vector", - "suite": null + "suite": "tests/release_upgrades/test_full_server.py" } ] } diff --git a/scripts/upgrade_database_probe.py b/scripts/upgrade_database_probe.py index 0188b77823..53618388b2 100644 --- a/scripts/upgrade_database_probe.py +++ b/scripts/upgrade_database_probe.py @@ -39,8 +39,18 @@ async def inspect(backend: str, database: Path, *, add_revision: bool) -> dict[s with sqlite3.connect(database) as connection: if add_revision: connection.execute(REVISION_DDL) - rows = connection.execute("SELECT name, sql FROM sqlite_schema WHERE type='table' ORDER BY name").fetchall() - return {"engine_version": sqlite3.sqlite_version, "driver_version": version("aiosqlite"), "tables": dict(rows)} + objects = { + plural: dict( + connection.execute("SELECT name, sql FROM sqlite_schema WHERE type=? ORDER BY name", (kind,)) + ) + for kind, plural in ( + ("table", "tables"), + ("index", "indexes"), + ("view", "views"), + ("trigger", "triggers"), + ) + } + return {"engine_version": sqlite3.sqlite_version, "driver_version": version("aiosqlite"), **objects} # Use the driver's public API, not a current-checkout Profile that could run # startup guards or mutate an old schema before the target CLI is tested. @@ -79,11 +89,17 @@ async def inspect(backend: str, database: Path, *, add_revision: bool) -> dict[s for name in names: cursor.execute("SHOW CREATE TABLE `" + name.replace("`", "``") + "`") tables[name] = cursor.fetchone()[1] + cursor.execute("SHOW FULL TABLES WHERE Table_type = 'VIEW'") + views = {} + for name in sorted(row[0] for row in cursor.fetchall()): + cursor.execute("SHOW CREATE VIEW `" + name.replace("`", "``") + "`") + views[name] = cursor.fetchone()[1] return { "engine_version": engine, "driver_version": version("pymysql"), **details, "tables": tables, + "views": views, } finally: if instance is not None: diff --git a/scripts/upgrade_release_gate.py b/scripts/upgrade_release_gate.py index cf0a3cbc4d..0ba593cd7f 100644 --- a/scripts/upgrade_release_gate.py +++ b/scripts/upgrade_release_gate.py @@ -53,8 +53,8 @@ }) VERSION = r"\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?" NOTICE = { - "en": "This release supports fresh installations only; upgrades of existing Server databases are not supported.", - "zh": "此版本仅支持新建部署，尚不支持已有 Server 数据库升级。", # noqa: RUF001 -- Chinese prose + "en": "Upgrades from 1.3.0 require passing full Server acceptance on SQLite, seekDB and OceanBase. This migration path is not yet verified; do not upgrade existing databases.", + "zh": "1.3.0 升级必须通过 SQLite、seekDB 和 OceanBase 的完整 Server 验收。当前迁移路径尚未验证，请勿升级已有数据库。", # noqa: RUF001 -- Chinese prose } @@ -100,7 +100,7 @@ def load_policy(root: Path) -> dict[str, Any]: require( match is not None and match[1] == policy["target_version"], "Upgrade policy target differs from API release" ) - require(policy["mode"] in {"fresh-install-only", "supported-upgrades"}, "Unknown upgrade policy mode") + require(policy["mode"] in {"upgrade-required", "supported-upgrades"}, "Unknown upgrade policy mode") review = policy["schema_review"] require(set(review) == {"persistence_sha256", "rationale"}, "Invalid schema review fields") require( @@ -124,8 +124,12 @@ def load_policy(root: Path) -> dict[str, Any]: require(isinstance(profiles, list) and profiles, "Automatic sources require backend profiles") for profile in profiles: require(set(profile) == {"backend", "configuration", "suite"}, "Invalid automatic backend profile") - supported = policy["mode"] == "supported-upgrades" - require(bool(cases or profiles) if supported else not cases, "Upgrade mode and support matrix disagree") + require(bool(cases or profiles), "Upgrade acceptance requires a nonempty matrix") + required_backends = {"sqlite", "seekdb", "oceanbase"} + covered = {case["backend"] for case in cases if case.get("source_version") == "1.3.0"} | ( + {profile["backend"] for profile in profiles} if automatic and automatic["source_version"] == "1.3.0" else set() + ) + require(covered >= required_backends, "The 1.3.0 upgrade baseline requires SQLite, seekDB and OceanBase") identities: set[tuple[str, str, str]] = set() for case in cases: require(set(case) == {"source_version", "backend", "configuration", "suite"}, "Invalid upgrade case fields") @@ -139,9 +143,10 @@ def load_policy(root: Path) -> dict[str, Any]: identity = (case.get("source_version", "automatic"), case["backend"], case["configuration"]) require(identity not in identities, "Duplicate upgrade combination") identities.add(identity) - if case["suite"] is None: - require(not supported, "Supported upgrades require a real acceptance suite for every backend profile") - continue + require( + isinstance(case["suite"], str) and case["suite"], + "A real acceptance suite is required for every backend profile", + ) suite = (root / case["suite"]).resolve() require( suite.is_relative_to((root / "tests/release_upgrades").resolve()), @@ -154,7 +159,7 @@ def load_policy(root: Path) -> dict[str, Any]: def support_text(policy: dict[str, Any], language: str) -> str: - if policy["mode"] == "fresh-install-only": + if policy["mode"] == "upgrade-required": return NOTICE[language] title = ( "Supported database upgrades (other combinations are unsupported):" @@ -256,7 +261,7 @@ def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dic for key, value in os.environ.items() if not key.startswith(("POWERCONTEXT_", "PYTEST_")) and key not in {"PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV"} } - with tempfile.TemporaryDirectory(prefix="pc-upgrade-") as temporary: + with tempfile.TemporaryDirectory(prefix="pc-up-", dir="/tmp" if os.name != "nt" else None) as temporary: # noqa: S108 -- private mkdtemp directory; short embedded socket path directory = Path(temporary) extras = "cli,server,seekdb" if case["backend"] == "seekdb" else "cli,server" interpreters = {} @@ -284,7 +289,9 @@ def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dic metadata = {role: installed(python, environment, directory) for role, python in interpreters.items()} require(metadata["source"]["powercontext"] == case["source_version"], "Source release mismatch") require(metadata["target"]["powercontext"] == wheel_version(wheel), "Target wheel version mismatch") + (output / "installed.json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") environment.update({ + "PC_UPGRADE_REPORT_DIR": str(output), "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", "PC_UPGRADE_SOURCE_PYTHON": str(interpreters["source"]), "PC_UPGRADE_SOURCE_VERSION": case["source_version"], @@ -344,8 +351,22 @@ def gate(root: Path, wheel: Path, output: Path, *, release: bool = False, releas try: matrix = resolve_matrix(policy, actual_version) evidence["resolved_matrix"] = matrix + require(matrix["required_cases"], "No required upgrade acceptance cases") + failures = [] for index, case in enumerate(matrix["required_cases"]): - evidence["cases"].append(run_case(root, wheel, case, output / f"case-{index}")) + try: + evidence["cases"].append(run_case(root, wheel, case, output / f"case-{index}")) + except (GateError, OSError, ValueError, subprocess.SubprocessError, ElementTree.ParseError) as error: + # A broken backend must not prevent the other required backends + # from running and producing their own failure evidence. + failures.append(case["backend"]) + evidence["cases"].append({"case": case, "status": "failed", "error": str(error)}) + require(not failures, f"Required upgrade acceptance failed: {', '.join(failures)}") + if release: + require( + policy["mode"] == "supported-upgrades", + "Upgrade acceptance passed; update the reviewed support declaration before publication", + ) evidence["status"] = "passed" finally: (output / "result.json").write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") diff --git a/scripts/upgrade_release_matrix.py b/scripts/upgrade_release_matrix.py index dbb56b33ce..9ed8cbb731 100644 --- a/scripts/upgrade_release_matrix.py +++ b/scripts/upgrade_release_matrix.py @@ -63,7 +63,7 @@ def resolve_matrix(policy: dict[str, Any], target: str) -> dict[str, Any]: automatic = policy["automatic_sources"] explicit = policy["cases"] if automatic is None and not explicit: - return {"candidate_cases": [], "required_cases": []} + raise ValueError("Upgrade acceptance requires a nonempty matrix") # noqa: TRY003 catalog = fetch_catalog() if not isinstance(catalog.get("releases"), dict) or not catalog["releases"]: raise ValueError("Empty or malformed PyPI release catalog") # noqa: TRY003 @@ -84,7 +84,5 @@ def resolve_matrix(policy: dict[str, Any], target: str) -> dict[str, Any]: "catalog_url": CATALOG_URL, "catalog_sha256": hashlib.sha256(json.dumps(catalog, sort_keys=True).encode()).hexdigest(), "candidate_cases": cases, - # Fresh-install-only records discovery, never reports unexecuted upgrades - # as accepted. Switching mode requires real suites for every profile. - "required_cases": cases if policy["mode"] == "supported-upgrades" else [], + "required_cases": cases, } diff --git a/tests/conftest.py b/tests/conftest.py index 91bda3b638..d8fd628be4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -161,6 +161,10 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item def pytest_ignore_collect(collection_path: Path, config: pytest.Config) -> bool | None: + # Installed-release suites run only through the mandatory artifact gate. + # The gate uses --confcutdir to exclude this repository conftest entirely. + if collection_path.resolve() == (Path(__file__).parent / "release_upgrades").resolve(): + return True # Alembic executes env.py as migration configuration; doctest collection # must not import it as a test module. if collection_path.resolve() == _DATABASE_MIGRATION_ENV.resolve(): diff --git a/tests/release_upgrades/README.md b/tests/release_upgrades/README.md index 4e7eddff93..a30d9d1b9b 100644 --- a/tests/release_upgrades/README.md +++ b/tests/release_upgrades/README.md @@ -1,8 +1,9 @@ # Installed-release upgrade acceptance -`release/upgrade-support.json` is the explicit upgrade support policy. Currently it declares -`fresh-install-only`: **no production database upgrade combination is supported**. Discovered candidates in this mode -are not accepted upgrades. Full migration integration is tracked in #2154. +`release/upgrade-support.json` requires **positive** 1.3.0 upgrades on SQLite, embedded seekDB, and OceanBase. +The current `upgrade-required` mode keeps the upgrade support declaration unverified while requiring every case +to run and pass. Missing full migration integration (#2154) fails the gate. `supported-upgrades` runs the same +matrix and enables publication after the reviewed support declaration is updated. `fresh-install-only` is rejected. ## Fixed source and current target @@ -18,11 +19,12 @@ source wheels, invalid artifact metadata, catalog outages, and a target at or be Evidence records the catalog digest, exact source artifact, target artifact digest, and expanded matrix. The resolver requires universal `py3-none-any` wheels; a distribution format change must extend it explicitly. -Currently the three profiles have `suite: null` and the mode remains `fresh-install-only`. Resolving candidates -executes and accepts **zero** positive upgrades. To activate supported upgrades, implement the complete suite -contract below for each backend, replace the null paths, set `mode` to `supported-upgrades`, and regenerate the -installation notices. A missing suite fails supported mode. Tests must fail when migration support is missing; -the runner never changes expected success into a rejection automatically. +All three profiles use `test_full_server.py`. It starts the installed 1.3.0 Server, writes data through HTTP, +stops it, seeds a pending processing intent using the installed SDK, and runs the target wheel's public +`server db-migrate status/plan/apply/verify` commands. Verification must report `state=ready` and `server_ready=true`. +The target Server must then start, preserve data and permissions, accept new writes, and support retrieval. +Partial migration success, missing framework/baseline support, and startup rejection all fail. These disposable +fixtures explicitly accept no backup; this is not a production upgrade procedure. `cases` remains available for additional exact source versions/configurations. Each case needs `source_version`, `backend`, `configuration`, and `suite`; overlapping cases fail. Optional features that create persistent objects @@ -45,6 +47,7 @@ The suite receives: - `PC_UPGRADE_BACKEND`, `PC_UPGRADE_CONFIGURATION`: the declared case. - `PC_UPGRADE_WORK`: disposable workspace for both databases, controlled inference fixtures, and environment files. - `PC_UPGRADE_EVIDENCE`: destination for backend evidence JSON. +- `PC_UPGRADE_REPORT_DIR`: persistent per-case command logs, fixture snapshots and schema inventories. - `PC_UPGRADE_OCEANBASE_URL`, `PC_UPGRADE_OCEANBASE_FRESH_URL`: dedicated disposable MySQL-tenant databases for the upgraded and fresh-install comparisons. CI supplies both; local positive OceanBase suites must supply their own isolated databases. Each suite must isolate/reset its own fixtures when multiple configurations run. @@ -58,9 +61,9 @@ Provide these behavioral tests (additional tests are allowed): | `test_target_startup` | Start the target Server against the migrated database and verify readiness. | | `test_read_write_and_retrieval` | Read the old records, write new ones, and retrieve both through public interfaces. | | `test_authorization` | Verify permitted and denied access and preservation of relevant grants. | -| `test_candidates_and_processing` | Verify retained Candidates and task consumption, including idempotency. | +| `test_candidates_and_processing` | Verify retained Candidates and CAS approval; load and acknowledge the old pending intent using the target installed SDK, including idempotency. This does not execute a model-driven Worker. | | `test_preserved_history_and_cursors` | Verify exact history/references and unchanged or declared-transformed cursor and receipt semantics. | -| `test_fresh_upgrade_equivalence` | Initialize a fresh target database; compare schema/backend objects and behavior with the upgraded database, explicitly allowing only registered retained/archive objects. | +| `test_fresh_upgrade_equivalence` | Initialize a fresh target database; compare schema/backend objects and behavior with the upgraded database. The current suite has no retained/archive object exclusions. | Use a session fixture for the shared lifecycle; do not depend on pytest's test execution order. All collected tests must execute and pass. Empty collection, missing phases, skips (including missing backends), xfail, failure, command @@ -75,24 +78,14 @@ records installed Python/package versions, target wheel SHA-256, policy SHA-256, Evidence is created during the gate run in a new directory. There is no option to import old successful reports. Test names and metadata do not themselves prove correctness: reviewers must inspect the assertions and confirm -they follow the public operations and compare actual persisted data. This contract is intended to host the real -upgrade acceptance added with #2154, without claiming that it already exists. - -For example, after implementing and reviewing actual suites, automatic selection would have this shape: - -```json -{ - "strategy": "fixed-version", - "source_version": "1.3.0", - "profiles": [ - {"backend": "sqlite", "configuration": "full-server-no-vector", "suite": "tests/release_upgrades/test_sqlite.py"}, - {"backend": "seekdb", "configuration": "full-server-no-vector", "suite": "tests/release_upgrades/test_seekdb.py"}, - {"backend": "oceanbase", "configuration": "full-server-no-vector", "suite": "tests/release_upgrades/test_oceanbase.py"} - ] -} -``` - -These suite paths illustrate the contract; they do not exist yet. Regenerate installation support regions with +they follow actual installed operations and compare persisted data. The profile exercises explicit writes and FTS, +not vector storage or model-generated content. The processing fixture tests persisted intent compatibility and +acknowledgment, not Worker execution. Add separately reviewed profiles before claiming those optional paths. +SQLite inventories include indexes, views and triggers; MySQL-family table DDL includes indexes and constraints, +and views are collected separately. Comparison normalizes whitespace, identifier quoting and auto-increment +counters only; structural differences fail rather than being silently excluded. + +Regenerate installation support regions with `uv run python scripts/upgrade_release_gate.py --write-docs` and run `make upgrade-policy-check`. When persistence inputs change, calculate `managed_digest(ROOT)` from `upgrade_release_gate`, review the compatibility impact, and update both the digest and rationale in `schema_review`. The inputs include persistence, optional code storage, @@ -109,23 +102,9 @@ Repository administrators must require **Upgrade release gate** in branch protec that setting. Because the release workflow starts on `release.published`, it can block package/asset publication, not the earlier manual creation of a GitHub Release; run the prepublication command before publishing that page. -## Actual historical rejection regressions - -The PR/master and publication jobs run `ci_legacy_upgrade_rejection.py` for **SQLite, embedded seekDB, and OceanBase** -against a database created and populated by installed PowerContext 1.3.0. It uses separate source/target environments, one disposable -database per backend, sequential process ownership, and public HTTP writes/readback before testing target CLI behavior. -The installed drivers inventory the actual tables and record each environment's engine, driver, and binding versions. -OceanBase uses the container's dedicated `pc_upgrade_test` database and records its MySQL tenant mode and direct -connection. Its public migration status command supplies `--evidence-dir` and `--lock-coordination single-host`, -as required by OceanBase maintenance. The negative runner only accepts the local test database and verifies it is empty before starting the -old Server; it refuses to overwrite existing data. No external service or credentials are needed. A missing embedded -binding or unavailable OceanBase instance fails the job; neither backend is skipped. - -The regressions require historical startup rejection, full-baseline migration rejection, and rejection after adding -only the prototype revision table to the disposable fixture. The last operation is deliberately **not a migration**. -SeekDB and OceanBase permit either the known CHECK-reflection rejection or unknown-baseline rejection, so fixing #2160 does not -require retaining the parser defect. These tests do not certify that an upgrade succeeds. - -CI invokes them with `--respect-policy` after the main gate. Once supported-path acceptance is activated, these -historical rejection assertions are recorded as `not-applicable` instead of failing because migration was fixed. -The preceding positive gate must still execute all required version/backend cases and accept no skips. +## Historical rejection regressions + +`scripts/ci_legacy_upgrade_rejection.py` remains a diagnostic tool for historical rejection behavior. PR and release +workflows do not invoke it as upgrade acceptance. An expected rejection or its `--respect-policy` result can never +replace a successful positive upgrade. The gate attempts every required backend even after an earlier case fails, +then fails overall and archives each case's logs and result. Empty matrices and missing suites are errors. diff --git a/tests/release_upgrades/processing_fixture.py b/tests/release_upgrades/processing_fixture.py new file mode 100644 index 0000000000..87afd80ffd --- /dev/null +++ b/tests/release_upgrades/processing_fixture.py @@ -0,0 +1,60 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exercise stored invocation format compatibility using each installed SDK. + +Run with the selected installed interpreter and -I, with Server processes stopped. +No schema creation or direct SQL writes: the installed repository owns its rows. +""" + +import asyncio +import json +import sys +from pathlib import Path + +from powercontext.builtin.persistence.oceanbase import OceanBaseProfile +from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository +from powercontext.builtin.persistence.seekdb import SeekDBProfile +from powercontext.builtin.persistence.sqlite import SQLiteProfile +from powercontext.server.settings import ServerSettings + + +async def main(): + operation, scope = sys.argv[1:3] + config = ServerSettings().database + if config.kind == "sqlite": + storage_context = SQLiteProfile.open(config, tables=()) + elif config.kind == "seekdb": + storage_context = SeekDBProfile.open(config, tables=()) + else: + storage_context = OceanBaseProfile.open(config, tables=()) + repository = ArtifactProcessingIntentRepository() + binding = "topic-memory-source-window" + async with storage_context as storage, storage.database.transaction() as connection: + if operation == "seed": + row = await repository.request(connection, scope, binding) + result = row.model_dump(mode="json") + else: + row = await repository.load(connection, scope, binding) + assert row is not None and row.requested_generation > row.handled_generation + before = row.model_dump(mode="json") + after = await repository.acknowledge(connection, scope, binding, row.requested_generation) + repeated = await repository.acknowledge(connection, scope, binding, row.requested_generation) + assert after == repeated, "Acknowledgment must be idempotent" + result = {"before": before, "after": after.model_dump(mode="json")} + return result + + +if __name__ == "__main__": + Path(sys.argv[3]).write_text(json.dumps(asyncio.run(main()), indent=2) + "\n", encoding="utf-8") diff --git a/tests/release_upgrades/test_full_server.py b/tests/release_upgrades/test_full_server.py new file mode 100644 index 0000000000..7212eec944 --- /dev/null +++ b/tests/release_upgrades/test_full_server.py @@ -0,0 +1,465 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Installed 1.3.0 -> built wheel acceptance; every required phase must succeed. + +No checkout runtime imports, migration fallbacks, skips or expected failures. +Fixtures establish dependencies rather than relying on test execution order. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import signal +import socket +import subprocess +import time +from contextlib import contextmanager +from pathlib import Path +from urllib.parse import urlsplit + +import httpx +import pytest + +ROOT = Path(__file__).resolve().parents[2] +TOKEN = "isolated-upgrade-acceptance-token" # noqa: S105 -- disposable fixture, never a deployment credential + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +def console(python): + return str(python.with_name("powercontext.exe" if os.name == "nt" else "powercontext")) + + +def command(arguments, environment, work, log): + with log.open("w", encoding="utf-8") as stream: + result = subprocess.run( + arguments, cwd=work, env=environment, stdout=stream, stderr=subprocess.STDOUT, timeout=1800, check=False + ) + assert result.returncode == 0, f"Command failed ({result.returncode}); see {log}" + return log.read_text(encoding="utf-8") + + +def post(client, path, body): + response = client.post(path, json=body) + assert response.is_success, f"{path}: HTTP {response.status_code}: {response.text}" + return response.json() + + +@contextmanager +def server(python, environment, work, log): + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + port = reservation.getsockname()[1] + with log.open("w", encoding="utf-8") as stream: + process = subprocess.Popen( + [console(python), "server", "run", "--no-env-file", "--host", "127.0.0.1", "--port", str(port)], + cwd=work, + env=environment, + stdout=stream, + stderr=subprocess.STDOUT, + start_new_session=os.name != "nt", + text=True, + ) + try: + with httpx.Client( + base_url=f"http://127.0.0.1:{port}", + headers={"Authorization": f"Bearer {TOKEN}"}, + trust_env=False, + timeout=30, + ) as client: + deadline = time.monotonic() + 180 + while True: + assert process.poll() is None, f"Server exited before readiness; see {log}" + try: + response = client.get("/health/ready") + if response.status_code == 200 and response.json().get("status") in {"ready", "degraded"}: + break + except httpx.TransportError: + pass + assert time.monotonic() < deadline, f"Server readiness timed out; see {log}" + time.sleep(0.2) + yield client + finally: + if process.poll() is None: + if os.name == "nt": + process.terminate() + else: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + if os.name == "nt": + process.kill() + else: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + + +def environment_for(case, *, fresh=False): + environment = { + key: value + for key, value in os.environ.items() + if not key.startswith(("POWERCONTEXT_", "PYTEST_", "OPENAI_", "ANTHROPIC_", "OTEL_")) + and key not in {"PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV"} + } + environment.update({ + "POWERCONTEXT_HOME": str(case["work"] / ("fresh-home" if fresh else "home")), + "POWERCONTEXT_SERVER_AUTH_TOKEN": TOKEN, + "POWERCONTEXT_SERVER_ACCESS_MODE": "enforced", + "POWERCONTEXT_SERVER_CURSOR_SIGNING_SECRET": "isolated-upgrade-cursor-signing-key-0001", + "POWERCONTEXT_SERVER_RUNTIME_ARTIFACT_PROCESSING_ROLE": "api", + "POWERCONTEXT_SERVER_MCP_ENABLED": "false", + "NO_PROXY": "localhost,127.0.0.1", + "no_proxy": "localhost,127.0.0.1", + }) + database = case["work"] / ("fresh" if fresh else "database") + backend = case["backend"] + environment["POWERCONTEXT_SERVER_DATABASE_KIND"] = backend + if backend == "sqlite": + environment["POWERCONTEXT_SERVER_DATABASE_URL"] = f"sqlite+aiosqlite:///{database}" + elif backend == "seekdb": + environment["POWERCONTEXT_SERVER_DATABASE_PATH"] = str(database) + else: + name = "PC_UPGRADE_OCEANBASE_FRESH_URL" if fresh else "PC_UPGRADE_OCEANBASE_URL" + url = os.environ[name] + parsed = urlsplit(url) + expected = "/pc_upgrade_fresh" if fresh else "/pc_upgrade_test" + assert parsed.scheme == "mysql+aoceanbase" and parsed.hostname in {"127.0.0.1", "localhost"} + assert parsed.path == expected, "Only dedicated disposable OceanBase databases may be used" + environment["POWERCONTEXT_SERVER_DATABASE_URL"] = url + environment["PC_UPGRADE_OCEANBASE_URL"] = url + return environment + + +def snapshot(case, role, name, *, fresh=False): + path = case["report"] / f"{name}-schema.json" + command( + [ + str(case[role]), + "-I", + str(ROOT / "scripts/upgrade_database_probe.py"), + "--backend", + case["backend"], + "--database", + str(case["work"] / ("fresh" if fresh else "database")), + "--output", + str(path), + ], + environment_for(case, fresh=fresh), + case["work"], + case["report"] / f"{name}-probe.log", + ) + return json.loads(path.read_text()) + + +def task_state(case, role, operation, scope): + path = case["report"] / f"task-{operation}.json" + command( + [ + str(case[role]), + "-I", + str(ROOT / "tests/release_upgrades/processing_fixture.py"), + operation, + scope, + str(path), + ], + environment_for(case), + case["work"], + case["report"] / f"task-{operation}.log", + ) + return json.loads(path.read_text()) + + +def seed(client, *, candidate_prefix): + scopes = [ + post( + client, + "/v1/scopes", + { + "title": f"Upgrade fixture {i}", + "summary": "Installed release acceptance", + "idempotency_key": f"upgrade-scope-{i}", + }, + ) + for i in range(3) + ] + scope = scopes[0]["scope_id"] + memory = [ + post(client, "/v1/memory/remember", {"scope_id": scope, "kind": "fact", "text": text})["records"][0] + for text in ("Upgrade fixture preserves historical storage.", "Upgrade fixture retains exact references.") + ] + captured = post( + client, + "/v1/sources/content", + { + "scope_id": scope, + "source_id": "upgrade-evidence", + "content": "Historical source evidence for upgrade candidate review.", + }, + )["source"] + proposal = { + "situation": "A release upgrades storage", + "action": "Run migration acceptance", + "outcome": "History survives", + "lesson": "Validate real old databases", + } + body = {"scope_id": scope, "proposal": proposal, "source_refs": [captured], "artifact_refs": []} + candidate = post(client, "/v1/experience/propose", body) + first = candidate + candidate = post( + client, + f"{candidate_prefix}/revise", + { + **body, + "candidate_id": candidate["candidate_id"], + "expected_version": candidate["version"], + "proposal": {**proposal, "lesson": "Validate every required backend"}, + }, + ) + published = post(client, "/v1/experience/propose", body) + published = post( + client, + f"{candidate_prefix}/approve", + {"scope_id": scope, "candidate_id": published["candidate_id"], "expected_version": published["version"]}, + ) + artifact = post(client, "/v1/experience/get", {"scope_id": scope, "artifact": published["result_artifact"]}) + principal = client.get("/v1/access/me").json()["principal"] + grant = { + "subject": principal, + "resource": {"type": "scope", "scope_id": scope}, + "role": "scope.viewer", + "idempotency_key": "upgrade-viewer", + } + binding = post(client, "/v1/access/bindings/create", grant) + first_page = client.get("/v1/scopes", params={"limit": 1}).json() + cursor = first_page["next_cursor"] + assert cursor + continuation = client.get("/v1/scopes", params={"limit": 1, "cursor": cursor}).json() + assert continuation["items"] + readback = post(client, "/v1/memory/entries/list", {"scope_id": scope})["entries"] + assert {digest(item) for item in readback} >= {digest(item) for item in memory} + return { + "scope": scope, + "memory": memory, + "candidate": candidate, + "first_candidate": first, + "published": published, + "artifact": artifact, + "grant": grant, + "binding": binding, + "cursor": cursor, + "continuation": continuation, + } + + +@pytest.fixture(scope="session") +def case(): + # Missing gate inputs are errors, never skips or an implicit local database. + assert os.environ["PC_UPGRADE_SOURCE_VERSION"] == "1.3.0", "Add a reviewed adapter for another source release" + return { + "source": Path(os.environ["PC_UPGRADE_SOURCE_PYTHON"]), + "target": Path(os.environ["PC_UPGRADE_TARGET_PYTHON"]), + "backend": os.environ["PC_UPGRADE_BACKEND"], + "work": Path(os.environ["PC_UPGRADE_WORK"]), + "report": Path(os.environ["PC_UPGRADE_REPORT_DIR"]), + } + + +@pytest.fixture(scope="session") +def historical(case): + if case["backend"] == "oceanbase": + assert not snapshot(case, "source", "empty")["tables"], "Refusing to write into a nonempty test database" + assert not snapshot(case, "target", "fresh-empty", fresh=True)["tables"] + with server(case["source"], environment_for(case), case["work"], case["report"] / "source-server.log") as client: + value = seed(client, candidate_prefix="/v1/artifact-candidates") + value["task"] = task_state(case, "source", "seed", value["scope"]) + value["schema"] = snapshot(case, "source", "source") + assert {"pc_artifact_candidate_heads", "pc_artifact_candidate_versions", "pc_scopes", "pc_artifacts"} <= set( + value["schema"]["tables"] + ) + (case["report"] / "source-fixture.json").write_text(json.dumps(value, indent=2) + "\n") + return value + + +@pytest.fixture(scope="session") +def migrated(case, historical): + environment = environment_for(case) + env_file = case["work"] / "deployment.env" + env_file.write_text( + "".join(f"{key}={value}\n" for key, value in environment.items() if key.startswith("POWERCONTEXT_")) + ) + common = ["--env-file", str(env_file)] + if case["backend"] == "oceanbase": + common += ["--evidence-dir", str(case["report"] / "maintenance"), "--lock-coordination", "single-host"] + + def migration(operation, *options): + text = command( + [console(case["target"]), "server", "db-migrate", operation, *common, *options], + environment, + case["work"], + case["report"] / f"migration-{operation}.log", + ) + result = json.loads(text) + assert not result.get("error"), result + return result + + migration("status") + plan = migration("plan", "--backup", "skip") + assert plan.get("plan_id"), "The public plan must identify exactly what will be applied" + # These are disposable fixtures; all Server processes are stopped. This + # explicit no-backup choice is never a recommendation for production. + migration( + "apply", + "--backup", + "skip", + "--plan-id", + plan["plan_id"], + "--yes", + "--maintenance-confirmed", + "--accept-no-backup", + ) + verified = migration("verify") + assert verified.get("state") == "ready" and verified.get("server_ready") is True, ( + "Full Server readiness is required; registered-bundle verification is insufficient", + verified, + ) + return {"verified": verified, "schema": snapshot(case, "target", "migrated"), "historical": historical} + + +@pytest.fixture +def target(case, migrated): + with server(case["target"], environment_for(case), case["work"], case["report"] / "target-server.log") as client: + yield client + + +def test_source_release_database(historical): + assert historical["memory"] and historical["candidate"]["version"] == 2 + assert historical["task"]["requested_generation"] > historical["task"]["handled_generation"] + + +def test_public_migration_commands(migrated): + assert migrated["verified"]["server_ready"] is True + + +def test_target_startup(target): + response = target.get("/health/ready") + assert response.status_code == 200 + assert response.json()["status"] in {"ready", "degraded"} # No inference provider in this explicit-write profile. + + +def test_read_write_and_retrieval(target, historical): + scope = historical["scope"] + records = post(target, "/v1/memory/entries/list", {"scope_id": scope})["entries"] + assert {digest(item) for item in records} >= {digest(item) for item in historical["memory"]} + text = "Upgrade fixture accepts new writes after migration." + written = post(target, "/v1/memory/remember", {"scope_id": scope, "kind": "fact", "text": text})["records"] + assert written and written[0]["text"] == text + found = post(target, "/v1/memory/search", {"scope_id": scope, "query": "Upgrade fixture", "mode": "fts"})["hits"] + texts = {item["memory"]["text"] for item in found} + assert texts >= {text, *(item["text"] for item in historical["memory"])} + + +def test_authorization(target, historical): + assert target.get("/v1/scopes", headers={"Authorization": "Bearer invalid-fixture-token"}).status_code == 401 + # The stable idempotency key must resolve to the SAME persisted grant. + assert post(target, "/v1/access/bindings/create", historical["grant"]) == historical["binding"] + response = target.get(f"/v1/scopes/{historical['scope']}") + assert response.status_code == 200 + + +def test_candidates_and_processing(case, migrated): + historical = migrated["historical"] + # Native typed SDK loading/acknowledgment verifies stored invocation formats + # and generation idempotency; model-driven Worker quality is a separate suite. + state = task_state(case, "target", "verify", historical["scope"]) + assert {key: state["before"][key] for key in historical["task"]} == historical["task"] + assert state["after"]["handled_generation"] == historical["task"]["requested_generation"] + with server(case["target"], environment_for(case), case["work"], case["report"] / "candidate-server.log") as client: + candidate = historical["candidate"] + request = {"scope_id": historical["scope"], "candidate_id": candidate["candidate_id"]} + current = post(client, "/v1/candidates/get", request) + for key in ("candidate_id", "version", "status", "proposal", "source_refs", "artifact_refs"): + assert current[key] == candidate[key] + approved = post(client, "/v1/candidates/approve", {**request, "expected_version": candidate["version"]}) + assert approved["status"] == "approved" and approved["result_artifact"] + repeated = client.post("/v1/candidates/approve", json={**request, "expected_version": candidate["version"]}) + assert repeated.status_code == 409, "A stale approval must not publish a duplicate Artifact" + + +def test_preserved_history_and_cursors(target, historical): + expected = historical["continuation"] + page = target.get("/v1/scopes", params={"limit": 1, "cursor": historical["cursor"]}) + assert page.status_code == 200 + assert page.json() == expected + artifact = post( + target, + "/v1/experience/get", + {"scope_id": historical["scope"], "artifact": historical["published"]["result_artifact"]}, + ) + assert artifact == historical["artifact"] + history = post( + target, + "/v1/candidates/history", + {"scope_id": historical["scope"], "candidate_id": historical["candidate"]["candidate_id"]}, + )["versions"] + for version in (historical["first_candidate"], historical["candidate"]): + stored = next(item for item in history if item["version"] == version["version"]) + for key in ("version", "proposal", "source_refs", "artifact_refs"): + assert stored[key] == version[key] + + +def canonical_schema(snapshot): + def normalize(sql): + sql = re.sub(r"\bAUTO_INCREMENT=\d+", "", sql) + sql = re.sub(r'["`]([A-Za-z_][A-Za-z_0-9]*)["`]', r"\1", sql) + return " ".join(sql.split()) + + return { + kind: {name: normalize(sql or "") for name, sql in snapshot.get(kind, {}).items()} + for kind in ("tables", "indexes", "views", "triggers") + } + + +def test_fresh_upgrade_equivalence(case, migrated): + with server( + case["target"], environment_for(case, fresh=True), case["work"], case["report"] / "fresh-server.log" + ) as client: + fresh = seed(client, candidate_prefix="/v1/candidates") + hits = post( + client, "/v1/memory/search", {"scope_id": fresh["scope"], "query": "Upgrade fixture", "mode": "fts"} + )["hits"] + assert {item["memory"]["text"] for item in hits} >= {item["text"] for item in fresh["memory"]} + inventory = snapshot(case, "target", "fresh", fresh=True) + actual = canonical_schema(migrated["schema"]) + expected = canonical_schema(inventory) + assert actual == expected, ( + "Upgraded and fresh database objects differ; no blanket table/constraint exclusions allowed" + ) + evidence = { + "backend": case["backend"], + "engine_version": inventory["engine_version"], + "driver_version": inventory["driver_version"], + "source_schema": digest(migrated["historical"]["schema"]), + "target_schema": digest(actual), + "fresh_schema": digest(expected), + } + if case["backend"] == "oceanbase": + evidence.update(tenant_mode=inventory["tenant_mode"], proxy_version=inventory["proxy_version"]) + Path(os.environ["PC_UPGRADE_EVIDENCE"]).write_text(json.dumps(evidence, indent=2) + "\n") diff --git a/tests/test_release_version.py b/tests/test_release_version.py index 74a5bca627..b7542c699e 100644 --- a/tests/test_release_version.py +++ b/tests/test_release_version.py @@ -102,8 +102,8 @@ def test_release_updates_installation_references_and_preserves_history(release_r assert f"To install {version} in a separate environment" in english assert f"Python 包版本为 `{version}`" in chinese assert f"安装 {version} 到独立环境" in chinese - assert "upgrades of existing Server databases are not supported" in english - assert "尚不支持已有 Server 数据库升级" in chinese + assert "Upgrades from 1.3.0 require passing full Server acceptance" in english + assert "1.3.0 升级必须通过 SQLite、seekDB 和 OceanBase 的完整 Server 验收" in chinese dsh = (release_repo / "integrations/dsh/plugins/powercontext/README.md").read_text(encoding="utf-8") assert "Release 1.1.0 includes direct-operation Scope failure handling" in dsh for relative, original in preserved.items(): diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py index 28525694aa..66a77dc776 100644 --- a/tests/test_upgrade_release_gate.py +++ b/tests/test_upgrade_release_gate.py @@ -41,11 +41,16 @@ def release(tmp_path, monkeypatch): policy = { "format": 2, "target_version": "2.0.0", - "mode": "fresh-install-only", + "mode": "upgrade-required", "schema_review": {"persistence_sha256": gate.managed_digest(tmp_path), "rationale": "No upgrades supported"}, "cases": [], "automatic_sources": None, } + suite = tmp_path / "tests/release_upgrades/test_example.py" + suite.parent.mkdir(parents=True) + suite.write_text("# Gate orchestration fixture; not real upgrade acceptance.\n") + automatic_policy(policy) + monkeypatch.setattr(gate, "run_case", lambda _root, _wheel, case, _output: {"case": case, "status": "passed"}) (tmp_path / gate.MANIFEST).write_text(json.dumps(policy)) monkeypatch.setattr( matrix, @@ -74,11 +79,14 @@ def release(tmp_path, monkeypatch): return tmp_path, wheel, policy -def test_fresh_install_policy_does_not_claim_upgrade_acceptance(release): +def test_required_mode_executes_three_cases_but_cannot_publish_as_supported(release): root, wheel, policy = release - gate.gate(root, wheel, root / "evidence", release=True, release_notes=gate.support_text(policy, "en")) + with pytest.raises(gate.GateError, match="support declaration"): + gate.gate(root, wheel, root / "evidence", release=True, release_notes=gate.support_text(policy, "en")) result = json.loads((root / "evidence/result.json").read_text()) - assert result["mode"] == "fresh-install-only" and result["cases"] == [] + assert result["status"] == "failed" + assert len(result["cases"]) == 3 + assert all(case["status"] == "passed" for case in result["cases"]) assert result["target_wheel_sha256"] == gate.sha256(wheel) @@ -106,8 +114,9 @@ def test_schema_change_needs_a_new_compatibility_review(release): def test_supported_mode_cannot_succeed_with_an_empty_matrix(release): root, _, policy = release policy["mode"] = "supported-upgrades" + policy["automatic_sources"] = None (root / gate.MANIFEST).write_text(json.dumps(policy)) - with pytest.raises(gate.GateError, match="matrix disagree"): + with pytest.raises(gate.GateError, match="nonempty matrix"): gate.load_policy(root) @@ -130,17 +139,16 @@ def test_schema_owners_outside_persistence_need_compatibility_review(release, re def supported_case(root, policy): - suite = root / "tests/release_upgrades/test_example.py" - suite.parent.mkdir(parents=True) - suite.write_text("# This fixture is only used to exercise gate failures.\n") policy["mode"] = "supported-upgrades" + policy["automatic_sources"] = None policy["cases"] = [ { "source_version": "1.3.0", - "backend": "sqlite", + "backend": backend, "configuration": "full-server", - "suite": suite.relative_to(root).as_posix(), + "suite": "tests/release_upgrades/test_example.py", } + for backend in ("sqlite", "seekdb", "oceanbase") ] (root / gate.MANIFEST).write_text(json.dumps(policy)) gate.check_docs(root, policy, write=True) @@ -157,9 +165,12 @@ def fail(*_args): raise gate.GateError(blocker) monkeypatch.setattr(gate, "run_case", fail) - with pytest.raises(gate.GateError, match=blocker): + with pytest.raises(gate.GateError, match="Required upgrade acceptance failed"): gate.gate(root, wheel, root / "evidence") - assert json.loads((root / "evidence/result.json").read_text())["status"] == "failed" + evidence = json.loads((root / "evidence/result.json").read_text()) + assert evidence["status"] == "failed" + assert len(evidence["cases"]) == 3 + assert all(case["status"] == "failed" and blocker in case["error"] for case in evidence["cases"]) def test_missing_suite_cannot_be_advertised(release): @@ -240,7 +251,11 @@ def automatic_policy(policy): "strategy": "fixed-version", "source_version": "1.3.0", "profiles": [ - {"backend": backend, "configuration": "full-server-no-vector", "suite": None} + { + "backend": backend, + "configuration": "full-server-no-vector", + "suite": "tests/release_upgrades/test_example.py", + } for backend in ("sqlite", "seekdb", "oceanbase") ], } @@ -257,7 +272,7 @@ def test_fixed_source_tracks_each_target_on_all_three_backends(release, monkeypa assert [(case["source_version"], case["backend"]) for case in resolved["candidate_cases"]] == [ ("1.3.0", backend) for backend in ("sqlite", "seekdb", "oceanbase") ] - assert resolved["required_cases"] == [] # Candidate selection does not establish upgrade support. + assert resolved["required_cases"] == resolved["candidate_cases"] assert all(case["source_artifact"]["sha256"] == "a" * 64 for case in resolved["candidate_cases"]) @@ -321,7 +336,7 @@ def unavailable(): def test_switching_to_supported_upgrades_requires_all_backend_suites(release): root, _, policy = release automatic_policy(policy) - policy["mode"] = "supported-upgrades" + policy["automatic_sources"]["profiles"][0]["suite"] = None (root / gate.MANIFEST).write_text(json.dumps(policy)) with pytest.raises(gate.GateError, match="every backend profile"): gate.load_policy(root) @@ -383,14 +398,10 @@ def test_upgrade_jobs_require_real_backends_and_always_clean_up(workflow_name, j i for i, step in enumerate(steps) if step.get("uses") == "./.github/actions/setup-upgrade-oceanbase" ) gate_index = next(i for i, step in enumerate(steps) if "run_upgrade_release_gate.py" in step.get("run", "")) - regression_index = next( - i for i, step in enumerate(steps) if "ci_legacy_upgrade_rejection.py" in step.get("run", "") - ) - assert setup_index < gate_index < regression_index - for index in (setup_index, gate_index, regression_index): + assert setup_index < gate_index + for index in (setup_index, gate_index): assert "if" not in steps[index] and not steps[index].get("continue-on-error") - assert "for backend in sqlite seekdb oceanbase; do" in steps[regression_index]["run"] - assert "--respect-policy" in steps[regression_index]["run"] + assert not any("ci_legacy_upgrade_rejection.py" in step.get("run", "") for step in steps) assert "/pc_upgrade_test?" in job["env"]["PC_UPGRADE_OCEANBASE_URL"] assert "/pc_upgrade_fresh?" in job["env"]["PC_UPGRADE_OCEANBASE_FRESH_URL"] cleanup = next(step for step in steps if "docker rm --force upgrade-oceanbase" in step.get("run", "")) @@ -407,3 +418,40 @@ def test_oceanbase_fixture_selects_the_oceanbase_server_profile(tmp_path, monkey settings = ServerSettings() assert settings.database.kind == "oceanbase" assert settings.database.url.get_secret_value() == url + + +@pytest.mark.parametrize("mode", ["fresh-install-only", "upgrade-required", "supported-upgrades"]) +def test_declarations_cannot_bypass_missing_backend_or_suite(release, mode): + root, _, policy = release + policy["mode"] = mode + policy["automatic_sources"]["profiles"] = policy["automatic_sources"]["profiles"][:2] + (root / gate.MANIFEST).write_text(json.dumps(policy)) + with pytest.raises(gate.GateError): + gate.load_policy(root) + + +def test_fresh_install_only_is_not_an_upgrade_escape_hatch(release): + root, _, policy = release + policy["mode"] = "fresh-install-only" + (root / gate.MANIFEST).write_text(json.dumps(policy)) + with pytest.raises(gate.GateError, match="mode"): + gate.load_policy(root) + + +def test_failed_backend_does_not_skip_other_required_backends(release, monkeypatch): + root, wheel, _ = release + executed = [] + + def execute(_root, _wheel, case, _output): + executed.append(case["backend"]) + if case["backend"] == "sqlite": + raise gate.GateError("migration_required") + return {"case": case, "status": "passed"} + + monkeypatch.setattr(gate, "run_case", execute) + with pytest.raises(gate.GateError, match="failed: sqlite"): + gate.gate(root, wheel, root / "evidence") + assert executed == ["sqlite", "seekdb", "oceanbase"] + result = json.loads((root / "evidence/result.json").read_text()) + assert result["status"] == "failed" + assert [case["status"] for case in result["cases"]] == ["failed", "passed", "passed"] From 7941a900ec1b592c8d4a00bbe3e06185195190dc Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 22 Oct 2026 15:35:59 +0800 Subject: [PATCH 29/94] test(release): use embedded-compatible Server roles --- scripts/upgrade_release_gate.py | 2 ++ tests/release_upgrades/test_full_server.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/upgrade_release_gate.py b/scripts/upgrade_release_gate.py index 0ba593cd7f..7122864f33 100644 --- a/scripts/upgrade_release_gate.py +++ b/scripts/upgrade_release_gate.py @@ -312,6 +312,8 @@ def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dic "--confcutdir", str(root / "tests/release_upgrades"), "--strict-markers", + "-p", + "no:cacheprovider", "-o", "xfail_strict=true", "-ra", diff --git a/tests/release_upgrades/test_full_server.py b/tests/release_upgrades/test_full_server.py index 7212eec944..bbd655c5de 100644 --- a/tests/release_upgrades/test_full_server.py +++ b/tests/release_upgrades/test_full_server.py @@ -124,7 +124,7 @@ def environment_for(case, *, fresh=False): "POWERCONTEXT_SERVER_AUTH_TOKEN": TOKEN, "POWERCONTEXT_SERVER_ACCESS_MODE": "enforced", "POWERCONTEXT_SERVER_CURSOR_SIGNING_SECRET": "isolated-upgrade-cursor-signing-key-0001", - "POWERCONTEXT_SERVER_RUNTIME_ARTIFACT_PROCESSING_ROLE": "api", + "POWERCONTEXT_SERVER_RUNTIME_ARTIFACT_PROCESSING_ROLE": "all", "POWERCONTEXT_SERVER_MCP_ENABLED": "false", "NO_PROXY": "localhost,127.0.0.1", "no_proxy": "localhost,127.0.0.1", From 10efc32f221717944a80da0fc91d228e1e663f36 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 22 Oct 2026 16:10:23 +0800 Subject: [PATCH 30/94] fix(release): normalize review hashes and validate all artifacts --- .github/workflows/master.yml | 4 +- .github/workflows/release.yml | 2 +- Makefile | 6 +- docs/en/development/releasing.md | 17 ++- docs/zh/development/releasing.md | 14 ++- scripts/run_upgrade_release_gate.py | 122 +++++++++++++++++-- scripts/upgrade_release_gate.py | 6 +- tests/release_upgrades/README.md | 14 ++- tests/test_upgrade_publication.py | 182 ++++++++++++++++++++++++++++ tests/test_upgrade_release_gate.py | 34 ++++++ 10 files changed, 380 insertions(+), 21 deletions(-) create mode 100644 tests/test_upgrade_publication.py diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index eb4ac2e4a6..76f4f12789 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -37,9 +37,9 @@ jobs: - name: Verify upgrade policy and gate regressions run: | make upgrade-policy-check - uv run pytest -q tests/test_upgrade_release_gate.py + uv run pytest -q tests/test_upgrade_release_gate.py tests/test_upgrade_publication.py - name: Build the target artifact - run: uv build --wheel --out-dir dist + run: uv build --out-dir dist - name: Start the isolated upgrade OceanBase environment uses: ./.github/actions/setup-upgrade-oceanbase diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2dddbb90d0..792b68c168 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -110,7 +110,7 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: release-dists - path: dist/* + path: ${{ runner.temp }}/upgrade-release-evidence/validated-dist/* if-no-files-found: error retention-days: 30 diff --git a/Makefile b/Makefile index 6f968cea9d..19482ef3d3 100644 --- a/Makefile +++ b/Makefile @@ -243,7 +243,7 @@ upgrade-policy-check: ## Verify the declared upgrade matrix, persistence review, @uv run python scripts/upgrade_release_gate.py .PHONY: upgrade-release-test -upgrade-release-test: ## Run installed-release acceptance for the single wheel in dist (fresh-install-only makes no upgrade claim). +upgrade-release-test: ## Validate distributions in dist and require successful installed-release upgrades. @uv run python scripts/run_upgrade_release_gate.py .PHONY: clean-build @@ -253,9 +253,7 @@ clean-build: ## Clean build artifacts .PHONY: publish publish: ## Publish after release upgrade acceptance; UPGRADE_RELEASE_NOTES must name the reviewed release notes file. - @uv run python scripts/run_upgrade_release_gate.py --release --release-notes "$$UPGRADE_RELEASE_NOTES" - @echo "🚀 Publishing." - @uv publish dist/* + @uv run python scripts/run_upgrade_release_gate.py --release --publish --release-notes "$$UPGRADE_RELEASE_NOTES" .PHONY: build-and-publish build-and-publish: build publish ## Build and publish. diff --git a/docs/en/development/releasing.md b/docs/en/development/releasing.md index db765663cc..1f76f902e9 100644 --- a/docs/en/development/releasing.md +++ b/docs/en/development/releasing.md @@ -144,16 +144,29 @@ The suite contract and evidence requirements are in `tests/release_upgrades/READ `make check` and `make release-check` validate this policy and the bilingual installation notices. Changes to managed persistence inputs require a compatibility review with an updated digest and rationale. A digest is a -review trigger, not evidence that a migration is correct. `make version-bump` also updates the policy target; +review trigger, not evidence that a migration is correct. Managed text hashes normalize CRLF to LF so checkout +line endings do not require a new review; distribution hashes always cover the exact archive bytes. `make version-bump` also updates the policy target; regenerate the support notices with `uv run python scripts/upgrade_release_gate.py --write-docs` after matrix changes. -After building exactly one wheel into `dist/`, execute the declared upgrade cases: +Build exactly one wheel and its matching source distribution into `dist/`, then execute the declared upgrade cases: ```bash make upgrade-release-test uv run python scripts/run_upgrade_release_gate.py --release --release-notes /path/to/release-notes.md ``` +The publication input directory must contain exactly one wheel and one matching `.tar.gz` source distribution; +leftover distributions and unexpected files fail validation (`uv`'s `.gitignore` is not an artifact). The runner checks +source metadata and builds a wheel from the source archive, comparing all member contents except the derived +`RECORD` listing. A stale source archive with the same version therefore cannot bypass validation. Archive timestamps +and compression do not affect this comparison. A build-tool change that alters wheel contents requires rebuilding +the complete pair, not ignoring the mismatch. + +The runner tests a private snapshot, records exact artifact SHA-256s in `publication.json`, and exports only passing +files to the evidence directory's `validated-dist/`. CI uploads that set. `make publish` uses `--release --publish` to +upload those exact paths in the same invocation; subsequent changes to the original `dist/` cannot join the upload. +No packages are uploaded by validation commands without `--publish`. + The release check requires the actual wheel version to match the policy and the release notes to include the exact English support declaration from the installation guide. It installs each exact source release and the target wheel in separate environments. Missing tests, skipped backends, failures, and missing evidence block the check. Static diff --git a/docs/zh/development/releasing.md b/docs/zh/development/releasing.md index 6c5af20b7d..4f65ff503b 100644 --- a/docs/zh/development/releasing.md +++ b/docs/zh/development/releasing.md @@ -134,16 +134,26 @@ DSH README 中说明某项能力已包含在 `1.1.0` 的文字同样属于历史 验收契约和证据要求见仓库中的 `tests/release_upgrades/README.md`。 `make check` 和 `make release-check` 会核对策略及双语安装说明。受管持久化文件变化时，必须审查兼容性影响， -更新摘要和理由。摘要用于触发审查，本身不证明迁移正确。`make version-bump` 同时更新策略中的目标版本； +更新摘要和理由。摘要用于触发审查，本身不证明迁移正确。受管文本的哈希会将 CRLF 归一为 LF，避免检出行尾 +差异误报兼容性变更；发行包哈希仍覆盖完整原始字节。`make version-bump` 同时更新策略中的目标版本； 支持矩阵变化后，运行 `uv run python scripts/upgrade_release_gate.py --write-docs` 更新安装说明中的支持范围。 -将唯一的目标 wheel 构建到 `dist/` 后，执行声明的升级组合验收： +将唯一的目标 wheel 和配套源码包构建到 `dist/` 后，执行声明的升级组合验收： ```bash make upgrade-release-test uv run python scripts/run_upgrade_release_gate.py --release --release-notes /path/to/release-notes.md ``` +发布输入目录必须只有一个 wheel 和一个配套的 `.tar.gz` 源码包；残留发行包和其他文件都会使检查失败， +`uv` 生成的 `.gitignore` 不作为发行产物。门禁检查源码包元数据，并从该源码包实际重建 wheel，逐项比较归档中的 +文件内容，仅排除派生的 `RECORD` 哈希清单。因此，同版本但内容过期的源码包也无法绕过检查。归档时间戳和压缩 +方式不影响比较；构建工具变化导致内容不一致时，需要重新构建整组产物，不能忽略差异。 + +门禁验收输入文件的独立副本，在 `publication.json` 中记录实际 SHA-256，并且只有全部通过才将文件输出到证据目录 +下的 `validated-dist/`，供 CI 上传。`make publish` 使用 `--release --publish` 在同一次调用中上传这些确切路径； +验收期间原 `dist/` 的变化不会混入上传。未指定 `--publish` 的验收命令不会发布软件包。 + 发布检查要求实际 wheel 版本与策略一致，发布说明包含安装指南中的完整英文支持声明。每个组合分别安装准确的 旧版发行包和目标 wheel。测试缺失、后端跳过、执行失败或证据缺失都会阻断检查。仅通过静态策略检查不表示 存在升级路径。普通 PR 构建允许使用开发版本；`--release` 必须验证最终 tag 构建的 wheel，不能覆盖版本号。 diff --git a/scripts/run_upgrade_release_gate.py b/scripts/run_upgrade_release_gate.py index b585530f15..6dd7467d66 100644 --- a/scripts/run_upgrade_release_gate.py +++ b/scripts/run_upgrade_release_gate.py @@ -12,14 +12,118 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Select the built wheel and run the same upgrade gate locally and in CI.""" +"""Validate the complete distribution set and gate its publication on upgrades.""" import argparse +import hashlib import json +import shutil +import subprocess +import tarfile import tempfile +import zipfile +from email.parser import BytesParser from pathlib import Path -from upgrade_release_gate import ROOT, GateError, gate, require +from upgrade_release_gate import ROOT, GateError, gate, require, sha256, wheel_version + + +def wheel_contents(wheel: Path) -> dict[str, str]: + """Compare installed payloads, excluding only the derived RECORD hash listing.""" + with zipfile.ZipFile(wheel) as archive: + names = [item.filename for item in archive.infolist() if not item.is_dir()] + require(len(set(names)) == len(names), "Duplicate wheel members") + return { + name: hashlib.sha256(archive.read(name)).hexdigest() + for name in names + if not name.endswith(".dist-info/RECORD") + } + + +def validate_sdist(sdist: Path, wheel: Path, work: Path) -> None: + expected = wheel_version(wheel) + require(sdist.name == f"powercontext-{expected}.tar.gz", "Source distribution version differs from the wheel") + with tarfile.open(sdist, "r:gz") as archive: + names = [item for item in archive.getmembers() if item.name == f"powercontext-{expected}/PKG-INFO"] + require(len(names) == 1 and names[0].isfile(), "Source distribution requires one root PKG-INFO") + stream = archive.extractfile(names[0]) + if stream is None: + raise GateError("Cannot read source distribution metadata") # noqa: TRY003 -- operator diagnostic + metadata = BytesParser().parsebytes(stream.read()) + require( + metadata["Name"] == "powercontext" and metadata["Version"] == expected, + "Source distribution metadata differs from the wheel", + ) + rebuilt = work / "rebuilt" + rebuilt.mkdir() + with (work / "sdist-build.log").open("w", encoding="utf-8") as log: + subprocess.run( # noqa: S603 -- reviewed local distribution and fixed build tool; no shell + ["uv", "build", "--wheel", str(sdist), "--out-dir", str(rebuilt)], # noqa: S607 -- project tool on PATH + stdout=log, + stderr=subprocess.STDOUT, + timeout=600, + check=True, + ) + wheels = list(rebuilt.glob("*.whl")) + require(len(wheels) == 1, "Source distribution must build exactly one wheel") + require(wheels[0].name == wheel.name, "Source distribution produces a different wheel filename") + require( + wheel_contents(wheels[0]) == wheel_contents(wheel), + "Source distribution builds different wheel contents; rebuild the complete publication set", + ) + + +def stage_distributions(dist: Path, staging: Path, *, release: bool) -> tuple[Path, list[Path]]: + """Freeze all publication inputs before testing; later dist changes cannot be uploaded.""" + files = sorted(path for path in dist.iterdir() if path.name != ".gitignore") + require( + all(path.is_file() and not path.is_symlink() for path in files), "Distribution inputs must be regular files" + ) + wheels = [path for path in files if path.suffix == ".whl"] + sources = [path for path in files if path.name.endswith(".tar.gz")] + require(len(wheels) == 1, "Build exactly one release wheel before running upgrade acceptance") + require(len(sources) <= 1, "Build at most one source distribution") + require(set(files) == set(wheels + sources), "Unexpected files in publication directory") + require(not release or len(sources) == 1, "Release publication requires one wheel and one source distribution") + staging.mkdir() + staged = [] + for path in files: + copied = staging / path.name + shutil.copyfile(path, copied) + staged.append(copied) + wheel = staging / wheels[0].name + if sources: + validate_sdist(staging / sources[0].name, wheel, staging.parent) + return wheel, staged + + +def accept_distributions(dist: Path, output: Path, *, release: bool, release_notes: str | None) -> list[Path]: + # A private snapshot prevents a leftover or concurrent dist write from joining + # publication after acceptance. Keep the same bytes through testing and upload. + require(not output.exists(), "Use a fresh evidence directory") + with tempfile.TemporaryDirectory(prefix="pc-release-inputs-") as temporary: + work = Path(temporary) + try: + wheel, files = stage_distributions(dist, work / "inputs", release=release) + expected = {path.name: sha256(path) for path in files} + gate(ROOT, wheel, output, release=release, release_notes=release_notes) + require({path.name: sha256(path) for path in files} == expected, "Distribution changed during acceptance") + validated = output / "validated-dist" + validated.mkdir() + accepted = [] + for path in files: + destination = validated / path.name + shutil.copyfile(path, destination) + require(sha256(destination) == expected[path.name], "Validated distribution copy changed") + accepted.append(destination) + (output / "publication.json").write_text( + json.dumps({"artifacts": expected}, indent=2) + "\n", encoding="utf-8" + ) + return accepted + finally: + if (work / "sdist-build.log").is_file(): + output.mkdir(parents=True, exist_ok=True) + shutil.copyfile(work / "sdist-build.log", output / "sdist-build.log") def main() -> None: @@ -27,20 +131,22 @@ def main() -> None: parser.add_argument("--dist", type=Path, default=Path("dist")) parser.add_argument("--output", type=Path) parser.add_argument("--release", action="store_true") + parser.add_argument( + "--publish", action="store_true", help="Upload only this run's validated files; requires --release" + ) notes = parser.add_mutually_exclusive_group() notes.add_argument("--release-event", type=Path) notes.add_argument("--release-notes", type=Path) args = parser.parse_args() try: - wheels = list(args.dist.glob("*.whl")) - require(len(wheels) == 1, "Build exactly one release wheel before running upgrade acceptance") + require(not args.publish or args.release, "--publish requires --release") release_notes = None if args.release_event: release_notes = json.loads(args.release_event.read_text(encoding="utf-8"))["release"]["body"] or "" elif args.release_notes: release_notes = args.release_notes.read_text(encoding="utf-8") - output = args.output or Path(tempfile.mkdtemp(prefix="pc-upgrade-evidence-")) / "report" - gate(ROOT, wheels[0].resolve(), output.resolve(), release=args.release, release_notes=release_notes) + output = (args.output or Path(tempfile.mkdtemp(prefix="pc-upgrade-evidence-")) / "report").resolve() + accepted = accept_distributions(args.dist.resolve(), output, release=args.release, release_notes=release_notes) result = json.loads((output / "result.json").read_text(encoding="utf-8")) print( f"Upgrade gate passed: {result['mode']}; " @@ -48,7 +154,9 @@ def main() -> None: f"{len(result['cases'])} accepted upgrade combinations. " f"Evidence: {output / 'result.json'}" ) - except (GateError, OSError, ValueError) as error: + if args.publish: + subprocess.run(["uv", "publish", *map(str, accepted)], check=True) # noqa: S603, S607 -- project tool, exact validated files + except (GateError, OSError, ValueError, subprocess.SubprocessError, tarfile.TarError, zipfile.BadZipFile) as error: raise SystemExit(f"Upgrade gate failed: {error}") from error # noqa: TRY003 -- operator diagnostic diff --git a/scripts/upgrade_release_gate.py b/scripts/upgrade_release_gate.py index 7122864f33..b2255ad72e 100644 --- a/scripts/upgrade_release_gate.py +++ b/scripts/upgrade_release_gate.py @@ -76,10 +76,11 @@ def managed_digest(root: Path) -> str: This deliberately over-approximates schema changes. It is a review trigger, not a claim to prove SQL or data-format compatibility from Python syntax. + Normalize checkout line endings; artifact hashes must retain their exact bytes. """ inputs = [root / relative for relative in MANAGED_INPUTS] files = { - path.relative_to(root).as_posix(): sha256(path) + path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes().replace(b"\r\n", b"\n")).hexdigest() for location in inputs for path in (location.rglob("*") if location.is_dir() else [location]) if path.is_file() and path.suffix in {".py", ".sql", ".json"} @@ -107,7 +108,8 @@ def load_policy(root: Path) -> dict[str, Any]: isinstance(review["rationale"], str) and review["rationale"].strip(), "Schema compatibility rationale required" ) require( - review["persistence_sha256"] == managed_digest(root), "Persistence changed: update the compatibility review" + review["persistence_sha256"] == managed_digest(root), + "Persistence changed (file set or content; CRLF/LF normalized): update the compatibility review", ) cases = policy["cases"] require(isinstance(cases, list), "Upgrade cases must be a list") diff --git a/tests/release_upgrades/README.md b/tests/release_upgrades/README.md index a30d9d1b9b..d5344831a2 100644 --- a/tests/release_upgrades/README.md +++ b/tests/release_upgrades/README.md @@ -93,7 +93,7 @@ Server identity and authorization, and shared limits used in table definitions. `MANAGED_INPUTS` when introducing them. The digest intentionally includes non-schema code: it forces a review of changes, but does not infer semantic compatibility or replace real upgrade tests. -Build exactly one wheel into `dist/`, then run `make upgrade-release-test`. Before release, additionally run +Build one wheel and its matching `.tar.gz` source distribution into `dist/`, then run `make upgrade-release-test`. Before release, additionally run `uv run python scripts/run_upgrade_release_gate.py --release --release-notes /path/to/notes.md`. The notes must contain the exact English support declaration generated in the installation guide. The release workflow performs this check against the GitHub event's notes and gates both PyPI publication and release assets on the same tested wheel. @@ -108,3 +108,15 @@ not the earlier manual creation of a GitHub Release; run the prepublication comm workflows do not invoke it as upgrade acceptance. An expected rejection or its `--respect-policy` result can never replace a successful positive upgrade. The gate attempts every required backend even after an earlier case fails, then fails overall and archives each case's logs and result. Empty matrices and missing suites are errors. + +## Publication artifacts + +The runner validates every distribution input. Publication requires exactly one wheel and one source archive with +matching names and metadata. It rebuilds the source archive with `uv build --wheel` and compares wheel member bytes, +excluding only the derived `RECORD` file. A same-version stale source archive fails just like an older-version one. +PRs may test a wheel alone, but the PR workflow builds both formats to exercise this validation too. + +Acceptance uses a private copy of those files. Only a passing run creates `validated-dist/` and `publication.json` +in its evidence directory. CI uploads that directory; local `make publish` calls the runner with `--release --publish` +and uploads its exact returned paths, never a fresh glob of `dist/`. Failed acceptance produces no publication set. +Managed persistence hashes normalize CRLF/LF checkout line endings; artifact SHA-256s remain byte-exact. diff --git a/tests/test_upgrade_publication.py b/tests/test_upgrade_publication.py new file mode 100644 index 0000000000..3065054d84 --- /dev/null +++ b/tests/test_upgrade_publication.py @@ -0,0 +1,182 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exercise publication inputs with actual offline sdist builds; intercept uploads.""" + +import io +import json +import sys +import tarfile +import zipfile +from pathlib import Path + +import pytest +import run_upgrade_release_gate as runner +from upgrade_release_gate import GateError, sha256 + + +def write_wheel(path, content=b"current implementation", *, version="2.0.0"): + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("powercontext/__init__.py", content) + archive.writestr(f"powercontext-{version}.dist-info/METADATA", f"Name: powercontext\nVersion: {version}\n") + archive.writestr(f"powercontext-{version}.dist-info/WHEEL", "Wheel-Version: 1.0\nTag: py3-none-any\n") + archive.writestr(f"powercontext-{version}.dist-info/RECORD", "") + + +def write_source(path, content=b"current implementation", *, version="2.0.0", metadata_version=None): + # No external build dependencies or network. uv invokes a real local PEP 517 + # backend so stale same-version source changes alter the rebuilt wheel. + backend = f"""import pathlib, zipfile + +def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): + name = "powercontext-{version}-py3-none-any.whl" + with zipfile.ZipFile(pathlib.Path(wheel_directory) / name, "w") as archive: + archive.writestr("powercontext/__init__.py", pathlib.Path("payload.py").read_bytes()) + archive.writestr("powercontext-{version}.dist-info/METADATA", "Name: powercontext\\nVersion: {version}\\n") + archive.writestr("powercontext-{version}.dist-info/WHEEL", "Wheel-Version: 1.0\\nTag: py3-none-any\\n") + archive.writestr("powercontext-{version}.dist-info/RECORD", "") + return name +""" + files = { + "PKG-INFO": f"Name: powercontext\nVersion: {metadata_version or version}\n".encode(), + "pyproject.toml": b'[build-system]\nrequires = []\nbuild-backend = "fixture_backend"\nbackend-path = ["."]\n', + "fixture_backend.py": backend.encode(), + "payload.py": content, + } + with tarfile.open(path, "w:gz") as archive: + for name, data in files.items(): + member = tarfile.TarInfo(f"powercontext-{version}/{name}") + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + + +@pytest.fixture +def publication(tmp_path, monkeypatch): + monkeypatch.setenv("UV_OFFLINE", "true") + dist = tmp_path / "dist" + dist.mkdir() + wheel = dist / "powercontext-2.0.0-py3-none-any.whl" + source = dist / "powercontext-2.0.0.tar.gz" + write_wheel(wheel) + write_source(source) + (dist / ".gitignore").write_text("*") + return dist, wheel, source, tmp_path / "evidence" + + +@pytest.mark.parametrize( + "invalid", ["old-version", "metadata-mismatch", "same-version-stale-content", "extra-file", "missing-sdist"] +) +def test_invalid_publication_set_never_reaches_acceptance(publication, monkeypatch, invalid): + dist, _wheel, source, output = publication + if invalid == "old-version": + source.unlink() + write_source(dist / "powercontext-1.3.0.tar.gz", version="1.3.0") + elif invalid == "metadata-mismatch": + write_source(source, metadata_version="1.3.0") + elif invalid == "same-version-stale-content": + write_source(source, b"old implementation") + elif invalid == "extra-file": + (dist / "leftover.zip").write_bytes(b"unchecked archive") + else: + source.unlink() + + def unexpected_gate(*_args, **_kwargs): + pytest.fail("Invalid publication inputs must fail before upgrade acceptance") + + monkeypatch.setattr(runner, "gate", unexpected_gate) + with pytest.raises(GateError): + runner.accept_distributions(dist, output, release=True, release_notes="fixture") + assert not (output / "validated-dist").exists() + + +def test_failed_upgrade_cannot_produce_publishable_files(publication, monkeypatch): + dist, _, _, output = publication + + def fail(*_args, **_kwargs): + raise GateError("migration_required") + + monkeypatch.setattr(runner, "gate", fail) + with pytest.raises(GateError, match="migration_required"): + runner.accept_distributions(dist, output, release=True, release_notes="fixture") + assert not (output / "validated-dist").exists() + + +def test_publish_uses_only_tested_snapshot_even_if_dist_changes(publication, monkeypatch): + dist, wheel, source, output = publication + expected = {path.name: sha256(path) for path in (wheel, source)} + notes = dist.parent / "notes.md" + notes.write_text("fixture release notes") + + def accept(_root, tested_wheel, evidence, **_kwargs): + assert sha256(tested_wheel) == expected[wheel.name] + # A concurrent build must not replace bytes being tested or introduce an + # unchecked package into the later upload glob. + write_wheel(wheel, b"replacement after validation") + write_source(source, b"replacement after validation") + write_source(dist / "powercontext-1.3.0.tar.gz", version="1.3.0") + evidence.mkdir() + (evidence / "result.json").write_text( + json.dumps({ + "mode": "supported-upgrades", + "resolved_matrix": {"candidate_cases": [1, 2, 3]}, + "cases": [1, 2, 3], + }) + ) + + monkeypatch.setattr(runner, "gate", accept) + original_run = runner.subprocess.run + uploads = [] + + def intercept_upload(command, **kwargs): + if command[:2] == ["uv", "publish"]: + uploads.extend(Path(name) for name in command[2:]) + return None + return original_run(command, **kwargs) + + monkeypatch.setattr(runner.subprocess, "run", intercept_upload) + monkeypatch.setattr( + sys, + "argv", + [ + "run_upgrade_release_gate.py", + "--dist", + str(dist), + "--output", + str(output), + "--release", + "--publish", + "--release-notes", + str(notes), + ], + ) + runner.main() + assert {path.name: sha256(path) for path in uploads} == expected + assert all(path.parent == output / "validated-dist" for path in uploads) + assert json.loads((output / "publication.json").read_text())["artifacts"] == expected + + +def test_wheel_only_pr_build_still_runs_upgrade_acceptance(publication, monkeypatch): + dist, wheel, source, output = publication + source.unlink() + tested = [] + + def accept(_root, tested_wheel, evidence, **kwargs): + tested.append(sha256(tested_wheel)) + assert not kwargs["release"] + evidence.mkdir() + + monkeypatch.setattr(runner, "gate", accept) + accepted = runner.accept_distributions(dist, output, release=False, release_notes=None) + assert tested == [sha256(wheel)] + assert [path.name for path in accepted] == [wheel.name] diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py index 66a77dc776..49fc331039 100644 --- a/tests/test_upgrade_release_gate.py +++ b/tests/test_upgrade_release_gate.py @@ -376,6 +376,7 @@ def test_publishing_jobs_depend_on_nonoptional_preupload_upgrade_gate(): assert "--release --release-event" in steps[index]["run"] upload = next(i for i, step in enumerate(steps) if step.get("with", {}).get("name") == "release-dists") assert index < upload + assert steps[upload]["with"]["path"] == "${{ runner.temp }}/upgrade-release-evidence/validated-dist/*" for name in ("pypi-publish", "release-assets"): assert workflow["jobs"][name]["needs"] == "release-build" @@ -455,3 +456,36 @@ def execute(_root, _wheel, case, _output): result = json.loads((root / "evidence/result.json").read_text()) assert result["status"] == "failed" assert [case["status"] for case in result["cases"]] == ["failed", "passed", "passed"] + + +@pytest.mark.parametrize( + "relative", + [ + "src/powercontext/builtin/persistence/tables.py", + "src/powercontext/builtin/persistence/revision.sql", + "src/powercontext/builtin/code/layout.json", + "src/powercontext/server/authz/repository.py", + "src/powercontext/server/identity.py", + "src/powercontext/limits.py", + ], +) +def test_checkout_line_endings_preserve_compatibility_review(release, relative): + root, _, policy = release + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"first line\nsecond line\n") + policy["schema_review"]["persistence_sha256"] = gate.managed_digest(root) + (root / gate.MANIFEST).write_text(json.dumps(policy)) + path.write_bytes(b"first line\r\nsecond line\r\n") + assert gate.load_policy(root)["schema_review"] == policy["schema_review"] + path.write_bytes(b"first line\r\nchanged content\r\n") + with pytest.raises(gate.GateError, match="file set or content; CRLF/LF normalized"): + gate.load_policy(root) + + +def test_artifact_hashes_do_not_normalize_bytes(tmp_path): + artifact = tmp_path / "artifact" + artifact.write_bytes(b"payload\n") + before = gate.sha256(artifact) + artifact.write_bytes(b"payload\r\n") + assert gate.sha256(artifact) != before From 8b7ec35a71208afba5e21676c769b78eb98b5bbd Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 16:17:53 +0800 Subject: [PATCH 31/94] feat(migrations): recognize complete released SQLite Server schemas Freeze the full schema captured from the installed public 1.3.0 wheel and distinguish it from the limited artifact migration prototype. Read-only maintenance commands identify the source but keep writes blocked until a complete path is registered. Constraint: No schema stamps, data writes, readiness claims or changes to released migration resources. Scope-risk: Exact recognition covers only the captured SQLite shape; other layouts remain unknown. Tested: Linux baseline/runner suites 55 passed; Windows baseline suite 11 passed; actual installed-release snapshot unchanged after plan/verify/apply; full Linux ty for Python 3.11 and changed-file pre-commit hooks passed. --- .../server_baselines/1.3.0/sqlite.json | 585 ++++++++++++++++++ .../server_baselines/README.md | 36 ++ .../migrations/server_baselines.py | 60 ++ .../builtin/persistence/migrations/sqlite.py | 8 + .../test_server_migration_baselines.py | 126 ++++ 5 files changed, 815 insertions(+) create mode 100644 src/powercontext/builtin/persistence/migration_resources/server_baselines/1.3.0/sqlite.json create mode 100644 src/powercontext/builtin/persistence/migration_resources/server_baselines/README.md create mode 100644 src/powercontext/builtin/persistence/migrations/server_baselines.py create mode 100644 tests/builtin/persistence/test_server_migration_baselines.py diff --git a/src/powercontext/builtin/persistence/migration_resources/server_baselines/1.3.0/sqlite.json b/src/powercontext/builtin/persistence/migration_resources/server_baselines/1.3.0/sqlite.json new file mode 100644 index 0000000000..ab00fc2b4c --- /dev/null +++ b/src/powercontext/builtin/persistence/migration_resources/server_baselines/1.3.0/sqlite.json @@ -0,0 +1,585 @@ +{ + "format": "powercontext.server-baseline.v1", + "release": "1.3.0", + "backend": "sqlite", + "source_artifact": { + "filename": "powercontext-1.3.0-py3-none-any.whl", + "sha256": "8da7983763f3a6ab80b413c770a4973a46c557a54e1a1fa675099e58d47305e5", + "uploaded_at": "2026-10-12T03:55:56.682348Z" + }, + "capture": { + "python": "3.14.3", + "sqlite": "3.50.4", + "sqlalchemy": "2.1.6", + "access_mode": "enforced", + "processing_role": "all", + "initialization": "Installed release Server; no model or embedding provider" + }, + "schema_fingerprint": "11e8770068a5fe32dfb25681564dbbe586ba0fd0238ed01adf1b09dde8a49f22", + "inventory": [ + [ + "index", + "ix_pc_artifact_tags_family_key", + "pc_artifact_tags", + "CREATE INDEX ix_pc_artifact_tags_family_key ON pc_artifact_tags (scope_id, family, tag_key_hash, target_type, artifact_id, target_id)" + ], + [ + "index", + "ix_pc_artifact_tags_key", + "pc_artifact_tags", + "CREATE INDEX ix_pc_artifact_tags_key ON pc_artifact_tags (scope_id, tag_key_hash, family, target_type, artifact_id, target_id)" + ], + [ + "index", + "ix_pc_decision_observations_operation", + "pc_decision_observations", + "CREATE INDEX ix_pc_decision_observations_operation ON pc_decision_observations (scope_id, operation_id, created_at)" + ], + [ + "index", + "ix_pc_decision_observations_scope_created", + "pc_decision_observations", + "CREATE INDEX ix_pc_decision_observations_scope_created ON pc_decision_observations (scope_id, created_at, observation_id)" + ], + [ + "index", + "ix_pc_dream_dispatch", + "pc_dream_runs", + "CREATE INDEX ix_pc_dream_dispatch ON pc_dream_runs (scope_id, operation, status, request_generation)" + ], + [ + "index", + "ix_pc_dream_list", + "pc_dream_runs", + "CREATE INDEX ix_pc_dream_list ON pc_dream_runs (scope_id, accepted_at, run_id)" + ], + [ + "index", + "ix_pc_processing_intent_binding_sequence", + "pc_artifact_processing_intents", + "CREATE INDEX ix_pc_processing_intent_binding_sequence ON pc_artifact_processing_intents (binding_name, pending_sequence)" + ], + [ + "index", + "ix_pc_recurrence_observation_key", + "pc_recurrence_observation", + "CREATE INDEX ix_pc_recurrence_observation_key ON pc_recurrence_observation (scope_id, signature_key_hash)" + ], + [ + "index", + "ix_pc_recurrence_observation_outcome", + "pc_recurrence_observation", + "CREATE INDEX ix_pc_recurrence_observation_outcome ON pc_recurrence_observation (scope_id, task_outcome_source_type, task_outcome_source_id, task_outcome_position)" + ], + [ + "index", + "ix_pc_recurrence_observation_revision", + "pc_recurrence_observation", + "CREATE INDEX ix_pc_recurrence_observation_revision ON pc_recurrence_observation (scope_id, family, artifact_id, revision)" + ], + [ + "table", + "pc_access_audit", + "pc_access_audit", + "CREATE TABLE pc_access_audit (\n\tcursor INTEGER NOT NULL, \n\tevent_id VARCHAR(64) NOT NULL, \n\toccurred_at VARCHAR(32) NOT NULL, \n\trequest_id VARCHAR(128), \n\ttransport VARCHAR(16) NOT NULL, \n\toperation VARCHAR(128) NOT NULL, \n\tprincipal_type VARCHAR(16) NOT NULL, \n\tprincipal_id VARCHAR(255) NOT NULL, \n\tprincipal_description TEXT, \n\tactor_type VARCHAR(16), \n\tactor_id VARCHAR(255), \n\tactor_description TEXT, \n\taction VARCHAR(64) NOT NULL, \n\tresource_type VARCHAR(16) NOT NULL, \n\tdeployment_id VARCHAR(128), \n\tscope_id VARCHAR(256), \n\tfamily VARCHAR(128), \n\tartifact_id VARCHAR(128), \n\tselector_type VARCHAR(32), \n\tselector_entry_id VARCHAR(128), \n\tallowed BOOLEAN NOT NULL, \n\treason_code VARCHAR(64) NOT NULL, \n\tpolicy_revision VARCHAR(64), \n\tmatched_subject_type VARCHAR(16), \n\tmatched_subject_id VARCHAR(255), \n\tmatched_subject_description TEXT, \n\tbinding_id VARCHAR(64), \n\ttarget_type VARCHAR(16), \n\ttarget_id VARCHAR(255), \n\ttarget_description TEXT, \n\trole VARCHAR(32), \n\texpected_version INTEGER, \n\tresult_version INTEGER, \n\tPRIMARY KEY (cursor), \n\tUNIQUE (event_id)\n)" + ], + [ + "table", + "pc_access_idempotency", + "pc_access_idempotency", + "CREATE TABLE pc_access_idempotency (\n\tactor_id VARCHAR(255) NOT NULL, \n\tidempotency_key_hash VARCHAR(64) NOT NULL, \n\toperation VARCHAR(64) NOT NULL, \n\tpayload_hash VARCHAR(64) NOT NULL, \n\tresult_binding_id VARCHAR(64), \n\tsecondary_binding_id VARCHAR(64), \n\tPRIMARY KEY (actor_id, idempotency_key_hash), \n\tCONSTRAINT uq_pc_access_idempotency_actor_key UNIQUE (actor_id, idempotency_key_hash)\n)" + ], + [ + "table", + "pc_access_owners", + "pc_access_owners", + "CREATE TABLE pc_access_owners (\n\towner_kind VARCHAR(16) NOT NULL, \n\tobject_key_hash VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128), \n\tcandidate_id VARCHAR(128), \n\ttarget_artifact_id VARCHAR(128), \n\tselector_type VARCHAR(32), \n\tselector_entry_id VARCHAR(128), \n\towner_type VARCHAR(16) NOT NULL, \n\towner_id VARCHAR(255) NOT NULL, \n\towner_description TEXT, \n\testablished_at VARCHAR(32), \n\tpolicy_revision VARCHAR(64), \n\tidempotency_key VARCHAR(255) NOT NULL, \n\tPRIMARY KEY (owner_kind, object_key_hash), \n\tCONSTRAINT ck_pc_access_owners_kind CHECK ((owner_kind = 'artifact' AND artifact_id IS NOT NULL AND candidate_id IS NULL AND target_artifact_id IS NULL AND established_at IS NOT NULL AND policy_revision IS NOT NULL) OR (owner_kind = 'candidate' AND candidate_id IS NOT NULL AND artifact_id IS NULL AND selector_type IS NULL AND selector_entry_id IS NULL AND established_at IS NULL AND policy_revision IS NULL))\n)" + ], + [ + "table", + "pc_access_relationship_heads", + "pc_access_relationship_heads", + "CREATE TABLE pc_access_relationship_heads (\n\tname VARCHAR(32) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tPRIMARY KEY (name), \n\tCONSTRAINT ck_pc_access_relationship_heads_revision_nonnegative CHECK (revision >= 0)\n)" + ], + [ + "table", + "pc_access_relationships", + "pc_access_relationships", + "CREATE TABLE pc_access_relationships (\n\tbinding_id VARCHAR(64) NOT NULL, \n\tsubject_type VARCHAR(16) NOT NULL, \n\tsubject_id VARCHAR(255) NOT NULL, \n\tsubject_description TEXT, \n\tresource_key_hash VARCHAR(64) NOT NULL, \n\tresource_type VARCHAR(16) NOT NULL, \n\tdeployment_id VARCHAR(128), \n\tscope_id VARCHAR(256), \n\tfamily VARCHAR(128), \n\tartifact_id VARCHAR(128), \n\tselector_type VARCHAR(32), \n\tselector_entry_id VARCHAR(128), \n\trole VARCHAR(32) NOT NULL, \n\tsingleton_key VARCHAR(64), \n\tgranted_by_type VARCHAR(16) NOT NULL, \n\tgranted_by_id VARCHAR(255) NOT NULL, \n\tgranted_by_description TEXT, \n\treason TEXT, \n\tcreated_at VARCHAR(32) NOT NULL, \n\texpires_at VARCHAR(32), \n\tstate VARCHAR(16) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tpolicy_revision VARCHAR(64) NOT NULL, \n\tidempotency_key VARCHAR(255) NOT NULL, \n\trevoked_at VARCHAR(32), \n\trevoked_by_type VARCHAR(16), \n\trevoked_by_id VARCHAR(255), \n\trevoked_by_description TEXT, \n\tPRIMARY KEY (binding_id), \n\tCONSTRAINT ck_pc_access_relationships_version_positive CHECK (version > 0), \n\tUNIQUE (singleton_key)\n)" + ], + [ + "table", + "pc_agent_skill_targets", + "pc_agent_skill_targets", + "CREATE TABLE pc_agent_skill_targets (\n\tscope_id VARCHAR(256) NOT NULL, \n\ttarget_id VARCHAR(64) NOT NULL, \n\tdisplay_name VARCHAR(128) NOT NULL, \n\tagent_kind VARCHAR(32) NOT NULL, \n\tinstallation_scope VARCHAR(16) NOT NULL, \n\tdelivery_mode VARCHAR(16) NOT NULL, \n\tinstallation_id VARCHAR(128), \n\tstate VARCHAR(16) NOT NULL, \n\tenrollment_token_digest VARCHAR(64), \n\tenrollment_expires_at DATETIME, \n\tcredential_subject VARCHAR(128), \n\tcredential_verifier VARCHAR(64), \n\treceiver_version VARCHAR(64), \n\tenvironment_fingerprint VARCHAR(64), \n\tmachine_hostname VARCHAR(255), \n\tworkspace_name VARCHAR(128), \n\tlast_seen_at DATETIME, \n\tgeneration BIGINT NOT NULL, \n\tcreated_at DATETIME NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, target_id), \n\tCONSTRAINT uq_pc_agent_skill_targets_installation UNIQUE (scope_id, agent_kind, installation_scope, installation_id), \n\tCONSTRAINT uq_pc_agent_skill_targets_enrollment_token UNIQUE (enrollment_token_digest), \n\tCONSTRAINT uq_pc_agent_skill_targets_credential_subject UNIQUE (credential_subject), \n\tCONSTRAINT uq_pc_agent_skill_targets_credential_verifier UNIQUE (credential_verifier), \n\tCONSTRAINT ck_pc_agent_skill_targets_agent_kind CHECK (agent_kind IN ('codex', 'claude_code')), \n\tCONSTRAINT ck_pc_agent_skill_targets_installation_scope CHECK (installation_scope IN ('project')), \n\tCONSTRAINT ck_pc_agent_skill_targets_delivery_mode CHECK (delivery_mode = 'agent_pull'), \n\tCONSTRAINT ck_pc_agent_skill_targets_state CHECK (state IN ('pending', 'active', 'revoked')), \n\tCONSTRAINT ck_pc_agent_skill_targets_state_payload CHECK ((state = 'pending' AND enrollment_token_digest IS NOT NULL AND enrollment_expires_at IS NOT NULL AND installation_id IS NULL AND credential_subject IS NULL AND credential_verifier IS NULL) OR (state = 'active' AND enrollment_token_digest IS NULL AND enrollment_expires_at IS NULL AND installation_id IS NOT NULL AND credential_subject IS NOT NULL AND credential_verifier IS NOT NULL) OR (state = 'revoked' AND enrollment_token_digest IS NULL AND enrollment_expires_at IS NULL AND credential_verifier IS NULL)), \n\tCONSTRAINT ck_pc_agent_skill_targets_generation_nonnegative CHECK (generation >= 0)\n)" + ], + [ + "table", + "pc_artifact_candidate_heads", + "pc_artifact_candidate_heads", + "CREATE TABLE pc_artifact_candidate_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tcandidate_id VARCHAR(128) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tstatus VARCHAR(16) NOT NULL, \n\tresult_family VARCHAR(128), \n\tresult_artifact_id VARCHAR(128), \n\tresult_revision INTEGER, \n\tdecision_reason TEXT, \n\tPRIMARY KEY (scope_id, candidate_id), \n\tFOREIGN KEY(scope_id, candidate_id, version) REFERENCES pc_artifact_candidate_versions (scope_id, candidate_id, version) ON DELETE RESTRICT, \n\tFOREIGN KEY(scope_id, result_family, result_artifact_id, result_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_candidate_heads_status CHECK (status IN ('pending', 'approved', 'rejected')), \n\tCONSTRAINT ck_pc_artifact_candidate_heads_terminal_result CHECK ((status = 'approved' AND result_family IS NOT NULL AND result_artifact_id IS NOT NULL AND result_revision > 0 AND decision_reason IS NULL) OR (status = 'rejected' AND result_family IS NULL AND result_artifact_id IS NULL AND result_revision IS NULL AND decision_reason IS NOT NULL) OR (status = 'pending' AND result_family IS NULL AND result_artifact_id IS NULL AND result_revision IS NULL AND decision_reason IS NULL))\n)" + ], + [ + "table", + "pc_artifact_candidate_versions", + "pc_artifact_candidate_versions", + "CREATE TABLE pc_artifact_candidate_versions (\n\tscope_id VARCHAR(256) NOT NULL, \n\tcandidate_id VARCHAR(128) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tproposal BLOB NOT NULL, \n\tsource_refs BLOB NOT NULL, \n\tartifact_refs BLOB NOT NULL, \n\ttarget_family VARCHAR(128), \n\ttarget_artifact_id VARCHAR(128), \n\ttarget_revision INTEGER, \n\treason TEXT, \n\tPRIMARY KEY (scope_id, candidate_id, version), \n\tFOREIGN KEY(scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_candidate_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_artifact_candidate_versions_target_complete CHECK ((target_family IS NULL AND target_artifact_id IS NULL AND target_revision IS NULL) OR (target_family IS NOT NULL AND target_artifact_id IS NOT NULL AND target_revision > 0))\n)" + ], + [ + "table", + "pc_artifact_fts", + "pc_artifact_fts", + "CREATE VIRTUAL TABLE pc_artifact_fts USING fts5(\n scope_id UNINDEXED,\n family UNINDEXED,\n artifact_id UNINDEXED,\n revision UNINDEXED,\n searchable_text,\n tokenize='unicode61'\n)" + ], + [ + "table", + "pc_artifact_fts_config", + "pc_artifact_fts_config", + "CREATE TABLE 'pc_artifact_fts_config'(k PRIMARY KEY, v) WITHOUT ROWID" + ], + [ + "table", + "pc_artifact_fts_content", + "pc_artifact_fts_content", + "CREATE TABLE 'pc_artifact_fts_content'(id INTEGER PRIMARY KEY, c0, c1, c2, c3, c4)" + ], + [ + "table", + "pc_artifact_fts_data", + "pc_artifact_fts_data", + "CREATE TABLE 'pc_artifact_fts_data'(id INTEGER PRIMARY KEY, block BLOB)" + ], + [ + "table", + "pc_artifact_fts_docsize", + "pc_artifact_fts_docsize", + "CREATE TABLE 'pc_artifact_fts_docsize'(id INTEGER PRIMARY KEY, sz BLOB)" + ], + [ + "table", + "pc_artifact_fts_idx", + "pc_artifact_fts_idx", + "CREATE TABLE 'pc_artifact_fts_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID" + ], + [ + "table", + "pc_artifact_heads", + "pc_artifact_heads", + "CREATE TABLE pc_artifact_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsearchable_text TEXT, \n\tlifecycle_state VARCHAR(16) DEFAULT 'active' NOT NULL, \n\treplacement_artifact_id VARCHAR(128), \n\tmerged_into_id VARCHAR(128), \n\tgovernance_generation BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_heads_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_artifact_heads_lifecycle_state CHECK (lifecycle_state IN ('active', 'deprecated', 'retired')), \n\tCONSTRAINT ck_pc_artifact_heads_governance_generation_nonnegative CHECK (governance_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_heads_replacement_deprecated CHECK (replacement_artifact_id IS NULL OR lifecycle_state = 'deprecated'), \n\tCONSTRAINT ck_pc_artifact_heads_merge_target CHECK (merged_into_id IS NULL OR (lifecycle_state = 'deprecated' AND merged_into_id <> artifact_id AND replacement_artifact_id IS NULL))\n)" + ], + [ + "table", + "pc_artifact_lineage_artifacts", + "pc_artifact_lineage_artifacts", + "CREATE TABLE pc_artifact_lineage_artifacts (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tordinal INTEGER NOT NULL, \n\tupstream_family VARCHAR(128) NOT NULL, \n\tupstream_artifact_id VARCHAR(128) NOT NULL, \n\tupstream_revision INTEGER NOT NULL, \n\tis_merge_input BOOLEAN DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision, ordinal), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE CASCADE, \n\tFOREIGN KEY(scope_id, upstream_family, upstream_artifact_id, upstream_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT\n)" + ], + [ + "table", + "pc_artifact_lineage_sources", + "pc_artifact_lineage_sources", + "CREATE TABLE pc_artifact_lineage_sources (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tordinal INTEGER NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision, ordinal), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE CASCADE, \n\tFOREIGN KEY(scope_id, source_type, source_id) REFERENCES pc_sources (scope_id, source_type, source_id) ON DELETE RESTRICT\n)" + ], + [ + "table", + "pc_artifact_processing_auto_wave_targets", + "pc_artifact_processing_auto_wave_targets", + "CREATE TABLE pc_artifact_processing_auto_wave_targets (\n\twave_id VARCHAR(36) NOT NULL, \n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tcompleted BOOLEAN DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (wave_id, binding_name, scope_id), \n\tCONSTRAINT ck_pc_artifact_processing_auto_wave_target_source_positive CHECK (source_through >= 1)\n)" + ], + [ + "table", + "pc_artifact_processing_binding_states", + "pc_artifact_processing_binding_states", + "CREATE TABLE pc_artifact_processing_binding_states (\n\tbinding_name VARCHAR(128) NOT NULL, \n\tlast_auto_wave_completed_at DATETIME, \n\tlast_schedule_checkpoint_at DATETIME, \n\tscan_generation BIGINT DEFAULT '0' NOT NULL, \n\tscan_in_progress BOOLEAN DEFAULT '0' NOT NULL, \n\tscan_upper_pending_sequence BIGINT, \n\tPRIMARY KEY (binding_name), \n\tCONSTRAINT ck_pc_processing_scan_generation CHECK (scan_generation >= 0)\n)" + ], + [ + "table", + "pc_artifact_processing_intents", + "pc_artifact_processing_intents", + "CREATE TABLE pc_artifact_processing_intents (\n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tpending_sequence BIGINT NOT NULL, \n\tdirty_generation BIGINT DEFAULT '0' NOT NULL, \n\tclean_generation BIGINT DEFAULT '0' NOT NULL, \n\trequested_generation BIGINT DEFAULT '0' NOT NULL, \n\thandled_generation BIGINT DEFAULT '0' NOT NULL, \n\tlast_auto_scan_generation BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (binding_name, scope_id), \n\tCONSTRAINT ck_pc_processing_intent_sequence CHECK (pending_sequence > 0), \n\tCONSTRAINT ck_pc_processing_intent_dirty CHECK (clean_generation >= 0 AND clean_generation <= dirty_generation), \n\tCONSTRAINT ck_pc_processing_intent_request CHECK (handled_generation >= 0 AND handled_generation <= requested_generation), \n\tCONSTRAINT ck_pc_processing_intent_scan CHECK (last_auto_scan_generation >= 0), \n\tUNIQUE (pending_sequence)\n)" + ], + [ + "table", + "pc_artifact_processing_leases", + "pc_artifact_processing_leases", + "CREATE TABLE pc_artifact_processing_leases (\n\tsupervisor_group VARCHAR(64) NOT NULL, \n\tholder_id VARCHAR(36) NOT NULL, \n\tsupervisor_generation BIGINT NOT NULL, \n\tlease_expires_at DATETIME, \n\tPRIMARY KEY (supervisor_group), \n\tCONSTRAINT ck_pc_artifact_processing_leases_generation_positive CHECK (supervisor_generation > 0)\n)" + ], + [ + "table", + "pc_artifact_processing_migration_receipts", + "pc_artifact_processing_migration_receipts", + "CREATE TABLE pc_artifact_processing_migration_receipts (\n\tmigration_id VARCHAR(64) NOT NULL, \n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_snapshot BLOB NOT NULL, \n\trequested_generation BIGINT NOT NULL, \n\tdirty_generation BIGINT NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tPRIMARY KEY (migration_id, binding_name, scope_id)\n)" + ], + [ + "table", + "pc_artifact_processing_pending", + "pc_artifact_processing_pending", + "CREATE TABLE pc_artifact_processing_pending (\n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tflush_generation BIGINT DEFAULT '0' NOT NULL, \n\thandled_flush_generation BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (binding_name, scope_id), \n\tCONSTRAINT ck_pc_artifact_processing_pending_source_positive CHECK (source_through >= 1), \n\tCONSTRAINT ck_pc_artifact_processing_pending_flush_nonnegative CHECK (flush_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_processing_pending_handled_nonnegative CHECK (handled_flush_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_processing_pending_handled_not_ahead CHECK (handled_flush_generation <= flush_generation)\n)" + ], + [ + "table", + "pc_artifact_processing_schema", + "pc_artifact_processing_schema", + "CREATE TABLE pc_artifact_processing_schema (\n\tsingleton INTEGER NOT NULL, \n\tschema_version INTEGER NOT NULL, \n\tsource_version INTEGER DEFAULT '1490' NOT NULL, \n\tphase VARCHAR(32) NOT NULL, \n\tmigration_id VARCHAR(64) NOT NULL, \n\tconfig_manifest TEXT NOT NULL, \n\tPRIMARY KEY (singleton), \n\tCONSTRAINT ck_pc_processing_schema_singleton CHECK (singleton = 1)\n)" + ], + [ + "table", + "pc_artifact_processing_sequences", + "pc_artifact_processing_sequences", + "CREATE TABLE pc_artifact_processing_sequences (\n\tsingleton INTEGER NOT NULL, \n\tsequence BIGINT NOT NULL, \n\tPRIMARY KEY (singleton), \n\tCONSTRAINT ck_pc_processing_sequence CHECK (singleton = 1 AND sequence >= 0)\n)" + ], + [ + "table", + "pc_artifact_publications", + "pc_artifact_publications", + "CREATE TABLE pc_artifact_publications (\n\ttarget_scope_id VARCHAR(256) NOT NULL, \n\ttarget_family VARCHAR(128) NOT NULL, \n\ttarget_artifact_id VARCHAR(128) NOT NULL, \n\ttarget_revision INTEGER NOT NULL, \n\tsource_scope_id VARCHAR(256) NOT NULL, \n\tsource_family VARCHAR(128) NOT NULL, \n\tsource_artifact_id VARCHAR(128) NOT NULL, \n\tsource_revision INTEGER NOT NULL, \n\tcontent_digest VARCHAR(64) NOT NULL, \n\tidempotency_key VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (target_scope_id, target_family, target_artifact_id, target_revision), \n\tFOREIGN KEY(target_scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tFOREIGN KEY(source_scope_id, source_family, source_artifact_id, source_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT uq_pc_artifact_publications_request UNIQUE (target_scope_id, idempotency_key)\n)" + ], + [ + "table", + "pc_artifact_tags", + "pc_artifact_tags", + "CREATE TABLE pc_artifact_tags (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\ttarget_type VARCHAR(12) NOT NULL, \n\ttarget_id VARCHAR(128) NOT NULL, \n\ttag_key_hash BLOB NOT NULL, \n\ttag_key VARCHAR(128) NOT NULL, \n\ttag VARCHAR(64) NOT NULL, \n\tassigned_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, target_type, target_id, tag_key_hash), \n\tFOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads (scope_id, family, artifact_id) ON DELETE CASCADE, \n\tCONSTRAINT ck_pc_artifact_tags_target CHECK ((target_type = 'artifact' AND target_id = artifact_id) OR (target_type = 'memory_entry' AND family = 'memory'))\n)" + ], + [ + "table", + "pc_artifacts", + "pc_artifacts", + "CREATE TABLE pc_artifacts (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)" + ], + [ + "table", + "pc_atomic_memory_current", + "pc_atomic_memory_current", + "CREATE TABLE pc_atomic_memory_current (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tstate_version BIGINT NOT NULL, \n\tcontent_hash VARCHAR(64) NOT NULL, \n\tprojection_format VARCHAR(64) NOT NULL, \n\tkind TEXT NOT NULL, \n\ttext TEXT NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\ttag_keys TEXT NOT NULL, \n\tembedding BLOB, \n\tprofile_fingerprint VARCHAR(64), \n\tembedding_input_hash VARCHAR(64), \n\tPRIMARY KEY (scope_id, artifact_id), \n\tCONSTRAINT ck_pc_atomic_memory_current_revision CHECK (revision > 0), \n\tCONSTRAINT ck_pc_atomic_memory_current_state_version CHECK (state_version >= 0), \n\tCONSTRAINT ck_pc_atomic_memory_current_embedding_metadata CHECK ((embedding IS NULL AND profile_fingerprint IS NULL AND embedding_input_hash IS NULL) OR (embedding IS NOT NULL AND profile_fingerprint IS NOT NULL AND embedding_input_hash IS NOT NULL))\n)" + ], + [ + "table", + "pc_atomic_memory_current_fts", + "pc_atomic_memory_current_fts", + "CREATE VIRTUAL TABLE pc_atomic_memory_current_fts USING fts5(scope_id UNINDEXED, artifact_id UNINDEXED, revision UNINDEXED, state_version UNINDEXED, content_hash UNINDEXED, projection_format UNINDEXED, kind UNINDEXED, text UNINDEXED, searchable_text, tag_keys UNINDEXED, identity_token, tokenize='unicode61')" + ], + [ + "table", + "pc_atomic_memory_current_fts_config", + "pc_atomic_memory_current_fts_config", + "CREATE TABLE 'pc_atomic_memory_current_fts_config'(k PRIMARY KEY, v) WITHOUT ROWID" + ], + [ + "table", + "pc_atomic_memory_current_fts_content", + "pc_atomic_memory_current_fts_content", + "CREATE TABLE 'pc_atomic_memory_current_fts_content'(id INTEGER PRIMARY KEY, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10)" + ], + [ + "table", + "pc_atomic_memory_current_fts_data", + "pc_atomic_memory_current_fts_data", + "CREATE TABLE 'pc_atomic_memory_current_fts_data'(id INTEGER PRIMARY KEY, block BLOB)" + ], + [ + "table", + "pc_atomic_memory_current_fts_docsize", + "pc_atomic_memory_current_fts_docsize", + "CREATE TABLE 'pc_atomic_memory_current_fts_docsize'(id INTEGER PRIMARY KEY, sz BLOB)" + ], + [ + "table", + "pc_atomic_memory_current_fts_idx", + "pc_atomic_memory_current_fts_idx", + "CREATE TABLE 'pc_atomic_memory_current_fts_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID" + ], + [ + "table", + "pc_connector_checkpoints", + "pc_connector_checkpoints", + "CREATE TABLE pc_connector_checkpoints (\n\tscope_id VARCHAR(256) NOT NULL, \n\tbinding_id VARCHAR(256) NOT NULL, \n\tconnector_name VARCHAR(128) NOT NULL, \n\tconnector_version VARCHAR(128) NOT NULL, \n\tcheckpoint BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, binding_id)\n)" + ], + [ + "table", + "pc_decision_observations", + "pc_decision_observations", + "CREATE TABLE pc_decision_observations (\n\tscope_id VARCHAR(256) NOT NULL, \n\tobservation_id VARCHAR(128) NOT NULL, \n\toperation_id VARCHAR(256) NOT NULL, \n\tcreated_at DATETIME NOT NULL, \n\tconsumer VARCHAR(128) NOT NULL, \n\tpolicy_id VARCHAR(256) NOT NULL, \n\tpolicy_version VARCHAR(64) NOT NULL, \n\tmode VARCHAR(16) NOT NULL, \n\tprivacy_boundary VARCHAR(32) NOT NULL, \n\tcoverage VARCHAR(16) NOT NULL, \n\tverdict VARCHAR(16) NOT NULL, \n\tassessment_source VARCHAR(32) NOT NULL, \n\tfinal_action VARCHAR(128) NOT NULL, \n\tused_fallback BOOLEAN NOT NULL, \n\tfallback_reason VARCHAR(128), \n\tprovider_id VARCHAR(128), \n\tbackend_model_id VARCHAR(256), \n\tmodel_policy_id VARCHAR(256), \n\trequests BIGINT NOT NULL, \n\tinput_tokens BIGINT, \n\toutput_tokens BIGINT, \n\tlatency_ms FLOAT, \n\tpayload BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, observation_id), \n\tCONSTRAINT ck_pc_decision_observations_requests_nonnegative CHECK (requests >= 0), \n\tCONSTRAINT ck_pc_decision_observations_input_tokens_nonnegative CHECK (input_tokens IS NULL OR input_tokens >= 0), \n\tCONSTRAINT ck_pc_decision_observations_output_tokens_nonnegative CHECK (output_tokens IS NULL OR output_tokens >= 0), \n\tCONSTRAINT ck_pc_decision_observations_latency_nonnegative CHECK (latency_ms IS NULL OR latency_ms >= 0)\n)" + ], + [ + "table", + "pc_dream_runs", + "pc_dream_runs", + "CREATE TABLE pc_dream_runs (\n\tscope_id VARCHAR(256) NOT NULL, \n\trun_id VARCHAR(64) NOT NULL, \n\tprincipal_key VARCHAR(64) NOT NULL, \n\tidempotency_key VARCHAR(128) NOT NULL, \n\trequest_digest VARCHAR(71) NOT NULL, \n\toperation VARCHAR(32) NOT NULL, \n\tstatus VARCHAR(16) NOT NULL, \n\taccepted_at BIGINT NOT NULL, \n\tgeneration INTEGER NOT NULL, \n\trequest_generation BIGINT NOT NULL, \n\tpayload BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, run_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tCONSTRAINT uq_pc_dream_idempotency UNIQUE (scope_id, principal_key, idempotency_key), \n\tCONSTRAINT ck_pc_dream_generation CHECK (generation >= 0)\n)" + ], + [ + "table", + "pc_external_skill_registrations", + "pc_external_skill_registrations", + "CREATE TABLE pc_external_skill_registrations (\n\tscope_id VARCHAR(256) NOT NULL, \n\texternal_skill_id VARCHAR(128) NOT NULL, \n\tprovider VARCHAR(128) NOT NULL, \n\tagent_kind VARCHAR(128) NOT NULL, \n\thost_id VARCHAR(128) NOT NULL, \n\tinstallation_scope VARCHAR(16) NOT NULL, \n\tlocator VARCHAR(2000) NOT NULL, \n\tlocator_hash VARCHAR(64) NOT NULL, \n\tfingerprint VARCHAR(64) NOT NULL, \n\tname VARCHAR(128) NOT NULL, \n\tdescription VARCHAR(2000) NOT NULL, \n\tPRIMARY KEY (scope_id, external_skill_id), \n\tCONSTRAINT uq_pc_external_skill_registrations_binding UNIQUE (scope_id, provider, host_id, installation_scope, locator_hash), \n\tCONSTRAINT ck_pc_external_skill_registrations_scope CHECK (installation_scope IN ('user', 'project', 'plugin'))\n)" + ], + [ + "table", + "pc_memory_entry_heads", + "pc_memory_entry_heads", + "CREATE TABLE pc_memory_entry_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tmemory_artifact_id VARCHAR(128) NOT NULL, \n\thead_revision INTEGER NOT NULL, \n\tentry_id VARCHAR(128) NOT NULL, \n\tentry_version_id VARCHAR(128) NOT NULL, \n\tentry_content_hash VARCHAR(64) NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\tPRIMARY KEY (scope_id, memory_artifact_id, entry_id), \n\tFOREIGN KEY(scope_id, memory_artifact_id, entry_id, entry_version_id) REFERENCES pc_memory_entry_versions (scope_id, memory_artifact_id, entry_id, entry_version_id) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_memory_entry_heads_revision_positive CHECK (head_revision > 0)\n)" + ], + [ + "table", + "pc_memory_entry_versions", + "pc_memory_entry_versions", + "CREATE TABLE pc_memory_entry_versions (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tmemory_artifact_id VARCHAR(128) NOT NULL, \n\tentry_id VARCHAR(128) NOT NULL, \n\tentry_version_id VARCHAR(128) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tprevious_version_id VARCHAR(128), \n\tkind VARCHAR(128) NOT NULL, \n\ttext TEXT NOT NULL, \n\tsource_refs BLOB NOT NULL, \n\tartifact_refs BLOB NOT NULL, \n\tentry_content_hash VARCHAR(64) NOT NULL, \n\tcreated_in_revision INTEGER NOT NULL, \n\tPRIMARY KEY (scope_id, memory_artifact_id, entry_version_id), \n\tCONSTRAINT uq_pc_memory_entry_versions_logical_version UNIQUE (scope_id, memory_artifact_id, entry_id, version), \n\tCONSTRAINT uq_pc_memory_entry_versions_identity UNIQUE (scope_id, memory_artifact_id, entry_id, entry_version_id), \n\tCONSTRAINT ck_pc_memory_entry_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_memory_entry_versions_revision_positive CHECK (created_in_revision > 0)\n)" + ], + [ + "table", + "pc_memory_source_windows", + "pc_memory_source_windows", + "CREATE TABLE pc_memory_source_windows (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\twindow_limit BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id), \n\tCONSTRAINT ck_pc_memory_source_window_positive CHECK (source_through > 0 AND window_limit > 0)\n)" + ], + [ + "table", + "pc_model_usage_daily", + "pc_model_usage_daily", + "CREATE TABLE pc_model_usage_daily (\n\tscope_id VARCHAR(256) NOT NULL, \n\tusage_date DATE NOT NULL, \n\tpurpose VARCHAR(64) NOT NULL, \n\toperation VARCHAR(16) NOT NULL, \n\trequests BIGINT NOT NULL, \n\tinput_tokens BIGINT NOT NULL, \n\toutput_tokens BIGINT NOT NULL, \n\tinput_complete BOOLEAN NOT NULL, \n\toutput_complete BOOLEAN NOT NULL, \n\tPRIMARY KEY (scope_id, usage_date, purpose, operation), \n\tCONSTRAINT ck_pc_model_usage_daily_operation CHECK (operation IN ('generation', 'embedding')), \n\tCONSTRAINT ck_pc_model_usage_daily_requests_nonnegative CHECK (requests >= 0), \n\tCONSTRAINT ck_pc_model_usage_daily_input_nonnegative CHECK (input_tokens >= 0), \n\tCONSTRAINT ck_pc_model_usage_daily_output_nonnegative CHECK (output_tokens >= 0)\n)" + ], + [ + "table", + "pc_profile_policies", + "pc_profile_policies", + "CREATE TABLE pc_profile_policies (\n\tscope_id VARCHAR(256) NOT NULL, \n\tgeneration_enabled BOOLEAN NOT NULL, \n\tactivation_mode VARCHAR(32) NOT NULL, \n\tpending_candidate_id VARCHAR(128), \n\tversion BIGINT NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tFOREIGN KEY(scope_id, pending_candidate_id) REFERENCES pc_artifact_candidate_heads (scope_id, candidate_id) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_profile_policies_activation_mode CHECK (activation_mode IN ('automatic', 'review_required')), \n\tCONSTRAINT ck_pc_profile_policies_version_positive CHECK (version > 0)\n)" + ], + [ + "table", + "pc_recall_effort_daily", + "pc_recall_effort_daily", + "CREATE TABLE pc_recall_effort_daily (\n\tscope_id VARCHAR(256) NOT NULL, \n\tusage_date DATE NOT NULL, \n\tpolicy_id VARCHAR(128) NOT NULL, \n\tassessment VARCHAR(64) NOT NULL, \n\tpreparations BIGINT NOT NULL CONSTRAINT ck_recall_effort_preparations_nonnegative CHECK (preparations >= 0), \n\trounds BIGINT NOT NULL CONSTRAINT ck_recall_effort_rounds_nonnegative CHECK (rounds >= 0), \n\texpanded_preparations BIGINT NOT NULL CONSTRAINT ck_recall_effort_expanded_preparations_nonnegative CHECK (expanded_preparations >= 0), \n\tadmission_expansions BIGINT NOT NULL CONSTRAINT ck_recall_effort_admission_expansions_nonnegative CHECK (admission_expansions >= 0), \n\tpolicy_floor_expansions BIGINT NOT NULL CONSTRAINT ck_recall_effort_policy_floor_expansions_nonnegative CHECK (policy_floor_expansions >= 0), \n\tcandidate_round_samples BIGINT NOT NULL CONSTRAINT ck_recall_effort_candidate_round_samples_nonnegative CHECK (candidate_round_samples >= 0), \n\tcandidates_assessed BIGINT NOT NULL CONSTRAINT ck_recall_effort_candidates_assessed_nonnegative CHECK (candidates_assessed >= 0), \n\tfinal_candidate_pool BIGINT NOT NULL CONSTRAINT ck_recall_effort_final_candidate_pool_nonnegative CHECK (final_candidate_pool >= 0), \n\tadded_embeddings BIGINT NOT NULL CONSTRAINT ck_recall_effort_added_embeddings_nonnegative CHECK (added_embeddings >= 0), \n\tadded_generation_calls BIGINT NOT NULL CONSTRAINT ck_recall_effort_added_generation_calls_nonnegative CHECK (added_generation_calls >= 0), \n\ttruncated_items BIGINT NOT NULL CONSTRAINT ck_recall_effort_truncated_items_nonnegative CHECK (truncated_items >= 0), \n\tdropped_items BIGINT NOT NULL CONSTRAINT ck_recall_effort_dropped_items_nonnegative CHECK (dropped_items >= 0), \n\tdropped_below_min_bytes BIGINT NOT NULL CONSTRAINT ck_recall_effort_dropped_below_min_bytes_nonnegative CHECK (dropped_below_min_bytes >= 0), \n\tdropped_no_fitting_truncation BIGINT NOT NULL CONSTRAINT ck_recall_effort_dropped_no_fitting_truncation_nonnegative CHECK (dropped_no_fitting_truncation >= 0), \n\tPRIMARY KEY (scope_id, usage_date, policy_id, assessment), \n\tCONSTRAINT ck_pc_recall_effort_daily_drop_total CHECK (dropped_items = dropped_below_min_bytes + dropped_no_fitting_truncation)\n)" + ], + [ + "table", + "pc_recall_token_daily", + "pc_recall_token_daily", + "CREATE TABLE pc_recall_token_daily (\n\tscope_id VARCHAR(256) NOT NULL, \n\tusage_date DATE NOT NULL, \n\testimator_id VARCHAR(128) NOT NULL, \n\testimator_version VARCHAR(64) NOT NULL, \n\tpreparations BIGINT NOT NULL, \n\tready_preparations BIGINT NOT NULL, \n\tcomparable_preparations BIGINT NOT NULL, \n\tbaseline_tokens BIGINT NOT NULL, \n\trecalled_tokens BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, usage_date, estimator_id, estimator_version), \n\tCONSTRAINT ck_pc_recall_token_daily_preparations_nonnegative CHECK (preparations >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_ready_nonnegative CHECK (ready_preparations >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_comparable_nonnegative CHECK (comparable_preparations >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_comparable_ready CHECK (comparable_preparations <= ready_preparations), \n\tCONSTRAINT ck_pc_recall_token_daily_ready_total CHECK (ready_preparations <= preparations), \n\tCONSTRAINT ck_pc_recall_token_daily_baseline_nonnegative CHECK (baseline_tokens >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_recalled_nonnegative CHECK (recalled_tokens >= 0)\n)" + ], + [ + "table", + "pc_receipt_migration_review", + "pc_receipt_migration_review", + "CREATE TABLE pc_receipt_migration_review (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\treason VARCHAR(64) NOT NULL, \n\tPRIMARY KEY (scope_id, source_id)\n)" + ], + [ + "table", + "pc_recurrence_match", + "pc_recurrence_match", + "CREATE TABLE pc_recurrence_match (\n\tscope_id VARCHAR(256) NOT NULL, \n\tmatch_key VARCHAR(71) NOT NULL, \n\ttask_outcome_source_type VARCHAR(128) NOT NULL, \n\ttask_outcome_source_id VARCHAR(256) NOT NULL, \n\ttask_outcome_position BIGINT NOT NULL, \n\tfailure_item_kind VARCHAR(16) NOT NULL, \n\tfailure_item_index INTEGER NOT NULL, \n\tfailure_item_digest VARCHAR(71) NOT NULL, \n\tcandidate_set_mode VARCHAR(32) NOT NULL, \n\tcandidate_set_digest VARCHAR(71) NOT NULL, \n\tresult VARCHAR(16) NOT NULL, \n\ttarget_family VARCHAR(128), \n\ttarget_artifact_id VARCHAR(128), \n\ttarget_revision INTEGER, \n\tsignature_key TEXT, \n\tpayload BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, match_key), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tCONSTRAINT uq_pc_recurrence_match_identity UNIQUE (scope_id, task_outcome_source_type, task_outcome_source_id, failure_item_kind, failure_item_index), \n\tCONSTRAINT ck_pc_recurrence_match_result CHECK (result IN ('matched', 'unmatched', 'ambiguous')), \n\tCONSTRAINT ck_pc_recurrence_match_candidate_set_mode CHECK (candidate_set_mode IN ('handoff_citations', 'scope_heads')), \n\tCONSTRAINT ck_pc_recurrence_match_failure_item_kind CHECK (failure_item_kind IN ('observation', 'check')), \n\tCONSTRAINT ck_pc_recurrence_match_position_positive CHECK (task_outcome_position > 0), \n\tCONSTRAINT ck_pc_recurrence_match_item_index_nonnegative CHECK (failure_item_index >= 0)\n)" + ], + [ + "table", + "pc_recurrence_observation", + "pc_recurrence_observation", + "CREATE TABLE pc_recurrence_observation (\n\tscope_id VARCHAR(256) NOT NULL, \n\tobservation_id VARCHAR(71) NOT NULL, \n\tselection_key VARCHAR(71) NOT NULL, \n\tverdict_key VARCHAR(71) NOT NULL, \n\tevent VARCHAR(16) NOT NULL, \n\tmatch_basis VARCHAR(8) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsignature_key TEXT NOT NULL, \n\tsignature_key_hash BLOB NOT NULL, \n\ttask_outcome_source_type VARCHAR(128) NOT NULL, \n\ttask_outcome_source_id VARCHAR(256) NOT NULL, \n\ttask_outcome_position BIGINT NOT NULL, \n\tpayload BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, observation_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tCONSTRAINT uq_pc_recurrence_observation_selection UNIQUE (scope_id, selection_key), \n\tCONSTRAINT uq_pc_recurrence_observation_verdict UNIQUE (scope_id, verdict_key), \n\tCONSTRAINT ck_pc_recurrence_observation_event CHECK (event IN ('selected', 'recurred', 'avoided')), \n\tCONSTRAINT ck_pc_recurrence_observation_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_recurrence_observation_position_positive CHECK (task_outcome_position > 0)\n)" + ], + [ + "table", + "pc_scope_bindings", + "pc_scope_bindings", + "CREATE TABLE pc_scope_bindings (\n\tintegration VARCHAR(128) NOT NULL, \n\tkind VARCHAR(64) NOT NULL, \n\texternal_id VARCHAR(256) NOT NULL, \n\texternal_id_search TEXT NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (integration, kind, external_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT\n)" + ], + [ + "table", + "pc_scope_context_references", + "pc_scope_context_references", + "CREATE TABLE pc_scope_context_references (\n\tscope_id VARCHAR(256) NOT NULL, \n\treferenced_scope_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (scope_id, referenced_scope_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tFOREIGN KEY(referenced_scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_scope_context_references_not_self CHECK (scope_id <> referenced_scope_id)\n)" + ], + [ + "table", + "pc_scope_creation_requests", + "pc_scope_creation_requests", + "CREATE TABLE pc_scope_creation_requests (\n\tidempotency_key VARCHAR(256) NOT NULL, \n\trequest_digest VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (idempotency_key), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT\n)" + ], + [ + "table", + "pc_scope_external_references", + "pc_scope_external_references", + "CREATE TABLE pc_scope_external_references (\n\tscope_id VARCHAR(256) NOT NULL, \n\tordinal INTEGER NOT NULL, \n\tkind VARCHAR(128) NOT NULL, \n\tvalue VARCHAR(2000) NOT NULL, \n\tvalue_search TEXT NOT NULL, \n\tvalue_digest VARCHAR(64) NOT NULL, \n\tPRIMARY KEY (scope_id, ordinal), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tCONSTRAINT uq_pc_scope_external_references_value UNIQUE (scope_id, kind, value_digest), \n\tCONSTRAINT ck_pc_scope_external_references_ordinal_nonnegative CHECK (ordinal >= 0)\n)" + ], + [ + "table", + "pc_scope_settings", + "pc_scope_settings", + "CREATE TABLE pc_scope_settings (\n\tname VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (name), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT\n)" + ], + [ + "table", + "pc_scopes", + "pc_scopes", + "CREATE TABLE pc_scopes (\n\tscope_id VARCHAR(256) NOT NULL, \n\ttitle VARCHAR(256) NOT NULL, \n\tsummary VARCHAR(2000) NOT NULL, \n\tscope_id_search TEXT NOT NULL, \n\ttitle_search TEXT NOT NULL, \n\tsummary_search TEXT NOT NULL, \n\tparent_scope_id VARCHAR(256), \n\tversion INTEGER NOT NULL, \n\tPRIMARY KEY (scope_id), \n\tFOREIGN KEY(parent_scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_scopes_version_positive CHECK (version > 0)\n)" + ], + [ + "table", + "pc_server_identity", + "pc_server_identity", + "CREATE TABLE pc_server_identity (\n\tsingleton_key INTEGER NOT NULL, \n\tserver_id VARCHAR(36) NOT NULL, \n\tPRIMARY KEY (singleton_key), \n\tCONSTRAINT ck_pc_server_identity_singleton CHECK (singleton_key = 1), \n\tUNIQUE (server_id)\n)" + ], + [ + "table", + "pc_skill_packages", + "pc_skill_packages", + "CREATE TABLE pc_skill_packages (\n\tscope_id VARCHAR(256) NOT NULL, \n\ttree_digest VARCHAR(64) NOT NULL, \n\tarchive_digest VARCHAR(64) NOT NULL, \n\tarchive_bytes BLOB NOT NULL, \n\tmanifest BLOB NOT NULL, \n\tfile_count INTEGER NOT NULL, \n\tuncompressed_size BIGINT NOT NULL, \n\tarchive_size BIGINT NOT NULL, \n\tcreated_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, tree_digest), \n\tCONSTRAINT ck_pc_skill_packages_file_count CHECK (file_count > 0 AND file_count <= 256), \n\tCONSTRAINT ck_pc_skill_packages_uncompressed_size CHECK (uncompressed_size > 0 AND uncompressed_size <= 4194304), \n\tCONSTRAINT ck_pc_skill_packages_archive_size CHECK (archive_size > 0 AND archive_size <= 5242880)\n)" + ], + [ + "table", + "pc_skill_publications", + "pc_skill_publications", + "CREATE TABLE pc_skill_publications (\n\tscope_id VARCHAR(256) NOT NULL, \n\ttarget_id VARCHAR(64) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\tdesired_state VARCHAR(16) DEFAULT 'published' NOT NULL, \n\tdesired_revision INTEGER NOT NULL, \n\tdesired_tree_digest VARCHAR(64) NOT NULL, \n\tobserved_revision INTEGER, \n\tobserved_tree_digest VARCHAR(64), \n\tobserved_generation BIGINT, \n\tdestination TEXT, \n\tstate VARCHAR(32) NOT NULL, \n\tselected_runtime_variant VARCHAR(128), \n\tenvironment_fingerprint VARCHAR(64), \n\tlast_error_code VARCHAR(128), \n\tobserved_at DATETIME, \n\tgeneration BIGINT NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, target_id, artifact_id), \n\tCONSTRAINT ck_pc_skill_publications_desired_revision_positive CHECK (desired_revision > 0), \n\tCONSTRAINT ck_pc_skill_publications_observed_revision_positive CHECK (observed_revision IS NULL OR observed_revision > 0), \n\tCONSTRAINT ck_pc_skill_publications_desired_state CHECK (desired_state IN ('published', 'unpublished')), \n\tCONSTRAINT ck_pc_skill_publications_state CHECK (state IN ('unpublished', 'pending', 'current', 'update_available', 'delivery_failed', 'conflict', 'drifted', 'incompatible')), \n\tCONSTRAINT ck_pc_skill_publications_observed_generation_nonnegative CHECK (observed_generation IS NULL OR observed_generation >= 0), \n\tCONSTRAINT ck_pc_skill_publications_generation_nonnegative CHECK (generation >= 0)\n)" + ], + [ + "table", + "pc_source_cursors", + "pc_source_cursors", + "CREATE TABLE pc_source_cursors (\n\tscope_id VARCHAR(256) NOT NULL, \n\tbinding_name VARCHAR(128) NOT NULL, \n\tcursor BLOB NOT NULL, \n\tgeneration BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, binding_name), \n\tCONSTRAINT ck_pc_source_cursors_generation_nonnegative CHECK (generation >= 0)\n)" + ], + [ + "table", + "pc_source_definition_manifests", + "pc_source_definition_manifests", + "CREATE TABLE pc_source_definition_manifests (\n\tdefinition_name VARCHAR(128) NOT NULL, \n\tdefinition_version VARCHAR(128) NOT NULL, \n\tfingerprint VARCHAR(71) NOT NULL, \n\tmanifest BLOB NOT NULL, \n\tPRIMARY KEY (definition_name, definition_version), \n\tCONSTRAINT uq_pc_source_definition_manifest_fingerprint UNIQUE (definition_name, fingerprint)\n)" + ], + [ + "table", + "pc_source_journal_heads", + "pc_source_journal_heads", + "CREATE TABLE pc_source_journal_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tposition BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id), \n\tCONSTRAINT ck_pc_source_journal_heads_position_nonnegative CHECK (position >= 0)\n)" + ], + [ + "table", + "pc_sources", + "pc_sources", + "CREATE TABLE pc_sources (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tpayload BLOB NOT NULL, \n\tjournal_position BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id), \n\tCONSTRAINT uq_pc_sources_scope_journal_position UNIQUE (scope_id, journal_position)\n)" + ], + [ + "table", + "pc_topic_memory_active_chunks", + "pc_topic_memory_active_chunks", + "CREATE TABLE pc_topic_memory_active_chunks (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tchunk_ordinal INTEGER NOT NULL, \n\tstart_offset INTEGER NOT NULL, \n\tend_offset INTEGER NOT NULL, \n\tchunk_text TEXT NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\tpolicy_version VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, chunk_ordinal), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_topic_memory_revision_publications (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_family CHECK (family = 'topic-memory'), \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_ordinal_nonnegative CHECK (chunk_ordinal >= 0), \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_start_nonnegative CHECK (start_offset >= 0), \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_offsets CHECK (end_offset > start_offset)\n)" + ], + [ + "table", + "pc_topic_memory_active_topics", + "pc_topic_memory_active_topics", + "CREATE TABLE pc_topic_memory_active_topics (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\ttitle TEXT NOT NULL, \n\tsummary TEXT NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\tsource_count INTEGER NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_topic_memory_revision_publications (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_topic_memory_active_topics_family CHECK (family = 'topic-memory'), \n\tCONSTRAINT ck_pc_topic_memory_active_topics_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_topic_memory_active_topics_sources_nonnegative CHECK (source_count >= 0)\n)" + ], + [ + "table", + "pc_topic_memory_chunk_fts", + "pc_topic_memory_chunk_fts", + "CREATE VIRTUAL TABLE pc_topic_memory_chunk_fts USING fts5(\n scope_id UNINDEXED, artifact_id UNINDEXED, revision UNINDEXED,\n chunk_ordinal UNINDEXED, start_offset UNINDEXED,\n title UNINDEXED, summary UNINDEXED, chunk_text UNINDEXED,\n searchable_text, tokenize='unicode61'\n)" + ], + [ + "table", + "pc_topic_memory_chunk_fts_config", + "pc_topic_memory_chunk_fts_config", + "CREATE TABLE 'pc_topic_memory_chunk_fts_config'(k PRIMARY KEY, v) WITHOUT ROWID" + ], + [ + "table", + "pc_topic_memory_chunk_fts_content", + "pc_topic_memory_chunk_fts_content", + "CREATE TABLE 'pc_topic_memory_chunk_fts_content'(id INTEGER PRIMARY KEY, c0, c1, c2, c3, c4, c5, c6, c7, c8)" + ], + [ + "table", + "pc_topic_memory_chunk_fts_data", + "pc_topic_memory_chunk_fts_data", + "CREATE TABLE 'pc_topic_memory_chunk_fts_data'(id INTEGER PRIMARY KEY, block BLOB)" + ], + [ + "table", + "pc_topic_memory_chunk_fts_docsize", + "pc_topic_memory_chunk_fts_docsize", + "CREATE TABLE 'pc_topic_memory_chunk_fts_docsize'(id INTEGER PRIMARY KEY, sz BLOB)" + ], + [ + "table", + "pc_topic_memory_chunk_fts_idx", + "pc_topic_memory_chunk_fts_idx", + "CREATE TABLE 'pc_topic_memory_chunk_fts_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID" + ], + [ + "table", + "pc_topic_memory_fts_index", + "pc_topic_memory_fts_index", + "CREATE TABLE pc_topic_memory_fts_index (\n\tsingleton INTEGER NOT NULL, \n\tschema_version INTEGER NOT NULL, \n\tPRIMARY KEY (singleton), \n\tCONSTRAINT ck_pc_topic_memory_fts_index_singleton CHECK (singleton = 1)\n)" + ], + [ + "table", + "pc_topic_memory_processing_targets", + "pc_topic_memory_processing_targets", + "CREATE TABLE pc_topic_memory_processing_targets (\n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\ttarget_request_generation BIGINT NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tcaptured_flush_generation BIGINT NOT NULL, \n\tobserved_dirty_generation BIGINT NOT NULL, \n\tPRIMARY KEY (binding_name, scope_id), \n\tCONSTRAINT ck_pc_topic_processing_target CHECK (target_request_generation > 0 AND source_through >= 0 AND captured_flush_generation >= 0 AND observed_dirty_generation >= 0)\n)" + ], + [ + "table", + "pc_topic_memory_retrieval_shape", + "pc_topic_memory_retrieval_shape", + "CREATE TABLE pc_topic_memory_retrieval_shape (\n\tsingleton INTEGER NOT NULL, \n\tshape VARCHAR(16) NOT NULL, \n\tprofile_fingerprint VARCHAR(64), \n\tPRIMARY KEY (singleton), \n\tCONSTRAINT ck_pc_topic_memory_retrieval_shape_singleton CHECK (singleton = 1), \n\tCONSTRAINT ck_pc_topic_memory_retrieval_shape_value CHECK (shape IN ('fts', 'hybrid')), \n\tCONSTRAINT ck_pc_topic_memory_retrieval_shape_profile CHECK ((shape = 'fts' AND profile_fingerprint IS NULL) OR (shape = 'hybrid' AND profile_fingerprint IS NOT NULL))\n)" + ], + [ + "table", + "pc_topic_memory_revision_publications", + "pc_topic_memory_revision_publications", + "CREATE TABLE pc_topic_memory_revision_publications (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tpublished_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_topic_memory_publications_family CHECK (family = 'topic-memory'), \n\tCONSTRAINT ck_pc_topic_memory_publications_revision_positive CHECK (revision > 0)\n)" + ], + [ + "table", + "pc_topic_memory_topic_fts", + "pc_topic_memory_topic_fts", + "CREATE VIRTUAL TABLE pc_topic_memory_topic_fts USING fts5(\n scope_id UNINDEXED, artifact_id UNINDEXED, revision UNINDEXED,\n title UNINDEXED, summary UNINDEXED, searchable_text, tokenize='unicode61'\n)" + ], + [ + "table", + "pc_topic_memory_topic_fts_config", + "pc_topic_memory_topic_fts_config", + "CREATE TABLE 'pc_topic_memory_topic_fts_config'(k PRIMARY KEY, v) WITHOUT ROWID" + ], + [ + "table", + "pc_topic_memory_topic_fts_content", + "pc_topic_memory_topic_fts_content", + "CREATE TABLE 'pc_topic_memory_topic_fts_content'(id INTEGER PRIMARY KEY, c0, c1, c2, c3, c4, c5)" + ], + [ + "table", + "pc_topic_memory_topic_fts_data", + "pc_topic_memory_topic_fts_data", + "CREATE TABLE 'pc_topic_memory_topic_fts_data'(id INTEGER PRIMARY KEY, block BLOB)" + ], + [ + "table", + "pc_topic_memory_topic_fts_docsize", + "pc_topic_memory_topic_fts_docsize", + "CREATE TABLE 'pc_topic_memory_topic_fts_docsize'(id INTEGER PRIMARY KEY, sz BLOB)" + ], + [ + "table", + "pc_topic_memory_topic_fts_idx", + "pc_topic_memory_topic_fts_idx", + "CREATE TABLE 'pc_topic_memory_topic_fts_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID" + ], + [ + "table", + "pc_topic_memory_work_budgets", + "pc_topic_memory_work_budgets", + "CREATE TABLE pc_topic_memory_work_budgets (\n\tscope_id VARCHAR(256) NOT NULL, \n\tbinding_name VARCHAR(128) NOT NULL, \n\tsource_after BIGINT NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tattempt_id VARCHAR(36) NOT NULL, \n\tattempts INTEGER NOT NULL, \n\trequests BIGINT NOT NULL, \n\ttokens BIGINT NOT NULL, \n\tfailure_code VARCHAR(64) NOT NULL, \n\tPRIMARY KEY (scope_id, binding_name, source_after), \n\tCONSTRAINT ck_pc_topic_budget_window CHECK (source_after >= 0 AND source_through > source_after), \n\tCONSTRAINT ck_pc_topic_budget_usage CHECK (attempts > 0 AND requests >= 0 AND tokens >= 0)\n)" + ], + [ + "trigger", + "pc_atomic_memory_fts_delete", + "pc_atomic_memory_current", + "CREATE TRIGGER pc_atomic_memory_fts_delete AFTER DELETE ON pc_atomic_memory_current BEGIN DELETE FROM pc_atomic_memory_current_fts WHERE identity_token MATCH ('i' || lower(hex(old.scope_id)) || 'x' || lower(hex(old.artifact_id))); END" + ], + [ + "trigger", + "pc_atomic_memory_fts_insert", + "pc_atomic_memory_current", + "CREATE TRIGGER pc_atomic_memory_fts_insert AFTER INSERT ON pc_atomic_memory_current BEGIN INSERT INTO pc_atomic_memory_current_fts (scope_id, artifact_id, revision, state_version, content_hash, projection_format, kind, text, searchable_text, tag_keys, identity_token) VALUES (new.scope_id, new.artifact_id, new.revision, new.state_version, new.content_hash, new.projection_format, new.kind, new.text, new.searchable_text, new.tag_keys, 'i' || lower(hex(new.scope_id)) || 'x' || lower(hex(new.artifact_id))); END" + ], + [ + "trigger", + "pc_atomic_memory_fts_update", + "pc_atomic_memory_current", + "CREATE TRIGGER pc_atomic_memory_fts_update AFTER UPDATE ON pc_atomic_memory_current BEGIN DELETE FROM pc_atomic_memory_current_fts WHERE identity_token MATCH ('i' || lower(hex(old.scope_id)) || 'x' || lower(hex(old.artifact_id))); INSERT INTO pc_atomic_memory_current_fts (scope_id, artifact_id, revision, state_version, content_hash, projection_format, kind, text, searchable_text, tag_keys, identity_token) VALUES (new.scope_id, new.artifact_id, new.revision, new.state_version, new.content_hash, new.projection_format, new.kind, new.text, new.searchable_text, new.tag_keys, 'i' || lower(hex(new.scope_id)) || 'x' || lower(hex(new.artifact_id))); END" + ] + ] +} diff --git a/src/powercontext/builtin/persistence/migration_resources/server_baselines/README.md b/src/powercontext/builtin/persistence/migration_resources/server_baselines/README.md new file mode 100644 index 0000000000..992147ad10 --- /dev/null +++ b/src/powercontext/builtin/persistence/migration_resources/server_baselines/README.md @@ -0,0 +1,36 @@ +# Complete Server source baselines + +Each resource records the complete `sqlite_schema` inventory produced by a +specific installed public wheel. Inventories include indexes, constraints, +triggers, virtual tables, and their SQLite shadow tables. They contain schema +definitions and artifact provenance, with no business rows or deployment paths. + +Recognition identifies a source schema. It does not validate its business data, +adopt a schema revision, authorize writes, establish Server readiness, or promise +an executable upgrade path. The maintenance runner keeps an identified source +blocked until the registered migration chain covers that complete source and its +domain preconditions. + +## SQLite 1.3.0 + +`1.3.0/sqlite.json` was captured from the public +`powercontext-1.3.0-py3-none-any.whl`, SHA-256 +`8da7983763f3a6ab80b413c770a4973a46c557a54e1a1fa675099e58d47305e5`. +The wheel was installed with its `cli,server` extras in an isolated environment. +A real Server initialized a new file with enforced Access, all Processing roles, +and no model or embedding provider. Public HTTP operations created Scopes, +Atomic Memory, a Source, pending and approved Candidates, and an Access binding +with idempotency receipts. After stopping the Server, SQLite's backup API +produced the database used for schema capture. + +The resource records the Python, SQLite, and SQLAlchemy capture versions. Its +94 explicit objects comprise 81 tables, 10 indexes, and 3 triggers. The complete +schema fingerprint is +`11e8770068a5fe32dfb25681564dbbe586ba0fd0238ed01adf1b09dde8a49f22`. +This shape differs from the historical four-table migration prototype. + +The recognizer uses exact definitions. A different dependency version, optional +feature, missing FTS helper, manually altered constraint, or extra persistent +object may produce an unregistered shape. Such a shape remains unknown; a +semantically similar subset is never sufficient for adoption. Additional source +shapes require their own installed-release capture and review. diff --git a/src/powercontext/builtin/persistence/migrations/server_baselines.py b/src/powercontext/builtin/persistence/migrations/server_baselines.py new file mode 100644 index 0000000000..e5f70ebe33 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_baselines.py @@ -0,0 +1,60 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Recognize complete installed-release schema shapes, independently of readiness.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path + +from sqlalchemy import Connection + +from .bundle import sqlite_inventory +from .models import MigrationError, digest + +_BASELINES = Path(__file__).parents[1] / "migration_resources/server_baselines" + + +@dataclass(frozen=True) +class ServerBaseline: + """A recognized schema source, not a validated data state or executable path.""" + + release: str + schema_fingerprint: str + + +def recognize_sqlite_server_baseline(connection: Connection) -> ServerBaseline | None: + """Match the complete schema, including FTS shadows and optional-feature objects. + + No objects are excluded: in particular, an existing control table cannot be + ignored to stamp a partially migrated database as a released source. + """ + fingerprint = digest(sqlite_inventory(connection)) + matches: list[ServerBaseline] = [] + for path in sorted(_BASELINES.glob("*/sqlite.json")): + source = json.loads(path.read_text(encoding="utf-8")) + if ( + source["format"] != "powercontext.server-baseline.v1" + or source["backend"] != "sqlite" + or source["release"] != path.parent.name + or digest(source["inventory"]) != source["schema_fingerprint"] + ): + raise MigrationError("invalid_bundle", "The frozen Server baseline resource is inconsistent.") + if fingerprint == source["schema_fingerprint"]: + matches.append(ServerBaseline(release=source["release"], schema_fingerprint=fingerprint)) + if len(matches) > 1: + raise MigrationError("invalid_bundle", "Multiple releases claim the same complete Server baseline.") + return matches[0] if matches else None diff --git a/src/powercontext/builtin/persistence/migrations/sqlite.py b/src/powercontext/builtin/persistence/migrations/sqlite.py index 2664f2f440..983c55a702 100644 --- a/src/powercontext/builtin/persistence/migrations/sqlite.py +++ b/src/powercontext/builtin/persistence/migrations/sqlite.py @@ -39,6 +39,7 @@ from .bundle import ALEMBIC_CONTEXT_LOCK, MigrationBundle, sqlite_inventory from .locking import local_migration_lock from .models import MigrationError, MigrationPlan, MigrationResult, digest +from .server_baselines import recognize_sqlite_server_baseline _VERSION_DDL = ( "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, " @@ -285,6 +286,13 @@ def _inspect(self, connection: Connection, **options: Any) -> MigrationPlan: elif inventory: revision = self.bundle.baselines.get(fingerprint) if revision is None: + source = recognize_sqlite_server_baseline(connection) + if source is not None: + raise MigrationError( + "server_migration_unavailable", + f"Complete Server {source.release} schema recognized; " + "the installed bundle has no registered full Server upgrade path. No changes were made.", + ) raise MigrationError("unknown_baseline", "Schema is not registered; no changes were made.") self._verify_retained(inventory, revision) self._verify_integrity(connection) diff --git a/tests/builtin/persistence/test_server_migration_baselines.py b/tests/builtin/persistence/test_server_migration_baselines.py new file mode 100644 index 0000000000..44b2d86fc5 --- /dev/null +++ b/tests/builtin/persistence/test_server_migration_baselines.py @@ -0,0 +1,126 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Recognize complete released schemas without stamping or promising an upgrade.""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest +from sqlalchemy import create_engine + +from powercontext.builtin.persistence.migrations import MigrationError, SQLiteMigrationRunner +from powercontext.builtin.persistence.migrations.deployment import production_bundle +from powercontext.builtin.persistence.migrations.server_baselines import recognize_sqlite_server_baseline + +RESOURCE = ( + Path(__file__).resolve().parents[3] + / "src/powercontext/builtin/persistence/migration_resources/server_baselines/1.3.0/sqlite.json" +) + + +@pytest.fixture +def released_database(tmp_path: Path) -> Path: + database = tmp_path / "released.db" + inventory = json.loads(RESOURCE.read_text(encoding="utf-8"))["inventory"] + with sqlite3.connect(database) as connection: + # Creating an FTS virtual table also creates its frozen shadow tables. + for kind in ("table", "index", "trigger", "view"): + for object_kind, name, _, sql in inventory: + if ( + object_kind == kind + and connection.execute("SELECT 1 FROM sqlite_schema WHERE name=?", (name,)).fetchone() is None + ): + connection.execute(sql) + connection.execute( + "INSERT INTO pc_artifacts (scope_id,family,artifact_id,revision,content) " + "VALUES ('scope','atomic-memory','historical',1,?)", + (b'{"historical":"immutable"}',), + ) + return database + + +def recognize(database: Path): + engine = create_engine("sqlite://", creator=lambda: sqlite3.connect(database.as_uri() + "?mode=ro", uri=True)) + try: + with engine.connect() as connection: + connection.exec_driver_sql("PRAGMA query_only=ON") + return recognize_sqlite_server_baseline(connection) + finally: + engine.dispose() + + +def test_complete_release_schema_is_recognized_without_writing_or_claiming_readiness(released_database: Path) -> None: + before = released_database.read_bytes() + files = set(released_database.parent.iterdir()) + baseline = recognize(released_database) + assert baseline is not None + assert baseline.release == "1.3.0" + assert baseline.schema_fingerprint == "11e8770068a5fe32dfb25681564dbbe586ba0fd0238ed01adf1b09dde8a49f22" + runner = SQLiteMigrationRunner(released_database, production_bundle()) + for operation in (runner.plan, runner.verify, runner.apply): + with pytest.raises(MigrationError, match="server_migration_unavailable"): + operation() + assert released_database.read_bytes() == before + assert set(released_database.parent.iterdir()) == files + + +@pytest.mark.parametrize( + "ddl", + [ + "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) PRIMARY KEY)", + "CREATE TABLE pc_unregistered_persistent_state (id INTEGER PRIMARY KEY)", + "CREATE VIEW unregistered_view AS SELECT artifact_id FROM pc_artifacts", + "DROP INDEX ix_pc_artifact_tags_key", + "CREATE INDEX unregistered_index ON pc_artifacts (family)", + "ALTER TABLE pc_artifacts ADD COLUMN unexpected TEXT", + "ALTER TABLE pc_artifact_heads RENAME TO altered_heads", + "CREATE TRIGGER unregistered_trigger AFTER DELETE ON pc_artifacts BEGIN SELECT 1; END", + ], +) +def test_unregistered_schema_changes_are_not_recognized(released_database: Path, ddl: str) -> None: + with sqlite3.connect(released_database) as connection: + connection.execute(ddl) + before = released_database.read_bytes() + assert recognize(released_database) is None + assert released_database.read_bytes() == before + + +def test_four_table_subset_is_not_a_complete_server_baseline(tmp_path: Path) -> None: + database = tmp_path / "subset.db" + selected = {"pc_artifacts", "pc_artifact_heads", "pc_artifact_tags", "pc_artifact_candidate_versions"} + inventory = json.loads(RESOURCE.read_text(encoding="utf-8"))["inventory"] + with sqlite3.connect(database) as connection: + for kind, name, _, sql in inventory: + if kind == "table" and name in selected: + connection.execute(sql) + before = database.read_bytes() + assert recognize(database) is None + with pytest.raises(MigrationError, match="unknown_baseline"): + SQLiteMigrationRunner(database, production_bundle()).plan() + assert database.read_bytes() == before + + +def test_weakened_constraint_with_identical_columns_is_not_recognized(released_database: Path) -> None: + with sqlite3.connect(released_database) as connection: + original = connection.execute("SELECT sql FROM sqlite_schema WHERE name='pc_artifact_heads'").fetchone()[0] + assert "CHECK (revision > 0)" in original + connection.execute("DROP TABLE pc_artifact_heads") + connection.execute(original.replace("CHECK (revision > 0)", "CHECK (revision >= 0)")) + before = released_database.read_bytes() + assert recognize(released_database) is None + assert released_database.read_bytes() == before From 3a8b38be1037672010f0bba1597060a9994c4dd3 Mon Sep 17 00:00:00 2001 From: Mao Meng <1707647656@qq.com> Date: Thu, 22 Oct 2026 16:39:41 +0800 Subject: [PATCH 32/94] fix(sources): preserve stable remote manifest fingerprints Keep the pre-upgrade validation schema unless aliases rename a required payload key, so unchanged Definitions with Decimal or optional aliased fields retain their registered fingerprints while generated aliases still emit a matching schema. --- src/powercontext/sources/observations.py | 66 +++++++++++++- tests/test_source_observations.py | 110 ++++++++++++++++++++++- 2 files changed, 170 insertions(+), 6 deletions(-) diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 3531a6ab81..596eff9b71 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -17,10 +17,20 @@ from __future__ import annotations import hashlib -from typing import Any, Literal +from typing import Any, Literal, get_args, get_origin import rfc8785 -from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, field_validator, model_validator +from pydantic import ( + AliasChoices, + AliasPath, + BaseModel, + ConfigDict, + Field, + JsonValue, + TypeAdapter, + field_validator, + model_validator, +) from powercontext.errors import ( InvalidSourceDefinitionError, @@ -172,7 +182,9 @@ def _manifest_for_definition( *, transport: bool, ) -> SourceDefinitionManifest: - if transport: + if transport and _validation_schema_matches_payload(definition.source_class): + source_schema = _json_object(definition.source_class.model_json_schema()) + elif transport: source_schema = _json_object( definition.source_class.model_json_schema( mode="serialization", @@ -232,6 +244,54 @@ def project_source_for_transport( ) +def _validation_schema_matches_payload( + source_class: type[BaseModel], + seen: set[type[BaseModel]] | None = None, +) -> bool: + """Return whether validation-mode keys already describe the serialized payload.""" + + visited = set() if seen is None else seen + if source_class in visited: + return True + visited.add(source_class) + serialize_by_alias = bool(source_class.model_config.get("serialize_by_alias")) + for name, field in source_class.model_fields.items(): + if field.exclude: + continue + serialization_key = (field.serialization_alias or field.alias or name) if serialize_by_alias else name + if field.is_required() and _validation_schema_key(name, field) != serialization_key: + return False + for nested in _nested_models(field.annotation): + if not _validation_schema_matches_payload(nested, visited): + return False + return True + + +def _validation_schema_key(name: str, field: Any) -> str | None: + alias = field.validation_alias + if isinstance(alias, str): + return alias + if isinstance(alias, AliasChoices): + return next((choice for choice in alias.choices if isinstance(choice, str)), None) + if isinstance(alias, AliasPath): + return name # Pydantic keeps the field name for path-based validation. + if alias is not None: + return None + return field.alias if field.alias is not None else name + + +def _nested_models(annotation: object) -> tuple[type[BaseModel], ...]: + origin = get_origin(annotation) + if origin is None: + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return (annotation,) + return () + nested: list[type[BaseModel]] = [] + for argument in get_args(annotation): + nested.extend(_nested_models(argument)) + return tuple(nested) + + def _source_definition_fingerprint( *, name: str, diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index 93cb1b7f52..cf3ff5cee0 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -17,11 +17,13 @@ import asyncio import hashlib import urllib.request +from decimal import Decimal from typing import Self import pytest import rfc8785 from pydantic import ConfigDict, Field, ValidationError, field_serializer +from pydantic.alias_generators import to_camel from referencing.exceptions import Unresolvable from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository @@ -222,6 +224,110 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_optional_alias_keeps_the_preexisting_manifest_fingerprint(tmp_path) -> None: + class VersionedSource(Source): + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + body: str + + class VersionedAdapter: + name = "versioned-source" + input_class = str + source_class = VersionedSource + + async def resolve(self, value: str, /) -> VersionedSource: + return VersionedSource( + name="versioned", + definition_version="2", + materialization=SourceMaterialization.CAPTURED, + body=value, + ) + + async def read(self, source: VersionedSource, /) -> str: + return source.body + + async def scenario() -> None: + definition = AdapterSourceDefinition(VersionedAdapter(), version="2") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("Retained.") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + baseline = { + "name": definition.name, + "version": definition.version, + "source_schema": VersionedSource.model_json_schema(), + "projections": [], + } + assert manifest.source_schema == VersionedSource.model_json_schema() + assert manifest.fingerprint == f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" + assert observation.payload["definition_version"] == "2" + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'optional-alias.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "versioned" + + asyncio.run(scenario()) + + +def test_decimal_source_manifest_fingerprint_survives_upgrade(tmp_path) -> None: + class AmountSource(Source): + amount: Decimal + + class AmountAdapter: + name = "amount-source" + input_class = str + source_class = AmountSource + + async def resolve(self, value: str, /) -> AmountSource: + return AmountSource( + name="amount", + definition_version="1", + materialization=SourceMaterialization.CAPTURED, + amount=Decimal(value), + ) + + async def read(self, source: AmountSource, /) -> str: + return str(source.amount) + + async def scenario() -> None: + definition = AdapterSourceDefinition(AmountAdapter(), version="1") + registry = SourceDefinitionRegistry((definition,)) + source = await registry.resolve("1.25") + manifest = manifest_for_definition(definition) + observation = project_source_for_transport(registry, source) + baseline = { + "name": definition.name, + "version": definition.version, + "source_schema": AmountSource.model_json_schema(), + "projections": [], + } + baseline_fingerprint = f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" + assert manifest.source_schema == AmountSource.model_json_schema() + assert manifest.fingerprint == baseline_fingerprint + assert observation.payload["amount"] == "1.25" + + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'decimal-upgrade.db'}") + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: + contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) + assert await contexts.register_source_definition(manifest) == manifest + receipt = await contexts.submit_source_observation( + SubmitSourceObservation(scope_id="scope-a", observation=observation) + ) + assert receipt.source_ref.source_id == "amount" + + asyncio.run(scenario()) + + def test_recursive_source_transport_preserves_canonical_nested_payload() -> None: class RecursiveSource(Source): name: str = Field(serialization_alias="source_name") @@ -324,9 +430,7 @@ async def scenario() -> None: for source_class in (ExcludedIdentitySource, TransformedIdentitySource): definition = AdapterSourceDefinition(Adapter(source_class), version="1") registry = SourceDefinitionRegistry((definition,)) - assert manifest_for_definition(definition).source_schema == source_class.model_json_schema( - mode="serialization", by_alias=False - ) + assert manifest_for_definition(definition).source_schema == source_class.model_json_schema() source = await registry.resolve("identity") with pytest.raises(ValidationError): project_source_for_transport(registry, source) From 8a8952050569886146bdc2a8900979c9959ee431 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 16:44:17 +0800 Subject: [PATCH 33/94] feat(migrations): register immutable MySQL schema layout fragments Resolve schema identification and recovery prefixes from the registered Alembic chain instead of a second fixed revision list. Freeze the unchanged historical DDL, reject duplicate or incomplete declarations, and bind appended fixture resources to their own revision checksums. Execute parameter-free DDL without driver interpolation so remainder CHECKs and percent literals survive actual fresh initialization. Keep exact frozen resource bytes consistent across Windows and Linux checkouts. Constraint: Existing p0001-p0005 canonical resource bytes/checksums and historical schema/prefix definitions remain unchanged. Complete Server upgrade and readiness stay blocked. Scope-risk: The extension fixture verifies real SQLite execution and MySQL schema recognition; full future MySQL wrapper and domain integration remain pending. Tested: Linux migration suites with real seekDB 128 passed/13 skipped; Windows recognition/CHECK suites 69 passed/4 skipped; two real-driver regressions failed before the DDL fix and passed afterward; full Linux Ty/Python 3.11 and changed-file hooks passed. --- .gitattributes | 4 + docs/en/docs/operate/database-migrations.md | 21 ++- docs/zh/docs/operate/database-migrations.md | 9 +- .../builtin/persistence/migrations/bundle.py | 8 ++ .../builtin/persistence/migrations/mysql.py | 8 +- .../persistence/migrations/mysql_schema.py | 85 +++++------ .../migrations/resources/manifest.json | 5 +- .../migrations/resources/mysql_layouts.json | 31 ++++ .../test_mysql_migration_checks.py | 18 +++ .../test_mysql_migration_schema.py | 133 ++++++++++++++++-- 10 files changed, 266 insertions(+), 56 deletions(-) create mode 100644 src/powercontext/builtin/persistence/migrations/resources/mysql_layouts.json diff --git a/.gitattributes b/.gitattributes index 54cbafd6b3..f934560f8e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,3 +6,7 @@ integrations/dsh/plugins/powercontext/lib/*.js text eol=lf integrations/dsh/plugins/powercontext/lib/*.d.ts text eol=lf # Generated guidance compares canonical bytes; keep user notes outside its markers untouched. integrations/**/skills/powercontext-project-context/** text eol=lf +# Migration evidence hashes exact resource bytes on every platform. +src/powercontext/builtin/persistence/migrations/resources/** text eol=lf +src/powercontext/builtin/persistence/migrations/mysql_resources/** text eol=lf +src/powercontext/builtin/persistence/migration_resources/** text eol=lf diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 5e40364468..1c5e670fd7 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -19,6 +19,11 @@ The default `--bundle artifacts` ships frozen resources for Phase A acceptance o `pc_artifact_candidate_versions`, and `pc_artifact_tags`. Only empty targets and registered historical layouts are accepted. A complete Server database containing unmanaged objects is rejected. +The complete SQLite schema captured from the installed public 1.3.0 wheel is recognized separately, including its FTS +shadow tables, indexes, and triggers. Maintenance reports `server_migration_unavailable` for this source because a complete +Server upgrade path is not registered. Recognition does not stamp a revision, validate domain data, or authorize writes. +Other complete schemas remain unknown unless their exact shape is registered. + `ready` means this migration bundle passed its schema and data checks. Output explicitly includes `readiness_scope=registered_bundle` and `server_ready=false`: it does not establish complete Server, index, legacy-task, or cluster readiness. Do not substitute it for a production upgrade or initialization of the complete business database. @@ -344,10 +349,24 @@ weaken coordination and continue. These locks do not stop business writers. seekdb/OceanBase DDL may commit implicitly, so one Python transaction cannot make the entire upgrade atomic. Recovery recognizes only the registered sequence: `p0001` creates the four frozen tables and two indexes; `p0002` adds the two nullable `MEDIUMBLOB` citation columns; `p0003` removes the tag-family CHECK. Actual schema, data checks, and persisted -maintenance evidence must establish the known completed prefix before continuing. Alembic advances the version only +maintenance evidence must establish the known completed prefix before continuing. `p0004` adds exact-revision Artifact +handles; `p0005` adds Source evidence storage without attesting historical content. Alembic advances the version only after the revision's postconditions pass. Unknown objects, layouts, or partial transitions block execution instead of being optimistically stamped. +MySQL schema steps are registered in frozen layout fragments selected by the `mysql_layouts` list in `manifest.json`. +The fragments together must cover exactly the Alembic chain, without duplicate declarations. New revisions append +immutable fragments and declare their files in `revision_resources`; they do not edit previously released fragments. +Optional checkpoints compare the accumulated steps with independently +frozen complete layouts. Reflection, next-revision prefix recovery, and version-table validation use that same registered +chain; adding a revision does not require changing a second version list in the recognizer. These schema registrations +do not replace domain conversions, compatibility checks, or full Server readiness. + +The reviewed plan binds the registration through the complete bundle checksum. Active maintenance requires the same +bundle for retry. Released revision resources and their recorded checksums remain immutable. +Parameter-free DDL uses statement-scoped driver execution options so CHECK remainder operators and percent-containing +SQL literals reach the backend unchanged. + Repositories run after schema checks. Versioned resources and controlled rebuilds for complete business full-text/vector indexes, plus legacy-task readiness, remain outside this acceptance bundle. diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index bed350885c..d734f8f071 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -12,6 +12,8 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 `powercontext server db-migrate status/plan/apply/verify` 使用部署配置中的持久 SQLite、嵌入式 seekdb 或 OceanBase MySQL 模式数据库，不接受任意测试 bundle 路径。维护命令以显式环境文件为准，应使用部署的环境文件，不依赖临时 shell 环境覆盖。随 wheel 发布的冻结资源用于阶段 A 的 Artifact 四表迁移：`pc_artifacts`、`pc_artifact_heads`、`pc_artifact_candidate_versions`、`pc_artifact_tags`。仅接受空目标和已登记的历史结构；完整 Server 数据库含有尚未纳管的对象时拒绝执行。 +维护入口可以单独识别由公开 1.3.0 wheel 实际初始化的完整 SQLite 结构，包括 FTS 影子表、索引和触发器。由于完整 Server 升级路径尚未注册，该来源返回 `server_migration_unavailable`。识别不会登记 revision、验证领域数据或允许写入；其他完整结构仍需登记精确形状。 + 输出中的 `ready` 仅表示当前迁移 bundle 已通过结构和数据验证；`readiness_scope=registered_bundle`、`server_ready=false` 明确区分它与完整 Server、索引、旧任务及集群可用。不要以此替代生产升级或完整业务库初始化。普通业务启动的全面版本门禁和完整历史基线尚未纳管。已通过 OceanBase AI 4.6.3 的 ODP 入口完成隔离四表验收，覆盖固定维护主机互斥、DDL 提交后中断的显式确认恢复、数据保留与重复执行。这不代表完整 Server 升级、崩溃后远端 DDL 自动判定或完整 Fork 恢复已通过验收。 MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、`*`、`/`、`%`、一元正负号及嵌套括号，并保留算术、比较和布尔运算的优先级。未支持的 CHECK 运算符或函数仍被拒绝，错误会标明反射的表名、约束名及未支持的语法，避免将合法分组误报为括号不匹配。表达式解析成功不代表历史结构已登记，也不会绕过迁移或数据完整性检查。 @@ -216,7 +218,12 @@ powercontext server run --env-file deployment.env --role all 维护执行器复用配置，使用专用连接承载 Alembic；冻结历史 SQL 不依赖当前 Repository 或应用 metadata。seekdb 从启动引擎前到关闭引擎后持有规范化目录的锁。OceanBase 在每条 DDL 前后验证操作系统锁、主机绑定、持久运行记录及维护连接。正常结束时先关闭迁移主连接，才清理运行记录并释放文件锁；引擎随后关闭。执行中断或关闭连接失败均保留记录；后续执行需要准确的旧运行 token，以及 DBA 确认远端执行已结束的人工声明。PC 不从本机进程退出或本地锁可获取推断远端执行结束。锁文件被替换、连接丢失或旧运行未确认时阻止继续。平台不支持原生文件锁时拒绝执行，包括运行时降级为软锁的情况，不重连或静默弱化互斥继续迁移。这些锁不阻止业务写入者。 -seekdb／OceanBase DDL 可能隐式提交，不能用 Python 事务承诺整个迁移原子化。恢复只识别已登记顺序：`p0001` 创建冻结四表及两个索引；`p0002` 添加两个可空的 `MEDIUMBLOB` 引用列；`p0003` 删除标签 family 的 CHECK。继续执行前须用实际结构、数据检查和持久维护证据证明已完成的已知前缀；revision 的后置条件通过后才推进 Alembic 版本。未知对象、结构或中间状态拒绝执行，不乐观 stamp。 +seekdb／OceanBase DDL 可能隐式提交，不能用 Python 事务承诺整个迁移原子化。恢复只识别已登记顺序：`p0001` 创建冻结四表及两个索引；`p0002` 添加两个可空的 `MEDIUMBLOB` 引用列；`p0003` 删除标签 family 的 CHECK；`p0004` 添加指向精确 revision 的 Artifact handle；`p0005` 添加 Source 证据存储，不追认历史内容的可信度。继续执行前须用实际结构、数据检查和持久维护证据证明已完成的已知前缀；revision 的后置条件通过后才推进 Alembic 版本。未知对象、结构或中间状态拒绝执行，不乐观 stamp。 + +MySQL 结构步骤通过冻结的布局片段注册，由 `manifest.json` 的 `mysql_layouts` 列表选择，合起来必须恰好覆盖 Alembic 版本链，且不能重复声明。新增 revision 追加不可改写的片段，并在 `revision_resources` 中声明其文件，不修改已发布片段。可选 checkpoint 将累积步骤与独立冻结的完整结构对照。反射、下一 revision 的已知前缀恢复和版本表校验使用同一注册链；新增 revision 无需修改识别器中的另一份版本列表。这些结构注册不能代替领域转换、兼容性检查或完整 Server readiness。 + +计划通过完整 bundle 摘要绑定注册资源，未完成维护的重试要求使用同一 bundle。已发布 revision 资源及其记录摘要保持不可改写。 +无参数 DDL 使用单语句驱动选项，确保 CHECK 的取余运算符和 SQL 字面量中的百分号原样到达后端。 Repository 仅在结构满足后运行。完整业务的全文／向量索引版本管理及受控重建、旧任务 readiness 尚未包含在本验收 bundle 中。 diff --git a/src/powercontext/builtin/persistence/migrations/bundle.py b/src/powercontext/builtin/persistence/migrations/bundle.py index c38bc2671e..1921362ab3 100644 --- a/src/powercontext/builtin/persistence/migrations/bundle.py +++ b/src/powercontext/builtin/persistence/migrations/bundle.py @@ -67,6 +67,14 @@ def __init__(self, directory: Path) -> None: self.revisions = tuple(item.revision for item in revisions) self.head = self.revisions[-1] manifest = json.loads(resources["manifest.json"]) + mysql_layouts = manifest.get("mysql_layouts", []) + if ( + not isinstance(mysql_layouts, list) + or any(not isinstance(name, str) or name not in resources for name in mysql_layouts) + or len(set(mysql_layouts)) != len(mysql_layouts) + ): + raise MigrationError("invalid_bundle", "MySQL layout registration is outside the frozen bundle.") + self.mysql_layouts: tuple[str, ...] = tuple(mysql_layouts) self.affected_objects = tuple(manifest.get("affected_objects", ())) self.retained_objects = tuple(manifest.get("retained_objects", ())) self.retained_fingerprints = manifest.get("retained_fingerprints", {}) diff --git a/src/powercontext/builtin/persistence/migrations/mysql.py b/src/powercontext/builtin/persistence/migrations/mysql.py index abfccff3a6..0006db5711 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql.py +++ b/src/powercontext/builtin/persistence/migrations/mysql.py @@ -416,7 +416,7 @@ def upgrade(revision: str) -> None: raise MigrationError("recovery_required", "The stored intent does not match the schema transition.") for statement in recovery.pending_statements: guard() - connection.exec_driver_sql(statement) + connection.exec_driver_sql(statement, execution_options={"no_parameters": True}) # DDL may already have committed; rollback cannot undo it. guard() self.schema.recovery(mysql_inventory(connection), source) @@ -456,7 +456,11 @@ def _inspect( inventory = ( mysql_inventory(connection) if connection is not None else {"tables": {}, "views": [], "triggers": []} ) - initialized, current = read_revision(connection, inventory) if connection is not None else (False, None) + initialized, current = ( + read_revision(connection, inventory, revisions=self.bundle.revisions) + if connection is not None + else (False, None) + ) self._validate_evidence(evidence) current, adopt = self._recognize_source(inventory, initialized, current, evidence) if initialized and self.bundle.pending(current) and evidence is None: diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py index a3112f44ac..f5118a5567 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql_schema.py +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -14,7 +14,7 @@ """Recognize the frozen MySQL layouts without importing application metadata. -Only the reviewed linear p0001--p0005 history is supported. DDL recovery accepts +Registered resources describe the complete linear history. DDL recovery accepts an exact prefix of the next revision, never an arbitrary collection of objects. The caller owns maintenance evidence, backup, the pinned database lock and the version row; this module never writes or stamps a revision. @@ -37,20 +37,7 @@ _LEXEME = re.compile( r"\s+|`(?:``|[^`])*`|'(?:''|\\.|[^'\\])*'|[A-Za-z_][A-Za-z_0-9$]*|[0-9]+|>=|<=|<>|!=|[(),.=<>+*/%\^&|~!?:@-]" ) -_REVISIONS = ("p0001", "p0002", "p0003", "p0004", "p0005") -_CITATION_TABLES = ("pc_artifacts", "pc_artifact_candidate_versions") -_FAMILY_CHECK = "ck_pc_artifact_tags_family" _CONTROL_TABLE = "pc_schema_revision" -_ARTIFACT_HANDLES_DDL = """CREATE TABLE pc_artifact_handles ( - scope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, - family VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, - handle_key VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, - artifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, - revision INTEGER NOT NULL, - PRIMARY KEY (scope_id, family, handle_key), - FOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, - CONSTRAINT ck_pc_artifact_handles_revision_positive CHECK (revision > 0) -)""" _PHYSICAL_VALUES = frozenset({ "row_format", "compression", @@ -137,13 +124,15 @@ def _reflect_tables(connection: Connection, database: str, exclude: frozenset[st inventory["tables"][name] = table -def read_revision(connection: Connection, inventory: dict[str, Any]) -> tuple[bool, str | None]: +def read_revision( + connection: Connection, inventory: dict[str, Any], *, revisions: tuple[str, ...] +) -> tuple[bool, str | None]: """Validate Alembic's only control table before reading its zero/one head.""" _business_inventory(inventory) if _CONTROL_TABLE not in inventory["tables"]: return False, None rows = connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision LIMIT 2").all() - if len(rows) > 1 or (rows and rows[0][0] not in _REVISIONS): + if len(rows) > 1 or (rows and rows[0][0] not in revisions): raise MigrationError("incompatible_schema", "The database has multiple or unknown migration heads.") return True, rows[0][0] if rows else None @@ -152,36 +141,52 @@ class MySQLSchema: """Validate source, intermediate and post-DDL shapes from frozen resources.""" def __init__(self, bundle: MigrationBundle) -> None: - if bundle.revisions != _REVISIONS: - raise MigrationError("invalid_bundle", "MySQL schema recognition does not cover this revision chain.") self.bundle = bundle - baseline = json.loads((bundle.snapshot_directory / "schemas/pre_dream.json").read_text())["mysql"] - current = json.loads((bundle.snapshot_directory / "schemas/v1_1_0.json").read_text())["mysql"] - evidence = json.loads((bundle.snapshot_directory / "schemas/atomic_memory_evidence_v1.json").read_text())[ - "mysql" - ] - self._statements = { - "p0001": tuple(baseline), - "p0002": tuple(f"ALTER TABLE {table} ADD COLUMN memory_citations MEDIUMBLOB" for table in _CITATION_TABLES), - "p0003": (f"ALTER TABLE pc_artifact_tags DROP CHECK {_FAMILY_CHECK}",), - "p0004": (_ARTIFACT_HANDLES_DDL,), - "p0005": tuple(evidence[:-1]), - } + if not bundle.mysql_layouts: + raise MigrationError("invalid_bundle", "MySQL layout steps are not registered in this bundle.") + self._statements: dict[str, tuple[str, ...]] = {} + checkpoints: dict[str, str] = {} + for resource in bundle.mysql_layouts: + layouts = json.loads((bundle.snapshot_directory / resource).read_text(encoding="utf-8")) + if ( + not isinstance(layouts, dict) + or layouts.get("format") != "powercontext.mysql-layouts.v1" + or not isinstance(layouts.get("revisions"), dict) + or not isinstance(layouts.get("checkpoints"), dict) + or set(layouts["revisions"]) & self._statements.keys() + or set(layouts["checkpoints"]) & checkpoints.keys() + or any( + not isinstance(steps, list) or any(not isinstance(step, str) or not step.strip() for step in steps) + for steps in layouts["revisions"].values() + ) + or any( + not isinstance(name, str) or name not in bundle.resources + for name in layouts["checkpoints"].values() + ) + ): + raise MigrationError("invalid_bundle", "MySQL layout registration is invalid or duplicated.") + self._statements.update({revision: tuple(steps) for revision, steps in layouts["revisions"].items()}) + checkpoints.update(layouts["checkpoints"]) + if set(self._statements) != set(bundle.revisions) or not set(checkpoints) <= set(bundle.revisions): + raise MigrationError("invalid_bundle", "MySQL layout registration does not cover the revision chain.") inventory = _empty_inventory() self._prefixes: dict[str, list[dict[str, Any]]] = {} self._schemas: dict[str, dict[str, Any]] = {} - for revision in _REVISIONS: + for revision in bundle.revisions: prefixes = [copy.deepcopy(inventory)] for statement in self._statements[revision]: _apply_statement(inventory, statement) prefixes.append(copy.deepcopy(inventory)) self._prefixes[revision] = prefixes self._schemas[revision] = copy.deepcopy(inventory) - frozen_current = _empty_inventory() - for statement in current: - _apply_statement(frozen_current, statement) - if frozen_current != self._schemas["p0004"]: - raise MigrationError("invalid_bundle", "Frozen MySQL layouts disagree with the registered revision steps.") + for revision, resource in checkpoints.items(): + frozen = _empty_inventory() + for statement in json.loads((bundle.snapshot_directory / resource).read_text(encoding="utf-8"))["mysql"]: + _apply_statement(frozen, statement) + if frozen != self._schemas[revision]: + raise MigrationError( + "invalid_bundle", "Frozen MySQL layouts disagree with the registered revision steps." + ) @staticmethod def fingerprint(inventory: dict[str, Any]) -> str: @@ -211,12 +216,10 @@ def recovery(self, inventory: dict[str, Any], revision: str | None) -> MySQLReco for advancing the version row, and is not itself recovery authorization. """ inventory = _business_inventory(inventory) - if revision is None: - target = _REVISIONS[0] - elif revision not in _REVISIONS or revision == _REVISIONS[-1]: + pending = self.bundle.pending(revision) + if not pending: raise MigrationError("incompatible_schema", "There is no registered next MySQL revision to recover.") - else: - target = _REVISIONS[_REVISIONS.index(revision) + 1] + target = pending[0] for position, expected in enumerate(self._prefixes[target]): if _matches(inventory, expected): return MySQLRecovery(target, self._statements[target][position:]) diff --git a/src/powercontext/builtin/persistence/migrations/resources/manifest.json b/src/powercontext/builtin/persistence/migrations/resources/manifest.json index dc2b373564..63887addc7 100644 --- a/src/powercontext/builtin/persistence/migrations/resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/resources/manifest.json @@ -56,5 +56,8 @@ }, "retained_revisions": { "pc_retained_p0002_artifact_tags": "p0003" - } + }, + "mysql_layouts": [ + "mysql_layouts.json" + ] } diff --git a/src/powercontext/builtin/persistence/migrations/resources/mysql_layouts.json b/src/powercontext/builtin/persistence/migrations/resources/mysql_layouts.json new file mode 100644 index 0000000000..6442266801 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/mysql_layouts.json @@ -0,0 +1,31 @@ +{ + "format": "powercontext.mysql-layouts.v1", + "revisions": { + "p0001": [ + "CREATE TABLE pc_artifacts (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent MEDIUMBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)", + "CREATE TABLE pc_artifact_heads (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsearchable_text MEDIUMTEXT, \n\tlifecycle_state VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'active', \n\treplacement_artifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tgovernance_generation BIGINT NOT NULL DEFAULT '0', \n\tPRIMARY KEY (scope_id, family, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_heads_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_artifact_heads_lifecycle_state CHECK (lifecycle_state IN ('active', 'deprecated', 'retired')), \n\tCONSTRAINT ck_pc_artifact_heads_governance_generation_nonnegative CHECK (governance_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_heads_replacement_deprecated CHECK (replacement_artifact_id IS NULL OR lifecycle_state = 'deprecated')\n)", + "CREATE TABLE pc_artifact_candidate_versions (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tcandidate_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tversion INTEGER NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tproposal MEDIUMBLOB NOT NULL, \n\tsource_refs MEDIUMBLOB NOT NULL, \n\tartifact_refs MEDIUMBLOB NOT NULL, \n\ttarget_family VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\ttarget_artifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\ttarget_revision INTEGER, \n\treason MEDIUMTEXT, \n\tPRIMARY KEY (scope_id, candidate_id, version), \n\tFOREIGN KEY(scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_candidate_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_artifact_candidate_versions_target_complete CHECK ((target_family IS NULL AND target_artifact_id IS NULL AND target_revision IS NULL) OR (target_family IS NOT NULL AND target_artifact_id IS NOT NULL AND target_revision > 0))\n)", + "CREATE TABLE pc_artifact_tags (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttarget_type VARCHAR(12) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttarget_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttag_key_hash BINARY(32) NOT NULL, \n\ttag_key VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttag VARCHAR(64) NOT NULL, \n\tassigned_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, target_type, target_id, tag_key_hash), \n\tFOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads (scope_id, family, artifact_id) ON DELETE CASCADE, \n\tCONSTRAINT ck_pc_artifact_tags_family CHECK (family IN ('memory', 'experience', 'skill', 'handoff')), \n\tCONSTRAINT ck_pc_artifact_tags_target CHECK ((target_type = 'artifact' AND target_id = artifact_id) OR (target_type = 'memory_entry' AND family = 'memory'))\n)", + "CREATE INDEX ix_pc_artifact_tags_family_key ON pc_artifact_tags (scope_id, family, tag_key_hash, target_type, artifact_id, target_id)", + "CREATE INDEX ix_pc_artifact_tags_key ON pc_artifact_tags (scope_id, tag_key_hash, family, target_type, artifact_id, target_id)" + ], + "p0002": [ + "ALTER TABLE pc_artifacts ADD COLUMN memory_citations MEDIUMBLOB", + "ALTER TABLE pc_artifact_candidate_versions ADD COLUMN memory_citations MEDIUMBLOB" + ], + "p0003": [ + "ALTER TABLE pc_artifact_tags DROP CHECK ck_pc_artifact_tags_family" + ], + "p0004": [ + "CREATE TABLE pc_artifact_handles (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\thandle_key VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\trevision INTEGER NOT NULL,\n\tPRIMARY KEY (scope_id, family, handle_key),\n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT,\n\tCONSTRAINT ck_pc_artifact_handles_revision_positive CHECK (revision > 0)\n)" + ], + "p0005": [ + "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", + "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", + "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_type VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tmaterial_digest VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tdefinition_name MEDIUMTEXT NOT NULL, \n\tdefinition_version MEDIUMTEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tdeclaration LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)" + ] + }, + "checkpoints": { + "p0004": "schemas/v1_1_0.json" + } +} diff --git a/tests/builtin/persistence/test_mysql_migration_checks.py b/tests/builtin/persistence/test_mysql_migration_checks.py index fc779a9d93..c0e4a7d6ce 100644 --- a/tests/builtin/persistence/test_mysql_migration_checks.py +++ b/tests/builtin/persistence/test_mysql_migration_checks.py @@ -192,6 +192,10 @@ def test_real_seekdb_percent_checks_validate_data_and_plan( layout = json.loads(path.read_text()) layout["mysql"].append(ddl) path.write_text(json.dumps(layout, ensure_ascii=False)) + path = directory / "mysql_layouts.json" + layouts = json.loads(path.read_text()) + layouts["revisions"]["p0001"].append(ddl) + path.write_text(json.dumps(layouts), encoding="utf-8") bundle = MigrationBundle(directory) schema = MySQLSchema(bundle) connections = MaintenanceConnections(SeekDBConfig(path=short_tmp_path / "seekdb")) @@ -219,6 +223,20 @@ def invalidate(connection, _identity, guard): schema.validate_data(connection, "p0001") with _seekdb_scenario_lifetime(monkeypatch): + fresh = MySQLMigrationRunner( + MaintenanceConnections(SeekDBConfig(path=short_tmp_path / "fresh_seekdb")), + bundle, + evidence_directory=short_tmp_path / "fresh_evidence", + ) + fresh_plan = fresh.plan(backup_policy="skip") + fresh.apply( + plan_id=fresh_plan.plan_id, + accepted=True, + maintenance_confirmed=True, + backup_policy="skip", + accept_no_backup=True, + ) + assert fresh.verify().state == "ready" connections.run(prepare, writable=True) assert runner.plan().state == "migration_required" connections.run(invalidate, writable=True) diff --git a/tests/builtin/persistence/test_mysql_migration_schema.py b/tests/builtin/persistence/test_mysql_migration_schema.py index 268a9a1cb8..9cf278d367 100644 --- a/tests/builtin/persistence/test_mysql_migration_schema.py +++ b/tests/builtin/persistence/test_mysql_migration_schema.py @@ -19,6 +19,7 @@ import json import os import re +import shutil from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -26,9 +27,10 @@ import pytest from sqlalchemy import Connection, create_engine -from powercontext.builtin.persistence.migrations.bundle import MigrationBundle +from powercontext.builtin.persistence.migrations import SQLiteMigrationRunner +from powercontext.builtin.persistence.migrations.bundle import MigrationBundle, sqlite_inventory from powercontext.builtin.persistence.migrations.connections import MaintenanceConnections -from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.models import MigrationError, digest from powercontext.builtin.persistence.migrations.mysql_schema import MySQLSchema, mysql_inventory, read_revision from powercontext.builtin.persistence.seekdb import SeekDBConfig @@ -87,6 +89,108 @@ def schema() -> MySQLSchema: return MySQLSchema(MigrationBundle(_RESOURCES)) +_NEXT_TABLE = "CREATE TABLE pc_extension_marker (id INTEGER NOT NULL, PRIMARY KEY (id))" + + +@pytest.fixture +def extended_bundle(tmp_path: Path) -> MigrationBundle: + directory = tmp_path / "bundle" + shutil.copytree(_RESOURCES, directory) + database = tmp_path / "fingerprint.db" + runner = SQLiteMigrationRunner(database, MigrationBundle(_RESOURCES)) + runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True) + engine = create_engine("sqlite:///" + str(database)) + try: + with engine.begin() as connection: + connection.exec_driver_sql(_NEXT_TABLE) + fingerprint = digest( + sqlite_inventory(connection, exclude=frozenset({"pc_schema_revision", *runner.bundle.retained_objects})) + ) + finally: + engine.dispose() + manifest_path = directory / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["revision_resources"]["fixture_next"] = ["mysql_fixture_next.json"] + manifest["sqlite_fingerprints"]["fixture_next"] = [fingerprint] + manifest["mysql_layouts"].append("mysql_fixture_next.json") + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + (directory / "mysql_fixture_next.json").write_text( + json.dumps({ + "format": "powercontext.mysql-layouts.v1", + "revisions": {"fixture_next": [_NEXT_TABLE]}, + "checkpoints": {}, + }), + encoding="utf-8", + ) + (directory / "versions/fixture_next.py").write_text( + "from alembic import op\n" + "revision = 'fixture_next'\n" + "down_revision = 'p0005'\n" + f"def upgrade():\n op.execute({_NEXT_TABLE!r})\n", + encoding="utf-8", + ) + return MigrationBundle(directory) + + +def test_registered_next_revision_uses_the_same_recognition_and_sqlite_executor( + tmp_path: Path, extended_bundle: MigrationBundle +) -> None: + schema = MySQLSchema(extended_bundle) + layouts = json.loads((extended_bundle.snapshot_directory / "mysql_layouts.json").read_text()) + tables = _show_tables(citations=2, drop_family_check=True) + for revision in ("p0004", "p0005"): + for statement in layouts["revisions"][revision]: + tables[statement.split()[2]] = statement + source = _ReflectedDatabase(tables) + inventory = _inventory(source) + assert schema.identify(inventory) == "p0005" + recovery = schema.recovery(inventory, "p0005") + assert recovery.revision == "fixture_next" + assert recovery.pending_statements == (_NEXT_TABLE,) + source.tables["pc_extension_marker"] = _NEXT_TABLE + inventory = _inventory(source) + assert schema.identify(inventory) == "fixture_next" + schema.verify(inventory, "fixture_next") + assert schema.recovery(inventory, "p0005").completed_revision == "fixture_next" + assert extended_bundle.pending("p0003") == ("p0004", "p0005", "fixture_next") + source.tables["pc_schema_revision"] = ( + "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, PRIMARY KEY (version_num))" + ) + source.heads = ["fixture_next"] + assert read_revision(cast(Connection, source), _inventory(source), revisions=extended_bundle.revisions) == ( + True, + "fixture_next", + ) + + runner = SQLiteMigrationRunner(tmp_path / "adjacent.db", MigrationBundle(_RESOURCES)) + runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True) + adjacent = SQLiteMigrationRunner(runner.database, extended_bundle) + plan = adjacent.plan() + assert plan.revisions == ("fixture_next",) + adjacent.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + direct = SQLiteMigrationRunner(tmp_path / "direct.db", extended_bundle) + direct.apply(plan_id=direct.plan().plan_id, accepted=True, maintenance_confirmed=True) + assert adjacent.verify().revision == direct.verify().revision == "fixture_next" + assert adjacent.plan().schema_fingerprint == direct.plan().schema_fingerprint + + +@pytest.mark.parametrize("change", ["missing", "unregistered", "duplicate"]) +def test_mysql_layout_registration_must_cover_exactly_the_revision_chain( + extended_bundle: MigrationBundle, change: str +) -> None: + path = extended_bundle.directory / "mysql_fixture_next.json" + layouts = json.loads(path.read_text()) + if change == "missing": + del layouts["revisions"]["fixture_next"] + elif change == "unregistered": + layouts["revisions"]["unregistered"] = [] + else: + layouts["revisions"]["p0001"] = [] + path.write_text(json.dumps(layouts), encoding="utf-8") + with pytest.raises(MigrationError, match="invalid_bundle"): + MySQLSchema(MigrationBundle(extended_bundle.directory)) + + def _append_clause(statement: str, clause: str) -> str: return statement.replace("\n)", ",\n\t" + clause + "\n)") @@ -268,28 +372,34 @@ def test_unknown_objects_or_changed_integrity_constraints_cannot_be_adopted(sche def test_version_table_is_validated_before_reading_and_is_not_a_business_table(schema: MySQLSchema) -> None: database = _ReflectedDatabase(_show_tables()) - assert read_revision(cast(Connection, database), _inventory(database)) == (False, None) + assert read_revision(cast(Connection, database), _inventory(database), revisions=schema.bundle.revisions) == ( + False, + None, + ) database.tables["pc_schema_revision"] = ( "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, " "CONSTRAINT pc_schema_revision_pkc PRIMARY KEY (version_num)\n)" ) - assert read_revision(cast(Connection, database), _inventory(database)) == (True, None) + assert read_revision(cast(Connection, database), _inventory(database), revisions=schema.bundle.revisions) == ( + True, + None, + ) database.heads = ["p0001"] inventory = _inventory(database) - assert read_revision(cast(Connection, database), inventory) == (True, "p0001") + assert read_revision(cast(Connection, database), inventory, revisions=schema.bundle.revisions) == (True, "p0001") assert schema.identify(inventory) == "p0001" schema.verify(inventory, "p0001") database.heads = ["p0001", "p0002"] with pytest.raises(MigrationError, match="multiple or unknown"): - read_revision(cast(Connection, database), inventory) + read_revision(cast(Connection, database), inventory, revisions=schema.bundle.revisions) database.heads = ["unknown"] with pytest.raises(MigrationError, match="multiple or unknown"): - read_revision(cast(Connection, database), inventory) + read_revision(cast(Connection, database), inventory, revisions=schema.bundle.revisions) database.tables["pc_schema_revision"] = _append_clause( database.tables["pc_schema_revision"], "KEY unexpected (version_num)" ) with pytest.raises(MigrationError, match="version table"): - read_revision(cast(Connection, database), _inventory(database)) + read_revision(cast(Connection, database), _inventory(database), revisions=schema.bundle.revisions) @pytest.mark.parametrize("violation", ["foreign_key", "check"]) @@ -317,13 +427,16 @@ def test_real_seekdb_frozen_schema_reflection_and_revision_recovery(short_tmp_pa def migrate(connection, _identity, guard) -> None: assert connection is not None - assert read_revision(connection, mysql_inventory(connection)) == (False, None) + assert read_revision(connection, mysql_inventory(connection), revisions=schema.bundle.revisions) == ( + False, + None, + ) connection.exec_driver_sql( "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, " "CONSTRAINT pc_schema_revision_pkc PRIMARY KEY (version_num))" ) connection.commit() - assert read_revision(connection, mysql_inventory(connection)) == (True, None) + assert read_revision(connection, mysql_inventory(connection), revisions=schema.bundle.revisions) == (True, None) source = None for target in ("p0001", "p0002", "p0003"): while True: From 881335ffe2f85f8975f8e5d688c3a89b30945735 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 17:31:10 +0800 Subject: [PATCH 34/94] feat(migrations): recognize complete released seekDB Server schemas Preserve FULLTEXT index identity, parser and token-size properties during native reflection, including four actual 1.3.0 definitions. Freeze the complete installed-release seekDB inventory and report the unavailable full Server upgrade without stamping a prototype revision. Constraint: p0001-p0005 resources, checksums and normalized layouts remain unchanged; full Server readiness stays disabled. Scope-risk: the baseline covers only the captured seekDB configuration, not OceanBase or optional embedding profiles. Unknown FULLTEXT properties reject and special indexes cannot be hidden as generated foreign-key indexes. Tested: FULLTEXT regressions fail before the fix; Windows 90 passed/28 external skips; Linux with real seekDB 105 passed/13 OceanBase skips. Reopened real installed 1.3.0 source retains all 56 tables, four FULLTEXT indexes and raw DDL; plan/verify/apply reject with every table's data hash unchanged and no maintenance evidence written. Full Python 3.11 Ty, changed-file hooks and immutable-resource comparison passed. --- .../server_baselines/1.3.0/seekdb.json | 11142 ++++++++++++++++ .../server_baselines/README.md | 23 +- .../builtin/persistence/migrations/mysql.py | 9 + .../persistence/migrations/mysql_schema.py | 41 +- .../migrations/server_baselines.py | 21 +- .../test_mysql_migration_schema.py | 63 + .../test_server_migration_baselines.py | 51 +- .../schemas/seekdb_v1_3_0_fulltext.json | 13 + 8 files changed, 11356 insertions(+), 7 deletions(-) create mode 100644 src/powercontext/builtin/persistence/migration_resources/server_baselines/1.3.0/seekdb.json create mode 100644 tests/fixtures/database_migrations/schemas/seekdb_v1_3_0_fulltext.json diff --git a/src/powercontext/builtin/persistence/migration_resources/server_baselines/1.3.0/seekdb.json b/src/powercontext/builtin/persistence/migration_resources/server_baselines/1.3.0/seekdb.json new file mode 100644 index 0000000000..fa18d68aba --- /dev/null +++ b/src/powercontext/builtin/persistence/migration_resources/server_baselines/1.3.0/seekdb.json @@ -0,0 +1,11142 @@ +{ + "format": "powercontext.server-baseline.v1", + "release": "1.3.0", + "backend": "seekdb", + "source_artifact": { + "filename": "powercontext-1.3.0-py3-none-any.whl", + "sha256": "8da7983763f3a6ab80b413c770a4973a46c557a54e1a1fa675099e58d47305e5", + "uploaded_at": "2026-10-12T03:55:56.682348Z" + }, + "capture": { + "python": "3.11.15", + "sqlalchemy": "2.0.51", + "pylibseekdb": "1.4.0.post1", + "engine": "5.7.25-OceanBase seekdb-v1.4.0.0", + "access_mode": "enforced", + "processing_role": "all", + "initialization": "Installed release Server; no model or embedding provider" + }, + "schema_fingerprint": "041ee0d1cbf114ddafab1df096f9e7debbff1edf5d3a459cf7db847b640ff76c", + "inventory": { + "tables": { + "pc_access_audit": { + "columns": { + "cursor": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [ + "auto_increment" + ] + }, + "event_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "occurred_at": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "request_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "transport": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "operation": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "principal_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "principal_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "principal_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "actor_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "actor_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "actor_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "action": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "resource_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "deployment_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selector_type": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selector_entry_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "allowed": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "reason_code": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "policy_revision": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "matched_subject_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "matched_subject_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "matched_subject_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "binding_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "role": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "expected_version": { + "type": [ + "int", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "result_version": { + "type": [ + "int", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "cursor", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": { + "event_id": { + "columns": [ + [ + "event_id", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_access_idempotency": { + "columns": { + "actor_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "idempotency_key_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "operation": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result_binding_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "secondary_binding_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "actor_id", + null, + "asc" + ], + [ + "idempotency_key_hash", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": { + "uq_pc_access_idempotency_actor_key": { + "columns": [ + [ + "actor_id", + null, + "asc" + ], + [ + "idempotency_key_hash", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_access_owners": { + "columns": { + "owner_kind": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "object_key_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "candidate_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selector_type": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selector_entry_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "owner_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "owner_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "owner_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "established_at": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "policy_revision": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "idempotency_key": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "owner_kind", + null, + "asc" + ], + [ + "object_key_hash", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_access_owners_kind": { + "expression": [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "owner_kind" + ], + [ + "literal", + "artifact" + ] + ], + [ + "is_not_null", + [ + "column", + "artifact_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "candidate_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "target_artifact_id" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "established_at" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "policy_revision" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "owner_kind" + ], + [ + "literal", + "candidate" + ] + ], + [ + "is_not_null", + [ + "column", + "candidate_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "artifact_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "selector_type" + ] + ] + ], + [ + "is_null", + [ + "column", + "selector_entry_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "established_at" + ] + ] + ], + [ + "is_null", + [ + "column", + "policy_revision" + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_access_relationship_heads": { + "columns": { + "name": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "name", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_access_relationship_heads_revision_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_access_relationships": { + "columns": { + "binding_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "subject_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "subject_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "subject_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "resource_key_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "resource_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "deployment_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selector_type": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selector_entry_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "role": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "singleton_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "granted_by_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "granted_by_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "granted_by_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "reason": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "created_at": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "expires_at": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "state": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "policy_revision": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "idempotency_key": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revoked_at": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revoked_by_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revoked_by_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revoked_by_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "binding_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_access_relationships_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "singleton_key": { + "columns": [ + [ + "singleton_key", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_agent_skill_targets": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "display_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "agent_kind": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "installation_scope": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "delivery_mode": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "installation_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "state": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "enrollment_token_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "enrollment_expires_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "credential_subject": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "credential_verifier": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "receiver_version": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "environment_fingerprint": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "machine_hostname": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "workspace_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "last_seen_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "created_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "updated_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "target_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_agent_skill_targets_agent_kind": { + "expression": [ + "in", + [ + "column", + "agent_kind" + ], + [ + [ + "literal", + "codex" + ], + [ + "literal", + "claude_code" + ] + ] + ], + "enforced": true + }, + "ck_pc_agent_skill_targets_installation_scope": { + "expression": [ + "binary", + "=", + [ + "column", + "installation_scope" + ], + [ + "literal", + "project" + ] + ], + "enforced": true + }, + "ck_pc_agent_skill_targets_delivery_mode": { + "expression": [ + "binary", + "=", + [ + "column", + "delivery_mode" + ], + [ + "literal", + "agent_pull" + ] + ], + "enforced": true + }, + "ck_pc_agent_skill_targets_state": { + "expression": [ + "in", + [ + "column", + "state" + ], + [ + [ + "literal", + "pending" + ], + [ + "literal", + "active" + ], + [ + "literal", + "revoked" + ] + ] + ], + "enforced": true + }, + "ck_pc_agent_skill_targets_state_payload": { + "expression": [ + "binary", + "or", + [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "state" + ], + [ + "literal", + "pending" + ] + ], + [ + "is_not_null", + [ + "column", + "enrollment_token_digest" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "enrollment_expires_at" + ] + ] + ], + [ + "is_null", + [ + "column", + "installation_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "credential_subject" + ] + ] + ], + [ + "is_null", + [ + "column", + "credential_verifier" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "state" + ], + [ + "literal", + "active" + ] + ], + [ + "is_null", + [ + "column", + "enrollment_token_digest" + ] + ] + ], + [ + "is_null", + [ + "column", + "enrollment_expires_at" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "installation_id" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "credential_subject" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "credential_verifier" + ] + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "state" + ], + [ + "literal", + "revoked" + ] + ], + [ + "is_null", + [ + "column", + "enrollment_token_digest" + ] + ] + ], + [ + "is_null", + [ + "column", + "enrollment_expires_at" + ] + ] + ], + [ + "is_null", + [ + "column", + "credential_verifier" + ] + ] + ] + ], + "enforced": true + }, + "ck_pc_agent_skill_targets_generation_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_agent_skill_targets_installation": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "agent_kind", + null, + "asc" + ], + [ + "installation_scope", + null, + "asc" + ], + [ + "installation_id", + null, + "asc" + ] + ], + "unique": true + }, + "uq_pc_agent_skill_targets_enrollment_token": { + "columns": [ + [ + "enrollment_token_digest", + null, + "asc" + ] + ], + "unique": true + }, + "uq_pc_agent_skill_targets_credential_subject": { + "columns": [ + [ + "credential_subject", + null, + "asc" + ] + ], + "unique": true + }, + "uq_pc_agent_skill_targets_credential_verifier": { + "columns": [ + [ + "credential_verifier", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_artifact_candidate_heads": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "candidate_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "status": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result_family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result_revision": { + "type": [ + "int", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "decision_reason": { + "type": [ + "mediumtext", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "candidate_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "candidate_id", + "version" + ], + "referred_schema": null, + "referred_table": "pc_artifact_candidate_versions", + "referred_columns": [ + "scope_id", + "candidate_id", + "version" + ], + "ondelete": "restrict", + "onupdate": "restrict" + }, + { + "columns": [ + "scope_id", + "result_family", + "result_artifact_id", + "result_revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_artifact_candidate_heads_status": { + "expression": [ + "in", + [ + "column", + "status" + ], + [ + [ + "literal", + "pending" + ], + [ + "literal", + "approved" + ], + [ + "literal", + "rejected" + ] + ] + ], + "enforced": true + }, + "ck_pc_artifact_candidate_heads_terminal_result": { + "expression": [ + "binary", + "or", + [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "status" + ], + [ + "literal", + "approved" + ] + ], + [ + "is_not_null", + [ + "column", + "result_family" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "result_artifact_id" + ] + ] + ], + [ + "binary", + ">", + [ + "column", + "result_revision" + ], + [ + "number", + 0 + ] + ] + ], + [ + "is_null", + [ + "column", + "decision_reason" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "status" + ], + [ + "literal", + "rejected" + ] + ], + [ + "is_null", + [ + "column", + "result_family" + ] + ] + ], + [ + "is_null", + [ + "column", + "result_artifact_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "result_revision" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "decision_reason" + ] + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "status" + ], + [ + "literal", + "pending" + ] + ], + [ + "is_null", + [ + "column", + "result_family" + ] + ] + ], + [ + "is_null", + [ + "column", + "result_artifact_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "result_revision" + ] + ] + ], + [ + "is_null", + [ + "column", + "decision_reason" + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_candidate_versions": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "candidate_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "proposal": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_refs": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "artifact_refs": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "target_family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_revision": { + "type": [ + "int", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "reason": { + "type": [ + "mediumtext", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "candidate_id", + null, + "asc" + ], + [ + "version", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "target_family", + "target_artifact_id", + "target_revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_artifact_candidate_versions_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_artifact_candidate_versions_target_complete": { + "expression": [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "is_null", + [ + "column", + "target_family" + ] + ], + [ + "is_null", + [ + "column", + "target_artifact_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "target_revision" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "is_not_null", + [ + "column", + "target_family" + ] + ], + [ + "is_not_null", + [ + "column", + "target_artifact_id" + ] + ] + ], + [ + "binary", + ">", + [ + "column", + "target_revision" + ], + [ + "number", + 0 + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_heads": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "searchable_text": { + "type": [ + "mediumtext", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "lifecycle_state": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": [ + "literal", + "active" + ], + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "replacement_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "merged_into_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "governance_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_artifact_heads_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_artifact_heads_lifecycle_state": { + "expression": [ + "in", + [ + "column", + "lifecycle_state" + ], + [ + [ + "literal", + "active" + ], + [ + "literal", + "deprecated" + ], + [ + "literal", + "retired" + ] + ] + ], + "enforced": true + }, + "ck_pc_artifact_heads_governance_generation_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "governance_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_artifact_heads_replacement_deprecated": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "replacement_artifact_id" + ] + ], + [ + "binary", + "=", + [ + "column", + "lifecycle_state" + ], + [ + "literal", + "deprecated" + ] + ] + ], + "enforced": true + }, + "ck_pc_artifact_heads_merge_target": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "merged_into_id" + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "lifecycle_state" + ], + [ + "literal", + "deprecated" + ] + ], + [ + "binary", + "<>", + [ + "column", + "merged_into_id" + ], + [ + "column", + "artifact_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "replacement_artifact_id" + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_artifact_heads_fts": { + "columns": [ + [ + "searchable_text", + null, + "asc" + ] + ], + "unique": false, + "kind": "fulltext", + "parser": "space", + "parser_properties": { + "min_token_size": 3, + "max_token_size": 84 + } + } + } + }, + "pc_artifact_lineage_artifacts": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "ordinal": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "upstream_family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "upstream_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "upstream_revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "is_merge_input": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": [ + "literal", + "0" + ], + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ], + [ + "ordinal", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "upstream_family", + "upstream_artifact_id", + "upstream_revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + }, + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": {} + }, + "pc_artifact_lineage_sources": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "ordinal": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_type": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ], + [ + "ordinal", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "source_type", + "source_id" + ], + "referred_schema": null, + "referred_table": "pc_sources", + "referred_columns": [ + "scope_id", + "source_type", + "source_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + }, + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": {} + }, + "pc_artifact_processing_auto_wave_targets": { + "columns": { + "wave_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_through": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "completed": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": [ + "literal", + "0" + ], + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "wave_id", + null, + "asc" + ], + [ + "binding_name", + null, + "asc" + ], + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_artifact_processing_auto_wave_target_source_positive": { + "expression": [ + "binary", + ">=", + [ + "column", + "source_through" + ], + [ + "number", + 1 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_processing_binding_states": { + "columns": { + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "last_auto_wave_completed_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "last_schedule_checkpoint_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "scan_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "scan_in_progress": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": [ + "literal", + "0" + ], + "charset": null, + "collation": null, + "extra": [] + }, + "scan_upper_pending_sequence": { + "type": [ + "bigint", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "binding_name", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_processing_scan_generation": { + "expression": [ + "binary", + ">=", + [ + "column", + "scan_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_processing_intents": { + "columns": { + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "pending_sequence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "dirty_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "clean_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "requested_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "handled_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "last_auto_scan_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "binding_name", + null, + "asc" + ], + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_processing_intent_sequence": { + "expression": [ + "binary", + ">", + [ + "column", + "pending_sequence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_processing_intent_dirty": { + "expression": [ + "binary", + "and", + [ + "binary", + ">=", + [ + "column", + "clean_generation" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + "<=", + [ + "column", + "clean_generation" + ], + [ + "column", + "dirty_generation" + ] + ] + ], + "enforced": true + }, + "ck_pc_processing_intent_request": { + "expression": [ + "binary", + "and", + [ + "binary", + ">=", + [ + "column", + "handled_generation" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + "<=", + [ + "column", + "handled_generation" + ], + [ + "column", + "requested_generation" + ] + ] + ], + "enforced": true + }, + "ck_pc_processing_intent_scan": { + "expression": [ + "binary", + ">=", + [ + "column", + "last_auto_scan_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "pending_sequence": { + "columns": [ + [ + "pending_sequence", + null, + "asc" + ] + ], + "unique": true + }, + "ix_pc_processing_intent_binding_sequence": { + "columns": [ + [ + "binding_name", + null, + "asc" + ], + [ + "pending_sequence", + null, + "asc" + ] + ], + "unique": false + } + } + }, + "pc_artifact_processing_leases": { + "columns": { + "supervisor_group": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "holder_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "supervisor_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "lease_expires_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "supervisor_group", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_artifact_processing_leases_generation_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "supervisor_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_processing_migration_receipts": { + "columns": { + "migration_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_snapshot": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "requested_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "dirty_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_through": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "migration_id", + null, + "asc" + ], + [ + "binding_name", + null, + "asc" + ], + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": {} + }, + "pc_artifact_processing_pending": { + "columns": { + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_through": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "flush_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "handled_flush_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "binding_name", + null, + "asc" + ], + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_artifact_processing_pending_source_positive": { + "expression": [ + "binary", + ">=", + [ + "column", + "source_through" + ], + [ + "number", + 1 + ] + ], + "enforced": true + }, + "ck_pc_artifact_processing_pending_flush_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "flush_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_artifact_processing_pending_handled_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "handled_flush_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_artifact_processing_pending_handled_not_ahead": { + "expression": [ + "binary", + "<=", + [ + "column", + "handled_flush_generation" + ], + [ + "column", + "flush_generation" + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_processing_schema": { + "columns": { + "singleton": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "schema_version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": [ + "number", + 1490 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "phase": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "migration_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "config_manifest": { + "type": [ + "text", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "singleton", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_processing_schema_singleton": { + "expression": [ + "binary", + "=", + [ + "column", + "singleton" + ], + [ + "number", + 1 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_processing_sequences": { + "columns": { + "singleton": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "sequence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "singleton", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_processing_sequence": { + "expression": [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "singleton" + ], + [ + "number", + 1 + ] + ], + [ + "binary", + ">=", + [ + "column", + "sequence" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_publications": { + "columns": { + "target_scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "content_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "idempotency_key": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "target_scope_id", + null, + "asc" + ], + [ + "target_family", + null, + "asc" + ], + [ + "target_artifact_id", + null, + "asc" + ], + [ + "target_revision", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "source_scope_id", + "source_family", + "source_artifact_id", + "source_revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + }, + { + "columns": [ + "target_scope_id", + "target_family", + "target_artifact_id", + "target_revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": { + "uq_pc_artifact_publications_request": { + "columns": [ + [ + "target_scope_id", + null, + "asc" + ], + [ + "idempotency_key", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_artifact_tags": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_type": { + "type": [ + "varchar", + [ + "12" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "tag_key_hash": { + "type": [ + "binary", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "tag_key": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "tag": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "assigned_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "target_type", + null, + "asc" + ], + [ + "target_id", + null, + "asc" + ], + [ + "tag_key_hash", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family", + "artifact_id" + ], + "referred_schema": null, + "referred_table": "pc_artifact_heads", + "referred_columns": [ + "scope_id", + "family", + "artifact_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_artifact_tags_target": { + "expression": [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "target_type" + ], + [ + "literal", + "artifact" + ] + ], + [ + "binary", + "=", + [ + "column", + "target_id" + ], + [ + "column", + "artifact_id" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "target_type" + ], + [ + "literal", + "memory_entry" + ] + ], + [ + "binary", + "=", + [ + "column", + "family" + ], + [ + "literal", + "memory" + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_artifact_tags_family_key": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "tag_key_hash", + null, + "asc" + ], + [ + "target_type", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "target_id", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_artifact_tags_key": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "tag_key_hash", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "target_type", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "target_id", + null, + "asc" + ] + ], + "unique": false + } + } + }, + "pc_artifacts": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "content": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": {} + }, + "pc_atomic_memory_current": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "state_version": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "content_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "projection_format": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "kind": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "searchable_text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "tag_keys": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "embedding": { + "type": [ + "blob", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "profile_fingerprint": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "embedding_input_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_atomic_memory_current_revision": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_atomic_memory_current_state_version": { + "expression": [ + "binary", + ">=", + [ + "column", + "state_version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_atomic_memory_current_embedding_metadata": { + "expression": [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "is_null", + [ + "column", + "embedding" + ] + ], + [ + "is_null", + [ + "column", + "profile_fingerprint" + ] + ] + ], + [ + "is_null", + [ + "column", + "embedding_input_hash" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "is_not_null", + [ + "column", + "embedding" + ] + ], + [ + "is_not_null", + [ + "column", + "profile_fingerprint" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "embedding_input_hash" + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_atomic_memory_current_fts": { + "columns": [ + [ + "searchable_text", + null, + "asc" + ] + ], + "unique": false, + "kind": "fulltext", + "parser": "space", + "parser_properties": { + "min_token_size": 3, + "max_token_size": 84 + } + } + } + }, + "pc_connector_checkpoints": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "binding_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "connector_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "connector_version": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "checkpoint": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "binding_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": {} + }, + "pc_decision_observations": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "observation_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "operation_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "created_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "consumer": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "policy_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "policy_version": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "mode": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "privacy_boundary": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "coverage": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "verdict": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "assessment_source": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "final_action": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "used_fallback": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "fallback_reason": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "provider_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "backend_model_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "model_policy_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "requests": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "input_tokens": { + "type": [ + "bigint", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "output_tokens": { + "type": [ + "bigint", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "latency_ms": { + "type": [ + "float", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "observation_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_decision_observations_requests_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "requests" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_decision_observations_input_tokens_nonnegative": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "input_tokens" + ] + ], + [ + "binary", + ">=", + [ + "column", + "input_tokens" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + }, + "ck_pc_decision_observations_output_tokens_nonnegative": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "output_tokens" + ] + ], + [ + "binary", + ">=", + [ + "column", + "output_tokens" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + }, + "ck_pc_decision_observations_latency_nonnegative": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "latency_ms" + ] + ], + [ + "binary", + ">=", + [ + "column", + "latency_ms" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_decision_observations_operation": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "operation_id", + null, + "asc" + ], + [ + "created_at", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_decision_observations_scope_created": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "created_at", + null, + "asc" + ], + [ + "observation_id", + null, + "asc" + ] + ], + "unique": false + } + } + }, + "pc_dream_runs": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "run_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "principal_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "idempotency_key": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "request_digest": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "operation": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "status": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "accepted_at": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "generation": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "request_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "run_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_dream_generation": { + "expression": [ + "binary", + ">=", + [ + "column", + "generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_dream_idempotency": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "principal_key", + null, + "asc" + ], + [ + "idempotency_key", + null, + "asc" + ] + ], + "unique": true + }, + "ix_pc_dream_list": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "accepted_at", + null, + "asc" + ], + [ + "run_id", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_dream_dispatch": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "operation", + null, + "asc" + ], + [ + "status", + null, + "asc" + ], + [ + "request_generation", + null, + "asc" + ] + ], + "unique": false + } + } + }, + "pc_external_skill_registrations": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "external_skill_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "provider": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "agent_kind": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "host_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "installation_scope": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "locator": { + "type": [ + "varchar", + [ + "2000" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "locator_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fingerprint": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "description": { + "type": [ + "varchar", + [ + "2000" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "external_skill_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_external_skill_registrations_scope": { + "expression": [ + "in", + [ + "column", + "installation_scope" + ], + [ + [ + "literal", + "user" + ], + [ + "literal", + "project" + ], + [ + "literal", + "plugin" + ] + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_external_skill_registrations_binding": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "provider", + null, + "asc" + ], + [ + "host_id", + null, + "asc" + ], + [ + "installation_scope", + null, + "asc" + ], + [ + "locator_hash", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_memory_entry_heads": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "memory_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "head_revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "entry_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "entry_version_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "entry_content_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "searchable_text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "memory_artifact_id", + null, + "asc" + ], + [ + "entry_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "memory_artifact_id", + "entry_id", + "entry_version_id" + ], + "referred_schema": null, + "referred_table": "pc_memory_entry_versions", + "referred_columns": [ + "scope_id", + "memory_artifact_id", + "entry_id", + "entry_version_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_memory_entry_heads_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "head_revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_memory_entry_versions": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "memory_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "entry_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "entry_version_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "previous_version_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "kind": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "source_refs": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "artifact_refs": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "entry_content_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "created_in_revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "memory_artifact_id", + null, + "asc" + ], + [ + "entry_version_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_memory_entry_versions_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_memory_entry_versions_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "created_in_revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_memory_entry_versions_logical_version": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "memory_artifact_id", + null, + "asc" + ], + [ + "entry_id", + null, + "asc" + ], + [ + "version", + null, + "asc" + ] + ], + "unique": true + }, + "uq_pc_memory_entry_versions_identity": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "memory_artifact_id", + null, + "asc" + ], + [ + "entry_id", + null, + "asc" + ], + [ + "entry_version_id", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_memory_source_windows": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_through": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "window_limit": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_memory_source_window_positive": { + "expression": [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "source_through" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + ">", + [ + "column", + "window_limit" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_model_usage_daily": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "usage_date": { + "type": [ + "date", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "purpose": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "operation": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "requests": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "input_tokens": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "output_tokens": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "input_complete": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "output_complete": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "usage_date", + null, + "asc" + ], + [ + "purpose", + null, + "asc" + ], + [ + "operation", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_model_usage_daily_operation": { + "expression": [ + "in", + [ + "column", + "operation" + ], + [ + [ + "literal", + "generation" + ], + [ + "literal", + "embedding" + ] + ] + ], + "enforced": true + }, + "ck_pc_model_usage_daily_requests_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "requests" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_model_usage_daily_input_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "input_tokens" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_model_usage_daily_output_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "output_tokens" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_profile_policies": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "generation_enabled": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "activation_mode": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "pending_candidate_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "version": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "updated_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + }, + { + "columns": [ + "scope_id", + "pending_candidate_id" + ], + "referred_schema": null, + "referred_table": "pc_artifact_candidate_heads", + "referred_columns": [ + "scope_id", + "candidate_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_profile_policies_activation_mode": { + "expression": [ + "in", + [ + "column", + "activation_mode" + ], + [ + [ + "literal", + "automatic" + ], + [ + "literal", + "review_required" + ] + ] + ], + "enforced": true + }, + "ck_pc_profile_policies_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_recall_effort_daily": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "usage_date": { + "type": [ + "date", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "policy_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "assessment": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "preparations": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "rounds": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "expanded_preparations": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "admission_expansions": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "policy_floor_expansions": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "candidate_round_samples": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "candidates_assessed": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "final_candidate_pool": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "added_embeddings": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "added_generation_calls": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "truncated_items": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "dropped_items": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "dropped_below_min_bytes": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "dropped_no_fitting_truncation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "usage_date", + null, + "asc" + ], + [ + "policy_id", + null, + "asc" + ], + [ + "assessment", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_recall_effort_preparations_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "preparations" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_rounds_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "rounds" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_expanded_preparations_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "expanded_preparations" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_admission_expansions_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "admission_expansions" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_policy_floor_expansions_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "policy_floor_expansions" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_candidate_round_samples_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "candidate_round_samples" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_candidates_assessed_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "candidates_assessed" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_final_candidate_pool_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "final_candidate_pool" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_added_embeddings_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "added_embeddings" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_added_generation_calls_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "added_generation_calls" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_truncated_items_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "truncated_items" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_dropped_items_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "dropped_items" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_dropped_below_min_bytes_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "dropped_below_min_bytes" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_dropped_no_fitting_truncation_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "dropped_no_fitting_truncation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recall_effort_daily_drop_total": { + "expression": [ + "binary", + "=", + [ + "column", + "dropped_items" + ], + [ + "binary", + "+", + [ + "column", + "dropped_below_min_bytes" + ], + [ + "column", + "dropped_no_fitting_truncation" + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_recall_token_daily": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "usage_date": { + "type": [ + "date", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "estimator_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "estimator_version": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "preparations": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "ready_preparations": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "comparable_preparations": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "baseline_tokens": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "recalled_tokens": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "usage_date", + null, + "asc" + ], + [ + "estimator_id", + null, + "asc" + ], + [ + "estimator_version", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_recall_token_daily_preparations_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "preparations" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recall_token_daily_ready_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "ready_preparations" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recall_token_daily_comparable_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "comparable_preparations" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recall_token_daily_comparable_ready": { + "expression": [ + "binary", + "<=", + [ + "column", + "comparable_preparations" + ], + [ + "column", + "ready_preparations" + ] + ], + "enforced": true + }, + "ck_pc_recall_token_daily_ready_total": { + "expression": [ + "binary", + "<=", + [ + "column", + "ready_preparations" + ], + [ + "column", + "preparations" + ] + ], + "enforced": true + }, + "ck_pc_recall_token_daily_baseline_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "baseline_tokens" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recall_token_daily_recalled_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "recalled_tokens" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_receipt_migration_review": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "reason": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "source_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": {} + }, + "pc_recurrence_match": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "match_key": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "task_outcome_source_type": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "task_outcome_source_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "task_outcome_position": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "failure_item_kind": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "failure_item_index": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "failure_item_digest": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "candidate_set_mode": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "candidate_set_digest": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "target_family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_revision": { + "type": [ + "int", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "signature_key": { + "type": [ + "mediumtext", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "match_key", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_recurrence_match_result": { + "expression": [ + "in", + [ + "column", + "result" + ], + [ + [ + "literal", + "matched" + ], + [ + "literal", + "unmatched" + ], + [ + "literal", + "ambiguous" + ] + ] + ], + "enforced": true + }, + "ck_pc_recurrence_match_candidate_set_mode": { + "expression": [ + "in", + [ + "column", + "candidate_set_mode" + ], + [ + [ + "literal", + "handoff_citations" + ], + [ + "literal", + "scope_heads" + ] + ] + ], + "enforced": true + }, + "ck_pc_recurrence_match_failure_item_kind": { + "expression": [ + "in", + [ + "column", + "failure_item_kind" + ], + [ + [ + "literal", + "observation" + ], + [ + "literal", + "check" + ] + ] + ], + "enforced": true + }, + "ck_pc_recurrence_match_position_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "task_outcome_position" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recurrence_match_item_index_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "failure_item_index" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_recurrence_match_identity": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "task_outcome_source_type", + null, + "asc" + ], + [ + "task_outcome_source_id", + null, + "asc" + ], + [ + "failure_item_kind", + null, + "asc" + ], + [ + "failure_item_index", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_recurrence_observation": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "observation_id": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selection_key": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "verdict_key": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "event": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "match_basis": { + "type": [ + "varchar", + [ + "8" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "signature_key": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "signature_key_hash": { + "type": [ + "binary", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "task_outcome_source_type": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "task_outcome_source_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "task_outcome_position": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "observation_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_recurrence_observation_event": { + "expression": [ + "in", + [ + "column", + "event" + ], + [ + [ + "literal", + "selected" + ], + [ + "literal", + "recurred" + ], + [ + "literal", + "avoided" + ] + ] + ], + "enforced": true + }, + "ck_pc_recurrence_observation_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recurrence_observation_position_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "task_outcome_position" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_recurrence_observation_selection": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "selection_key", + null, + "asc" + ] + ], + "unique": true + }, + "uq_pc_recurrence_observation_verdict": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "verdict_key", + null, + "asc" + ] + ], + "unique": true + }, + "ix_pc_recurrence_observation_revision": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_recurrence_observation_key": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "signature_key_hash", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_recurrence_observation_outcome": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "task_outcome_source_type", + null, + "asc" + ], + [ + "task_outcome_source_id", + null, + "asc" + ], + [ + "task_outcome_position", + null, + "asc" + ] + ], + "unique": false + } + } + }, + "pc_scope_bindings": { + "columns": { + "integration": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "kind": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "external_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "external_id_search": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "integration", + null, + "asc" + ], + [ + "kind", + null, + "asc" + ], + [ + "external_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": {} + }, + "pc_scope_context_references": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "referenced_scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "referenced_scope_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + }, + { + "columns": [ + "referenced_scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_scope_context_references_not_self": { + "expression": [ + "binary", + "<>", + [ + "column", + "scope_id" + ], + [ + "column", + "referenced_scope_id" + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_scope_creation_requests": { + "columns": { + "idempotency_key": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "request_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "idempotency_key", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": {} + }, + "pc_scope_external_references": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "ordinal": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "kind": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "value": { + "type": [ + "varchar", + [ + "2000" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "value_search": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "value_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "ordinal", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_scope_external_references_ordinal_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "ordinal" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_scope_external_references_value": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "kind", + null, + "asc" + ], + [ + "value_digest", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_scope_settings": { + "columns": { + "name": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "name", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": {} + }, + "pc_scopes": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "title": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "summary": { + "type": [ + "varchar", + [ + "2000" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "scope_id_search": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "title_search": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "summary_search": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "parent_scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "parent_scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_scopes_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_server_identity": { + "columns": { + "singleton_key": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "server_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "singleton_key", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_server_identity_singleton": { + "expression": [ + "binary", + "=", + [ + "column", + "singleton_key" + ], + [ + "number", + 1 + ] + ], + "enforced": true + } + }, + "indexes": { + "server_id": { + "columns": [ + [ + "server_id", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_skill_packages": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "tree_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "archive_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "archive_bytes": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "manifest": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "file_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "uncompressed_size": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "archive_size": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "created_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "tree_digest", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_skill_packages_file_count": { + "expression": [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "file_count" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + "<=", + [ + "column", + "file_count" + ], + [ + "number", + 256 + ] + ] + ], + "enforced": true + }, + "ck_pc_skill_packages_uncompressed_size": { + "expression": [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "uncompressed_size" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + "<=", + [ + "column", + "uncompressed_size" + ], + [ + "number", + 4194304 + ] + ] + ], + "enforced": true + }, + "ck_pc_skill_packages_archive_size": { + "expression": [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "archive_size" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + "<=", + [ + "column", + "archive_size" + ], + [ + "number", + 5242880 + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_skill_publications": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "desired_state": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": [ + "literal", + "published" + ], + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "desired_revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "desired_tree_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "observed_revision": { + "type": [ + "int", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "observed_tree_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "observed_generation": { + "type": [ + "bigint", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "destination": { + "type": [ + "mediumtext", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "state": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selected_runtime_variant": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "environment_fingerprint": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "last_error_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "observed_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "updated_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "target_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_skill_publications_desired_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "desired_revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_skill_publications_observed_revision_positive": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "observed_revision" + ] + ], + [ + "binary", + ">", + [ + "column", + "observed_revision" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + }, + "ck_pc_skill_publications_desired_state": { + "expression": [ + "in", + [ + "column", + "desired_state" + ], + [ + [ + "literal", + "published" + ], + [ + "literal", + "unpublished" + ] + ] + ], + "enforced": true + }, + "ck_pc_skill_publications_state": { + "expression": [ + "in", + [ + "column", + "state" + ], + [ + [ + "literal", + "unpublished" + ], + [ + "literal", + "pending" + ], + [ + "literal", + "current" + ], + [ + "literal", + "update_available" + ], + [ + "literal", + "delivery_failed" + ], + [ + "literal", + "conflict" + ], + [ + "literal", + "drifted" + ], + [ + "literal", + "incompatible" + ] + ] + ], + "enforced": true + }, + "ck_pc_skill_publications_observed_generation_nonnegative": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "observed_generation" + ] + ], + [ + "binary", + ">=", + [ + "column", + "observed_generation" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + }, + "ck_pc_skill_publications_generation_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_source_cursors": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "cursor": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "binding_name", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_source_cursors_generation_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_source_definition_manifests": { + "columns": { + "definition_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "definition_version": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fingerprint": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "manifest": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "definition_name", + null, + "asc" + ], + [ + "definition_version", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": { + "uq_pc_source_definition_manifest_fingerprint": { + "columns": [ + [ + "definition_name", + null, + "asc" + ], + [ + "fingerprint", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_source_journal_heads": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "position": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_source_journal_heads_position_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "position" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_sources": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_type": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "journal_position": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "source_type", + null, + "asc" + ], + [ + "source_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": { + "uq_pc_sources_scope_journal_position": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "journal_position", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_topic_memory_active_chunks": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "chunk_ordinal": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "start_offset": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "end_offset": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "chunk_text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "searchable_text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "policy_version": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "chunk_ordinal", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_topic_memory_revision_publications", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_topic_memory_active_chunks_family": { + "expression": [ + "binary", + "=", + [ + "column", + "family" + ], + [ + "literal", + "topic-memory" + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_active_chunks_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_active_chunks_ordinal_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "chunk_ordinal" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_active_chunks_start_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "start_offset" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_active_chunks_offsets": { + "expression": [ + "binary", + ">", + [ + "column", + "end_offset" + ], + [ + "column", + "start_offset" + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_topic_memory_active_chunks_fts": { + "columns": [ + [ + "searchable_text", + null, + "asc" + ] + ], + "unique": false, + "kind": "fulltext", + "parser": "space", + "parser_properties": { + "min_token_size": 3, + "max_token_size": 84 + } + } + } + }, + "pc_topic_memory_active_topics": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "title": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "summary": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "searchable_text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "source_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_topic_memory_revision_publications", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_topic_memory_active_topics_family": { + "expression": [ + "binary", + "=", + [ + "column", + "family" + ], + [ + "literal", + "topic-memory" + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_active_topics_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_active_topics_sources_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "source_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_topic_memory_active_topics_fts": { + "columns": [ + [ + "searchable_text", + null, + "asc" + ] + ], + "unique": false, + "kind": "fulltext", + "parser": "space", + "parser_properties": { + "min_token_size": 3, + "max_token_size": 84 + } + } + } + }, + "pc_topic_memory_processing_targets": { + "columns": { + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_request_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_through": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "captured_flush_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "observed_dirty_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "binding_name", + null, + "asc" + ], + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_topic_processing_target": { + "expression": [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "target_request_generation" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + ">=", + [ + "column", + "source_through" + ], + [ + "number", + 0 + ] + ] + ], + [ + "binary", + ">=", + [ + "column", + "captured_flush_generation" + ], + [ + "number", + 0 + ] + ] + ], + [ + "binary", + ">=", + [ + "column", + "observed_dirty_generation" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_topic_memory_retrieval_shape": { + "columns": { + "singleton": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "shape": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "profile_fingerprint": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "singleton", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_topic_memory_retrieval_shape_singleton": { + "expression": [ + "binary", + "=", + [ + "column", + "singleton" + ], + [ + "number", + 1 + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_retrieval_shape_value": { + "expression": [ + "in", + [ + "column", + "shape" + ], + [ + [ + "literal", + "fts" + ], + [ + "literal", + "hybrid" + ] + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_retrieval_shape_profile": { + "expression": [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "shape" + ], + [ + "literal", + "fts" + ] + ], + [ + "is_null", + [ + "column", + "profile_fingerprint" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "shape" + ], + [ + "literal", + "hybrid" + ] + ], + [ + "is_not_null", + [ + "column", + "profile_fingerprint" + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_topic_memory_revision_publications": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "published_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_topic_memory_publications_family": { + "expression": [ + "binary", + "=", + [ + "column", + "family" + ], + [ + "literal", + "topic-memory" + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_publications_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_topic_memory_work_budgets": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_after": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_through": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "attempt_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "attempts": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "requests": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "tokens": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "failure_code": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "binding_name", + null, + "asc" + ], + [ + "source_after", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_topic_budget_window": { + "expression": [ + "binary", + "and", + [ + "binary", + ">=", + [ + "column", + "source_after" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + ">", + [ + "column", + "source_through" + ], + [ + "column", + "source_after" + ] + ] + ], + "enforced": true + }, + "ck_pc_topic_budget_usage": { + "expression": [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "attempts" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + ">=", + [ + "column", + "requests" + ], + [ + "number", + 0 + ] + ] + ], + [ + "binary", + ">=", + [ + "column", + "tokens" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + } + }, + "views": [], + "triggers": [] + } +} diff --git a/src/powercontext/builtin/persistence/migration_resources/server_baselines/README.md b/src/powercontext/builtin/persistence/migration_resources/server_baselines/README.md index 992147ad10..0f8c0d77b4 100644 --- a/src/powercontext/builtin/persistence/migration_resources/server_baselines/README.md +++ b/src/powercontext/builtin/persistence/migration_resources/server_baselines/README.md @@ -1,8 +1,8 @@ # Complete Server source baselines -Each resource records the complete `sqlite_schema` inventory produced by a +Each resource records the complete backend schema inventory produced by a specific installed public wheel. Inventories include indexes, constraints, -triggers, virtual tables, and their SQLite shadow tables. They contain schema +triggers, virtual tables, and SQLite shadow tables where applicable. They contain schema definitions and artifact provenance, with no business rows or deployment paths. Recognition identifies a source schema. It does not validate its business data, @@ -34,3 +34,22 @@ feature, missing FTS helper, manually altered constraint, or extra persistent object may produce an unregistered shape. Such a shape remains unknown; a semantically similar subset is never sufficient for adoption. Additional source shapes require their own installed-release capture and review. + +## seekDB 1.3.0 + +`1.3.0/seekdb.json` comes from the same public wheel with its `cli,server,seekdb` +extras, `pylibseekdb==1.4.0.post1`, and engine +`5.7.25-OceanBase seekdb-v1.4.0.0`. A real installed Server initialized the +database with enforced Access, all Processing roles, and no model or embedding +provider. The same public HTTP fixture populated historical records. After the +Server exited, a maintenance connection captured all 56 tables. Their raw +`SHOW CREATE TABLE` definitions and row counts remained unchanged during the +corrected reflection. + +The normalized inventory includes all four FULLTEXT indexes, their `space` +parser, and `min_token_size=3,max_token_size=84` properties. Its fingerprint is +`041ee0d1cbf114ddafab1df096f9e7debbff1edf5d3a459cf7db847b640ff76c`. +Generated foreign-key names and physical storage options are normalized by the +MySQL reflector; index identity and search semantics are retained. Recognition +requires the complete matching inventory. It does not cover optional embedding +profiles or establish an OceanBase baseline. diff --git a/src/powercontext/builtin/persistence/migrations/mysql.py b/src/powercontext/builtin/persistence/migrations/mysql.py index 0006db5711..ccdbe14972 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql.py +++ b/src/powercontext/builtin/persistence/migrations/mysql.py @@ -35,6 +35,7 @@ from .connections import BackendIdentity, MaintenanceConnections from .models import MigrationError, MigrationPlan, MigrationResult, digest from .mysql_schema import MySQLSchema, mysql_inventory, read_revision +from .server_baselines import recognize_mysql_server_baseline BackupPolicy = Literal["auto", "manual", "skip"] BackupFactory = Callable[[Connection, BackendIdentity, Path], BackupProvider] @@ -462,6 +463,14 @@ def _inspect( else (False, None) ) self._validate_evidence(evidence) + backend = identity.product + if not initialized and (backend == "seekdb" or backend == "oceanbase"): + source = recognize_mysql_server_baseline(inventory, backend=backend) + if source is not None: + raise MigrationError( + "server_migration_unavailable", + f"The complete {source.release} Server schema is recognized, but its full upgrade is unavailable.", + ) current, adopt = self._recognize_source(inventory, initialized, current, evidence) if initialized and self.bundle.pending(current) and evidence is None: raise MigrationError("recovery_required", "The original external maintenance evidence is missing.") diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py index f5118a5567..ba7a507b75 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql_schema.py +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -567,6 +567,42 @@ def _index_definition(part: list[str], constraint: str | None = None) -> tuple[s return index_name, {"columns": _key_columns(columns), "unique": unique} +def _fulltext_definition(part: list[str]) -> tuple[str, dict[str, Any]]: + if part[1:2] not in (["key"], ["index"]): + raise MigrationError("incompatible_schema", "An unsupported FULLTEXT index definition was discovered.") + columns, end = _group(part, 3) + parser = None + properties: dict[str, int] = {} + if part[end : end + 2] == ["with", "parser"]: + end += 2 + if end >= len(part) or not re.fullmatch(r"[a-z_][a-z_0-9$]*", part[end]): + raise MigrationError("incompatible_schema", "A FULLTEXT parser has no valid identity.") + parser = part[end] + end += 1 + if part[end : end + 2] == ["parser_properties", "="]: + arguments, end = _group(part, end + 2) + for option in _parts(arguments): + if ( + len(option) != 3 + or option[0] not in {"min_token_size", "max_token_size"} + or option[0] in properties + or option[1] != "=" + or not option[2].isdigit() + ): + raise MigrationError( + "incompatible_schema", "An unsupported FULLTEXT parser property was discovered." + ) + properties[option[0]] = int(option[2]) + _physical(part[end:]) + return part[2], { + "columns": _key_columns(columns), + "unique": False, + "kind": "fulltext", + "parser": parser, + "parser_properties": properties, + } + + def _table_clause( table: dict[str, Any], part: list[str], defaults: dict[str, str | None], database: str | None ) -> None: @@ -588,6 +624,9 @@ def _table_clause( elif part[0] in {"key", "index", "unique"}: index_name, definition = _index_definition(part, constraint) _put_named(table["indexes"], index_name, definition) + elif part[0] == "fulltext": + index_name, definition = _fulltext_definition(part) + _put_named(table["indexes"], index_name, definition) elif constraint is None: _put_named(table["columns"], part[0], _column(part, defaults)) else: @@ -634,7 +673,7 @@ def _matches(actual: dict[str, Any], expected: dict[str, Any]) -> bool: specification[option] = None automatic_keys: set[tuple[str, ...]] = set() for index_name, index in tuple(table["indexes"].items()): - if index_name in expected_table["indexes"] or index["unique"]: + if index_name in expected_table["indexes"] or index["unique"] or index.get("kind") is not None: continue columns = tuple(key[0] for key in index["columns"]) foreign_columns = {tuple(key["columns"]) for key in table["foreign_keys"]} diff --git a/src/powercontext/builtin/persistence/migrations/server_baselines.py b/src/powercontext/builtin/persistence/migrations/server_baselines.py index e5f70ebe33..e7736c34f2 100644 --- a/src/powercontext/builtin/persistence/migrations/server_baselines.py +++ b/src/powercontext/builtin/persistence/migrations/server_baselines.py @@ -19,6 +19,7 @@ import json from dataclasses import dataclass from pathlib import Path +from typing import Any, Literal from sqlalchemy import Connection @@ -42,13 +43,27 @@ def recognize_sqlite_server_baseline(connection: Connection) -> ServerBaseline | No objects are excluded: in particular, an existing control table cannot be ignored to stamp a partially migrated database as a released source. """ - fingerprint = digest(sqlite_inventory(connection)) + return _recognize(digest(sqlite_inventory(connection)), backend="sqlite") + + +def recognize_mysql_server_baseline( + inventory: dict[str, Any], *, backend: Literal["seekdb", "oceanbase"] +) -> ServerBaseline | None: + """Match a complete native inventory against captures of the same backend. + + FULLTEXT parser configuration participates in the fingerprint. A capture + from embedded seekDB does not establish an OceanBase source baseline. + """ + return _recognize(digest(inventory), backend=backend) + + +def _recognize(fingerprint: str, *, backend: Literal["sqlite", "seekdb", "oceanbase"]) -> ServerBaseline | None: matches: list[ServerBaseline] = [] - for path in sorted(_BASELINES.glob("*/sqlite.json")): + for path in sorted(_BASELINES.glob(f"*/{backend}.json")): source = json.loads(path.read_text(encoding="utf-8")) if ( source["format"] != "powercontext.server-baseline.v1" - or source["backend"] != "sqlite" + or source["backend"] != backend or source["release"] != path.parent.name or digest(source["inventory"]) != source["schema_fingerprint"] ): diff --git a/tests/builtin/persistence/test_mysql_migration_schema.py b/tests/builtin/persistence/test_mysql_migration_schema.py index 9cf278d367..926862e918 100644 --- a/tests/builtin/persistence/test_mysql_migration_schema.py +++ b/tests/builtin/persistence/test_mysql_migration_schema.py @@ -226,6 +226,69 @@ def _inventory(database: _ReflectedDatabase) -> dict[str, Any]: return mysql_inventory(cast(Connection, database)) +def _released_fulltext_tables() -> dict[str, str]: + path = Path(__file__).parents[2] / "fixtures/database_migrations/schemas/seekdb_v1_3_0_fulltext.json" + tables = json.loads(path.read_text(encoding="utf-8"))["tables"] + return {name: ddl.replace("`test`.", "`TestDB`.") for name, ddl in tables.items()} + + +def test_released_fulltext_keys_preserve_search_semantics_in_the_inventory() -> None: + inventory = _inventory(_ReflectedDatabase(_released_fulltext_tables())) + for name, table in inventory["tables"].items(): + assert "fulltext" not in table["columns"] + assert table["indexes"][f"ix_{name}_fts"] == { + "columns": [["searchable_text", None, "asc"]], + "unique": False, + "kind": "fulltext", + "parser": "space", + "parser_properties": {"min_token_size": 3, "max_token_size": 84}, + } + + +@pytest.mark.parametrize( + ("old", "new"), + [ + ("WITH PARSER space", "WITH PARSER ngram"), + ("min_token_size=3", "min_token_size=4"), + ("max_token_size=84", "max_token_size=80"), + ("ix_pc_artifact_heads_fts", "ix_other_fts"), + ], +) +def test_fulltext_semantic_changes_have_distinct_fingerprints(old: str, new: str) -> None: + tables = _released_fulltext_tables() + original = digest(_inventory(_ReflectedDatabase(tables))) + tables["pc_artifact_heads"] = tables["pc_artifact_heads"].replace(old, new) + assert digest(_inventory(_ReflectedDatabase(tables))) != original + + +@pytest.mark.parametrize( + "properties", + [ + "unknown_option=3,max_token_size=84", + "min_token_size='3',max_token_size=84", + "min_token_size=3,min_token_size=4", + "min_token_size=3+1,max_token_size=84", + ], +) +def test_fulltext_unknown_parser_properties_are_rejected(properties: str) -> None: + tables = _released_fulltext_tables() + tables["pc_artifact_heads"] = tables["pc_artifact_heads"].replace("min_token_size=3,max_token_size=84", properties) + with pytest.raises(MigrationError, match="incompatible_schema"): + _inventory(_ReflectedDatabase(tables)) + + +def test_fulltext_key_cannot_be_hidden_as_an_automatic_foreign_key_index(schema: MySQLSchema) -> None: + tables = _show_tables() + tables["pc_artifact_heads"] = _append_clause( + tables["pc_artifact_heads"], + "FULLTEXT KEY extra_search (scope_id, family, artifact_id, revision)", + ) + inventory = _inventory(_ReflectedDatabase(tables)) + assert inventory["tables"]["pc_artifact_heads"]["indexes"]["extra_search"]["kind"] == "fulltext" + with pytest.raises(MigrationError, match="complete layout"): + schema.identify(inventory) + + @pytest.mark.parametrize( ("citations", "drop_family", "revision"), [(0, False, "p0001"), (2, False, "p0002"), (2, True, "p0003")] ) diff --git a/tests/builtin/persistence/test_server_migration_baselines.py b/tests/builtin/persistence/test_server_migration_baselines.py index 44b2d86fc5..15dc4d6ece 100644 --- a/tests/builtin/persistence/test_server_migration_baselines.py +++ b/tests/builtin/persistence/test_server_migration_baselines.py @@ -16,6 +16,7 @@ from __future__ import annotations +import copy import json import sqlite3 from pathlib import Path @@ -25,7 +26,10 @@ from powercontext.builtin.persistence.migrations import MigrationError, SQLiteMigrationRunner from powercontext.builtin.persistence.migrations.deployment import production_bundle -from powercontext.builtin.persistence.migrations.server_baselines import recognize_sqlite_server_baseline +from powercontext.builtin.persistence.migrations.server_baselines import ( + recognize_mysql_server_baseline, + recognize_sqlite_server_baseline, +) RESOURCE = ( Path(__file__).resolve().parents[3] @@ -124,3 +128,48 @@ def test_weakened_constraint_with_identical_columns_is_not_recognized(released_d before = released_database.read_bytes() assert recognize(released_database) is None assert released_database.read_bytes() == before + + +def test_complete_seekdb_release_inventory_is_recognized_only_for_its_captured_backend() -> None: + resource = json.loads(RESOURCE.with_name("seekdb.json").read_text(encoding="utf-8")) + inventory = resource["inventory"] + before = copy.deepcopy(inventory) + baseline = recognize_mysql_server_baseline(inventory, backend="seekdb") + assert baseline is not None + assert baseline.release == "1.3.0" + assert baseline.schema_fingerprint == resource["schema_fingerprint"] + assert recognize_mysql_server_baseline(inventory, backend="oceanbase") is None + assert inventory == before + + +@pytest.mark.parametrize( + "change", ["table", "control", "column", "constraint", "index", "parser", "properties", "view", "trigger", "subset"] +) +def test_incomplete_or_altered_seekdb_inventories_are_not_released_baselines(change: str) -> None: + inventory = json.loads(RESOURCE.with_name("seekdb.json").read_text(encoding="utf-8"))["inventory"] + table = inventory["tables"]["pc_artifact_heads"] + if change == "table": + del inventory["tables"]["pc_sources"] + elif change == "control": + inventory["tables"]["pc_schema_revision"] = {} + elif change == "column": + table["columns"]["revision"]["nullable"] = True + elif change == "constraint": + table["checks"]["ck_pc_artifact_heads_revision_positive"]["enforced"] = False + elif change == "index": + del table["indexes"]["ix_pc_artifact_heads_fts"] + elif change == "parser": + table["indexes"]["ix_pc_artifact_heads_fts"]["parser"] = "ngram" + elif change == "properties": + table["indexes"]["ix_pc_artifact_heads_fts"]["parser_properties"]["min_token_size"] = 4 + elif change == "view": + inventory["views"] = ["unknown_view"] + elif change == "trigger": + inventory["triggers"] = [{"Trigger": "unknown_trigger", "Table": "pc_artifacts"}] + else: + inventory["tables"] = { + name: shape for name, shape in inventory["tables"].items() if name.startswith("pc_artifact") + } + before = copy.deepcopy(inventory) + assert recognize_mysql_server_baseline(inventory, backend="seekdb") is None + assert inventory == before diff --git a/tests/fixtures/database_migrations/schemas/seekdb_v1_3_0_fulltext.json b/tests/fixtures/database_migrations/schemas/seekdb_v1_3_0_fulltext.json new file mode 100644 index 0000000000..a18f9be511 --- /dev/null +++ b/tests/fixtures/database_migrations/schemas/seekdb_v1_3_0_fulltext.json @@ -0,0 +1,13 @@ +{ + "source_release": "1.3.0", + "source_wheel_sha256": "8da7983763f3a6ab80b413c770a4973a46c557a54e1a1fa675099e58d47305e5", + "capture": "Installed public release Server with enforced Access, all Processing roles, and no model or embedding provider; public HTTP fixture; maintenance reflection after process exit.", + "backend": "5.7.25-OceanBase seekdb-v1.4.0.0", + "pylibseekdb": "1.4.0.post1", + "tables": { + "pc_artifact_heads": "CREATE TABLE `pc_artifact_heads` (\n `scope_id` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `family` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `artifact_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `revision` int(11) NOT NULL,\n `searchable_text` mediumtext DEFAULT NULL,\n `lifecycle_state` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'active',\n `replacement_artifact_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `merged_into_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `governance_generation` bigint(20) NOT NULL DEFAULT '0',\n PRIMARY KEY (`scope_id`, `family`, `artifact_id`),\n CONSTRAINT `pc_artifact_heads_OBFK_1792659503573247` FOREIGN KEY (`scope_id`, `family`, `artifact_id`, `revision`) REFERENCES `test`.`pc_artifacts`(`scope_id`, `family`, `artifact_id`, `revision`) ON UPDATE RESTRICT ON DELETE RESTRICT ,\n FULLTEXT KEY `ix_pc_artifact_heads_fts` (`searchable_text`) WITH PARSER space PARSER_PROPERTIES=(min_token_size=3,max_token_size=84) BLOCK_SIZE 16384,\n CONSTRAINT `ck_pc_artifact_heads_revision_positive` CHECK ((`revision` > 0)),\n CONSTRAINT `ck_pc_artifact_heads_lifecycle_state` CHECK ((`lifecycle_state` in ('active','deprecated','retired'))),\n CONSTRAINT `ck_pc_artifact_heads_governance_generation_nonnegative` CHECK ((`governance_generation` >= 0)),\n CONSTRAINT `ck_pc_artifact_heads_replacement_deprecated` CHECK (((`replacement_artifact_id` is null) or (`lifecycle_state` = 'deprecated'))),\n CONSTRAINT `ck_pc_artifact_heads_merge_target` CHECK (((`merged_into_id` is null) or ((`lifecycle_state` = 'deprecated') and (`merged_into_id` <> `artifact_id`) and (`replacement_artifact_id` is null))))\n) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0", + "pc_atomic_memory_current": "CREATE TABLE `pc_atomic_memory_current` (\n `scope_id` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `artifact_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `revision` bigint(20) NOT NULL,\n `state_version` bigint(20) NOT NULL,\n `content_hash` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `projection_format` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `kind` mediumtext NOT NULL,\n `text` mediumtext NOT NULL,\n `searchable_text` mediumtext NOT NULL,\n `tag_keys` mediumtext NOT NULL,\n `embedding` blob DEFAULT NULL,\n `profile_fingerprint` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `embedding_input_hash` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n PRIMARY KEY (`scope_id`, `artifact_id`),\n FULLTEXT KEY `ix_pc_atomic_memory_current_fts` (`searchable_text`) WITH PARSER space PARSER_PROPERTIES=(min_token_size=3,max_token_size=84) BLOCK_SIZE 16384,\n CONSTRAINT `ck_pc_atomic_memory_current_revision` CHECK ((`revision` > 0)),\n CONSTRAINT `ck_pc_atomic_memory_current_state_version` CHECK ((`state_version` >= 0)),\n CONSTRAINT `ck_pc_atomic_memory_current_embedding_metadata` CHECK ((((`embedding` is null) and (`profile_fingerprint` is null) and (`embedding_input_hash` is null)) or ((`embedding` is not null) and (`profile_fingerprint` is not null) and (`embedding_input_hash` is not null))))\n) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0", + "pc_topic_memory_active_chunks": "CREATE TABLE `pc_topic_memory_active_chunks` (\n `scope_id` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `family` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `artifact_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `revision` int(11) NOT NULL,\n `chunk_ordinal` int(11) NOT NULL,\n `start_offset` int(11) NOT NULL,\n `end_offset` int(11) NOT NULL,\n `chunk_text` mediumtext NOT NULL,\n `searchable_text` mediumtext NOT NULL,\n `policy_version` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n PRIMARY KEY (`scope_id`, `artifact_id`, `chunk_ordinal`),\n CONSTRAINT `pc_topic_memory_active_chunks_OBFK_1792659503857007` FOREIGN KEY (`scope_id`, `family`, `artifact_id`, `revision`) REFERENCES `test`.`pc_topic_memory_revision_publications`(`scope_id`, `family`, `artifact_id`, `revision`) ON UPDATE RESTRICT ON DELETE RESTRICT ,\n FULLTEXT KEY `ix_pc_topic_memory_active_chunks_fts` (`searchable_text`) WITH PARSER space PARSER_PROPERTIES=(min_token_size=3,max_token_size=84) BLOCK_SIZE 16384,\n CONSTRAINT `ck_pc_topic_memory_active_chunks_family` CHECK ((`family` = 'topic-memory')),\n CONSTRAINT `ck_pc_topic_memory_active_chunks_revision_positive` CHECK ((`revision` > 0)),\n CONSTRAINT `ck_pc_topic_memory_active_chunks_ordinal_nonnegative` CHECK ((`chunk_ordinal` >= 0)),\n CONSTRAINT `ck_pc_topic_memory_active_chunks_start_nonnegative` CHECK ((`start_offset` >= 0)),\n CONSTRAINT `ck_pc_topic_memory_active_chunks_offsets` CHECK ((`end_offset` > `start_offset`))\n) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0", + "pc_topic_memory_active_topics": "CREATE TABLE `pc_topic_memory_active_topics` (\n `scope_id` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `family` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `artifact_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `revision` int(11) NOT NULL,\n `title` mediumtext NOT NULL,\n `summary` mediumtext NOT NULL,\n `searchable_text` mediumtext NOT NULL,\n `source_count` int(11) NOT NULL,\n PRIMARY KEY (`scope_id`, `artifact_id`),\n CONSTRAINT `pc_topic_memory_active_topics_OBFK_1792659503833750` FOREIGN KEY (`scope_id`, `family`, `artifact_id`, `revision`) REFERENCES `test`.`pc_topic_memory_revision_publications`(`scope_id`, `family`, `artifact_id`, `revision`) ON UPDATE RESTRICT ON DELETE RESTRICT ,\n FULLTEXT KEY `ix_pc_topic_memory_active_topics_fts` (`searchable_text`) WITH PARSER space PARSER_PROPERTIES=(min_token_size=3,max_token_size=84) BLOCK_SIZE 16384,\n CONSTRAINT `ck_pc_topic_memory_active_topics_family` CHECK ((`family` = 'topic-memory')),\n CONSTRAINT `ck_pc_topic_memory_active_topics_revision_positive` CHECK ((`revision` > 0)),\n CONSTRAINT `ck_pc_topic_memory_active_topics_sources_nonnegative` CHECK ((`source_count` >= 0))\n) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0" + } +} From 54f88027e289ff791352fe4d23f33c6f0fb5d909 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 17:58:16 +0800 Subject: [PATCH 35/94] feat(migrations): execute registered MySQL revision wrappers Load frozen contributed wrappers through the existing Alembic executor and reject incomplete or divergent history before opening a target. Preserve historical execution inputs and stable per-revision checksums. Constraint: One linear revision chain; no changes to released p0001-p0005 resources. Scope-risk: Future MySQL revision execution only; full Server upgrades remain unavailable. Tested: Linux migration selection 154 passed, 13 skipped; final schema suite with real seekDB 48 passed; changed-file pre-commit including Ty/Python 3.11 passed. --- docs/en/docs/operate/database-migrations.md | 8 ++ docs/zh/docs/operate/database-migrations.md | 2 + .../builtin/persistence/migrations/bundle.py | 16 ++++ .../builtin/persistence/migrations/mysql.py | 25 +++++- .../test_mysql_migration_schema.py | 81 ++++++++++++++++++- 5 files changed, 130 insertions(+), 2 deletions(-) diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 1c5e670fd7..ed1a2d6b42 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -362,6 +362,14 @@ frozen complete layouts. Reflection, next-revision prefix recovery, and version- chain; adding a revision does not require changing a second version list in the recognizer. These schema registrations do not replace domain conversions, compatibility checks, or full Server readiness. +New MySQL execution wrappers are also registered in `manifest.json`, using a `mysql_wrappers` object that maps a +revision to its frozen Python resource. List that resource in the revision's `revision_resources` dependencies. +The wrapper declares the same `revision` and `down_revision` as the common chain and calls +`context.config.attributes["execute_revision"](revision)`. Historical execution wrappers cannot be overridden. +The executor validates complete, linear wrapper coverage before opening a maintenance target; missing or divergent +execution history is an invalid bundle rather than a failure discovered after earlier revisions have committed. +Wrapper bytes participate in both the reviewed bundle and the revision's execution checksum. + The reviewed plan binds the registration through the complete bundle checksum. Active maintenance requires the same bundle for retry. Released revision resources and their recorded checksums remain immutable. Parameter-free DDL uses statement-scoped driver execution options so CHECK remainder operators and percent-containing diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index d734f8f071..cb1a449e16 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -222,6 +222,8 @@ seekdb／OceanBase DDL 可能隐式提交，不能用 Python 事务承诺整个 MySQL 结构步骤通过冻结的布局片段注册，由 `manifest.json` 的 `mysql_layouts` 列表选择，合起来必须恰好覆盖 Alembic 版本链，且不能重复声明。新增 revision 追加不可改写的片段，并在 `revision_resources` 中声明其文件，不修改已发布片段。可选 checkpoint 将累积步骤与独立冻结的完整结构对照。反射、下一 revision 的已知前缀恢复和版本表校验使用同一注册链；新增 revision 无需修改识别器中的另一份版本列表。这些结构注册不能代替领域转换、兼容性检查或完整 Server readiness。 +新增 MySQL 执行入口也在 `manifest.json` 中登记：`mysql_wrappers` 对象将 revision 映射到冻结的 Python 资源，并将该文件列入对应 revision 的 `revision_resources` 依赖。入口声明与公共链一致的 `revision` 和 `down_revision`，调用 `context.config.attributes["execute_revision"](revision)`，不能覆盖历史执行入口。执行器在打开维护目标前验证完整、线性的入口覆盖；缺失或分叉的执行历史直接判为无效 bundle，不等到前序 revision 提交后才发现。入口字节同时纳入计划的 bundle 摘要和该 revision 的执行摘要。 + 计划通过完整 bundle 摘要绑定注册资源，未完成维护的重试要求使用同一 bundle。已发布 revision 资源及其记录摘要保持不可改写。 无参数 DDL 使用单语句驱动选项，确保 CHECK 的取余运算符和 SQL 字面量中的百分号原样到达后端。 diff --git a/src/powercontext/builtin/persistence/migrations/bundle.py b/src/powercontext/builtin/persistence/migrations/bundle.py index 1921362ab3..71a2e956f1 100644 --- a/src/powercontext/builtin/persistence/migrations/bundle.py +++ b/src/powercontext/builtin/persistence/migrations/bundle.py @@ -18,6 +18,7 @@ import hashlib import json +import re import threading from pathlib import Path from tempfile import TemporaryDirectory @@ -75,6 +76,7 @@ def __init__(self, directory: Path) -> None: ): raise MigrationError("invalid_bundle", "MySQL layout registration is outside the frozen bundle.") self.mysql_layouts: tuple[str, ...] = tuple(mysql_layouts) + self.mysql_wrappers = self._mysql_wrappers(manifest, resources) self.affected_objects = tuple(manifest.get("affected_objects", ())) self.retained_objects = tuple(manifest.get("retained_objects", ())) self.retained_fingerprints = manifest.get("retained_fingerprints", {}) @@ -100,6 +102,20 @@ def __init__(self, directory: Path) -> None: self.checksums[item.revision] = digest(inputs) self.checksum = digest(self.resources) + def _mysql_wrappers(self, manifest: dict[str, Any], resources: dict[str, bytes]) -> dict[str, str]: + wrappers = manifest.get("mysql_wrappers", {}) + if not isinstance(wrappers, dict) or any( + revision not in self.revisions + or re.fullmatch(r"[A-Za-z0-9_-]+", revision) is None + or not isinstance(name, str) + or name not in resources + or not name.endswith(".py") + or name not in manifest["revision_resources"][revision] + for revision, name in wrappers.items() + ): + raise MigrationError("invalid_bundle", "MySQL execution registration is outside its frozen revision.") + return dict(wrappers) + def assert_unchanged(self) -> None: resources = { path.relative_to(self.directory).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() diff --git a/src/powercontext/builtin/persistence/migrations/mysql.py b/src/powercontext/builtin/persistence/migrations/mysql.py index ccdbe14972..6e807238e7 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql.py +++ b/src/powercontext/builtin/persistence/migrations/mysql.py @@ -27,6 +27,9 @@ from alembic import command from alembic.config import Config +from alembic.script import ScriptDirectory +from alembic.script.revision import RevisionError +from alembic.util.exc import CommandError from pydantic import BaseModel, ConfigDict, model_validator from sqlalchemy import Connection @@ -175,12 +178,19 @@ def __init__( path.relative_to(self._history_directory).as_posix(): path.read_bytes() for path in sorted(self._history_directory.rglob("*.py")) } - self._history_bytes = resources + self._history_bytes = resources.copy() + for revision, resource in bundle.mysql_wrappers.items(): + if any(name.startswith(f"versions/{revision}_") for name in resources): + raise MigrationError("invalid_bundle", "A registered MySQL wrapper overrides historical execution.") + # Prefix the filename with its revision to keep the existing + # per-revision checksum contract, including contributed wrappers. + resources[f"versions/{revision}_registered.py"] = (bundle.snapshot_directory / resource).read_bytes() self._snapshot = TemporaryDirectory(prefix="pc-mysql-migrations-") for name, content in resources.items(): target = Path(self._snapshot.name) / name target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(content) + self._validate_history() hashes = {name: hashlib.sha256(content).hexdigest() for name, content in resources.items()} self.checksum = digest({"bundle": bundle.checksum, "mysql_history": hashes}) self.checksums = { @@ -192,6 +202,19 @@ def __init__( for revision, checksum in bundle.checksums.items() } + def _validate_history(self) -> None: + try: + history = list(reversed(list(ScriptDirectory(self._snapshot.name).walk_revisions()))) + except (CommandError, RevisionError, KeyError) as error: + raise MigrationError("invalid_bundle", "MySQL execution history is incomplete or ambiguous.") from error + previous = None + if tuple(item.revision for item in history) != self.bundle.revisions: + raise MigrationError("invalid_bundle", "MySQL execution history does not cover the registered chain.") + for item in history: + if item.down_revision != previous or item.dependencies: + raise MigrationError("invalid_bundle", "MySQL execution history differs from the registered chain.") + previous = item.revision + def _assert_unchanged(self) -> None: self.bundle.assert_unchanged() resources = { diff --git a/tests/builtin/persistence/test_mysql_migration_schema.py b/tests/builtin/persistence/test_mysql_migration_schema.py index 926862e918..c05c6f81dd 100644 --- a/tests/builtin/persistence/test_mysql_migration_schema.py +++ b/tests/builtin/persistence/test_mysql_migration_schema.py @@ -16,6 +16,7 @@ from __future__ import annotations +import asyncio import json import os import re @@ -31,6 +32,7 @@ from powercontext.builtin.persistence.migrations.bundle import MigrationBundle, sqlite_inventory from powercontext.builtin.persistence.migrations.connections import MaintenanceConnections from powercontext.builtin.persistence.migrations.models import MigrationError, digest +from powercontext.builtin.persistence.migrations.mysql import MySQLMigrationRunner from powercontext.builtin.persistence.migrations.mysql_schema import MySQLSchema, mysql_inventory, read_revision from powercontext.builtin.persistence.seekdb import SeekDBConfig @@ -110,9 +112,10 @@ def extended_bundle(tmp_path: Path) -> MigrationBundle: engine.dispose() manifest_path = directory / "manifest.json" manifest = json.loads(manifest_path.read_text()) - manifest["revision_resources"]["fixture_next"] = ["mysql_fixture_next.json"] + manifest["revision_resources"]["fixture_next"] = ["mysql_fixture_next.json", "mysql_wrappers/fixture_next.py"] manifest["sqlite_fingerprints"]["fixture_next"] = [fingerprint] manifest["mysql_layouts"].append("mysql_fixture_next.json") + manifest["mysql_wrappers"] = {"fixture_next": "mysql_wrappers/fixture_next.py"} manifest_path.write_text(json.dumps(manifest), encoding="utf-8") (directory / "mysql_fixture_next.json").write_text( json.dumps({ @@ -129,6 +132,14 @@ def extended_bundle(tmp_path: Path) -> MigrationBundle: f"def upgrade():\n op.execute({_NEXT_TABLE!r})\n", encoding="utf-8", ) + (directory / "mysql_wrappers").mkdir() + (directory / "mysql_wrappers/fixture_next.py").write_text( + "from alembic import context\n" + "revision = 'fixture_next'\n" + "down_revision = 'p0005'\n" + "def upgrade():\n context.config.attributes['execute_revision'](revision)\n", + encoding="utf-8", + ) return MigrationBundle(directory) @@ -191,6 +202,74 @@ def test_mysql_layout_registration_must_cover_exactly_the_revision_chain( MySQLSchema(MigrationBundle(extended_bundle.directory)) +@pytest.mark.parametrize("change", ["missing", "predecessor", "unbound", "override"]) +def test_mysql_execution_registration_rejects_incomplete_history_before_opening_a_target( + extended_bundle: MigrationBundle, tmp_path: Path, change: str +) -> None: + path = extended_bundle.directory / "manifest.json" + manifest = json.loads(path.read_text()) + if change == "missing": + del manifest["mysql_wrappers"] + elif change == "predecessor": + wrapper = extended_bundle.directory / "mysql_wrappers/fixture_next.py" + wrapper.write_text(wrapper.read_text().replace("'p0005'", "'p0004'"), encoding="utf-8") + elif change == "unbound": + manifest["revision_resources"]["fixture_next"].remove("mysql_wrappers/fixture_next.py") + else: + manifest["mysql_wrappers"]["p0005"] = "mysql_wrappers/fixture_next.py" + path.write_text(json.dumps(manifest), encoding="utf-8") + target = tmp_path / "absent-seekdb" + evidence = tmp_path / "absent-evidence" + with pytest.raises(MigrationError, match="invalid_bundle"): + MySQLMigrationRunner( + MaintenanceConnections(SeekDBConfig(path=target)), + MigrationBundle(extended_bundle.directory), + evidence_directory=evidence, + ) + assert not target.exists() + assert not evidence.exists() + + +@pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") +def test_registered_next_revision_executes_directly_and_sequentially_on_real_seekdb( + extended_bundle: MigrationBundle, short_tmp_path: Path +) -> None: + binding = pytest.importorskip("pylibseekdb") + leases = [] + try: + fingerprints = [] + for adjacent in (False, True): + target = short_tmp_path / ("adjacent" if adjacent else "direct") + # Retain a native engine lease while public maintenance calls open + # independent connections; restart behavior has separate acceptance. + leases.append(asyncio.run(binding.aopen(str(target)))) + connections = MaintenanceConnections(SeekDBConfig(path=target)) + evidence = short_tmp_path / (target.name + "-evidence") + previous = None + if adjacent: + previous = MySQLMigrationRunner(connections, MigrationBundle(_RESOURCES), evidence_directory=evidence) + previous.apply(plan_id=previous.plan().plan_id, accepted=True, maintenance_confirmed=True) + runner = MySQLMigrationRunner(connections, extended_bundle, evidence_directory=evidence) + if previous is not None: + assert all(runner.checksums[revision] == checksum for revision, checksum in previous.checksums.items()) + plan = runner.plan(backup_policy="skip") + assert plan.revisions == (("fixture_next",) if adjacent else extended_bundle.revisions) + result = runner.apply( + plan_id=plan.plan_id, + backup_policy="skip", + accepted=True, + maintenance_confirmed=True, + accept_no_backup=True, + ) + assert result.changed and runner.verify().revision == "fixture_next" + assert not runner.apply().changed + fingerprints.append(runner.plan().schema_fingerprint) + assert fingerprints[0] == fingerprints[1] + finally: + for lease in leases: + lease.close() + + def _append_clause(statement: str, clause: str) -> str: return statement.replace("\n)", ",\n\t" + clause + "\n)") From 9e0088b9712daa886e87973d1d11ddebdf65f5c5 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 18:19:08 +0800 Subject: [PATCH 36/94] fix(migrations): bind plans to target Processing declarations Include the runtime canonical Processing manifest in the existing deployment configuration digest so capability or supervisor ownership changes invalidate an accepted plan before database creation. Constraint: Reuse the startup manifest; do not print provider credentials or change domain state. Scope-risk: Existing operator plans must be regenerated because their configuration binding changes. Full historical Processing validation remains unavailable. Tested: Two public CLI regressions fail on the old adapter; Windows CLI 34 passed, 1 platform skip; Linux CLI 35 passed; changed-file hooks and full Ty/Python 3.11 passed. --- docs/en/docs/operate/database-migrations.md | 4 +++ docs/zh/docs/operate/database-migrations.md | 2 ++ .../persistence/migrations/deployment.py | 23 ++++++++++-- tests/test_database_migration_cli.py | 35 +++++++++++++++++++ 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index ed1a2d6b42..091969629f 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -107,6 +107,10 @@ Plans bind target identity, source/target revisions, actual schema, frozen resou service scope, and migration coordination. Diagnostics do not print configuration or credential values. Expected failures return stable JSON categories and a nonzero exit status. +Configuration binding includes the target's canonical Processing ownership mode, capabilities and declared automatic +bindings. Changing those declarations requires a new plan; an accepted plan cannot be applied under a different +Processing declaration. This binding does not validate or migrate the historical Processing state. + Within the supported acceptance scope, interactive users can invoke `apply` directly. PC-managed backup is the default. The command displays the plan, backup method, maintenance impact, and stopped-write responsibility, then asks for one confirmation before execution. It does not ask about each SQL statement: diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index cb1a449e16..97693891ea 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -72,6 +72,8 @@ powercontext server db-migrate verify --env-file deployment.env 计划绑定目标身份、源／目标 revision、实际结构摘要、冻结脚本摘要、配置摘要、备份策略、服务范围和迁移协调范围。配置与凭据值不输出到诊断中；预期错误返回稳定 JSON 类别和非零退出码。 +配置绑定包含目标部署的规范 Processing 归属模式、能力和自动绑定声明。改变这些声明后必须重新生成计划，不能使用旧计划执行。此绑定本身不验证或迁移历史 Processing 状态。 + 在支持范围内，终端用户可直接执行 `apply`，默认 PC 自动备份。命令展示计划、备份方法、维护影响及停写责任，确认一次后执行，不逐条 SQL 询问： ```bash diff --git a/src/powercontext/builtin/persistence/migrations/deployment.py b/src/powercontext/builtin/persistence/migrations/deployment.py index 86977dfaa4..d1313639b8 100644 --- a/src/powercontext/builtin/persistence/migrations/deployment.py +++ b/src/powercontext/builtin/persistence/migrations/deployment.py @@ -18,13 +18,15 @@ import os from pathlib import Path -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal from sqlalchemy.engine import make_url from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.seekdb import SeekDBConfig from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime.config import BuiltinConfig +from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest from .bundle import MigrationBundle from .connections import MaintenanceConnections @@ -45,6 +47,21 @@ def production_bundle() -> MigrationBundle: MigrationRunner = SQLiteMigrationRunner | MySQLMigrationRunner | TopicMemoryRecoveryMigrationRunner +def _migration_configuration(settings: ServerSettings) -> dict[str, Any]: + config = BuiltinConfig( + code=settings.code, + runtime=settings.runtime, + database=settings.database, + handoff_report=settings.handoff_report, + inference=settings.inference, + external_skills=settings.external_skills, + ) + return { + "database": settings.database.model_dump(mode="json", exclude={"url", "echo"}), + "processing_manifest": canonical_processing_manifest(config), + } + + def deployment_runner( settings: ServerSettings, *, @@ -89,7 +106,7 @@ def deployment_runner( ), production_bundle(), evidence_directory=evidence_dir, - configuration=database.model_dump(mode="json", exclude={"url", "echo"}), + configuration=_migration_configuration(settings), ) if not isinstance(database, SQLiteConfig): raise MigrationError("unsupported_backend", "The database has no registered migration adapter.") @@ -103,6 +120,6 @@ def deployment_runner( return SQLiteMigrationRunner( target, production_bundle(), - configuration=database.model_dump(exclude={"url", "echo"}), + configuration=_migration_configuration(settings), evidence_directory=evidence_dir, ) diff --git a/tests/test_database_migration_cli.py b/tests/test_database_migration_cli.py index 8a188d0f37..7bb443d633 100644 --- a/tests/test_database_migration_cli.py +++ b/tests/test_database_migration_cli.py @@ -161,6 +161,41 @@ def test_read_only_commands_never_create_missing_target(tmp_path, monkeypatch): assert not database.parent.exists() +@pytest.mark.parametrize( + "declaration", + [ + """POWERCONTEXT_SERVER_RUNTIME_ARTIFACT_PROCESSING_FAMILIES='["memory"]'""", + "POWERCONTEXT_SERVER_RUNTIME_ARTIFACT_PROCESSING_SUPERVISOR_MODE=dedicated", + ], +) +def test_processing_ownership_changes_invalidate_an_accepted_migration_plan(tmp_path, monkeypatch, declaration): + database, command = _configure(tmp_path, monkeypatch) + original = _invoke(command, "plan", tmp_path) + assert original.exit_code == 0, original.output + reviewed = json.loads(original.output) + environment = tmp_path / "deployment.env" + environment.write_text(environment.read_text(encoding="utf-8") + declaration + "\n", encoding="utf-8") + changed = _invoke(command, "plan", tmp_path) + assert changed.exit_code == 0, changed.output + current = json.loads(changed.output) + assert current["configuration_digest"] != reviewed["configuration_digest"] + assert current["plan_id"] != reviewed["plan_id"] + rejected = _invoke( + command, + "apply", + tmp_path, + "--backup", + "auto", + "--plan-id", + reviewed["plan_id"], + "--yes", + "--maintenance-confirmed", + ) + assert rejected.exit_code == 1, rejected.output + assert json.loads(rejected.output)["error"] == "plan_changed" + assert not database.parent.exists() + + def test_sqlite_url_authority_is_rejected_without_creating_a_local_target(tmp_path, monkeypatch): _database, command = _configure(tmp_path, monkeypatch) monkeypatch.chdir(tmp_path) From bcd365738f9d140cd01290598a59bb813250a9ac Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 18:44:30 +0800 Subject: [PATCH 37/94] feat(migrations): recognize complete OceanBase Server baseline Preserve OceanBase AUTO_INCREMENT_MODE in native schema inventory while excluding the changing counter. Freeze the complete installed 1.3.0 OceanBase schema with source wheel and driver provenance. Constraint: historical p0001-p0005 resources and execution checksums are unchanged. Scope-risk: recognition still blocks unavailable full Server upgrades before writes. Tested: Linux real seekDB/OceanBase migration selection 217 passed, 1 skipped; final baseline/schema selection 83 passed, 2 external skips; full Linux Ty/Python 3.11 and changed-file hooks pass; actual source schema/data hashes remain unchanged after plan/verify/apply rejection. --- docs/en/docs/operate/database-migrations.md | 7 +- docs/zh/docs/operate/database-migrations.md | 2 +- .../server_baselines/1.3.0/oceanbase.json | 11144 ++++++++++++++++ .../server_baselines/README.md | 21 + .../persistence/migrations/mysql_schema.py | 23 +- .../test_mysql_migration_schema.py | 22 + .../test_server_migration_baselines.py | 37 +- .../schemas/oceanbase_v1_3_0_audit.json | 7 + 8 files changed, 11251 insertions(+), 12 deletions(-) create mode 100644 src/powercontext/builtin/persistence/migration_resources/server_baselines/1.3.0/oceanbase.json create mode 100644 tests/fixtures/database_migrations/schemas/oceanbase_v1_3_0_audit.json diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 091969629f..1a82d4724e 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -19,8 +19,11 @@ The default `--bundle artifacts` ships frozen resources for Phase A acceptance o `pc_artifact_candidate_versions`, and `pc_artifact_tags`. Only empty targets and registered historical layouts are accepted. A complete Server database containing unmanaged objects is rejected. -The complete SQLite schema captured from the installed public 1.3.0 wheel is recognized separately, including its FTS -shadow tables, indexes, and triggers. Maintenance reports `server_migration_unavailable` for this source because a complete +Complete schemas captured from the installed public 1.3.0 wheel on SQLite, seekDB and OceanBase CE 4.3.5.6 are recognized +separately. SQLite recognition includes its FTS shadow tables, indexes and triggers; the two MySQL-compatible captures +each include 56 tables and four FULLTEXT indexes with parser settings. OceanBase also preserves its native +`AUTO_INCREMENT_MODE`, independently of the changing counter. Each inventory is registered only for its captured backend. +Maintenance reports `server_migration_unavailable` for these sources because a complete Server upgrade path is not registered. Recognition does not stamp a revision, validate domain data, or authorize writes. Other complete schemas remain unknown unless their exact shape is registered. diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 97693891ea..32c4881c61 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -12,7 +12,7 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 `powercontext server db-migrate status/plan/apply/verify` 使用部署配置中的持久 SQLite、嵌入式 seekdb 或 OceanBase MySQL 模式数据库，不接受任意测试 bundle 路径。维护命令以显式环境文件为准，应使用部署的环境文件，不依赖临时 shell 环境覆盖。随 wheel 发布的冻结资源用于阶段 A 的 Artifact 四表迁移：`pc_artifacts`、`pc_artifact_heads`、`pc_artifact_candidate_versions`、`pc_artifact_tags`。仅接受空目标和已登记的历史结构；完整 Server 数据库含有尚未纳管的对象时拒绝执行。 -维护入口可以单独识别由公开 1.3.0 wheel 实际初始化的完整 SQLite 结构，包括 FTS 影子表、索引和触发器。由于完整 Server 升级路径尚未注册，该来源返回 `server_migration_unavailable`。识别不会登记 revision、验证领域数据或允许写入；其他完整结构仍需登记精确形状。 +维护入口可以分别识别由公开 1.3.0 wheel 在 SQLite、seekDB 和 OceanBase CE 4.3.5.6 上实际初始化的完整结构。SQLite 包含 FTS 影子表、索引和触发器；两个 MySQL 兼容后端各有 56 张表、4 个保留解析器配置的全文索引。OceanBase 的原生 `AUTO_INCREMENT_MODE` 也参与结构校验，与不断变化的计数器分开处理。各清单仅适用于实际采集的后端。由于完整 Server 升级路径尚未注册，这些来源返回 `server_migration_unavailable`。识别不会登记 revision、验证领域数据或允许写入；其他完整结构仍需登记精确形状。 输出中的 `ready` 仅表示当前迁移 bundle 已通过结构和数据验证；`readiness_scope=registered_bundle`、`server_ready=false` 明确区分它与完整 Server、索引、旧任务及集群可用。不要以此替代生产升级或完整业务库初始化。普通业务启动的全面版本门禁和完整历史基线尚未纳管。已通过 OceanBase AI 4.6.3 的 ODP 入口完成隔离四表验收，覆盖固定维护主机互斥、DDL 提交后中断的显式确认恢复、数据保留与重复执行。这不代表完整 Server 升级、崩溃后远端 DDL 自动判定或完整 Fork 恢复已通过验收。 diff --git a/src/powercontext/builtin/persistence/migration_resources/server_baselines/1.3.0/oceanbase.json b/src/powercontext/builtin/persistence/migration_resources/server_baselines/1.3.0/oceanbase.json new file mode 100644 index 0000000000..70449250ec --- /dev/null +++ b/src/powercontext/builtin/persistence/migration_resources/server_baselines/1.3.0/oceanbase.json @@ -0,0 +1,11144 @@ +{ + "format": "powercontext.server-baseline.v1", + "release": "1.3.0", + "backend": "oceanbase", + "source_artifact": { + "filename": "powercontext-1.3.0-py3-none-any.whl", + "sha256": "8da7983763f3a6ab80b413c770a4973a46c557a54e1a1fa675099e58d47305e5", + "uploaded_at": "2026-10-12T03:55:56.682348Z" + }, + "capture": { + "python": "3.11.15", + "sqlalchemy": "2.0.51", + "driver": "aiomysql", + "engine": "5.7.25-OceanBase_CE-v4.3.5.6", + "image_digest": "sha256:31086a6900c21c479c2bcd942b6a28c53b17a51f4e9b9eb8eafcc596adfcd2e3", + "access_mode": "enforced", + "processing_role": "all", + "initialization": "Installed release Server; no model or embedding provider; direct MySQL tenant connection" + }, + "schema_fingerprint": "049414cd6208742a92fdbc1a2965bf37c397ce7e7f1823e7d89ab6445040fedb", + "inventory": { + "tables": { + "pc_access_audit": { + "columns": { + "cursor": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [ + "auto_increment" + ] + }, + "event_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "occurred_at": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "request_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "transport": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "operation": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "principal_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "principal_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "principal_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "actor_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "actor_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "actor_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "action": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "resource_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "deployment_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selector_type": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selector_entry_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "allowed": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "reason_code": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "policy_revision": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "matched_subject_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "matched_subject_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "matched_subject_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "binding_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "role": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "expected_version": { + "type": [ + "int", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "result_version": { + "type": [ + "int", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "cursor", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": { + "event_id": { + "columns": [ + [ + "event_id", + null, + "asc" + ] + ], + "unique": true + } + }, + "auto_increment_mode": "order" + }, + "pc_access_idempotency": { + "columns": { + "actor_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "idempotency_key_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "operation": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result_binding_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "secondary_binding_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "actor_id", + null, + "asc" + ], + [ + "idempotency_key_hash", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": { + "uq_pc_access_idempotency_actor_key": { + "columns": [ + [ + "actor_id", + null, + "asc" + ], + [ + "idempotency_key_hash", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_access_owners": { + "columns": { + "owner_kind": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "object_key_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "candidate_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selector_type": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selector_entry_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "owner_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "owner_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "owner_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "established_at": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "policy_revision": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "idempotency_key": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "owner_kind", + null, + "asc" + ], + [ + "object_key_hash", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_access_owners_kind": { + "expression": [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "owner_kind" + ], + [ + "literal", + "artifact" + ] + ], + [ + "is_not_null", + [ + "column", + "artifact_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "candidate_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "target_artifact_id" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "established_at" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "policy_revision" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "owner_kind" + ], + [ + "literal", + "candidate" + ] + ], + [ + "is_not_null", + [ + "column", + "candidate_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "artifact_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "selector_type" + ] + ] + ], + [ + "is_null", + [ + "column", + "selector_entry_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "established_at" + ] + ] + ], + [ + "is_null", + [ + "column", + "policy_revision" + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_access_relationship_heads": { + "columns": { + "name": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "name", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_access_relationship_heads_revision_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_access_relationships": { + "columns": { + "binding_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "subject_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "subject_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "subject_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "resource_key_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "resource_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "deployment_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selector_type": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selector_entry_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "role": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "singleton_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "granted_by_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "granted_by_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "granted_by_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "reason": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "created_at": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "expires_at": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "state": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "policy_revision": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "idempotency_key": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revoked_at": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revoked_by_type": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revoked_by_id": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revoked_by_description": { + "type": [ + "text", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "binding_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_access_relationships_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "singleton_key": { + "columns": [ + [ + "singleton_key", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_agent_skill_targets": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "display_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "agent_kind": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "installation_scope": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "delivery_mode": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "installation_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "state": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "enrollment_token_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "enrollment_expires_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "credential_subject": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "credential_verifier": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "receiver_version": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "environment_fingerprint": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "machine_hostname": { + "type": [ + "varchar", + [ + "255" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "workspace_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "last_seen_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "created_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "updated_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "target_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_agent_skill_targets_agent_kind": { + "expression": [ + "in", + [ + "column", + "agent_kind" + ], + [ + [ + "literal", + "codex" + ], + [ + "literal", + "claude_code" + ] + ] + ], + "enforced": true + }, + "ck_pc_agent_skill_targets_installation_scope": { + "expression": [ + "binary", + "=", + [ + "column", + "installation_scope" + ], + [ + "literal", + "project" + ] + ], + "enforced": true + }, + "ck_pc_agent_skill_targets_delivery_mode": { + "expression": [ + "binary", + "=", + [ + "column", + "delivery_mode" + ], + [ + "literal", + "agent_pull" + ] + ], + "enforced": true + }, + "ck_pc_agent_skill_targets_state": { + "expression": [ + "in", + [ + "column", + "state" + ], + [ + [ + "literal", + "pending" + ], + [ + "literal", + "active" + ], + [ + "literal", + "revoked" + ] + ] + ], + "enforced": true + }, + "ck_pc_agent_skill_targets_state_payload": { + "expression": [ + "binary", + "or", + [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "state" + ], + [ + "literal", + "pending" + ] + ], + [ + "is_not_null", + [ + "column", + "enrollment_token_digest" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "enrollment_expires_at" + ] + ] + ], + [ + "is_null", + [ + "column", + "installation_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "credential_subject" + ] + ] + ], + [ + "is_null", + [ + "column", + "credential_verifier" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "state" + ], + [ + "literal", + "active" + ] + ], + [ + "is_null", + [ + "column", + "enrollment_token_digest" + ] + ] + ], + [ + "is_null", + [ + "column", + "enrollment_expires_at" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "installation_id" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "credential_subject" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "credential_verifier" + ] + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "state" + ], + [ + "literal", + "revoked" + ] + ], + [ + "is_null", + [ + "column", + "enrollment_token_digest" + ] + ] + ], + [ + "is_null", + [ + "column", + "enrollment_expires_at" + ] + ] + ], + [ + "is_null", + [ + "column", + "credential_verifier" + ] + ] + ] + ], + "enforced": true + }, + "ck_pc_agent_skill_targets_generation_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_agent_skill_targets_installation": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "agent_kind", + null, + "asc" + ], + [ + "installation_scope", + null, + "asc" + ], + [ + "installation_id", + null, + "asc" + ] + ], + "unique": true + }, + "uq_pc_agent_skill_targets_enrollment_token": { + "columns": [ + [ + "enrollment_token_digest", + null, + "asc" + ] + ], + "unique": true + }, + "uq_pc_agent_skill_targets_credential_subject": { + "columns": [ + [ + "credential_subject", + null, + "asc" + ] + ], + "unique": true + }, + "uq_pc_agent_skill_targets_credential_verifier": { + "columns": [ + [ + "credential_verifier", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_artifact_candidate_heads": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "candidate_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "status": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result_family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result_revision": { + "type": [ + "int", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "decision_reason": { + "type": [ + "mediumtext", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "candidate_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "candidate_id", + "version" + ], + "referred_schema": null, + "referred_table": "pc_artifact_candidate_versions", + "referred_columns": [ + "scope_id", + "candidate_id", + "version" + ], + "ondelete": "restrict", + "onupdate": "restrict" + }, + { + "columns": [ + "scope_id", + "result_family", + "result_artifact_id", + "result_revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_artifact_candidate_heads_status": { + "expression": [ + "in", + [ + "column", + "status" + ], + [ + [ + "literal", + "pending" + ], + [ + "literal", + "approved" + ], + [ + "literal", + "rejected" + ] + ] + ], + "enforced": true + }, + "ck_pc_artifact_candidate_heads_terminal_result": { + "expression": [ + "binary", + "or", + [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "status" + ], + [ + "literal", + "approved" + ] + ], + [ + "is_not_null", + [ + "column", + "result_family" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "result_artifact_id" + ] + ] + ], + [ + "binary", + ">", + [ + "column", + "result_revision" + ], + [ + "number", + 0 + ] + ] + ], + [ + "is_null", + [ + "column", + "decision_reason" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "status" + ], + [ + "literal", + "rejected" + ] + ], + [ + "is_null", + [ + "column", + "result_family" + ] + ] + ], + [ + "is_null", + [ + "column", + "result_artifact_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "result_revision" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "decision_reason" + ] + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "status" + ], + [ + "literal", + "pending" + ] + ], + [ + "is_null", + [ + "column", + "result_family" + ] + ] + ], + [ + "is_null", + [ + "column", + "result_artifact_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "result_revision" + ] + ] + ], + [ + "is_null", + [ + "column", + "decision_reason" + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_candidate_versions": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "candidate_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "proposal": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_refs": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "artifact_refs": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "target_family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_revision": { + "type": [ + "int", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "reason": { + "type": [ + "mediumtext", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "candidate_id", + null, + "asc" + ], + [ + "version", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "target_family", + "target_artifact_id", + "target_revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_artifact_candidate_versions_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_artifact_candidate_versions_target_complete": { + "expression": [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "is_null", + [ + "column", + "target_family" + ] + ], + [ + "is_null", + [ + "column", + "target_artifact_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "target_revision" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "is_not_null", + [ + "column", + "target_family" + ] + ], + [ + "is_not_null", + [ + "column", + "target_artifact_id" + ] + ] + ], + [ + "binary", + ">", + [ + "column", + "target_revision" + ], + [ + "number", + 0 + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_heads": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "searchable_text": { + "type": [ + "mediumtext", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "lifecycle_state": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": [ + "literal", + "active" + ], + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "replacement_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "merged_into_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "governance_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_artifact_heads_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_artifact_heads_lifecycle_state": { + "expression": [ + "in", + [ + "column", + "lifecycle_state" + ], + [ + [ + "literal", + "active" + ], + [ + "literal", + "deprecated" + ], + [ + "literal", + "retired" + ] + ] + ], + "enforced": true + }, + "ck_pc_artifact_heads_governance_generation_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "governance_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_artifact_heads_replacement_deprecated": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "replacement_artifact_id" + ] + ], + [ + "binary", + "=", + [ + "column", + "lifecycle_state" + ], + [ + "literal", + "deprecated" + ] + ] + ], + "enforced": true + }, + "ck_pc_artifact_heads_merge_target": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "merged_into_id" + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "lifecycle_state" + ], + [ + "literal", + "deprecated" + ] + ], + [ + "binary", + "<>", + [ + "column", + "merged_into_id" + ], + [ + "column", + "artifact_id" + ] + ] + ], + [ + "is_null", + [ + "column", + "replacement_artifact_id" + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_artifact_heads_fts": { + "columns": [ + [ + "searchable_text", + null, + "asc" + ] + ], + "unique": false, + "kind": "fulltext", + "parser": "space", + "parser_properties": { + "min_token_size": 3, + "max_token_size": 84 + } + } + } + }, + "pc_artifact_lineage_artifacts": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "ordinal": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "upstream_family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "upstream_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "upstream_revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "is_merge_input": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": [ + "literal", + "0" + ], + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ], + [ + "ordinal", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "upstream_family", + "upstream_artifact_id", + "upstream_revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + }, + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": {} + }, + "pc_artifact_lineage_sources": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "ordinal": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_type": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ], + [ + "ordinal", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "source_type", + "source_id" + ], + "referred_schema": null, + "referred_table": "pc_sources", + "referred_columns": [ + "scope_id", + "source_type", + "source_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + }, + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": {} + }, + "pc_artifact_processing_auto_wave_targets": { + "columns": { + "wave_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_through": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "completed": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": [ + "literal", + "0" + ], + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "wave_id", + null, + "asc" + ], + [ + "binding_name", + null, + "asc" + ], + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_artifact_processing_auto_wave_target_source_positive": { + "expression": [ + "binary", + ">=", + [ + "column", + "source_through" + ], + [ + "number", + 1 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_processing_binding_states": { + "columns": { + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "last_auto_wave_completed_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "last_schedule_checkpoint_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "scan_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "scan_in_progress": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": [ + "literal", + "0" + ], + "charset": null, + "collation": null, + "extra": [] + }, + "scan_upper_pending_sequence": { + "type": [ + "bigint", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "binding_name", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_processing_scan_generation": { + "expression": [ + "binary", + ">=", + [ + "column", + "scan_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_processing_intents": { + "columns": { + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "pending_sequence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "dirty_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "clean_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "requested_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "handled_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "last_auto_scan_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "binding_name", + null, + "asc" + ], + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_processing_intent_sequence": { + "expression": [ + "binary", + ">", + [ + "column", + "pending_sequence" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_processing_intent_dirty": { + "expression": [ + "binary", + "and", + [ + "binary", + ">=", + [ + "column", + "clean_generation" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + "<=", + [ + "column", + "clean_generation" + ], + [ + "column", + "dirty_generation" + ] + ] + ], + "enforced": true + }, + "ck_pc_processing_intent_request": { + "expression": [ + "binary", + "and", + [ + "binary", + ">=", + [ + "column", + "handled_generation" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + "<=", + [ + "column", + "handled_generation" + ], + [ + "column", + "requested_generation" + ] + ] + ], + "enforced": true + }, + "ck_pc_processing_intent_scan": { + "expression": [ + "binary", + ">=", + [ + "column", + "last_auto_scan_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "pending_sequence": { + "columns": [ + [ + "pending_sequence", + null, + "asc" + ] + ], + "unique": true + }, + "ix_pc_processing_intent_binding_sequence": { + "columns": [ + [ + "binding_name", + null, + "asc" + ], + [ + "pending_sequence", + null, + "asc" + ] + ], + "unique": false + } + } + }, + "pc_artifact_processing_leases": { + "columns": { + "supervisor_group": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "holder_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "supervisor_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "lease_expires_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "supervisor_group", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_artifact_processing_leases_generation_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "supervisor_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_processing_migration_receipts": { + "columns": { + "migration_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_snapshot": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "requested_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "dirty_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_through": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "migration_id", + null, + "asc" + ], + [ + "binding_name", + null, + "asc" + ], + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": {} + }, + "pc_artifact_processing_pending": { + "columns": { + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_through": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "flush_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "handled_flush_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": [ + "number", + 0 + ], + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "binding_name", + null, + "asc" + ], + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_artifact_processing_pending_source_positive": { + "expression": [ + "binary", + ">=", + [ + "column", + "source_through" + ], + [ + "number", + 1 + ] + ], + "enforced": true + }, + "ck_pc_artifact_processing_pending_flush_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "flush_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_artifact_processing_pending_handled_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "handled_flush_generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_artifact_processing_pending_handled_not_ahead": { + "expression": [ + "binary", + "<=", + [ + "column", + "handled_flush_generation" + ], + [ + "column", + "flush_generation" + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_processing_schema": { + "columns": { + "singleton": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "schema_version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": [ + "number", + 1490 + ], + "charset": null, + "collation": null, + "extra": [] + }, + "phase": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "migration_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "config_manifest": { + "type": [ + "text", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "singleton", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_processing_schema_singleton": { + "expression": [ + "binary", + "=", + [ + "column", + "singleton" + ], + [ + "number", + 1 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_processing_sequences": { + "columns": { + "singleton": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "sequence": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "singleton", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_processing_sequence": { + "expression": [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "singleton" + ], + [ + "number", + 1 + ] + ], + [ + "binary", + ">=", + [ + "column", + "sequence" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_artifact_publications": { + "columns": { + "target_scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "content_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "idempotency_key": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "target_scope_id", + null, + "asc" + ], + [ + "target_family", + null, + "asc" + ], + [ + "target_artifact_id", + null, + "asc" + ], + [ + "target_revision", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "source_scope_id", + "source_family", + "source_artifact_id", + "source_revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + }, + { + "columns": [ + "target_scope_id", + "target_family", + "target_artifact_id", + "target_revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": { + "uq_pc_artifact_publications_request": { + "columns": [ + [ + "target_scope_id", + null, + "asc" + ], + [ + "idempotency_key", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_artifact_tags": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_type": { + "type": [ + "varchar", + [ + "12" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "tag_key_hash": { + "type": [ + "binary", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "tag_key": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "tag": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "assigned_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "target_type", + null, + "asc" + ], + [ + "target_id", + null, + "asc" + ], + [ + "tag_key_hash", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family", + "artifact_id" + ], + "referred_schema": null, + "referred_table": "pc_artifact_heads", + "referred_columns": [ + "scope_id", + "family", + "artifact_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_artifact_tags_target": { + "expression": [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "target_type" + ], + [ + "literal", + "artifact" + ] + ], + [ + "binary", + "=", + [ + "column", + "target_id" + ], + [ + "column", + "artifact_id" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "target_type" + ], + [ + "literal", + "memory_entry" + ] + ], + [ + "binary", + "=", + [ + "column", + "family" + ], + [ + "literal", + "memory" + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_artifact_tags_family_key": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "tag_key_hash", + null, + "asc" + ], + [ + "target_type", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "target_id", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_artifact_tags_key": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "tag_key_hash", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "target_type", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "target_id", + null, + "asc" + ] + ], + "unique": false + } + } + }, + "pc_artifacts": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "content": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": {} + }, + "pc_atomic_memory_current": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "state_version": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "content_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "projection_format": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "kind": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "searchable_text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "tag_keys": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "embedding": { + "type": [ + "blob", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "profile_fingerprint": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "embedding_input_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_atomic_memory_current_revision": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_atomic_memory_current_state_version": { + "expression": [ + "binary", + ">=", + [ + "column", + "state_version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_atomic_memory_current_embedding_metadata": { + "expression": [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "is_null", + [ + "column", + "embedding" + ] + ], + [ + "is_null", + [ + "column", + "profile_fingerprint" + ] + ] + ], + [ + "is_null", + [ + "column", + "embedding_input_hash" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "and", + [ + "is_not_null", + [ + "column", + "embedding" + ] + ], + [ + "is_not_null", + [ + "column", + "profile_fingerprint" + ] + ] + ], + [ + "is_not_null", + [ + "column", + "embedding_input_hash" + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_atomic_memory_current_fts": { + "columns": [ + [ + "searchable_text", + null, + "asc" + ] + ], + "unique": false, + "kind": "fulltext", + "parser": "space", + "parser_properties": { + "min_token_size": 3, + "max_token_size": 84 + } + } + } + }, + "pc_connector_checkpoints": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "binding_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "connector_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "connector_version": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "checkpoint": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "binding_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": {} + }, + "pc_decision_observations": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "observation_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "operation_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "created_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "consumer": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "policy_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "policy_version": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "mode": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "privacy_boundary": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "coverage": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "verdict": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "assessment_source": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "final_action": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "used_fallback": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "fallback_reason": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "provider_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "backend_model_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "model_policy_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "requests": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "input_tokens": { + "type": [ + "bigint", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "output_tokens": { + "type": [ + "bigint", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "latency_ms": { + "type": [ + "float", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "observation_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_decision_observations_requests_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "requests" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_decision_observations_input_tokens_nonnegative": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "input_tokens" + ] + ], + [ + "binary", + ">=", + [ + "column", + "input_tokens" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + }, + "ck_pc_decision_observations_output_tokens_nonnegative": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "output_tokens" + ] + ], + [ + "binary", + ">=", + [ + "column", + "output_tokens" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + }, + "ck_pc_decision_observations_latency_nonnegative": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "latency_ms" + ] + ], + [ + "binary", + ">=", + [ + "column", + "latency_ms" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_decision_observations_scope_created": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "created_at", + null, + "asc" + ], + [ + "observation_id", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_decision_observations_operation": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "operation_id", + null, + "asc" + ], + [ + "created_at", + null, + "asc" + ] + ], + "unique": false + } + } + }, + "pc_dream_runs": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "run_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "principal_key": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "idempotency_key": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "request_digest": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "operation": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "status": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "accepted_at": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "generation": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "request_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "run_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_dream_generation": { + "expression": [ + "binary", + ">=", + [ + "column", + "generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_dream_idempotency": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "principal_key", + null, + "asc" + ], + [ + "idempotency_key", + null, + "asc" + ] + ], + "unique": true + }, + "ix_pc_dream_list": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "accepted_at", + null, + "asc" + ], + [ + "run_id", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_dream_dispatch": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "operation", + null, + "asc" + ], + [ + "status", + null, + "asc" + ], + [ + "request_generation", + null, + "asc" + ] + ], + "unique": false + } + } + }, + "pc_external_skill_registrations": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "external_skill_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "provider": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "agent_kind": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "host_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "installation_scope": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "locator": { + "type": [ + "varchar", + [ + "2000" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "locator_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fingerprint": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "description": { + "type": [ + "varchar", + [ + "2000" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "external_skill_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_external_skill_registrations_scope": { + "expression": [ + "in", + [ + "column", + "installation_scope" + ], + [ + [ + "literal", + "user" + ], + [ + "literal", + "project" + ], + [ + "literal", + "plugin" + ] + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_external_skill_registrations_binding": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "provider", + null, + "asc" + ], + [ + "host_id", + null, + "asc" + ], + [ + "installation_scope", + null, + "asc" + ], + [ + "locator_hash", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_memory_entry_heads": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "memory_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "head_revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "entry_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "entry_version_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "entry_content_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "searchable_text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "memory_artifact_id", + null, + "asc" + ], + [ + "entry_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "memory_artifact_id", + "entry_id", + "entry_version_id" + ], + "referred_schema": null, + "referred_table": "pc_memory_entry_versions", + "referred_columns": [ + "scope_id", + "memory_artifact_id", + "entry_id", + "entry_version_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_memory_entry_heads_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "head_revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_memory_entry_versions": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "memory_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "entry_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "entry_version_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "previous_version_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "kind": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "source_refs": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "artifact_refs": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "entry_content_hash": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "created_in_revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "memory_artifact_id", + null, + "asc" + ], + [ + "entry_version_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_memory_entry_versions_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_memory_entry_versions_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "created_in_revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_memory_entry_versions_logical_version": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "memory_artifact_id", + null, + "asc" + ], + [ + "entry_id", + null, + "asc" + ], + [ + "version", + null, + "asc" + ] + ], + "unique": true + }, + "uq_pc_memory_entry_versions_identity": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "memory_artifact_id", + null, + "asc" + ], + [ + "entry_id", + null, + "asc" + ], + [ + "entry_version_id", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_memory_source_windows": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_through": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "window_limit": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_memory_source_window_positive": { + "expression": [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "source_through" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + ">", + [ + "column", + "window_limit" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_model_usage_daily": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "usage_date": { + "type": [ + "date", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "purpose": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "operation": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "requests": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "input_tokens": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "output_tokens": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "input_complete": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "output_complete": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "usage_date", + null, + "asc" + ], + [ + "purpose", + null, + "asc" + ], + [ + "operation", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_model_usage_daily_operation": { + "expression": [ + "in", + [ + "column", + "operation" + ], + [ + [ + "literal", + "generation" + ], + [ + "literal", + "embedding" + ] + ] + ], + "enforced": true + }, + "ck_pc_model_usage_daily_requests_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "requests" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_model_usage_daily_input_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "input_tokens" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_model_usage_daily_output_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "output_tokens" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_profile_policies": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "generation_enabled": { + "type": [ + "tinyint", + [ + "1" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "activation_mode": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "pending_candidate_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "version": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "updated_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + }, + { + "columns": [ + "scope_id", + "pending_candidate_id" + ], + "referred_schema": null, + "referred_table": "pc_artifact_candidate_heads", + "referred_columns": [ + "scope_id", + "candidate_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_profile_policies_activation_mode": { + "expression": [ + "in", + [ + "column", + "activation_mode" + ], + [ + [ + "literal", + "automatic" + ], + [ + "literal", + "review_required" + ] + ] + ], + "enforced": true + }, + "ck_pc_profile_policies_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_recall_effort_daily": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "usage_date": { + "type": [ + "date", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "policy_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "assessment": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "preparations": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "rounds": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "expanded_preparations": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "admission_expansions": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "policy_floor_expansions": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "candidate_round_samples": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "candidates_assessed": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "final_candidate_pool": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "added_embeddings": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "added_generation_calls": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "truncated_items": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "dropped_items": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "dropped_below_min_bytes": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "dropped_no_fitting_truncation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "usage_date", + null, + "asc" + ], + [ + "policy_id", + null, + "asc" + ], + [ + "assessment", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_recall_effort_preparations_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "preparations" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_rounds_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "rounds" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_expanded_preparations_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "expanded_preparations" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_admission_expansions_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "admission_expansions" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_policy_floor_expansions_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "policy_floor_expansions" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_candidate_round_samples_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "candidate_round_samples" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_candidates_assessed_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "candidates_assessed" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_final_candidate_pool_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "final_candidate_pool" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_added_embeddings_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "added_embeddings" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_added_generation_calls_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "added_generation_calls" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_truncated_items_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "truncated_items" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_dropped_items_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "dropped_items" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_dropped_below_min_bytes_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "dropped_below_min_bytes" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_recall_effort_dropped_no_fitting_truncation_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "dropped_no_fitting_truncation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recall_effort_daily_drop_total": { + "expression": [ + "binary", + "=", + [ + "column", + "dropped_items" + ], + [ + "binary", + "+", + [ + "column", + "dropped_below_min_bytes" + ], + [ + "column", + "dropped_no_fitting_truncation" + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_recall_token_daily": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "usage_date": { + "type": [ + "date", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "estimator_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "estimator_version": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "preparations": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "ready_preparations": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "comparable_preparations": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "baseline_tokens": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "recalled_tokens": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "usage_date", + null, + "asc" + ], + [ + "estimator_id", + null, + "asc" + ], + [ + "estimator_version", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_recall_token_daily_preparations_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "preparations" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recall_token_daily_ready_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "ready_preparations" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recall_token_daily_comparable_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "comparable_preparations" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recall_token_daily_comparable_ready": { + "expression": [ + "binary", + "<=", + [ + "column", + "comparable_preparations" + ], + [ + "column", + "ready_preparations" + ] + ], + "enforced": true + }, + "ck_pc_recall_token_daily_ready_total": { + "expression": [ + "binary", + "<=", + [ + "column", + "ready_preparations" + ], + [ + "column", + "preparations" + ] + ], + "enforced": true + }, + "ck_pc_recall_token_daily_baseline_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "baseline_tokens" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recall_token_daily_recalled_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "recalled_tokens" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_receipt_migration_review": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "reason": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "source_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": {} + }, + "pc_recurrence_match": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "match_key": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "task_outcome_source_type": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "task_outcome_source_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "task_outcome_position": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "failure_item_kind": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "failure_item_index": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "failure_item_digest": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "candidate_set_mode": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "candidate_set_digest": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "result": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "target_family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_revision": { + "type": [ + "int", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "signature_key": { + "type": [ + "mediumtext", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "match_key", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_recurrence_match_result": { + "expression": [ + "in", + [ + "column", + "result" + ], + [ + [ + "literal", + "matched" + ], + [ + "literal", + "unmatched" + ], + [ + "literal", + "ambiguous" + ] + ] + ], + "enforced": true + }, + "ck_pc_recurrence_match_candidate_set_mode": { + "expression": [ + "in", + [ + "column", + "candidate_set_mode" + ], + [ + [ + "literal", + "handoff_citations" + ], + [ + "literal", + "scope_heads" + ] + ] + ], + "enforced": true + }, + "ck_pc_recurrence_match_failure_item_kind": { + "expression": [ + "in", + [ + "column", + "failure_item_kind" + ], + [ + [ + "literal", + "observation" + ], + [ + "literal", + "check" + ] + ] + ], + "enforced": true + }, + "ck_pc_recurrence_match_position_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "task_outcome_position" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recurrence_match_item_index_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "failure_item_index" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_recurrence_match_identity": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "task_outcome_source_type", + null, + "asc" + ], + [ + "task_outcome_source_id", + null, + "asc" + ], + [ + "failure_item_kind", + null, + "asc" + ], + [ + "failure_item_index", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_recurrence_observation": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "observation_id": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selection_key": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "verdict_key": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "event": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "match_basis": { + "type": [ + "varchar", + [ + "8" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "signature_key": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "signature_key_hash": { + "type": [ + "binary", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "task_outcome_source_type": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "task_outcome_source_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "task_outcome_position": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "observation_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_recurrence_observation_event": { + "expression": [ + "in", + [ + "column", + "event" + ], + [ + [ + "literal", + "selected" + ], + [ + "literal", + "recurred" + ], + [ + "literal", + "avoided" + ] + ] + ], + "enforced": true + }, + "ck_pc_recurrence_observation_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_recurrence_observation_position_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "task_outcome_position" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_recurrence_observation_selection": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "selection_key", + null, + "asc" + ] + ], + "unique": true + }, + "uq_pc_recurrence_observation_verdict": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "verdict_key", + null, + "asc" + ] + ], + "unique": true + }, + "ix_pc_recurrence_observation_revision": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_recurrence_observation_key": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "signature_key_hash", + null, + "asc" + ] + ], + "unique": false + }, + "ix_pc_recurrence_observation_outcome": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "task_outcome_source_type", + null, + "asc" + ], + [ + "task_outcome_source_id", + null, + "asc" + ], + [ + "task_outcome_position", + null, + "asc" + ] + ], + "unique": false + } + } + }, + "pc_scope_bindings": { + "columns": { + "integration": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "kind": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "external_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "external_id_search": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "integration", + null, + "asc" + ], + [ + "kind", + null, + "asc" + ], + [ + "external_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": {} + }, + "pc_scope_context_references": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "referenced_scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "referenced_scope_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + }, + { + "columns": [ + "referenced_scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_scope_context_references_not_self": { + "expression": [ + "binary", + "<>", + [ + "column", + "scope_id" + ], + [ + "column", + "referenced_scope_id" + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_scope_creation_requests": { + "columns": { + "idempotency_key": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "request_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "idempotency_key", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": {} + }, + "pc_scope_external_references": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "ordinal": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "kind": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "value": { + "type": [ + "varchar", + [ + "2000" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "value_search": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "value_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "ordinal", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "cascade", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_scope_external_references_ordinal_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "ordinal" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "uq_pc_scope_external_references_value": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "kind", + null, + "asc" + ], + [ + "value_digest", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_scope_settings": { + "columns": { + "name": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "name", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": {}, + "indexes": {} + }, + "pc_scopes": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "title": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "summary": { + "type": [ + "varchar", + [ + "2000" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "scope_id_search": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "title_search": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "summary_search": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "parent_scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "version": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "parent_scope_id" + ], + "referred_schema": null, + "referred_table": "pc_scopes", + "referred_columns": [ + "scope_id" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_scopes_version_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "version" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_server_identity": { + "columns": { + "singleton_key": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "server_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "singleton_key", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_server_identity_singleton": { + "expression": [ + "binary", + "=", + [ + "column", + "singleton_key" + ], + [ + "number", + 1 + ] + ], + "enforced": true + } + }, + "indexes": { + "server_id": { + "columns": [ + [ + "server_id", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_skill_packages": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "tree_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "archive_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "archive_bytes": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "manifest": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "file_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "uncompressed_size": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "archive_size": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "created_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "tree_digest", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_skill_packages_file_count": { + "expression": [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "file_count" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + "<=", + [ + "column", + "file_count" + ], + [ + "number", + 256 + ] + ] + ], + "enforced": true + }, + "ck_pc_skill_packages_uncompressed_size": { + "expression": [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "uncompressed_size" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + "<=", + [ + "column", + "uncompressed_size" + ], + [ + "number", + 4194304 + ] + ] + ], + "enforced": true + }, + "ck_pc_skill_packages_archive_size": { + "expression": [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "archive_size" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + "<=", + [ + "column", + "archive_size" + ], + [ + "number", + 5242880 + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_skill_publications": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_id": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "desired_state": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": [ + "literal", + "published" + ], + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "desired_revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "desired_tree_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "observed_revision": { + "type": [ + "int", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "observed_tree_digest": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "observed_generation": { + "type": [ + "bigint", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "destination": { + "type": [ + "mediumtext", + [] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "state": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "selected_runtime_variant": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "environment_fingerprint": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "last_error_code": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "observed_at": { + "type": [ + "datetime", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "updated_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "target_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_skill_publications_desired_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "desired_revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_skill_publications_observed_revision_positive": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "observed_revision" + ] + ], + [ + "binary", + ">", + [ + "column", + "observed_revision" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + }, + "ck_pc_skill_publications_desired_state": { + "expression": [ + "in", + [ + "column", + "desired_state" + ], + [ + [ + "literal", + "published" + ], + [ + "literal", + "unpublished" + ] + ] + ], + "enforced": true + }, + "ck_pc_skill_publications_state": { + "expression": [ + "in", + [ + "column", + "state" + ], + [ + [ + "literal", + "unpublished" + ], + [ + "literal", + "pending" + ], + [ + "literal", + "current" + ], + [ + "literal", + "update_available" + ], + [ + "literal", + "delivery_failed" + ], + [ + "literal", + "conflict" + ], + [ + "literal", + "drifted" + ], + [ + "literal", + "incompatible" + ] + ] + ], + "enforced": true + }, + "ck_pc_skill_publications_observed_generation_nonnegative": { + "expression": [ + "binary", + "or", + [ + "is_null", + [ + "column", + "observed_generation" + ] + ], + [ + "binary", + ">=", + [ + "column", + "observed_generation" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + }, + "ck_pc_skill_publications_generation_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_source_cursors": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "cursor": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "binding_name", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_source_cursors_generation_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "generation" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_source_definition_manifests": { + "columns": { + "definition_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "definition_version": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "fingerprint": { + "type": [ + "varchar", + [ + "71" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "manifest": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "definition_name", + null, + "asc" + ], + [ + "definition_version", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": { + "uq_pc_source_definition_manifest_fingerprint": { + "columns": [ + [ + "definition_name", + null, + "asc" + ], + [ + "fingerprint", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_source_journal_heads": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "position": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_source_journal_heads_position_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "position" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_sources": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_type": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload": { + "type": [ + "mediumblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "journal_position": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "source_type", + null, + "asc" + ], + [ + "source_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": { + "uq_pc_sources_scope_journal_position": { + "columns": [ + [ + "scope_id", + null, + "asc" + ], + [ + "journal_position", + null, + "asc" + ] + ], + "unique": true + } + } + }, + "pc_topic_memory_active_chunks": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "chunk_ordinal": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "start_offset": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "end_offset": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "chunk_text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "searchable_text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "policy_version": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "chunk_ordinal", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_topic_memory_revision_publications", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_topic_memory_active_chunks_family": { + "expression": [ + "binary", + "=", + [ + "column", + "family" + ], + [ + "literal", + "topic-memory" + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_active_chunks_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_active_chunks_ordinal_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "chunk_ordinal" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_active_chunks_start_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "start_offset" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_active_chunks_offsets": { + "expression": [ + "binary", + ">", + [ + "column", + "end_offset" + ], + [ + "column", + "start_offset" + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_topic_memory_active_chunks_fts": { + "columns": [ + [ + "searchable_text", + null, + "asc" + ] + ], + "unique": false, + "kind": "fulltext", + "parser": "space", + "parser_properties": { + "min_token_size": 3, + "max_token_size": 84 + } + } + } + }, + "pc_topic_memory_active_topics": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "title": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "summary": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "searchable_text": { + "type": [ + "mediumtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + }, + "source_count": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_topic_memory_revision_publications", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_topic_memory_active_topics_family": { + "expression": [ + "binary", + "=", + [ + "column", + "family" + ], + [ + "literal", + "topic-memory" + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_active_topics_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_active_topics_sources_nonnegative": { + "expression": [ + "binary", + ">=", + [ + "column", + "source_count" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": { + "ix_pc_topic_memory_active_topics_fts": { + "columns": [ + [ + "searchable_text", + null, + "asc" + ] + ], + "unique": false, + "kind": "fulltext", + "parser": "space", + "parser_properties": { + "min_token_size": 3, + "max_token_size": 84 + } + } + } + }, + "pc_topic_memory_processing_targets": { + "columns": { + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_request_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_through": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "captured_flush_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "observed_dirty_generation": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "binding_name", + null, + "asc" + ], + [ + "scope_id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_topic_processing_target": { + "expression": [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "target_request_generation" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + ">=", + [ + "column", + "source_through" + ], + [ + "number", + 0 + ] + ] + ], + [ + "binary", + ">=", + [ + "column", + "captured_flush_generation" + ], + [ + "number", + 0 + ] + ] + ], + [ + "binary", + ">=", + [ + "column", + "observed_dirty_generation" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_topic_memory_retrieval_shape": { + "columns": { + "singleton": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "shape": { + "type": [ + "varchar", + [ + "16" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "profile_fingerprint": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "singleton", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_topic_memory_retrieval_shape_singleton": { + "expression": [ + "binary", + "=", + [ + "column", + "singleton" + ], + [ + "number", + 1 + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_retrieval_shape_value": { + "expression": [ + "in", + [ + "column", + "shape" + ], + [ + [ + "literal", + "fts" + ], + [ + "literal", + "hybrid" + ] + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_retrieval_shape_profile": { + "expression": [ + "binary", + "or", + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "shape" + ], + [ + "literal", + "fts" + ] + ], + [ + "is_null", + [ + "column", + "profile_fingerprint" + ] + ] + ], + [ + "binary", + "and", + [ + "binary", + "=", + [ + "column", + "shape" + ], + [ + "literal", + "hybrid" + ] + ], + [ + "is_not_null", + [ + "column", + "profile_fingerprint" + ] + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_topic_memory_revision_publications": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "published_at": { + "type": [ + "datetime", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ] + ], + "foreign_keys": [ + { + "columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "referred_schema": null, + "referred_table": "pc_artifacts", + "referred_columns": [ + "scope_id", + "family", + "artifact_id", + "revision" + ], + "ondelete": "restrict", + "onupdate": "restrict" + } + ], + "checks": { + "ck_pc_topic_memory_publications_family": { + "expression": [ + "binary", + "=", + [ + "column", + "family" + ], + [ + "literal", + "topic-memory" + ] + ], + "enforced": true + }, + "ck_pc_topic_memory_publications_revision_positive": { + "expression": [ + "binary", + ">", + [ + "column", + "revision" + ], + [ + "number", + 0 + ] + ], + "enforced": true + } + }, + "indexes": {} + }, + "pc_topic_memory_work_budgets": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "binding_name": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "source_after": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_through": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "attempt_id": { + "type": [ + "varchar", + [ + "36" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "attempts": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "requests": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "tokens": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "failure_code": { + "type": [ + "varchar", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "binding_name", + null, + "asc" + ], + [ + "source_after", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": { + "ck_pc_topic_budget_window": { + "expression": [ + "binary", + "and", + [ + "binary", + ">=", + [ + "column", + "source_after" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + ">", + [ + "column", + "source_through" + ], + [ + "column", + "source_after" + ] + ] + ], + "enforced": true + }, + "ck_pc_topic_budget_usage": { + "expression": [ + "binary", + "and", + [ + "binary", + "and", + [ + "binary", + ">", + [ + "column", + "attempts" + ], + [ + "number", + 0 + ] + ], + [ + "binary", + ">=", + [ + "column", + "requests" + ], + [ + "number", + 0 + ] + ] + ], + [ + "binary", + ">=", + [ + "column", + "tokens" + ], + [ + "number", + 0 + ] + ] + ], + "enforced": true + } + }, + "indexes": {} + } + }, + "views": [], + "triggers": [] + } +} diff --git a/src/powercontext/builtin/persistence/migration_resources/server_baselines/README.md b/src/powercontext/builtin/persistence/migration_resources/server_baselines/README.md index 0f8c0d77b4..5cf57a22b1 100644 --- a/src/powercontext/builtin/persistence/migration_resources/server_baselines/README.md +++ b/src/powercontext/builtin/persistence/migration_resources/server_baselines/README.md @@ -53,3 +53,24 @@ Generated foreign-key names and physical storage options are normalized by the MySQL reflector; index identity and search semantics are retained. Recognition requires the complete matching inventory. It does not cover optional embedding profiles or establish an OceanBase baseline. + +## OceanBase 1.3.0 + +`1.3.0/oceanbase.json` was captured independently from an installed public +1.3.0 Server using a direct MySQL tenant connection to +`5.7.25-OceanBase_CE-v4.3.5.6`. The container image digest is +`sha256:31086a6900c21c479c2bcd942b6a28c53b17a51f4e9b9eb8eafcc596adfcd2e3`. +The source wheel, enforced Access, all Processing roles, disabled model and +embedding providers, and public HTTP fixture match the other source captures. + +All 56 tables and four FULLTEXT indexes participate in its fingerprint +`049414cd6208742a92fdbc1a2965bf37c397ce7e7f1823e7d89ab6445040fedb`. +The native audit table's `AUTO_INCREMENT_MODE='ORDER'` is retained as a schema +property, distinct from the changing auto-increment counter. Unknown or +duplicate mode declarations reject. This source differs from the seekDB +inventory and is registered only for OceanBase. + +The stopped Server's table definitions, counts and data hashes were unchanged +after reflection and `plan/verify/apply` rejection. This is complete source +recognition with `server_migration_unavailable`, not a successful upgrade, +runtime-readiness check, ODP test or multi-node acceptance. diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py index ba7a507b75..0164460dc5 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql_schema.py +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -373,6 +373,25 @@ def _validate_physical_value(option: str, value: str) -> None: raise MigrationError("incompatible_schema", "An unsupported table engine or index organization was discovered.") +def _table_options(tokens: list[str]) -> tuple[dict[str, str | None], str | None]: + positions = [position for position, option in enumerate(tokens) if option == "auto_increment_mode"] + if not positions: + return _physical(tokens), None + if len(positions) != 1: + raise MigrationError("incompatible_schema", "Duplicate AUTO_INCREMENT_MODE declarations were discovered.") + start = positions[0] + position = start + 1 + if tokens[position : position + 1] == ["="]: + position += 1 + if position >= len(tokens): + raise MigrationError("incompatible_schema", "AUTO_INCREMENT_MODE has no value.") + value = tokens[position] + mode = (_literal(value) if value.startswith("'") else value).lower() + if mode not in {"order", "noorder"}: + raise MigrationError("incompatible_schema", "An unsupported AUTO_INCREMENT_MODE was discovered.") + return _physical(tokens[:start] + tokens[position + 1 :]), mode + + def _column(tokens: list[str], defaults: dict[str, str | None]) -> dict[str, Any]: if len(tokens) < 2: raise MigrationError("incompatible_schema", "A column definition is incomplete.") @@ -526,8 +545,10 @@ def _parse_table(statement: str, *, database: str | None = None) -> tuple[str, d if schema is not None: raise MigrationError("incompatible_schema", "A table belongs to an unexpected database.") body, position = _group(tokens, position) - defaults = _physical(tokens[position:]) + defaults, auto_increment_mode = _table_options(tokens[position:]) table: dict[str, Any] = {"columns": {}, "primary_key": [], "foreign_keys": [], "checks": {}, "indexes": {}} + if auto_increment_mode is not None: + table["auto_increment_mode"] = auto_increment_mode for part in _parts(body): _table_clause(table, part, defaults, database) table["foreign_keys"].sort(key=digest) diff --git a/tests/builtin/persistence/test_mysql_migration_schema.py b/tests/builtin/persistence/test_mysql_migration_schema.py index c05c6f81dd..4ad9e45fdd 100644 --- a/tests/builtin/persistence/test_mysql_migration_schema.py +++ b/tests/builtin/persistence/test_mysql_migration_schema.py @@ -419,6 +419,28 @@ def test_named_unique_constraints_and_native_keys_have_the_same_fingerprint(sche schema.identify(native) +def test_native_oceanbase_auto_increment_mode_is_preserved_while_the_counter_can_advance() -> None: + path = Path(__file__).parents[2] / "fixtures/database_migrations/schemas/oceanbase_v1_3_0_audit.json" + statement = json.loads(path.read_text())["tables"]["pc_access_audit"] + original = _inventory(_ReflectedDatabase({"pc_access_audit": statement})) + assert original["tables"]["pc_access_audit"]["auto_increment_mode"] == "order" + advanced = _inventory( + _ReflectedDatabase({"pc_access_audit": statement.replace("AUTO_INCREMENT = 37", "AUTO_INCREMENT = 900")}) + ) + assert advanced == original + unordered = _inventory(_ReflectedDatabase({"pc_access_audit": statement.replace("'ORDER'", "'NOORDER'")})) + assert unordered["tables"]["pc_access_audit"]["auto_increment_mode"] == "noorder" + assert digest(unordered) != digest(original) + + +@pytest.mark.parametrize("mode", ["'FUTURE'", "'ORDER' AUTO_INCREMENT_MODE='NOORDER'"]) +def test_unknown_or_duplicate_auto_increment_mode_is_not_silently_discarded(mode: str) -> None: + path = Path(__file__).parents[2] / "fixtures/database_migrations/schemas/oceanbase_v1_3_0_audit.json" + statement = json.loads(path.read_text())["tables"]["pc_access_audit"].replace("'ORDER'", mode) + with pytest.raises(MigrationError, match="incompatible_schema"): + _inventory(_ReflectedDatabase({"pc_access_audit": statement})) + + @pytest.mark.parametrize("applied", range(7)) def test_baseline_recovery_accepts_only_the_completed_statement_prefix(schema: MySQLSchema, applied: int) -> None: inventory = _inventory(_ReflectedDatabase(_show_tables(baseline_prefix=applied))) diff --git a/tests/builtin/persistence/test_server_migration_baselines.py b/tests/builtin/persistence/test_server_migration_baselines.py index 15dc4d6ece..2351140870 100644 --- a/tests/builtin/persistence/test_server_migration_baselines.py +++ b/tests/builtin/persistence/test_server_migration_baselines.py @@ -130,23 +130,38 @@ def test_weakened_constraint_with_identical_columns_is_not_recognized(released_d assert released_database.read_bytes() == before -def test_complete_seekdb_release_inventory_is_recognized_only_for_its_captured_backend() -> None: - resource = json.loads(RESOURCE.with_name("seekdb.json").read_text(encoding="utf-8")) +@pytest.mark.parametrize("backend", ["seekdb", "oceanbase"]) +def test_complete_mysql_release_inventory_is_recognized_only_for_its_captured_backend(backend) -> None: + resource = json.loads(RESOURCE.with_name(f"{backend}.json").read_text(encoding="utf-8")) inventory = resource["inventory"] before = copy.deepcopy(inventory) - baseline = recognize_mysql_server_baseline(inventory, backend="seekdb") + baseline = recognize_mysql_server_baseline(inventory, backend=backend) assert baseline is not None assert baseline.release == "1.3.0" assert baseline.schema_fingerprint == resource["schema_fingerprint"] - assert recognize_mysql_server_baseline(inventory, backend="oceanbase") is None + other_backend = "oceanbase" if backend == "seekdb" else "seekdb" + assert recognize_mysql_server_baseline(inventory, backend=other_backend) is None assert inventory == before +@pytest.mark.parametrize("backend", ["seekdb", "oceanbase"]) @pytest.mark.parametrize( - "change", ["table", "control", "column", "constraint", "index", "parser", "properties", "view", "trigger", "subset"] + "change", + [ + "table", + "control", + "column", + "constraint", + "index", + "parser", + "properties", + "view", + "trigger", + "subset", + ], ) -def test_incomplete_or_altered_seekdb_inventories_are_not_released_baselines(change: str) -> None: - inventory = json.loads(RESOURCE.with_name("seekdb.json").read_text(encoding="utf-8"))["inventory"] +def test_incomplete_or_altered_mysql_inventories_are_not_released_baselines(change: str, backend) -> None: + inventory = json.loads(RESOURCE.with_name(f"{backend}.json").read_text(encoding="utf-8"))["inventory"] table = inventory["tables"]["pc_artifact_heads"] if change == "table": del inventory["tables"]["pc_sources"] @@ -171,5 +186,11 @@ def test_incomplete_or_altered_seekdb_inventories_are_not_released_baselines(cha name: shape for name, shape in inventory["tables"].items() if name.startswith("pc_artifact") } before = copy.deepcopy(inventory) - assert recognize_mysql_server_baseline(inventory, backend="seekdb") is None + assert recognize_mysql_server_baseline(inventory, backend=backend) is None assert inventory == before + + +def test_changed_audit_auto_increment_mode_is_not_a_released_oceanbase_baseline() -> None: + inventory = json.loads(RESOURCE.with_name("oceanbase.json").read_text(encoding="utf-8"))["inventory"] + inventory["tables"]["pc_access_audit"]["auto_increment_mode"] = "noorder" + assert recognize_mysql_server_baseline(inventory, backend="oceanbase") is None diff --git a/tests/fixtures/database_migrations/schemas/oceanbase_v1_3_0_audit.json b/tests/fixtures/database_migrations/schemas/oceanbase_v1_3_0_audit.json new file mode 100644 index 0000000000..3159c569ce --- /dev/null +++ b/tests/fixtures/database_migrations/schemas/oceanbase_v1_3_0_audit.json @@ -0,0 +1,7 @@ +{ + "tables": { + "pc_access_audit": "CREATE TABLE `pc_access_audit` (\n `cursor` int(11) NOT NULL AUTO_INCREMENT,\n `event_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `occurred_at` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `request_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `transport` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `operation` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `principal_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `principal_id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `principal_description` text DEFAULT NULL,\n `actor_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `actor_id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `actor_description` text DEFAULT NULL,\n `action` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `resource_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `deployment_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `scope_id` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `family` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `artifact_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `selector_type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `selector_entry_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `allowed` tinyint(1) NOT NULL,\n `reason_code` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `policy_revision` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `matched_subject_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `matched_subject_id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `matched_subject_description` text DEFAULT NULL,\n `binding_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `target_type` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `target_id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `target_description` text DEFAULT NULL,\n `role` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `expected_version` int(11) DEFAULT NULL,\n `result_version` int(11) DEFAULT NULL,\n PRIMARY KEY (`cursor`),\n UNIQUE KEY `event_id` (`event_id`) BLOCK_SIZE 16384 LOCAL\n) ORGANIZATION INDEX AUTO_INCREMENT = 37 AUTO_INCREMENT_MODE = 'ORDER' DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' REPLICA_NUM = 1 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE ENABLE_MACRO_BLOCK_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0" + }, + "engine": "5.7.25-OceanBase_CE-v4.3.5.6", + "source_release": "1.3.0" +} From dc6e1885e1a9db2ba5bcd7d2cc35015d5cd98fb9 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 22 Oct 2026 19:18:19 +0800 Subject: [PATCH 38/94] fix(migrations): validate durable task ownership before admission --- .../remote-access-implementation.md | 12 +------ docs/en/docs/operate/database-migrations.md | 2 +- .../remote-access-implementation.md | 10 +----- docs/zh/docs/operate/database-migrations.md | 2 +- release/upgrade-support.json | 2 +- .../server_resources/dream/manifest.json | 12 +++---- .../server_resources/server_domain.py | 35 +++++++++++++++---- .../test_full_server_migrations.py | 12 +++++++ 8 files changed, 51 insertions(+), 36 deletions(-) diff --git a/docs/en/development/remote-access-implementation.md b/docs/en/development/remote-access-implementation.md index f0fe6c4b07..79077568d8 100644 --- a/docs/en/development/remote-access-implementation.md +++ b/docs/en/development/remote-access-implementation.md @@ -141,17 +141,7 @@ SQLite URI interpretation; uppercase schemes, leading spaces, and literal filena storage. Pooling keeps one connection for nonpersistent storage; offline maintenance, cursor-secret persistence, and subprocess workers consume that same persistence classification. -The unified migration implementation from [RFC #1771](../rfcs/1771-unified-database-migrations.md) currently -covers only a registered four-table Artifact bundle. It does not manage `pc_server_identity`, gate ordinary Server -startup, or establish complete Server readiness; its maintenance commands reject a complete business database with -unmanaged tables. Do not use that partial bundle to migrate a Server database. - -When unified migration takes ownership of the complete Server schema, identity table creation must move from startup -DDL into an immutable managed revision, with schema verification before Runtime composition. If discovery is already -released, the supported historical baseline must include the existing identity table and preserve its singleton value; -if still unmerged at framework enablement, discovery must ship that revision alongside the model. Deployment identity -remains distinct from `pc_schema_revision`, and schema adoption or upgrades must not rotate `server_id`. Clone rotation -remains an explicit offline operation. No independent schema-version or migration-readiness marker is added for identity. +[Full Server migration](../docs/operate/database-migrations.md) includes `pc_server_identity` in the frozen 1.3.0 baseline and shared revision chain, checking the schema before Runtime composition. Upgrades preserve the existing singleton value. Deployment identity remains independent of `pc_schema_revision`; upgrading does not rotate `server_id`. Clone rotation remains an explicit offline operation, with no separate identity schema version or readiness marker. A partial Artifact bundle cannot establish full Server readiness. Treat a restored backup as the same deployment and keep its ID. When a backup is used to create an independent clone, stop every Server process using the clone database and rotate only the clone: diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 0f7fbf4f21..6d3404cb58 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -20,7 +20,7 @@ Only successful schema, domain, projection, and task validation returns `state=r Stop all writers and drain queued/running 1.3.0 Dream tasks before upgrading: the target cannot execute the old prompt contract. Terminal task payloads and pending Processing counters are preserved; old worker leases are fenced. Registered capability additions retain the old automatic-processing choices instead of replacing ownership with the current configuration. -Old Candidate tables are retained as `pc_legacy_artifact_candidate_heads` and `pc_legacy_artifact_candidate_versions` for migration-copy verification. They and the backup are not live write targets; subsequent business changes are not compared against old snapshots. New historical Artifact provenance fields remain unknown, and Source evidence is not inferred from old references. Projection rebuilds and verification scan their tables; budget downtime and capacity for backups and retained copies. +Old Candidate tables are retained as `pc_retained_dream_candidate_heads` and `pc_retained_dream_candidate_versions` for migration-copy verification. They and the backup are not live write targets; subsequent business changes are not compared against old snapshots. New historical Artifact provenance fields remain unknown, and Source evidence is not inferred from old references. Projection rebuilds and verification scan their tables; budget downtime and capacity for backups and retained copies. Ordinary Server startup checks readiness without repairing an existing database. Empty local persistent SQLite/seekDB storage initializes through the same revision chain; OceanBase requires an explicit maintenance job first. In-memory SQLite remains transient runtime storage. A changed schema or inconsistent projection requires maintenance, not restart-based repair. diff --git a/docs/zh/development/remote-access-implementation.md b/docs/zh/development/remote-access-implementation.md index 7c70469b80..8f2b656aa6 100644 --- a/docs/zh/development/remote-access-implementation.md +++ b/docs/zh/development/remote-access-implementation.md @@ -130,15 +130,7 @@ SQLite 存储分类使用 dialect 的实际连接参数和解码后的 URI，包 不会归一化大写 scheme、前导空格或文件名中的控制字符。`SQLiteConfig.is_in_memory` 区分内存存储；`SQLiteConfig.is_persistent` 同时排除内存与临时 存储。非持久存储的连接池保留一个连接；离线维护、cursor secret 持久化及子进程 worker 统一使用该持久性分类。 -[RFC #1771](../rfcs/1771-unified-database-migrations.md) 对应的统一迁移实现目前只覆盖已注册的四张 Artifact 表。 -它尚未管理 `pc_server_identity`、接管普通 Server 启动检查或证明完整 Server readiness；维护命令会拒绝包含未管理表的 -完整业务数据库。不能使用这个局部 bundle 迁移 Server 数据库。 - -统一迁移接管完整 Server schema 时，identity table 创建必须从启动 DDL 移到不可变的受管 revision，并在 Runtime -组装前验证 schema。若 discovery 已发布，支持的历史 baseline 必须包含现有 identity table，并保留 singleton 值； -若在框架启用时仍未合并，discovery 必须随 model 一起提交该 revision。deployment identity 与 `pc_schema_revision` -保持独立，schema 纳管或升级不能轮换 `server_id`。clone 轮换仍是显式离线操作，identity 不另设 schema version 或 -migration readiness marker。 +[完整 Server 迁移](../docs/operate/database-migrations.md)已将 `pc_server_identity` 纳入冻结的 1.3.0 基线和共享版本链，并在 Runtime 组装前检查结构。迁移保留原有 singleton 值；deployment identity 与 `pc_schema_revision` 独立，升级不会轮换 `server_id`。clone 轮换仍是显式离线操作，identity 不另设 schema version 或 readiness marker。局部 Artifact bundle 不能证明完整 Server 就绪。 把备份恢复为原 deployment 时应保留原 ID。若用备份创建独立 clone，请停止所有使用 clone 数据库的 Server 进程，然后只在 clone 上轮换： diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 1eaa80bb52..19b4bce250 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -18,7 +18,7 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 升级前必须停止所有写入者并排空 1.3.0 中 queued/running Dream 任务；新版不能执行旧提示词契约。终态任务保留原始 payload，待处理 Processing intent 的计数保留，旧 worker lease 被 fencing。迁移按登记规则扩展处理能力，保留原有自动处理选择，不根据当前配置覆盖旧所有权。 -Candidate 旧表保留为 `pc_legacy_artifact_candidate_heads` 和 `pc_legacy_artifact_candidate_versions`，用于迁移当次的复制验证。它们与迁移备份都不是线上写入目标；升级后的正常修改不再与旧快照比较。历史 Artifact 新增的来源时间/操作者字段保持未知，Source evidence 不凭历史引用虚构声明。投影重建和数据校验会扫描相关表；请为停机时间、备份与保留副本安排容量。 +Candidate 旧表保留为 `pc_retained_dream_candidate_heads` 和 `pc_retained_dream_candidate_versions`，用于迁移当次的复制验证。它们与迁移备份都不是线上写入目标；升级后的正常修改不再与旧快照比较。历史 Artifact 新增的来源时间/操作者字段保持未知，Source evidence 不凭历史引用虚构声明。投影重建和数据校验会扫描相关表；请为停机时间、备份与保留副本安排容量。 普通 Server 启动先做只读就绪检查，不修补已有库。空的本地持久 SQLite/seekDB 可通过同一版本链初始化；OceanBase 必须先执行显式维护命令初始化。内存 SQLite 仍为临时运行时存储。迁移后发现结构或投影被篡改，必须停写维护，不能靠重启修复。 diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 02af28a089..5003b93785 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,7 +3,7 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "41fe0fdca76591fc4ce51ed6efa6ace8c910a476a68f34710f0c14c24843e8d3", + "persistence_sha256": "7eb8a861925bbb8fc9080d97e9bae83d772280d786d9a05d40d6bec8e7c9e734", "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history and pending intents with projection rebuilds and read-only startup admission. Publication requires installed-release upgrade, repeat-apply, old-startup rejection and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles." }, "cases": [], diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/dream/manifest.json b/src/powercontext/builtin/persistence/migrations/server_resources/dream/manifest.json index 01cdc9c6ae..445f144be7 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/dream/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/server_resources/dream/manifest.json @@ -4,9 +4,9 @@ "migration_rfc": 1771, "framework_implementation_pr": 1772, "registration": { - "state": "awaiting_full_database_bundle", - "revision": null, - "down_revision": null, + "state": "registered", + "revision": "s1400", + "down_revision": "s1300", "server_ready": false }, "execution_mode": "maintenance", @@ -55,8 +55,8 @@ ], "deprecation_policy": "Explicit direct-removal exception in RFC 1715; coordinated client upgrade", "aliases": false, - "removal_release": null, - "minimum_client_version": null + "removal_release": "1.4.0", + "minimum_client_version": "1.4.0" }, "task_formats": { "dream_runs": { @@ -78,7 +78,7 @@ "frozen_execution": "Never rewrite prompt/model identity, started_at, deadline_at, retry counters or failure outcomes" }, "processing_intents": "Preserve all fields; initialize consecutive_dream_attempts to zero", - "other_queues_and_leases": "Full framework inventory must validate, convert or drain before registration", + "other_queues_and_leases": "Full Server preflight validates registered Processing bindings and counters; the revision fences the shared processing lease. Other historical schemas and optional feature queues are not adopted.", "processing_manifest": "Require the target deployment manifest; preserve explicit capabilities, ownership mode and historical automatic bindings; add canonical Handoff/Prompt bindings; allow only declared Handoff/Prompt capability additions and reject incompatible targets before writes" }, "framework_requirements": [ diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/server_domain.py b/src/powercontext/builtin/persistence/migrations/server_resources/server_domain.py index 241ad03d90..515405a689 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/server_domain.py +++ b/src/powercontext/builtin/persistence/migrations/server_resources/server_domain.py @@ -14,6 +14,7 @@ """Registered Server 1.3 maintenance and live readiness requirements.""" +import hashlib import json import runpy from pathlib import Path @@ -54,6 +55,7 @@ def preflight(connection, *, configuration): # collection data needs its separately declared conversion, never a stamp. finish(assert_atomic_memory_migration_ready(pinned(connection))) if "pc_artifact_candidate_versions" in names: + _validate_intent_bindings(connection) module = dream() try: module["validate_dream_tasks"](connection) @@ -138,11 +140,9 @@ def validate(connection, *, configuration): ) if actual != expected: raise MigrationError("projection_migration_required", "The registered search projections are not current.") - manifest = json.loads( - connection.exec_driver_sql( - "SELECT config_manifest FROM pc_artifact_processing_schema WHERE singleton=1" - ).scalar_one() - ) + frozen = dream() + manifest = frozen["_processing_manifest"](connection, frozen["_schema"](), upgraded=True) + _validate_intent_bindings(connection) target = configuration.get("processing_manifest") if target and any(manifest[key] != target[key] for key in ("bindings", "mode", "capabilities")): raise MigrationError( @@ -150,8 +150,18 @@ def validate(connection, *, configuration): ) from powercontext.builtin.persistence.dream import DreamRepository - for scope, run in connection.exec_driver_sql("SELECT scope_id,run_id FROM pc_dream_runs"): - finish(DreamRepository().get(pinned(connection), scope, run)) + for row in connection.exec_driver_sql( + "SELECT scope_id,run_id,status,operation,generation,request_generation,principal_key,idempotency_key FROM pc_dream_runs" + ).mappings(): + record = finish(DreamRepository().get(pinned(connection), row["scope_id"], row["run_id"])) + if ( + any(getattr(record.run, key) != row[key] for key in ("scope_id", "run_id", "status", "operation")) + or record.generation != row["generation"] + or record.request_generation != row["request_generation"] + or hashlib.sha256(record.principal_id.encode()).hexdigest() != row["principal_key"] + or (record.request is not None and record.request.idempotency_key != row["idempotency_key"]) + ): + raise MigrationError("unsupported_task_format", "A Dream payload disagrees with its durable task identity.") from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository for scope, binding in connection.exec_driver_sql( @@ -217,3 +227,14 @@ def _validate_sqlite_fts(connection): raise MigrationError( "projection_verification_failed", "A full-text helper differs from its authoritative current rows." ) + + +def _validate_intent_bindings(connection): + manifest = json.loads( + connection.exec_driver_sql( + "SELECT config_manifest FROM pc_artifact_processing_schema WHERE singleton=1" + ).scalar_one() + ) + for (binding,) in connection.exec_driver_sql("SELECT DISTINCT binding_name FROM pc_artifact_processing_intents"): + if binding not in manifest["bindings"]: + raise MigrationError("unsupported_task_format", "A pending Processing intent has no registered binding.") diff --git a/tests/builtin/persistence/test_full_server_migrations.py b/tests/builtin/persistence/test_full_server_migrations.py index 1923b7f6fb..f88c31abc0 100644 --- a/tests/builtin/persistence/test_full_server_migrations.py +++ b/tests/builtin/persistence/test_full_server_migrations.py @@ -205,3 +205,15 @@ def test_captured_oceanbase_baseline_preserves_ordered_audit_cursor(): changed["tables"]["pc_access_audit"]["auto_increment_mode"] = "noorder" with pytest.raises(MigrationError, match="incompatible_schema"): schema.identify(changed) + + +def test_unregistered_processing_intent_blocks_before_any_upgrade(source): + with closing(sqlite3.connect(source.database)) as connection, connection: + connection.execute( + "INSERT INTO pc_artifact_processing_intents (binding_name,scope_id,pending_sequence,dirty_generation,clean_generation,requested_generation,handled_generation,last_auto_scan_generation) VALUES ('unknown-handler','scope',1,0,0,1,0,0)" + ) + before = dump(source.database) + with pytest.raises(MigrationError, match="unsupported_task_format"): + source.plan() + assert dump(source.database) == before + assert not source.evidence_path.exists() From 29f0512734e94b8b3ce7583cfe2b81b1dd3a4cc6 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 19:22:16 +0800 Subject: [PATCH 39/94] fix(migrations): preserve Server resource checksums across checkouts Pin full Server migration resources to LF, matching the other immutable bundles. Windows autocrlf otherwise changes raw resource hashes and accepted plan identity. Constraint: Frozen Git resource bytes remain unchanged. Scope-risk: Checkout line endings only. Tested: All 21 canonical resource payloads match their Git objects; git diff --check; git check-attr eol. --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index f934560f8e..290931d115 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,4 +9,5 @@ integrations/**/skills/powercontext-project-context/** text eol=lf # Migration evidence hashes exact resource bytes on every platform. src/powercontext/builtin/persistence/migrations/resources/** text eol=lf src/powercontext/builtin/persistence/migrations/mysql_resources/** text eol=lf +src/powercontext/builtin/persistence/migrations/server_resources/** text eol=lf src/powercontext/builtin/persistence/migration_resources/** text eol=lf From 0a89ec706db5f016c3efa698ca3fbe831f549f11 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 22 Oct 2026 19:40:03 +0800 Subject: [PATCH 40/94] test(upgrades): shut down embedded servers gracefully --- tests/release_upgrades/test_full_server.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/release_upgrades/test_full_server.py b/tests/release_upgrades/test_full_server.py index 77e0409003..10b8effb81 100644 --- a/tests/release_upgrades/test_full_server.py +++ b/tests/release_upgrades/test_full_server.py @@ -98,10 +98,9 @@ def server(python, environment, work, log): yield client finally: if process.poll() is None: - if os.name == "nt": - process.terminate() - else: - os.killpg(process.pid, signal.SIGTERM) + # Let the Server close SQL sessions and its embedded engine in + # order. Signalling the whole group also kills seekDB mid-close. + process.terminate() try: process.wait(timeout=30) except subprocess.TimeoutExpired: From 72b16a7711884a0d4421ea25eb25ccadc534fbb4 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 19:44:40 +0800 Subject: [PATCH 41/94] test(migrations): align startup guards with full Server readiness Expect the unified migration_required rejection for an incomplete prototype revision, preserving the assertion that startup leaves the database unchanged. Update the bilingual Dream workflow and Processing plan explanations to match the registered full Server bundle. Constraint: Startup still rejects partial managed schemas without repair. Scope-risk: Test expectation and documentation only. Tested: Linux Candidate/full Server suites: 50 passed; full Ty/Python 3.11; applicable pre-commit hooks. --- docs/en/docs/operate/database-migrations.md | 3 ++- docs/en/docs/workflows/review-dream-revisions.md | 2 +- docs/zh/docs/operate/database-migrations.md | 2 +- docs/zh/docs/workflows/review-dream-revisions.md | 2 +- tests/e2e/test_candidate_schema_upgrade.py | 3 ++- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 6d3404cb58..32c37ff6b2 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -105,7 +105,8 @@ categories and a nonzero exit status. Configuration binding includes the target's canonical Processing ownership mode, capabilities and declared automatic bindings. Changing those declarations requires a new plan; an accepted plan cannot be applied under a different -Processing declaration. This binding does not validate or migrate the historical Processing state. +Processing declaration. Plan binding alone does not validate or migrate historical Processing state; the full Server +bundle separately validates the stored declaration and pending intents through its registered domain steps. Within the supported acceptance scope, interactive users can invoke `apply` directly. PC-managed backup is the default. The command displays the plan, backup method, maintenance impact, and stopped-write responsibility, then asks for one diff --git a/docs/en/docs/workflows/review-dream-revisions.md b/docs/en/docs/workflows/review-dream-revisions.md index e3e0ebae39..de9cf7fb17 100644 --- a/docs/en/docs/workflows/review-dream-revisions.md +++ b/docs/en/docs/workflows/review-dream-revisions.md @@ -21,7 +21,7 @@ The unified Candidate routes and physical table rename ship together, including Before stopping all writers, stop new Dream admissions and let the old runtime finish every queued/running Dream run. Migration rejects unfinished runs with `dream_tasks_require_drain`; it does not change frozen prompts, model configuration, retry budgets or deadlines. Completed runs and pending Candidates remain readable and reviewable after upgrade. The migration also updates Handoff/Prompt processing bindings explicitly; ordinary startup does not repair the processing manifest. An unsupported manifest blocks migration with `unknown_processing_manifest`. -Production upgrade is not enabled by the Dream contribution alone. [PR #1772](https://github.com/oceanbase/powercontext/pull/1772) must register the complete Server baseline, revision chain, legacy-task checks and backend acceptance. Its four-table SQLite prototype does not establish Server readiness. This branch rejects legacy Dream storage with `migration_required` and versioned databases with `migration_framework_not_ready` until full framework readiness is integrated; do not bypass these errors by stamping a revision or creating empty replacement tables. Registered retained tables and recovery objects remain under the framework's retention policy. +The [full Server migration bundle](../operate/database-migrations.md) registers the complete 1.3.0 baseline, revision chain and legacy-task checks for the no-vector, Code-disabled profile. Stop writers and drain queued or running legacy Dream tasks before maintenance. Startup validates the complete managed database and rejects incomplete revisions with `migration_required`; stamping a revision or creating empty replacement tables cannot establish readiness. The partial Artifact bundle still reports `server_ready=false`. Consult the installation guide for the release's verified upgrade support. Registered retained tables and recovery objects remain under the framework's retention policy. ## Prompt configuration diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 19b4bce250..9bb7b13c18 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -82,7 +82,7 @@ powercontext server db-migrate verify --env-file deployment.env 计划绑定目标身份、源／目标 revision、实际结构摘要、冻结脚本摘要、配置摘要、备份策略、服务范围和迁移协调范围。配置与凭据值不输出到诊断中；预期错误返回稳定 JSON 类别和非零退出码。 -配置绑定包含目标部署的规范 Processing 归属模式、能力和自动绑定声明。改变这些声明后必须重新生成计划，不能使用旧计划执行。此绑定本身不验证或迁移历史 Processing 状态。 +配置绑定包含目标部署的规范 Processing 归属模式、能力和自动绑定声明。改变这些声明后必须重新生成计划，不能使用旧计划执行。计划绑定本身不验证或迁移历史 Processing 状态；完整 Server bundle 另由已登记的领域步骤校验持久化声明与待处理 intent。 在支持范围内，终端用户可直接执行 `apply`，默认 PC 自动备份。命令展示计划、备份方法、维护影响及停写责任，确认一次后执行，不逐条 SQL 询问： diff --git a/docs/zh/docs/workflows/review-dream-revisions.md b/docs/zh/docs/workflows/review-dream-revisions.md index f110875db5..6b641691b7 100644 --- a/docs/zh/docs/workflows/review-dream-revisions.md +++ b/docs/zh/docs/workflows/review-dream-revisions.md @@ -21,7 +21,7 @@ Handoff 批准即发布新版本，后续显式 Continue 按既有 latest 规则 停止全部写入前，先停止新增 Dream 请求，并由旧程序将 queued/running Dream Run 处理到终态。未完成任务会使迁移返回 `dream_tasks_require_drain`；迁移不改写冻结的提示词、模型配置、重试预算或截止时间。已完成 Run 和 pending Candidate 保留，升级后可继续读取与审核。Handoff/Prompt 的 Processing 绑定也由显式迁移更新，普通启动不修补 manifest；不支持的 manifest 返回 `unknown_processing_manifest` 并阻止迁移。 -Dream 提供的变更资源本身不启用生产升级。[PR #1772](https://github.com/oceanbase/powercontext/pull/1772) 还需完成完整 Server 基线、revision 链登记、历史任务检查与后端验收，其四表 SQLite 原型不代表 Server 就绪。在完整框架就绪检查接入前，本分支对旧 Dream 结构返回 `migration_required`，对已登记 revision 的数据库返回 `migration_framework_not_ready`；不能通过手工 stamp 或创建空替代表绕过。已登记旧表与恢复对象按统一框架的保留策略管理。 +[完整 Server 迁移 bundle](../operate/database-migrations.md) 为无向量、Code 关闭的配置登记了完整 1.3.0 基线、revision 链和历史任务检查。维护前须停止写入，并排空 queued 或 running 的旧 Dream 任务。启动时校验完整受管数据库，对未完成的 revision 返回 `migration_required`；手工 stamp 或创建空替代表不能建立就绪状态。局部 Artifact bundle 仍返回 `server_ready=false`。具体版本已验证的升级支持范围以安装指南为准。已登记旧表与恢复对象按统一框架的保留策略管理。 ## Prompt 配置 diff --git a/tests/e2e/test_candidate_schema_upgrade.py b/tests/e2e/test_candidate_schema_upgrade.py index a735d9cd6f..a1d2fe822f 100644 --- a/tests/e2e/test_candidate_schema_upgrade.py +++ b/tests/e2e/test_candidate_schema_upgrade.py @@ -29,6 +29,7 @@ upgrade_dream_storage, validate_dream_processing_manifest, ) +from powercontext.builtin.persistence.migrations import MigrationError from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.persistence.tables import PROFILE_POLICIES_TABLE from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts @@ -415,7 +416,7 @@ async def scenario(): connection.execute("CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL PRIMARY KEY)") connection.execute("INSERT INTO pc_schema_revision VALUES ('p0003')") before = snapshot(path) - with pytest.raises(DreamSchemaNotReadyError, match="migration_framework_not_ready"): + with pytest.raises(MigrationError, match="migration_required"): async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{path}"))): pytest.fail("A version-table marker cannot establish Server readiness") assert snapshot(path) == before From 69fd6ee1ba0ae3ead209b3de0080a5c77fe072e7 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 20:10:12 +0800 Subject: [PATCH 42/94] test(migrations): exercise installed background workers after upgrade Require real installed Memory Worker execution, usage reporting, exact Source citations and FTS retrieval in each release upgrade case. Constraint: controlled local inference; no vector or real-model quality claim. Tested: SQLite installed-release suite 11 passed; gate/publication regressions 65 passed; applicable hooks and Python 3.11 Ty passed. --- scripts/upgrade_release_gate.py | 1 + tests/release_upgrades/README.md | 11 +- tests/release_upgrades/test_full_server.py | 169 ++++++++++++++++++++- 3 files changed, 173 insertions(+), 8 deletions(-) diff --git a/scripts/upgrade_release_gate.py b/scripts/upgrade_release_gate.py index acd98f99b4..c97812ae48 100644 --- a/scripts/upgrade_release_gate.py +++ b/scripts/upgrade_release_gate.py @@ -50,6 +50,7 @@ "test_read_write_and_retrieval", "test_authorization", "test_candidates_and_processing", + "test_background_processing", "test_preserved_history_and_cursors", "test_fresh_upgrade_equivalence", }) diff --git a/tests/release_upgrades/README.md b/tests/release_upgrades/README.md index 8929310598..71caf38861 100644 --- a/tests/release_upgrades/README.md +++ b/tests/release_upgrades/README.md @@ -63,7 +63,8 @@ Provide these behavioral tests (additional tests are allowed): | `test_target_startup` | Start the target Server against the migrated database and verify readiness. | | `test_read_write_and_retrieval` | Read the old records, write new ones, and retrieve both through public interfaces. | | `test_authorization` | Verify permitted and denied access and preservation of relevant grants. | -| `test_candidates_and_processing` | Verify retained Candidates and CAS approval; load and acknowledge the old pending intent using the target installed SDK, including idempotency. This does not execute a model-driven Worker. | +| `test_candidates_and_processing` | Verify retained Candidates and CAS approval; load and acknowledge the old pending intent using the target installed SDK, including idempotency. | +| `test_background_processing` | Run the installed target's actual Memory Worker against a local controlled inference endpoint; capture a new Source, verify the published Memory's exact Source reference and FTS retrieval, and require the child Worker's usage report. | | `test_preserved_history_and_cursors` | Verify exact history/references and unchanged or declared-transformed cursor and receipt semantics. | | `test_fresh_upgrade_equivalence` | Initialize a fresh target database; compare schema/backend objects and behavior with the upgraded database. The current suite has no retained/archive object exclusions. | @@ -80,9 +81,11 @@ records installed Python/package versions, target wheel SHA-256, policy SHA-256, Evidence is created during the gate run in a new directory. There is no option to import old successful reports. Test names and metadata do not themselves prove correctness: reviewers must inspect the assertions and confirm -they follow actual installed operations and compare persisted data. The profile exercises explicit writes and FTS, -not vector storage or model-generated content. The processing fixture tests persisted intent compatibility and -acknowledgment, not Worker execution. Add separately reviewed profiles before claiming those optional paths. +they follow actual installed operations and compare persisted data. The profile exercises explicit writes, FTS and +actual background Memory processing with controlled model responses. Both installed releases declare the Memory +capability and use the same processing ownership; the Worker check changes only the execution interval after migration. +The separate SDK fixture tests the old pending intent's format and acknowledgment. Controlled responses establish +execution and evidence preservation, not real-model quality. Vector storage needs separately reviewed profiles. SQLite inventories include indexes, views and triggers; MySQL-family table DDL includes indexes and constraints, and views are collected separately. MySQL comparison parses columns, indexes (including parser properties), constraints, and CHECK expressions, ignoring only physical formatting and backend-generated constraint names. Views and triggers are collected independently. SQLite normalizes whitespace and identifier quoting. Structural differences fail rather than being silently excluded. diff --git a/tests/release_upgrades/test_full_server.py b/tests/release_upgrades/test_full_server.py index 10b8effb81..12f1aa5138 100644 --- a/tests/release_upgrades/test_full_server.py +++ b/tests/release_upgrades/test_full_server.py @@ -24,11 +24,14 @@ import json import os import re +import shlex import signal import socket import subprocess +import threading import time from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from urllib.parse import urlsplit @@ -37,6 +40,99 @@ ROOT = Path(__file__).resolve().parents[2] TOKEN = "isolated-upgrade-acceptance-token" # noqa: S105 -- disposable fixture, never a deployment credential +AUTOMATIC_MEMORY = "Upgrade fixture background processing retains source evidence." + + +def objects(value): + if isinstance(value, dict): + yield value + for child in value.values(): + yield from objects(child) + elif isinstance(value, list): + for child in value: + yield from objects(child) + elif isinstance(value, str): + try: + decoded = json.loads(value) + except ValueError: + try: + decoded, _ = json.JSONDecoder().raw_decode(value[value.index("{") :]) + except ValueError: + return + if isinstance(decoded, (dict, list)): + yield from objects(decoded) + + +@pytest.fixture(scope="session") +def model_gateway(): + stages = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, format, *args): # noqa: A002 -- stdlib signature + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + payload = next(obj for obj in objects(body["messages"]) if "evidence" in obj) + if "proposal" in payload: + stages.append("reconcile") + proposal = payload["proposal"] + duplicate = next((item for item in payload["related"] if item["text"] == proposal["text"]), None) + value = { + "action": "noop" if duplicate else "create", + "compared_ids": [item["item_id"] for item in payload["related"]], + "target_ids": [duplicate["item_id"]] if duplicate else [], + "content": None if duplicate else {"kind": proposal["kind"], "text": proposal["text"]}, + "evidence_ids": [] if duplicate else proposal["evidence_ids"], + "reason": "Controlled upgrade acceptance", + } + else: + stages.append("extract") + value = { + "candidates": [ + {"kind": "fact", "text": AUTOMATIC_MEMORY, "evidence_ids": [item["evidence_id"]]} + for item in payload["evidence"] + if AUTOMATIC_MEMORY in json.dumps(item["content"]) + ] + } + output = json.dumps(value) + tools = body.get("tools", []) + message = {"role": "assistant", "content": output} + if tools: + message = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "upgrade-generation", + "type": "function", + "function": {"name": tools[0]["function"]["name"], "arguments": output}, + } + ], + } + response = json.dumps({ + "id": "upgrade-fixture", + "object": "chat.completion", + "created": 1, + "model": "upgrade-fixture", + "choices": [{"index": 0, "message": message, "finish_reason": "tool_calls" if tools else "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 10, "total_tokens": 20}, + }).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(response))) + self.end_headers() + self.wfile.write(response) + + gateway = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=gateway.serve_forever, daemon=True) + thread.start() + try: + yield {"url": f"http://127.0.0.1:{gateway.server_port}/v1", "stages": stages} + finally: + gateway.shutdown() + gateway.server_close() + thread.join(timeout=5) def digest(value): @@ -124,6 +220,11 @@ def environment_for(case, *, fresh=False): "POWERCONTEXT_SERVER_ACCESS_MODE": "enforced", "POWERCONTEXT_SERVER_CURSOR_SIGNING_SECRET": "isolated-upgrade-cursor-signing-key-0001", "POWERCONTEXT_SERVER_RUNTIME_ARTIFACT_PROCESSING_ROLE": "all", + "POWERCONTEXT_SERVER_RUNTIME_ARTIFACT_PROCESSING_FAMILIES": '["memory"]', + "POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS": "3600", + "POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL": "openai-chat:upgrade-fixture", + "POWERCONTEXT_SERVER_INFERENCE_GENERATION_BASE_URL": case["model"]["url"], + "OPENAI_API_KEY": "isolated-upgrade-fixture", "POWERCONTEXT_SERVER_MCP_ENABLED": "false", "NO_PROXY": "localhost,127.0.0.1", "no_proxy": "localhost,127.0.0.1", @@ -270,7 +371,7 @@ def seed(client, *, candidate_prefix): @pytest.fixture(scope="session") -def case(): +def case(model_gateway): # Missing gate inputs are errors, never skips or an implicit local database. assert os.environ["PC_UPGRADE_SOURCE_VERSION"] == "1.3.0", "Add a reviewed adapter for another source release" return { @@ -279,6 +380,7 @@ def case(): "backend": os.environ["PC_UPGRADE_BACKEND"], "work": Path(os.environ["PC_UPGRADE_WORK"]), "report": Path(os.environ["PC_UPGRADE_REPORT_DIR"]), + "model": model_gateway, } @@ -319,7 +421,9 @@ def migrated(case, historical, rejected_legacy_startup): environment = environment_for(case) env_file = case["work"] / "deployment.env" env_file.write_text( - "".join(f"{key}={value}\n" for key, value in environment.items() if key.startswith("POWERCONTEXT_")) + "".join( + f"{key}={shlex.quote(value)}\n" for key, value in environment.items() if key.startswith("POWERCONTEXT_") + ) ) common = ["--env-file", str(env_file)] if case["backend"] == "oceanbase": @@ -392,6 +496,56 @@ def target(case, migrated): yield client +@pytest.fixture(scope="session") +def processed_background(case, migrated): + historical = migrated["historical"] + environment = environment_for(case) + environment["POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS"] = "0.05" + source_id = "upgrade-background-worker" + stages = case["model"]["stages"] + start = len(stages) + with server(case["target"], environment, case["work"], case["report"] / "worker-server.log") as client: + post( + client, + "/v1/sources/content", + {"scope_id": historical["scope"], "source_id": source_id, "content": AUTOMATIC_MEMORY}, + ) + deadline = time.monotonic() + 90 + while True: + entries = post(client, "/v1/memory/entries/list", {"scope_id": historical["scope"]})["entries"] + generated = next((item for item in entries if item["text"] == AUTOMATIC_MEMORY), None) + metrics = client.get("/metrics") + metrics.raise_for_status() + report = re.search( + r'^powercontext_server_model_usage_worker_reports_total\{outcome="received"\} (\S+)', + metrics.text, + re.MULTILINE, + ) + if generated and report and float(report[1]) > 0: + break + assert time.monotonic() < deadline, "The installed target Worker did not publish and report completion" + time.sleep(0.2) + artifact = generated["artifact"] + revision = client.get( + f"/v1/scopes/{historical['scope']}/artifacts/{artifact['family']}/" + f"{artifact['artifact_id']}/revisions/{artifact['revision']}" + ) + revision.raise_for_status() + assert source_id in {item["source_id"] for item in revision.json()["sources"]} + assert {"extract", "reconcile"} <= set(stages[start:]) + found = post( + client, + "/v1/memory/search", + {"scope_id": historical["scope"], "query": "background processing", "mode": "fts"}, + )["hits"] + assert AUTOMATIC_MEMORY in {item["memory"]["text"] for item in found} + return { + "model_stages": sorted(set(stages[start:])), + "worker_report_received": True, + "source_evidence_verified": True, + } + + def test_source_release_database(historical): assert historical["memory"] and historical["candidate"]["version"] == 2 assert historical["task"]["requested_generation"] > historical["task"]["handled_generation"] @@ -458,6 +612,10 @@ def test_candidates_and_processing(case, migrated): assert stale.status_code == 409, "A stale revision must not publish a duplicate Artifact" +def test_background_processing(processed_background): + assert processed_background["worker_report_received"] and processed_background["source_evidence_verified"] + + def test_preserved_history_and_cursors(target, historical): expected = historical["continuation"] page = target.get("/v1/scopes", params={"limit": 1, "cursor": historical["cursor"]}) @@ -507,12 +665,14 @@ def normalize(sql): } -def test_fresh_upgrade_equivalence(case, migrated): +def test_fresh_upgrade_equivalence(case, migrated, processed_background): if case["backend"] == "oceanbase": environment = environment_for(case, fresh=True) env_file = case["work"] / "fresh.env" env_file.write_text( - "".join(f"{key}={value}\n" for key, value in environment.items() if key.startswith("POWERCONTEXT_")) + "".join( + f"{key}={shlex.quote(value)}\n" for key, value in environment.items() if key.startswith("POWERCONTEXT_") + ) ) common = [ "--env-file", @@ -570,6 +730,7 @@ def test_fresh_upgrade_equivalence(case, migrated): "source_schema": digest(migrated["historical"]["schema"]), "target_schema": digest(actual), "fresh_schema": digest(expected), + "background_processing": processed_background, } if case["backend"] == "oceanbase": evidence.update(tenant_mode=inventory["tenant_mode"], proxy_version=inventory["proxy_version"]) From c615a901276f6b93f720fa74e809ef2f956f2a53 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 20:54:57 +0800 Subject: [PATCH 43/94] fix(migrations): allow worker takeover after Server upgrade Expire the revoked migration term so OceanBase can acquire a new timed lease. Initialize the complete Desktop Server fixture before opening its separate Access context. Constraint: old workers remain fenced; no runtime lease rules or unknown-schema admission is weakened. Tested: 134 passed, 2 backend skips; regression fails before fix; Windows provider startup passes; applicable hooks, gate/publication regressions and full Python 3.11 Ty pass. --- desktop/tests/real_server.py | 3 +++ release/upgrade-support.json | 2 +- .../server_resources/schemas/mysql-s1400.json | 2 +- .../versions/s1400_full_server.py | 5 ++-- .../test_full_server_migrations.py | 23 +++++++++++++++++++ 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/desktop/tests/real_server.py b/desktop/tests/real_server.py index fb2808e591..5ed528fcf2 100644 --- a/desktop/tests/real_server.py +++ b/desktop/tests/real_server.py @@ -196,12 +196,15 @@ async def run(): if not config.get("provider"): await server.serve() return + from powercontext.builtin.persistence.migrations.server_readiness import prepare_server_storage from powercontext.server.authz import PrincipalRef from powercontext.server.authz.composition import open_builtin_access_control from powercontext.server.settings import AccessControlConfig admin = PrincipalRef(type="service", id="desktop-fixture-admin") + # Bootstrap the complete Server before the separate Access context opens it. + await asyncio.to_thread(prepare_server_storage, settings) async with open_builtin_access_control( settings.database, bootstrap_administrators=(admin,), deployment_id="desktop-fixture" ) as access: diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 5003b93785..3a619cd6ec 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,7 +3,7 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "7eb8a861925bbb8fc9080d97e9bae83d772280d786d9a05d40d6bec8e7c9e734", + "persistence_sha256": "a7b708fac60ffc4debdde16d52de2b0e882ccb719c22d7964fc0c8197e160b52", "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history and pending intents with projection rebuilds and read-only startup admission. Publication requires installed-release upgrade, repeat-apply, old-startup rejection and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles." }, "cases": [], diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1400.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1400.json index e298f57acb..91b7920c33 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1400.json +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1400.json @@ -43,7 +43,7 @@ "CREATE TABLE `pc_context_bootstrap_receipts` (\n `receipt_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `scope_id` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `event_key` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `request_digest` varchar(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `integration` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `lifecycle` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `profile` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `state` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `reason` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `max_bytes` int(11) NOT NULL,\n `package_digest` varchar(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `content_bytes` int(11) NOT NULL,\n `truncated` tinyint(1) NOT NULL,\n `items` mediumblob NOT NULL,\n `created_at` datetime NOT NULL,\n PRIMARY KEY (`receipt_id`),\n FOREIGN KEY (`scope_id`) REFERENCES `pc_scopes`(`scope_id`) ON UPDATE RESTRICT ON DELETE CASCADE ,\n UNIQUE KEY `uq_pc_context_bootstrap_receipts_event` (`event_key`) BLOCK_SIZE 16384 LOCAL,\n KEY `ix_pc_context_bootstrap_receipts_created_at` (`created_at`) BLOCK_SIZE 16384 LOCAL,\n CONSTRAINT `ck_pc_context_bootstrap_receipts_state` CHECK ((`state` in ('pending','injected','skipped','failed'))),\n CONSTRAINT `ck_pc_context_bootstrap_receipts_budget` CHECK (((`max_bytes` >= 512) and (`max_bytes` <= 8192))),\n CONSTRAINT `ck_pc_context_bootstrap_receipts_content_bytes` CHECK ((`content_bytes` >= 0))\n) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0", "CREATE TABLE `pc_lexical_projection_versions` (\n `projection` varchar(128) NOT NULL,\n `analyzer_version` varchar(64) NOT NULL,\n PRIMARY KEY (`projection`)\n) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0", "CREATE TABLE `pc_topic_memory_work_blocks` (\n `block_id` varchar(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `attempt_id` varchar(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `scope_id` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `binding_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `source_after` bigint(20) NOT NULL,\n `source_through` bigint(20) DEFAULT NULL,\n `budget_source_through` bigint(20) DEFAULT NULL,\n `legacy_source_through` bigint(20) DEFAULT NULL,\n `target_source_through` bigint(20) NOT NULL,\n `attempts` int(11) NOT NULL,\n `requests` bigint(20) NOT NULL,\n `tokens` bigint(20) NOT NULL,\n `last_error_code` varchar(64) DEFAULT NULL,\n `error_code` varchar(64) NOT NULL,\n `blocked_at` datetime NOT NULL,\n `retried_at` datetime DEFAULT NULL,\n `retry_request_generation` bigint(20) DEFAULT NULL,\n PRIMARY KEY (`block_id`),\n UNIQUE KEY `uq_pc_topic_block_identity` (`scope_id`, `binding_name`, `source_after`, `attempt_id`) BLOCK_SIZE 16384 LOCAL,\n CONSTRAINT `ck_pc_topic_block_window` CHECK (((`source_after` >= 0) and (`target_source_through` > `source_after`) and ((`source_through` is null) or ((`source_through` > `source_after`) and (`source_through` <= `target_source_through`))) and ((`budget_source_through` is null) or ((`budget_source_through` > `source_after`) and (`budget_source_through` <= `target_source_through`))) and ((`source_through` is null) or (`budget_source_through` is null) or (`source_through` <= `budget_source_through`)) and ((`legacy_source_through` is null) or ((`legacy_source_through` > `source_after`) and (`legacy_source_through` <= `target_source_through`))))),\n CONSTRAINT `ck_pc_topic_block_usage` CHECK (((`attempts` > 0) and (`requests` >= 0) and (`tokens` >= 0))),\n CONSTRAINT `ck_pc_topic_block_acceptance` CHECK ((((`retried_at` is null) and (`retry_request_generation` is null)) or ((`retried_at` is not null) and (`retry_request_generation` is not null) and (`retry_request_generation` > 0))))\n) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0", - "UPDATE pc_artifact_processing_leases SET supervisor_generation=supervisor_generation+1, holder_id='migration-s1400', lease_expires_at=NULL WHERE holder_id <> 'migration-s1400' OR lease_expires_at IS NOT NULL" + "UPDATE pc_artifact_processing_leases SET supervisor_generation=supervisor_generation+1, holder_id='migration-s1400', lease_expires_at='1970-01-01 00:00:00' WHERE holder_id <> 'migration-s1400' OR lease_expires_at IS NULL OR lease_expires_at <> '1970-01-01 00:00:00'" ], "changes": [ { diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/versions/s1400_full_server.py b/src/powercontext/builtin/persistence/migrations/server_resources/versions/s1400_full_server.py index 1d05effb34..4526ca6191 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/versions/s1400_full_server.py +++ b/src/powercontext/builtin/persistence/migrations/server_resources/versions/s1400_full_server.py @@ -67,7 +67,8 @@ def upgrade(): for kind, _name, owner, ddl in target: if kind == "index" and owner in rebuild | added: connection.exec_driver_sql(ddl) - # A stopped worker's old fence must not be valid in the target protocol. + # Revoke the old fence and leave an expired term for timed-lease takeover. + # NULL denotes a non-expiring single-process term and blocks OceanBase. connection.exec_driver_sql( - "UPDATE pc_artifact_processing_leases SET supervisor_generation=supervisor_generation+1, holder_id='migration-s1400', lease_expires_at=NULL" + "UPDATE pc_artifact_processing_leases SET supervisor_generation=supervisor_generation+1, holder_id='migration-s1400', lease_expires_at='1970-01-01 00:00:00.000000'" ) diff --git a/tests/builtin/persistence/test_full_server_migrations.py b/tests/builtin/persistence/test_full_server_migrations.py index f88c31abc0..e0371584e8 100644 --- a/tests/builtin/persistence/test_full_server_migrations.py +++ b/tests/builtin/persistence/test_full_server_migrations.py @@ -14,6 +14,7 @@ """Complete frozen Server upgrades, rollback, recovery and admission behavior.""" +import asyncio import hashlib import json import sqlite3 @@ -28,6 +29,8 @@ from powercontext.builtin.persistence.migrations.deployment import server_bundle from powercontext.builtin.persistence.migrations.models import MigrationError from powercontext.builtin.persistence.migrations.sqlite import SQLiteMigrationRunner +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.supervision import ArtifactProcessingLeaseRepository from powercontext.builtin.runtime.config import BuiltinConfig from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest @@ -86,6 +89,26 @@ def test_complete_upgrade_preserves_history_backup_and_repeated_apply(source): assert dump(source.database) == after +def test_migration_revokes_old_fence_and_allows_expiring_worker_lease(source): + with closing(sqlite3.connect(source.database)) as connection, connection: + connection.execute( + "INSERT INTO pc_artifact_processing_leases (supervisor_group,holder_id,supervisor_generation,lease_expires_at) VALUES ('global','old-worker',7,NULL)" + ) + apply(source) + + async def scenario(): + repository = ArtifactProcessingLeaseRepository() + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{source.database}") + async with SQLiteProfile.open(config, tables=()) as profile: + async with profile.database.transaction() as connection: + acquired = await repository.try_acquire(connection, "new-worker", 15) + assert acquired is not None, "The migration fence must allow an OceanBase-style lease takeover" + assert acquired.supervisor_generation > 7 + assert acquired.lease_expires_at is not None + + asyncio.run(scenario()) + + def test_domain_failure_rolls_back_and_reuses_original_recovery_point(source, monkeypatch): original = source.bundle.domains._invoke From 3c3e86651caf675f9668e5500b02fd60bf730a97 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 22 Oct 2026 20:59:21 +0800 Subject: [PATCH 44/94] refactor(migrations): separate Source evidence feature from upgrade framework --- docs/en/docs/operate/database-migrations.md | 8 +- docs/zh/docs/operate/database-migrations.md | 8 +- .../powercontext_connector_opendal/source.py | 2 - release/upgrade-support.json | 4 +- src/powercontext/__init__.py | 4 - .../artifacts/atomic_memory/service.py | 33 +- .../persistence/atomic_memory_evidence.py | 211 ------ .../atomic_memory_evidence_schema.py | 59 -- .../persistence/atomic_memory_index.py | 52 +- .../persistence/atomic_memory_readiness.py | 12 +- .../topic_memory_recovery/schema.json | 18 - .../migrations/atomic_memory_evidence.py | 130 ---- .../versions/p0005_atomic_memory_evidence.py | 25 - .../migrations/resources/manifest.json | 17 +- .../migrations/resources/mysql_layouts.json | 5 - .../schemas/atomic_memory_evidence_v1.json | 14 - .../versions/p0005_atomic_memory_evidence.py | 30 - .../server_resources/evidence_domain.py | 34 - .../migrations/server_resources/manifest.json | 34 +- .../server_resources/mysql/versions/s1401.py | 24 - .../schemas/atomic_memory_evidence_v1.json | 14 - .../server_resources/schemas/mysql-s1401.json | 259 -------- .../versions/s1401_source_evidence.py | 32 - .../builtin/persistence/schema.py | 10 +- .../builtin/persistence/source_definitions.py | 36 +- .../builtin/persistence/sources.py | 612 +----------------- .../builtin/runtime/atomic_memory.py | 22 - .../builtin/runtime/atomic_memory_rebuild.py | 58 +- .../builtin/runtime/composition.py | 14 +- src/powercontext/sources/__init__.py | 3 - src/powercontext/sources/definitions.py | 63 +- src/powercontext/sources/evidence.py | 60 -- src/powercontext/sources/models.py | 2 - src/powercontext/sources/observations.py | 85 +-- .../test_full_server_migrations.py | 3 +- .../test_mysql_migration_runner.py | 22 +- .../test_mysql_migration_schema.py | 18 +- tests/builtin/persistence/test_sources.py | 565 +--------------- ...t_atomic_memory_evidence_neutral_recall.py | 131 ---- .../test_atomic_memory_projection_rebuild.py | 17 +- .../e2e/test_atomic_memory_source_evidence.py | 416 ------------ tests/e2e/test_observability.py | 9 +- tests/test_source_observations.py | 355 +--------- 43 files changed, 69 insertions(+), 3461 deletions(-) delete mode 100644 src/powercontext/builtin/persistence/atomic_memory_evidence.py delete mode 100644 src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py delete mode 100644 src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py delete mode 100644 src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0005_atomic_memory_evidence.py delete mode 100644 src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json delete mode 100644 src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_memory_evidence.py delete mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/evidence_domain.py delete mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/s1401.py delete mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/schemas/atomic_memory_evidence_v1.json delete mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1401.json delete mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/versions/s1401_source_evidence.py delete mode 100644 src/powercontext/sources/evidence.py delete mode 100644 tests/e2e/test_atomic_memory_evidence_neutral_recall.py delete mode 100644 tests/e2e/test_atomic_memory_source_evidence.py diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 32c37ff6b2..da2a65fed1 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -14,13 +14,13 @@ The design follows [RFC #1771](https://github.com/oceanbase/powercontext/pull/17 `powercontext server db-migrate status/plan/apply/verify` default to `--bundle server` and use the configured persistent SQLite, embedded seekDB, or OceanBase MySQL database. Explicit environment files are authoritative; arbitrary bundle paths are not accepted. -The full Server path registers the actual 1.3.0 no-vector, Code-disabled configuration as `s1300 → s1400 → s1401`. It covers the complete schema, Candidate renaming and retained history, Dream task formats, Processing ownership, full-text projections, and Source evidence. Unknown layouts, extra objects, vector/Code configurations, and unregistered sources fail before writes. Older Memory collection conversion is outside this path. +The full Server path registers the actual 1.3.0 no-vector, Code-disabled configuration as `s1300 → s1400`. It covers the complete schema, Candidate renaming and retained history, Dream task formats, Processing ownership, full-text projections, and existing Source references. Unknown layouts, extra objects, vector/Code configurations, and unregistered sources fail before writes. Older Memory collection conversion is outside this path. Only successful schema, domain, projection, and task validation returns `state=ready`, `readiness_scope=server`, and `server_ready=true`. Publication also requires installed-artifact acceptance on all three backends. An implemented path is not evidence that a release passed acceptance; consult the installation guide and release gate for support status. Stop all writers and drain queued/running 1.3.0 Dream tasks before upgrading: the target cannot execute the old prompt contract. Terminal task payloads and pending Processing counters are preserved; old worker leases are fenced. Registered capability additions retain the old automatic-processing choices instead of replacing ownership with the current configuration. -Old Candidate tables are retained as `pc_retained_dream_candidate_heads` and `pc_retained_dream_candidate_versions` for migration-copy verification. They and the backup are not live write targets; subsequent business changes are not compared against old snapshots. New historical Artifact provenance fields remain unknown, and Source evidence is not inferred from old references. Projection rebuilds and verification scan their tables; budget downtime and capacity for backups and retained copies. +Old Candidate tables are retained as `pc_retained_dream_candidate_heads` and `pc_retained_dream_candidate_versions` for migration-copy verification. They and the backup are not live write targets; subsequent business changes are not compared against old snapshots. New historical Artifact provenance fields remain unknown, and existing Source references are preserved. Projection rebuilds and verification scan their tables; budget downtime and capacity for backups and retained copies. Ordinary Server startup checks readiness without repairing an existing database. Empty local persistent SQLite/seekDB storage initializes through the same revision chain; OceanBase requires an explicit maintenance job first. In-memory SQLite remains transient runtime storage. A changed schema or inconsistent projection requires maintenance, not restart-based repair. @@ -351,7 +351,7 @@ seekdb/OceanBase DDL may commit implicitly, so one Python transaction cannot mak recognizes only the registered sequence: `p0001` creates the four frozen tables and two indexes; `p0002` adds the two nullable `MEDIUMBLOB` citation columns; `p0003` removes the tag-family CHECK. Actual schema, data checks, and persisted maintenance evidence must establish the known completed prefix before continuing. `p0004` adds exact-revision Artifact -handles; `p0005` adds Source evidence storage without attesting historical content. Alembic advances the version only +handles. Alembic advances the version only after the revision's postconditions pass. Unknown objects, layouts, or partial transitions block execution instead of being optimistically stamped. @@ -376,7 +376,7 @@ bundle for retry. Released revision resources and their recorded checksums remai Parameter-free DDL uses statement-scoped driver execution options so CHECK remainder operators and percent-containing SQL literals reach the backend unchanged. -Repositories run after schema checks. The full Server `s1300 → s1400 → s1401` chain registers exact per-statement schema transitions and recovery prefixes through `mysql_state_layouts`. Domain steps attach to the shared revision and use its connection, lock, and recovery point; verification precedes version advancement. Read-only live checks are separate from migration-copy validation. Vector configurations still require separate registration. +Repositories run after schema checks. The full Server `s1300 → s1400` chain registers exact per-statement schema transitions and recovery prefixes through `mysql_state_layouts`. Domain steps attach to the shared revision and use its connection, lock, and recovery point; verification precedes version advancement. Read-only live checks are separate from migration-copy validation. Vector configurations still require separate registration. One version table offers no generic run-ID resume. After interruption, inspect actual schema/data through read-only commands. Ambiguous state returns `recovery_required`; never blindly stamp or replace the original recovery point with a diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 9bb7b13c18..e89bf0ce91 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -12,13 +12,13 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 `powercontext server db-migrate status/plan/apply/verify` 默认选择 `--bundle server`，使用部署配置中的持久 SQLite、嵌入式 seekDB 或 OceanBase MySQL 模式数据库。维护命令以显式环境文件为准，不接受任意测试 bundle 路径。 -完整 Server 路径登记了真实 1.3.0 的无向量、Code 关闭配置，版本链为 `s1300 → s1400 → s1401`。它覆盖完整数据库结构、Candidate 改名与历史保留、Dream 任务格式、Processing 所有权声明、全文检索投影和 Source evidence。未知结构、额外对象、不同向量/Code 配置及未登记来源均在写入前拒绝。更早的 Memory 集合转换不属于此路径。 +完整 Server 路径登记了真实 1.3.0 的无向量、Code 关闭配置，版本链为 `s1300 → s1400`。它覆盖完整数据库结构、Candidate 改名与历史保留、Dream 任务格式、Processing 所有权声明、全文检索投影和既有 Source 引用。未知结构、额外对象、不同向量/Code 配置及未登记来源均在写入前拒绝。更早的 Memory 集合转换不属于此路径。 只有结构、领域数据、搜索投影和任务校验全部通过，才返回 `state=ready`、`readiness_scope=server`、`server_ready=true`。发布还必须通过真实安装包在三个后端上的完整验收；实现路径存在不等于某次发布已通过验收，支持状态以安装指南和发布门禁为准。 升级前必须停止所有写入者并排空 1.3.0 中 queued/running Dream 任务；新版不能执行旧提示词契约。终态任务保留原始 payload，待处理 Processing intent 的计数保留，旧 worker lease 被 fencing。迁移按登记规则扩展处理能力，保留原有自动处理选择，不根据当前配置覆盖旧所有权。 -Candidate 旧表保留为 `pc_retained_dream_candidate_heads` 和 `pc_retained_dream_candidate_versions`，用于迁移当次的复制验证。它们与迁移备份都不是线上写入目标；升级后的正常修改不再与旧快照比较。历史 Artifact 新增的来源时间/操作者字段保持未知，Source evidence 不凭历史引用虚构声明。投影重建和数据校验会扫描相关表；请为停机时间、备份与保留副本安排容量。 +Candidate 旧表保留为 `pc_retained_dream_candidate_heads` 和 `pc_retained_dream_candidate_versions`，用于迁移当次的复制验证。它们与迁移备份都不是线上写入目标；升级后的正常修改不再与旧快照比较。历史 Artifact 新增的来源时间/操作者字段保持未知，既有 Source 引用保持不变。投影重建和数据校验会扫描相关表；请为停机时间、备份与保留副本安排容量。 普通 Server 启动先做只读就绪检查，不修补已有库。空的本地持久 SQLite/seekDB 可通过同一版本链初始化；OceanBase 必须先执行显式维护命令初始化。内存 SQLite 仍为临时运行时存储。迁移后发现结构或投影被篡改，必须停写维护，不能靠重启修复。 @@ -230,7 +230,7 @@ powercontext server run --env-file deployment.env --role all 维护执行器复用配置，使用专用连接承载 Alembic；冻结历史 SQL 不依赖当前 Repository 或应用 metadata。seekdb 从启动引擎前到关闭引擎后持有规范化目录的锁。OceanBase 在每条 DDL 前后验证操作系统锁、主机绑定、持久运行记录及维护连接。正常结束时先关闭迁移主连接，才清理运行记录并释放文件锁；引擎随后关闭。执行中断或关闭连接失败均保留记录；后续执行需要准确的旧运行 token，以及 DBA 确认远端执行已结束的人工声明。PC 不从本机进程退出或本地锁可获取推断远端执行结束。锁文件被替换、连接丢失或旧运行未确认时阻止继续。平台不支持原生文件锁时拒绝执行，包括运行时降级为软锁的情况，不重连或静默弱化互斥继续迁移。这些锁不阻止业务写入者。 -seekdb／OceanBase DDL 可能隐式提交，不能用 Python 事务承诺整个迁移原子化。局部 Artifact 原型只识别已登记顺序：`p0001` 创建冻结四表及两个索引；`p0002` 添加两个可空的 `MEDIUMBLOB` 引用列；`p0003` 删除标签 family 的 CHECK；`p0004` 添加指向精确 revision 的 Artifact handle；`p0005` 添加 Source 证据存储，不追认历史内容的可信度。继续执行前须用实际结构、数据检查和持久维护证据证明已完成的已知前缀；revision 的后置条件通过后才推进 Alembic 版本。未知对象、结构或中间状态拒绝执行，不乐观 stamp。 +seekdb／OceanBase DDL 可能隐式提交，不能用 Python 事务承诺整个迁移原子化。局部 Artifact 原型只识别已登记顺序：`p0001` 创建冻结四表及两个索引；`p0002` 添加两个可空的 `MEDIUMBLOB` 引用列；`p0003` 删除标签 family 的 CHECK；`p0004` 添加指向精确 revision 的 Artifact handle。继续执行前须用实际结构、数据检查和持久维护证据证明已完成的已知前缀；revision 的后置条件通过后才推进 Alembic 版本。未知对象、结构或中间状态拒绝执行，不乐观 stamp。 MySQL 结构步骤通过冻结的布局片段注册，由 `manifest.json` 的 `mysql_layouts` 列表选择，合起来必须恰好覆盖 Alembic 版本链，且不能重复声明。新增 revision 追加不可改写的片段，并在 `revision_resources` 中声明其文件，不修改已发布片段。可选 checkpoint 将累积步骤与独立冻结的完整结构对照。反射、下一 revision 的已知前缀恢复和版本表校验使用同一注册链；新增 revision 无需修改识别器中的另一份版本列表。这些结构注册不能代替领域转换、兼容性检查或完整 Server readiness。 @@ -239,7 +239,7 @@ MySQL 结构步骤通过冻结的布局片段注册，由 `manifest.json` 的 `m 计划通过完整 bundle 摘要绑定注册资源，未完成维护的重试要求使用同一 bundle。已发布 revision 资源及其记录摘要保持不可改写。 无参数 DDL 使用单语句驱动选项，确保 CHECK 的取余运算符和 SQL 字面量中的百分号原样到达后端。 -Repository 仅在结构满足后运行。完整 Server 的 `s1300 → s1400 → s1401` 链用 `mysql_state_layouts` 登记逐语句的精确结构变化和可恢复前缀。领域步骤附着在共享 revision 上，使用同一连接、锁和恢复点；校验成功后才推进版本。只读 live 校验与迁移当次的保留副本校验分离。向量配置仍需独立登记。 +Repository 仅在结构满足后运行。完整 Server 的 `s1300 → s1400` 链用 `mysql_state_layouts` 登记逐语句的精确结构变化和可恢复前缀。领域步骤附着在共享 revision 上，使用同一连接、锁和恢复点；校验成功后才推进版本。只读 live 校验与迁移当次的保留副本校验分离。向量配置仍需独立登记。 单版本表不提供通用 run ID 续跑。中断后先用只读命令检查真实结构和数据；含混状态返回 `recovery_required`，不盲目 stamp、不用部分状态的备份替换原恢复点。库外摘要在 DDL 前记录执行及备份信息；重试保留原维护窗口和恢复点。在恢复条件允许时可显式重新确认 manual／skip 策略，但不绕过结构验证，也不替代必需的进度证据。 diff --git a/integrations/opendal/src/powercontext_connector_opendal/source.py b/integrations/opendal/src/powercontext_connector_opendal/source.py index c089367cba..5ab48cf7d6 100644 --- a/integrations/opendal/src/powercontext_connector_opendal/source.py +++ b/integrations/opendal/src/powercontext_connector_opendal/source.py @@ -23,7 +23,6 @@ from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, - MemoryEvidenceDeclaration, Source, SourceMaterialization, SourceProjection, @@ -93,7 +92,6 @@ class TextFileSnapshotSourceDefinition: name = TEXT_FILE_SNAPSHOT_SOURCE_NAME version = "1" source_class = TextFileSnapshotSource - memory_evidence: MemoryEvidenceDeclaration | None = None def __init__(self) -> None: projection = cast(SourceProjection[TextFileSnapshotSource], TextFileEvidenceProjection()) diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 5003b93785..e0b90bb72f 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "7eb8a861925bbb8fc9080d97e9bae83d772280d786d9a05d40d6bec8e7c9e734", - "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history and pending intents with projection rebuilds and read-only startup admission. Publication requires installed-release upgrade, repeat-apply, old-startup rejection and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles." + "persistence_sha256": "ffb5fbbc6940414a73fc30309b076dd338858a9cae0484ec5cb2c43799b58b61", + "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/__init__.py b/src/powercontext/__init__.py index f7589beabc..7448a04133 100644 --- a/src/powercontext/__init__.py +++ b/src/powercontext/__init__.py @@ -51,14 +51,12 @@ ) from powercontext.sources import ( AdapterSourceDefinition, - MemoryEvidenceDeclaration, Source, SourceAdapter, SourceCatalog, SourceCatalogBackend, SourceDefinition, SourceDefinitionRegistry, - SourceEvidence, SourceMaterialization, SourceProjection, SourceProjectionKey, @@ -92,7 +90,6 @@ "InvalidSourceProjectionError", "InvalidSourceReferenceError", "InvalidSourceResultError", - "MemoryEvidenceDeclaration", "PolicyTransition", "PowerContext", "PowerContextError", @@ -107,7 +104,6 @@ "SourceDefinitionNotFoundError", "SourceDefinitionRegistry", "SourceError", - "SourceEvidence", "SourceMaterialization", "SourceNotFoundError", "SourceProjection", diff --git a/src/powercontext/builtin/artifacts/atomic_memory/service.py b/src/powercontext/builtin/artifacts/atomic_memory/service.py index b6ad370439..cee995cd31 100644 --- a/src/powercontext/builtin/artifacts/atomic_memory/service.py +++ b/src/powercontext/builtin/artifacts/atomic_memory/service.py @@ -21,7 +21,6 @@ from typing import TYPE_CHECKING, Any, Protocol, cast from sqlalchemy.ext.asyncio import AsyncConnection -from typing_extensions import override from powercontext.artifacts import ArtifactLineage, ArtifactRef from powercontext.builtin.artifacts.atomic_memory.errors import ( @@ -54,7 +53,6 @@ from powercontext.builtin.artifacts.merge import ArtifactMergeSecurity, ArtifactMergeService, MergeTags from powercontext.builtin.artifacts.merge_models import ( ArtifactMergeErrors, - ArtifactMergeMutationResult, ArtifactMergePlan, ArtifactMergeRead, ArtifactMergeRecord, @@ -89,9 +87,6 @@ async def publish( prepared: PreparedAtomicMemoryProjection, ) -> None: ... async def remove(self, connection: AsyncConnection, scope_id: str, artifact_id: str) -> None: ... - async def capture_evidence( - self, connection: AsyncConnection, scope_id: str, record: AtomicMemoryRecord - ) -> None: ... ATOMIC_MERGE_ERRORS = ArtifactMergeErrors( @@ -287,32 +282,6 @@ async def remove(self, connection: AsyncConnection, scope_id: str, artifact_id: await self.projections.remove(connection, scope_id, artifact_id) -class _AtomicMemoryMergeService(ArtifactMergeService): - """Accept evidence with every new body, including retired restoration results. - - Shared merges, the Atomic facade and processing all use this commit seam. - State-only writes must not attest old bodies. Capture joins the caller's - transaction so its failure rolls back authority, governance and projections. - """ - - @override - async def _commit_locked( - self, - connection: AsyncConnection, - prepared: PreparedArtifactMerge, - current: dict[str, ArtifactMergeRecord], - context: Any, - direct_source: SourceRef | None, - ) -> ArtifactMergeMutationResult: - result = await super()._commit_locked(connection, prepared, current, context, direct_source) - revisions = {write.artifact_id for write in prepared.plan.writes if write.draft is not None} - projections = cast(AtomicMemoryMergeAdapter, self.adapter).projections - for record in result.records: - if record.ref.artifact_id in revisions: - await projections.capture_evidence(connection, prepared.plan.scope_id, _atomic_record(record)) - return result - - class _AtomicPreviewAdapter: def __init__(self, signer): self.signer = signer @@ -339,7 +308,7 @@ def __init__( ) -> None: self.artifacts, self.states, self.security = artifacts, states, security self.projections, self.merge_tags, self.preview_signer = projections, merge_tags, preview_signer - self.shared = _AtomicMemoryMergeService( + self.shared = ArtifactMergeService( artifacts=artifacts, adapter=AtomicMemoryMergeAdapter(projections), security=security, diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/atomic_memory_evidence.py deleted file mode 100644 index 99d5c38311..0000000000 --- a/src/powercontext/builtin/persistence/atomic_memory_evidence.py +++ /dev/null @@ -1,211 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Immutable per-revision evidence inputs and their neutral internal annotation.""" - -from __future__ import annotations - -import json -from collections.abc import Sequence -from typing import Literal - -import rfc8785 -from aiosqlite import Connection as SQLiteConnection -from pydantic import BaseModel, ConfigDict, ValidationError -from sqlalchemy import insert, select -from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncConnection - -from powercontext.artifacts import ArtifactRef -from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE -from powercontext.builtin.persistence.sources import SourceRepository, _require_identity -from powercontext.limits import MAX_SCOPE_ID_LENGTH -from powercontext.sources import SourceRef -from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence - - -class AtomicMemoryEvidenceAnnotation(BaseModel): - """Exact revision facts, not a trust score or recursively inferred evidence.""" - - model_config = ConfigDict(extra="forbid", frozen=True) - - scope_id: str - artifact_ref: ArtifactRef - sources: tuple[SourceEvidence, ...] - format: Literal["powercontext.atomic-memory.evidence.v1"] = "powercontext.atomic-memory.evidence.v1" - - -class AtomicMemoryEvidenceRepository: - """Join caller transactions; retained inputs are never deleted with the index. - - The annotation is derived from retained inputs at read time. There is no - separate cache to confuse with accepted history or join into search. - """ - - async def save( - self, - connection: AsyncConnection, - scope_id: str, - artifact_ref: ArtifactRef, - sources: Sequence[SourceEvidence], - ) -> AtomicMemoryEvidenceAnnotation: - artifact_ref = self._reference(scope_id, artifact_ref) - payload = await self._payload(connection, scope_id, artifact_ref) - if payload is not None: - return self._decode(scope_id, artifact_ref, payload) or self._neutral(scope_id, artifact_ref, sources) - accepted = [] - for source in sources: - value = source.model_dump(mode="json") - try: - declaration = MemoryEvidenceDeclaration.model_validate(value.get("declaration")) - except ValidationError: - declaration = MemoryEvidenceDeclaration() - accepted.append(SourceEvidence.model_validate({**value, "declaration": declaration})) - annotation = AtomicMemoryEvidenceAnnotation( - scope_id=scope_id, artifact_ref=artifact_ref, sources=tuple(accepted) - ) - if connection.dialect.name == "sqlite": - driver = (await connection.get_raw_connection()).driver_connection - if isinstance(driver, SQLiteConnection) and not driver.in_transaction: - # SQLite legacy transaction mode does not BEGIN on SELECT; - # SAVEPOINT must not accidentally own and commit the write. - await connection.exec_driver_sql("BEGIN") - try: - async with connection.begin_nested(): - await connection.execute( - insert(ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE).values( - scope_id=scope_id, - artifact_id=artifact_ref.artifact_id, - revision=artifact_ref.revision, - sources=rfc8785.dumps([source.model_dump(mode="json") for source in annotation.sources]), - ) - ) - except IntegrityError: - payload = await self._payload(connection, scope_id, artifact_ref) - if payload is None: - raise - return self._decode(scope_id, artifact_ref, payload) or self._neutral(scope_id, artifact_ref, sources) - return annotation - - async def read( - self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef - ) -> AtomicMemoryEvidenceAnnotation | None: - artifact_ref = self._reference(scope_id, artifact_ref) - payload = await self._payload(connection, scope_id, artifact_ref) - return None if payload is None else self._decode(scope_id, artifact_ref, payload) - - async def verify( - self, - connection: AsyncConnection, - scope_id: str, - artifact_ref: ArtifactRef, - source_refs: Sequence[SourceRef], - *, - sources: SourceRepository | None = None, - ) -> AtomicMemoryEvidenceAnnotation: - """Derive exact facts from retained inputs, never today's declarations. - - Missing or damaged declaration records are neutral. Missing Sources, - corrupt authoritative payloads and unequal exact materials remain errors. - No accepted snapshot is saved or repaired by this operation. - """ - - from powercontext.builtin.persistence.atomic_memory_index import AtomicMemoryIndexError - from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY - - artifact_ref = self._reference(scope_id, artifact_ref) - repository = SourceRepository(BUILTIN_SOURCE_REGISTRY) if sources is None else sources - materials = await repository.historical_evidence(connection, scope_id, source_refs) - annotation = await self.read(connection, scope_id, artifact_ref) - if annotation is not None: - by_ref = {(item.source_ref.source_type, item.source_ref.source_id): item for item in materials} - for accepted in annotation.sources: - material = by_ref.get((accepted.source_ref.source_type, accepted.source_ref.source_id)) - if material is not None and accepted.material_digest != material.material_digest: - raise AtomicMemoryIndexError( - "evidence-material", "Historical evidence does not match the exact Source material" - ) - if annotation is None or tuple(item.source_ref for item in annotation.sources) != tuple( - item.source_ref for item in materials - ): - return self._neutral(scope_id, artifact_ref, materials) - verified = tuple( - accepted - if ( - accepted.materialization == material.materialization - and accepted.definition_name == material.definition_name - and accepted.definition_version == material.definition_version - and accepted.manifest_fingerprint == material.manifest_fingerprint - and accepted.declaration == material.declaration - ) - else accepted.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) - for accepted, material in zip(annotation.sources, materials, strict=True) - ) - return annotation.model_copy(update={"sources": verified}) - - @staticmethod - def _reference(scope_id: str, artifact_ref: ArtifactRef) -> ArtifactRef: - _require_identity("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) - reference = ArtifactRef.model_validate(artifact_ref.model_dump()) - if reference.family != "atomic-memory": - from powercontext.builtin.persistence.errors import InvalidRepositoryArgumentError - - raise InvalidRepositoryArgumentError("artifact_ref", "must reference an Atomic Memory revision") - return reference - - @staticmethod - async def _payload(connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef) -> object | None: - table = ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE - return await connection.scalar( - select(table.c.sources).where( - table.c.scope_id == scope_id, - table.c.artifact_id == artifact_ref.artifact_id, - table.c.revision == artifact_ref.revision, - ) - ) - - @staticmethod - def _decode(scope_id: str, artifact_ref: ArtifactRef, payload: object) -> AtomicMemoryEvidenceAnnotation | None: - try: - if not isinstance(payload, (bytes, bytearray, memoryview)): - return None - values = json.loads(bytes(payload)) - if not isinstance(values, list): - return None - sources = [] - for value in values: - if not isinstance(value, dict): - return None - try: - declaration = MemoryEvidenceDeclaration.model_validate(value.get("declaration")) - except ValidationError: - declaration = MemoryEvidenceDeclaration() - sources.append(SourceEvidence.model_validate({**value, "declaration": declaration})) - return AtomicMemoryEvidenceAnnotation(scope_id=scope_id, artifact_ref=artifact_ref, sources=tuple(sources)) - except (ValueError, TypeError): - # An unreadable declaration snapshot conveys no historical claim. - # This does not catch SQL errors or actual Source resolution errors. - return None - - @staticmethod - def _neutral( - scope_id: str, artifact_ref: ArtifactRef, sources: Sequence[SourceEvidence] - ) -> AtomicMemoryEvidenceAnnotation: - # Preserve a damaged row exactly, without replacing it with today's - # stronger configuration. A caller can only receive neutral facts. - return AtomicMemoryEvidenceAnnotation( - scope_id=scope_id, - artifact_ref=artifact_ref, - sources=tuple(source.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) for source in sources), - ) diff --git a/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py b/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py deleted file mode 100644 index 22846565c1..0000000000 --- a/src/powercontext/builtin/persistence/atomic_memory_evidence_schema.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Retained exact-version evidence, independent of disposable search projections.""" - -from sqlalchemy import BigInteger, CheckConstraint, Column, LargeBinary, Table, Text -from sqlalchemy.dialects.mysql import LONGBLOB, MEDIUMTEXT - -from powercontext.builtin.persistence.tables import SHARED_METADATA, identity_string -from powercontext.limits import ( - MAX_ARTIFACT_ID_LENGTH, - MAX_SCOPE_ID_LENGTH, - MAX_SOURCE_ID_LENGTH, - MAX_SOURCE_TYPE_LENGTH, -) - -ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION = "p0005" -ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE = Table( - "pc_atomic_memory_evidence_schema", - SHARED_METADATA, - Column("schema_version", identity_string(32), primary_key=True), -) -ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE = Table( - "pc_atomic_memory_evidence_history", - SHARED_METADATA, - Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), - Column("artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), - Column("revision", BigInteger, primary_key=True), - Column("sources", LargeBinary().with_variant(LONGBLOB(), "mysql"), nullable=False), - CheckConstraint("revision > 0", name="ck_pc_atomic_memory_evidence_history_revision"), -) -SOURCE_EVIDENCE_BINDINGS_TABLE = Table( - "pc_source_evidence_bindings", - SHARED_METADATA, - Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), - Column("source_type", identity_string(MAX_SOURCE_TYPE_LENGTH), primary_key=True), - Column("source_id", identity_string(MAX_SOURCE_ID_LENGTH), primary_key=True), - Column("material_digest", identity_string(71), nullable=False), - Column("definition_name", Text().with_variant(MEDIUMTEXT(), "mysql"), nullable=False), - Column("definition_version", Text().with_variant(MEDIUMTEXT(), "mysql"), nullable=False), - Column("manifest_fingerprint", identity_string(71)), - Column("declaration", LargeBinary().with_variant(LONGBLOB(), "mysql"), nullable=False), -) -ATOMIC_MEMORY_EVIDENCE_TABLES = ( - ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE, - ATOMIC_MEMORY_EVIDENCE_HISTORY_TABLE, - SOURCE_EVIDENCE_BINDINGS_TABLE, -) diff --git a/src/powercontext/builtin/persistence/atomic_memory_index.py b/src/powercontext/builtin/persistence/atomic_memory_index.py index 45d356d6de..7cb942c861 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/atomic_memory_index.py @@ -18,7 +18,6 @@ import json import math -from collections.abc import Sequence from dataclasses import dataclass, replace from hashlib import sha256 from typing import Any, Literal, Protocol @@ -41,16 +40,9 @@ lexical_search_score, ) from powercontext.builtin.inference import EmbeddingModel -from powercontext.builtin.persistence.atomic_memory_evidence import ( - AtomicMemoryEvidenceAnnotation, - AtomicMemoryEvidenceRepository, -) -from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES from powercontext.builtin.persistence.atomic_memory_index_schema import ATOMIC_MEMORY_PROJECTION_FORMAT -from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.tables import ARTIFACT_TAGS_TABLE from powercontext.builtin.tags import TagFilter -from powercontext.sources.evidence import SourceEvidence AtomicMemorySearchMode = Literal["fts", "vector", "hybrid"] ATOMIC_MEMORY_RRF_CONSTANT = 60 @@ -194,19 +186,6 @@ async def replace( async def delete(self, connection: AsyncConnection, scope_id: str, artifact_id: str, /) -> None: ... - async def save_evidence( - self, - connection: AsyncConnection, - scope_id: str, - artifact_ref: ArtifactRef, - sources: Sequence[SourceEvidence], - /, - ) -> AtomicMemoryEvidenceAnnotation: ... - - async def read_evidence( - self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / - ) -> AtomicMemoryEvidenceAnnotation | None: ... - async def search( self, connection: AsyncConnection, scope_id: str, request: AtomicMemorySearchRequest, / ) -> AtomicMemorySearchChannels: ... @@ -232,11 +211,9 @@ def __init__( index: AtomicMemoryIndex, *, embedding_model: EmbeddingModel | None = None, - sources: SourceRepository | None = None, ) -> None: self.index = index self._embedding_model = embedding_model - self._sources = sources @property def profile_fingerprint(self) -> str | None: @@ -286,17 +263,6 @@ async def prepare(self, content: BaseModel) -> PreparedAtomicMemoryProjection: embedding_input_hash=None if vector is None else atomic_memory_embedding_input_hash(kind, body), ) - async def capture_evidence(self, connection: AsyncConnection, scope_id: str, record: Any) -> None: - """Accept history only for a committed body write, never state-only republishing.""" - - artifact = record.artifact - if ( - self._sources is not None - and await self.index.read_evidence(connection, scope_id, artifact.as_ref()) is None - ): - evidence = await self._sources.evidence(connection, scope_id, artifact.lineage.sources) - await self.index.save_evidence(connection, scope_id, artifact.as_ref(), evidence) - async def publish( self, connection: AsyncConnection, @@ -573,28 +539,12 @@ def __init__(self, table: Table, profile: EmbeddingProfile | None) -> None: if profile is not None and (profile.dimension < 1 or profile.distance != "l2"): raise AtomicMemoryIndexError("embedding-profile", "Atomic Memory requires a positive L2 embedding profile") self.table = table - self.tables: tuple[Table, ...] = (table, *ATOMIC_MEMORY_EVIDENCE_TABLES) - self._evidence = AtomicMemoryEvidenceRepository() + self.tables: tuple[Table, ...] = (table,) self.profile = profile self.capabilities = AtomicMemoryIndexCapabilities( vector=profile is not None, hybrid=profile is not None, embedding_profile=profile ) - async def save_evidence( - self, - connection: AsyncConnection, - scope_id: str, - artifact_ref: ArtifactRef, - sources: Sequence[SourceEvidence], - /, - ) -> AtomicMemoryEvidenceAnnotation: - return await self._evidence.save(connection, scope_id, artifact_ref, sources) - - async def read_evidence( - self, connection: AsyncConnection, scope_id: str, artifact_ref: ArtifactRef, / - ) -> AtomicMemoryEvidenceAnnotation | None: - return await self._evidence.read(connection, scope_id, artifact_ref) - async def validate_current(self, connection: AsyncConnection, /) -> None: """Audit vector deployment readiness at startup, outside ordinary bounded search.""" profile = self.capabilities.embedding_profile diff --git a/src/powercontext/builtin/persistence/atomic_memory_readiness.py b/src/powercontext/builtin/persistence/atomic_memory_readiness.py index 5dfc3356c5..ff12a47d4b 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_readiness.py +++ b/src/powercontext/builtin/persistence/atomic_memory_readiness.py @@ -25,12 +25,7 @@ from sqlalchemy import inspect, text from sqlalchemy.ext.asyncio import AsyncConnection -from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES from powercontext.builtin.persistence.errors import PersistenceError -from powercontext.builtin.persistence.migrations.atomic_memory_evidence import ( - AtomicMemoryEvidenceSchemaError, - require_atomic_memory_evidence_schema, -) LEGACY_CITATION_COLUMN = "memory_citations" _LEGACY_CITATION_TABLES = ("pc_artifacts", "pc_artifact_candidate_versions", "pc_candidate_versions") @@ -95,16 +90,11 @@ async def legacy_foreign_keys(connection: AsyncConnection, tables: set[str]) -> async def atomic_memory_readiness_issues( connection: AsyncConnection, tables: set[str] | None = None, *, collections_removed: bool = True ) -> list[str]: - """Check versioned schema and public migration completion without reading retained history.""" + """Check public migration completion, allowing collections only during offline pre-removal acceptance.""" if tables is None: tables = set(await connection.run_sync(lambda value: inspect(value).get_table_names())) issues: list[str] = [] - if "pc_artifacts" in tables or any(table.name in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): - try: - await require_atomic_memory_evidence_schema(connection, tables) - except AtomicMemoryEvidenceSchemaError as error: - issues.append(str(error)) if ( collections_removed and "pc_artifacts" in tables diff --git a/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json b/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json index f7159e2595..9d6c8f65cc 100644 --- a/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json +++ b/src/powercontext/builtin/persistence/migration_resources/topic_memory_recovery/schema.json @@ -101,30 +101,12 @@ "pc_atomic_memory_current_fts_idx", "CREATE TABLE 'pc_atomic_memory_current_fts_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID" ], - [ - "table", - "pc_atomic_memory_evidence_history", - "pc_atomic_memory_evidence_history", - "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)" - ], - [ - "table", - "pc_atomic_memory_evidence_schema", - "pc_atomic_memory_evidence_schema", - "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (schema_version)\n)" - ], [ "table", "pc_memory_artifact_archive", "pc_memory_artifact_archive", "CREATE TABLE pc_memory_artifact_archive (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent BLOB NOT NULL, \n\tmemory_citations BLOB, \n\tmetadata BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)" ], - [ - "table", - "pc_source_evidence_bindings", - "pc_source_evidence_bindings", - "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tmaterial_digest VARCHAR(71) NOT NULL, \n\tdefinition_name TEXT NOT NULL, \n\tdefinition_version TEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71), \n\tdeclaration BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)" - ], [ "trigger", "pc_atomic_memory_fts_delete", diff --git a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py deleted file mode 100644 index d23f81fd2e..0000000000 --- a/src/powercontext/builtin/persistence/migrations/atomic_memory_evidence.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Apply the frozen additive p0005 revision to runtime-owned evidence tables. - -The production bundle covers a historical subset of business tables. Runtime -profiles therefore apply this same frozen additive revision independently of -that bundle's whole-database inventory, without stamping unrelated schema or -backfilling declarations for old Source/Artifact rows. SQLite DDL joins the -caller's transaction; MySQL DDL may commit and is resumed by exact postconditions. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -from aiosqlite import Connection as SQLiteConnection -from sqlalchemy import inspect, select -from sqlalchemy.ext.asyncio import AsyncConnection - -from powercontext.builtin.persistence.atomic_memory_evidence_schema import ( - ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE, - ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION, - ATOMIC_MEMORY_EVIDENCE_TABLES, -) -from powercontext.builtin.persistence.errors import PersistenceError - - -class AtomicMemoryEvidenceSchemaError(PersistenceError): - """An evidence schema is not the registered versioned layout.""" - - def __init__(self, detail: str) -> None: - message = { - "missing": "Atomic Memory evidence schema is incomplete; apply the registered p0005 migration", - "version": "Atomic Memory evidence schema version is unsupported", - "shape": "Atomic Memory evidence table has an incompatible shape", - }[detail] - super().__init__(message) - - -async def apply_atomic_memory_evidence_migration(connection: AsyncConnection) -> None: - """Add only missing p0005 tables, validate exact shapes, then mark ready. - - No reads or writes of Source/Artifact history are needed, and no current - declaration is used to invent accepted historical inputs. - """ - - if connection.dialect.name == "sqlite": - driver = (await connection.get_raw_connection()).driver_connection - if isinstance(driver, SQLiteConnection) and not driver.in_transaction: - await connection.exec_driver_sql("BEGIN IMMEDIATE") - dialect = "sqlite" if connection.dialect.name == "sqlite" else "mysql" - resources = Path(__file__).parent / "resources/schemas/atomic_memory_evidence_v1.json" - statements = json.loads(resources.read_text())[dialect] - tables = set(await connection.run_sync(lambda sync: inspect(sync).get_table_names())) - for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): - if table.name in tables: - await _require_shape(connection, table.name, statement, dialect) - marker = ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE - versions = ( - tuple((await connection.execute(select(marker.c.schema_version))).scalars()) if marker.name in tables else () - ) - if versions and versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): - raise AtomicMemoryEvidenceSchemaError("version") - if versions and any(table.name not in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): - # A ready marker cannot authorize recreating missing retained history. - raise AtomicMemoryEvidenceSchemaError("missing") - for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): - if table.name not in tables: - await connection.exec_driver_sql(statement) - await _require_shape(connection, table.name, statement, dialect) - if not versions: - await connection.exec_driver_sql(statements[-1]) - - -async def require_atomic_memory_evidence_schema(connection: AsyncConnection, tables: set[str] | None = None) -> None: - """Check retained evidence readiness without DDL or historical payload reads.""" - - if tables is None: - tables = set(await connection.run_sync(lambda sync: inspect(sync).get_table_names())) - if any(table.name not in tables for table in ATOMIC_MEMORY_EVIDENCE_TABLES): - raise AtomicMemoryEvidenceSchemaError("missing") - dialect = "sqlite" if connection.dialect.name == "sqlite" else "mysql" - resources = Path(__file__).parent / "resources/schemas/atomic_memory_evidence_v1.json" - statements = json.loads(resources.read_text())[dialect] - for table, statement in zip(ATOMIC_MEMORY_EVIDENCE_TABLES, statements[:-1], strict=True): - await _require_shape(connection, table.name, statement, dialect) - versions = tuple((await connection.execute(select(ATOMIC_MEMORY_EVIDENCE_SCHEMA_TABLE.c.schema_version))).scalars()) - if versions != (ATOMIC_MEMORY_EVIDENCE_SCHEMA_VERSION,): - raise AtomicMemoryEvidenceSchemaError("version") - - -async def _require_shape(connection: AsyncConnection, name: str, statement: str, dialect: str) -> None: - if dialect == "sqlite": - actual = await connection.exec_driver_sql( - "SELECT type, name, sql FROM sqlite_master WHERE (name = ? OR tbl_name = ?) " - "AND substr(name, 1, 7) != 'sqlite_' ORDER BY type, name", - (name, name), - ) - if actual.all() != [("table", name, statement)]: - raise AtomicMemoryEvidenceSchemaError("shape") - return - # Reuse the frozen MySQL inventory parser so physical formatting accepted - # by the shared backend does not become a second schema convention. - from powercontext.builtin.persistence.migrations.models import MigrationError - from powercontext.builtin.persistence.migrations.mysql_schema import _matches, _parse_table - - actual = (await connection.exec_driver_sql(f"SHOW CREATE TABLE `{name}`")).one()[1] - try: - actual_name, actual_table = _parse_table(actual) - expected_name, expected_table = _parse_table(statement) - except MigrationError as error: - raise AtomicMemoryEvidenceSchemaError("shape") from error - if not _matches( - {"tables": {actual_name: actual_table}, "views": [], "triggers": []}, - {"tables": {expected_name: expected_table}, "views": [], "triggers": []}, - ): - raise AtomicMemoryEvidenceSchemaError("shape") diff --git a/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0005_atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0005_atomic_memory_evidence.py deleted file mode 100644 index 2550f917ed..0000000000 --- a/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0005_atomic_memory_evidence.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Create the frozen additive evidence layout without backfilling trust.""" - -from alembic import context, op - -revision = "p0005" -down_revision = "p0004" - - -def upgrade() -> None: - context.config.attributes["execute_revision"](revision) - op.execute("INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')") diff --git a/src/powercontext/builtin/persistence/migrations/resources/manifest.json b/src/powercontext/builtin/persistence/migrations/resources/manifest.json index 63887addc7..92f6ba7340 100644 --- a/src/powercontext/builtin/persistence/migrations/resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/resources/manifest.json @@ -5,16 +5,12 @@ ], "p0002": [], "p0003": [], - "p0004": [], - "p0005": [ - "schemas/atomic_memory_evidence_v1.json" - ] + "p0004": [] }, "sqlite_baselines": { "e1cae089a8a565291a7a1f782cea1f6c6e1a6cdd9eab75ef0c4ca61dfda453cb": "p0001", "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d": "p0003", - "8cb2d41e53cde751c8d0ef4023c344ec5ec78d9e579366c1c93d17de30aa5ea4": "p0004", - "5d2a458f6cb278e4fbff5aa7cef9429bcb940be5fb0a29649be50c30482e00e1": "p0005" + "8cb2d41e53cde751c8d0ef4023c344ec5ec78d9e579366c1c93d17de30aa5ea4": "p0004" }, "sqlite_fingerprints": { "p0001": [ @@ -30,10 +26,6 @@ "p0004": [ "f65244be963a382a62565118ff874cd775ab1ea33cf51a49881b40391e47c051", "8cb2d41e53cde751c8d0ef4023c344ec5ec78d9e579366c1c93d17de30aa5ea4" - ], - "p0005": [ - "10b8f62310f90e4e267e795efef0aa6e6716c1ecad515ab745e783507f6496f3", - "5d2a458f6cb278e4fbff5aa7cef9429bcb940be5fb0a29649be50c30482e00e1" ] }, "affected_objects": [ @@ -41,10 +33,7 @@ "pc_artifact_heads", "pc_artifact_candidate_versions", "pc_artifact_tags", - "pc_artifact_handles", - "pc_atomic_memory_evidence_schema", - "pc_atomic_memory_evidence_history", - "pc_source_evidence_bindings" + "pc_artifact_handles" ], "retained_objects": [ "pc_retained_p0002_artifact_tags" diff --git a/src/powercontext/builtin/persistence/migrations/resources/mysql_layouts.json b/src/powercontext/builtin/persistence/migrations/resources/mysql_layouts.json index 6442266801..6d6bda0e94 100644 --- a/src/powercontext/builtin/persistence/migrations/resources/mysql_layouts.json +++ b/src/powercontext/builtin/persistence/migrations/resources/mysql_layouts.json @@ -18,11 +18,6 @@ ], "p0004": [ "CREATE TABLE pc_artifact_handles (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\thandle_key VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\trevision INTEGER NOT NULL,\n\tPRIMARY KEY (scope_id, family, handle_key),\n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT,\n\tCONSTRAINT ck_pc_artifact_handles_revision_positive CHECK (revision > 0)\n)" - ], - "p0005": [ - "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", - "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", - "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_type VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tmaterial_digest VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tdefinition_name MEDIUMTEXT NOT NULL, \n\tdefinition_version MEDIUMTEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tdeclaration LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)" ] }, "checkpoints": { diff --git a/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json b/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json deleted file mode 100644 index e4af1dc164..0000000000 --- a/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_memory_evidence_v1.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sqlite": [ - "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", - "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", - "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tmaterial_digest VARCHAR(71) NOT NULL, \n\tdefinition_name TEXT NOT NULL, \n\tdefinition_version TEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71), \n\tdeclaration BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", - "INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')" - ], - "mysql": [ - "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", - "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", - "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_type VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tmaterial_digest VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tdefinition_name MEDIUMTEXT NOT NULL, \n\tdefinition_version MEDIUMTEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tdeclaration LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", - "INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')" - ] -} diff --git a/src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_memory_evidence.py b/src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_memory_evidence.py deleted file mode 100644 index 9190834ba0..0000000000 --- a/src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_memory_evidence.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Frozen additive evidence revision; never synthesize historical declarations.""" - -import json -from pathlib import Path - -from alembic import op - -revision = "p0005" -down_revision = "p0004" - - -def upgrade() -> None: - statements = json.loads((Path(__file__).parents[1] / "schemas/atomic_memory_evidence_v1.json").read_text()) - connection = op.get_bind() - for statement in statements["sqlite" if connection.dialect.name == "sqlite" else "mysql"]: - connection.exec_driver_sql(statement) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/evidence_domain.py b/src/powercontext/builtin/persistence/migrations/server_resources/evidence_domain.py deleted file mode 100644 index 4a1cdff5bb..0000000000 --- a/src/powercontext/builtin/persistence/migrations/server_resources/evidence_domain.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Read-only admission of retained exact-version evidence.""" - -from powercontext.builtin.persistence.atomic_memory_readiness import assert_atomic_memory_migration_ready -from powercontext.builtin.persistence.migrations.pinned import finish, pinned - - -def preflight(connection, *, configuration): - pass - - -def apply(connection, *, configuration): - pass - - -def verify(connection, *, configuration): - validate(connection, configuration=configuration) - - -def validate(connection, *, configuration): - finish(assert_atomic_memory_migration_ready(pinned(connection))) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json index bbb3c30169..78d996e08a 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json @@ -12,11 +12,6 @@ "dream/manifest.json", "mysql_operations.py", "schemas/mysql-s1400.json" - ], - "s1401": [ - "schemas/atomic_memory_evidence_v1.json", - "mysql_operations.py", - "schemas/mysql-s1401.json" ] }, "sqlite_baselines": { @@ -28,9 +23,6 @@ ], "s1400": [ "9064ecb0c6233884e64312a1a9a48a6c52b7c379724fa83f9b77b90ed1ee0381" - ], - "s1401": [ - "c2684f7c7a3476ebefd9f0c7b837765299002379948f1a08f369943a0d461f09" ] }, "affected_objects": [ @@ -122,10 +114,7 @@ "pc_topic_memory_work_blocks", "pc_topic_memory_work_budgets", "pc_retained_dream_candidate_heads", - "pc_retained_dream_candidate_versions", - "pc_atomic_memory_evidence_schema", - "pc_atomic_memory_evidence_history", - "pc_source_evidence_bindings" + "pc_retained_dream_candidate_versions" ], "readiness_scope": "server", "retained_objects": [ @@ -152,29 +141,11 @@ "seekdb", "oceanbase" ] - }, - { - "id": "source_evidence", - "revision": "s1401", - "kind": "data", - "module": "evidence_domain.py", - "depends_on": [ - "server_upgrade" - ], - "backends": [ - "sqlite", - "seekdb", - "oceanbase" - ] } ], "domain_readiness": { "s1400": [ "server_upgrade" - ], - "s1401": [ - "server_upgrade", - "source_evidence" ] }, "retained_fingerprints": { @@ -187,8 +158,7 @@ }, "mysql_state_layouts": [ "schemas/mysql-s1300.json", - "schemas/mysql-s1400.json", - "schemas/mysql-s1401.json" + "schemas/mysql-s1400.json" ], "mysql_history": "mysql" } diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/s1401.py b/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/s1401.py deleted file mode 100644 index fc511f26e5..0000000000 --- a/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/s1401.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Initialize the frozen MySQL baseline with declared partial-DDL recovery.""" - -from alembic import context - -revision = "s1401" -down_revision = "s1400" - - -def upgrade() -> None: - context.config.attributes["execute_revision"](revision) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/atomic_memory_evidence_v1.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/atomic_memory_evidence_v1.json deleted file mode 100644 index fe76b18f3a..0000000000 --- a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/atomic_memory_evidence_v1.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sqlite": [ - "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", - "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", - "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tmaterial_digest VARCHAR(71) NOT NULL, \n\tdefinition_name TEXT NOT NULL, \n\tdefinition_version TEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71), \n\tdeclaration BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", - "INSERT INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')" - ], - "mysql": [ - "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", - "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", - "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_type VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tmaterial_digest VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tdefinition_name MEDIUMTEXT NOT NULL, \n\tdefinition_version MEDIUMTEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tdeclaration LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", - "INSERT IGNORE INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')" - ] -} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1401.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1401.json deleted file mode 100644 index c31364d8d4..0000000000 --- a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1401.json +++ /dev/null @@ -1,259 +0,0 @@ -{ - "format": "powercontext.mysql-state-layouts.v1", - "revisions": { - "s1401": { - "statements": [ - "CREATE TABLE pc_atomic_memory_evidence_schema (\n\tschema_version VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tPRIMARY KEY (schema_version)\n)", - "CREATE TABLE pc_atomic_memory_evidence_history (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision BIGINT NOT NULL, \n\tsources LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, revision), \n\tCONSTRAINT ck_pc_atomic_memory_evidence_history_revision CHECK (revision > 0)\n)", - "CREATE TABLE pc_source_evidence_bindings (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_type VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tsource_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tmaterial_digest VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tdefinition_name MEDIUMTEXT NOT NULL, \n\tdefinition_version MEDIUMTEXT NOT NULL, \n\tmanifest_fingerprint VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tdeclaration LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id)\n)", - "INSERT IGNORE INTO pc_atomic_memory_evidence_schema (schema_version) VALUES ('p0005')" - ], - "changes": [ - { - "pc_atomic_memory_evidence_schema": { - "columns": { - "schema_version": { - "type": [ - "varchar", - [ - "32" - ] - ], - "nullable": false, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - } - }, - "primary_key": [ - [ - "schema_version", - null, - "asc" - ] - ], - "foreign_keys": [], - "checks": {}, - "indexes": {} - } - }, - { - "pc_atomic_memory_evidence_history": { - "columns": { - "scope_id": { - "type": [ - "varchar", - [ - "256" - ] - ], - "nullable": false, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - }, - "artifact_id": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": false, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - }, - "revision": { - "type": [ - "bigint", - [] - ], - "nullable": false, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "sources": { - "type": [ - "longblob", - [] - ], - "nullable": false, - "default": null, - "charset": null, - "collation": null, - "extra": [] - } - }, - "primary_key": [ - [ - "scope_id", - null, - "asc" - ], - [ - "artifact_id", - null, - "asc" - ], - [ - "revision", - null, - "asc" - ] - ], - "foreign_keys": [], - "checks": { - "ck_pc_atomic_memory_evidence_history_revision": { - "expression": [ - "binary", - ">", - [ - "column", - "revision" - ], - [ - "number", - 0 - ] - ], - "enforced": true - } - }, - "indexes": {} - } - }, - { - "pc_source_evidence_bindings": { - "columns": { - "scope_id": { - "type": [ - "varchar", - [ - "256" - ] - ], - "nullable": false, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - }, - "source_type": { - "type": [ - "varchar", - [ - "128" - ] - ], - "nullable": false, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - }, - "source_id": { - "type": [ - "varchar", - [ - "256" - ] - ], - "nullable": false, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - }, - "material_digest": { - "type": [ - "varchar", - [ - "71" - ] - ], - "nullable": false, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - }, - "definition_name": { - "type": [ - "mediumtext", - [] - ], - "nullable": false, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "definition_version": { - "type": [ - "mediumtext", - [] - ], - "nullable": false, - "default": null, - "charset": null, - "collation": null, - "extra": [] - }, - "manifest_fingerprint": { - "type": [ - "varchar", - [ - "71" - ] - ], - "nullable": true, - "default": null, - "charset": "utf8mb4", - "collation": "utf8mb4_bin", - "extra": [] - }, - "declaration": { - "type": [ - "longblob", - [] - ], - "nullable": false, - "default": null, - "charset": null, - "collation": null, - "extra": [] - } - }, - "primary_key": [ - [ - "scope_id", - null, - "asc" - ], - [ - "source_type", - null, - "asc" - ], - [ - "source_id", - null, - "asc" - ] - ], - "foreign_keys": [], - "checks": {}, - "indexes": {} - } - }, - {} - ] - } - } -} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/versions/s1401_source_evidence.py b/src/powercontext/builtin/persistence/migrations/server_resources/versions/s1401_source_evidence.py deleted file mode 100644 index 00f87b1f59..0000000000 --- a/src/powercontext/builtin/persistence/migrations/server_resources/versions/s1401_source_evidence.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Retained source evidence without inventing declarations for old records.""" - -import json -from pathlib import Path - -from alembic import op - -revision = "s1401" -down_revision = "s1400" - - -def upgrade(): - connection = op.get_bind() - statements = json.loads((Path(__file__).parents[1] / "schemas/atomic_memory_evidence_v1.json").read_text())[ - connection.dialect.name - ] - for statement in statements: - connection.exec_driver_sql(statement) diff --git a/src/powercontext/builtin/persistence/schema.py b/src/powercontext/builtin/persistence/schema.py index 8af018a110..93bd0678c3 100644 --- a/src/powercontext/builtin/persistence/schema.py +++ b/src/powercontext/builtin/persistence/schema.py @@ -21,12 +21,9 @@ from sqlalchemy import MetaData, Table, inspect from sqlalchemy.ext.asyncio import AsyncConnection -from powercontext.builtin.persistence.atomic_memory_evidence_schema import ATOMIC_MEMORY_EVIDENCE_TABLES -from powercontext.builtin.persistence.migrations.atomic_memory_evidence import apply_atomic_memory_evidence_migration - async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) -> None: - """Create selected tables, applying retained evidence through its frozen revision.""" + """Create selected tables only for unmanaged storage; managed schemas are read-only.""" names = await connection.run_sync(lambda sync: set(inspect(sync).get_table_names())) if "pc_schema_revision" in names: @@ -38,12 +35,7 @@ async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) ) return groups: dict[MetaData, list[Table]] = {} - evidence_names = {table.name for table in ATOMIC_MEMORY_EVIDENCE_TABLES} - if any(table.name in evidence_names or table.name == "pc_sources" for table in tables): - await apply_atomic_memory_evidence_migration(connection) for table in tables: - if table.name in evidence_names: - continue groups.setdefault(table.metadata, []).append(table) for metadata, selected in groups.items(): diff --git a/src/powercontext/builtin/persistence/source_definitions.py b/src/powercontext/builtin/persistence/source_definitions.py index beffb3f87f..9e5d4a8640 100644 --- a/src/powercontext/builtin/persistence/source_definitions.py +++ b/src/powercontext/builtin/persistence/source_definitions.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Immutable historical manifests owned by the Source registration boundary.""" +"""Persistence for worker-owned Source Definition manifests.""" from __future__ import annotations @@ -26,18 +26,15 @@ from powercontext.builtin.persistence.codec import dump_model, load_model, stored_bytes from powercontext.builtin.persistence.errors import ( IdentityMismatchError, - InvalidStoredColumnError, - InvalidStoredPayloadError, RepositoryNotFoundError, StoredPayloadConflictError, ) from powercontext.builtin.persistence.tables import SOURCE_DEFINITION_MANIFESTS_TABLE -from powercontext.sources import SourceDefinition, SourceDefinitionManifest -from powercontext.sources.observations import _manifest_for_definition +from powercontext.sources import SourceDefinitionManifest class SourceDefinitionManifestRepository: - """Retain native and worker manifests without changing their identities.""" + """Register immutable worker-owned Definition manifests by name and version.""" async def register( self, @@ -62,9 +59,7 @@ async def register( ) except IntegrityError: existing = await self.find(connection, manifest.name, manifest.version) - if existing is None: - raise - if existing != manifest: + if existing is None or existing != manifest: raise StoredPayloadConflictError( "source-definition-manifest", (manifest.name, manifest.version), @@ -72,29 +67,6 @@ async def register( return existing return manifest - async def register_native( - self, - connection: AsyncConnection, - definition: SourceDefinition[Any, Any, Any], - /, - ) -> SourceDefinitionManifest | None: - """Retain the existing manifest representation for trusted native material. - - This internal registration cannot replace an older manifest. A missing, - invalid or conflicting ancillary manifest leaves the Source declaration - neutral; database failures still abort the caller's transaction. External - registration keeps its existing active-Definition ownership guard. - """ - - try: - manifest = _manifest_for_definition(definition, transport=False) - except Exception: # Native schema providers may not expose a valid manifest. - return None - try: - return await self.register(connection, manifest) - except (IdentityMismatchError, InvalidStoredColumnError, InvalidStoredPayloadError, StoredPayloadConflictError): - return None - async def get( self, connection: AsyncConnection, diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index 88ff8e331e..ba3cd56c56 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -18,16 +18,10 @@ from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence from contextlib import suppress -from hashlib import sha256 from typing import Any, Literal, cast import rfc8785 -from jsonschema import Draft202012Validator -from jsonschema.exceptions import SchemaError -from jsonschema.exceptions import ValidationError as JsonSchemaValidationError -from pydantic import AliasChoices, AliasGenerator, BaseModel, ConfigDict, JsonValue, TypeAdapter -from referencing import Registry -from referencing.exceptions import Unresolvable +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter from sqlalchemy import func, insert, select, tuple_, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection @@ -35,7 +29,6 @@ from powercontext.builtin.artifacts.experience import EXPERIENCE_INCUBATION_CURSOR_NAME from powercontext.builtin.artifacts.profile.models import PROFILE_SOURCE_WINDOW_BINDING from powercontext.builtin.artifacts.topic_memory import TOPIC_MEMORY_SOURCE_WINDOW_BINDING -from powercontext.builtin.persistence.atomic_memory_evidence_schema import SOURCE_EVIDENCE_BINDINGS_TABLE from powercontext.builtin.persistence.codec import load_model, stored_bytes from powercontext.builtin.persistence.errors import ( IdentityMismatchError, @@ -46,22 +39,12 @@ StoredPayloadConflictError, ) from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository -from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository from powercontext.builtin.persistence.tables import SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE -from powercontext.builtin.sources.content import CONTENT_SOURCE_NAME, ContentSource +from powercontext.builtin.sources.content import ContentSource from powercontext.builtin.triggers import SOURCE_WINDOW_TRIGGER_NAME from powercontext.errors import SourceDefinitionNotFoundError from powercontext.limits import MAX_SCOPE_ID_LENGTH -from powercontext.sources import ( - MemoryEvidenceDeclaration, - Source, - SourceAdapter, - SourceDefinitionManifest, - SourceDefinitionRegistry, - SourceEvidence, - SourceObservation, - SourceRef, -) +from powercontext.sources import Source, SourceAdapter, SourceDefinitionRegistry, SourceObservation, SourceRef _AnySourceAdapter = SourceAdapter[Any, Any, Any] _JSON_OBJECT = TypeAdapter(dict[str, JsonValue]) @@ -79,9 +62,6 @@ class _StoredSourcePayload(BaseModel): encoding: Literal["powercontext-source-v1"] representation: Literal["native", "observation"] value: dict[str, JsonValue] - # Server-owned header permits historical reads without adapter code. Never hash it. - source: Source | None = None - source_keys: dict[str, str] | None = None class StoredSource(BaseModel): @@ -161,8 +141,6 @@ async def add_with_status( return StoredSource(ref=ref, value=source, journal_position=stored.journal_position), False if stored.value != source: raise StoredPayloadConflictError("source", (scope_id, ref)) - if representation == "native": - await self._retain_native_header(connection, scope_id, ref, existing, source) return stored, False position = await _next_journal_position(connection, scope_id) @@ -187,7 +165,6 @@ async def add_with_status( if stored.value != source: raise StoredPayloadConflictError("source", (scope_id, ref)) from None return stored, False - await self._bind_memory_evidence(connection, scope_id, ref, source) # Centralized here so capture, record projection and import all publish # discoverable input atomically. Replayed Source identities do not dirty # a binding again. Disabled processors simply retain ordinary dirty. @@ -256,425 +233,6 @@ async def get_many( raise RepositoryNotFoundError("source", (scope_id, ref)) return tuple(decoded[(ref.source_type, ref.source_id)] for ref in ordered) - async def evidence( - self, - connection: AsyncConnection, - scope_id: str, - refs: Sequence[SourceRef], - /, - ) -> tuple[SourceEvidence, ...]: - """Read accepted declarations against exact stored Source material. - - Source reads retain their existing errors and never become generation - eligibility checks. Missing or invalid declaration bindings are neutral; - database failures propagate. No current registry declaration or provider - value is substituted for historical registration facts. - """ - - return await self.historical_evidence(connection, scope_id, refs) - - async def historical_evidence( - self, connection: AsyncConnection, scope_id: str, refs: Sequence[SourceRef], / - ) -> tuple[SourceEvidence, ...]: - """Check retained material without loading today's native Definition. - - Only the stored representation selects decoding. Canonical JSON retains - all native fields, rather than round-tripping through a newer model. - This does not supply a decoder or provider value for ordinary reads. - """ - - _require_identity("scope_id", scope_id, MAX_SCOPE_ID_LENGTH) - identities = tuple(dict.fromkeys((ref.source_type, ref.source_id) for ref in refs)) - if not identities: - return () - rows = ( - ( - await connection.execute( - select(SOURCES_TABLE).where( - SOURCES_TABLE.c.scope_id == scope_id, - tuple_(SOURCES_TABLE.c.source_type, SOURCES_TABLE.c.source_id).in_(identities), - ) - ) - ) - .mappings() - .all() - ) - by_ref = {(row["source_type"], row["source_id"]): row for row in rows} - materials: list[SourceEvidence] = [] - fingerprints: dict[tuple[str, str], str] = {} - for ref in identities: - reference = SourceRef(source_type=ref[0], source_id=ref[1]) - row = by_ref.get(ref) - if row is None: - raise RepositoryNotFoundError("source", (scope_id, reference)) - source, material, verifiable = await self._historical_material(connection, reference, row) - if not verifiable: - fingerprints[ref] = "" # Never authorize a retained declaration for this capture. - if isinstance(source, SourceObservation): - fingerprints[ref] = source.definition_fingerprint - elif ref[0] == CONTENT_SOURCE_NAME: - # The server-owned Content witness is the sole permitted material - # mutation; this exclusion is the same one used at acceptance. - internal = material.get("internal") - if ( - isinstance(internal, dict) - and internal.get("role") == "lineage_only" - and internal.get("operation") in {"artifact_create", "artifact_replace"} - ): - continue - material = {key: value for key, value in material.items() if key != "handoff_receipt"} - materials.append( - SourceEvidence( - source_ref=reference, - materialization=source.materialization, - material_digest=_source_material_digest_material(material), - definition_name=ref[0], - definition_version=source.definition_version, - ) - ) - return await self._bound_evidence(connection, scope_id, tuple(materials), fingerprints) - - async def _historical_material( - self, connection: AsyncConnection, reference: SourceRef, row: Mapping[Any, Any] - ) -> tuple[Source, dict[str, JsonValue], bool]: - """Validate the stored representation and its retained schema, not a live model.""" - - payload = stored_bytes(row["payload"], column="payload") - try: - envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=reference.source_type) - except InvalidStoredPayloadError: - return await self._historical_legacy_material(connection, reference, row, payload) - material = envelope.value - source = ( - await self._historical_canonical_source(connection, reference, row, material) - if envelope.representation == "native" and envelope.source is None - else None - ) - if source is None: - source = self._historical_source(reference, envelope) - decoded = SourceRef( - source_type=source.source_type if isinstance(source, SourceObservation) else reference.source_type, - source_id=source.name, - ) - if reference != decoded: - raise IdentityMismatchError("source", reference, decoded) - table = SOURCE_EVIDENCE_BINDINGS_TABLE - binding = ( - await connection.execute( - select(table.c.manifest_fingerprint, table.c.material_digest).where( - table.c.scope_id == row["scope_id"], - table.c.source_type == reference.source_type, - table.c.source_id == reference.source_id, - table.c.definition_name == reference.source_type, - table.c.definition_version == source.definition_version, - ) - ) - ).one_or_none() - manifest = await _find_evidence_manifest(connection, reference.source_type, source.definition_version) - if manifest is not None and binding is not None and manifest.fingerprint == binding.manifest_fingerprint: - try: - Draft202012Validator.check_schema(manifest.source_schema) - value = source.payload if isinstance(source, SourceObservation) else material - Draft202012Validator(manifest.source_schema, registry=Registry()).validate(value) - except JsonSchemaValidationError as error: - if envelope.representation == "native" and binding.material_digest == _source_material_digest_material( - material - ): - # Older releases bound declarations without checking serialization - # compatibility. Exact accepted bytes remain readable, but neutral. - return source, material, False - raise InvalidStoredPayloadError( - "source", reference.source_type, "payload does not match the historical schema" - ) from error - except (SchemaError, Unresolvable): - self._validate_unbound_material(reference, envelope, source) - else: - self._validate_unbound_material(reference, envelope, source) - return source, material, True - - async def _historical_legacy_material( - self, - connection: AsyncConnection, - reference: SourceRef, - row: Mapping[Any, Any], - payload: bytes, - ) -> tuple[Source, dict[str, JsonValue], bool]: - """Read bare legacy material without guessing absent identity fields.""" - try: - material = _JSON_OBJECT.validate_json(payload, strict=True) - except ValueError as error: - raise InvalidStoredPayloadError("source", reference.source_type, "payload is not JSON") from error - source = await self._historical_canonical_source(connection, reference, row, material) - if source is not None: - return source, material, False - try: - self._registry.definition_for_name(reference.source_type) - except SourceDefinitionNotFoundError: - source = load_model(Source, payload, kind="source", name=reference.source_type) - if not {"name", "definition_version", "materialization"} <= source.model_fields_set: - raise InvalidStoredPayloadError( - "source", reference.source_type, "legacy native identity requires its registered decoder" - ) from None - if source.name != reference.source_id: - raise IdentityMismatchError("source", reference.source_id, source.name) from None - else: - source = self._decode_row(row).value - return source, material, True - - async def _historical_canonical_source( - self, - connection: AsyncConnection, - reference: SourceRef, - row: Mapping[Any, Any], - material: dict[str, JsonValue], - ) -> Source | None: - """Recover evidence identity from canonical retained material, never a live alias.""" - - if not {"name", "definition_version", "materialization"} <= material.keys(): - return None - if material["name"] != reference.source_id: - return None - digest_material = material - if reference.source_type == CONTENT_SOURCE_NAME: - digest_material = {key: value for key, value in material.items() if key != "handoff_receipt"} - material_digest = _source_material_digest_material(digest_material) - table = SOURCE_EVIDENCE_BINDINGS_TABLE - binding = ( - await connection.execute( - select(table.c.definition_name, table.c.definition_version, table.c.material_digest).where( - table.c.scope_id == row["scope_id"], - table.c.source_type == reference.source_type, - table.c.source_id == reference.source_id, - ) - ) - ).one_or_none() - if binding is not None: - if ( - binding.definition_name != reference.source_type - or binding.material_digest != material_digest - or binding.definition_version != material["definition_version"] - ): - return None - version = binding.definition_version - elif isinstance(material["definition_version"], str): - # Canonical retained identity is explicit even when its historical - # evidence binding predates the evidence tables. It stays neutral. - version = material["definition_version"] - else: - return None - try: - definition = self._registry.definition_for_name(reference.source_type) - except SourceDefinitionNotFoundError: - definition = None - if ( - definition is not None - and definition.version == version - and any( - ( - ( - definition.source_class.model_fields[key].alias is not None - and definition.source_class.model_fields[key].alias != key - ) - or ( - definition.source_class.model_fields[key].serialization_alias is not None - and definition.source_class.model_fields[key].serialization_alias != key - ) - or ( - definition.source_class.model_fields[key].validation_alias is not None - and definition.source_class.model_fields[key].validation_alias != key - ) - ) - for key in ("definition_version", "materialization") - ) - ): - # The current exact-version codec is required to disambiguate retained identity keys. - return None - try: - return Source.model_validate({ - "name": reference.source_id, - "definition_version": version, - "materialization": material["materialization"], - "description": material.get("description"), - }) - except (TypeError, ValueError): - return None - - def _historical_source(self, reference: SourceRef, envelope: _StoredSourcePayload) -> Source: - """Decode identity independently of retained material and today's business defaults.""" - - material = envelope.value - if envelope.representation == "observation": - return load_model(SourceObservation, rfc8785.dumps(material), kind="source", name=reference.source_type) - if envelope.source is not None: - keys = envelope.source_keys or {key: key for key in ("name", "definition_version", "materialization")} - for key, alias in keys.items(): - value = getattr(envelope.source, key) - if alias not in material or material[alias] != value: - raise IdentityMismatchError("source-envelope", value, material.get(alias)) - return envelope.source - # Legacy aliased rows need their registered decoder once; canonical rows do not. - if {"name", "definition_version", "materialization"} <= material.keys() and material[ - "name" - ] == reference.source_id: - definition = None - with suppress(SourceDefinitionNotFoundError): - definition = self._registry.definition_for_name(reference.source_type) - if definition is None or not _claims_identity_keys(definition.source_class): - return load_model(Source, rfc8785.dumps(material), kind="source", name=reference.source_type) - with suppress(SourceDefinitionNotFoundError): - definition = self._registry.definition_for_name(reference.source_type) - if any( - (definition.source_class.model_fields[key].serialization_alias or key) != key - for key in ("name", "definition_version", "materialization") - ): - source, _ = self._decode_envelope(reference.source_type, envelope) - return source - header = material - with suppress(SourceDefinitionNotFoundError): - definition = self._registry.definition_for_name(reference.source_type) - header = _normalize_native_material(definition.source_class, material) - if not {"name", "definition_version", "materialization"} <= header.keys(): - raise InvalidStoredPayloadError( - "source", reference.source_type, "legacy native identity requires its registered decoder" - ) - try: - return load_model(Source, rfc8785.dumps(header), kind="source", name=reference.source_type) - except InvalidStoredPayloadError: - source, _ = self._decode_envelope(reference.source_type, envelope) - return source - - def _validate_unbound_material(self, reference: SourceRef, envelope: _StoredSourcePayload, source: Source) -> None: - """Keep supported exact-read errors when no historical schema is available. - - A live decoder validates only material of its own version; it never - contributes a declaration or reinterprets a valid historical binding. - """ - - if envelope.representation == "observation": - return # SourceObservation already validated its envelope and payload. - try: - definition = self._registry.definition_for_name(reference.source_type) - except SourceDefinitionNotFoundError: - return - if _claims_identity_keys(definition.source_class): - # The retained canonical identity/header is authoritative; a live business alias may claim it. - return - if envelope.source is None and any( - (definition.source_class.model_fields[key].serialization_alias or key) != key - for key in ("name", "definition_version", "materialization") - ): - # Today's identity aliases cannot validate an older unbound representation. - return - if definition.version == source.definition_version: - self._decode_envelope(reference.source_type, envelope) - - @staticmethod - async def _bound_evidence( - connection: AsyncConnection, - scope_id: str, - materials: tuple[SourceEvidence, ...], - fingerprints: Mapping[tuple[str, str], str], - ) -> tuple[SourceEvidence, ...]: - if not materials: - return () - table = SOURCE_EVIDENCE_BINDINGS_TABLE - rows = ( - ( - await connection.execute( - select(table).where( - table.c.scope_id == scope_id, - tuple_(table.c.source_type, table.c.source_id).in_( - tuple((value.source_ref.source_type, value.source_ref.source_id) for value in materials) - ), - ) - ) - ) - .mappings() - .all() - ) - bindings = {(row["source_type"], row["source_id"]): row for row in rows} - manifests: dict[tuple[str, str], SourceDefinitionManifest | None] = {} - evidence: list[SourceEvidence] = [] - for value in materials: - identity = (value.source_ref.source_type, value.source_ref.source_id) - binding = bindings.get(identity) - if binding is None or ( - binding["material_digest"] != value.material_digest - or binding["definition_name"] != value.definition_name - or binding["definition_version"] != value.definition_version - or not isinstance(binding["manifest_fingerprint"], str) - ): - evidence.append(value) - continue - definition_identity = (value.definition_name, value.definition_version) - if definition_identity not in manifests: - manifests[definition_identity] = await _find_evidence_manifest(connection, *definition_identity) - manifest = manifests[definition_identity] - if ( - manifest is None - or manifest.fingerprint != binding["manifest_fingerprint"] - or (identity in fingerprints and fingerprints[identity] != manifest.fingerprint) - ): - evidence.append(value) - continue - try: - declaration = MemoryEvidenceDeclaration.model_validate_json( - stored_bytes(binding["declaration"], column="declaration") - ) - except (InvalidStoredColumnError, TypeError, ValueError): - declaration = MemoryEvidenceDeclaration() - evidence.append( - value.model_copy( - update={ - "manifest_fingerprint": manifest.fingerprint, - "declaration": declaration, - } - ) - ) - return tuple(evidence) - - async def _bind_memory_evidence( - self, - connection: AsyncConnection, - scope_id: str, - ref: SourceRef, - source: Source, - ) -> None: - """Freeze registration evidence only for a newly accepted Source row.""" - - if isinstance(source, SourceObservation): - manifest = await _find_evidence_manifest(connection, ref.source_type, source.definition_version) - if manifest is not None and manifest.fingerprint != source.definition_fingerprint: - manifest = None - else: - definition = self._registry.definition_for_source(source) - manifest = await SourceDefinitionManifestRepository().register_native(connection, definition) - if manifest is not None: - # A native decoder can accept representations its generated schema - # cannot describe. Such captures are valid, but cannot attest evidence. - try: - Draft202012Validator.check_schema(manifest.source_schema) - Draft202012Validator(manifest.source_schema, registry=Registry()).validate( - source.model_dump(mode="json", by_alias=True) - ) - except (JsonSchemaValidationError, SchemaError, Unresolvable): - manifest = None - declaration = ( - MemoryEvidenceDeclaration() if manifest is None else self._registry.memory_evidence_for_source(source) - ) - await connection.execute( - insert(SOURCE_EVIDENCE_BINDINGS_TABLE).values( - scope_id=scope_id, - source_type=ref.source_type, - source_id=ref.source_id, - material_digest=_source_material_digest(source), - definition_name=ref.source_type, - definition_version=source.definition_version, - manifest_fingerprint=None if manifest is None else manifest.fingerprint, - declaration=declaration.model_dump_json().encode("utf-8"), - ) - ) - async def list( self, connection: AsyncConnection, @@ -836,41 +394,6 @@ def _decode_row(self, row: Mapping[Any, Any]) -> StoredSource: journal_position=int(row["journal_position"]), ) - async def _retain_native_header( - self, connection: AsyncConnection, scope_id: str, ref: SourceRef, row: Mapping[Any, Any], source: Source - ) -> None: - """Enrich only an identical replay's unreconstructible legacy header. - - The retained value, binding, manifest, digest and journal position never change. - This grants no authority and does not publish another processing intent. - """ - - payload = stored_bytes(row["payload"], column="payload") - try: - envelope = load_model(_StoredSourcePayload, payload, kind="source-envelope", name=ref.source_type) - except InvalidStoredPayloadError: - envelope = _StoredSourcePayload( - encoding="powercontext-source-v1", - representation="native", - value=_JSON_OBJECT.validate_json(payload, strict=True), - ) - if envelope.source is not None: - return - header = _source_header(source) - with suppress(InvalidStoredPayloadError): - if load_model(Source, rfc8785.dumps(envelope.value), kind="source", name=ref.source_type) == header: - return - enriched = envelope.model_copy(update={"source": header, "source_keys": _source_keys(source, envelope.value)}) - await connection.execute( - update(SOURCES_TABLE) - .where( - SOURCES_TABLE.c.scope_id == scope_id, - SOURCES_TABLE.c.source_type == ref.source_type, - SOURCES_TABLE.c.source_id == ref.source_id, - ) - .values(payload=rfc8785.dumps(enriched.model_dump(mode="json", exclude_none=True))) - ) - def _decode_envelope( self, source_type: str, @@ -889,15 +412,11 @@ def _decode_envelope( definition = self._definition_by_name(source_type) source = load_model( definition.source_class, - rfc8785.dumps(_normalize_native_material(definition.source_class, envelope.value)), + payload, kind="source", name=source_type, ) self._registry.definition_for_source(source) - if envelope.source is not None: - for key in ("name", "definition_version", "materialization"): - if getattr(envelope.source, key) != getattr(source, key): - raise IdentityMismatchError("source-envelope", envelope.source, _source_header(source)) return source, SourceRef(source_type=definition.name, source_id=source.name) def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Source, SourceRef]: @@ -912,13 +431,9 @@ def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Sour ) decoded = SourceRef(source_type=source.source_type, source_id=source.name) else: - try: - material = _JSON_OBJECT.validate_json(payload, strict=True) - except ValueError as error: - raise InvalidStoredPayloadError("source", source_type, "payload is not JSON") from error source = load_model( definition.source_class, - rfc8785.dumps(_normalize_native_material(definition.source_class, material)), + payload, kind="source", name=source_type, ) @@ -927,113 +442,6 @@ def _decode_legacy_payload(self, source_type: str, payload: bytes) -> tuple[Sour return cast(Source, source), decoded -def _source_material_digest(source: Source) -> str: - # Preserve every material field except the narrow server witness upgrade. - material = source.model_dump(mode="json", by_alias=True) - if isinstance(source, ContentSource): - material.pop("handoff_receipt", None) - return f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}" - - -def _management_write_source(source: Source) -> bool: - return ( - isinstance(source, ContentSource) - and source.internal is not None - and source.internal.role == "lineage_only" - and source.internal.operation in {"artifact_create", "artifact_replace"} - ) - - -async def _find_evidence_manifest( - connection: AsyncConnection, name: str, version: str -) -> SourceDefinitionManifest | None: - try: - return await SourceDefinitionManifestRepository().find(connection, name, version) - except (IdentityMismatchError, InvalidStoredColumnError, InvalidStoredPayloadError): - return None - - -def _source_material_digest_material(material: Mapping[str, JsonValue]) -> str: - return f"sha256:{sha256(rfc8785.dumps(material)).hexdigest()}" - - -def _source_header(source: Source) -> Source: - return Source(**{key: getattr(source, key) for key in Source.model_fields}) - - -def _source_keys(source: Source, material: dict[str, JsonValue] | None = None) -> dict[str, str]: - """Retain the exact serialized spelling used for each native identity.""" - keys: dict[str, str] = {} - for key in ("name", "definition_version", "materialization"): - field = type(source).model_fields[key] - preferred = field.serialization_alias or key - candidates = [preferred] - if isinstance(field.validation_alias, str): - candidates.append(field.validation_alias) - elif isinstance(field.validation_alias, AliasChoices): - candidates.extend(choice for choice in field.validation_alias.choices if isinstance(choice, str)) - if field.alias is not None: - candidates.append(field.alias) - generator = type(source).model_config.get("alias_generator") - if isinstance(generator, AliasGenerator): - generator = generator.serialization_alias or generator.alias - if generator is not None: - candidates.append(generator(key)) - if material is not None: - keys[key] = next( - (alias for alias in dict.fromkeys(candidates) if material.get(alias) == getattr(source, key)), - preferred, - ) - else: - keys[key] = preferred - return keys - - -def _claims_identity_keys(source_class: type[Source]) -> bool: - """Report whether a business field serializes to a canonical identity spelling. - - A headerless envelope cannot distinguish an identity value from a business - value with that name. When the registered model claims the spelling for a - different field, only its declared aliases are authoritative. - """ - - return any( - name not in Source.model_fields and key in (field.alias, field.validation_alias, field.serialization_alias) - for name, field in source_class.model_fields.items() - for key in Source.model_fields - ) - - -def _normalize_native_material(source_class: type[Source], material: dict[str, JsonValue]) -> dict[str, JsonValue]: - """Read pre-canonical inherited envelope aliases, never business-field aliases. - - Derive spellings from the registered generator, not a version/key allowlist. - Explicit adapter aliases and conflicting identities retain strict validation. - """ - - generator = source_class.model_config.get("alias_generator") - if isinstance(generator, AliasGenerator): - generator = generator.serialization_alias or generator.alias - if generator is None: - return material - normalized = dict(material) - for key in Source.model_fields: - field = source_class.model_fields[key] - if field.validation_alias != key and field.alias_priority != 1: - continue - alias = generator(key) - if any( - field_name != key and alias in (field.alias, field.validation_alias, field.serialization_alias) - for field_name, field in source_class.model_fields.items() - ): - continue - if alias != key and alias in normalized: - if key in normalized and normalized[key] != normalized[alias]: - raise IdentityMismatchError("source", normalized[key], normalized[alias]) - normalized[key] = normalized.pop(alias) - return normalized - - def _dump_source( source: Source, *, @@ -1045,16 +453,8 @@ def _dump_source( encoding="powercontext-source-v1", representation=representation, value=_JSON_OBJECT.validate_python(source.model_dump(mode="json", by_alias=True)), - source=None if representation == "observation" else _source_header(source), - source_keys=None if representation == "observation" else _source_keys(source), ) - if envelope.source_keys is not None: - for key, alias in envelope.source_keys.items(): - if alias not in envelope.value: - raise InvalidStoredPayloadError("source", name, f"serialized native identity omits {key!r}") - if envelope.value[alias] != getattr(source, key): - raise InvalidStoredPayloadError("source", name, f"serialized native identity changes {key!r}") - return rfc8785.dumps(envelope.model_dump(mode="json", exclude_none=True)) + return rfc8785.dumps(envelope.model_dump(mode="json")) except (TypeError, ValueError) as error: raise InvalidStoredPayloadError("source", name, "value is not JSON serializable") from error diff --git a/src/powercontext/builtin/runtime/atomic_memory.py b/src/powercontext/builtin/runtime/atomic_memory.py index ba99ef4e02..bbffb84a58 100644 --- a/src/powercontext/builtin/runtime/atomic_memory.py +++ b/src/powercontext/builtin/runtime/atomic_memory.py @@ -68,10 +68,6 @@ ) from powercontext.builtin.persistence.artifacts import ArtifactRepository, with_artifact_content from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository -from powercontext.builtin.persistence.atomic_memory_evidence import ( - AtomicMemoryEvidenceAnnotation, - AtomicMemoryEvidenceRepository, -) from powercontext.builtin.persistence.atomic_memory_index import ( ATOMIC_MEMORY_RRF_CONSTANT, AtomicMemoryIndex, @@ -255,7 +251,6 @@ def __init__( self.publisher = AtomicMemoryProjectionPublisher( index, embedding_model=embedding_model, - sources=artifacts.source_repository, ) self.service = AtomicMemoryService( artifacts=artifacts, @@ -306,23 +301,6 @@ async def get(self, artifact_id: str, *, revision: int | None = None, context=No connection, self.scope_id, artifact_id, context, revision=revision ) - async def evidence(self, artifact_id: str, *, revision: int, context=None) -> AtomicMemoryEvidenceAnnotation: - """Inspect neutral evidence facts for one authorized exact body revision. - - A missing historical snapshot remains unknown; current registration - cannot attest a revision that did not accept a declaration. - """ - application = self.application - async with self._read(context) as connection: - record = await application.service.get(connection, self.scope_id, artifact_id, context, revision=revision) - return await AtomicMemoryEvidenceRepository().verify( - connection, - self.scope_id, - record.ref, - record.artifact.lineage.sources, - sources=application.artifacts.source_repository, - ) - def list_query( self, connection: AsyncConnection, diff --git a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py index 1dfe6e6e3f..716480b6e3 100644 --- a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py +++ b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Offline current-projection repair and verification of retained Atomic history.""" +"""Explicit offline repair of only the current Atomic Memory projection.""" from __future__ import annotations @@ -25,14 +25,12 @@ from sqlalchemy.engine import RowMapping from sqlalchemy.ext.asyncio import AsyncConnection -from powercontext.artifacts import ArtifactRef from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemory, AtomicMemoryRecord, AtomicMemoryStateValue from powercontext.builtin.artifacts.memory.canonical import canonical_json from powercontext.builtin.artifacts.search import analyze_text from powercontext.builtin.inference import EmbeddingModel from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository -from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceRepository from powercontext.builtin.persistence.atomic_memory_index import ( AtomicMemoryIndex, AtomicMemoryIndexError, @@ -51,7 +49,6 @@ from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex from powercontext.builtin.persistence.revision_metadata_schema import ensure_artifact_revision_metadata_schema -from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE @@ -118,42 +115,6 @@ async def _identity_batch( return tuple((str(row.scope_id), str(row.artifact_id)) for row in rows) -async def _verify_history( - connection: AsyncConnection, artifacts: ArtifactRepository, batch_size: int, sources: SourceRepository | None -) -> dict[str, int]: - table = ARTIFACTS_TABLE - evidence = AtomicMemoryEvidenceRepository() - after: tuple[str, str, int] | None = None - revisions = neutral_sources = 0 - while True: - statement = select(table.c.scope_id, table.c.artifact_id, table.c.revision).where( - table.c.family == AtomicMemory.family - ) - if after is not None: - statement = statement.where(tuple_(table.c.scope_id, table.c.artifact_id, table.c.revision) > after) - rows = ( - await connection.execute( - statement.order_by(table.c.scope_id, table.c.artifact_id, table.c.revision).limit(batch_size) - ) - ).all() - if not rows: - break - for row in rows: - scope_id, artifact_id, revision = str(row.scope_id), str(row.artifact_id), int(row.revision) - reference = ArtifactRef(family=AtomicMemory.family, artifact_id=artifact_id, revision=revision) - artifact = await artifacts.get(connection, scope_id, reference) - annotation = await evidence.verify( - connection, scope_id, reference, artifact.lineage.sources, sources=sources - ) - revisions += 1 - neutral_sources += sum( - source.declaration.authority == "untrusted" and source.declaration.verification == "unknown" - for source in annotation.sources - ) - after = scope_id, artifact_id, revision - return {"historical_revisions": revisions, "neutral_sources": neutral_sources} - - async def _record( connection: AsyncConnection, artifacts: ArtifactRepository, scope_id: str, artifact_id: str ) -> AtomicMemoryRecord: @@ -215,20 +176,12 @@ async def _projection_differences( async def verify_atomic_memory_current( - connection: AsyncConnection, - index: AtomicMemoryIndex, - /, - *, - batch_size: int = 100, - sources: SourceRepository | None = None, + connection: AsyncConnection, index: AtomicMemoryIndex, /, *, batch_size: int = 100 ) -> AtomicMemoryProjectionRebuildReport: - """Check current projections and every exact revision's retained evidence.""" + """Check every current identity against authority, including post-migration creations.""" - if isinstance(batch_size, bool) or not 1 <= batch_size <= 1000: - raise AtomicMemoryIndexError("batch-size", "Projection rebuild batch size must be between 1 and 1000") await _require_authority(connection) artifacts = ArtifactRepository((AtomicMemory,)) - history_counts = await _verify_history(connection, artifacts, batch_size, sources) profile = index.capabilities.embedding_profile fingerprint = None if profile is None else atomic_memory_profile_fingerprint(profile) errors: list[str] = [] @@ -266,7 +219,7 @@ async def verify_atomic_memory_current( return AtomicMemoryProjectionRebuildReport( ready=not errors, profile_fingerprint=fingerprint, - counts={"active_rows": active, "obsolete_rows": obsolete, **history_counts}, + counts={"active_rows": active, "obsolete_rows": obsolete}, errors=tuple(errors), ) @@ -290,7 +243,6 @@ async def rebuild_atomic_memory_projection( maintenance_confirmed: bool, embedding_model: EmbeddingModel | None = None, batch_size: int = 100, - sources: SourceRepository | None = None, ) -> AtomicMemoryProjectionRebuildReport: """Repair current without changing revisions, Family state or retained history. @@ -359,7 +311,7 @@ async def rebuild_atomic_memory_projection( ) await connection.execute(index.table.delete().where(_obsolete_current(index))) removed += obsolete - report = await verify_atomic_memory_current(connection, index, batch_size=batch_size, sources=sources) + report = await verify_atomic_memory_current(connection, index, batch_size=batch_size) return report.model_copy( update={ "counts": { diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index 3d19b29bce..ea9ab6f550 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -211,7 +211,7 @@ ) from powercontext.builtin.statistics import ModelUsagePurpose from powercontext.errors import InvalidSourceProjectionError, SourceProjectionNotFoundError -from powercontext.sources import AdapterSourceDefinition, Source, SourceDefinitionRegistry, SourceProjectionKey +from powercontext.sources import Source, SourceDefinitionRegistry, SourceProjectionKey if TYPE_CHECKING: from pydantic_ai.models import Model @@ -905,17 +905,7 @@ def _validate_processing_source_registry( type(source_registry) is not SourceDefinitionRegistry or len(definitions) != len(builtin_definitions) or any( - not ( - definition is expected - or ( - type(definition) is AdapterSourceDefinition - and type(expected) is AdapterSourceDefinition - and definition.adapter is expected.adapter - and definition.version == expected.version - and definition.projections is expected.projections - ) - ) - for definition, expected in zip(definitions, builtin_definitions, strict=True) + definition is not expected for definition, expected in zip(definitions, builtin_definitions, strict=True) ) ): raise BuiltinConfigurationError("artifact-processing-source-registry") diff --git a/src/powercontext/sources/__init__.py b/src/powercontext/sources/__init__.py index 5469244670..91e1a27b08 100644 --- a/src/powercontext/sources/__init__.py +++ b/src/powercontext/sources/__init__.py @@ -30,7 +30,6 @@ SourceDefinitionRegistry, SourceProjection, ) -from powercontext.sources.evidence import MemoryEvidenceDeclaration, SourceEvidence from powercontext.sources.models import Source, SourceMaterialization, SourceProjectionKey, SourceRef from powercontext.sources.observations import ( SourceDefinitionManifest, @@ -54,7 +53,6 @@ "ConnectorRunSession", "ConnectorRunStatus", "ConnectorSubmissionStatus", - "MemoryEvidenceDeclaration", "Source", "SourceAdapter", "SourceCatalog", @@ -62,7 +60,6 @@ "SourceDefinition", "SourceDefinitionManifest", "SourceDefinitionRegistry", - "SourceEvidence", "SourceMaterialization", "SourceObservation", "SourceProjection", diff --git a/src/powercontext/sources/definitions.py b/src/powercontext/sources/definitions.py index 18f6bd5162..d9c54572dc 100644 --- a/src/powercontext/sources/definitions.py +++ b/src/powercontext/sources/definitions.py @@ -18,7 +18,6 @@ from collections.abc import Iterable, Mapping from dataclasses import dataclass -from types import MappingProxyType from typing import Any, Generic, Protocol, TypeVar, cast from pydantic import BaseModel, JsonValue, TypeAdapter @@ -35,7 +34,6 @@ SourceProjectionNotFoundError, ) from powercontext.sources.adapters import SourceAdapter -from powercontext.sources.evidence import MemoryEvidenceDeclaration from powercontext.sources.models import Source, SourceProjectionKey InputT = TypeVar("InputT") @@ -44,7 +42,6 @@ _JSON_VALUE = TypeAdapter(JsonValue) _AnySourceAdapter = SourceAdapter[Any, Any, Any] -_NEUTRAL_MEMORY_EVIDENCE = MemoryEvidenceDeclaration() class SourceProjection(Protocol[SourceT]): @@ -63,7 +60,6 @@ class SourceDefinition(SourceAdapter[InputT, SourceT, ValueT_co], Protocol[Input version: str projections: tuple[SourceProjection[SourceT], ...] - memory_evidence: MemoryEvidenceDeclaration | None @dataclass(frozen=True, slots=True) @@ -73,7 +69,6 @@ class AdapterSourceDefinition(Generic[InputT, SourceT, ValueT_co]): adapter: SourceAdapter[InputT, SourceT, ValueT_co] version: str = "1" projections: tuple[SourceProjection[SourceT], ...] = () - memory_evidence: MemoryEvidenceDeclaration | None = None @property def input_class(self) -> type[InputT]: @@ -98,28 +93,14 @@ async def read(self, source: SourceT, /) -> ValueT_co: class SourceDefinitionRegistry: - """Provide one immutable routing and trusted declaration view. - - Native Definitions optionally supply ``memory_evidence``. Deployment-owned - ``remote_declarations`` keys are exact ``(name, version, fingerprint)`` - manifest identities, never worker payload or projection values. Declarations - are validated and copied at construction; invalid or unavailable values are - neutral and never prevent registration of an otherwise valid Definition. - """ - - def __init__( - self, - definitions: Iterable[_AnySourceDefinition], - /, - *, - remote_declarations: Mapping[tuple[str, str, str], MemoryEvidenceDeclaration] | None = None, - ) -> None: + """Provide one immutable routing view for Source persistence and consumers.""" + + def __init__(self, definitions: Iterable[_AnySourceDefinition], /) -> None: by_input: dict[type[object], _AnySourceDefinition] = {} by_source: dict[type[Source], _AnySourceDefinition] = {} by_name: dict[str, _AnySourceDefinition] = {} projections: dict[type[Source], Mapping[SourceProjectionKey, SourceProjection[Any]]] = {} registered: list[_AnySourceDefinition] = [] - memory_evidence: dict[type[Source], MemoryEvidenceDeclaration] = {} for definition in definitions: input_class, source_class = _validate_definition(definition) @@ -141,11 +122,6 @@ def __init__( by_source[source_class] = definition by_name[definition.name] = definition projections[source_class] = projection_routes - try: - declaration = getattr(definition, "memory_evidence", None) - except Exception: # Declaration providers may be unavailable; routing remains valid. - declaration = None - memory_evidence[source_class] = _snapshot_memory_evidence(declaration) registered.append(definition) self._definitions = tuple(registered) @@ -153,10 +129,6 @@ def __init__( self._by_source = by_source self._by_name = by_name self._projections = projections - self._memory_evidence = MappingProxyType(memory_evidence) - self._remote_declarations = MappingProxyType({ - key: _snapshot_memory_evidence(value) for key, value in (remote_declarations or {}).items() - }) @classmethod def from_adapters(cls, adapters: Iterable[_AnySourceAdapter], /) -> SourceDefinitionRegistry: @@ -189,23 +161,6 @@ def definition_for_source(self, source: object, /) -> _AnySourceDefinition: ) return definition - def memory_evidence_for_source(self, source: Source, /) -> MemoryEvidenceDeclaration: - """Read the frozen trusted declaration using the exact registered route.""" - - from powercontext.sources.observations import SourceObservation - - if isinstance(source, SourceObservation): - return self.memory_evidence_for_manifest( - source.source_type, source.definition_version, source.definition_fingerprint - ) - self.definition_for_source(source) - return self._memory_evidence[type(source)] - - def memory_evidence_for_manifest(self, name: str, version: str, fingerprint: str, /) -> MemoryEvidenceDeclaration: - """Return only deployment-configured evidence for an exact manifest.""" - - return self._remote_declarations.get((name, version, fingerprint), _NEUTRAL_MEMORY_EVIDENCE) - async def resolve(self, value: object, /) -> Source: input_class = type(value) try: @@ -248,18 +203,6 @@ def project(self, source: Source, key: SourceProjectionKey, /) -> JsonValue: raise InvalidSourceProjectionError(key.name, "result", "must match the declared output schema") from error -def _snapshot_memory_evidence(value: object) -> MemoryEvidenceDeclaration: - if value is None: - return _NEUTRAL_MEMORY_EVIDENCE - try: - # Revalidate even constructed models, and never retain caller-owned data. - if isinstance(value, MemoryEvidenceDeclaration): - value = value.model_dump(mode="python") - return MemoryEvidenceDeclaration.model_validate(value) - except (TypeError, ValueError): - return _NEUTRAL_MEMORY_EVIDENCE - - def _validate_definition(definition: object) -> tuple[type[object], type[Source]]: definition_type = type(definition) input_class = getattr(definition, "input_class", None) diff --git a/src/powercontext/sources/evidence.py b/src/powercontext/sources/evidence.py deleted file mode 100644 index 33eebfed9f..0000000000 --- a/src/powercontext/sources/evidence.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Typed declarations and exact material facts accepted at Source registration.""" - -from __future__ import annotations - -from typing import Literal - -from pydantic import BaseModel, ConfigDict, Field - -from powercontext.sources.models import SourceMaterialization, SourceRef - - -class MemoryEvidenceDeclaration(BaseModel): - """A trusted adapter's claim, not a score, permission or verification engine. - - ``declaration_version`` identifies this contract, independently of the - Definition version and Artifact content revision. Missing or unsupported - declarations resolve to ``untrusted`` / ``unknown``. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - authority: Literal["untrusted", "user_asserted", "repository_attested", "system_attested"] = "untrusted" - verification: Literal["unknown", "verified", "not_verified"] = "unknown" - declaration_version: Literal["1"] = "1" - - -class SourceEvidence(BaseModel): - """Retain one exact Source material binding without duplicating its body. - - The digest checks equality of canonical Source JSON, excluding only the - server-owned ContentSource handoff receipt witness. It authenticates neither - the Source nor its declaration. Each Source retains its own declaration. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - source_ref: SourceRef - materialization: SourceMaterialization - material_digest: str - definition_name: str - definition_version: str - manifest_fingerprint: str | None = None - declaration: MemoryEvidenceDeclaration = Field(default_factory=MemoryEvidenceDeclaration) - - -__all__ = ["MemoryEvidenceDeclaration", "SourceEvidence"] diff --git a/src/powercontext/sources/models.py b/src/powercontext/sources/models.py index 0f1a4a26fe..6e2cd48df6 100644 --- a/src/powercontext/sources/models.py +++ b/src/powercontext/sources/models.py @@ -60,8 +60,6 @@ def validate_key_part(cls, value: str, info) -> str: class Source(BaseModel): """Base value for an adapter-owned Source description.""" - model_config = ConfigDict(populate_by_name=True) - name: str definition_version: str = "1" materialization: SourceMaterialization diff --git a/src/powercontext/sources/observations.py b/src/powercontext/sources/observations.py index 596eff9b71..576e29fb79 100644 --- a/src/powercontext/sources/observations.py +++ b/src/powercontext/sources/observations.py @@ -17,20 +17,10 @@ from __future__ import annotations import hashlib -from typing import Any, Literal, get_args, get_origin +from typing import Any, Literal import rfc8785 -from pydantic import ( - AliasChoices, - AliasPath, - BaseModel, - ConfigDict, - Field, - JsonValue, - TypeAdapter, - field_validator, - model_validator, -) +from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, field_validator, model_validator from powercontext.errors import ( InvalidSourceDefinitionError, @@ -171,28 +161,9 @@ def projection(self, key: SourceProjectionKey, /) -> JsonValue: def manifest_for_definition(definition: SourceDefinition[Any, Any, Any], /) -> SourceDefinitionManifest: - """Build a declaration matching the canonical remote payload representation.""" - - return _manifest_for_definition(definition, transport=True) + """Build the immutable declaration transported by a remote worker.""" - -def _manifest_for_definition( - definition: SourceDefinition[Any, Any, Any], - /, - *, - transport: bool, -) -> SourceDefinitionManifest: - if transport and _validation_schema_matches_payload(definition.source_class): - source_schema = _json_object(definition.source_class.model_json_schema()) - elif transport: - source_schema = _json_object( - definition.source_class.model_json_schema( - mode="serialization", - by_alias=bool(definition.source_class.model_config.get("serialize_by_alias")), - ) - ) - else: - source_schema = _json_object(definition.source_class.model_json_schema(mode="serialization")) + source_schema = _json_object(definition.source_class.model_json_schema()) projections = tuple( SourceProjectionManifest( key=SourceProjectionKey(name=projection.name, version=projection.version), @@ -244,54 +215,6 @@ def project_source_for_transport( ) -def _validation_schema_matches_payload( - source_class: type[BaseModel], - seen: set[type[BaseModel]] | None = None, -) -> bool: - """Return whether validation-mode keys already describe the serialized payload.""" - - visited = set() if seen is None else seen - if source_class in visited: - return True - visited.add(source_class) - serialize_by_alias = bool(source_class.model_config.get("serialize_by_alias")) - for name, field in source_class.model_fields.items(): - if field.exclude: - continue - serialization_key = (field.serialization_alias or field.alias or name) if serialize_by_alias else name - if field.is_required() and _validation_schema_key(name, field) != serialization_key: - return False - for nested in _nested_models(field.annotation): - if not _validation_schema_matches_payload(nested, visited): - return False - return True - - -def _validation_schema_key(name: str, field: Any) -> str | None: - alias = field.validation_alias - if isinstance(alias, str): - return alias - if isinstance(alias, AliasChoices): - return next((choice for choice in alias.choices if isinstance(choice, str)), None) - if isinstance(alias, AliasPath): - return name # Pydantic keeps the field name for path-based validation. - if alias is not None: - return None - return field.alias if field.alias is not None else name - - -def _nested_models(annotation: object) -> tuple[type[BaseModel], ...]: - origin = get_origin(annotation) - if origin is None: - if isinstance(annotation, type) and issubclass(annotation, BaseModel): - return (annotation,) - return () - nested: list[type[BaseModel]] = [] - for argument in get_args(annotation): - nested.extend(_nested_models(argument)) - return tuple(nested) - - def _source_definition_fingerprint( *, name: str, diff --git a/tests/builtin/persistence/test_full_server_migrations.py b/tests/builtin/persistence/test_full_server_migrations.py index f88c31abc0..aa7a53a793 100644 --- a/tests/builtin/persistence/test_full_server_migrations.py +++ b/tests/builtin/persistence/test_full_server_migrations.py @@ -70,7 +70,7 @@ def dump(path): def test_complete_upgrade_preserves_history_backup_and_repeated_apply(source): before = dump(source.database) plan = source.plan() - assert plan.source_revision == "s1300" and plan.target_revision == "s1401" + assert plan.source_revision == "s1300" and plan.target_revision == "s1400" assert dump(source.database) == before result = apply(source) assert result.readiness_scope == "server" and result.backup_state == "completed" @@ -79,7 +79,6 @@ def test_complete_upgrade_preserves_history_backup_and_repeated_apply(source): assert connection.execute( "SELECT content,created_at,created_by,restored_from_revision,rollback_reason FROM pc_artifacts" ).fetchall() == [(b'{"history":"unchanged"}', None, None, None, None)] - assert connection.execute("SELECT COUNT(*) FROM pc_atomic_memory_evidence_history").fetchone() == (0,) after = dump(source.database) assert not apply(source).changed assert source.verify().readiness_scope == "server" diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index 71022d4a3c..945c6b78ce 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -299,10 +299,10 @@ def test_mysql_initialization_and_shared_evidence_noop(backend: _BackendTarget) assert result.state == "ready" assert result.changed assert result.backup_state == "not_required" - assert result.revision == "p0005" + assert result.revision == "p0004" replacement = backend.runner() assert replacement.plan().state == "ready" - assert replacement.verify().revision == "p0005" + assert replacement.verify().revision == "p0004" assert not replacement.apply().changed @@ -322,7 +322,7 @@ def test_mysql_operator_backup_choices_preserve_baseline_data_without_fork( result = _apply(runner, policy=policy) assert result.backup_state == ("user_confirmed" if policy == "manual" else "skipped") state = _read_state(backend) - assert state["revision"] == "p0005" + assert state["revision"] == "p0004" assert state["content"] == [(b'{"content":"preserved"}',)] assert state["tags"] == [("Keep",)] assert "memory_citations" in state["columns"]["pc_artifacts"] @@ -390,7 +390,7 @@ def test_mysql_committed_partial_ddl_resumes_from_shared_original_evidence(backe replacement = _acknowledge_known_completed_interruption(backend, recovery) assert _apply(replacement, policy="manual").state == "ready" state = _read_state(backend) - assert state["revision"] == "p0005" + assert state["revision"] == "p0004" assert state["content"] == [(b'{"content":"preserved"}',)] assert state["tags"] == [("Keep",)] assert replacement.verify().state == "ready" @@ -436,7 +436,7 @@ def test_mysql_empty_source_recovers_baseline_and_citation_ddl_without_probing_b ) result = _apply(replacement) assert result.backup_state == "not_required" - assert replacement.verify().revision == "p0005" + assert replacement.verify().revision == "p0004" assert _read_state(backend)["content"] == [] @@ -453,7 +453,7 @@ def test_mysql_baseline_adoption_recovers_an_empty_committed_version_table(backe assert recovered.adopt_baseline replacement = _acknowledge_known_completed_interruption(backend, recovered) assert _apply(replacement, policy="manual").state == "ready" - assert replacement.verify().revision == "p0005" + assert replacement.verify().revision == "p0004" assert _read_state(backend)["tags"] == [("Keep",)] @@ -479,7 +479,7 @@ def interrupt(journal, evidence) -> None: assert recovery.source_revision == "p0002" replacement = _acknowledge_known_completed_interruption(backend, recovery) assert _apply(replacement, policy="manual").state == "ready" - assert replacement.verify().revision == "p0005" + assert replacement.verify().revision == "p0004" assert _read_state(backend)["content"] == [(b'{"content":"preserved"}',)] @@ -488,7 +488,7 @@ def test_mysql_completed_head_with_an_interrupted_run_requires_exact_acknowledge pytest.skip("embedded seekdb closes its engine and has no remote in-flight run receipt") _load_baseline(backend) assert _apply(backend.runner(), policy="manual").state == "ready" - assert backend.runner().verify().revision == "p0005" + assert backend.runner().verify().revision == "p0004" def committed_write_then_interrupt( connection: Connection | None, _identity: BackendIdentity, verify: Callable[[], None] @@ -502,12 +502,12 @@ def committed_write_then_interrupt( with pytest.raises(_InterruptedDDL): backend.connections().run(committed_write_then_interrupt, writable=True) before = _read_state(backend) - assert before["revision"] == "p0005" + assert before["revision"] == "p0004" assert before["tags"] == [("Confirmed",)] replacement = backend.runner(backup_factory=_forbidden_automatic_backup) recovery = replacement.plan() assert recovery.state == "recovery_required" - assert recovery.source_revision == "p0005" + assert recovery.source_revision == "p0004" assert recovery.revisions == () pending = (recovery.coordination or {}).get("pending_run_id") assert pending @@ -526,6 +526,6 @@ def committed_write_then_interrupt( assert result.state == "ready" assert not result.changed assert _read_state(backend) == before - assert backend.runner().verify().revision == "p0005" + assert backend.runner().verify().revision == "p0004" assert not backend.runner().apply().changed assert "pending_run_id" not in (backend.runner().plan().coordination or {}) diff --git a/tests/builtin/persistence/test_mysql_migration_schema.py b/tests/builtin/persistence/test_mysql_migration_schema.py index 4ad9e45fdd..b938216016 100644 --- a/tests/builtin/persistence/test_mysql_migration_schema.py +++ b/tests/builtin/persistence/test_mysql_migration_schema.py @@ -128,7 +128,7 @@ def extended_bundle(tmp_path: Path) -> MigrationBundle: (directory / "versions/fixture_next.py").write_text( "from alembic import op\n" "revision = 'fixture_next'\n" - "down_revision = 'p0005'\n" + "down_revision = 'p0004'\n" f"def upgrade():\n op.execute({_NEXT_TABLE!r})\n", encoding="utf-8", ) @@ -136,7 +136,7 @@ def extended_bundle(tmp_path: Path) -> MigrationBundle: (directory / "mysql_wrappers/fixture_next.py").write_text( "from alembic import context\n" "revision = 'fixture_next'\n" - "down_revision = 'p0005'\n" + "down_revision = 'p0004'\n" "def upgrade():\n context.config.attributes['execute_revision'](revision)\n", encoding="utf-8", ) @@ -149,21 +149,21 @@ def test_registered_next_revision_uses_the_same_recognition_and_sqlite_executor( schema = MySQLSchema(extended_bundle) layouts = json.loads((extended_bundle.snapshot_directory / "mysql_layouts.json").read_text()) tables = _show_tables(citations=2, drop_family_check=True) - for revision in ("p0004", "p0005"): + for revision in ("p0004",): for statement in layouts["revisions"][revision]: tables[statement.split()[2]] = statement source = _ReflectedDatabase(tables) inventory = _inventory(source) - assert schema.identify(inventory) == "p0005" - recovery = schema.recovery(inventory, "p0005") + assert schema.identify(inventory) == "p0004" + recovery = schema.recovery(inventory, "p0004") assert recovery.revision == "fixture_next" assert recovery.pending_statements == (_NEXT_TABLE,) source.tables["pc_extension_marker"] = _NEXT_TABLE inventory = _inventory(source) assert schema.identify(inventory) == "fixture_next" schema.verify(inventory, "fixture_next") - assert schema.recovery(inventory, "p0005").completed_revision == "fixture_next" - assert extended_bundle.pending("p0003") == ("p0004", "p0005", "fixture_next") + assert schema.recovery(inventory, "p0004").completed_revision == "fixture_next" + assert extended_bundle.pending("p0003") == ("p0004", "fixture_next") source.tables["pc_schema_revision"] = ( "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, PRIMARY KEY (version_num))" ) @@ -212,11 +212,11 @@ def test_mysql_execution_registration_rejects_incomplete_history_before_opening_ del manifest["mysql_wrappers"] elif change == "predecessor": wrapper = extended_bundle.directory / "mysql_wrappers/fixture_next.py" - wrapper.write_text(wrapper.read_text().replace("'p0005'", "'p0004'"), encoding="utf-8") + wrapper.write_text(wrapper.read_text().replace("'p0004'", "'p0003'"), encoding="utf-8") elif change == "unbound": manifest["revision_resources"]["fixture_next"].remove("mysql_wrappers/fixture_next.py") else: - manifest["mysql_wrappers"]["p0005"] = "mysql_wrappers/fixture_next.py" + manifest["mysql_wrappers"]["p0004"] = "mysql_wrappers/fixture_next.py" path.write_text(json.dumps(manifest), encoding="utf-8") target = tmp_path / "absent-seekdb" evidence = tmp_path / "absent-evidence" diff --git a/tests/builtin/persistence/test_sources.py b/tests/builtin/persistence/test_sources.py index 86f86f4f64..22e080499f 100644 --- a/tests/builtin/persistence/test_sources.py +++ b/tests/builtin/persistence/test_sources.py @@ -15,33 +15,22 @@ from __future__ import annotations import asyncio -import json -from urllib.parse import quote, unquote import pytest -from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator -from pydantic.alias_generators import to_camel -from sqlalchemy import delete, insert, select, update +from sqlalchemy import insert from powercontext import InvalidSourceDefinitionError from powercontext.builtin.persistence import ( InvalidRepositoryArgumentError, - InvalidStoredPayloadError, RepositoryNotFoundError, StoredPayloadConflictError, ) -from powercontext.builtin.persistence.atomic_memory_evidence_schema import SOURCE_EVIDENCE_BINDINGS_TABLE from powercontext.builtin.persistence.codec import dump_model from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import SHARED_TABLES, SOURCES_TABLE -from powercontext.builtin.runtime import SubmitSourceObservation from powercontext.builtin.runtime.relational import RelationalContexts -from powercontext.builtin.sources.content import CONTENT_SOURCE_ADAPTER, ContentSource from powercontext.sources import ( - AdapterSourceDefinition, - MemoryEvidenceDeclaration, - Source, SourceDefinitionRegistry, SourceMaterialization, SourceObservation, @@ -57,49 +46,6 @@ ) -class SerializationAliasInput(BaseModel): - source_id: str - body: str - - -class SerializationAliasSource(Source): - body: str = Field(serialization_alias="wire_text") - - -class CamelCaseSource(SerializationAliasSource): - model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) - - -class UpperCaseSource(SerializationAliasSource): - model_config = ConfigDict(alias_generator=str.upper, populate_by_name=True) - - -class NativeCamelCaseSource(SerializationAliasSource): - model_config = ConfigDict(alias_generator=to_camel) - - body: str - - -class SerializationAliasAdapter: - name = "serialization-alias" - input_class = SerializationAliasInput - - def __init__(self, source_class: type[SerializationAliasSource]) -> None: - self.source_class = source_class - - async def resolve(self, value: SerializationAliasInput, /) -> SerializationAliasSource: - return self.source_class( - name=value.source_id, - definition_version="2", - materialization=SourceMaterialization.CAPTURED, - description="Remote alias description.", - body=value.body, - ) - - async def read(self, source: SerializationAliasSource, /) -> str: - return source.body - - def test_two_source_adapters_share_one_repository_and_journal() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): @@ -203,473 +149,6 @@ async def scenario() -> None: asyncio.run(scenario()) -@pytest.mark.parametrize("source_class", (SerializationAliasSource, CamelCaseSource, UpperCaseSource)) -def test_remote_observation_ingestion_accepts_serialization_aliases( - source_class: type[SerializationAliasSource], -) -> None: - async def scenario() -> None: - worker = SourceDefinitionRegistry(( - AdapterSourceDefinition(SerializationAliasAdapter(source_class), version="2"), - )) - source = await worker.resolve(SerializationAliasInput(source_id="remote-1", body="Remote note text.")) - manifest = manifest_for_definition(worker.definition_for_name("serialization-alias")) - observation = project_source_for_transport(worker, source) - - async with repository_profile() as (profile, _repositories): - contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) - await contexts.register_source_definition(manifest) - receipt = await contexts.submit_source_observation( - SubmitSourceObservation(scope_id="scope-a", observation=observation) - ) - context = await contexts.get("scope-a") - stored = await context.sources.get(observation) - - assert receipt.source_ref == SourceRef(source_type="serialization-alias", source_id="remote-1") - assert isinstance(stored, SourceObservation) - assert stored.payload == { - "name": "remote-1", - "definition_version": "2", - "materialization": "captured", - "description": "Remote alias description.", - "body": "Remote note text.", - } - - asyncio.run(scenario()) - - -def test_native_generated_alias_source_survives_read_replay_and_restart(tmp_path) -> None: - async def scenario() -> None: - adapter = SerializationAliasAdapter(NativeCamelCaseSource) - registry = SourceDefinitionRegistry((AdapterSourceDefinition(adapter, version="2"),)) - value = SerializationAliasInput(source_id="native-2", body="Keep the native version and body.") - source = await adapter.resolve(value) - # Keep the persisted version nondefault regardless of the constructor's alias rules. - source.definition_version = "2" - config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'native-alias.db'}") - - async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: - contexts = RelationalContexts(database=profile.database, source_registry=registry) - context = await contexts.get("scope-a") - assert await context.sources.add(source) == source - loaded = await context.sources.get(source) - assert loaded == source - assert await context.sources.read(loaded) == value.body - assert await context.sources.add(source) == source - assert await context.sources.journal.position(source) == 1 - - async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: - contexts = RelationalContexts(database=profile.database, source_registry=registry) - context = await contexts.get("scope-a") - loaded = await context.sources.get(source) - assert loaded == source - assert loaded.definition_version == "2" - resolved = await context.sources.resolve(value) - assert resolved == source - assert await context.sources.add(resolved) == source - assert await context.sources.journal.position(resolved) == 1 - - asyncio.run(scenario()) - - -@pytest.mark.parametrize("current_version", ["1", "2"]) -def test_canonical_history_survives_current_identity_aliases(current_version) -> None: - class CurrentSource(SerializationAliasSource): - name: str = Field(alias="source_name") - label: str = Field(alias="name") - - async def scenario() -> None: - original = SourceRepository((SerializationAliasAdapter(SerializationAliasSource),)) - source = SerializationAliasSource( - name="canonical", materialization=SourceMaterialization.CAPTURED, body="Retained." - ) - material = source.model_dump(mode="json", by_alias=True) - payload = json.dumps({ - "encoding": "powercontext-source-v1", - "representation": "native", - "value": material, - }).encode() - async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: - stored = await original.add(connection, "scope-a", source) - await connection.execute( - update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) - ) - await connection.execute(delete(SOURCE_EVIDENCE_BINDINGS_TABLE)) - expected = await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) - current = SourceRepository( - SourceDefinitionRegistry(( - AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version=current_version), - )) - ) - assert await current.historical_evidence(connection, "scope-a", (stored.ref,)) == expected - assert expected[0].definition_version == "1" - assert expected[0].declaration == MemoryEvidenceDeclaration() - - asyncio.run(scenario()) - - -@pytest.mark.parametrize("has_header", [False, True]) -def test_business_name_alias_does_not_reinterpret_canonical_history(has_header: bool) -> None: - class CurrentSource(SerializationAliasSource): - label: str = Field(alias="name") - - async def scenario() -> None: - original = SourceRepository((SerializationAliasAdapter(SerializationAliasSource),)) - current = SourceRepository( - SourceDefinitionRegistry((AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version="1"),)) - ) - source = SerializationAliasSource( - name="canonical-business-alias", - materialization=SourceMaterialization.CAPTURED, - body="Retained.", - ) - material = source.model_dump(mode="json", by_alias=True) - value = {"encoding": "powercontext-source-v1", "representation": "native", "value": material} - if has_header: - value.update( - source=Source( - name=source.name, - definition_version=source.definition_version, - materialization=source.materialization, - ).model_dump(mode="json", exclude_none=True), - source_keys={ - "name": "name", - "definition_version": "definition_version", - "materialization": "materialization", - }, - ) - async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: - stored = await original.add(connection, "scope-a", source) - await connection.execute( - update(SOURCES_TABLE) - .where(SOURCES_TABLE.c.source_id == source.name) - .values(payload=json.dumps(value).encode()) - ) - await connection.execute(delete(SOURCE_EVIDENCE_BINDINGS_TABLE)) - expected = await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) - assert await current.historical_evidence(connection, "scope-a", (stored.ref,)) == expected - - asyncio.run(scenario()) - - -@pytest.mark.parametrize("business_version", ["1", "2"]) -def test_bound_aliased_history_does_not_treat_business_fields_as_identity(business_version) -> None: - class AliasedSource(NativeCamelCaseSource): - definition_version: str = Field(alias="source_version") - materialization: SourceMaterialization = Field(alias="source_materialization") - business_version: str = Field(alias="definition_version") - business_kind: str = Field(alias="materialization") - - async def scenario() -> None: - declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") - registry = SourceDefinitionRegistry(( - AdapterSourceDefinition(SerializationAliasAdapter(AliasedSource), version="2", memory_evidence=declaration), - )) - repository = SourceRepository(registry) - source = AliasedSource( - name="aliased-history", - source_version="2", - source_materialization=SourceMaterialization.CAPTURED, - definition_version=business_version, - materialization="referenced", - body="Retained.", - ) - async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: - stored = await repository.add(connection, "scope-a", source) - expected = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) - payload = json.dumps({ - "encoding": "powercontext-source-v1", - "representation": "native", - "value": source.model_dump(mode="json", by_alias=True), - }).encode() - await connection.execute( - update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) - ) - assert await repository.historical_evidence(connection, "scope-a", (stored.ref,)) == expected - assert expected[0].definition_version == "2" - assert expected[0].materialization is SourceMaterialization.CAPTURED - assert expected[0].declaration == declaration - - asyncio.run(scenario()) - - -def test_unbound_versioned_canonical_history_ignores_new_definition_aliases() -> None: - class CurrentSource(SerializationAliasSource): - model_config = ConfigDict(populate_by_name=True) - name: str = Field(alias="source_name") - label: str = Field(alias="name") - - async def scenario() -> None: - old_source = SerializationAliasSource( - name="old-version-two", - definition_version="2", - materialization=SourceMaterialization.CAPTURED, - body="Retained without an evidence binding.", - ) - old_repository = SourceRepository(()) - - class CurrentAdapter: - name = "serialization-alias" - input_class = SerializationAliasInput - source_class = CurrentSource - - async def resolve(self, value: SerializationAliasInput, /) -> CurrentSource: - raise AssertionError # Historical evidence must not invoke the current decoder. - - async def read(self, source: CurrentSource, /) -> str: - return source.body - - registry = SourceDefinitionRegistry((AdapterSourceDefinition(CurrentAdapter(), version="3"),)) - async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: - # Model a canonical Source accepted before evidence bindings existed. - await connection.execute( - insert(SOURCES_TABLE).values( - scope_id="scope-a", - source_type="serialization-alias", - source_id=old_source.name, - payload=json.dumps({ - "encoding": "powercontext-source-v1", - "representation": "native", - "value": old_source.model_dump(mode="json", by_alias=False), - }).encode(), - journal_position=1, - ) - ) - ref = SourceRef(source_type="serialization-alias", source_id=old_source.name) - baseline = await old_repository.historical_evidence(connection, "scope-a", (ref,)) - upgraded = await SourceRepository(registry).historical_evidence(connection, "scope-a", (ref,)) - assert upgraded == baseline - assert upgraded[0].definition_version == "2" - assert upgraded[0].declaration == MemoryEvidenceDeclaration() - - asyncio.run(scenario()) - - -def test_headerless_content_receipt_preserves_accepted_material_digest() -> None: - async def scenario() -> None: - repository = SourceRepository((CONTENT_SOURCE_ADAPTER,)) - source = ContentSource( - name="receipt-digest", - materialization=SourceMaterialization.CAPTURED, - content="Receipt content.", - handoff_receipt=True, - ) - async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: - stored = await repository.add(connection, "scope-a", source) - original = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) - material = source.model_dump(mode="json", by_alias=True) - payload = json.dumps({ - "encoding": "powercontext-source-v1", - "representation": "native", - "value": material, - }).encode() - await connection.execute( - update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) - ) - restored = await repository.historical_evidence(connection, "scope-a", (stored.ref,)) - assert restored == original - - asyncio.run(scenario()) - - -def test_precanonical_generated_alias_source_rows_remain_readable_and_replayable() -> None: - async def scenario() -> None: - registry = SourceDefinitionRegistry(( - AdapterSourceDefinition(SerializationAliasAdapter(NativeCamelCaseSource), version="2"), - )) - repository = SourceRepository(registry) - source = NativeCamelCaseSource( - name="legacy-2", definition_version="2", materialization=SourceMaterialization.CAPTURED, body="Retained." - ) - material = source.model_dump(mode="json", by_alias=True) - payload = json.dumps({ - "encoding": "powercontext-source-v1", - "representation": "native", - "value": material, - }).encode() - async with ( - repository_profile() as (profile, _repositories), - profile.database.transaction() as connection, - ): - await connection.execute( - insert(SOURCES_TABLE).values( - scope_id="scope-a", - source_type="serialization-alias", - source_id=source.name, - payload=payload, - journal_position=1, - ) - ) - ref = SourceRef(source_type="serialization-alias", source_id=source.name) - with pytest.raises(InvalidStoredPayloadError): - await SourceRepository(()).historical_evidence(connection, "scope-a", (ref,)) - loaded = await repository.get(connection, "scope-a", ref) - assert loaded.value == source - assert await repository.add(connection, "scope-a", source) == loaded - assert await repository.journal_position(connection, "scope-a") == 1 - - asyncio.run(scenario()) - - -def test_excluded_native_identity_is_rejected_before_capture() -> None: - class ExcludedVersionSource(NativeCamelCaseSource): - definition_version: str = Field(default="1", exclude=True) - - async def scenario() -> None: - repository = SourceRepository((SerializationAliasAdapter(ExcludedVersionSource),)) - source = ExcludedVersionSource( - name="excluded", materialization=SourceMaterialization.CAPTURED, body="Retained." - ) - async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: - with pytest.raises(InvalidStoredPayloadError): - await repository.add(connection, "scope-a", source) - ref = SourceRef(source_type="serialization-alias", source_id=source.name) - with pytest.raises(RepositoryNotFoundError): - await repository.get(connection, "scope-a", ref) - assert await repository.journal_position(connection, "scope-a") == 0 - - asyncio.run(scenario()) - - -@pytest.mark.parametrize("identity_field", ["name", "definition_version", "materialization"]) -def test_encoded_native_identity_is_rejected_before_capture(identity_field) -> None: - class EncodedSource(NativeCamelCaseSource): - @field_serializer("name", "definition_version", "materialization") - def encode_identity(self, value, info): - if info.field_name == identity_field: - return quote(f"wire:{value}", safe="") - return value - - @field_validator("name", "definition_version", "materialization", mode="before") - @classmethod - def decode_identity(cls, value, info): - if info.field_name == identity_field: - return unquote(value).removeprefix("wire:") - return value - - async def scenario() -> None: - repository = SourceRepository((SerializationAliasAdapter(EncodedSource),)) - source = EncodedSource(name="note/42", materialization=SourceMaterialization.CAPTURED, body="Retained.") - assert EncodedSource.model_validate_json(source.model_dump_json(by_alias=True)) == source - async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: - with pytest.raises(InvalidStoredPayloadError): - await repository.add(connection, "scope-a", source) - ref = SourceRef(source_type="serialization-alias", source_id=source.name) - with pytest.raises(RepositoryNotFoundError): - await repository.get(connection, "scope-a", ref) - assert await repository.journal_position(connection, "scope-a") == 0 - - asyncio.run(scenario()) - - -def test_explicit_name_alias_evidence_survives_restart_without_the_native_definition(tmp_path) -> None: - class AliasedSource(NativeCamelCaseSource): - name: str = Field(alias="source_name") - business_name: str = Field(alias="name") - description: str | None = Field(default="captured description", exclude=True) - - class UpdatedSource(AliasedSource): - description: str | None = Field(default="current description", exclude=True) - - async def scenario() -> None: - declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") - registry = SourceDefinitionRegistry(( - AdapterSourceDefinition( - SerializationAliasAdapter(AliasedSource), - version="2", - memory_evidence=declaration, - ), - )) - source = AliasedSource( - source_name="native-alias", - name="business label", - definition_version="2", - materialization=SourceMaterialization.CAPTURED, - body="Retained.", - ) - config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'explicit-alias.db'}") - repository = SourceRepository(registry) - async with ( - SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, - profile.database.transaction() as connection, - ): - stored = await repository.add(connection, "scope-a", source) - assert (await repository.get(connection, "scope-a", stored.ref)).value == source - assert await repository.add(connection, "scope-a", source) == stored - evidence = await repository.evidence(connection, "scope-a", (stored.ref,)) - assert evidence[0].declaration == declaration - async with ( - SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, - profile.database.transaction() as connection, - ): - updated = SourceDefinitionRegistry(( - AdapterSourceDefinition(SerializationAliasAdapter(UpdatedSource), version="2"), - )) - loaded = await SourceRepository(updated).get(connection, "scope-a", stored.ref) - assert loaded.value.description == "current description" - assert await SourceRepository(()).historical_evidence(connection, "scope-a", (stored.ref,)) == evidence - - asyncio.run(scenario()) - - -@pytest.mark.parametrize("representation", ["enveloped", "bare"]) -def test_replay_retains_historical_native_identity_spelling(representation) -> None: - from pydantic import AliasChoices - - class OldSource(NativeCamelCaseSource): - name: str = Field(alias="legacy_name") - - class CurrentSource(NativeCamelCaseSource): - name: str = Field( - validation_alias=AliasChoices("source_name", "legacy_name"), - serialization_alias="source_name", - ) - - async def scenario() -> None: - original_registry = SourceDefinitionRegistry(( - AdapterSourceDefinition(SerializationAliasAdapter(OldSource), version="2"), - )) - current_registry = SourceDefinitionRegistry(( - AdapterSourceDefinition(SerializationAliasAdapter(CurrentSource), version="2"), - )) - original = SourceRepository(original_registry) - current = SourceRepository(current_registry) - source = OldSource( - legacy_name="legacy-replay", - definition_version="2", - materialization=SourceMaterialization.CAPTURED, - body="Retained exactly.", - ) - async with repository_profile() as (profile, _repositories), profile.database.transaction() as connection: - stored = await original.add(connection, "scope-a", source) - material = source.model_dump(mode="json", by_alias=True) - if representation == "bare": - payload = json.dumps(material).encode() - else: - payload = json.dumps({ - "encoding": "powercontext-source-v1", - "representation": "native", - "value": material, - }).encode() - await connection.execute( - update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) - ) - loaded = await current.get(connection, "scope-a", stored.ref) - assert loaded.value.name == source.name - replay = await current.add(connection, "scope-a", loaded.value) - assert replay.ref == stored.ref - assert await current.get(connection, "scope-a", stored.ref) == loaded - evidence = await current.historical_evidence(connection, "scope-a", (stored.ref,)) - assert evidence[0].source_ref == stored.ref - raw = await connection.scalar( - select(SOURCES_TABLE.c.payload).where(SOURCES_TABLE.c.source_id == source.name) - ) - assert isinstance(raw, bytes) - envelope = json.loads(raw) - assert envelope["source_keys"]["name"] == "legacy_name" - - asyncio.run(scenario()) - - def test_repository_rejects_a_scope_outside_the_relational_identity_baseline() -> None: async def scenario() -> None: async with repository_profile() as (profile, repositories): @@ -832,45 +311,3 @@ async def add(scope_id: str, source: NoteSource): assert following.journal_position == 2 asyncio.run(scenario()) - - -def test_nested_uppercase_source_aliases_survive_read_replay_and_restart(tmp_path) -> None: - class UpperNested(Source): - model_config = ConfigDict(alias_generator=str.upper) - body: str - - class NestedNativeSource(NativeCamelCaseSource): - nested: UpperNested - - async def scenario() -> None: - adapter = SerializationAliasAdapter(NestedNativeSource) - registry = SourceDefinitionRegistry((AdapterSourceDefinition(adapter, version="2"),)) - repository = SourceRepository(registry) - source = NestedNativeSource( - name="nested-legacy", - definition_version="2", - materialization=SourceMaterialization.CAPTURED, - body="Outer.", - nested=UpperNested( - name="inner", - materialization=SourceMaterialization.REFERENCED, - body="Nested.", - ), - ) - config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'nested-alias.db'}") - async with ( - SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, - profile.database.transaction() as connection, - ): - original = await repository.add(connection, "scope-a", source) - async with ( - SQLiteProfile.open(config, tables=SHARED_TABLES) as profile, - profile.database.transaction() as connection, - ): - loaded = await repository.get(connection, "scope-a", original.ref) - assert loaded.value == source - assert await repository.add(connection, "scope-a", source) == original - evidence = await repository.historical_evidence(connection, "scope-a", (original.ref,)) - assert evidence[0].source_ref == original.ref - - asyncio.run(scenario()) diff --git a/tests/e2e/test_atomic_memory_evidence_neutral_recall.py b/tests/e2e/test_atomic_memory_evidence_neutral_recall.py deleted file mode 100644 index c3ad6ee50b..0000000000 --- a/tests/e2e/test_atomic_memory_evidence_neutral_recall.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Accepted Source declarations do not alter recall or consumer revision citations.""" - -from __future__ import annotations - -import asyncio -import json -from dataclasses import replace -from typing import cast - -import httpx -import pytest - -from powercontext.builtin.artifacts.handoff import PrepareHandoff -from powercontext.builtin.runtime import ( - BuiltinConfig, - HandoffArtifactCitation, - PrepareContextRequest, - RuntimeConfig, - open_builtin_contexts, - open_builtin_runtime, -) -from powercontext.builtin.scope import ScopeDraft -from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION -from powercontext.server.app import ServerApplication, create_app -from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceRef -from powercontext.sources.evidence import MemoryEvidenceDeclaration -from tests.builtin.persistence.contract import NoteAdapter, NoteInput -from tests.e2e.atomic_memory_migration_backend import MigrationBackend -from tests.e2e.test_atomic_memory_source_evidence import _commit -from tests.e2e.test_handoff_runtime import _EchoHandoffPipeline - -pytestmark = pytest.mark.parametrize("migration_backend", ["sqlite", "oceanbase"], indirect=True) - - -def test_source_declarations_preserve_recall_and_consumers(migration_backend: MigrationBackend) -> None: - async def scenario() -> None: - config = BuiltinConfig( - database=migration_backend.config, - runtime=RuntimeConfig(artifact_processing_families=(), atomic_memory_related_mode="fts"), - ) - arms = [] - neutral = MemoryEvidenceDeclaration() - candidates = ( - (SourceRef(source_type="note", source_id="note"), "memory-a", "alpha acceptance"), - (SourceRef(source_type="content", source_id="source"), "memory-b", "alpha guidance"), - ) - for declaration in ( - neutral, - MemoryEvidenceDeclaration(authority="system_attested", verification="verified"), - ): - registry = SourceDefinitionRegistry(( - replace(CONTENT_SOURCE_DEFINITION, memory_evidence=declaration), - AdapterSourceDefinition(NoteAdapter()), - )) - async with open_builtin_runtime( - config, - scheduler_path=migration_backend.directory / "consumer-scheduler.db", - source_registry=registry, - handoff_pipeline=_EchoHandoffPipeline(), - ) as runtime: - assert runtime.scopes is not None and runtime.atomic_memory is not None - scope = await runtime.scopes.create( - ScopeDraft( - title="Recall", - summary="Evidence neutrality.", - idempotency_key=declaration.authority, - ) - ) - scope_id = scope.scope_id - async with open_builtin_contexts(config, source_registry=registry) as contexts: - await contexts.records.capture_source(scope_id, "content", "source", "alpha guidance", {}) - context = await contexts.get(scope_id) - note = await context.sources.resolve(NoteInput(note_id="note", body="alpha acceptance")) - await context.sources.add(note) - memories = [] - for source, artifact_id, text in candidates: - memory = await _commit(contexts, scope_id, artifact_id, text, sources=(source,)) - annotation = await contexts.atomic_memory.for_scope(scope_id).evidence(artifact_id, revision=1) - assert annotation.artifact_ref == memory.ref - expected = declaration if source.source_type == "content" else neutral - assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( - (source, expected), - ) - memories.append(memory) - page = await runtime.atomic_memory.for_scope(scope_id).search("alpha", mode="text", limit=2) - assert tuple((hit.hit.artifact_ref, hit.text, hit.matched_by) for hit in page.hits) == tuple( - (memory.ref, text, ("text",)) for memory, (_, _, text) in zip(memories, candidates, strict=True) - ) - recalled = tuple((hit.hit.artifact_ref.model_dump(mode="json"), hit.text) for hit in page.hits) - app = create_app(application=cast(ServerApplication, runtime)) - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app), base_url="http://testserver" - ) as client: - response = await client.post("/v1/memory/search", json={"scope_id": scope_id, "query": "alpha"}) - assert response.status_code == 200, response.text - public = response.json() - assert ( - tuple((item["memory"]["artifact"], item["memory"]["text"]) for item in public["hits"]) == recalled - ) - prepared = await runtime.context.for_scope(scope_id).prepare(PrepareContextRequest(query="alpha")) - assert prepared.status == "ready" and prepared.content is not None - items = json.loads(prepared.content.splitlines()[-2])["items"] - assert tuple((item["citation"]["artifact_ref"], item["content"]) for item in items) == recalled - citation = HandoffArtifactCitation(artifact_ref=page.hits[-1].hit.artifact_ref) - handoffs = runtime.handoff.for_scope(scope_id) - finalized = await handoffs.finalize( - await handoffs.prepare( - PrepareHandoff(objective="Keep the recalled revision.", evidence=(citation,)) - ) - ) - assert finalized.content.state[0].citations == (citation,) - assert finalized.content.next_action is not None - assert finalized.content.next_action.citations == (citation,) - arms.append((page.hits, public, prepared.content.replace(scope_id, "scope"), finalized.content)) - assert arms[0] == arms[1] - - asyncio.run(scenario()) diff --git a/tests/e2e/test_atomic_memory_projection_rebuild.py b/tests/e2e/test_atomic_memory_projection_rebuild.py index 5263ff1a0c..3075699659 100644 --- a/tests/e2e/test_atomic_memory_projection_rebuild.py +++ b/tests/e2e/test_atomic_memory_projection_rebuild.py @@ -15,7 +15,6 @@ from __future__ import annotations import asyncio -from dataclasses import replace from pathlib import Path import pytest @@ -30,9 +29,6 @@ from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection -from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION -from powercontext.sources import SourceDefinitionRegistry -from powercontext.sources.evidence import MemoryEvidenceDeclaration from tests.e2e.atomic_memory_migration_fixture import SCOPE, _atomic, _seed_and_migrate from tests.legacy_memory import add_legacy_citation_columns @@ -40,12 +36,6 @@ @pytest.mark.parametrize("retention", ["complete", "archive_only", "entries_only", "removed", "unreadable"]) def test_rebuild_uses_current_authority_independently_of_legacy_retention(tmp_path: Path, retention: str) -> None: config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'projection.db'}") - strong_registry = SourceDefinitionRegistry(( - replace( - CONTENT_SOURCE_DEFINITION, - memory_evidence=MemoryEvidenceDeclaration(authority="system_attested", verification="verified"), - ), - )) async def scenario() -> None: await _seed_and_migrate(config) @@ -53,8 +43,6 @@ async def scenario() -> None: memory = contexts.atomic_memory.for_scope(SCOPE) current = await memory.get(_atomic("alpha", 2).artifact_id) historical = await memory.get(_atomic("alpha", 1).artifact_id, revision=1) - annotation = await memory.evidence(historical.ref.artifact_id, revision=1) - assert all(source.declaration == MemoryEvidenceDeclaration() for source in annotation.sources) async with SQLiteProfile.open(config, tables=()) as profile: async with profile.database.transaction() as connection: if retention in {"entries_only", "removed"}: @@ -75,13 +63,10 @@ async def scenario() -> None: ) assert report.ready, report.errors assert report.counts["rebuilt_rows"] == 2 - assert report.counts["historical_revisions"] == 3 - assert report.counts["neutral_sources"] == 4 - async with open_builtin_contexts(BuiltinConfig(database=config), source_registry=strong_registry) as contexts: + async with open_builtin_contexts(BuiltinConfig(database=config)) as contexts: memory = contexts.atomic_memory.for_scope(SCOPE) assert await memory.get(current.ref.artifact_id) == current assert await memory.get(historical.ref.artifact_id, revision=1) == historical - assert await memory.evidence(historical.ref.artifact_id, revision=1) == annotation result = await memory.search("Tasks A and C", mode="text") assert current.ref in tuple(hit.hit.artifact_ref for hit in result.hits) diff --git a/tests/e2e/test_atomic_memory_source_evidence.py b/tests/e2e/test_atomic_memory_source_evidence.py deleted file mode 100644 index 5d674241d1..0000000000 --- a/tests/e2e/test_atomic_memory_source_evidence.py +++ /dev/null @@ -1,416 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Versioned source evidence acceptance on disposable SQLite and OceanBase.""" - -from __future__ import annotations - -import asyncio -import json -from dataclasses import replace -from typing import cast - -import pytest -from pydantic import Field, field_serializer, field_validator -from sqlalchemy import event, text, update -from sqlalchemy.exc import SQLAlchemyError - -from powercontext.artifacts import ArtifactLineage -from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent -from powercontext.builtin.artifacts.atomic_memory.extraction import ( - AtomicMemoryCandidate, - AtomicMemoryExtractionOutput, -) -from powercontext.builtin.inference import GenerationResult -from powercontext.builtin.persistence.tables import SOURCES_TABLE -from powercontext.builtin.runtime import ( - BuiltinConfig, - SubmitSourceObservation, - open_builtin_contexts, - open_builtin_runtime, -) -from powercontext.builtin.runtime.atomic_memory_rebuild import rebuild_atomic_memory_projection -from powercontext.builtin.runtime.relational import RelationalContexts -from powercontext.builtin.scope import ScopeDraft -from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION -from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry, SourceMaterialization, SourceRef -from powercontext.sources.evidence import MemoryEvidenceDeclaration -from powercontext.sources.observations import manifest_for_definition, project_source_for_transport -from tests.builtin.persistence.contract import NoteAdapter, NoteInput, NoteSource -from tests.e2e.atomic_memory_migration_backend import MigrationBackend -from tests.e2e.dream_support import atomic_memory_pipeline - -pytestmark = pytest.mark.parametrize("migration_backend", ["sqlite", "oceanbase"], indirect=True) - - -async def _commit(contexts, scope_id, artifact_id, text_value, *, sources=(), artifacts=(), expected_revision=None): - application = contexts.atomic_memory - async with contexts.database.transaction() as connection: - plan = await application.service.inspect_change( - connection, - scope_id, - artifact_id, - AtomicMemoryContent(kind="fact", text=text_value), - None, - expected_revision=expected_revision, - lineage=ArtifactLineage(sources=sources, artifacts=artifacts), - ) - prepared = await application.service.prepare_change(plan) - async with contexts.database.transaction() as connection: - result = await application.service.commit(connection, prepared, None) - return result.primary - - -@pytest.mark.parametrize("declared", [False, True]) -def test_unverifiable_native_serializer_keeps_memory_evidence_neutral(migration_backend, declared) -> None: - class SerializedNote(NoteSource): - count: int = 12 - - @field_serializer("count") - def serialize_count(self, value): - return str(value) - - @field_validator("count", mode="before") - @classmethod - def parse_count(cls, value): - return int(value) - - class SerializedAdapter(NoteAdapter): - source_class = SerializedNote - - declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") - registry = SourceDefinitionRegistry(( - AdapterSourceDefinition(SerializedAdapter(), memory_evidence=declaration if declared else None), - )) - config = BuiltinConfig(database=migration_backend.config) - - async def scenario(): - async with open_builtin_contexts(config, source_registry=registry) as contexts: - context = await contexts.get("scope") - source = SerializedNote(name="serialized", materialization=SourceMaterialization.CAPTURED, body="Retained.") - await context.sources.add(source) - assert await context.sources.get(source) == source - assert await context.sources.add(source) == source - ref = SourceRef(source_type="note", source_id=source.name) - record = await _commit(contexts, "scope", "serialized", "Retained fact.", sources=(ref,)) - evidence = await contexts.atomic_memory.for_scope("scope").evidence("serialized", revision=1) - assert evidence.artifact_ref == record.ref - assert evidence.sources[0].source_ref == ref - assert evidence.sources[0].declaration == MemoryEvidenceDeclaration() - assert evidence.sources[0].manifest_fingerprint is None - if declared: - # Retain the binding shape accepted before schema compatibility checks. - async with contexts.database.transaction() as connection: - await connection.execute( - text( - "UPDATE pc_source_evidence_bindings SET manifest_fingerprint = " - "(SELECT fingerprint FROM pc_source_definition_manifests WHERE definition_name = 'note'), " - "declaration = :declaration WHERE scope_id = 'scope' AND source_id = 'serialized'" - ), - {"declaration": declaration.model_dump_json().encode()}, - ) - await _commit(contexts, "scope", "legacy", "Previously accepted fact.", sources=(ref,)) - legacy = await contexts.atomic_memory.for_scope("scope").evidence("legacy", revision=1) - assert legacy.sources == evidence.sources - async with open_builtin_contexts(config, source_registry=SourceDefinitionRegistry(())) as contexts: - report = await rebuild_atomic_memory_projection( - contexts.database, - contexts.atomic_memory.index, - maintenance_confirmed=True, - sources=contexts.repositories.sources, - ) - assert report.ready, report.errors - assert await contexts.atomic_memory.for_scope("scope").evidence("serialized", revision=1) == evidence - - asyncio.run(scenario()) - - -@pytest.mark.parametrize("has_header", [False, True]) -@pytest.mark.parametrize("historical_version", ["1", "2"]) -def test_canonical_historical_evidence_survives_business_alias_rebuild( - migration_backend: MigrationBackend, has_header: bool, historical_version: str -) -> None: - class CurrentNote(NoteSource): - name: str = Field(alias="source_name") - label: str = Field(alias="name") - - class CurrentAdapter(NoteAdapter): - source_class = CurrentNote - - config = BuiltinConfig(database=migration_backend.config) - old_registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter(), version=historical_version),)) - current_registry = SourceDefinitionRegistry(( - AdapterSourceDefinition(CurrentAdapter(), version=str(int(historical_version) + 1)), - )) - - async def scenario(): - source = NoteSource( - name=f"canonical-v{historical_version}", - definition_version=historical_version, - materialization=SourceMaterialization.CAPTURED, - body="Retained.", - ) - ref = SourceRef(source_type="note", source_id=source.name) - async with open_builtin_contexts(config, source_registry=old_registry) as contexts: - context = await contexts.get("scope") - await context.sources.add(source) - await _commit(contexts, "scope", "canonical", "Canonical historical fact.", sources=(ref,)) - material = source.model_dump(mode="json", by_alias=True) - payload_value = { - "encoding": "powercontext-source-v1", - "representation": "native", - "value": material, - } - if has_header: - payload_value.update( - source=source.model_dump( - mode="json", - by_alias=False, - include={"name", "definition_version", "materialization"}, - ), - source_keys={ - "name": "name", - "definition_version": "definition_version", - "materialization": "materialization", - }, - ) - payload = json.dumps(payload_value).encode() - async with contexts.database.transaction() as connection: - await connection.execute( - update(SOURCES_TABLE).where(SOURCES_TABLE.c.source_id == source.name).values(payload=payload) - ) - - async with open_builtin_contexts(config, source_registry=current_registry) as contexts: - before = await contexts.atomic_memory.for_scope("scope").evidence("canonical", revision=1) - assert before.sources[0].source_ref == ref - assert before.sources[0].declaration == MemoryEvidenceDeclaration() - report = await rebuild_atomic_memory_projection( - contexts.database, - contexts.atomic_memory.index, - maintenance_confirmed=True, - sources=contexts.repositories.sources, - ) - assert report.ready, report.errors - after = await contexts.atomic_memory.for_scope("scope").evidence("canonical", revision=1) - assert after == before - - asyncio.run(scenario()) - - -def test_mixed_evidence_survives_failed_revision_restart_and_rebuild(migration_backend: MigrationBackend) -> None: - strong = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") - asserted = MemoryEvidenceDeclaration(authority="user_asserted", verification="not_verified") - remote = MemoryEvidenceDeclaration(authority="system_attested", verification="verified") - - class RemoteNoteAdapter(NoteAdapter): - name = "remote-note" - - worker = SourceDefinitionRegistry((AdapterSourceDefinition(RemoteNoteAdapter(), memory_evidence=remote),)) - manifest = manifest_for_definition(worker.definition_for_name("remote-note")) - registry = SourceDefinitionRegistry( - ( - replace(CONTENT_SOURCE_DEFINITION, memory_evidence=strong), - AdapterSourceDefinition(NoteAdapter(), memory_evidence=asserted), - ), - remote_declarations={(manifest.name, manifest.version, manifest.fingerprint): remote}, - ) - config = BuiltinConfig(database=migration_backend.config) - refs = ( - SourceRef(source_type="content", source_id="cafe\u0301"), - SourceRef(source_type="note", source_id="native"), - SourceRef(source_type="remote-note", source_id="remote"), - ) - - async def scenario() -> None: - async with open_builtin_contexts(config, source_registry=registry) as contexts: - context = await contexts.get("scope") - await contexts.records.capture_source( - "scope", "content", "cafe\u0301", "Release received.", {"authority": "untrusted"} - ) - native = await context.sources.resolve(NoteInput(note_id="native", body="User correction.")) - await context.sources.add(native) - source = await worker.resolve(NoteInput(note_id="remote", body="Remote release evidence.")) - await contexts.register_source_definition(manifest) - await contexts.submit_source_observation( - SubmitSourceObservation(scope_id="scope", observation=project_source_for_transport(worker, source)) - ) - original = await _commit(contexts, "scope", "rule", "Release rule one.", sources=refs) - memory = contexts.atomic_memory.for_scope("scope") - annotation = await memory.evidence("rule", revision=1) - assert annotation.artifact_ref == original.ref - assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( - (refs[0], strong), - (refs[1], asserted), - (refs[2], remote), - ) - - def reject_snapshot(connection, cursor, statement, *args): - if statement.lstrip().upper().startswith("INSERT INTO PC_ATOMIC_MEMORY_EVIDENCE_HISTORY"): - raise SQLAlchemyError(statement) - - engine = contexts.database.engine.sync_engine - event.listen(engine, "before_cursor_execute", reject_snapshot) - try: - with pytest.raises(SQLAlchemyError): - await _commit(contexts, "scope", "rule", "Failed revision.", expected_revision=1) - finally: - event.remove(engine, "before_cursor_execute", reject_snapshot) - assert await memory.get("rule") == original - revised = await _commit(contexts, "scope", "rule", "Rule two.", sources=(refs[2],), expected_revision=1) - current = await memory.evidence("rule", revision=2) - assert current.sources == (annotation.sources[2],) - neutral_registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION, AdapterSourceDefinition(NoteAdapter()))) - async with open_builtin_contexts(config, source_registry=neutral_registry) as contexts: - await contexts.records.capture_source( - "other", refs[0].source_type, refs[0].source_id, "Other Scope release.", {} - ) - other = await _commit(contexts, "other", "rule", "Other Scope rule.", sources=(refs[0],)) - other_annotation = await contexts.atomic_memory.for_scope("other").evidence("rule", revision=1) - assert other_annotation.scope_id == "other" - assert tuple((item.source_ref, item.declaration) for item in other_annotation.sources) == ( - (refs[0], MemoryEvidenceDeclaration()), - ) - assert other_annotation.sources[0].material_digest != annotation.sources[0].material_digest - - async def assert_retained(contexts, historical): - memory = contexts.atomic_memory.for_scope("scope") - assert await memory.evidence("rule", revision=1) == historical - assert await memory.evidence("rule", revision=2) == current - assert (await memory.get("rule", revision=1)).artifact == original.artifact - assert (await memory.get("rule")).artifact == revised.artifact - other_memory = contexts.atomic_memory.for_scope("other") - assert await other_memory.evidence("rule", revision=1) == other_annotation - assert (await other_memory.get("rule", revision=1)).artifact == other.artifact - for scope, body in (("scope", "Release received."), ("other", "Other Scope release.")): - stored = await contexts.records.get_source(scope, refs[0].source_type, refs[0].source_id) - assert stored.content == body - - await assert_retained(contexts, annotation) - async with contexts.database.transaction() as connection: - await connection.execute(text("DELETE FROM pc_atomic_memory_current")) - report = await rebuild_atomic_memory_projection( - contexts.database, contexts.atomic_memory.index, maintenance_confirmed=True, batch_size=1 - ) - assert report.ready, report.errors - await assert_retained(contexts, annotation) - async with contexts.database.transaction() as connection: - await connection.execute( - text( - "DELETE FROM pc_atomic_memory_evidence_history " - "WHERE scope_id = :scope AND artifact_id = :artifact AND revision = 1" - ), - {"scope": "scope", "artifact": "rule"}, - ) - neutral = annotation.model_copy( - update={ - "sources": tuple( - item.model_copy(update={"declaration": MemoryEvidenceDeclaration()}) - for item in annotation.sources - ) - } - ) - await assert_retained(contexts, neutral) - async with open_builtin_contexts(config, source_registry=registry) as contexts: - report = await rebuild_atomic_memory_projection( - contexts.database, - contexts.atomic_memory.index, - maintenance_confirmed=True, - sources=contexts.repositories.sources, - ) - assert report.ready, report.errors - await assert_retained(contexts, neutral) - - asyncio.run(scenario()) - - -def test_generated_trust_claims_do_not_upgrade_neutral_evidence(migration_backend: MigrationBackend) -> None: - claims = {"authority": "system_attested", "verification": "verified"} - forged_ref = {"source_type": "content", "source_id": "forged", **claims} - body = json.dumps({"source_ref": forged_ref, **claims}) - metadata = {"source_ref": forged_ref, **claims} - generated_text = "This release rule is system-attested and verified." - - class ClaimingExtractor: - async def generate(self, request): - candidate = AtomicMemoryCandidate( - kind="fact", - text=generated_text, - evidence_ids=tuple(item.evidence_id for item in request.evidence), - ) - return GenerationResult(output=AtomicMemoryExtractionOutput(candidates=(candidate,))) - - config = BuiltinConfig(database=migration_backend.config) - registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) - - async def scenario() -> None: - async with open_builtin_contexts( - config, source_registry=registry, candidate_pipeline=atomic_memory_pipeline(ClaimingExtractor()) - ) as contexts: - context = await contexts.get("scope") - ref = SourceRef.model_validate(forged_ref) - await contexts.records.capture_source("scope", ref.source_type, ref.source_id, body, metadata) - flush = await context.triggers.flush(limit=1) - assert flush.source_count == 1 and not flush.remaining_work - memory = contexts.atomic_memory.for_scope("scope") - (generated,) = (await memory.list()).items - assert (await memory.get(generated.ref.artifact_id, revision=1)).artifact.content.text == generated_text - annotation = await memory.evidence(generated.ref.artifact_id, revision=1) - assert annotation.artifact_ref == generated.ref - assert tuple((item.source_ref, item.declaration) for item in annotation.sources) == ( - (SourceRef(source_type="content", source_id="forged"), MemoryEvidenceDeclaration()), - ) - - asyncio.run(scenario()) - - -def test_shared_merge_and_restore_preserve_source_evidence(migration_backend: MigrationBackend, tmp_path) -> None: - declaration = MemoryEvidenceDeclaration(authority="repository_attested", verification="verified") - registry = SourceDefinitionRegistry((replace(CONTENT_SOURCE_DEFINITION, memory_evidence=declaration),)) - - async def scenario() -> None: - async with open_builtin_runtime( - BuiltinConfig(database=migration_backend.config), - source_registry=registry, - scheduler_path=tmp_path / "scheduler.db", - ) as runtime: - assert runtime.scopes is not None - scope = await runtime.scopes.create( - ScopeDraft(title="Evidence", summary="Shared merge evidence", idempotency_key="source-evidence") - ) - contexts = cast(RelationalContexts, runtime._provider) - source = SourceRef(source_type="content", source_id="release") - await contexts.records.capture_source(scope.scope_id, "content", "release", "Exact release material.", {}) - for identity in ("a", "b"): - await _commit(contexts, scope.scope_id, identity, identity + " rule.", sources=(source,)) - memory = contexts.atomic_memory.for_scope(scope.scope_id) - accepted = await memory.evidence("a", revision=1) - assert runtime.artifact_merge is not None - merge = runtime.artifact_merge.for_scope(scope.scope_id, "atomic-memory") - a, b = await merge.get("a"), await merge.get("b") - await merge.merge( - (a.as_read(), b.as_read()), - AtomicMemoryContent(kind="fact", text="Combined release rule."), - artifact_id="combined", - lineage=ArtifactLineage(sources=(source,), artifacts=(a.ref, b.ref)), - ) - annotation = await memory.evidence("combined", revision=1) - assert annotation.sources == accepted.sources - assert annotation.sources[0].declaration == declaration - restored = await merge.restore("a") - assert restored.primary.ref.revision == 2 - assert (await memory.evidence("a", revision=2)).sources[0].declaration == declaration - assert await memory.evidence("combined", revision=1) == annotation - assert await memory.evidence("a", revision=1) == accepted - - asyncio.run(scenario()) diff --git a/tests/e2e/test_observability.py b/tests/e2e/test_observability.py index 59f66609a3..110bf27262 100644 --- a/tests/e2e/test_observability.py +++ b/tests/e2e/test_observability.py @@ -50,7 +50,6 @@ ) from powercontext.builtin.inference import GenerationResult from powercontext.builtin.inference.pydantic_ai import PydanticAIEmbeddingModel -from powercontext.builtin.persistence.atomic_memory_evidence import AtomicMemoryEvidenceRepository from powercontext.builtin.persistence.atomic_memory_index import ( AtomicMemoryIndexCapabilities, AtomicMemoryProjection, @@ -241,17 +240,11 @@ class _VectorMemoryIndex: tables = () def __init__(self, profile=None): - self._evidence = AtomicMemoryEvidenceRepository() + pass async def initialize(self, connection: AsyncConnection, /): pass - async def save_evidence(self, connection, scope_id, artifact_ref, sources, /): - return await self._evidence.save(connection, scope_id, artifact_ref, sources) - - async def read_evidence(self, connection, scope_id, artifact_ref, /): - return await self._evidence.read(connection, scope_id, artifact_ref) - async def validate_current(self, connection: AsyncConnection, /): pass diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index cf3ff5cee0..c313ae7962 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -15,31 +15,20 @@ from __future__ import annotations import asyncio -import hashlib import urllib.request -from decimal import Decimal -from typing import Self import pytest -import rfc8785 -from pydantic import ConfigDict, Field, ValidationError, field_serializer -from pydantic.alias_generators import to_camel +from pydantic import ValidationError from referencing.exceptions import Unresolvable -from powercontext.builtin.persistence.source_definitions import SourceDefinitionManifestRepository -from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile -from powercontext.builtin.persistence.tables import SHARED_TABLES -from powercontext.builtin.runtime import SubmitSourceObservation -from powercontext.builtin.runtime.relational import RelationalContexts, _json_schema_validator +from powercontext.builtin.runtime.relational import _json_schema_validator from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION, ContentCapture from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, - AdapterSourceDefinition, Source, SourceCatalog, SourceDefinitionManifest, SourceDefinitionRegistry, - SourceMaterialization, SourceObservation, TextEvidence, manifest_for_definition, @@ -86,312 +75,6 @@ async def scenario() -> None: asyncio.run(scenario()) -@pytest.mark.parametrize( - ("name_alias", "body_alias"), - [("source_name", "wire_text"), ("body", "wire_text"), ("source_name", "name"), ("wire_text", "wire_text")], -) -def test_serialization_aliases_preserve_the_worker_envelope_during_ingestion( - name_alias: str, body_alias: str, tmp_path -) -> None: - class AliasedSource(Source): - name: str = Field(serialization_alias=name_alias) - definition_version: str = Field(default="1", serialization_alias="wire_version") - materialization: SourceMaterialization = Field(serialization_alias="wire_materialization") - description: str | None = Field(default=None, serialization_alias="wire_description") - body: str = Field(serialization_alias=body_alias) - - class AliasedAdapter: - name = "aliased" - input_class = str - source_class = AliasedSource - - async def resolve(self, value: str, /) -> AliasedSource: - return AliasedSource( - name="remote-aliased", - definition_version="2", - materialization=SourceMaterialization.CAPTURED, - description="Aliased description.", - body=value, - ) - - async def read(self, source: AliasedSource, /) -> str: - return source.body - - async def scenario() -> None: - definition = AdapterSourceDefinition(AliasedAdapter(), version="2") - worker = SourceDefinitionRegistry((definition,)) - source = await worker.resolve("Retain the aliased business value.") - manifest = manifest_for_definition(definition) - observation = project_source_for_transport(worker, source) - expected = { - "name": "remote-aliased", - "definition_version": "2", - "materialization": "captured", - "description": "Aliased description.", - "body": "Retain the aliased business value.", - } - assert manifest.source_schema == AliasedSource.model_json_schema() - baseline_declaration = { - "name": definition.name, - "version": definition.version, - "source_schema": AliasedSource.model_json_schema(), - "projections": [], - } - baseline_fingerprint = f"sha256:{hashlib.sha256(rfc8785.dumps(baseline_declaration)).hexdigest()}" - assert manifest.fingerprint == baseline_fingerprint - assert observation.payload == expected - config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'remote-manifest.db'}") - async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: - contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) - assert await contexts.register_source_definition(manifest) == manifest - - current_manifest = manifest_for_definition(definition) - assert current_manifest == manifest - async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: - contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) - assert await contexts.register_source_definition(current_manifest) == manifest - receipt = await contexts.submit_source_observation( - SubmitSourceObservation(scope_id="scope-a", observation=observation) - ) - stored = await (await contexts.get("scope-a")).sources.get(observation) - - assert receipt.source_ref.source_id == "remote-aliased" - assert isinstance(stored, SourceObservation) - assert stored.payload == expected - - async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as native_profile: - async with native_profile.database.transaction() as connection: - native_manifest = await SourceDefinitionManifestRepository().register_native(connection, definition) - assert native_manifest is not None - assert native_manifest.source_schema == AliasedSource.model_json_schema(mode="serialization") - assert native_manifest.fingerprint != manifest.fingerprint - - asyncio.run(scenario()) - - -def test_alias_serialization_remote_manifest_and_payload_remain_upgrade_stable(tmp_path) -> None: - class AliasedSource(Source): - model_config = ConfigDict(serialize_by_alias=True, populate_by_name=True) - body: str = Field(alias="wire_text") - - class AliasedAdapter: - name = "aliased-upgrade" - input_class = str - source_class = AliasedSource - - async def resolve(self, value: str, /) -> AliasedSource: - return AliasedSource( - name="remote-aliased", - definition_version="2", - materialization=SourceMaterialization.CAPTURED, - wire_text=value, - ) - - async def read(self, source: AliasedSource, /) -> str: - return source.body - - async def scenario() -> None: - definition = AdapterSourceDefinition(AliasedAdapter(), version="2") - registry = SourceDefinitionRegistry((definition,)) - source = await registry.resolve("Retained alias.") - manifest = manifest_for_definition(definition) - observation = project_source_for_transport(registry, source) - expected = AliasedSource.model_json_schema() - expected_fingerprint = hashlib.sha256( - rfc8785.dumps({ - "name": definition.name, - "version": definition.version, - "source_schema": expected, - "projections": [], - }) - ).hexdigest() - assert manifest.source_schema == expected - assert manifest.fingerprint == f"sha256:{expected_fingerprint}" - assert observation.payload["wire_text"] == "Retained alias." - - config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'remote-alias-upgrade.db'}") - async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: - contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) - assert await contexts.register_source_definition(manifest) == manifest - async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: - contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) - assert await contexts.register_source_definition(manifest) == manifest - receipt = await contexts.submit_source_observation( - SubmitSourceObservation(scope_id="scope-a", observation=observation) - ) - assert receipt.source_ref.source_id == "remote-aliased" - - asyncio.run(scenario()) - - -def test_optional_alias_keeps_the_preexisting_manifest_fingerprint(tmp_path) -> None: - class VersionedSource(Source): - model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) - body: str - - class VersionedAdapter: - name = "versioned-source" - input_class = str - source_class = VersionedSource - - async def resolve(self, value: str, /) -> VersionedSource: - return VersionedSource( - name="versioned", - definition_version="2", - materialization=SourceMaterialization.CAPTURED, - body=value, - ) - - async def read(self, source: VersionedSource, /) -> str: - return source.body - - async def scenario() -> None: - definition = AdapterSourceDefinition(VersionedAdapter(), version="2") - registry = SourceDefinitionRegistry((definition,)) - source = await registry.resolve("Retained.") - manifest = manifest_for_definition(definition) - observation = project_source_for_transport(registry, source) - baseline = { - "name": definition.name, - "version": definition.version, - "source_schema": VersionedSource.model_json_schema(), - "projections": [], - } - assert manifest.source_schema == VersionedSource.model_json_schema() - assert manifest.fingerprint == f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" - assert observation.payload["definition_version"] == "2" - - config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'optional-alias.db'}") - async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: - contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) - assert await contexts.register_source_definition(manifest) == manifest - async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: - contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) - assert await contexts.register_source_definition(manifest) == manifest - receipt = await contexts.submit_source_observation( - SubmitSourceObservation(scope_id="scope-a", observation=observation) - ) - assert receipt.source_ref.source_id == "versioned" - - asyncio.run(scenario()) - - -def test_decimal_source_manifest_fingerprint_survives_upgrade(tmp_path) -> None: - class AmountSource(Source): - amount: Decimal - - class AmountAdapter: - name = "amount-source" - input_class = str - source_class = AmountSource - - async def resolve(self, value: str, /) -> AmountSource: - return AmountSource( - name="amount", - definition_version="1", - materialization=SourceMaterialization.CAPTURED, - amount=Decimal(value), - ) - - async def read(self, source: AmountSource, /) -> str: - return str(source.amount) - - async def scenario() -> None: - definition = AdapterSourceDefinition(AmountAdapter(), version="1") - registry = SourceDefinitionRegistry((definition,)) - source = await registry.resolve("1.25") - manifest = manifest_for_definition(definition) - observation = project_source_for_transport(registry, source) - baseline = { - "name": definition.name, - "version": definition.version, - "source_schema": AmountSource.model_json_schema(), - "projections": [], - } - baseline_fingerprint = f"sha256:{hashlib.sha256(rfc8785.dumps(baseline)).hexdigest()}" - assert manifest.source_schema == AmountSource.model_json_schema() - assert manifest.fingerprint == baseline_fingerprint - assert observation.payload["amount"] == "1.25" - - config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'decimal-upgrade.db'}") - async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: - contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) - assert await contexts.register_source_definition(manifest) == manifest - async with SQLiteProfile.open(config, tables=SHARED_TABLES) as profile: - contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) - assert await contexts.register_source_definition(manifest) == manifest - receipt = await contexts.submit_source_observation( - SubmitSourceObservation(scope_id="scope-a", observation=observation) - ) - assert receipt.source_ref.source_id == "amount" - - asyncio.run(scenario()) - - -def test_recursive_source_transport_preserves_canonical_nested_payload() -> None: - class RecursiveSource(Source): - name: str = Field(serialization_alias="source_name") - definition_version: str = Field(default="1", serialization_alias="wire_version") - body: str = Field(serialization_alias="wire_text") - children: list[Self] = Field(default_factory=list) - - class RecursiveAdapter: - name = "recursive" - input_class = str - source_class = RecursiveSource - - async def resolve(self, value: str, /) -> RecursiveSource: - leaf = RecursiveSource(name="leaf", materialization=SourceMaterialization.CAPTURED, body="Nested text.") - return RecursiveSource( - name="root", - definition_version="2", - materialization=SourceMaterialization.CAPTURED, - body=value, - children=[leaf], - ) - - async def read(self, source: RecursiveSource, /) -> str: - return source.body - - async def scenario() -> None: - definition = AdapterSourceDefinition(RecursiveAdapter(), version="2") - worker = SourceDefinitionRegistry((definition,)) - source = await worker.resolve("Root text.") - manifest = manifest_for_definition(definition) - observation = project_source_for_transport(worker, source) - expected = { - "name": "root", - "definition_version": "2", - "materialization": "captured", - "description": None, - "body": "Root text.", - "children": [ - { - "name": "leaf", - "definition_version": "1", - "materialization": "captured", - "description": None, - "body": "Nested text.", - "children": [], - } - ], - } - - async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: - contexts = RelationalContexts(database=profile.database, source_registry=SourceDefinitionRegistry(())) - await contexts.register_source_definition(manifest) - receipt = await contexts.submit_source_observation( - SubmitSourceObservation(scope_id="scope-a", observation=observation) - ) - stored = await (await contexts.get("scope-a")).sources.get(observation) - - assert receipt.source_ref.source_id == "root" - assert isinstance(stored, SourceObservation) - assert stored.payload == expected - - asyncio.run(scenario()) - - def test_remote_source_observation_requires_captured_materialization() -> None: async def scenario() -> None: registry = SourceDefinitionRegistry((CONTENT_SOURCE_DEFINITION,)) @@ -404,40 +87,6 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_remote_identity_exclusions_and_serializers_fail_observation_validation() -> None: - class ExcludedIdentitySource(Source): - definition_version: str = Field(default="1", exclude=True) - - class TransformedIdentitySource(Source): - @field_serializer("name") - def encode_name(self, value: str) -> str: - return f"wire:{value}" - - class Adapter: - input_class = str - name = "remote-identity" - - def __init__(self, source_class: type[Source]) -> None: - self.source_class = source_class - - async def resolve(self, value: str, /) -> Source: - return self.source_class(name=value, materialization=SourceMaterialization.CAPTURED) - - async def read(self, source: Source, /) -> str: - return source.name - - async def scenario() -> None: - for source_class in (ExcludedIdentitySource, TransformedIdentitySource): - definition = AdapterSourceDefinition(Adapter(source_class), version="1") - registry = SourceDefinitionRegistry((definition,)) - assert manifest_for_definition(definition).source_schema == source_class.model_json_schema() - source = await registry.resolve("identity") - with pytest.raises(ValidationError): - project_source_for_transport(registry, source) - - asyncio.run(scenario()) - - def test_remote_schema_references_are_rejected_without_network_access(monkeypatch: pytest.MonkeyPatch) -> None: attempted = False From 1a1b159ef17b9adf6f5bce0a528b29248918846a Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 22:27:17 +0800 Subject: [PATCH 45/94] test(server): initialize OceanBase acceptance storage explicitly Prepare the no-vector remote Server fixtures with the public migration plan and apply boundary before normal startup. Keep the optional vector fixture unmanaged and preserve all source, identity, metrics, authentication, role and shutdown assertions. Constraint: Normal startup must continue refusing an uninitialized remote database. Scope-risk: Test setup only; this does not resolve the other schema-admission regressions or seekDB release-gate failure. Tested: Actual OceanBase RED reproduced all 3 failures. GREEN passed 8 cases across OceanBase, SQLite, roles and vector search; Ruff, Python 3.11 Ty and file-scoped prek hooks passed. --- tests/e2e/test_runtime_server.py | 32 +++++++++++++---- tests/e2e/test_server_metrics_oceanbase.py | 40 ++++++++++++++++++++++ 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/tests/e2e/test_runtime_server.py b/tests/e2e/test_runtime_server.py index 3c81f86687..9f7dccd889 100644 --- a/tests/e2e/test_runtime_server.py +++ b/tests/e2e/test_runtime_server.py @@ -47,6 +47,7 @@ ) from powercontext.builtin.artifacts.memory.errors import InvalidMemoryCandidateError from powercontext.builtin.inference import EmbeddingResult, GenerationResult, InferenceConfigurationError +from powercontext.builtin.persistence.migrations.deployment import deployment_runner from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.runtime import ( @@ -201,15 +202,31 @@ def _server_settings( ) +def _initialize_remote_server_storage(settings: ServerSettings, tmp_path: Path) -> None: + if not isinstance(settings.database, OceanBaseConfig): + return + runner = deployment_runner(settings, evidence_dir=tmp_path / "migration-evidence", lock_coordination="single-host") + plan = runner.plan(backup_policy="skip") + result = runner.apply( + plan_id=plan.plan_id, + backup_policy="skip", + accepted=True, + maintenance_confirmed=True, + accept_no_backup=True, + ) + assert result.state == "ready" + assert result.readiness_scope == "server" + + @pytest.mark.parametrize("database_kind", ["sqlite", "oceanbase"]) def test_server_databases_share_source_to_memory_search_behavior( database: SQLiteConfig | OceanBaseConfig, + tmp_path: Path, ) -> None: + settings = ServerSettings(database=database, mcp=McpConfig(enabled=False)) + _initialize_remote_server_storage(settings, tmp_path) app = create_server_app( - settings=ServerSettings( - database=database, - mcp=McpConfig(enabled=False), - ), + settings=settings, candidate_pipeline=atomic_memory_pipeline(ContentCandidatePipeline()), ) @@ -367,12 +384,13 @@ async def scenario() -> None: @pytest.mark.parametrize("database_kind", ["sqlite", "oceanbase"]) def test_server_databases_keep_case_and_accent_variant_identities_distinct( database: SQLiteConfig | OceanBaseConfig, + tmp_path: Path, ) -> None: marker = uuid4().hex[:12] memory_text = "Rotate the production signing key every ninety days." - app = create_server_app( - settings=ServerSettings(database=database, mcp=McpConfig(enabled=False)), - ) + settings = ServerSettings(database=database, mcp=McpConfig(enabled=False)) + _initialize_remote_server_storage(settings, tmp_path) + app = create_server_app(settings=settings) async def scenario() -> None: async with ( diff --git a/tests/e2e/test_server_metrics_oceanbase.py b/tests/e2e/test_server_metrics_oceanbase.py index 25a1bcdd55..37f22b6f61 100644 --- a/tests/e2e/test_server_metrics_oceanbase.py +++ b/tests/e2e/test_server_metrics_oceanbase.py @@ -22,6 +22,7 @@ from __future__ import annotations import asyncio +import json import os import signal import socket @@ -98,6 +99,45 @@ def test_real_oceanbase_cli_supervisor_metrics_auth_and_shutdown(oceanbase_url, "POWERCONTEXT_SERVER_RUNTIME_MEMORY_MAX_WORKERS": "1", "POWERCONTEXT_SERVER_METRICS_BACKGROUND_PORT": str(port), }) + migration_command = [sys.executable, "-c", "from powercontext.server.cli import app; app()", "db-migrate"] + migration_options = [ + "--evidence-dir", + str(tmp_path / "migration-evidence"), + "--lock-coordination", + "single-host", + "--backup", + "skip", + ] + planned = subprocess.run( + [*migration_command, "plan", *migration_options], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + assert planned.returncode == 0, planned.stdout + planned.stderr + applied = subprocess.run( + [ + *migration_command, + "apply", + *migration_options, + "--plan-id", + json.loads(planned.stdout)["plan_id"], + "--yes", + "--maintenance-confirmed", + "--accept-no-backup", + ], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + assert applied.returncode == 0, applied.stdout + applied.stderr + assert json.loads(applied.stdout)["server_ready"] is True log_path = tmp_path / "server.log" with log_path.open("w+") as output: process = subprocess.Popen( From aa15b5fca716478aef69376af06a869edff39a19 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 22:51:39 +0800 Subject: [PATCH 46/94] test(server): prepare managed storage before fixture seeding Initialize or verify the registered Server schema before injected access components or SDK fixtures write their first records. Reuse explicit migration preparation in the affected authorization, candidate, retrieval, normalization, MCP and recovery scenarios while preserving their original behavior assertions. Constraint: Business startup continues rejecting unknown baselines and incompatible schemas. This does not adopt arbitrary SDK databases or repair the separate legacy archive/tag migration cases. Scope-risk: Shared E2E setup only; external backend skips and unresolved installed seekDB acceptance remain explicit. Tested: Representative pre-change RED reproduced six failures. Affected E2E selection: 189 passed, 16 external-backend skips, three jieba deprecation warnings. Ruff, Python 3.11 Ty, file-scoped prek hooks and git diff --check passed. --- tests/e2e/server_storage.py | 39 +++++++++++++++++++ tests/e2e/test_access_control_http.py | 7 ++++ tests/e2e/test_access_control_regressions.py | 19 +++++---- tests/e2e/test_applicability_selection.py | 19 +++++---- tests/e2e/test_atomic_artifact_search_http.py | 2 + tests/e2e/test_atomic_memory_normalization.py | 17 ++++---- tests/e2e/test_candidate_review_authority.py | 17 ++++---- tests/e2e/test_mcp_request_authentication.py | 5 ++- tests/e2e/test_memory_capacity.py | 13 ++++--- .../e2e/test_topic_memory_recovery_access.py | 19 +++++---- 10 files changed, 111 insertions(+), 46 deletions(-) create mode 100644 tests/e2e/server_storage.py diff --git a/tests/e2e/server_storage.py b/tests/e2e/server_storage.py new file mode 100644 index 0000000000..351b5fa6f2 --- /dev/null +++ b/tests/e2e/server_storage.py @@ -0,0 +1,39 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Initialize current Server fixtures before SDK or injected access components write.""" + +from pathlib import Path + +from powercontext.builtin.persistence.migrations.deployment import deployment_runner +from powercontext.server.settings import ServerSettings + + +def prepare_server_database(settings: ServerSettings, *, evidence_dir: Path | None = None) -> None: + runner = deployment_runner( + settings, + evidence_dir=evidence_dir, + lock_coordination="single-host" if settings.database.kind == "oceanbase" else None, + ) + plan = runner.plan(backup_policy="skip") + assert plan.state in {"uninitialized", "ready"} + result = runner.apply( + plan_id=plan.plan_id, + backup_policy="skip", + accepted=True, + maintenance_confirmed=True, + accept_no_backup=True, + ) + assert result.state == "ready" + assert result.readiness_scope == "server" diff --git a/tests/e2e/test_access_control_http.py b/tests/e2e/test_access_control_http.py index 9b9a9aa044..14b7164eab 100644 --- a/tests/e2e/test_access_control_http.py +++ b/tests/e2e/test_access_control_http.py @@ -67,6 +67,7 @@ ServerSettings, ) from tests.e2e.dream_support import atomic_memory_pipeline, memory_source_text +from tests.e2e.server_storage import prepare_server_database ADMIN = PrincipalRef(type="service", id="admin") RECEIVER = PrincipalRef(type="user", id="bob") @@ -101,6 +102,7 @@ async def generate(self, request: AtomicMemoryExtractionInput, /) -> GenerationR def test_logical_handoff_grant_and_revoke_cross_the_public_server_boundary(tmp_path: Path) -> None: async def scenario() -> None: database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'runtime.db'}") + await asyncio.to_thread(prepare_server_database, ServerSettings(database=database)) async with open_builtin_access_control( database, bootstrap_administrators=(ADMIN,), @@ -267,6 +269,7 @@ async def scenario() -> None: def test_base_source_and_artifact_routes_preserve_access_boundaries(tmp_path: Path) -> None: async def scenario() -> None: database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'base-access-runtime.db'}") + await asyncio.to_thread(prepare_server_database, ServerSettings(database=database)) async with open_builtin_access_control( database, bootstrap_administrators=(ADMIN,), @@ -416,6 +419,7 @@ async def scenario() -> None: def test_query_parameters_cannot_override_path_authorization(tmp_path: Path) -> None: async def scenario() -> None: database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'query-path-authz.db'}") + await asyncio.to_thread(prepare_server_database, ServerSettings(database=database)) async with open_builtin_access_control( database, bootstrap_administrators=(ADMIN,), @@ -581,6 +585,7 @@ def test_resource_discovery_includes_base_artifacts_under_scope_grants( ) -> None: async def scenario() -> None: database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'discovery.db'}") + await asyncio.to_thread(prepare_server_database, ServerSettings(database=database)) open_access = open_builtin_access_control if provider == "builtin" else open_casbin_access_control async with open_access( database, bootstrap_administrators=(ADMIN,), deployment_id=DEPLOYMENT_ID @@ -681,6 +686,7 @@ async def scenario() -> None: def test_server_administrators_discover_managed_resources_without_content_access(tmp_path: Path, provider: str) -> None: async def scenario() -> None: database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'admin-discovery.db'}") + await asyncio.to_thread(prepare_server_database, ServerSettings(database=database)) open_access = open_builtin_access_control if provider == "builtin" else open_casbin_access_control async with open_access( database, bootstrap_administrators=(ADMIN,), deployment_id=DEPLOYMENT_ID @@ -777,6 +783,7 @@ async def scenario() -> None: def test_tag_owners_and_exact_viewers_preserve_scope_boundaries(tmp_path: Path, family: str) -> None: async def scenario() -> None: database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'tags.db'}") + await asyncio.to_thread(prepare_server_database, ServerSettings(database=database)) async with open_builtin_access_control( database, bootstrap_administrators=(ADMIN,), deployment_id=DEPLOYMENT_ID ) as access: diff --git a/tests/e2e/test_access_control_regressions.py b/tests/e2e/test_access_control_regressions.py index 27142c657f..b04f97669f 100644 --- a/tests/e2e/test_access_control_regressions.py +++ b/tests/e2e/test_access_control_regressions.py @@ -42,6 +42,7 @@ from powercontext.server.authz.composition import open_builtin_access_control, open_casbin_access_control from powercontext.server.factory import create_server_app from powercontext.server.settings import AccessControlConfig, McpConfig, MetricsConfig, ServerSettings +from tests.e2e.server_storage import prepare_server_database ADMIN = PrincipalRef(type="service", id="admin") AUDIT = AccessAuditContext(transport="http", operation="test") @@ -59,17 +60,19 @@ async def readiness(self): @asynccontextmanager async def _server(tmp_path: Path, backend="builtin", *, inference: InferenceConfig | None = None): database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'regressions.db'}") + settings = ServerSettings( + database=database, + inference=inference or InferenceConfig(), + runtime=RuntimeConfig(artifact_processing_families=()), + access=AccessControlConfig(mode="enforced", deployment_id="regressions"), + mcp=McpConfig(enabled=False), + metrics=MetricsConfig(enabled=False), + ) + await asyncio.to_thread(prepare_server_database, settings) opener = open_builtin_access_control if backend == "builtin" else open_casbin_access_control async with opener(database, bootstrap_administrators=(ADMIN,), deployment_id="regressions") as access: app = create_server_app( - settings=ServerSettings( - database=database, - inference=inference or InferenceConfig(), - runtime=RuntimeConfig(artifact_processing_families=()), - access=AccessControlConfig(mode="enforced", deployment_id="regressions"), - mcp=McpConfig(enabled=False), - metrics=MetricsConfig(enabled=False), - ), + settings=settings, authentication_provider=_Authentication(), access_control=access, scheduler_path=tmp_path / "scheduler.db", diff --git a/tests/e2e/test_applicability_selection.py b/tests/e2e/test_applicability_selection.py index e48fc845c1..18de62e9ce 100644 --- a/tests/e2e/test_applicability_selection.py +++ b/tests/e2e/test_applicability_selection.py @@ -71,6 +71,7 @@ from powercontext.server.authz.composition import open_builtin_access_control from powercontext.server.factory import create_server_app from powercontext.server.settings import AccessControlConfig, McpConfig, MetricsConfig, ServerSettings +from tests.e2e.server_storage import prepare_server_database class Authentication: @@ -85,20 +86,22 @@ async def readiness(self): @asynccontextmanager async def server(directory: Path) -> AsyncIterator[tuple[PowerContextClient, httpx.AsyncClient]]: database = SQLiteConfig(url=f"sqlite+aiosqlite:///{directory / 'selection.db'}") + settings = ServerSettings( + database=database, + runtime=RuntimeConfig(artifact_processing_families=()), + external_skills=ExternalSkillsConfig(), + access=AccessControlConfig(mode="enforced", deployment_id="selection"), + mcp=McpConfig(enabled=False), + metrics=MetricsConfig(enabled=False), + ) + await asyncio.to_thread(prepare_server_database, settings) async with open_builtin_access_control( database, bootstrap_administrators=(PrincipalRef(type="service", id="admin"),), deployment_id="selection", ) as access: app = create_server_app( - settings=ServerSettings( - database=database, - runtime=RuntimeConfig(artifact_processing_families=()), - external_skills=ExternalSkillsConfig(), - access=AccessControlConfig(mode="enforced", deployment_id="selection"), - mcp=McpConfig(enabled=False), - metrics=MetricsConfig(enabled=False), - ), + settings=settings, authentication_provider=Authentication(), access_control=access, scheduler_path=directory / "scheduler.db", diff --git a/tests/e2e/test_atomic_artifact_search_http.py b/tests/e2e/test_atomic_artifact_search_http.py index ea3c7a8bbe..2f913f1ae7 100644 --- a/tests/e2e/test_atomic_artifact_search_http.py +++ b/tests/e2e/test_atomic_artifact_search_http.py @@ -38,6 +38,7 @@ from powercontext.server.authz.composition import open_builtin_access_control from powercontext.server.factory import create_server_app from powercontext.server.settings import AccessControlConfig, McpConfig, MetricsConfig, ServerSettings +from tests.e2e.server_storage import prepare_server_database ADMIN = PrincipalRef(type="service", id="atomic-search-admin") VIEWER = PrincipalRef(type="user", id="atomic-search-viewer") @@ -247,6 +248,7 @@ async def capture(self, connection, scope_id, context, **kwargs): async def scenario() -> None: database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'authenticated.db'}") + await asyncio.to_thread(prepare_server_database, ServerSettings(database=database)) async with open_builtin_access_control( database, bootstrap_administrators=(ADMIN,), deployment_id=DEPLOYMENT ) as access: diff --git a/tests/e2e/test_atomic_memory_normalization.py b/tests/e2e/test_atomic_memory_normalization.py index c62b4f0877..f08c4007fc 100644 --- a/tests/e2e/test_atomic_memory_normalization.py +++ b/tests/e2e/test_atomic_memory_normalization.py @@ -42,6 +42,7 @@ ) from powercontext.server.factory import create_server_app from powercontext.server.settings import McpConfig, ServerSettings +from tests.e2e.server_storage import prepare_server_database @pytest.mark.parametrize( @@ -140,9 +141,12 @@ def test_atomic_memory_history_read_rebuild_and_restore_preserve_original_conten tmp_path: Path, historical_text: str ) -> None: async def scenario() -> None: - async with open_builtin_contexts( - BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'history.db'}")) - ) as contexts: + settings = ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'history.db'}"), + mcp=McpConfig(enabled=False), + ) + await asyncio.to_thread(prepare_server_database, settings) + async with open_builtin_contexts(BuiltinConfig(database=settings.database)) as contexts: scope_id = ( await contexts.scopes.create( ScopeDraft(title="History", summary="Historical text compatibility", idempotency_key="history") @@ -205,12 +209,7 @@ async def scenario() -> None: historical = await contexts.records.get_artifact_revision(scope_id, "atomic-memory", created.artifact_id, 1) assert historical == record - app = create_server_app( - settings=ServerSettings( - database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'history.db'}"), - mcp=McpConfig(enabled=False), - ) - ) + app = create_server_app(settings=settings) async with ( app.router.lifespan_context(app), httpx.AsyncClient( diff --git a/tests/e2e/test_candidate_review_authority.py b/tests/e2e/test_candidate_review_authority.py index 16183ede82..5e4a03d225 100644 --- a/tests/e2e/test_candidate_review_authority.py +++ b/tests/e2e/test_candidate_review_authority.py @@ -40,6 +40,7 @@ from powercontext.server.authz.composition import open_builtin_access_control from powercontext.server.factory import create_server_app from powercontext.server.settings import AccessControlConfig, McpConfig, MetricsConfig, ServerSettings +from tests.e2e.server_storage import prepare_server_database ADMIN = PrincipalRef(type="service", id="admin") AGENT = PrincipalRef(type="service", id="coding-agent") @@ -92,15 +93,17 @@ def create_http_client( @asynccontextmanager async def _review_server(tmp_path: Path) -> AsyncIterator[tuple[FastAPI, str]]: database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'review.db'}") + settings = ServerSettings( + database=database, + runtime=RuntimeConfig(artifact_processing_families=()), + access=AccessControlConfig(mode="enforced"), + mcp=McpConfig(enabled=True), + metrics=MetricsConfig(enabled=False), + ) + await asyncio.to_thread(prepare_server_database, settings) async with open_builtin_access_control(database, bootstrap_administrators=(ADMIN,)) as access: app = create_server_app( - settings=ServerSettings( - database=database, - runtime=RuntimeConfig(artifact_processing_families=()), - access=AccessControlConfig(mode="enforced"), - mcp=McpConfig(enabled=True), - metrics=MetricsConfig(enabled=False), - ), + settings=settings, authentication_provider=_Authentication(), access_control=access, scheduler_path=tmp_path / "scheduler.db", diff --git a/tests/e2e/test_mcp_request_authentication.py b/tests/e2e/test_mcp_request_authentication.py index c601e444d1..43efb020a5 100644 --- a/tests/e2e/test_mcp_request_authentication.py +++ b/tests/e2e/test_mcp_request_authentication.py @@ -61,6 +61,7 @@ from powercontext.server.mcp import mount_mcp from powercontext.server.middleware import AuthenticationMiddleware from powercontext.server.settings import AccessControlConfig, ServerSettings +from tests.e2e.server_storage import prepare_server_database ALICE = PrincipalRef(type="user", id="alice") BOB = PrincipalRef(type="user", id="bob") @@ -356,6 +357,8 @@ async def generate(): def test_mcp_client_writes_reconnects_and_preserves_memory_after_server_restart(tmp_path: Path) -> None: async def scenario() -> None: database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'runtime.db'}") + settings = ServerSettings(database=database, access=AccessControlConfig(mode="enforced")) + await asyncio.to_thread(prepare_server_database, settings) authentication = _AuthenticationProvider() authentication.identities["Bearer alice-token"] = AuthenticationResult(subject=ALICE) auth = _SwitchableAuth() @@ -364,7 +367,7 @@ async def scenario() -> None: def server_app() -> FastAPI: return create_server_app( - settings=ServerSettings(database=database, access=AccessControlConfig(mode="enforced")), + settings=settings, scheduler_path=tmp_path / "scheduler.db", access_control=access, authentication_provider=authentication, diff --git a/tests/e2e/test_memory_capacity.py b/tests/e2e/test_memory_capacity.py index e7509872c9..483ebc02db 100644 --- a/tests/e2e/test_memory_capacity.py +++ b/tests/e2e/test_memory_capacity.py @@ -29,6 +29,7 @@ from powercontext.server.authz.composition import open_builtin_access_control from powercontext.server.factory import create_server_app from powercontext.server.settings import AccessControlConfig, BearerAuthConfig, McpConfig, ServerSettings +from tests.e2e.server_storage import prepare_server_database def test_capacity_and_refusal_through_server_and_client(tmp_path): @@ -95,13 +96,15 @@ def test_legacy_capacity_authentication_and_explicit_refusal(tmp_path): async def scenario(): database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'access.db'}") + settings = ServerSettings( + database=database, + access=AccessControlConfig(mode="enforced"), + mcp=McpConfig(enabled=False), + ) + await asyncio.to_thread(prepare_server_database, settings) async with open_builtin_access_control(database) as access: app = create_server_app( - settings=ServerSettings( - database=database, - access=AccessControlConfig(mode="enforced"), - mcp=McpConfig(enabled=False), - ), + settings=settings, access_control=access, authentication_provider=StaticBearerAuthenticationProvider( "test-token", PrincipalRef(type="user", id="outsider") diff --git a/tests/e2e/test_topic_memory_recovery_access.py b/tests/e2e/test_topic_memory_recovery_access.py index 3b25d4e8da..2ae619c8f3 100644 --- a/tests/e2e/test_topic_memory_recovery_access.py +++ b/tests/e2e/test_topic_memory_recovery_access.py @@ -26,6 +26,7 @@ from powercontext.server.factory import create_server_app from powercontext.server.settings import AccessControlConfig, McpConfig, MetricsConfig, ServerSettings from tests.builtin.runtime.test_topic_memory_terminal_frontier import _budget, _seed +from tests.e2e.server_storage import prepare_server_database from tests.e2e.test_access_control_regressions import ADMIN, _Authentication, _grant, _scope, _server from tests.e2e.test_topic_memory_terminal_frontier import _Handler, _Provider @@ -38,17 +39,19 @@ async def _recovery_server(tmp_path, backend, inference): yield app, client return database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'recovery.db'}") + settings = ServerSettings( + database=database, + inference=inference, + runtime=RuntimeConfig(artifact_processing_families=("topic-memory",)), + access=AccessControlConfig(mode="enforced", deployment_id="recovery", background_principal_id=ADMIN.id), + mcp=McpConfig(enabled=False), + metrics=MetricsConfig(enabled=False), + ) + await asyncio.to_thread(prepare_server_database, settings) async with open_builtin_access_control(database, bootstrap_administrators=(ADMIN,), deployment_id="recovery"): pass app = create_server_app( - settings=ServerSettings( - database=database, - inference=inference, - runtime=RuntimeConfig(artifact_processing_families=("topic-memory",)), - access=AccessControlConfig(mode="enforced", deployment_id="recovery", background_principal_id=ADMIN.id), - mcp=McpConfig(enabled=False), - metrics=MetricsConfig(enabled=False), - ), + settings=settings, authentication_provider=_Authentication(), scheduler_path=tmp_path / "scheduler.db", ) From bc235ffcbd55bcfef4c47cf7179c0b690510b441 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 23:10:58 +0800 Subject: [PATCH 47/94] fix(storage): preserve canonical schema in legacy tag maintenance Create the final SQLite tag table directly after renaming the legacy table. Preserve tag rows and rollback while avoiding the quoted final DDL produced by renaming a replacement table. Prepare managed real-backend fixtures and run explicit maintenance before ordinary Server admission. Refresh the persistence compatibility review without editing frozen revision resources. Constraint: Normal startup remains read-only and rejects incompatible schemas; this does not register an older Memory upgrade profile. Scope-risk: SQLite tag rebuilding and corresponding acceptance setup; MySQL maintenance behavior is unchanged. Tested: Pre-fix canonical regressions failed twice. Topic API, full Server migrations, release-gate and publication tests: 108 passed. Real seekDB/OceanBase tag upgrade and repeated startup: 4 passed. Ruff, Python 3.11 Ty, file-scoped prek including full Ty, upgrade policy and git diff --check passed. --- release/upgrade-support.json | 4 ++-- .../builtin/persistence/tag_schema.py | 14 ++++++------ tests/e2e/test_tag_schema_upgrade.py | 22 +++++++++++++++++++ tests/e2e/test_topic_memory_generic_api.py | 13 ++++++----- 4 files changed, 39 insertions(+), 14 deletions(-) diff --git a/release/upgrade-support.json b/release/upgrade-support.json index e276caa0c1..25c535f6f8 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "c9effa3d7af5cd54453076d9bacda97bc05e70f7c5e0eec0be3c2aa4071086b0", - "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles." + "persistence_sha256": "5ab61a61db3f66d2b9046e986f9b8af1149552f7abaf12c1c4f85e1c760af2b6", + "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback; frozen schema/revision resources remain unchanged. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/persistence/tag_schema.py b/src/powercontext/builtin/persistence/tag_schema.py index b0b98a64d5..aac906140e 100644 --- a/src/powercontext/builtin/persistence/tag_schema.py +++ b/src/powercontext/builtin/persistence/tag_schema.py @@ -22,7 +22,7 @@ async def ensure_topic_memory_tag_schema(connection: AsyncConnection) -> None: - """Upgrade the existing tag table during serialized deployment startup.""" + """Remove the legacy family constraint inside the caller's write transaction.""" if connection.dialect.name == "sqlite": # Start a real write transaction before DDL, including legacy sqlite3 mode. @@ -31,15 +31,15 @@ async def ensure_topic_memory_tag_schema(connection: AsyncConnection) -> None: if ddl is None or "ck_pc_artifact_tags_family" not in ddl: return create = str(CreateTable(ARTIFACT_TAGS_TABLE).compile(dialect=connection.dialect)) - await connection.exec_driver_sql( - create.replace("CREATE TABLE pc_artifact_tags", "CREATE TABLE pc_artifact_tags_topic_memory") - ) + # Create the final name directly: SQLite's rename rewrites its DDL with + # quotes, which would differ from the registered Server schema. + await connection.exec_driver_sql("ALTER TABLE pc_artifact_tags RENAME TO pc_artifact_tags_topic_memory") + await connection.exec_driver_sql(create) columns = ", ".join(column.name for column in ARTIFACT_TAGS_TABLE.columns) await connection.exec_driver_sql( - f"INSERT INTO pc_artifact_tags_topic_memory ({columns}) SELECT {columns} FROM pc_artifact_tags" # noqa: S608 + f"INSERT INTO pc_artifact_tags ({columns}) SELECT {columns} FROM pc_artifact_tags_topic_memory" # noqa: S608 ) - await connection.exec_driver_sql("DROP TABLE pc_artifact_tags") - await connection.exec_driver_sql("ALTER TABLE pc_artifact_tags_topic_memory RENAME TO pc_artifact_tags") + await connection.exec_driver_sql("DROP TABLE pc_artifact_tags_topic_memory") for index in ARTIFACT_TAGS_TABLE.indexes: await connection.run_sync(index.create) return diff --git a/tests/e2e/test_tag_schema_upgrade.py b/tests/e2e/test_tag_schema_upgrade.py index 9922c9eba7..1ba735dcad 100644 --- a/tests/e2e/test_tag_schema_upgrade.py +++ b/tests/e2e/test_tag_schema_upgrade.py @@ -30,11 +30,15 @@ from pydantic import SecretStr from sqlalchemy.engine import make_url +from powercontext.builtin.persistence.migrations.connections import MaintenanceConnections +from powercontext.builtin.persistence.migrations.pinned import finish, pinned from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile +from powercontext.builtin.persistence.tag_schema import ensure_topic_memory_tag_schema from powercontext.builtin.runtime import RuntimeConfig from powercontext.server.factory import create_server_app from powercontext.server.settings import AccessControlConfig, BearerAuthConfig, McpConfig, ServerSettings +from tests.e2e.server_storage import prepare_server_database @asynccontextmanager @@ -71,6 +75,9 @@ async def scenario(): access=AccessControlConfig(mode="disabled"), mcp=McpConfig(enabled=False), ) + await asyncio.to_thread( + prepare_server_database, settings, evidence_dir=tmp_path / "server-migration-evidence" + ) @asynccontextmanager async def client(): @@ -116,6 +123,21 @@ async def client(): f"ck_pc_artifact_tags_family CHECK (family IN ({families}))" ) + def upgrade_tags(): + def apply(connection, identity, guard): + assert connection is not None + guard() + finish(ensure_topic_memory_tag_schema(pinned(connection))) + connection.commit() + + MaintenanceConnections( + database, + evidence_directory=tmp_path / "tag-maintenance-evidence", + lock_coordination="single-host" if isinstance(database, OceanBaseConfig) else None, + ).run(apply, writable=True) + + await asyncio.to_thread(upgrade_tags) + # First startup still never uses Topic Memory; the second tests retry. for restart in range(2): async with client() as http: diff --git a/tests/e2e/test_topic_memory_generic_api.py b/tests/e2e/test_topic_memory_generic_api.py index 40c7728fbd..49656fb085 100644 --- a/tests/e2e/test_topic_memory_generic_api.py +++ b/tests/e2e/test_topic_memory_generic_api.py @@ -752,23 +752,26 @@ def test_legacy_tags_survive_transactional_upgrade_and_repeated_startup(tmp_path class InterruptedUpgrade(Exception): pass - async def interrupted_upgrade(): + async def upgrade(*, interrupt=False): engine = create_async_engine(f"sqlite+aiosqlite:///{database}") try: - with pytest.raises(InterruptedUpgrade): - async with engine.begin() as connection: - await ensure_topic_memory_tag_schema(connection) + async with engine.begin() as connection: + await ensure_topic_memory_tag_schema(connection) + if interrupt: raise InterruptedUpgrade finally: await engine.dispose() - asyncio.run(interrupted_upgrade()) + with pytest.raises(InterruptedUpgrade): + asyncio.run(upgrade(interrupt=True)) with sqlite3.connect(database) as connection: assert ( "ck_pc_artifact_tags_family" in connection.execute("SELECT sql FROM sqlite_master WHERE name = 'pc_artifact_tags'").fetchone()[0] ) assert connection.execute("SELECT * FROM pc_artifact_tags").fetchall() == rows + # Repair the isolated legacy constraint before normal Server admission. + asyncio.run(upgrade()) configuration_tags = {} for restart in range(2): with TestClient(_app(tmp_path)) as client: From 110cecc9dbbe719bfacd4b33bf636c8eb371cb9a Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 23:29:11 +0800 Subject: [PATCH 48/94] test(server): migrate legacy fixtures before normal admission Initialize managed storage before opening an injected authorization repository. Give the background and synchronous configurations independent parameterized fixtures, preserving rejection and supervisor assertions without silently changing persisted Processing ownership. Build the old Profile scenario from the registered 1.3.0 schema and apply the shared deployment runner before Server startup. Preserve unknown historical revision metadata and verify the original Profile remains readable and replaceable. Constraint: Current revision markers never authorize schema repair during normal startup. This does not register arbitrary legacy layouts. Scope-risk: Two test fixtures only; all existing public behavior assertions remain. Tested: Both original failures reproduced on the parent revision. Both affected modules: 39 passed. Ruff, Python 3.11 Ty, file-scoped prek including full Ty, and git diff --check passed. --- tests/test_dashboard_profile_handoff.py | 62 ++++++++++++++++++++----- tests/test_processing_security.py | 22 +++++---- 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/tests/test_dashboard_profile_handoff.py b/tests/test_dashboard_profile_handoff.py index 4f9fb6a034..54e9fb8bfa 100644 --- a/tests/test_dashboard_profile_handoff.py +++ b/tests/test_dashboard_profile_handoff.py @@ -23,12 +23,17 @@ from urllib.parse import parse_qs, urlencode, urlsplit import pytest +from alembic import command from fastapi.testclient import TestClient from markdown_it import MarkdownIt +from sqlalchemy import create_engine from powercontext.builtin.artifacts.prompt import PROMPT_KEYS, PromptRegistry from powercontext.builtin.artifacts.prompt.builtin import builtin_prompt_definitions +from powercontext.builtin.persistence.migrations.deployment import deployment_runner, server_bundle from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime.config import BuiltinConfig +from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest from powercontext.server.dashboard.api import DashboardAPI from powercontext.server.dashboard.pagination import PAGE_SIZE from powercontext.server.factory import create_server_app @@ -194,27 +199,60 @@ def test_rollback_is_offered_when_access_control_is_disabled(tmp_path: Path) -> def test_existing_database_without_revision_metadata_columns_keeps_working(tmp_path: Path) -> None: + settings = ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path}/existing.db"), + dashboard=DashboardConfig(enabled=True), + mcp=McpConfig(enabled=False), + ) + def open_app(): return create_server_app( - settings=ServerSettings( - database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path}/existing.db"), - dashboard=DashboardConfig(enabled=True), - mcp=McpConfig(enabled=False), - ), + settings=settings, scheduler_path=tmp_path / "scheduler.db", ) - with TestClient(open_app()) as client: - scope = create_scope(client, "Existing database")["scope_id"] - _profile(client, scope, "# First profile\n\nOriginal preference.") - # A database written before revision metadata existed has none of these columns. - with sqlite3.connect(tmp_path / "existing.db") as connection: - for column in ("created_at", "created_by", "restored_from_revision", "rollback_reason"): - connection.execute(f"ALTER TABLE pc_artifacts DROP COLUMN {column}") + # Use the registered release layout, then migrate before normal admission. + scope = "existing" + content = json.dumps({ + "schema": "powercontext.profile.v1", + "media_type": "text/markdown", + "content": "# First profile\n\nOriginal preference.\n", + "generation": {"mode": "manual_create", "created_at": "2026-01-01T00:00:00Z"}, + }).encode() + engine = create_engine(f"sqlite:///{tmp_path}/existing.db") + bundle = server_bundle() + try: + with engine.begin() as connection: + config = bundle.config(connection) + config.attributes["configuration"] = {"processing_manifest": canonical_processing_manifest(BuiltinConfig())} + command.upgrade(config, "s1300") + connection.exec_driver_sql("DROP TABLE pc_schema_revision") + connection.exec_driver_sql( + "INSERT INTO pc_scopes (scope_id,title,summary,scope_id_search,title_search,summary_search,version) " + "VALUES (?, 'Existing database', '', ?, 'existing database', '', 1)", + (scope, scope), + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifacts (scope_id,family,artifact_id,revision,content) VALUES (?, 'profile', 'profile', 1, ?)", + (scope, content), + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_heads (scope_id,family,artifact_id,revision) VALUES (?, 'profile', 'profile', 1)", + (scope,), + ) + columns = {row[1] for row in connection.exec_driver_sql("PRAGMA table_info(pc_artifacts)")} + assert not columns & {"created_at", "created_by", "restored_from_revision", "rollback_reason"} + finally: + engine.dispose() + runner = deployment_runner(settings) + plan = runner.plan() + assert plan.source_revision == "s1300" + assert runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).state == "ready" with TestClient(open_app()) as client: path = f"/v1/scopes/{scope}/artifacts/profile/profile" current = client.get(path) assert current.status_code == 200, current.text + assert current.json()["content"]["content"] == "# First profile\n\nOriginal preference.\n" assert current.json()["created_at"] is None changed = client.put( path, diff --git a/tests/test_processing_security.py b/tests/test_processing_security.py index b3aeb57eb8..18ffeb1a22 100644 --- a/tests/test_processing_security.py +++ b/tests/test_processing_security.py @@ -22,6 +22,7 @@ import pytest from pydantic import SecretStr +from powercontext.builtin.persistence.migrations.deployment import deployment_runner from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.runtime.composition import BuiltinConfigurationError from powercontext.builtin.runtime.config import InferenceConfig @@ -81,28 +82,33 @@ async def scenario(): asyncio.run(scenario()) -def test_server_rejects_injected_authorization_before_registering_actual_background_workers(tmp_path): +@pytest.mark.parametrize("generation_model", ["test", None]) +def test_server_rejects_injected_authorization_before_registering_actual_background_workers(tmp_path, generation_model): async def scenario(): config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'server.db'}") settings = ServerSettings( database=config, auth=BearerAuthConfig(token=SecretStr("test-token")), access=AccessControlConfig(mode="enforced", background_principal_id="worker"), - inference=InferenceConfig(generation_model="test"), + inference=InferenceConfig(generation_model=generation_model), ) + runner = deployment_runner(settings) + plan = await asyncio.to_thread(runner.plan) + result = await asyncio.to_thread(runner.apply, plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + assert result.state == "ready" async with SQLiteProfile.open(config, tables=ACCESS_TABLES) as profile: repository = RelationalAccessRepository(profile.database) access = AccessControlService( BuiltinAuthorizationProvider(repository), relationships=repository, audit=repository ) app = create_server_app(settings=settings, access_control=access) - with pytest.raises(BuiltinConfigurationError, match="reconstructible Authorization Provider"): + if generation_model is not None: + with pytest.raises(BuiltinConfigurationError, match="reconstructible Authorization Provider"): + async with app.router.lifespan_context(app): + pass + else: async with app.router.lifespan_context(app): - pass - settings = settings.model_copy(update={"inference": InferenceConfig()}) - app = create_server_app(settings=settings, access_control=access) - async with app.router.lifespan_context(app): - assert app.state.application.artifact_processing_supervisor is None + assert app.state.application.artifact_processing_supervisor is None asyncio.run(scenario()) From 8c581a02afe79fa5d0563c454d5710dfe629d35e Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Thu, 22 Oct 2026 23:51:42 +0800 Subject: [PATCH 49/94] fix(migrations): register the offline Memory archive Add immutable s1401 schema resources for the existing Memory archive on SQLite, seekDB and OceanBase. Keep s1300/s1400 resources and checksums unchanged and use the shared planner, DDL recovery and live validators for the next revision. Fresh managed databases include the empty archive; repeated apply preserves its historical bytes. Use dedicated raw maintenance connections in legacy Atomic Memory test fixtures so deliberately unready data can be inspected and repaired before ordinary business admission. Initialize remote managed grant fixtures explicitly, preserving all existing migration, authority, receipt and HTTP assertions. Document the three-revision chain in both locales and refresh the compatibility review. Constraint: Older Memory source layouts and optional vector/Code profiles still require separate registration. No arbitrary adoption, history deletion, startup repair or support-mode change is introduced. Scope-risk: Managed archive schema and offline test fixture integration; installed-release acceptance remains required. Tested: Two pre-fix regressions failed. Full Server/grant selection: 29 passed, 26 external-backend skips. Evidence/vector/Topic/CLI/release selection: 169 passed, 68 external-backend skips. Real seekDB/OceanBase grant and adapter-startup scenarios: 4 passed, zero skips. Previous s1400 to s1401 adjacent upgrade, repeat apply and live verification passed with unchanged prior revision checksums. Full make check and git diff --check passed. --- docs/en/docs/operate/database-migrations.md | 6 +- docs/zh/docs/operate/database-migrations.md | 6 +- release/upgrade-support.json | 4 +- .../migrations/server_resources/manifest.json | 16 ++- .../server_resources/mysql/versions/s1401.py | 24 ++++ .../server_resources/schemas/mysql-s1401.json | 126 ++++++++++++++++++ .../schemas/sqlite-s1401.json | 3 + .../versions/s1401_memory_archive.py | 28 ++++ .../test_full_server_migrations.py | 29 +++- tests/e2e/atomic_memory_migration_backend.py | 55 ++++++-- .../e2e/test_atomic_memory_grant_migration.py | 12 ++ 11 files changed, 291 insertions(+), 18 deletions(-) create mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/s1401.py create mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1401.json create mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-s1401.json create mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/versions/s1401_memory_archive.py diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index da2a65fed1..031b17dfe3 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -14,7 +14,7 @@ The design follows [RFC #1771](https://github.com/oceanbase/powercontext/pull/17 `powercontext server db-migrate status/plan/apply/verify` default to `--bundle server` and use the configured persistent SQLite, embedded seekDB, or OceanBase MySQL database. Explicit environment files are authoritative; arbitrary bundle paths are not accepted. -The full Server path registers the actual 1.3.0 no-vector, Code-disabled configuration as `s1300 → s1400`. It covers the complete schema, Candidate renaming and retained history, Dream task formats, Processing ownership, full-text projections, and existing Source references. Unknown layouts, extra objects, vector/Code configurations, and unregistered sources fail before writes. Older Memory collection conversion is outside this path. +The full Server path registers the actual 1.3.0 no-vector, Code-disabled configuration as `s1300 → s1400 → s1401`. It covers the complete schema, Candidate renaming and retained history, Dream task formats, Processing ownership, full-text projections, and existing Source references. Unknown layouts, extra objects, vector/Code configurations, and unregistered sources fail before writes. Older Memory collection conversion is outside this path. Only successful schema, domain, projection, and task validation returns `state=ready`, `readiness_scope=server`, and `server_ready=true`. Publication also requires installed-artifact acceptance on all three backends. An implemented path is not evidence that a release passed acceptance; consult the installation guide and release gate for support status. @@ -22,6 +22,8 @@ Stop all writers and drain queued/running 1.3.0 Dream tasks before upgrading: th Old Candidate tables are retained as `pc_retained_dream_candidate_heads` and `pc_retained_dream_candidate_versions` for migration-copy verification. They and the backup are not live write targets; subsequent business changes are not compared against old snapshots. New historical Artifact provenance fields remain unknown, and existing Source references are preserved. Projection rebuilds and verification scan their tables; budget downtime and capacity for backups and retained copies. +Revision `s1401` registers `pc_memory_artifact_archive` in the managed target schema on all three backends. It stores frozen Memory history for offline maintenance and has no public Artifact read path. Fresh databases start with an empty archive; repeat migration preserves its bytes, and runtime admission verifies its registered structure. This revision does not register additional historical source layouts. + Ordinary Server startup checks readiness without repairing an existing database. Empty local persistent SQLite/seekDB storage initializes through the same revision chain; OceanBase requires an explicit maintenance job first. In-memory SQLite remains transient runtime storage. A changed schema or inconsistent projection requires maintenance, not restart-based repair. Existing fresh deployments outside the no-vector, Code-disabled profile retain domain bootstrap and startup checks; they are not automatically adopted into this chain or covered by its upgrade support. Changing configuration cannot bypass admission or implicitly add vector/Code objects to managed storage. @@ -376,7 +378,7 @@ bundle for retry. Released revision resources and their recorded checksums remai Parameter-free DDL uses statement-scoped driver execution options so CHECK remainder operators and percent-containing SQL literals reach the backend unchanged. -Repositories run after schema checks. The full Server `s1300 → s1400` chain registers exact per-statement schema transitions and recovery prefixes through `mysql_state_layouts`. Domain steps attach to the shared revision and use its connection, lock, and recovery point; verification precedes version advancement. Read-only live checks are separate from migration-copy validation. Vector configurations still require separate registration. +Repositories run after schema checks. The full Server `s1300 → s1400 → s1401` chain registers exact per-statement schema transitions and recovery prefixes through `mysql_state_layouts`. Domain steps attach to the shared revision and use its connection, lock, and recovery point; verification precedes version advancement. Read-only live checks are separate from migration-copy validation. Vector configurations still require separate registration. One version table offers no generic run-ID resume. After interruption, inspect actual schema/data through read-only commands. Ambiguous state returns `recovery_required`; never blindly stamp or replace the original recovery point with a diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index e89bf0ce91..7dc0e6ad89 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -12,7 +12,7 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 `powercontext server db-migrate status/plan/apply/verify` 默认选择 `--bundle server`，使用部署配置中的持久 SQLite、嵌入式 seekDB 或 OceanBase MySQL 模式数据库。维护命令以显式环境文件为准，不接受任意测试 bundle 路径。 -完整 Server 路径登记了真实 1.3.0 的无向量、Code 关闭配置，版本链为 `s1300 → s1400`。它覆盖完整数据库结构、Candidate 改名与历史保留、Dream 任务格式、Processing 所有权声明、全文检索投影和既有 Source 引用。未知结构、额外对象、不同向量/Code 配置及未登记来源均在写入前拒绝。更早的 Memory 集合转换不属于此路径。 +完整 Server 路径登记了真实 1.3.0 的无向量、Code 关闭配置，版本链为 `s1300 → s1400 → s1401`。它覆盖完整数据库结构、Candidate 改名与历史保留、Dream 任务格式、Processing 所有权声明、全文检索投影和既有 Source 引用。未知结构、额外对象、不同向量/Code 配置及未登记来源均在写入前拒绝。更早的 Memory 集合转换不属于此路径。 只有结构、领域数据、搜索投影和任务校验全部通过，才返回 `state=ready`、`readiness_scope=server`、`server_ready=true`。发布还必须通过真实安装包在三个后端上的完整验收；实现路径存在不等于某次发布已通过验收，支持状态以安装指南和发布门禁为准。 @@ -20,6 +20,8 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 Candidate 旧表保留为 `pc_retained_dream_candidate_heads` 和 `pc_retained_dream_candidate_versions`，用于迁移当次的复制验证。它们与迁移备份都不是线上写入目标；升级后的正常修改不再与旧快照比较。历史 Artifact 新增的来源时间/操作者字段保持未知，既有 Source 引用保持不变。投影重建和数据校验会扫描相关表；请为停机时间、备份与保留副本安排容量。 +`s1401` 在三种后端的托管目标结构中登记 `pc_memory_artifact_archive`。该表保存供离线维护使用的 Memory 历史快照，不提供公开 Artifact 读取入口。新数据库的归档表为空；重复迁移保留其原始字节，运行时准入检查验证登记的表结构。本 revision 不新增对其他历史来源结构的支持。 + 普通 Server 启动先做只读就绪检查，不修补已有库。空的本地持久 SQLite/seekDB 可通过同一版本链初始化；OceanBase 必须先执行显式维护命令初始化。内存 SQLite 仍为临时运行时存储。迁移后发现结构或投影被篡改，必须停写维护，不能靠重启修复。 无向量、Code 关闭之外的既有新建部署仍使用原领域初始化和启动检查，不会自动纳入此版本链；这些配置不属于已登记升级范围。受管数据库不能通过切换配置绕过检查或隐式增加向量/Code 表。 @@ -239,7 +241,7 @@ MySQL 结构步骤通过冻结的布局片段注册，由 `manifest.json` 的 `m 计划通过完整 bundle 摘要绑定注册资源，未完成维护的重试要求使用同一 bundle。已发布 revision 资源及其记录摘要保持不可改写。 无参数 DDL 使用单语句驱动选项，确保 CHECK 的取余运算符和 SQL 字面量中的百分号原样到达后端。 -Repository 仅在结构满足后运行。完整 Server 的 `s1300 → s1400` 链用 `mysql_state_layouts` 登记逐语句的精确结构变化和可恢复前缀。领域步骤附着在共享 revision 上，使用同一连接、锁和恢复点；校验成功后才推进版本。只读 live 校验与迁移当次的保留副本校验分离。向量配置仍需独立登记。 +Repository 仅在结构满足后运行。完整 Server 的 `s1300 → s1400 → s1401` 链用 `mysql_state_layouts` 登记逐语句的精确结构变化和可恢复前缀。领域步骤附着在共享 revision 上，使用同一连接、锁和恢复点；校验成功后才推进版本。只读 live 校验与迁移当次的保留副本校验分离。向量配置仍需独立登记。 单版本表不提供通用 run ID 续跑。中断后先用只读命令检查真实结构和数据；含混状态返回 `recovery_required`，不盲目 stamp、不用部分状态的备份替换原恢复点。库外摘要在 DDL 前记录执行及备份信息；重试保留原维护窗口和恢复点。在恢复条件允许时可显式重新确认 manual／skip 策略，但不绕过结构验证，也不替代必需的进度证据。 diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 25c535f6f8..1fb3f695eb 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "5ab61a61db3f66d2b9046e986f9b8af1149552f7abaf12c1c4f85e1c760af2b6", - "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback; frozen schema/revision resources remain unchanged. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles." + "persistence_sha256": "d67654098c4404726f70c7c262f0ecca969cf4691aa922ab59d099022de538e7", + "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback. New revision s1401 registers the frozen offline Memory archive on all three backends through the shared chain; s1300/s1400 resources and checksums remain unchanged, and archived history is preserved. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json index 78d996e08a..bc932749bb 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json @@ -12,6 +12,10 @@ "dream/manifest.json", "mysql_operations.py", "schemas/mysql-s1400.json" + ], + "s1401": [ + "schemas/sqlite-s1401.json", + "schemas/mysql-s1401.json" ] }, "sqlite_baselines": { @@ -23,6 +27,9 @@ ], "s1400": [ "9064ecb0c6233884e64312a1a9a48a6c52b7c379724fa83f9b77b90ed1ee0381" + ], + "s1401": [ + "8752beed9acfb66e03d4360f8652661107b58067ae7a9c461f1035345edf4f27" ] }, "affected_objects": [ @@ -114,7 +121,8 @@ "pc_topic_memory_work_blocks", "pc_topic_memory_work_budgets", "pc_retained_dream_candidate_heads", - "pc_retained_dream_candidate_versions" + "pc_retained_dream_candidate_versions", + "pc_memory_artifact_archive" ], "readiness_scope": "server", "retained_objects": [ @@ -146,6 +154,9 @@ "domain_readiness": { "s1400": [ "server_upgrade" + ], + "s1401": [ + "server_upgrade" ] }, "retained_fingerprints": { @@ -158,7 +169,8 @@ }, "mysql_state_layouts": [ "schemas/mysql-s1300.json", - "schemas/mysql-s1400.json" + "schemas/mysql-s1400.json", + "schemas/mysql-s1401.json" ], "mysql_history": "mysql" } diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/s1401.py b/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/s1401.py new file mode 100644 index 0000000000..1c524326af --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/mysql/versions/s1401.py @@ -0,0 +1,24 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Register offline Memory history through the shared MySQL DDL executor.""" + +from alembic import context + +revision = "s1401" +down_revision = "s1400" + + +def upgrade() -> None: + context.config.attributes["execute_revision"](revision) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1401.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1401.json new file mode 100644 index 0000000000..3e2fe4c39b --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1401.json @@ -0,0 +1,126 @@ +{ + "format": "powercontext.mysql-state-layouts.v1", + "revisions": { + "s1401": { + "statements": [ + "CREATE TABLE pc_memory_artifact_archive (\n\tscope_id VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent LONGBLOB NOT NULL, \n\tmemory_citations LONGBLOB, \n\tmetadata LONGBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)" + ], + "changes": [ + { + "pc_memory_artifact_archive": { + "columns": { + "scope_id": { + "type": [ + "varchar", + [ + "256" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "family": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "artifact_id": { + "type": [ + "varchar", + [ + "128" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "revision": { + "type": [ + "int", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "content": { + "type": [ + "longblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "memory_citations": { + "type": [ + "longblob", + [] + ], + "nullable": true, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "metadata": { + "type": [ + "longblob", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "scope_id", + null, + "asc" + ], + [ + "family", + null, + "asc" + ], + [ + "artifact_id", + null, + "asc" + ], + [ + "revision", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": {} + } + } + ] + } + } +} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-s1401.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-s1401.json new file mode 100644 index 0000000000..8e3b8d87c4 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/sqlite-s1401.json @@ -0,0 +1,3 @@ +{ + "create": "CREATE TABLE pc_memory_artifact_archive (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent BLOB NOT NULL, \n\tmemory_citations BLOB, \n\tmetadata BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)" +} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/versions/s1401_memory_archive.py b/src/powercontext/builtin/persistence/migrations/server_resources/versions/s1401_memory_archive.py new file mode 100644 index 0000000000..b75645dfee --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/versions/s1401_memory_archive.py @@ -0,0 +1,28 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Register offline Memory history without exposing it through live Artifacts.""" + +import json +from pathlib import Path + +from alembic import op + +revision = "s1401" +down_revision = "s1400" + + +def upgrade(): + resource = Path(__file__).parents[1] / "schemas/sqlite-s1401.json" + op.get_bind().exec_driver_sql(json.loads(resource.read_text())["create"]) diff --git a/tests/builtin/persistence/test_full_server_migrations.py b/tests/builtin/persistence/test_full_server_migrations.py index 9469020178..8b1db6857d 100644 --- a/tests/builtin/persistence/test_full_server_migrations.py +++ b/tests/builtin/persistence/test_full_server_migrations.py @@ -73,7 +73,8 @@ def dump(path): def test_complete_upgrade_preserves_history_backup_and_repeated_apply(source): before = dump(source.database) plan = source.plan() - assert plan.source_revision == "s1300" and plan.target_revision == "s1400" + assert plan.source_revision == "s1300" and plan.target_revision == "s1401" + assert plan.revisions == ("s1400", "s1401") assert dump(source.database) == before result = apply(source) assert result.readiness_scope == "server" and result.backup_state == "completed" @@ -88,6 +89,32 @@ def test_complete_upgrade_preserves_history_backup_and_repeated_apply(source): assert dump(source.database) == after +def test_archived_memory_history_survives_repeat_apply_and_runtime_admission(source): + apply(source) + metadata = json.dumps({ + "format": "powercontext.memory-artifact-archive.v1", + "lineage": {"sources": [], "artifacts": []}, + "incoming_references": [], + "collection": {"head": None, "owners": [], "tags": [], "bindings": [], "idempotency": []}, + }).encode() + archived = ("scope", "memory", "legacy", 2, b'{"history":"unchanged"}', b"[]", metadata) + with closing(sqlite3.connect(source.database)) as connection, connection: + connection.execute("INSERT INTO pc_memory_artifact_archive VALUES (?,?,?,?,?,?,?)", archived) + assert not apply(source).changed + assert source.verify().readiness_scope == "server" + + async def read_archive(): + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{source.database}") + async with ( + SQLiteProfile.open(config, tables=()) as profile, + profile.database.transaction() as connection, + ): + rows = await connection.exec_driver_sql("SELECT * FROM pc_memory_artifact_archive") + assert [tuple(row) for row in rows] == [archived] + + asyncio.run(read_archive()) + + def test_migration_revokes_old_fence_and_allows_expiring_worker_lease(source): with closing(sqlite3.connect(source.database)) as connection, connection: connection.execute( diff --git a/tests/e2e/atomic_memory_migration_backend.py b/tests/e2e/atomic_memory_migration_backend.py index e22f52c115..78b3ea875d 100644 --- a/tests/e2e/atomic_memory_migration_backend.py +++ b/tests/e2e/atomic_memory_migration_backend.py @@ -16,16 +16,23 @@ from __future__ import annotations -from contextlib import AbstractAsyncContextManager +from collections.abc import AsyncIterator +from contextlib import AsyncExitStack, asynccontextmanager from dataclasses import dataclass from pathlib import Path +from sqlalchemy.ext.asyncio import create_async_engine + from powercontext.builtin.artifacts.memory import EmbeddingProfile from powercontext.builtin.persistence.atomic_memory_index import AtomicMemoryIndex -from powercontext.builtin.persistence.oceanbase import OceanBaseProfile +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile +from powercontext.builtin.persistence.oceanbase import profile as oceanbase_profile from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile +from powercontext.builtin.persistence.seekdb import profile as seekdb_profile from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.sqlite import profile as sqlite_profile from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex from powercontext.builtin.runtime.config import DatabaseConfig @@ -36,16 +43,46 @@ class MigrationBackend: directory: Path -def migration_profile( +@asynccontextmanager +async def migration_profile( config: DatabaseConfig, *, load_vector_extension: bool = False, -) -> AbstractAsyncContextManager[SQLiteProfile | SeekDBProfile | OceanBaseProfile]: - if isinstance(config, SQLiteConfig): - return SQLiteProfile.open(config, tables=(), load_vector_extension=load_vector_extension) - if isinstance(config, SeekDBConfig): - return SeekDBProfile.open(config, tables=()) - return OceanBaseProfile.open(config, tables=()) +) -> AsyncIterator[SQLiteProfile | SeekDBProfile | OceanBaseProfile]: + # Maintenance must inspect and repair the deliberately unready fixture. + # Business profile admission would reject it before the migration can run. + async with AsyncExitStack() as resources: + if isinstance(config, SQLiteConfig): + sqlite_profile._create_database_directory(config.url) + engine = create_async_engine(config.url, echo=config.echo, hide_parameters=True) + sqlite_profile._configure_sqlite(engine, config, load_vector_extension=load_vector_extension) + profile_type = SQLiteProfile + elif isinstance(config, SeekDBConfig): + path = config.path.expanduser().resolve() + path.parent.mkdir(parents=True, exist_ok=True) + instance = await seekdb_profile._open_instance(seekdb_profile._load_binding(), path) + resources.callback(instance.close) + engine = seekdb_profile._create_engine(config, instance.connection_options()) + profile_type = SeekDBProfile + else: + assert isinstance(config, OceanBaseConfig) + oceanbase_profile._register_official_dialect() + engine = create_async_engine( + config.url.get_secret_value(), + connect_args={"init_command": "SET autocommit = 0"}, + echo=config.echo, + hide_parameters=True, + pool_pre_ping=config.pool_pre_ping, + ) + profile_type = OceanBaseProfile + database = AsyncDatabase.own(engine) + resources.push_async_callback(database.close) + if isinstance(config, SQLiteConfig): + await sqlite_profile._warm_sqlite(engine, config) + elif isinstance(config, OceanBaseConfig): + async with database.transaction() as connection: + await oceanbase_profile._require_mysql_tenant(connection) + yield profile_type(database=database, tables=()) def migration_index(config: DatabaseConfig, profile: EmbeddingProfile | None = None) -> AtomicMemoryIndex: diff --git a/tests/e2e/test_atomic_memory_grant_migration.py b/tests/e2e/test_atomic_memory_grant_migration.py index 5d20b82bd0..057adbe63a 100644 --- a/tests/e2e/test_atomic_memory_grant_migration.py +++ b/tests/e2e/test_atomic_memory_grant_migration.py @@ -64,6 +64,7 @@ from powercontext.server.middleware import AuthenticationMiddleware from powercontext.server.settings import AccessControlConfig, BearerAuthConfig, McpConfig, MetricsConfig, ServerSettings from tests.e2e.atomic_memory_migration_backend import MigrationBackend, migration_index, migration_profile +from tests.e2e.server_storage import prepare_server_database ACTOR = PrincipalRef(type="service", id="server-token", description="PowerContext static bearer") CREATED_AT = datetime(2026, 1, 1, tzinfo=UTC) @@ -289,6 +290,17 @@ async def _apply(migration_backend: MigrationBackend): def _prepare(migration_backend: MigrationBackend, **options): + prepare_server_database( + ServerSettings( + database=migration_backend.config, + runtime=RuntimeConfig(artifact_processing_families=()), + access=AccessControlConfig(mode="enforced"), + auth=BearerAuthConfig(enabled=True, token=SecretStr("migration-test-token")), + mcp=McpConfig(enabled=False), + metrics=MetricsConfig(enabled=False), + ), + evidence_dir=migration_backend.directory / "server-migration-evidence", + ) with _client(migration_backend) as client: scope_id = client.get("/v1/scopes/default").json()["scope_id"] return asyncio.run(_seed_legacy(migration_backend, scope_id, **options)) From f7a328c2b82535051ba4de75a132b31ae89173bd Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 00:25:46 +0800 Subject: [PATCH 50/94] fix(migrations): connect managed upgrades to local service maintenance Bind the owned local registration, effective configuration and target executable to full Server migration plans. Stop and protect the service through migration and live verification, then switch only the confirmed registration. Allow a reviewed service transition after the database is already ready so interrupted completion can resume. Constraint: Partial bundles retain their no-op behavior and cannot switch a business service; other writers still require explicit maintenance coordination. Scope-risk: Service-controller integration uses a simulated native manager with actual SQLite migration fixtures; this does not establish native platform acceptance. Tested: Windows focused migration/controller selection 95 passed, 2 platform skips; Linux same selection 97 passed; full make check passed; focused Python 3.11 Ty, Ruff and git diff --check passed. --- docs/en/docs/operate/database-migrations.md | 32 +-- docs/zh/docs/operate/database-migrations.md | 18 +- src/powercontext/server/database_migration.py | 44 ++++- tests/test_service_maintenance.py | 186 +++++++++++++++++- 4 files changed, 256 insertions(+), 24 deletions(-) diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 031b17dfe3..3f51aefd2e 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -299,15 +299,23 @@ activation suppressed until an explicit `service start`; an unverified migration They manage only the current user's PC-owned local service, not a cluster or other clients. `stop` suppresses automatic restart during maintenance; `start` restores startup. One `restart` cannot replace stopped-write migration. -The complete managed-upgrade design drains requests, stops the service and automatic restarts, takes the migration lock, -reinspects, applies backup policy, migrates, verifies, switches the launch definition to the accepted new executable, and -starts a previously running service with readiness checks. Migration failure keeps it stopped; a previously stopped -service stays stopped. Database verification and service startup are reported separately. Resolve startup failures without -rerunning a successful migration. - -`--manage-service` always covers only the accepted local service. It cannot replace `--maintenance-confirmed` for other -writers and shared nodes. The Phase A partial bundle does not establish complete Server readiness: when persistent changes -are needed, `--manage-service` returns `service_unsupported` rather than switching a business service. +For `--bundle server`, `--manage-service` binds the owned local registration and target executable to the reviewed plan. +Use the registered environment file and data directory; a different effective Server configuration is rejected before +stopping the service. Stop incoming traffic and drain work first. Apply suppresses automatic restarts, stops the local +service, takes the migration lock, reinspects, applies the backup policy, migrates and verifies, then switches the launch +definition to the accepted executable. It starts a previously running service and checks readiness; a previously stopped +service stays stopped. Migration failure keeps the old registration stopped and protected against automatic activation. + +Database verification and service startup are reported separately. `service_start_failed` with `database_state=ready` +means the schema migration succeeded; fix the startup problem and use `powercontext service start`. If execution stopped +after database verification but before switching the registration, review and apply a new `--manage-service` plan. It can +finish the service transition when the database is already ready, without rerunning completed revisions. + +`--manage-service` covers only the accepted local service. It cannot replace `--maintenance-confirmed` for other writers +and shared nodes. The partial Artifact and Topic recovery bundles reject service management when changes are needed; +an already-ready no-op does not touch the service. They cannot establish complete Server readiness or switch a business +service. Controller tests use a simulated native manager with a real +SQLite migration; they do not establish native service-manager acceptance on every supported operating system. For manual deployment, stop the old service first. After a supported complete migration succeeds, ensure the launch definition points to the new environment before starting it. For foreground deployment, exit the old process and use the @@ -334,7 +342,8 @@ without every handler supporting two table layouts. Legacy tasks require inspection in the read-only plan, after stopped-write locking, after migration, and before Worker startup. Unknown formats or unobservable sources must not be reported as an empty queue. Use declared compatible consumption, idempotent conversion, or bounded draining. Preserve task IDs, idempotency keys, Scope, retries, and domain -receipts. The Phase A bundle does not yet cover these complete business checks. +receipts. The partial Artifact bundle does not cover these complete business checks. The full Server path checks its +registered 1.3.0 Dream formats and rejects queued/running old tasks before migration; terminal payloads remain intact. ## Storage layers and recovery boundaries @@ -396,4 +405,5 @@ The existing Scope discovery startup migration inspects the actual search-column storage. If an earlier startup stopped after adding nullable columns or backfilling them, the next startup fills any remaining missing values from the original text and completes `NOT NULL` constraints. Columns already marked `NOT NULL` do not receive another `MODIFY COLUMN`. Original Scope titles, summaries, external references, and binding -identifiers are preserved. This compatibility repair is separate from the four-table unified migration bundle above. +identifiers are preserved. This compatibility repair remains for unmanaged profiles. Managed Server storage uses the +registered migration chain and read-only startup admission; its startup never invokes this repair to upgrade the schema. diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 7dc0e6ad89..b63657a29d 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -210,9 +210,20 @@ powercontext service restart 这些命令只管理当前用户归 PC 所有的服务，不管理集群或其他客户端。`stop` 禁止维护期间自动拉起旧服务；`start` 恢复启动。涉及迁移时不能用一次 `restart` 替代停写维护。 -完整升级的托管设计为：排空请求、停服并禁止自动重启 → 取迁移锁并复核 → 按策略备份 → 迁移 → 验证 → 将启动定义切换到已确认新版程序 → 启动原本运行的服务 → readiness。迁移失败保持停服；原本停止的服务保持停止。数据库验证成功而服务启动失败时分别报告，处理启动问题而不重跑迁移。 +`--bundle server` 的 `--manage-service` 将本机自有服务注册和目标可执行程序绑定到待确认的计划。 +必须使用注册时的环境文件和数据目录；实际 Server 配置不一致时，在停止服务前拒绝执行。 +先停止流量并排空旧任务，再应用计划。应用会抑制自动重启、停止本机服务、取得迁移锁、重新检查并执行备份策略， +完成迁移和数据库验证后才切换注册。原先运行的服务会重新启动并检查 readiness；原先停止的服务保持停止。 +迁移失败时保留旧注册和停止保护，不恢复自动启动。 -`--manage-service` 的范围始终只是已确认的本机服务，不能代替 `--maintenance-confirmed` 对其他写入者和共享库节点的声明。阶段 A 的有限 bundle 未完成完整 Server readiness 验收；需要持久化变更时，`--manage-service` 返回 `service_unsupported`，不能据此自动切换生产服务。 +数据库验证与服务启动分别报告。`service_start_failed` 且 `database_state=ready` 表示迁移已成功， +修复启动问题后使用 `powercontext service start`。如果在数据库验证之后、切换服务注册之前中断， +可以重新审阅并应用 `--manage-service` 计划；数据库已就绪时也能完成服务切换，不重复执行已完成的 revision。 + +`--manage-service` 仅管理已确认的本机服务，不能代替 `--maintenance-confirmed` 对其他写入者和共享库节点的声明。 +部分 Artifact 和 Topic recovery bundle 需要变更时返回 `service_unsupported`；已就绪的空操作不触碰服务。 +它们不能用于完整 Server readiness 或业务服务切换。 +控制器测试使用模拟的原生服务管理器和真实 SQLite 迁移，不代表所有支持操作系统的原生服务验收。 手动管理时，先停旧服务，在已支持的完整迁移成功后确认服务启动定义指向新版，再启动。前台部署则退出旧进程，使用新版环境的现有入口： @@ -251,4 +262,5 @@ Repository 仅在结构满足后运行。完整 Server 的 `s1300 → s1400 → 现有 Scope discovery 启动迁移会检查 MySQL 兼容存储中搜索列的实际可空性。如果上次启动在添加可空列或回填后中断， 下次启动会根据原文补齐剩余缺失值，再完成 `NOT NULL` 约束。已为 `NOT NULL` 的列不会再次执行 `MODIFY COLUMN`。 -Scope 的原始标题、摘要、外部引用和 binding 标识保持不变。此兼容修复独立于上述四表统一迁移 bundle。 +Scope 的原始标题、摘要、外部引用和 binding 标识保持不变。此兼容修复仅供未受管理的配置使用。 +受管理的 Server 存储使用登记的迁移链和只读启动准入，不会在启动时调用此修复来升级结构。 diff --git a/src/powercontext/server/database_migration.py b/src/powercontext/server/database_migration.py index 900461f398..5f194bb56c 100644 --- a/src/powercontext/server/database_migration.py +++ b/src/powercontext/server/database_migration.py @@ -33,7 +33,9 @@ resolve_server_environment_file, server_settings_context, ) +from powercontext.server.settings import ServerSettings from powercontext.service.controller import ServiceController +from powercontext.service.environment import ProtectedEnvironmentFileError, load_protected_environment_file from powercontext.service.model import ServiceError BackupPolicy = Literal["auto", "manual", "skip"] @@ -111,7 +113,9 @@ def plan( runner = deployment_runner( settings, evidence_dir=evidence_dir, lock_coordination=lock_coordination, bundle=bundle ) - service = _service_scope(_environment_file(env_file)) if manage_service else None + service = ( + _service_scope(_environment_file(env_file), bundle=bundle, settings=settings) if manage_service else None + ) reviewed = runner.plan(backup_policy=backup, service=service, shared_database=shared_database) _write_payload(reviewed) @@ -164,10 +168,12 @@ def apply( policy: BackupPolicy = backup or "auto" initial = runner.plan(backup_policy=policy, shared_database=shared_database) _check_previous_run(initial, acknowledge_previous_run) - if initial.state == "ready": + if initial.state == "ready" and (not manage_service or bundle != "server"): _write_payload(runner.verify()) return - service = _service_scope(_environment_file(env_file)) if manage_service else None + service = ( + _service_scope(_environment_file(env_file), bundle=bundle, settings=settings) if manage_service else None + ) reviewed = runner.plan(backup_policy=policy, service=service, shared_database=shared_database) _check_previous_run(reviewed, acknowledge_previous_run) if policy == "auto" and not reviewed.backup_available and reviewed.state != "uninitialized": @@ -273,12 +279,31 @@ def _write_confirmation(reviewed: MigrationPlan, policy: BackupPolicy) -> None: typer.echo("Old tables and recovery points are retained.") -def _service_scope(env_file: Path | None) -> dict[str, object]: - del env_file - raise MigrationError( - "service_unsupported", - "This partial acceptance bundle cannot establish complete Server readiness or switch a business service.", - ) +def _service_scope(env_file: Path | None, *, bundle: str, settings: ServerSettings) -> dict[str, object]: + if bundle != "server": + raise MigrationError( + "service_unsupported", + "This partial acceptance bundle cannot establish complete Server readiness or switch a business service.", + ) + summary = ServiceController().maintenance_summary(env_file=env_file) + registered = summary.definition + try: + environment = ( + {} + if registered.env_file is None + else load_protected_environment_file(Path(registered.env_file.path), expected=registered.env_file).values + ) + except ProtectedEnvironmentFileError as error: + raise MigrationError( + "service_configuration_mismatch", "The registered service environment changed; review its configuration." + ) from error + with server_settings_context(environment=environment, data_dir=Path(registered.data_dir)) as service_settings: + if settings != service_settings: + raise MigrationError( + "service_configuration_mismatch", + "Use the registered service environment and data directory for its database migration.", + ) + return summary.as_dict() def _apply_reviewed( @@ -311,6 +336,7 @@ def migrate() -> MigrationResult: raise MigrationError("service_unsupported", "The local service does not provide a stable maintenance identity.") with ServiceController().maintenance(expected_fingerprint=fingerprint, env_file=env_file) as session: result = migrate() + runner.verify() try: session.complete() except (ServiceError, OSError): diff --git a/tests/test_service_maintenance.py b/tests/test_service_maintenance.py index 5ebe4dcb85..6800af0cc3 100644 --- a/tests/test_service_maintenance.py +++ b/tests/test_service_maintenance.py @@ -23,9 +23,17 @@ from pathlib import Path import pytest +from alembic import command +from sqlalchemy import create_engine from typer.testing import CliRunner import powercontext.service.cli as service_cli +from powercontext.builtin.persistence.migrations.deployment import deployment_runner, server_bundle +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.cli.app import create_cli +from powercontext.server import database_migration +from powercontext.server.cli import app as server_app +from powercontext.server.configuration import server_settings_context from powercontext.service.adapters.base import service_python_executable from powercontext.service.adapters.systemd import SystemdUserAdapter from powercontext.service.adapters.windows import WindowsTaskSchedulerAdapter @@ -41,7 +49,183 @@ ServiceError, ) from powercontext.service.probe import probe_readiness -from tests.test_service import FakeAdapter, _definition, _LivenessHandler, _manager_probe, _serve +from tests.test_service import FakeAdapter, _definition, _LivenessHandler, _manager_probe, _secure_windows_file, _serve + + +@pytest.fixture +def managed_upgrade(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + for name in tuple(os.environ): + if name == "POWERCONTEXT_HOME" or name.startswith("POWERCONTEXT_SERVER_"): + monkeypatch.delenv(name) + database = tmp_path / "server.db" + environment = tmp_path / "deployment.env" + environment.write_text( + f"POWERCONTEXT_HOME={tmp_path.as_posix()}\n" + "POWERCONTEXT_SERVER_DATABASE_KIND=sqlite\n" + f"POWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite:///{database.as_posix()}\n", + encoding="utf-8", + ) + environment.chmod(0o600) + _secure_windows_file(environment) + with server_settings_context(env_file=environment) as settings: + runner = deployment_runner(settings) + bundle = server_bundle() + engine = create_engine(f"sqlite:///{database}") + try: + with engine.begin() as connection: + config = bundle.config(connection) + config.attributes["configuration"] = runner.configuration + command.upgrade(config, "s1300") + connection.exec_driver_sql("DROP TABLE pc_schema_revision") + finally: + engine.dispose() + controller, adapter = _installed(tmp_path) + assert adapter.definition is not None + from powercontext.service.environment import load_protected_environment_file + + adapter.definition = replace( + adapter.definition, + data_dir=str(tmp_path), + env_file=load_protected_environment_file(environment).identity, + ) + adapter.content = adapter.render(adapter.definition) + adapter.loaded_definition = adapter.definition + monkeypatch.setattr(database_migration, "ServiceController", lambda: controller) + cli = create_cli([server_app]) + + def invoke(action: str, *options: str): + return CliRunner().invoke( + cli, + ["server", "db-migrate", action, "--env-file", str(environment), "--manage-service", *options], + ) + + def apply(plan): + return invoke( + "apply", + "--backup", + "auto", + "--plan-id", + plan["plan_id"], + "--yes", + "--maintenance-confirmed", + ) + + return controller, adapter, runner, environment, invoke, apply + + +@pytest.mark.parametrize("running", [True, False]) +def test_full_server_migration_switches_service_and_preserves_original_running_state(managed_upgrade, running): + controller, adapter, runner, _environment, invoke, apply = managed_upgrade + if not running: + controller.stop() + original = adapter.definition + planned = invoke("plan") + assert planned.exit_code == 0, planned.output + plan = json.loads(planned.output) + assert plan["service"]["originally_running"] is running + assert adapter.definition == original + assert runner.plan().state == "migration_required" + + migrated = apply(plan) + + assert migrated.exit_code == 0, migrated.output + assert json.loads(migrated.output)["server_ready"] is True + assert runner.verify().state == "ready" + assert adapter.definition is not None + assert adapter.definition.python_executable == service_python_executable() + assert adapter.manager is (ManagerState.ACTIVE if running else ManagerState.INACTIVE) + assert adapter.suspended is not running + + +def test_failed_full_server_migration_keeps_service_stopped_and_can_resume(managed_upgrade, monkeypatch): + controller, adapter, runner, _environment, invoke, apply = managed_upgrade + original = adapter.definition + planned = invoke("plan") + assert planned.exit_code == 0, planned.output + plan = json.loads(planned.output) + from powercontext.builtin.persistence.migrations.domains import DomainRegistry + + original_invoke = DomainRegistry._invoke + + def interrupted(registry, step, operation, connection, configuration): + original_invoke(registry, step, operation, connection, configuration) + if step.id == "server_upgrade" and operation == "verify": + raise MigrationError("injected_failure", "Interrupted domain verification") + + with monkeypatch.context() as fault: + fault.setattr(DomainRegistry, "_invoke", interrupted) + failed = apply(plan) + assert failed.exit_code != 0 + assert json.loads(failed.output)["error"] == "injected_failure" + assert adapter.definition == original + assert adapter.manager is ManagerState.INACTIVE + assert adapter.suspended + with pytest.raises(ServiceError, match="not been verified"): + controller.start() + retry = invoke("plan") + assert retry.exit_code == 0, retry.output + resumed = apply(json.loads(retry.output)) + assert resumed.exit_code == 0, resumed.output + assert runner.verify().state == "ready" + assert adapter.manager is ManagerState.ACTIVE + + +def test_service_start_failure_reports_ready_database_and_retries_only_service(managed_upgrade, monkeypatch): + controller, adapter, runner, _environment, invoke, apply = managed_upgrade + planned = invoke("plan") + assert planned.exit_code == 0, planned.output + with monkeypatch.context() as fault: + fault.setattr(controller, "_readiness_probe", lambda _: ProbeResult(ProbeState.CONFLICT, "startup failed")) + failed = apply(json.loads(planned.output)) + assert failed.exit_code != 0 + result = json.loads(failed.output) + assert result["error"] == "service_start_failed" + assert result["database_state"] == "ready" + assert adapter.suspended + assert adapter.manager is ManagerState.INACTIVE + before = runner.database.read_bytes() + assert controller.start().ok + assert runner.database.read_bytes() == before + + +def test_database_ready_before_service_switch_can_complete_managed_retry(managed_upgrade): + controller, adapter, runner, environment, invoke, apply = managed_upgrade + summary = controller.maintenance_summary(env_file=environment) + with controller.maintenance(expected_fingerprint=summary.fingerprint, env_file=environment): + plan = runner.plan() + runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + original = adapter.definition + before = runner.database.read_bytes() + planned = invoke("plan") + assert planned.exit_code == 0, planned.output + completed = apply(json.loads(planned.output)) + assert completed.exit_code == 0, completed.output + assert adapter.definition != original + assert adapter.manager is ManagerState.ACTIVE + assert not controller.maintenance_path.exists() + assert runner.database.read_bytes() == before + + +def test_managed_migration_rejects_a_different_database_before_stopping_service(managed_upgrade, tmp_path): + controller, adapter, runner, _environment, _invoke, _apply = managed_upgrade + different = tmp_path / "different.env" + different.write_text( + "POWERCONTEXT_SERVER_DATABASE_KIND=sqlite\n" + f"POWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite:///{(tmp_path / 'other.db').as_posix()}\n", + encoding="utf-8", + ) + different.chmod(0o600) + _secure_windows_file(different) + result = CliRunner().invoke( + create_cli([server_app]), + ["server", "db-migrate", "plan", "--env-file", str(different), "--manage-service"], + ) + assert result.exit_code != 0 + assert adapter.manager is ManagerState.ACTIVE + assert not adapter.suspended + assert not controller.maintenance_path.exists() + assert not (tmp_path / "other.db").exists() + assert runner.plan().state == "migration_required" def _installed(tmp_path: Path, *, running: bool = True) -> tuple[ServiceController, FakeAdapter]: From 3ade3790c941672c90efa6dee70bd247922bdd30 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 00:42:32 +0800 Subject: [PATCH 51/94] fix(e2e): initialize managed OceanBase before Compose startup Run the public migration plan, accepted apply and verify in one stopped-writer maintenance container before starting the no-vector, Code-disabled OceanBase Server. Reuse one host identity and evidence directory. Preserve the optional unmanaged bootstrap and fail startup when maintenance fails. Update the local-service CLI help to match the supported full Server path. Constraint: This initialization belongs to the isolated disposable harness and does not turn ordinary remote startup into schema migration. Scope-risk: The local probe covers actual Compose initialization and HTTP readiness, not the complete model/workload acceptance suite. Tested: Real OceanBase CE 4.3.5.6 Compose initialization and public plan/apply/verify returned server_ready=true; Server health/ready returned HTTP 200. Both Compose configurations, sh -n, all applicable prek hooks including Ty, and git diff --check passed. Upgrade framework/release/publication regressions: 79 passed. --- e2e/bub/README.md | 6 ++++ e2e/bub/run.sh | 36 ++++++++++++++++++- src/powercontext/server/database_migration.py | 2 +- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/e2e/bub/README.md b/e2e/bub/README.md index ecfc3cfce7..139275ee3f 100644 --- a/e2e/bub/README.md +++ b/e2e/bub/README.md @@ -126,6 +126,12 @@ make harness-compose-acceptance \ ARGS='--id locomo-support-group --id project-database-decision' ``` +For the registered no-vector, Code-disabled OceanBase profile, Compose runs public `db-migrate plan/apply/verify` +commands before starting the Server. They share one maintenance container and `/data/migration-state`; ordinary remote +Server startup only checks readiness. The harness owns disposable volumes and stops business writers during setup. +Optional vector/Code profiles retain their existing unmanaged fresh bootstrap and are outside the registered upgrade +matrix. A maintenance failure fails startup rather than allowing a partially initialized Server to run. + Each selected workload writes the same layout: ```text diff --git a/e2e/bub/run.sh b/e2e/bub/run.sh index d6c1b16c64..491d7a3f41 100755 --- a/e2e/bub/run.sh +++ b/e2e/bub/run.sh @@ -90,6 +90,40 @@ show_startup_diagnostics() { docker compose $compose_files logs --no-color --timestamps --tail 200 >&2 || true } +initialize_database() { + if [ "$database" != oceanbase ]; then + return + fi + docker compose $compose_files up --detach --wait oceanbase || return + # Plan/apply/verify share one container, fixed host and evidence directory. + # Remote Server startup only verifies storage; it cannot initialize it. + docker compose $compose_files run --rm --no-deps --entrypoint python powercontext -c ' +import json +import subprocess + +from powercontext.server.settings import ServerSettings + +settings = ServerSettings() +if settings.inference.embedding_model is not None or settings.code.enabled: + print("Optional vector/Code profile retains its unmanaged fresh bootstrap; no registered upgrade is claimed.") + raise SystemExit(0) + +common = [ + "--evidence-dir", "/data/migration-state", + "--lock-coordination", "single-host", +] +command = ["powercontext", "server", "db-migrate"] +plan = json.loads(subprocess.check_output([*command, "plan", *common, "--backup", "auto"], text=True)) +print(json.dumps(plan), flush=True) +subprocess.run([ + *command, "apply", *common, + "--backup", "auto", "--plan-id", plan["plan_id"], + "--yes", "--maintenance-confirmed", +], check=True) +subprocess.run([*command, "verify", *common], check=True) +' +} + start_services() { attempt=1 max_attempts=1 @@ -98,7 +132,7 @@ start_services() { fi while :; do - if docker compose $compose_files up --detach --wait powercontext; then + if initialize_database && docker compose $compose_files up --detach --wait powercontext; then return else status=$? diff --git a/src/powercontext/server/database_migration.py b/src/powercontext/server/database_migration.py index 5f194bb56c..dbc21430f8 100644 --- a/src/powercontext/server/database_migration.py +++ b/src/powercontext/server/database_migration.py @@ -56,7 +56,7 @@ ] ManageService = Annotated[ bool, - typer.Option(help="Request local service maintenance; unavailable for the current partial acceptance bundle."), + typer.Option(help="Coordinate the owned local service through a full Server migration and readiness check."), ] SharedDatabase = Annotated[ bool, From 9b7bd48dddd674b0f4294c52998f993279893b59 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 01:02:31 +0800 Subject: [PATCH 52/94] test(migrations): exercise managed upgrades with native services Run public maintenance commands against installed package code, a frozen historical SQLite database and an isolated native service registration. Verify stopped-state preservation, explicit startup and ready-database service maintenance. Document the fixture and platform evidence boundaries. Constraint: No real personal-service registration or historical resource is modified. Scope-risk: Native acceptance uses uniquely named test services; each platform needs its own run. Tested: Windows native lifecycle suite: 9 passed, 2 macOS-only skips; focused new cases: 2 passed. Full prek run -a and Python 3.11 Ty passed. --- docs/en/docs/operate/database-migrations.md | 7 +- docs/zh/docs/operate/database-migrations.md | 4 +- .../native/test_personal_service_lifecycle.py | 144 +++++++++++++++++- 3 files changed, 150 insertions(+), 5 deletions(-) diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 3f51aefd2e..f7ab2efafc 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -314,8 +314,11 @@ finish the service transition when the database is already ready, without rerunn `--manage-service` covers only the accepted local service. It cannot replace `--maintenance-confirmed` for other writers and shared nodes. The partial Artifact and Topic recovery bundles reject service management when changes are needed; an already-ready no-op does not touch the service. They cannot establish complete Server readiness or switch a business -service. Controller tests use a simulated native manager with a real -SQLite migration; they do not establish native service-manager acceptance on every supported operating system. +service. Controller tests use a simulated native manager with a real SQLite migration. The native lifecycle suite also +exercises the installed CLI with isolated service registrations: it upgrades a stopped frozen SQLite baseline, checks +that it stays stopped until an explicit start, and completes service maintenance for an already-ready running database. +Each operating system requires its own native run. A frozen baseline fixture is distinct from upgrading a database +populated by an installed historical release. For manual deployment, stop the old service first. After a supported complete migration succeeds, ensure the launch definition points to the new environment before starting it. For foreground deployment, exit the old process and use the diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index b63657a29d..a55c04a5d9 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -223,7 +223,9 @@ powercontext service restart `--manage-service` 仅管理已确认的本机服务，不能代替 `--maintenance-confirmed` 对其他写入者和共享库节点的声明。 部分 Artifact 和 Topic recovery bundle 需要变更时返回 `service_unsupported`；已就绪的空操作不触碰服务。 它们不能用于完整 Server readiness 或业务服务切换。 -控制器测试使用模拟的原生服务管理器和真实 SQLite 迁移，不代表所有支持操作系统的原生服务验收。 +控制器测试使用模拟的原生服务管理器和真实 SQLite 迁移。原生生命周期套件还通过独立服务注册验证安装后的 CLI： +升级已停止服务的冻结 SQLite 基线，确认升级后仍保持停止并可显式启动；同时验证数据库已就绪时运行中服务的维护恢复。 +每个操作系统须分别执行原生测试。冻结基线夹具与实际历史版本安装并写入数据库后的升级验收是不同的证据。 手动管理时，先停旧服务，在已支持的完整迁移成功后确认服务启动定义指向新版，再启动。前台部署则退出旧进程，使用新版环境的现有入口： diff --git a/tests/native/test_personal_service_lifecycle.py b/tests/native/test_personal_service_lifecycle.py index 9d2c807c10..99836ccab7 100644 --- a/tests/native/test_personal_service_lifecycle.py +++ b/tests/native/test_personal_service_lifecycle.py @@ -19,24 +19,34 @@ import plistlib import re import socket +import sqlite3 import subprocess import sys import time import uuid import xml.etree.ElementTree as ET from collections.abc import Callable -from contextlib import suppress +from contextlib import closing, suppress from importlib.metadata import version from pathlib import Path from typing import Any import pytest - +from alembic import command +from sqlalchemy import create_engine +from typer.testing import CliRunner + +from powercontext.builtin.persistence.migrations.deployment import deployment_runner, server_bundle +from powercontext.cli.app import create_cli +from powercontext.server import database_migration +from powercontext.server.cli import app as server_app +from powercontext.server.configuration import server_settings_context from powercontext.service.adapters.base import NativeServiceAdapter, service_python_executable from powercontext.service.adapters.launchd import LaunchdUserAdapter from powercontext.service.adapters.systemd import SystemdUserAdapter from powercontext.service.adapters.windows import WindowsTaskSchedulerAdapter from powercontext.service.controller import ServiceController +from powercontext.service.environment import load_protected_environment_file from powercontext.service.model import ( DEFINITION_VERSION, OWNERSHIP_MARKER, @@ -135,6 +145,136 @@ def test_native_maintenance_preserves_registration_and_original_running_state(tm _cleanup(adapter) +def test_native_public_migration_upgrades_a_stopped_historical_database( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Exercise the installed CLI, frozen SQLite baseline and real native manager together.""" + + adapter = _native_adapter(suffix="database-upgrade") + controller = ServiceController(adapter) + environment = _environment_file(tmp_path) + database = tmp_path / "data" / "powercontext.db" + database.parent.mkdir(parents=True, exist_ok=True) + with server_settings_context(env_file=environment) as settings: + runner = deployment_runner(settings) + bundle = server_bundle() + engine = create_engine(f"sqlite:///{database.as_posix()}") + try: + with engine.begin() as connection: + config = bundle.config(connection) + config.attributes["configuration"] = runner.configuration + command.upgrade(config, "s1300") + connection.exec_driver_sql("DROP TABLE pc_schema_revision") + connection.exec_driver_sql( + "INSERT INTO pc_scopes (scope_id,title,summary,scope_id_search,title_search,summary_search,version) " + "VALUES ('historical','retained title','retained summary','historical','retained title','retained summary',1)" + ) + finally: + engine.dispose() + port = _service_port(tmp_path) + assert port is not None + # A stopped registration represents the historical deployment. Its database + # comes from frozen migration resources, not an installed 1.3.0 executable. + definition = ServiceDefinition( + ownership=OWNERSHIP_MARKER, + definition_version=DEFINITION_VERSION, + package_version="frozen-s1300-fixture", + python_executable=service_python_executable(), + endpoint=f"http://127.0.0.1:{port}", + data_dir=str(database.parent), + env_file=load_protected_environment_file(environment).identity, + ) + try: + adapter.write(adapter.render(definition)) + adapter.update_suspended() + controller.stop() + # Select only this uniquely named native registration; all manager, + # database, backup and readiness operations remain real. + monkeypatch.setattr(database_migration, "ServiceController", lambda: controller) + cli = create_cli([server_app]) + reviewed = _native_migration_command(cli, environment, "plan") + assert reviewed["state"] == "migration_required" + assert reviewed["service"]["originally_running"] is False + + result = _native_migration_command( + cli, + environment, + "apply", + "--backup", + "auto", + "--plan-id", + reviewed["plan_id"], + "--yes", + "--maintenance-confirmed", + ) + + assert result["server_ready"] is True + assert result["backup_state"] == "completed" + assert result["backup_ref"] and Path(result["backup_ref"]).is_file() + assert runner.verify().state == "ready" + assert adapter.manager_state() in {ManagerState.INACTIVE, ManagerState.FAILED} + with closing(sqlite3.connect(database)) as connection: + assert connection.execute("SELECT title,summary FROM pc_scopes WHERE scope_id='historical'").fetchone() == ( + "retained title", + "retained summary", + ) + assert controller.maintenance_path.exists() + assert controller.start().ok, _failure_diagnostics(adapter, tmp_path) + assert not controller.maintenance_path.exists() + finally: + _capture_native_failure(adapter, tmp_path) + with suppress(Exception): + controller.uninstall() + _cleanup(adapter) + + +def test_native_public_migration_of_ready_storage_restores_a_running_service( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A reviewed ready-database retry must still finish native service maintenance.""" + + adapter = _native_adapter(suffix="ready-database") + controller = ServiceController(adapter) + environment = _environment_file(tmp_path) + try: + assert controller.install(env_file=environment).ok + monkeypatch.setattr(database_migration, "ServiceController", lambda: controller) + cli = create_cli([server_app]) + reviewed = _native_migration_command(cli, environment, "plan") + assert reviewed["state"] == "ready" + assert reviewed["service"]["originally_running"] is True + + result = _native_migration_command( + cli, + environment, + "apply", + "--backup", + "auto", + "--plan-id", + reviewed["plan_id"], + "--yes", + "--maintenance-confirmed", + ) + + assert result["server_ready"] is True and result["changed"] is False + assert controller.status().ok, _failure_diagnostics(adapter, tmp_path) + assert adapter.manager_state() is ManagerState.ACTIVE + assert not controller.maintenance_path.exists() + finally: + _capture_native_failure(adapter, tmp_path) + with suppress(Exception): + controller.uninstall() + _cleanup(adapter) + + +def _native_migration_command(cli, environment: Path, action: str, *options: str) -> dict[str, Any]: + result = CliRunner().invoke( + cli, ["server", "db-migrate", action, "--env-file", str(environment), "--manage-service", *options] + ) + assert result.exit_code == 0, f"{result.output}\n{result.exception!r}" + return json.loads(result.output) + + def test_native_stopped_service_registration_updates_without_starting(tmp_path: Path) -> None: """Updating a stopped registration must leave activation to an explicit start.""" From 0270f631fe00b2317fe771adfa21205f9610b2d8 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 01:19:51 +0800 Subject: [PATCH 53/94] test(migrations): cover native full Server interruption recovery Interrupt the frozen Server upgrade after retained-copy creation, Candidate rename, Processing conversion and the following revision. Verify historical data and the original maintenance window/manual backup declaration survive recovery on actual seekDB and OceanBase. Exercise OceanBase cases without skips in the existing disposable CI environment. Constraint: Native restart behavior and verified backup restoration are not claimed by these fixtures. Scope-risk: The seekDB scenario retains one native engine lifetime; cross-process restart acceptance remains separate. Tested: Native seekDB/OceanBase recovery: 8 passed, zero skips. Workflow/routing: 45 passed. Full prek run -a, focused workflow hooks and Python 3.11 Ty passed. --- .github/workflows/e2e-harness.yml | 21 ++++ .../test_mysql_migration_runner.py | 110 +++++++++++++++++- 2 files changed, 130 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index c9d9e8074b..076e18a68a 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -175,6 +175,27 @@ jobs: tests/builtin/persistence/test_database_snapshots.py \ --junitxml=.powercontext/recall-effort-ci/regressions.xml -q -rs + - name: Recover the full Server chain on real OceanBase without skips + env: + POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- + mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 + run: | + docker exec recall-effort-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ + -Dpowercontext_test -e 'CREATE DATABASE pc_migration_probe_server_recovery' + timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -q -rs \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_full_server_interruption_preserves_history_and_original_maintenance \ + -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-recovery.xml + uv run --locked python - <<'PY' + from pathlib import Path + from xml.etree import ElementTree + + suites = ElementTree.parse(Path(".powercontext/recall-effort-ci/full-server-recovery.xml")).getroot() + cases = list(suites.iter("testcase")) + if len(cases) != 4 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): + raise SystemExit("All four native full Server recovery scenarios must pass without skips.") + PY + - name: Collect the database log if: always() run: | diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index 945c6b78ce..1d4d66dd47 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -38,12 +38,15 @@ from powercontext.builtin.persistence.migrations import mysql as mysql_migrations from powercontext.builtin.persistence.migrations.backup import BackupCapabilities, BackupProvider from powercontext.builtin.persistence.migrations.connections import BackendIdentity, MaintenanceConnections -from powercontext.builtin.persistence.migrations.deployment import production_bundle +from powercontext.builtin.persistence.migrations.deployment import production_bundle, server_bundle from powercontext.builtin.persistence.migrations.models import MigrationError, MigrationPlan from powercontext.builtin.persistence.migrations.mysql import MySQLMigrationRunner +from powercontext.builtin.persistence.migrations.mysql_schema import mysql_inventory from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.oceanbase import profile as oceanbase_profile from powercontext.builtin.persistence.seekdb import SeekDBConfig +from powercontext.builtin.runtime.config import BuiltinConfig +from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest @dataclass(frozen=True) @@ -372,6 +375,111 @@ def interrupt(_connection, _cursor, statement: str, _parameters, _context, _exec event.remove(Engine, "after_cursor_execute", interrupt) +def _full_server_runner( + backend: _BackendTarget, *, acknowledge_previous_run: str | None = None +) -> MySQLMigrationRunner: + return MySQLMigrationRunner( + backend.connections(acknowledge_previous_run=acknowledge_previous_run), + server_bundle(), + evidence_directory=backend.evidence_directory, + configuration={ + "processing_manifest": canonical_processing_manifest(BuiltinConfig()), + "embedding_enabled": False, + }, + ) + + +def _load_full_server_baseline(backend: _BackendTarget) -> None: + runner = _full_server_runner(backend) + + def load(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: + assert connection is not None + source = runner.schema.recovery(mysql_inventory(connection), None) + assert source.revision == "s1300" + for statement in source.pending_statements: + runner.schema.execute(connection, statement, runner.configuration) + connection.exec_driver_sql( + "INSERT INTO pc_scopes (scope_id,title,summary,scope_id_search,title_search,summary_search,version) " + "VALUES ('historical','retained title','retained summary','historical','retained title','retained summary',1)" + ) + connection.execute( + text( + "INSERT INTO pc_artifacts (scope_id,family,artifact_id,revision,content) " + "VALUES ('historical','custom-family','preserved',1,:content)" + ), + {"content": b'{"history":"unchanged"}'}, + ) + connection.commit() + + backend.connections().run(load, writable=True) + + +@pytest.mark.parametrize( + ("interruption", "recorded_revision"), + [ + ("CREATE TABLE pc_retained_dream_candidate_versions", "s1300"), + ("ALTER TABLE pc_artifact_candidate_versions RENAME TO pc_candidate_versions", "s1300"), + ("UPDATE pc_artifact_processing_schema SET config_manifest", "s1300"), + ("CREATE TABLE pc_memory_artifact_archive", "s1400"), + ], + ids=("retained-copy", "candidate-rename", "domain-processing", "next-revision"), +) +def test_mysql_full_server_interruption_preserves_history_and_original_maintenance( + backend: _BackendTarget, interruption: str, recorded_revision: str +) -> None: + """Native backend recovery covers the full Server chain, not just the Artifact prototype.""" + + _load_full_server_baseline(backend) + runner = _full_server_runner(backend) + original = runner.plan(backup_policy="manual") + assert original.source_revision == "s1300" and original.target_revision == "s1401" + _interrupt_after_ddl(runner, interruption) + journals = list(backend.evidence_directory.rglob("maintenance.json")) + assert len(journals) == 1 + interrupted = json.loads(journals[0].read_text()) + assert interrupted["state"] == "active" + assert interrupted["manual_ref"] == "operator://manual-backup" + recovery = _full_server_runner(backend).plan(backup_policy="manual") + assert recovery.state == "recovery_required" + assert recovery.source_revision == recorded_revision + token = (recovery.coordination or {}).get("pending_run_id") + if isinstance(backend.config, OceanBaseConfig): + assert token + with pytest.raises(MigrationError, match="recovery_required"): + _apply(_full_server_runner(backend), policy="manual") + # The injected exception happens after the backend returned from the + # statement and the owned connection closed. This known test interruption + # permits the required declaration; arbitrary remote interruptions do not. + resumed = _full_server_runner(backend, acknowledge_previous_run=token) + + result = _apply(resumed, policy="manual") + + assert result.readiness_scope == "server" and result.revision == "s1401" + assert result.backup_state == "user_confirmed" + assert result.backup_ref == interrupted["manual_ref"] + assert resumed.verify().state == "ready" + completed = json.loads(journals[0].read_text()) + assert completed["maintenance_window_id"] == interrupted["maintenance_window_id"] + assert completed["source_fingerprint"] == interrupted["source_fingerprint"] + assert completed["manual_ref"] == interrupted["manual_ref"] + assert completed["state"] == "complete" + + def inspect_history(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]): + assert connection is not None + return ( + connection.exec_driver_sql( + "SELECT content FROM pc_artifacts WHERE scope_id='historical' AND artifact_id='preserved'" + ).all(), + connection.exec_driver_sql("SELECT title,summary FROM pc_scopes WHERE scope_id='historical'").all(), + ) + + assert backend.connections().run(inspect_history) == ( + [(b'{"history":"unchanged"}',)], + [("retained title", "retained summary")], + ) + assert not _apply(_full_server_runner(backend), policy="manual").changed + + def _interrupt_after_committed_citation_ddl(backend: _BackendTarget) -> None: _load_baseline(backend) _interrupt_after_ddl(backend.runner(), "ALTER TABLE pc_artifacts ADD COLUMN memory_citations") From 752477fa71fd232bdbf74cd5d820379d50e335f7 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 01:30:09 +0800 Subject: [PATCH 54/94] fix(ci): retain bounded seekDB upgrade diagnostics Preserve native log tails and filesystem capacity from synthetic installed upgrade acceptance stores before temporary cleanup, including failed runs. Exclude database payloads and bound capture to 4 MiB per file and 16 MiB total. Constraint: Diagnostics do not change production CLI errors or acceptance criteria. Scope-risk: This preserves evidence; it does not fix the unresolved native seekDB upgrade failure. Tested: Two regressions failed before implementation. Release/publication tests passed 67 cases on Windows UTF-8 and Linux. Full prek run -a including Ruff and Ty passed. --- scripts/upgrade_release_gate.py | 58 ++++++++++++++++++++++++++++-- tests/test_upgrade_release_gate.py | 42 ++++++++++++++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/scripts/upgrade_release_gate.py b/scripts/upgrade_release_gate.py index c97812ae48..77303d4d3d 100644 --- a/scripts/upgrade_release_gate.py +++ b/scripts/upgrade_release_gate.py @@ -21,10 +21,13 @@ import json import os import re +import shutil import subprocess import sys import tempfile import zipfile +from collections.abc import Iterator +from contextlib import contextmanager from email.parser import Parser from pathlib import Path from typing import Any @@ -258,6 +261,58 @@ def wheel_version(wheel: Path) -> str: return str(metadata["Version"]) +def _preserve_seekdb_logs(directory: Path, output: Path) -> None: + """Retain bounded diagnostics from synthetic acceptance stores, excluding database files.""" + + remaining = 16 * 1024 * 1024 + files = [] + with zipfile.ZipFile(output / "seekdb-native-logs.zip", "w", compression=zipfile.ZIP_DEFLATED) as archive: + for name in ("database", "fresh"): + for path in sorted((directory / name).rglob("*.log*")): + if remaining == 0: + break + if path.is_symlink() or not path.is_file(): + continue + with path.open("rb") as source: + size = source.seek(0, os.SEEK_END) + length = min(size, 4 * 1024 * 1024, remaining) + source.seek(size - length) + content = source.read(length) + relative = path.relative_to(directory).as_posix() + archive.writestr(relative, content) + remaining -= len(content) + files.append({ + "path": relative, + "size_bytes": size, + "captured_bytes": len(content), + "truncated": len(content) < size, + }) + usage = shutil.disk_usage(directory) + manifest = { + "files": files, + "budget_exhausted": remaining == 0, + "filesystem": {"total_bytes": usage.total, "used_bytes": usage.used, "free_bytes": usage.free}, + } + (output / "seekdb-native-logs.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + + +@contextmanager +def case_workspace(backend: str, output: Path) -> Iterator[Path]: + # Keep embedded sockets under a short private path. Native logs must leave + # this directory before cleanup, including when acceptance raises or times out. + with tempfile.TemporaryDirectory(prefix="pc-up-", dir="/tmp" if os.name != "nt" else None) as temporary: # noqa: S108 -- private mkdtemp directory + directory = Path(temporary) + try: + yield directory + finally: + if backend == "seekdb": + try: + _preserve_seekdb_logs(directory, output) + except OSError as error: + # Diagnostics must not replace the acceptance failure. + print(f"seekDB native log collection failed: {type(error).__name__}", file=sys.stderr) + + def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dict[str, Any]: output.mkdir(parents=True, exist_ok=False) # Never inherit a developer's live deployment or pytest selection overrides. @@ -266,8 +321,7 @@ def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dic for key, value in os.environ.items() if not key.startswith(("POWERCONTEXT_", "PYTEST_")) and key not in {"PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV"} } - with tempfile.TemporaryDirectory(prefix="pc-up-", dir="/tmp" if os.name != "nt" else None) as temporary: # noqa: S108 -- private mkdtemp directory; short embedded socket path - directory = Path(temporary) + with case_workspace(case["backend"], output) as directory: extras = "cli,server,seekdb" if case["backend"] == "seekdb" else "cli,server" interpreters = {} artifact = case["source_artifact"] diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py index 49fc331039..290db8df0f 100644 --- a/tests/test_upgrade_release_gate.py +++ b/tests/test_upgrade_release_gate.py @@ -458,6 +458,48 @@ def execute(_root, _wheel, case, _output): assert [case["status"] for case in result["cases"]] == ["failed", "passed", "passed"] +def test_failed_seekdb_acceptance_keeps_native_logs_before_workspace_cleanup(tmp_path): + output = tmp_path / "evidence" + output.mkdir() + + with pytest.raises(gate.GateError, match="fixture migration failed"), gate.case_workspace("seekdb", output) as work: + native = work / "database/log" + native.mkdir(parents=True) + (native / "observer.log").write_bytes(b"native recovery diagnostic\n") + (work / "database/store").write_bytes(b"database payload must not become a log") + fresh = work / "fresh/log" + fresh.mkdir(parents=True) + (fresh / "observer.log.wf").write_bytes(b"fresh native diagnostic\n") + raise gate.GateError("fixture migration failed") # noqa: TRY003 -- synthetic acceptance failure + + assert not work.exists() + with zipfile.ZipFile(output / "seekdb-native-logs.zip") as archive: + assert set(archive.namelist()) == {"database/log/observer.log", "fresh/log/observer.log.wf"} + assert archive.read("database/log/observer.log") == b"native recovery diagnostic\n" + assert archive.read("fresh/log/observer.log.wf") == b"fresh native diagnostic\n" + manifest = json.loads((output / "seekdb-native-logs.json").read_text()) + assert len(manifest["files"]) == 2 + assert not any(item["truncated"] for item in manifest["files"]) + + +def test_seekdb_diagnostics_keep_a_bounded_tail_of_large_native_logs(tmp_path): + output = tmp_path / "evidence" + output.mkdir() + size = 5 * 1024 * 1024 + with gate.case_workspace("seekdb", output) as work: + native = work / "database/log" + native.mkdir(parents=True) + (native / "observer.log").write_bytes(b"x" * size + b"final diagnostic") + + with zipfile.ZipFile(output / "seekdb-native-logs.zip") as archive: + tail = archive.read("database/log/observer.log") + assert len(tail) <= 4 * 1024 * 1024 + assert tail.endswith(b"final diagnostic") + manifest = json.loads((output / "seekdb-native-logs.json").read_text()) + assert manifest["files"][0]["truncated"] is True + assert manifest["files"][0]["size_bytes"] == size + len(b"final diagnostic") + + @pytest.mark.parametrize( "relative", [ From 20f2c12333b888d6723452401794e0d90648da3a Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 02:23:12 +0800 Subject: [PATCH 55/94] feat(migrations): coordinate managed Processing maintenance Register Processing mode switches through shared target locks, backup policies, reviewed plans and durable recovery evidence. Delegate the compatibility CLI for managed databases while preserving unmanaged domain maintenance. Constraint: Preserve the full Server revision and atomic lease ownership updates. Scope-risk: Managed mode switches now require reviewed maintenance acceptance. Tested: Windows focused maintenance/CLI tests (10 passed); Linux focused maintenance/CLI tests (10 passed, one aiosqlite cleanup warning); expanded Windows/Linux suites (145/148 passed); native seekDB/OceanBase mode switches (2 passed); full hooks including ty. --- .github/workflows/e2e-harness.yml | 7 +- .../operate/artifact-processing-migration.md | 9 + docs/en/docs/operate/database-migrations.md | 32 +- .../operate/artifact-processing-migration.md | 8 + docs/zh/docs/operate/database-migrations.md | 26 +- .../persistence/migrations/connections.py | 51 ++- .../persistence/migrations/deployment.py | 48 ++- .../migrations/processing_configuration.py | 309 ++++++++++++++++++ src/powercontext/server/cli.py | 106 ++++++ src/powercontext/server/database_migration.py | 42 ++- .../test_mysql_migration_runner.py | 49 ++- tests/test_managed_processing_maintenance.py | 275 ++++++++++++++++ 12 files changed, 937 insertions(+), 25 deletions(-) create mode 100644 src/powercontext/builtin/persistence/migrations/processing_configuration.py create mode 100644 tests/test_managed_processing_maintenance.py diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 076e18a68a..cde768b214 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -175,7 +175,7 @@ jobs: tests/builtin/persistence/test_database_snapshots.py \ --junitxml=.powercontext/recall-effort-ci/regressions.xml -q -rs - - name: Recover the full Server chain on real OceanBase without skips + - name: Recover and maintain the full Server chain on real OceanBase without skips env: POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 @@ -185,6 +185,7 @@ jobs: -Dpowercontext_test -e 'CREATE DATABASE pc_migration_probe_server_recovery' timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -q -rs \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_full_server_interruption_preserves_history_and_original_maintenance \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_registered_processing_mode_switch_preserves_revision_and_history \ -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-recovery.xml uv run --locked python - <<'PY' from pathlib import Path @@ -192,8 +193,8 @@ jobs: suites = ElementTree.parse(Path(".powercontext/recall-effort-ci/full-server-recovery.xml")).getroot() cases = list(suites.iter("testcase")) - if len(cases) != 4 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): - raise SystemExit("All four native full Server recovery scenarios must pass without skips.") + if len(cases) != 5 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): + raise SystemExit("All five native full Server recovery and maintenance scenarios must pass without skips.") PY - name: Collect the database log diff --git a/docs/en/docs/operate/artifact-processing-migration.md b/docs/en/docs/operate/artifact-processing-migration.md index a80bb95686..9f342b71c1 100644 --- a/docs/en/docs/operate/artifact-processing-migration.md +++ b/docs/en/docs/operate/artifact-processing-migration.md @@ -9,6 +9,15 @@ marker is rejected before background processing starts. ## Plan and apply +For databases managed by `pc_schema_revision`, this command delegates to the registered +[Processing configuration maintenance](database-migrations.md#processing-configuration-maintenance) operation. +Its plan/apply/verify responses use the unified migration format. Apply additionally requires the reviewed +`--plan-id`, explicit `--backup` choice and `--yes` for automation, with the same backup declarations and +OceanBase coordination options as `db-migrate`. Complete any pending full Server upgrade first. +The domain-only procedure below applies to databases without a registered revision. +As with `db-migrate`, an explicit environment file is authoritative; otherwise the command discovers `.env` in the +current directory before inspecting ownership and executing the selected path. + Inspect the plan with the deployment configuration that will run after migration: ```shell diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index f7ab2efafc..62ef2cf7f2 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -36,10 +36,38 @@ operators/functions remain rejected; their errors identify the reflected table, construct instead of misreporting balanced parentheses. Parsing an expression does not register a historical layout or bypass migration/data-integrity checks. -The existing `server processing-migrate` remains available. Follow -[Migrate Artifact processing state](artifact-processing-migration.md); it does not yet forward to the unified entry point. +The existing `server processing-migrate` delegates managed databases to `db-migrate --bundle processing`. +Databases without a registered revision retain their domain-only migration command. +Follow [Migrate Artifact processing state](artifact-processing-migration.md). See [Deploy the Server](deploy-server.md) for service deployment. +## Processing configuration maintenance + +`--bundle processing` registers an offline ownership-configuration operation on a complete, current Server database. +It does not advance `pc_schema_revision` or adopt an older schema. Complete the full Server upgrade first. +Use the deployment's new configuration and a new migration ID for each mode switch: + +```bash +powercontext server db-migrate plan --bundle processing --env-file .env \ + --migration-id global-to-dedicated-1 --backup auto +powercontext server db-migrate apply --bundle processing --env-file .env \ + --migration-id global-to-dedicated-1 --backup auto --plan-id PLAN_ID \ + --maintenance-confirmed --yes +powercontext server db-migrate verify --bundle processing --env-file .env \ + --migration-id global-to-dedicated-1 +``` + +The operation uses the same target lock, backup policies, backend connections, and stopped-writer declarations as +schema migration. OceanBase still requires one fixed maintenance host, `--evidence-dir`, and +`--lock-coordination single-host`; recovery requires the exact previous-run acknowledgment when reported. +Local service switching is not part of this operation. Stop all writers and automatic restarts before applying. + +All mode-switch phases commit together. Failure before commit rolls back the ownership declaration and lease +invalidation together. Reopen the plan after an interruption and resume with the original configuration, migration ID, +backup policy, and evidence directory; the original maintenance window and recovery point are retained. +Successful retry does not invalidate leases again. Full Server validators run before and after the change. +An already matching configuration is a read-only no-op. + ## Topic Memory recovery bundle Select `--bundle topic-memory-recovery` for the versioned `powercontext.topic-memory-recovery.v1` conversion. It recognizes diff --git a/docs/zh/docs/operate/artifact-processing-migration.md b/docs/zh/docs/operate/artifact-processing-migration.md index 821e90dede..4aa8697399 100644 --- a/docs/zh/docs/operate/artifact-processing-migration.md +++ b/docs/zh/docs/operate/artifact-processing-migration.md @@ -8,6 +8,14 @@ Artifact Processing Supervisor。全新的空数据库自动初始化。已有 ## 查看计划并执行 +含有 `pc_schema_revision` 的受管数据库会转发到登记的 +[Processing 配置维护](database-migrations.md#processing-配置维护)操作，plan/apply/verify 使用统一迁移响应格式。 +自动化 apply 还须提供审阅后的 `--plan-id`、明确的 `--backup` 策略和 `--yes`， +并遵守 `db-migrate` 的备份声明及 OceanBase 协调选项。先完成尚未结束的整库升级。 +以下领域独立流程适用于未登记 revision 的数据库。 +与 `db-migrate` 一样，显式环境文件具有最高优先级；未指定时，先查找当前目录的 `.env`， +并在所有权检查和实际执行中使用同一份配置。 + 使用迁移后部署所采用的配置查看只读计划： ```shell diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index a55c04a5d9..75ab4d9c06 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -30,7 +30,31 @@ Candidate 旧表保留为 `pc_retained_dream_candidate_heads` 和 `pc_retained_d MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、`*`、`/`、`%`、一元正负号及嵌套括号，并保留算术、比较和布尔运算的优先级。未支持的 CHECK 运算符或函数仍被拒绝，错误会标明反射的表名、约束名及未支持的语法，避免将合法分组误报为括号不匹配。表达式解析成功不代表历史结构已登记，也不会绕过迁移或数据完整性检查。 -原有 `server processing-migrate` 保留，请按[迁移 Artifact 处理状态](artifact-processing-migration.md)使用。它尚未自动转发到统一入口。服务部署参考[部署 Server](deploy-server.md)。 +原有 `server processing-migrate` 在受管数据库上转发到 `db-migrate --bundle processing`；未登记 revision 的数据库仍保留领域独立迁移。具体步骤见[迁移 Artifact 处理状态](artifact-processing-migration.md)。服务部署参考[部署 Server](deploy-server.md)。 + +## Processing 配置维护 + +`--bundle processing` 在结构完整、已到当前 revision 的 Server 数据库上登记离线所有权配置操作。 +它不推进 `pc_schema_revision`，也不接管旧结构；必须先完成整库升级。 +使用切换后的部署配置，每次模式切换指定新的迁移 ID： + +```bash +powercontext server db-migrate plan --bundle processing --env-file .env \ + --migration-id global-to-dedicated-1 --backup auto +powercontext server db-migrate apply --bundle processing --env-file .env \ + --migration-id global-to-dedicated-1 --backup auto --plan-id PLAN_ID \ + --maintenance-confirmed --yes +powercontext server db-migrate verify --bundle processing --env-file .env \ + --migration-id global-to-dedicated-1 +``` + +该操作复用结构迁移的目标锁、备份策略、后端维护连接和停写声明。OceanBase 仍要求固定维护主机、 +`--evidence-dir` 和 `--lock-coordination single-host`；报告上次运行未结束时，恢复必须提供准确的运行确认 token。 +此操作不切换本地服务；执行前须自行停止全部写入者并暂停自动重启。 + +模式切换的所有阶段在一个事务中提交。提交前失败时，所有权声明和租约撤销一起回滚。 +中断后重新查看计划，保留原配置、迁移 ID、备份策略和证据目录恢复；原维护窗口和恢复点继续沿用。 +成功后的重试不会再次撤销租约。操作前后都运行完整 Server 校验；配置已匹配时只读返回。 ## Topic Memory 恢复 bundle diff --git a/src/powercontext/builtin/persistence/migrations/connections.py b/src/powercontext/builtin/persistence/migrations/connections.py index ef85571df8..83b93c39e0 100644 --- a/src/powercontext/builtin/persistence/migrations/connections.py +++ b/src/powercontext/builtin/persistence/migrations/connections.py @@ -18,13 +18,16 @@ import asyncio import socket +import sqlite3 from collections.abc import Callable +from contextlib import nullcontext from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import Literal, TypeVar -from sqlalchemy import Connection +from sqlalchemy import Connection, create_engine +from sqlalchemy.engine import make_url from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine from sqlalchemy.pool import NullPool @@ -33,6 +36,7 @@ from powercontext.builtin.persistence.oceanbase import profile as oceanbase_profile from powercontext.builtin.persistence.seekdb import SeekDBConfig from powercontext.builtin.persistence.seekdb import profile as seekdb_profile +from powercontext.builtin.persistence.sqlite import SQLiteConfig from .locking import local_migration_lock from .models import MigrationError, digest @@ -61,7 +65,7 @@ class MaintenanceConnections: def __init__( self, - config: SeekDBConfig | OceanBaseConfig, + config: SQLiteConfig | SeekDBConfig | OceanBaseConfig, *, evidence_directory: Path | None = None, lock_coordination: Literal["single-host"] | None = None, @@ -73,6 +77,8 @@ def __init__( self.acknowledge_previous_run = acknowledge_previous_run def coordination(self, identity: BackendIdentity) -> dict[str, str]: + if isinstance(self.config, SQLiteConfig): + return {"mode": "database-file"} if isinstance(self.config, SeekDBConfig): return {"mode": "engine-directory", "directory": str(self.config.path.expanduser().resolve())} coordination = { @@ -92,6 +98,8 @@ def run( *, writable: bool = False, ) -> _T: + if isinstance(self.config, SQLiteConfig): + return self._sqlite(operation, self.config, writable=writable) if isinstance(self.config, SeekDBConfig): path = self.config.path.expanduser().resolve() identity = BackendIdentity( @@ -115,6 +123,45 @@ def run( return asyncio.run(self._seekdb(operation, self.config, path, identity, initial_inode)) return asyncio.run(self._oceanbase(operation, self.config, writable=writable)) + @staticmethod + def _sqlite( + operation: Callable[[Connection | None, BackendIdentity, Callable[[], None]], _T], + config: SQLiteConfig, + *, + writable: bool, + ) -> _T: + url = make_url(config.url) + if ( + not config.is_persistent + or url.query + or not url.database + or url.username + or url.password + or url.host + or url.port + ): + raise MigrationError("unsupported_target", "Maintenance requires a regular persistent SQLite file URL.") + target = Path(url.database).absolute().resolve() + identity = BackendIdentity("sqlite", digest({"kind": "sqlite", "path": str(target)}), target.name) + if not target.exists(): + return operation(None, identity, lambda: None) + if not target.is_file() or target.stat().st_nlink != 1: + raise MigrationError("unsupported_target", "Use a regular SQLite file without hard-link aliases.") + with local_migration_lock(target) if writable else nullcontext(): + uri = target.as_uri() + ("?mode=rw" if writable else "?mode=ro") + engine = create_engine( + "sqlite://", creator=lambda: sqlite3.connect(uri, uri=True, isolation_level=None, timeout=0) + ) + try: + with engine.connect() as connection: + connection.exec_driver_sql("PRAGMA foreign_keys=ON") + if not writable: + connection.exec_driver_sql("PRAGMA query_only=ON") + connection.exec_driver_sql("BEGIN IMMEDIATE" if writable else "BEGIN") + return operation(connection, identity, lambda: None) + finally: + engine.dispose() + async def _seekdb( self, operation: Callable[[Connection | None, BackendIdentity, Callable[[], None]], _T], diff --git a/src/powercontext/builtin/persistence/migrations/deployment.py b/src/powercontext/builtin/persistence/migrations/deployment.py index 4556f6327d..7aa7999bff 100644 --- a/src/powercontext/builtin/persistence/migrations/deployment.py +++ b/src/powercontext/builtin/persistence/migrations/deployment.py @@ -32,6 +32,7 @@ from .connections import MaintenanceConnections from .models import MigrationError from .mysql import MySQLMigrationRunner +from .processing_configuration import ProcessingConfigurationMigrationRunner from .sqlite import SQLiteMigrationRunner from .topic_memory_recovery import TopicMemoryRecoveryMigrationRunner @@ -49,7 +50,12 @@ def production_bundle() -> MigrationBundle: return MigrationBundle(Path(__file__).parent / "resources") -MigrationRunner = SQLiteMigrationRunner | MySQLMigrationRunner | TopicMemoryRecoveryMigrationRunner +MigrationRunner = ( + SQLiteMigrationRunner + | MySQLMigrationRunner + | TopicMemoryRecoveryMigrationRunner + | ProcessingConfigurationMigrationRunner +) def _migration_configuration(settings: ServerSettings) -> dict[str, Any]: @@ -73,19 +79,21 @@ def deployment_runner( # noqa: C901 -- Ordered backend admission checks. evidence_dir: Path | None = None, lock_coordination: Literal["single-host"] | None = None, acknowledge_previous_run: str | None = None, - bundle: Literal["server", "artifacts", "topic-memory-recovery"] = "server", + bundle: Literal["server", "artifacts", "topic-memory-recovery", "processing"] = "server", + migration_id: str = "rfc1515", + batch_size: int = 100, ) -> MigrationRunner: """Use the real configured identity without opening a business profile.""" database = settings.database configuration = _migration_configuration(settings) - if bundle == "server": + if bundle in {"server", "processing"}: if settings.inference.embedding_model is not None or settings.code.enabled: raise MigrationError( "unsupported_upgrade_configuration", "The registered 1.3.0 Server path covers the no-vector, Code-disabled configuration.", ) configuration["embedding_enabled"] = False - selected = server_bundle() if bundle == "server" else production_bundle() + selected = server_bundle() if bundle in {"server", "processing"} else production_bundle() if bundle == "topic-memory-recovery": if not isinstance(database, (SQLiteConfig, SeekDBConfig, OceanBaseConfig)): raise MigrationError("unsupported_backend", "The database has no Topic recovery migration adapter.") @@ -111,13 +119,23 @@ def deployment_runner( # noqa: C901 -- Ordered backend admission checks. "coordination_required", "OceanBase requires --lock-coordination single-host; run all migrators on one fixed host and evidence directory.", ) - return MySQLMigrationRunner( - MaintenanceConnections( - database, + connections = MaintenanceConnections( + database, + evidence_directory=evidence_dir, + lock_coordination=lock_coordination, + acknowledge_previous_run=acknowledge_previous_run, + ) + if bundle == "processing": + return ProcessingConfigurationMigrationRunner( + connections, + selected, + configuration=configuration, evidence_directory=evidence_dir, - lock_coordination=lock_coordination, - acknowledge_previous_run=acknowledge_previous_run, - ), + migration_id=migration_id, + batch_size=batch_size, + ) + return MySQLMigrationRunner( + connections, selected, evidence_directory=evidence_dir, configuration=configuration, @@ -131,6 +149,16 @@ def deployment_runner( # noqa: C901 -- Ordered backend admission checks. ) # Match SQLite's URL normalization before the runner expands standalone Path inputs. target = Path(os.path.abspath(url.database)) + if bundle == "processing": + directory = evidence_dir or target.with_name(target.name + ".pc-migration-state") + return ProcessingConfigurationMigrationRunner( + MaintenanceConnections(database), + selected, + configuration=configuration, + evidence_directory=directory, + migration_id=migration_id, + batch_size=batch_size, + ) return SQLiteMigrationRunner( target, selected, diff --git a/src/powercontext/builtin/persistence/migrations/processing_configuration.py b/src/powercontext/builtin/persistence/migrations/processing_configuration.py new file mode 100644 index 0000000000..e3161e2459 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/processing_configuration.py @@ -0,0 +1,309 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Registered offline Processing configuration changes on current Server storage. + +This operation changes scheduling ownership, not the Alembic revision. Every +Processing phase joins one transaction: a failed mode switch cannot leave a +partially published configuration or an independently committed lease fence. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Literal +from uuid import uuid4 + +from sqlalchemy import Connection, text +from sqlalchemy.engine import make_url + +from powercontext.builtin.persistence import processing_migration +from powercontext.builtin.persistence.processing_migration import ProcessingSchemaNotReadyError +from powercontext.builtin.persistence.sqlite import SQLiteConfig + +from .backup import BackupContext, ForkBackupProvider, SQLiteBackupProvider +from .bundle import MigrationBundle, sqlite_inventory +from .connections import BackendIdentity, MaintenanceConnections +from .models import MigrationError, MigrationPlan, MigrationResult, digest +from .mysql import _Evidence, _Journal +from .mysql_schema import mysql_inventory +from .pinned import finish, pinned +from .server_readiness import check_server_schema +from .sqlite import SQLiteMigrationRunner + +OPERATION = "processing_configuration" +_OBJECTS = ( + "pc_artifact_processing_schema", + "pc_artifact_processing_leases", + "pc_artifact_processing_auto_wave_targets", +) + + +def _marker(connection: Connection) -> dict[str, Any]: + return dict( + connection + .execute( + text( + "SELECT schema_version,source_version,phase,migration_id,config_manifest " + "FROM pc_artifact_processing_schema WHERE singleton=1" + ) + ) + .mappings() + .one() + ) + + +class ProcessingConfigurationMigrationRunner: + """Share target locks, declarations, backup providers and durable evidence.""" + + def __init__( + self, + connections: MaintenanceConnections, + bundle: MigrationBundle, + *, + configuration: dict[str, Any], + evidence_directory: Path, + migration_id: str = "rfc1515", + batch_size: int = 100, + ) -> None: + self.connections = connections + self.bundle = bundle + self.configuration = json.loads(json.dumps(configuration)) + self.directory = evidence_directory.expanduser().resolve() + self.migration_id = migration_id + self.batch_size = batch_size + self.configuration_digest = digest({"configuration": configuration, "migration_id": migration_id}) + self.checksum = digest({ + "bundle": bundle.checksum, + "operation": OPERATION, + "implementation": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "processing": hashlib.sha256(Path(processing_migration.__file__).read_bytes()).hexdigest(), + }) + + def _journal(self, identity: BackendIdentity) -> _Journal: + return _Journal(self.directory / OPERATION, identity) + + def _inspect(self, connection: Connection | None, identity: BackendIdentity, **options) -> MigrationPlan: + self.bundle.assert_unchanged() + if connection is None: + raise MigrationError("migration_required", "Initialize or upgrade the full Server bundle first.") + check_server_schema(connection) + marker = _marker(connection) + fingerprint = digest({ + "schema": sqlite_inventory(connection) if identity.product == "sqlite" else mysql_inventory(connection), + "processing": marker, + }) + evidence = self._journal(identity).read() + active = evidence is not None and evidence.state != "complete" + if active and ( + evidence.bundle_checksum != self.checksum + or evidence.configuration_digest != self.configuration_digest + or evidence.source_revision != self.bundle.head + ): + raise MigrationError( + "recovery_required", "Resume with the original Processing operation and configuration." + ) + required = finish( + processing_migration.plan_processing_migration( + pinned(connection), config_manifest=self.configuration["processing_manifest"] + ) + ).required + if required and marker["migration_id"] == self.migration_id: + raise MigrationError( + "migration_id_reused", "A Processing configuration switch requires a new migration ID." + ) + policy = options.get("backup_policy", "auto") + if policy not in {"auto", "manual", "skip"}: + raise MigrationError("invalid_backup_policy", "Choose auto, manual or skip.") + if active and ( + evidence.backup_policy != policy + or ( + fingerprint != evidence.source_fingerprint and (required or marker["migration_id"] != self.migration_id) + ) + ): + raise MigrationError("recovery_required", "Keep the original Processing recovery point and backup policy.") + main_path = ( + self.directory / "maintenance.json" + if isinstance(self.connections.config, SQLiteConfig) + else self.directory / identity.database_id / "maintenance.json" + ) + if main_path.exists(): + try: + main = json.loads(main_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise MigrationError( + "recovery_required", "The original full Server evidence cannot be read." + ) from error + if not isinstance(main, dict) or main.get("state") != "complete": + raise MigrationError("recovery_required", "Complete the original full Server maintenance first.") + available = isinstance(self.connections.config, SQLiteConfig) + coordination = self.connections.coordination(identity) + recovering = active or "pending_run_id" in coordination + fields = { + "database_id": identity.database_id, + "source_revision": self.bundle.head, + "target_revision": self.bundle.head, + "schema_fingerprint": fingerprint, + "bundle_checksum": self.checksum, + "configuration_digest": self.configuration_digest, + "state": "recovery_required" if recovering else "migration_required" if required else "ready", + "revisions": (), + "domain_steps": (OPERATION,) if required or active else (), + "domain_validators": ("server_upgrade",), + "backup_policy": policy, + "backup_available": available, + "backup_reason": () + if available + else ("Native Fork recovery acceptance is unavailable; choose manual or skip.",), + "affected_objects": _OBJECTS, + "shared_database": options.get("shared_database", False), + "service": options.get("service"), + "coordination": coordination, + "readiness_scope": "server", + } + return MigrationPlan.model_validate({"plan_id": digest(fields), **fields}) + + def plan(self, **options) -> MigrationPlan: + return self.connections.run(lambda connection, identity, guard: self._inspect(connection, identity, **options)) + + def verify(self) -> MigrationResult: + plan = self.plan() + if plan.state != "ready": + raise MigrationError(plan.state, "Apply the registered Processing configuration operation first.") + return MigrationResult(revision=self.bundle.head, changed=False, readiness_scope="server") + + def _provider(self, connection: Connection, identity: BackendIdentity): + config = self.connections.config + if isinstance(config, SQLiteConfig): + return SQLiteBackupProvider(Path(str(make_url(config.url).database))) + return ForkBackupProvider( + connection, + database_name=identity.database_name, + expected_product="seekdb" if identity.product == "seekdb" else "oceanbase_ai", + directory=self.directory, + ) + + def apply( # noqa: C901 -- One transaction owns backup, configuration transition and recovery evidence. + self, + *, + plan_id: str | None = None, + accepted: bool = False, + maintenance_confirmed: bool = False, + backup_policy: Literal["auto", "manual", "skip"] = "auto", + backup_confirmed: bool = False, + accept_no_backup: bool = False, + backup_ref: str | None = None, + **options, + ) -> MigrationResult: + initial = self.plan(backup_policy=backup_policy, **options) + pending = (initial.coordination or {}).get("pending_run_id") + acknowledged = self.connections.acknowledge_previous_run + if pending is not None and pending != acknowledged: + raise MigrationError("recovery_required", "Acknowledge the exact previous remote maintenance run.") + if pending is None and acknowledged is not None: + raise MigrationError("stale_ack", "There is no pending run matching this acknowledgment.") + if initial.state == "ready": + return self.verify() + if options.get("service") is not None: + raise MigrationError("service_unsupported", "Use full Server maintenance to manage a local service.") + SQLiteMigrationRunner._require_acceptance( + initial, plan_id, accepted, maintenance_confirmed, backup_confirmed, accept_no_backup + ) + + def execute(connection, identity, guard): # noqa: C901 -- Keep the locked transaction and its recovery evidence together. + current = self._inspect(connection, identity, backup_policy=backup_policy, **options) + if current != initial: + raise MigrationError("plan_changed", "Processing state changed while acquiring the shared lock.") + if not current.domain_steps: + guard() + return MigrationResult(revision=self.bundle.head, changed=False, readiness_scope="server") + journal = self._journal(identity) + evidence = journal.read() + if evidence is None or evidence.state == "complete": + evidence = _Evidence( + state="preparing", + database_id=identity.database_id, + source_revision=self.bundle.head, + source_fingerprint=initial.schema_fingerprint, + bundle_checksum=self.checksum, + configuration_digest=self.configuration_digest, + last_revision=self.bundle.head, + maintenance_window_id=uuid4().hex, + backup_policy=backup_policy, + backup_state="pending", + ) + journal.write(evidence) + context = BackupContext( + database_id=identity.database_id, + source_revision=self.bundle.head, + bundle_checksum=self.checksum, + maintenance_window_id=evidence.maintenance_window_id, + objects=self.bundle.affected_objects, + writers_stopped=True, + ) + provider = self._provider(connection, identity) + # Backup must see committed state. The shared OS/engine lock remains held. + connection.commit() + if evidence.state == "preparing": + if backup_policy == "auto": + evidence.backup = provider.create_backup(context) + evidence.backup_state = "completed" + else: + evidence.backup_state = "user_confirmed" if backup_policy == "manual" else "skipped" + evidence.manual_ref = backup_ref + evidence.state = "active" + journal.write(evidence) + if evidence.backup is not None and provider.inspect_backup(evidence.backup).state != "completed": + raise MigrationError("backup_failed", "The original Processing recovery point is unavailable.") + guard() + connection.exec_driver_sql( + "BEGIN IMMEDIATE" if connection.dialect.name == "sqlite" else "START TRANSACTION" + ) + locked = self._inspect(connection, identity, backup_policy=backup_policy, **options) + if locked.schema_fingerprint != initial.schema_fingerprint: + raise MigrationError("plan_changed", "Processing state changed before the write transaction.") + try: + while not finish( + processing_migration.apply_processing_migration( + pinned(connection), + config_manifest=self.configuration["processing_manifest"], + migration_id=self.migration_id, + batch_size=self.batch_size, + ) + ).complete: + guard() + except ProcessingSchemaNotReadyError as error: + raise MigrationError("incompatible_processing_configuration", str(error)) from error + guard() + check_server_schema(connection) + self.bundle.domains.validate( + connection, self.bundle.head, backend=identity.product, configuration=self.configuration + ) + connection.commit() + evidence.state = "complete" + journal.write(evidence) + backup_state = evidence.backup_state + if backup_state == "pending": + raise MigrationError("backup_failed", "The Processing recovery point has not completed.") + return MigrationResult( + revision=self.bundle.head, + changed=True, + readiness_scope="server", + backup_state=backup_state, + backup_ref=evidence.backup.location if evidence.backup else evidence.manual_ref, + ) + + return self.connections.run(execute, writable=True) diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index 9e65dca352..5bbc87c1fd 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -26,6 +26,8 @@ import typer from pydantic import ValidationError +from sqlalchemy import inspect +from sqlalchemy.engine import make_url from sqlalchemy.exc import OperationalError from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile @@ -44,6 +46,7 @@ from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest from powercontext.cli.env_file import environment_context from powercontext.cli.inference_notice import write_inference_capability_notice +from powercontext.server import database_migration from powercontext.server.app import access_readiness_checks from powercontext.server.authz import PrincipalRef from powercontext.server.authz.composition import open_builtin_access_control @@ -117,17 +120,120 @@ def processing_migrate( str, typer.Option(help="Stable resume ID; use a new ID for an offline mode switch.") ] = "rfc1515", batch_size: Annotated[int, typer.Option(min=1, max=10000, help="Maximum rows committed per migration step.")] = 100, + evidence_dir: database_migration.EvidenceDir = None, + lock_coordination: database_migration.LockCoordination = None, + backup: Annotated[ + database_migration.BackupPolicy | None, typer.Option(help="Managed storage backup policy.") + ] = None, + plan_id: Annotated[str | None, typer.Option(help="Exact managed Processing plan to accept.")] = None, + yes: Annotated[bool, typer.Option("--yes", help="Accept the managed Processing plan.")] = False, + backup_confirmed: Annotated[bool, typer.Option(help="Declare a manual backup exists.")] = False, + accept_no_backup: Annotated[bool, typer.Option(help="Accept the no-backup risk.")] = False, + backup_ref: Annotated[str | None, typer.Option(help="Operator manual backup reference.")] = None, + acknowledge_previous_run: Annotated[ + str | None, typer.Option(help="OceanBase token confirming the previous remote maintenance ended.") + ] = None, ) -> None: """Plan, apply or verify the resumable artifact-processing schema migration.""" if action == "apply" and not maintenance_confirmed: raise typer.BadParameter("apply requires --maintenance-confirmed after stopping old workers and writes") # noqa: TRY003 + env_file = resolve_server_environment_file(env_file, discover=True) + with database_migration._operator_errors(), server_settings_context(env_file=env_file) as settings: + managed = _managed_storage(settings.database) + if managed: + if action == "plan": + database_migration.plan( + env_file=env_file, + evidence_dir=evidence_dir, + lock_coordination=lock_coordination, + backup=backup or "auto", + bundle="processing", + migration_id=migration_id, + batch_size=batch_size, + ) + elif action == "verify": + database_migration.verify( + env_file=env_file, + evidence_dir=evidence_dir, + lock_coordination=lock_coordination, + bundle="processing", + migration_id=migration_id, + batch_size=batch_size, + ) + else: + database_migration.apply( + env_file=env_file, + evidence_dir=evidence_dir, + lock_coordination=lock_coordination, + bundle="processing", + migration_id=migration_id, + batch_size=batch_size, + backup=backup, + plan_id=plan_id, + yes=yes, + maintenance_confirmed=maintenance_confirmed, + backup_confirmed=backup_confirmed, + accept_no_backup=accept_no_backup, + backup_ref=backup_ref, + acknowledge_previous_run=acknowledge_previous_run, + ) + return with server_settings_context(env_file=env_file) as settings: ready = asyncio.run(_processing_maintenance(settings, action, migration_id=migration_id, batch_size=batch_size)) if not ready: raise typer.Exit(code=1) +def _managed_storage(database: SQLiteConfig | SeekDBConfig | OceanBaseConfig) -> bool: + """Inspect ownership before a compatibility command may open a writable profile.""" + from powercontext.builtin.persistence.migrations.connections import MaintenanceConnections + from powercontext.builtin.persistence.migrations.mysql_schema import mysql_inventory + from powercontext.builtin.persistence.migrations.server_baselines import ( + recognize_mysql_server_baseline, + recognize_sqlite_server_baseline, + ) + + def owned(connection, backend): + if connection is None: + return False + if "pc_schema_revision" in inspect(connection).get_table_names(): + return True + source = ( + recognize_sqlite_server_baseline(connection) + if backend == "sqlite" + else recognize_mysql_server_baseline(mysql_inventory(connection), backend=backend) + ) + if source is not None: + raise database_migration.MigrationError( + "migration_required", + "Complete the registered full Server upgrade before Processing configuration maintenance.", + ) + return False + + if isinstance(database, SQLiteConfig) and not database.is_persistent: + return False + if not isinstance(database, SQLiteConfig) or not make_url(database.url).query: + return MaintenanceConnections(database).run( + lambda connection, identity, guard: owned(connection, identity.product) + ) + + async def inspect_sqlite(): + try: + async with ( + SQLiteProfile.open_readonly(database) as profile, + profile.database.transaction() as connection, + ): + return await connection.run_sync(lambda sync: owned(sync, "sqlite")) + except OperationalError as error: + code = getattr(error.orig, "sqlite_errorcode", None) + if isinstance(code, int) and code & 0xFF == SQLITE_CANTOPEN: + return False + raise + + return asyncio.run(inspect_sqlite()) + + async def _processing_maintenance( settings: ServerSettings, action: Literal["plan", "apply", "verify"], diff --git a/src/powercontext/server/database_migration.py b/src/powercontext/server/database_migration.py index dbc21430f8..295b0f2a80 100644 --- a/src/powercontext/server/database_migration.py +++ b/src/powercontext/server/database_migration.py @@ -40,7 +40,7 @@ BackupPolicy = Literal["auto", "manual", "skip"] Bundle = Annotated[ - Literal["server", "artifacts", "topic-memory-recovery"], + Literal["server", "artifacts", "topic-memory-recovery", "processing"], typer.Option(help="Select the registered domain bundle; readiness covers only that bundle."), ] EnvFile = Annotated[Path | None, typer.Option(help="Read the same environment file as the deployed Server.")] @@ -62,6 +62,10 @@ bool, typer.Option(help="Declare other Servers share this database; coordinate all writers through deployment tooling."), ] +MigrationID = Annotated[ + str, typer.Option(help="Processing maintenance resume ID; choose a new ID for each mode switch.") +] +BatchSize = Annotated[int, typer.Option(min=1, max=10000, help="Rows per Processing maintenance step.")] app = typer.Typer( name="db-migrate", @@ -93,9 +97,19 @@ def verify( evidence_dir: EvidenceDir = None, lock_coordination: LockCoordination = None, bundle: Bundle = "server", + migration_id: MigrationID = "rfc1515", + batch_size: BatchSize = 100, ) -> None: """Verify database readiness; this does not assert Server or cluster readiness.""" - _inspect(env_file, evidence_dir=evidence_dir, lock_coordination=lock_coordination, verify=True, bundle=bundle) + _inspect( + env_file, + evidence_dir=evidence_dir, + lock_coordination=lock_coordination, + verify=True, + bundle=bundle, + migration_id=migration_id, + batch_size=batch_size, + ) @app.command() @@ -107,11 +121,18 @@ def plan( manage_service: ManageService = False, shared_database: SharedDatabase = False, bundle: Bundle = "server", + migration_id: MigrationID = "rfc1515", + batch_size: BatchSize = 100, ) -> None: """Preview the target, resource digests, backup policy, and maintenance scope.""" with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: runner = deployment_runner( - settings, evidence_dir=evidence_dir, lock_coordination=lock_coordination, bundle=bundle + settings, + evidence_dir=evidence_dir, + lock_coordination=lock_coordination, + bundle=bundle, + migration_id=migration_id, + batch_size=batch_size, ) service = ( _service_scope(_environment_file(env_file), bundle=bundle, settings=settings) if manage_service else None @@ -155,6 +176,8 @@ def apply( manage_service: ManageService = False, shared_database: SharedDatabase = False, bundle: Bundle = "server", + migration_id: MigrationID = "rfc1515", + batch_size: BatchSize = 100, ) -> None: """Review once, then stop writes, back up, migrate, verify, and optionally switch a local service.""" with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: @@ -164,6 +187,8 @@ def apply( lock_coordination=lock_coordination, acknowledge_previous_run=acknowledge_previous_run, bundle=bundle, + migration_id=migration_id, + batch_size=batch_size, ) policy: BackupPolicy = backup or "auto" initial = runner.plan(backup_policy=policy, shared_database=shared_database) @@ -220,11 +245,18 @@ def _inspect( evidence_dir: Path | None, lock_coordination: Literal["single-host"] | None, verify: bool, - bundle: Literal["server", "artifacts", "topic-memory-recovery"] = "server", + bundle: Literal["server", "artifacts", "topic-memory-recovery", "processing"] = "server", + migration_id: str = "rfc1515", + batch_size: int = 100, ) -> None: with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: runner = deployment_runner( - settings, evidence_dir=evidence_dir, lock_coordination=lock_coordination, bundle=bundle + settings, + evidence_dir=evidence_dir, + lock_coordination=lock_coordination, + bundle=bundle, + migration_id=migration_id, + batch_size=batch_size, ) result = runner.verify() if verify else runner.plan() _write_payload(result) diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index 1d4d66dd47..ade74d44a5 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -38,15 +38,16 @@ from powercontext.builtin.persistence.migrations import mysql as mysql_migrations from powercontext.builtin.persistence.migrations.backup import BackupCapabilities, BackupProvider from powercontext.builtin.persistence.migrations.connections import BackendIdentity, MaintenanceConnections -from powercontext.builtin.persistence.migrations.deployment import production_bundle, server_bundle +from powercontext.builtin.persistence.migrations.deployment import deployment_runner, production_bundle, server_bundle from powercontext.builtin.persistence.migrations.models import MigrationError, MigrationPlan from powercontext.builtin.persistence.migrations.mysql import MySQLMigrationRunner from powercontext.builtin.persistence.migrations.mysql_schema import mysql_inventory from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.oceanbase import profile as oceanbase_profile from powercontext.builtin.persistence.seekdb import SeekDBConfig -from powercontext.builtin.runtime.config import BuiltinConfig +from powercontext.builtin.runtime.config import BuiltinConfig, RuntimeConfig from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest +from powercontext.server.settings import ServerSettings @dataclass(frozen=True) @@ -389,6 +390,50 @@ def _full_server_runner( ) +def test_mysql_registered_processing_mode_switch_preserves_revision_and_history(backend: _BackendTarget) -> None: + _load_full_server_baseline(backend) + _apply(_full_server_runner(backend), policy="manual") + settings = ServerSettings( + database=backend.config, + runtime=RuntimeConfig(artifact_processing_supervisor_mode="dedicated"), + ) + runner = deployment_runner( + settings, + bundle="processing", + evidence_dir=backend.evidence_directory, + lock_coordination="single-host" if isinstance(backend.config, OceanBaseConfig) else None, + migration_id="native-mode-switch", + ) + plan = runner.plan(backup_policy="manual") + assert plan.state == "migration_required" + result = runner.apply( + plan_id=plan.plan_id, + accepted=True, + maintenance_confirmed=True, + backup_policy="manual", + backup_confirmed=True, + backup_ref="operator://mode-backup", + ) + assert result.backup_state == "user_confirmed" and result.backup_ref == "operator://mode-backup" + assert runner.verify().state == "ready" + assert not runner.apply().changed + + def inspect_history(connection, identity, guard): + return ( + connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar_one(), + json.loads( + connection.exec_driver_sql( + "SELECT config_manifest FROM pc_artifact_processing_schema WHERE singleton=1" + ).scalar_one() + )["mode"], + connection.exec_driver_sql( + "SELECT content FROM pc_artifacts WHERE scope_id='historical' AND artifact_id='preserved'" + ).scalar_one(), + ) + + assert backend.connections().run(inspect_history) == ("s1401", "dedicated", b'{"history":"unchanged"}') + + def _load_full_server_baseline(backend: _BackendTarget) -> None: runner = _full_server_runner(backend) diff --git a/tests/test_managed_processing_maintenance.py b/tests/test_managed_processing_maintenance.py new file mode 100644 index 0000000000..5d544d90b8 --- /dev/null +++ b/tests/test_managed_processing_maintenance.py @@ -0,0 +1,275 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline configuration changes must share managed database maintenance.""" + +import json +import os +import sqlite3 +import subprocess +import sys +from contextlib import closing +from pathlib import Path + +import pytest +from alembic import command +from sqlalchemy import create_engine +from typer.testing import CliRunner + +from powercontext.builtin.persistence.migrations.deployment import deployment_runner, server_bundle +from powercontext.builtin.persistence.migrations.locking import local_migration_lock +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime.config import BuiltinConfig, RuntimeConfig +from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest +from powercontext.cli.app import create_cli +from powercontext.server.cli import app +from powercontext.server.settings import ServerSettings + + +@pytest.fixture +def managed(tmp_path): + path = tmp_path / "managed.db" + settings = ServerSettings(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{path}")) + runner = deployment_runner(settings) + plan = runner.plan() + runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + with closing(sqlite3.connect(path)) as connection, connection: + connection.execute( + "INSERT INTO pc_artifact_processing_leases " + "(supervisor_group,holder_id,supervisor_generation,lease_expires_at) VALUES ('global','old-worker',7,NULL)" + ) + return path, settings + + +def switched(settings): + return settings.model_copy(update={"runtime": RuntimeConfig(artifact_processing_supervisor_mode="dedicated")}) + + +def test_registered_mode_switch_backs_up_preserves_revision_and_fences_old_worker(managed): + path, settings = managed + runner = deployment_runner(switched(settings), bundle="processing") + plan = runner.plan() + assert plan.state == "migration_required" + with closing(sqlite3.connect(path)) as connection: + assert ( + json.loads(connection.execute("SELECT config_manifest FROM pc_artifact_processing_schema").fetchone()[0])[ + "mode" + ] + == "global" + ) + + result = runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + + assert result.backup_state == "completed" + assert result.backup_ref and Path(result.backup_ref).is_file() + with closing(sqlite3.connect(path)) as connection: + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("s1401",)] + assert ( + json.loads(connection.execute("SELECT config_manifest FROM pc_artifact_processing_schema").fetchone()[0])[ + "mode" + ] + == "dedicated" + ) + assert connection.execute("SELECT supervisor_generation FROM pc_artifact_processing_leases").fetchone()[0] > 7 + after = tuple(connection.iterdump()) + with closing(sqlite3.connect(result.backup_ref)) as connection: + assert ( + json.loads(connection.execute("SELECT config_manifest FROM pc_artifact_processing_schema").fetchone()[0])[ + "mode" + ] + == "global" + ) + assert deployment_runner(switched(settings)).verify().state == "ready" + assert not runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).changed + with closing(sqlite3.connect(path)) as connection: + assert tuple(connection.iterdump()) == after + + +def test_mode_switch_uses_the_whole_database_migration_lock(managed): + path, settings = managed + runner = deployment_runner(switched(settings), bundle="processing") + plan = runner.plan() + with local_migration_lock(path), pytest.raises(MigrationError, match="migration_locked"): + runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + assert deployment_runner(settings).verify().state == "ready" + + +def test_reused_mode_switch_id_is_rejected_before_creating_new_recovery_evidence(managed): + path, settings = managed + first = deployment_runner(switched(settings), bundle="processing", migration_id="first-switch") + plan = first.plan() + first.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + journal = next(path.parent.glob("*.pc-migration-state/processing_configuration/*/maintenance.json")) + completed = journal.read_bytes() + reused = deployment_runner(settings, bundle="processing", migration_id="first-switch") + with pytest.raises(MigrationError, match="migration_id_reused"): + reused.plan() + assert journal.read_bytes() == completed + reverse = deployment_runner(settings, bundle="processing", migration_id="second-switch") + plan = reverse.plan() + reverse.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + assert deployment_runner(settings).verify().state == "ready" + + +def test_process_exit_during_mode_switch_rolls_back_and_reuses_original_backup(managed): + path, settings = managed + runner = deployment_runner(switched(settings), bundle="processing") + plan = runner.plan() + environment = {key: value for key, value in os.environ.items() if not key.startswith("POWERCONTEXT_SERVER_")} + environment.update( + POWERCONTEXT_SERVER_DATABASE_KIND="sqlite", + POWERCONTEXT_SERVER_DATABASE_URL=f"sqlite+aiosqlite:///{path}", + POWERCONTEXT_SERVER_RUNTIME_ARTIFACT_PROCESSING_SUPERVISOR_MODE="dedicated", + ) + interrupted = subprocess.run( + [ + sys.executable, + "-c", + """ +import os +from sqlalchemy import event, Engine +from powercontext.cli.app import main +@event.listens_for(Engine, "after_cursor_execute") +def interrupt(connection, cursor, statement, parameters, context, executemany): + if statement.startswith("UPDATE pc_artifact_processing_leases "): + os._exit(73) +main() +""", + "server", + "db-migrate", + "apply", + "--bundle", + "processing", + "--plan-id", + plan.plan_id, + "--backup", + "auto", + "--yes", + "--maintenance-confirmed", + ], + env=environment, + capture_output=True, + text=True, + timeout=30, + ) + assert interrupted.returncode == 73, interrupted.stdout + interrupted.stderr + journal = next(path.parent.glob("*.pc-migration-state/processing_configuration/*/maintenance.json")) + original = json.loads(journal.read_text()) + assert original["state"] == "active" + with closing(sqlite3.connect(path)) as connection: + assert connection.execute("SELECT supervisor_generation FROM pc_artifact_processing_leases").fetchone() == (7,) + assert ( + json.loads(connection.execute("SELECT config_manifest FROM pc_artifact_processing_schema").fetchone()[0])[ + "mode" + ] + == "global" + ) + recovery = runner.plan() + assert recovery.state == "recovery_required" + result = runner.apply(plan_id=recovery.plan_id, accepted=True, maintenance_confirmed=True) + finished = json.loads(journal.read_text()) + assert finished["state"] == "complete" + assert finished["maintenance_window_id"] == original["maintenance_window_id"] + assert result.backup_ref == original["backup"]["location"] + assert deployment_runner(switched(settings)).verify().state == "ready" + + +def test_processing_alias_delegates_with_reviewed_plan_and_preserves_operator_migration_id(managed, monkeypatch): + path, _settings = managed + for key in tuple(os.environ): + if key.startswith("POWERCONTEXT_SERVER_"): + monkeypatch.delenv(key) + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_KIND", "sqlite") + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_URL", f"sqlite+aiosqlite:///{path}") + monkeypatch.setenv("POWERCONTEXT_SERVER_RUNTIME_ARTIFACT_PROCESSING_SUPERVISOR_MODE", "dedicated") + command = ["server", "processing-migrate", "--migration-id", "operator-switch"] + cli = create_cli([app]) + result = CliRunner().invoke(cli, [*command, "--action", "plan"]) + assert result.exit_code == 0, result.output + plan = json.loads(result.output) + assert plan["domain_steps"] == ["processing_configuration"] + denied = CliRunner().invoke(cli, [*command, "--action", "apply", "--maintenance-confirmed"]) + assert denied.exit_code != 0 + applied = CliRunner().invoke( + cli, + [ + *command, + "--action", + "apply", + "--maintenance-confirmed", + "--plan-id", + plan["plan_id"], + "--backup", + "auto", + "--yes", + ], + ) + assert applied.exit_code == 0, applied.output + with closing(sqlite3.connect(path)) as connection: + assert connection.execute("SELECT migration_id FROM pc_artifact_processing_schema").fetchone() == ( + "operator-switch", + ) + + +def test_processing_alias_does_not_modify_an_unversioned_complete_historical_server(tmp_path, monkeypatch): + path = tmp_path / "historical.db" + bundle = server_bundle() + engine = create_engine(f"sqlite:///{path}") + with engine.connect() as connection: + connection.exec_driver_sql("BEGIN IMMEDIATE") + config = bundle.config(connection) + config.attributes["configuration"] = {"processing_manifest": canonical_processing_manifest(BuiltinConfig())} + command.upgrade(config, "s1300") + connection.exec_driver_sql("DROP TABLE pc_schema_revision") + connection.commit() + engine.dispose() + original = path.read_bytes() + for key in tuple(os.environ): + if key.startswith("POWERCONTEXT_SERVER_"): + monkeypatch.delenv(key) + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_KIND", "sqlite") + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_URL", f"sqlite+aiosqlite:///{path}") + result = CliRunner().invoke( + create_cli([app]), + [ + "server", + "processing-migrate", + "--action", + "apply", + "--maintenance-confirmed", + "--migration-id", + "wrong-owner", + ], + ) + assert result.exit_code == 1, result.output + assert json.loads(result.output)["error"] == "migration_required" + assert path.read_bytes() == original + + +def test_processing_alias_inspection_and_execution_use_the_same_discovered_environment(managed, monkeypatch): + path, _settings = managed + monkeypatch.chdir(path.parent) + (path.parent / ".env").write_text( + f'POWERCONTEXT_SERVER_DATABASE_KIND=sqlite\nPOWERCONTEXT_SERVER_DATABASE_URL="sqlite+aiosqlite:///{path.as_posix()}"\n' + "POWERCONTEXT_SERVER_RUNTIME_ARTIFACT_PROCESSING_SUPERVISOR_MODE=dedicated\n", + encoding="utf-8", + ) + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_KIND", "sqlite") + other = path.parent / "other.db" + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_URL", f"sqlite+aiosqlite:///{other}") + result = CliRunner().invoke(create_cli([app]), ["server", "processing-migrate", "--action", "plan"]) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["domain_steps"] == ["processing_configuration"] + assert not other.exists() From 0a1056fbc56fd95f3c6ff17059c8ad924b566cd9 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 02:36:51 +0800 Subject: [PATCH 56/94] chore(release): review managed Processing maintenance Bind the persistence compatibility declaration to the registered managed Processing operation and its transaction, backup and alias ownership rules. Constraint: Frozen migration resources and upgrade-required policy are unchanged. Scope-risk: Exact installed-artifact acceptance remains required before release. Tested: Upgrade policy check; upgrade gate regressions on Windows and Linux (59 passed each); scoped hooks and full ty check. --- release/upgrade-support.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 1fb3f695eb..ed631da3fa 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "d67654098c4404726f70c7c262f0ecca969cf4691aa922ab59d099022de538e7", - "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback. New revision s1401 registers the frozen offline Memory archive on all three backends through the shared chain; s1300/s1400 resources and checksums remain unchanged, and archived history is preserved. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles." + "persistence_sha256": "aeb309fd603da7b66817332a0f86a5b49b277837d73ea1faf01a042ef6f37a1a", + "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback. New revision s1401 registers the frozen offline Memory archive on all three backends through the shared chain; s1300/s1400 resources and checksums remain unchanged, and archived history is preserved. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles. Managed Processing mode maintenance uses the canonical target lock, backup providers and durable evidence without changing pc_schema_revision or frozen migration resources. Its reviewed plan binds configuration and migration ID; one transaction publishes the mode and lease invalidation, and interrupted retries reuse the original recovery point. Compatibility commands delegate current managed Server storage and reject complete unversioned historical Server storage before domain writes; unmanaged domain behavior is retained. Atomic legacy maintenance/projection delegation is still required before framework activation." }, "cases": [], "automatic_sources": { From 70c73c1486f26273922d859e9294662db831a960 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 02:36:51 +0800 Subject: [PATCH 57/94] fix(ci): retain installed Desktop fixture Server diagnostics Preserve bounded synthetic Server warning/error logs and process status before temporary cleanup. Attach instance identity to failures and upload diagnostics for each Server so connection-isolation failures can be investigated. Constraint: Keep response-loss failures and existing acceptance criteria intact. Scope-risk: Diagnostic retention does not diagnose the earlier HTTP disconnect. Tested: Actual installed-wheel three-Server probe (two live Servers retain an injected response-loss failure; third normal shutdown); probe fails before retention; Ruff and ty on Windows; scoped hooks and full ty on Linux. --- .github/workflows/desktop.yml | 1 + desktop/tests/installed_fixture.py | 55 ++++++++++++++++++++++++------ desktop/tests/real_server.py | 2 +- 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 2ac1b7b9ba..63646ffdab 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -139,6 +139,7 @@ jobs: desktop/.artifacts/installed-ui-app.log desktop/.artifacts/installed-ui-processes.json desktop/.artifacts/installed-ui-window.png + desktop/.artifacts/installed-ui-server-*/ # Early validation failures can leave no installer or acceptance evidence. # Still reject missing artifacts when every preceding step succeeded. if-no-files-found: ${{ job.status == 'success' && 'error' || 'warn' }} diff --git a/desktop/tests/installed_fixture.py b/desktop/tests/installed_fixture.py index 015b2326a7..332d64f567 100644 --- a/desktop/tests/installed_fixture.py +++ b/desktop/tests/installed_fixture.py @@ -26,7 +26,7 @@ import time import zipfile from collections.abc import Iterator -from contextlib import contextmanager +from contextlib import contextmanager, suppress from pathlib import Path from typing import Any from urllib.parse import quote @@ -105,6 +105,7 @@ def isolated_server( "prefix": "", "port": port, "tls": False, + "uvicorn_log_level": "warning", "response_gate_path": str(response_gate) if response_gate else None, "response_loss_path": str(response_loss_counter) if response_loss_counter else None, } @@ -126,6 +127,13 @@ def isolated_server( creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0, ) control = control_pipe(process) + report = { + "pid": process.pid, + "port": port, + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), + "outcome": "failed", + "forced_kill": False, + } try: with httpx.Client(base_url=f"http://127.0.0.1:{port}", trust_env=False, timeout=10) as client: wait_ready(client, process) @@ -138,14 +146,41 @@ def isolated_server( }, ) response.raise_for_status() - yield client, response.json()["scope_id"], hashlib.sha256(wheel.read_bytes()).hexdigest() + yield client, response.json()["scope_id"], report["wheel_sha256"] + report["outcome"] = "passed" + except BaseException as error: + report["exception_type"] = type(error).__name__ + error.add_note(f"Installed fixture Server pid={process.pid}, port={port}") + raise finally: - if process.poll() is None: - control.write(b"stop\n") - control.flush() + report["before_cleanup_returncode"] = process.poll() + try: + if process.poll() is None: + with suppress(BrokenPipeError): + control.write(b"stop\n") + control.flush() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + report["forced_kill"] = True + process.kill() + process.wait(timeout=10) + finally: + report["after_cleanup_returncode"] = process.poll() + with suppress(BrokenPipeError): + control.close() + # Only the synthetic fixture's bounded logs survive temporary + # storage cleanup; database files and configuration do not. try: - process.wait(timeout=30) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=10) - control.close() + log.flush() + with (temp / "server.log").open("rb") as source: + size = source.seek(0, os.SEEK_END) + source.seek(max(0, size - 256 * 1024)) + tail = source.read(256 * 1024) + report["log_bytes"] = size + report["log_truncated"] = size > len(tail) + retained = Path(tempfile.mkdtemp(prefix="installed-ui-server-", dir=desktop / ".artifacts")) + (retained / "server.log").write_bytes(tail) + (retained / "process.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + except OSError: + print("Installed fixture Server diagnostics could not be retained.", file=sys.stderr) diff --git a/desktop/tests/real_server.py b/desktop/tests/real_server.py index 5ed528fcf2..dd7bc9161b 100644 --- a/desktop/tests/real_server.py +++ b/desktop/tests/real_server.py @@ -178,7 +178,7 @@ async def proxy(scope, receive, send): proxy, host="127.0.0.1", port=config["port"], - log_level="critical", + log_level=config.get("uvicorn_log_level", "critical"), access_log=False, ssl_keyfile=config["key"] if config["tls"] else None, ssl_certfile=config["certificate"] if config["tls"] else None, From 015026c96faae8f2f2bcb989bc3178343258be04 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 03:11:47 +0800 Subject: [PATCH 58/94] fix(migrations): delegate managed Atomic conversion to the Server chain Route managed and recognized complete historical Server storage through shared plan/apply/verify. Keep unmanaged conversion and reject legacy decision files on the managed path. Constraint: Preserve frozen revision resources and the no-vector, Code-disabled supported profile. Scope-risk: Compatibility CLI behavior and explicit backup declarations change only for managed Server storage. Tested: Windows and Linux focused CLI and release-gate selection each passed 107 tests; actual seekDB/OceanBase alias cases passed 2 tests without skips; scoped hooks and full Ty passed. --- .github/workflows/e2e-harness.yml | 5 +- .../docs/operate/atomic-memory-migration.md | 10 ++ docs/en/docs/operate/database-migrations.md | 6 + .../docs/operate/atomic-memory-migration.md | 9 ++ docs/zh/docs/operate/database-migrations.md | 5 + release/upgrade-support.json | 2 +- src/powercontext/server/cli.py | 60 +++++++- .../test_mysql_migration_runner.py | 60 ++++++++ tests/test_managed_atomic_maintenance.py | 141 ++++++++++++++++++ 9 files changed, 292 insertions(+), 6 deletions(-) create mode 100644 tests/test_managed_atomic_maintenance.py diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index cde768b214..00eb538217 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -186,6 +186,7 @@ jobs: timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -q -rs \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_full_server_interruption_preserves_history_and_original_maintenance \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_registered_processing_mode_switch_preserves_revision_and_history \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_atomic_compatibility_command_uses_shared_server_upgrade \ -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-recovery.xml uv run --locked python - <<'PY' from pathlib import Path @@ -193,8 +194,8 @@ jobs: suites = ElementTree.parse(Path(".powercontext/recall-effort-ci/full-server-recovery.xml")).getroot() cases = list(suites.iter("testcase")) - if len(cases) != 5 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): - raise SystemExit("All five native full Server recovery and maintenance scenarios must pass without skips.") + if len(cases) != 6 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): + raise SystemExit("All six native full Server recovery and maintenance scenarios must pass without skips.") PY - name: Collect the database log diff --git a/docs/en/docs/operate/atomic-memory-migration.md b/docs/en/docs/operate/atomic-memory-migration.md index eb970257b4..c778a2d938 100644 --- a/docs/en/docs/operate/atomic-memory-migration.md +++ b/docs/en/docs/operate/atomic-memory-migration.md @@ -14,6 +14,16 @@ legacy entry tables; complete per-entry verification is the job of `verify`. ## Run the maintenance task +On a database with `pc_schema_revision`, or a recognized complete 1.3.0 Server baseline, this compatibility command +delegates `plan`, `apply` and `verify` to `server db-migrate --bundle server`. It returns the shared plan/result format +and uses the same maintenance lock, backup policy and recovery evidence. Review the plan, then pass its `--plan-id`, +an explicit `--backup` policy, `--yes` and `--maintenance-confirmed` when changes are required. OceanBase also requires +the fixed-host coordination/evidence options described in [Database migrations](./database-migrations.md). +Already-ready storage is a read-only no-op. Candidate and Owner decision files apply only to the unmanaged legacy +conversion and are rejected on this registered Server path. This does not register pre-1.3.0 Memory layouts. + +The remaining instructions describe domain-only conversion for unmanaged databases: + Inspect the read-only plan with the new deployment's configuration: ```shell diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 62ef2cf7f2..d01dba1f31 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -41,6 +41,12 @@ Databases without a registered revision retain their domain-only migration comma Follow [Migrate Artifact processing state](artifact-processing-migration.md). See [Deploy the Server](deploy-server.md) for service deployment. +`server atomic-memory-migrate` delegates managed databases and recognized complete 1.3.0 Server baselines to +`db-migrate --bundle server`, including its reviewed-plan, backup and recovery requirements. It preserves domain-only +conversion for unmanaged legacy Memory layouts. Decision files for that conversion cannot modify the registered Server +chain. The compatibility command returns the shared Server plan/result for managed storage; an already-ready apply is a +read-only no-op. See [Migrate to Atomic Memory](atomic-memory-migration.md). + ## Processing configuration maintenance `--bundle processing` registers an offline ownership-configuration operation on a complete, current Server database. diff --git a/docs/zh/docs/operate/atomic-memory-migration.md b/docs/zh/docs/operate/atomic-memory-migration.md index e5e65fdccd..1f44efb60c 100644 --- a/docs/zh/docs/operate/atomic-memory-migration.md +++ b/docs/zh/docs/operate/atomic-memory-migration.md @@ -11,6 +11,15 @@ title: 迁移到 Atomic Memory ## 执行顺序 +数据库已有 `pc_schema_revision`，或匹配完整的 1.3.0 Server 基线时，这个兼容命令将 `plan`、`apply`、`verify` +转发到 `server db-migrate --bundle server`，返回共享计划和结果格式，并使用相同的维护锁、备份策略及恢复证据。 +需要变更时，先审阅计划，再提供对应的 `--plan-id`、显式 `--backup` 策略、`--yes` 和 `--maintenance-confirmed`。 +OceanBase 还要求[数据库迁移](./database-migrations.md)中说明的固定维护主机和证据目录选项。 +已就绪的数据库保持只读，不重复迁移。Candidate 和 Owner 决策文件只适用于未受管的旧 Memory 转换， +在已登记的 Server 路径上会被拒绝；该路径不因此增加对 1.3.0 之前 Memory 布局的支持。 + +下文描述未受管数据库的领域独立转换： + 先使用新版部署的配置查看只读计划： ```shell diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 75ab4d9c06..6e76a91954 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -32,6 +32,11 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 原有 `server processing-migrate` 在受管数据库上转发到 `db-migrate --bundle processing`；未登记 revision 的数据库仍保留领域独立迁移。具体步骤见[迁移 Artifact 处理状态](artifact-processing-migration.md)。服务部署参考[部署 Server](deploy-server.md)。 +`server atomic-memory-migrate` 在受管数据库或匹配完整 1.3.0 Server 基线的数据库上转发到 +`db-migrate --bundle server`，包括计划确认、备份和恢复要求；未受管的旧 Memory 布局继续使用领域独立转换。 +该转换的决策文件不能修改已登记的 Server 版本链。兼容命令在受管库上返回共享 Server 计划或结果， +已就绪时 apply 保持只读，不重复迁移。参见[迁移到 Atomic Memory](atomic-memory-migration.md)。 + ## Processing 配置维护 `--bundle processing` 在结构完整、已到当前 revision 的 Server 数据库上登记离线所有权配置操作。 diff --git a/release/upgrade-support.json b/release/upgrade-support.json index ed631da3fa..b322ec8876 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -4,7 +4,7 @@ "mode": "upgrade-required", "schema_review": { "persistence_sha256": "aeb309fd603da7b66817332a0f86a5b49b277837d73ea1faf01a042ef6f37a1a", - "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback. New revision s1401 registers the frozen offline Memory archive on all three backends through the shared chain; s1300/s1400 resources and checksums remain unchanged, and archived history is preserved. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles. Managed Processing mode maintenance uses the canonical target lock, backup providers and durable evidence without changing pc_schema_revision or frozen migration resources. Its reviewed plan binds configuration and migration ID; one transaction publishes the mode and lease invalidation, and interrupted retries reuse the original recovery point. Compatibility commands delegate current managed Server storage and reject complete unversioned historical Server storage before domain writes; unmanaged domain behavior is retained. Atomic legacy maintenance/projection delegation is still required before framework activation." + "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback. New revision s1401 registers the frozen offline Memory archive on all three backends through the shared chain; s1300/s1400 resources and checksums remain unchanged, and archived history is preserved. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles. Managed Processing mode maintenance uses the canonical target lock, backup providers and durable evidence without changing pc_schema_revision or frozen migration resources. Its reviewed plan binds configuration and migration ID; one transaction publishes the mode and lease invalidation, and interrupted retries reuse the original recovery point. Compatibility commands delegate current managed Server storage and reject complete unversioned historical Server storage before domain writes; unmanaged domain behavior is retained. The Atomic compatibility migration delegates managed databases and recognized complete Server baselines to the shared full Server chain; legacy decision files remain limited to unmanaged conversion. Atomic projection repair still requires shared coordination before framework activation." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index 5bbc87c1fd..ec99e836c9 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -185,7 +185,9 @@ def processing_migrate( raise typer.Exit(code=1) -def _managed_storage(database: SQLiteConfig | SeekDBConfig | OceanBaseConfig) -> bool: +def _managed_storage( + database: SQLiteConfig | SeekDBConfig | OceanBaseConfig, *, allow_historical: bool = False +) -> bool: """Inspect ownership before a compatibility command may open a writable profile.""" from powercontext.builtin.persistence.migrations.connections import MaintenanceConnections from powercontext.builtin.persistence.migrations.mysql_schema import mysql_inventory @@ -204,12 +206,12 @@ def owned(connection, backend): if backend == "sqlite" else recognize_mysql_server_baseline(mysql_inventory(connection), backend=backend) ) - if source is not None: + if source is not None and not allow_historical: raise database_migration.MigrationError( "migration_required", "Complete the registered full Server upgrade before Processing configuration maintenance.", ) - return False + return source is not None if isinstance(database, SQLiteConfig) and not database.is_persistent: return False @@ -336,6 +338,19 @@ def atomic_memory_migrate( Path | None, typer.Option(help="JSON attestations establishing Owners for ownerless legacy Memory entries."), ] = None, + evidence_dir: database_migration.EvidenceDir = None, + lock_coordination: database_migration.LockCoordination = None, + backup: Annotated[ + database_migration.BackupPolicy | None, typer.Option(help="Managed storage backup policy.") + ] = None, + plan_id: Annotated[str | None, typer.Option(help="Exact managed Server plan to accept.")] = None, + yes: Annotated[bool, typer.Option("--yes", help="Accept the managed Server plan.")] = False, + backup_confirmed: Annotated[bool, typer.Option(help="Declare a manual backup exists.")] = False, + accept_no_backup: Annotated[bool, typer.Option(help="Accept the no-backup risk.")] = False, + backup_ref: Annotated[str | None, typer.Option(help="Operator manual backup reference.")] = None, + acknowledge_previous_run: Annotated[ + str | None, typer.Option(help="OceanBase token confirming the previous remote maintenance ended.") + ] = None, ) -> None: """Archive Memory v1, convert its history and references to Atomic Memory, and remove it from public tables.""" @@ -353,6 +368,45 @@ def atomic_memory_migrate( ) except (OSError, ValueError, TypeError, KeyError) as error: raise typer.BadParameter(f"invalid decision file: {error}") from error # noqa: TRY003 + env_file = resolve_server_environment_file(env_file, discover=True) + with database_migration._operator_errors(), server_settings_context(env_file=env_file) as settings: + managed = _managed_storage(settings.database, allow_historical=True) + if managed: + if decisions is not None or owner_decisions is not None: + raise typer.BadParameter( # noqa: TRY003 + "decision files apply only to the unmanaged legacy Memory conversion, not the registered Server chain" + ) + if action == "plan": + database_migration.plan( + env_file=env_file, + evidence_dir=evidence_dir, + lock_coordination=lock_coordination, + backup=backup or "auto", + bundle="server", + ) + elif action == "verify": + database_migration.verify( + env_file=env_file, + evidence_dir=evidence_dir, + lock_coordination=lock_coordination, + bundle="server", + ) + else: + database_migration.apply( + env_file=env_file, + evidence_dir=evidence_dir, + lock_coordination=lock_coordination, + bundle="server", + backup=backup, + plan_id=plan_id, + yes=yes, + maintenance_confirmed=maintenance_confirmed, + backup_confirmed=backup_confirmed, + accept_no_backup=accept_no_backup, + backup_ref=backup_ref, + acknowledge_previous_run=acknowledge_previous_run, + ) + return with server_settings_context(env_file=env_file) as settings: try: ready = asyncio.run( diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index ade74d44a5..04f49fa77f 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -34,6 +34,7 @@ from sqlalchemy.engine import make_url from sqlalchemy.ext.asyncio import create_async_engine from sqlalchemy.pool import NullPool +from typer.testing import CliRunner from powercontext.builtin.persistence.migrations import mysql as mysql_migrations from powercontext.builtin.persistence.migrations.backup import BackupCapabilities, BackupProvider @@ -47,6 +48,8 @@ from powercontext.builtin.persistence.seekdb import SeekDBConfig from powercontext.builtin.runtime.config import BuiltinConfig, RuntimeConfig from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest +from powercontext.cli.app import create_cli +from powercontext.server.cli import app from powercontext.server.settings import ServerSettings @@ -434,6 +437,63 @@ def inspect_history(connection, identity, guard): assert backend.connections().run(inspect_history) == ("s1401", "dedicated", b'{"history":"unchanged"}') +def test_mysql_atomic_compatibility_command_uses_shared_server_upgrade(backend: _BackendTarget, monkeypatch) -> None: + _load_full_server_baseline(backend) + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_KIND", backend.config.kind) + if isinstance(backend.config, SeekDBConfig): + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_PATH", str(backend.config.path)) + else: + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_URL", backend.config.url.get_secret_value()) + cli = create_cli([app]) + arguments = [ + "server", + "atomic-memory-migrate", + "--evidence-dir", + str(backend.evidence_directory), + "--lock-coordination", + "single-host", + ] + inspected = CliRunner().invoke(cli, [*arguments, "--action", "plan", "--backup", "manual"]) + assert inspected.exit_code == 0, inspected.output + plan = json.loads(inspected.output) + assert plan["source_revision"] == "s1300" and plan["readiness_scope"] == "server" + applied = CliRunner().invoke( + cli, + [ + *arguments, + "--action", + "apply", + "--plan-id", + plan["plan_id"], + "--backup", + "manual", + "--backup-confirmed", + "--backup-ref", + "operator://native-atomic-backup", + "--maintenance-confirmed", + "--yes", + ], + ) + assert applied.exit_code == 0, applied.output + assert json.loads(applied.output)["backup_ref"] == "operator://native-atomic-backup" + verified = CliRunner().invoke(cli, [*arguments, "--action", "verify"]) + assert verified.exit_code == 0, verified.output + assert json.loads(verified.output)["readiness_scope"] == "server" + repeated = CliRunner().invoke(cli, [*arguments, "--action", "apply", "--maintenance-confirmed"]) + assert repeated.exit_code == 0, repeated.output + assert json.loads(repeated.output)["changed"] is False + + def inspect_history(connection, identity, guard): + return ( + connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar_one(), + connection.exec_driver_sql( + "SELECT content FROM pc_artifacts WHERE scope_id='historical' AND artifact_id='preserved'" + ).scalar_one(), + ) + + assert backend.connections().run(inspect_history) == ("s1401", b'{"history":"unchanged"}') + + def _load_full_server_baseline(backend: _BackendTarget) -> None: runner = _full_server_runner(backend) diff --git a/tests/test_managed_atomic_maintenance.py b/tests/test_managed_atomic_maintenance.py new file mode 100644 index 0000000000..f17cd23387 --- /dev/null +++ b/tests/test_managed_atomic_maintenance.py @@ -0,0 +1,141 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Atomic compatibility commands preserve managed Server migration ownership.""" + +import json +import sqlite3 +from contextlib import closing + +import pytest +from alembic import command +from sqlalchemy import create_engine +from typer.testing import CliRunner + +from powercontext.builtin.persistence.migrations.deployment import deployment_runner, server_bundle +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime.config import BuiltinConfig +from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest +from powercontext.cli.app import create_cli +from powercontext.server.cli import app +from powercontext.server.settings import ServerSettings + + +@pytest.fixture +def historical(tmp_path, monkeypatch): + path = tmp_path / "historical.db" + bundle = server_bundle() + engine = create_engine(f"sqlite:///{path}") + with engine.connect() as connection: + connection.exec_driver_sql("BEGIN IMMEDIATE") + config = bundle.config(connection) + config.attributes["configuration"] = {"processing_manifest": canonical_processing_manifest(BuiltinConfig())} + command.upgrade(config, "s1300") + connection.exec_driver_sql("DROP TABLE pc_schema_revision") + connection.exec_driver_sql( + "INSERT INTO pc_scopes (scope_id,title,summary,scope_id_search,title_search,summary_search,version) " + "VALUES ('scope','title','summary','scope','title','summary',1)" + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifacts (scope_id,family,artifact_id,revision,content) " + "VALUES ('scope','custom-family','historical',1,?)", + (b'{"history":"unchanged"}',), + ) + connection.commit() + engine.dispose() + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_KIND", "sqlite") + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_URL", f"sqlite+aiosqlite:///{path}") + return path + + +def upgraded(path): + runner = deployment_runner(ServerSettings(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{path}"))) + plan = runner.plan() + runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + + +def test_atomic_alias_upgrades_complete_historical_server_with_shared_plan_and_backup(historical): + cli = create_cli([app]) + arguments = ["server", "atomic-memory-migrate"] + original = historical.read_bytes() + with closing(sqlite3.connect(historical)) as connection: + original_dump = tuple(connection.iterdump()) + + inspected = CliRunner().invoke(cli, [*arguments, "--action", "plan"]) + assert inspected.exit_code == 0, inspected.output + plan = json.loads(inspected.output) + assert plan["source_revision"] == "s1300" + assert plan["target_revision"] == "s1401" + assert plan["readiness_scope"] == "server" + assert historical.read_bytes() == original + + denied = CliRunner().invoke(cli, [*arguments, "--action", "apply", "--maintenance-confirmed"]) + assert denied.exit_code != 0 + assert historical.read_bytes() == original + accepted = CliRunner().invoke( + cli, + [ + *arguments, + "--action", + "apply", + "--maintenance-confirmed", + "--plan-id", + plan["plan_id"], + "--backup", + "auto", + "--yes", + ], + ) + assert accepted.exit_code == 0, accepted.output + result = json.loads(accepted.output) + assert result["readiness_scope"] == "server" and result["backup_state"] == "completed" + with closing(sqlite3.connect(result["backup_ref"])) as connection: + assert tuple(connection.iterdump()) == original_dump + with closing(sqlite3.connect(historical)) as connection: + assert connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() == [("s1401",)] + assert connection.execute("SELECT content FROM pc_artifacts").fetchall() == [(b'{"history":"unchanged"}',)] + verified = CliRunner().invoke(cli, [*arguments, "--action", "verify"]) + assert verified.exit_code == 0, verified.output + assert json.loads(verified.output)["state"] == "ready" + after = historical.read_bytes() + ready_plan = CliRunner().invoke(cli, [*arguments, "--action", "plan"]) + assert ready_plan.exit_code == 0, ready_plan.output + assert json.loads(ready_plan.output)["state"] == "ready" + assert historical.read_bytes() == after + repeated = CliRunner().invoke(cli, [*arguments, "--action", "apply", "--maintenance-confirmed"]) + assert repeated.exit_code == 0, repeated.output + assert json.loads(repeated.output)["changed"] is False + assert historical.read_bytes() == after + + +def test_atomic_alias_rejects_unregistered_owner_decisions_on_managed_server(historical): + upgraded(historical) + decisions = historical.with_suffix(".json") + decisions.write_text('{"format":"powercontext.atomic-memory-owner-decisions.v1","decisions":[]}', encoding="utf-8") + original = historical.read_bytes() + result = CliRunner().invoke( + create_cli([app]), + [ + "server", + "atomic-memory-migrate", + "--action", + "apply", + "--maintenance-confirmed", + "--owner-decisions", + str(decisions), + ], + ) + assert result.exit_code != 0 + assert "decision files" in result.output + assert historical.read_bytes() == original From c0e585e5eeb84fc800e45c9f5d42ecb5659a9a63 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 03:12:04 +0800 Subject: [PATCH 59/94] test(desktop): retain diagnostics for both isolated Server harnesses Share bounded process and Server-log evidence between native adapter and installed UI fixtures. Record synthetic HTTP completion without paths, headers or bodies, and upload native fixture evidence after CI failure. Constraint: Preserve failures and acceptance criteria; retain no databases or credentials. Scope-risk: Test-harness diagnostics and graceful cleanup only; the Desktop disconnect and request timeout still require diagnosis. Tested: Actual installed-wheel three-Server probe preserved the deliberate response-loss failure and both failed-fixture logs, with clean shutdown; scoped hooks and full Ty passed. --- .github/workflows/desktop.yml | 1 + desktop/tests/installed_fixture.py | 86 ++++------- desktop/tests/real_server.py | 240 ++++++++++++++++++++--------- 3 files changed, 197 insertions(+), 130 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 63646ffdab..41795f52b8 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -132,6 +132,7 @@ jobs: desktop/.artifacts/windows-smoke.json desktop/.artifacts/ci-compatibility.json desktop/.artifacts/real-server.json + desktop/.artifacts/real-server-*/ desktop/.artifacts/real-cli.json desktop/.artifacts/installed-ui.json desktop/.artifacts/installed-ui.png diff --git a/desktop/tests/installed_fixture.py b/desktop/tests/installed_fixture.py index 332d64f567..037a4fc776 100644 --- a/desktop/tests/installed_fixture.py +++ b/desktop/tests/installed_fixture.py @@ -26,14 +26,14 @@ import time import zipfile from collections.abc import Iterator -from contextlib import contextmanager, suppress +from contextlib import contextmanager from pathlib import Path from typing import Any from urllib.parse import quote import httpx from fixture_qualification import fixture_profile -from real_server import HarnessFailure, control_pipe +from real_server import HarnessFailure, retained_fixture_server def fixture_compatibility_profile() -> str: @@ -89,6 +89,7 @@ def isolated_server( if len(wheels) != 1: raise HarnessFailure("installed_server_wheel_count") wheel = wheels[0] + wheel_digest = hashlib.sha256(wheel.read_bytes()).hexdigest() with tempfile.TemporaryDirectory(prefix="desktop-installed-server-") as directory: temp = Path(directory) wheel_root = temp / "wheel" @@ -126,61 +127,26 @@ def isolated_server( env=environment, creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0, ) - control = control_pipe(process) - report = { - "pid": process.pid, - "port": port, - "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), - "outcome": "failed", - "forced_kill": False, - } - try: - with httpx.Client(base_url=f"http://127.0.0.1:{port}", trust_env=False, timeout=10) as client: - wait_ready(client, process) - response = client.post( - "/v1/scopes", - json={ - "title": "Desktop installed CI", - "summary": "Synthetic installed UI acceptance", - "idempotency_key": "desktop-installed-ui-scope", - }, - ) - response.raise_for_status() - yield client, response.json()["scope_id"], report["wheel_sha256"] - report["outcome"] = "passed" - except BaseException as error: - report["exception_type"] = type(error).__name__ - error.add_note(f"Installed fixture Server pid={process.pid}, port={port}") - raise - finally: - report["before_cleanup_returncode"] = process.poll() - try: - if process.poll() is None: - with suppress(BrokenPipeError): - control.write(b"stop\n") - control.flush() - try: - process.wait(timeout=30) - except subprocess.TimeoutExpired: - report["forced_kill"] = True - process.kill() - process.wait(timeout=10) - finally: - report["after_cleanup_returncode"] = process.poll() - with suppress(BrokenPipeError): - control.close() - # Only the synthetic fixture's bounded logs survive temporary - # storage cleanup; database files and configuration do not. - try: - log.flush() - with (temp / "server.log").open("rb") as source: - size = source.seek(0, os.SEEK_END) - source.seek(max(0, size - 256 * 1024)) - tail = source.read(256 * 1024) - report["log_bytes"] = size - report["log_truncated"] = size > len(tail) - retained = Path(tempfile.mkdtemp(prefix="installed-ui-server-", dir=desktop / ".artifacts")) - (retained / "server.log").write_bytes(tail) - (retained / "process.json").write_text(json.dumps(report, indent=2), encoding="utf-8") - except OSError: - print("Installed fixture Server diagnostics could not be retained.", file=sys.stderr) + with ( + retained_fixture_server( + process, + log, + temp / "server.log", + desktop / ".artifacts", + label="installed-ui-server", + port=port, + wheel_sha256=wheel_digest, + ), + httpx.Client(base_url=f"http://127.0.0.1:{port}", trust_env=False, timeout=10) as client, + ): + wait_ready(client, process) + response = client.post( + "/v1/scopes", + json={ + "title": "Desktop installed CI", + "summary": "Synthetic installed UI acceptance", + "idempotency_key": "desktop-installed-ui-scope", + }, + ) + response.raise_for_status() + yield client, response.json()["scope_id"], wheel_digest diff --git a/desktop/tests/real_server.py b/desktop/tests/real_server.py index dd7bc9161b..43db2b8821 100644 --- a/desktop/tests/real_server.py +++ b/desktop/tests/real_server.py @@ -35,8 +35,10 @@ import threading import time import zipfile +from collections.abc import Iterator +from contextlib import contextmanager, suppress from pathlib import Path -from typing import IO +from typing import IO, TextIO class HarnessFailure(RuntimeError): @@ -132,6 +134,46 @@ def wait_for_release(): await send(message) +def measured_requests(app): + """Observe synthetic HTTP completion without recording request content.""" + request_number = 0 + + async def measured_proxy(scope, receive, send): + nonlocal request_number + if scope["type"] != "http": + await app(scope, receive, send) + return + request_number += 1 + number = request_number + started = time.perf_counter() + status = None + complete = False + print(json.dumps({"request": number, "phase": "start", "method": scope["method"]}), flush=True) + + async def observed_send(message): + nonlocal status + if message["type"] == "http.response.start": + status = message["status"] + await send(message) + + try: + await app(scope, receive, observed_send) + complete = True + finally: + print( + json.dumps({ + "request": number, + "phase": "finish", + "elapsed_ms": int((time.perf_counter() - started) * 1000), + "status": status, + "complete": complete, + }), + flush=True, + ) + + return measured_proxy + + def serve(config_path: Path) -> None: config = json.loads(config_path.read_text(encoding="utf-8")) sys.path.insert(0, config["wheel_root"]) @@ -175,7 +217,7 @@ async def proxy(scope, receive, send): server = uvicorn.Server( uvicorn.Config( - proxy, + measured_requests(proxy), host="127.0.0.1", port=config["port"], log_level=config.get("uvicorn_log_level", "critical"), @@ -228,6 +270,66 @@ def control_pipe(process: subprocess.Popen[bytes]) -> IO[bytes]: return process.stdin +@contextmanager +def retained_fixture_server( + process: subprocess.Popen[bytes], + log: TextIO, + log_path: Path, + artifact_dir: Path, + *, + label: str, + port: int, + wheel_sha256: str, +) -> Iterator[None]: + """Retain bounded synthetic diagnostics before either harness removes storage.""" + control = control_pipe(process) + report = { + "pid": process.pid, + "port": port, + "wheel_sha256": wheel_sha256, + "outcome": "failed", + "forced_kill": False, + } + try: + yield + report["outcome"] = "passed" + except BaseException as error: + report["exception_type"] = type(error).__name__ + error.add_note(f"Fixture {label} Server pid={process.pid}, port={port}") + raise + finally: + report["before_cleanup_returncode"] = process.poll() + try: + if process.poll() is None: + with suppress(BrokenPipeError): + control.write(b"stop\n") + control.flush() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + report["forced_kill"] = True + process.kill() + process.wait(timeout=10) + finally: + report["after_cleanup_returncode"] = process.poll() + with suppress(BrokenPipeError): + control.close() + # Databases, credentials and configuration stay in disposable storage. + try: + log.flush() + with log_path.open("rb") as source: + size = source.seek(0, os.SEEK_END) + source.seek(max(0, size - 256 * 1024)) + tail = source.read(256 * 1024) + report["log_bytes"] = size + report["log_truncated"] = size > len(tail) + retained = Path(tempfile.mkdtemp(prefix=label + "-", dir=artifact_dir)) + (retained / "server.log").write_bytes(tail) + (retained / "process.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + except OSError: + print("Fixture Server diagnostics could not be retained.", file=sys.stderr) + + def main() -> None: import httpx from fixture_qualification import fixture_profile @@ -272,6 +374,7 @@ def main() -> None: "prefix": prefix, "port": port, "tls": tls, + "uvicorn_log_level": "warning", "key": str(temp / "server-key.pem"), "certificate": str(temp / "server.pem"), } @@ -293,81 +396,78 @@ def main() -> None: env=allowed_env, creationflags=flags, ) - control = control_pipe(process) context = ssl.create_default_context(cafile=str(temp / "ca.pem")) if tls else True - try: - with httpx.Client( + with ( + retained_fixture_server( + process, + log, + case / "server.log", + artifact_dir, + label=f"real-server-{mode}", + port=port, + wheel_sha256=hashlib.sha256(wheel.read_bytes()).hexdigest(), + ), + httpx.Client( base_url=endpoint, verify=context, trust_env=False, timeout=3, headers={"Authorization": f"Bearer {token}"} if token else {}, - ) as client: - for _ in range(100): - if process.poll() is not None: - raise HarnessFailure( - "startup", (case / "server.log").read_text(encoding="utf-8")[-3000:] - ) - try: - if client.get("/health/ready").status_code == 200: - break - except httpx.HTTPError: - pass - time.sleep(0.2) - else: - raise HarnessFailure("readiness_timeout") - result = client.post( - "/v1/scopes", - json={ - "title": "Desktop synthetic fixture", - "summary": "Dedicated no-model validation", - "idempotency_key": "desktop-fixture-scope", - }, - ) - result.raise_for_status() - scope_id = result.json()["scope_id"] - fixture = { - "compatibility_profile": compatibility, - "response_loss_path": config["response_loss_path"], - "reader_token": config["reader_token"] if config["provider"] else None, - "identity_change_path": config["identity_change_path"] if config["provider"] else None, - "endpoint": endpoint, - "scope_id": scope_id, - "token": token, - "ca_pem": (temp / "ca.pem").read_text(encoding="utf-8") if tls else None, - } - fixture_path = case / "client.json" - fixture_path.write_text(json.dumps(fixture), encoding="utf-8") - probe = subprocess.run( - [str(executable), str(fixture_path)], - capture_output=True, - text=True, - timeout=120, - creationflags=flags, - ) - if probe.returncode: - raise HarnessFailure(mode, probe.stderr[-2000:]) - client.get("/health/live").raise_for_status() - reports.append({ - "mode": mode, - "result": "passed", - "serverAliveAfterClientExit": True, - "committedWriteWithLostResponseUnknownWithoutReplay": True, - "providerIdentityChangeInvalidatesContext": bool(config["provider"]), - "sameIdentityRevocationDeniesExactMemoryRead": bool(config["provider"]), - "sameTitleScopePagination51": bool(config["provider"]), - "performance": probe_measurement(probe.stdout), - }) - finally: - if process.poll() is None: - control.write(b"stop\n") - control.flush() + ) as client, + ): + for _ in range(100): + if process.poll() is not None: + raise HarnessFailure("startup", (case / "server.log").read_text(encoding="utf-8")[-3000:]) try: - process.wait(timeout=30) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=10) - control.close() + if client.get("/health/ready").status_code == 200: + break + except httpx.HTTPError: + pass + time.sleep(0.2) + else: + raise HarnessFailure("readiness_timeout") + result = client.post( + "/v1/scopes", + json={ + "title": "Desktop synthetic fixture", + "summary": "Dedicated no-model validation", + "idempotency_key": "desktop-fixture-scope", + }, + ) + result.raise_for_status() + scope_id = result.json()["scope_id"] + fixture = { + "compatibility_profile": compatibility, + "response_loss_path": config["response_loss_path"], + "reader_token": config["reader_token"] if config["provider"] else None, + "identity_change_path": config["identity_change_path"] if config["provider"] else None, + "endpoint": endpoint, + "scope_id": scope_id, + "token": token, + "ca_pem": (temp / "ca.pem").read_text(encoding="utf-8") if tls else None, + } + fixture_path = case / "client.json" + fixture_path.write_text(json.dumps(fixture), encoding="utf-8") + probe = subprocess.run( + [str(executable), str(fixture_path)], + capture_output=True, + text=True, + timeout=120, + creationflags=flags, + ) + if probe.returncode: + raise HarnessFailure(mode, probe.stderr[-2000:]) + client.get("/health/live").raise_for_status() + reports.append({ + "mode": mode, + "result": "passed", + "serverAliveAfterClientExit": True, + "committedWriteWithLostResponseUnknownWithoutReplay": True, + "providerIdentityChangeInvalidatesContext": bool(config["provider"]), + "sameIdentityRevocationDeniesExactMemoryRead": bool(config["provider"]), + "sameTitleScopePagination51": bool(config["provider"]), + "performance": probe_measurement(probe.stdout), + }) report = { "environment": { "os": platform.platform(), From 3a62282ea29063eccb1948daba89fba9416ea130 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 03:44:18 +0800 Subject: [PATCH 60/94] test(migrations): isolate native CLI capture from live SQL logging Pytest live logging suspends its capture streams during SQL emission. Running Typer isolation inside active pytest capture closed the CLI output buffer and failed the real seekDB acceptance at plan inspection. Suspend pytest capture around each CLI invocation while preserving SQL logs and assertions. Constraint: Preserve real backend execution and migration acceptance checks. Scope-risk: Native test capture boundaries only. Tested: Reproduced the exact CI ValueError with POWERCONTEXT_TEST_MIGRATION_SQL_LOG=1 and --log-cli-level=INFO; the same real seekDB case passes after this change. --- .../test_mysql_migration_runner.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index 04f49fa77f..40be1e215d 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -437,7 +437,9 @@ def inspect_history(connection, identity, guard): assert backend.connections().run(inspect_history) == ("s1401", "dedicated", b'{"history":"unchanged"}') -def test_mysql_atomic_compatibility_command_uses_shared_server_upgrade(backend: _BackendTarget, monkeypatch) -> None: +def test_mysql_atomic_compatibility_command_uses_shared_server_upgrade( + backend: _BackendTarget, monkeypatch, capsys +) -> None: _load_full_server_baseline(backend) monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_KIND", backend.config.kind) if isinstance(backend.config, SeekDBConfig): @@ -453,14 +455,18 @@ def test_mysql_atomic_compatibility_command_uses_shared_server_upgrade(backend: "--lock-coordination", "single-host", ] - inspected = CliRunner().invoke(cli, [*arguments, "--action", "plan", "--backup", "manual"]) + def invoke(options): + # Pytest live SQL logging suspends capture during emit. Keep its + # stream switching outside Typer's independent stdout isolation. + with capsys.disabled(): + return CliRunner().invoke(cli, [*arguments, *options]) + + inspected = invoke(["--action", "plan", "--backup", "manual"]) assert inspected.exit_code == 0, inspected.output plan = json.loads(inspected.output) assert plan["source_revision"] == "s1300" and plan["readiness_scope"] == "server" - applied = CliRunner().invoke( - cli, + applied = invoke( [ - *arguments, "--action", "apply", "--plan-id", @@ -476,10 +482,10 @@ def test_mysql_atomic_compatibility_command_uses_shared_server_upgrade(backend: ) assert applied.exit_code == 0, applied.output assert json.loads(applied.output)["backup_ref"] == "operator://native-atomic-backup" - verified = CliRunner().invoke(cli, [*arguments, "--action", "verify"]) + verified = invoke(["--action", "verify"]) assert verified.exit_code == 0, verified.output assert json.loads(verified.output)["readiness_scope"] == "server" - repeated = CliRunner().invoke(cli, [*arguments, "--action", "apply", "--maintenance-confirmed"]) + repeated = invoke(["--action", "apply", "--maintenance-confirmed"]) assert repeated.exit_code == 0, repeated.output assert json.loads(repeated.output)["changed"] is False From cbad742564df2b0b4858f68eeaab872b4d6f9be6 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 03:52:14 +0800 Subject: [PATCH 61/94] fix(migrations): coordinate managed Atomic projection repair Register atomic-projection maintenance and route the managed compatibility command through the shared lock, backup and durable recovery flow. Preserve immutable authority and the Server revision while repairing current rows and SQLite FTS identity tokens in one validated transaction. Constraint: Register only current managed no-vector, Code-disabled storage. Frozen migration resources and unmanaged repair behavior remain intact. Scope-risk: Native same-engine cases and manual backup declarations do not establish native crash/reopen or restoration. Installed artifact acceptance and optional profiles remain incomplete. Tested: Linux focused migration/CLI/release/projection selection 173 passed; Windows 172 passed and one symlink privilege skip; actual seekDB/OceanBase Processing, Atomic alias and projection selection 6 passed without skips under live SQL logging; scoped prek including full Ty; managed declaration digest and embedded workflow Python syntax. --- .github/workflows/e2e-harness.yml | 5 +- .../docs/operate/atomic-memory-migration.md | 9 + docs/en/docs/operate/database-migrations.md | 25 ++ .../docs/operate/atomic-memory-migration.md | 9 + docs/zh/docs/operate/database-migrations.md | 24 ++ release/upgrade-support.json | 4 +- .../migrations/atomic_projection.py | 164 +++++++++ .../persistence/migrations/deployment.py | 19 +- .../persistence/migrations/maintenance.py | 276 +++++++++++++++ .../migrations/processing_configuration.py | 316 +++--------------- .../migrations/server_readiness.py | 11 +- .../persistence/sqlite/atomic_memory_index.py | 11 + .../builtin/runtime/atomic_memory_rebuild.py | 39 ++- src/powercontext/server/cli.py | 56 ++++ src/powercontext/server/database_migration.py | 4 +- .../test_mysql_migration_runner.py | 67 ++++ tests/test_managed_atomic_projection.py | 225 +++++++++++++ 17 files changed, 984 insertions(+), 280 deletions(-) create mode 100644 src/powercontext/builtin/persistence/migrations/atomic_projection.py create mode 100644 src/powercontext/builtin/persistence/migrations/maintenance.py create mode 100644 tests/test_managed_atomic_projection.py diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 00eb538217..29801a650e 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -187,6 +187,7 @@ jobs: tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_full_server_interruption_preserves_history_and_original_maintenance \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_registered_processing_mode_switch_preserves_revision_and_history \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_atomic_compatibility_command_uses_shared_server_upgrade \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_registered_projection_repair_preserves_authority_and_revision \ -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-recovery.xml uv run --locked python - <<'PY' from pathlib import Path @@ -194,8 +195,8 @@ jobs: suites = ElementTree.parse(Path(".powercontext/recall-effort-ci/full-server-recovery.xml")).getroot() cases = list(suites.iter("testcase")) - if len(cases) != 6 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): - raise SystemExit("All six native full Server recovery and maintenance scenarios must pass without skips.") + if len(cases) != 7 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): + raise SystemExit("All seven native full Server recovery and maintenance scenarios must pass without skips.") PY - name: Collect the database log diff --git a/docs/en/docs/operate/atomic-memory-migration.md b/docs/en/docs/operate/atomic-memory-migration.md index c778a2d938..5bbf970505 100644 --- a/docs/en/docs/operate/atomic-memory-migration.md +++ b/docs/en/docs/operate/atomic-memory-migration.md @@ -241,6 +241,15 @@ Memory content restoration is a separate operation. ## Rebuild the current search projection +On managed Server storage, this command delegates to +[`db-migrate --bundle atomic-projection`](./database-migrations.md#atomic-memory-projection-maintenance). +Use `--action plan` to inspect without writes, then apply with `--plan-id`, explicit `--backup`, `--yes` and +`--maintenance-confirmed`. The registered no-vector repair retains a recovery point, commits the complete repair +in one transaction and returns the shared Server result. `--action verify` checks current readiness; an already +ready projection is a read-only no-op. This path does not register vector-profile changes. + +The remaining steps describe standalone repair on unmanaged storage: + After changing the embedding model, profile, dimension or normalization, or repairing current search data, back up the database and stop every API, host and Worker, their automatic restart and all input writes. Use the target deployment configuration: diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index d01dba1f31..11e9bd1e19 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -74,6 +74,31 @@ backup policy, and evidence directory; the original maintenance window and recov Successful retry does not invalidate leases again. Full Server validators run before and after the change. An already matching configuration is a read-only no-op. +## Atomic Memory projection maintenance + +`--bundle atomic-projection` repairs the current Atomic Memory rows and SQLite FTS helper on a current managed +Server database in the registered no-vector, Code-disabled configuration. It preserves the Server revision, +immutable content/history, lifecycle, formal tags, permissions and Source/Processing state. + +```bash +powercontext server db-migrate plan --bundle atomic-projection --env-file .env --backup auto +powercontext server db-migrate apply --bundle atomic-projection --env-file .env \ + --backup auto --plan-id PLAN_ID --maintenance-confirmed --yes +powercontext server db-migrate verify --bundle atomic-projection --env-file .env +``` + +The operation shares Processing maintenance's target lock, backup, stopped-writer declarations and durable recovery +transaction. Plan and verify are read-only. Apply repairs only derived rows and commits after full Server validation; +invalid authority or an unregistered layout is rejected. A ready projection is a read-only no-op. Interruption before +commit rolls back the repair; resume from a new plan with the original artifact, configuration, backup policy and +evidence directory to reuse its recovery point. Finish any incomplete maintenance operation before starting another. +OceanBase requires the same fixed-host coordination and previous-run acknowledgment as other maintenance. + +Inspection and repair scan the relevant Atomic history, heads, tags, lineage and current projection. Keyset batches +bound row buffering; this is an offline whole-projection operation. It does not register vector-profile changes or +optional Code configurations. The compatibility command `server atomic-memory-rebuild-projection` delegates managed +databases to this bundle and supports `--action plan/apply/verify` with the same confirmation and backup options. + ## Topic Memory recovery bundle Select `--bundle topic-memory-recovery` for the versioned `powercontext.topic-memory-recovery.v1` conversion. It recognizes diff --git a/docs/zh/docs/operate/atomic-memory-migration.md b/docs/zh/docs/operate/atomic-memory-migration.md index 1f44efb60c..6663f409af 100644 --- a/docs/zh/docs/operate/atomic-memory-migration.md +++ b/docs/zh/docs/operate/atomic-memory-migration.md @@ -212,6 +212,15 @@ RFC 1771 的结构版本。Atomic Memory 内容恢复接口不能替代数据库 ## 重建当前检索投影 +受管 Server 数据库上，该命令转发到 +[`db-migrate --bundle atomic-projection`](./database-migrations.md#atomic-memory-投影维护)。 +先用 `--action plan` 只读检查，再提供 `--plan-id`、显式 `--backup`、`--yes` 和 +`--maintenance-confirmed` 执行。已登记的无向量修复保留恢复点，在一个事务中提交完整修复， +返回共享 Server 结果。`--action verify` 检查 current 就绪状态；投影已就绪时只读返回。 +此路径不登记向量 profile 切换。 + +下文描述未受管数据库的领域独立修复： + 更换 embedding 模型、profile、维度或归一化方式，或修复当前检索数据时，先备份数据库，停止全部 API、宿主、Worker、自动重启和输入写入，然后使用目标部署配置执行： diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 6e76a91954..ecb0735141 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -61,6 +61,30 @@ powercontext server db-migrate verify --bundle processing --env-file .env \ 中断后重新查看计划，保留原配置、迁移 ID、备份策略和证据目录恢复；原维护窗口和恢复点继续沿用。 成功后的重试不会再次撤销租约。操作前后都运行完整 Server 校验；配置已匹配时只读返回。 +## Atomic Memory 投影维护 + +`--bundle atomic-projection` 在当前受管 Server 数据库上修复 Atomic Memory current 行和 SQLite FTS +辅助索引，适用于已登记的无向量、Code 关闭配置。它保留 Server revision、不可变正文与历史、生命周期、 +正式标签、权限和 Source/Processing 状态。 + +```bash +powercontext server db-migrate plan --bundle atomic-projection --env-file .env --backup auto +powercontext server db-migrate apply --bundle atomic-projection --env-file .env \ + --backup auto --plan-id PLAN_ID --maintenance-confirmed --yes +powercontext server db-migrate verify --bundle atomic-projection --env-file .env +``` + +该操作与 Processing 维护共用目标锁、备份、停写声明和持久恢复事务。plan 和 verify 只读； +apply 只修复派生行，完整 Server 校验通过后才提交。权威数据损坏或结构未登记时拒绝执行。 +投影已就绪则只读返回。提交前中断会回滚修复；使用新的计划、原软件产物、配置、备份策略和证据目录恢复， +继续沿用原恢复点。另一项维护未完成时，须先完成原操作。OceanBase 同样要求固定维护主机协调， +并在报告上次运行未结束时提供准确的确认 token。 + +检查和修复扫描相关 Atomic 历史、head、标签、lineage 及 current 投影，通过键集分页限制行缓冲； +这是停服执行的全投影操作，不登记向量 profile 切换或可选 Code 配置。 +兼容命令 `server atomic-memory-rebuild-projection` 在受管数据库上转发到此 bundle， +支持 `--action plan/apply/verify` 和同样的确认、备份选项。 + ## Topic Memory 恢复 bundle 选择 `--bundle topic-memory-recovery` 执行版本化的 `powercontext.topic-memory-recovery.v1` 转换。它识别恢复功能之前的 diff --git a/release/upgrade-support.json b/release/upgrade-support.json index b322ec8876..94e3b7df5f 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "aeb309fd603da7b66817332a0f86a5b49b277837d73ea1faf01a042ef6f37a1a", - "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback. New revision s1401 registers the frozen offline Memory archive on all three backends through the shared chain; s1300/s1400 resources and checksums remain unchanged, and archived history is preserved. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles. Managed Processing mode maintenance uses the canonical target lock, backup providers and durable evidence without changing pc_schema_revision or frozen migration resources. Its reviewed plan binds configuration and migration ID; one transaction publishes the mode and lease invalidation, and interrupted retries reuse the original recovery point. Compatibility commands delegate current managed Server storage and reject complete unversioned historical Server storage before domain writes; unmanaged domain behavior is retained. The Atomic compatibility migration delegates managed databases and recognized complete Server baselines to the shared full Server chain; legacy decision files remain limited to unmanaged conversion. Atomic projection repair still requires shared coordination before framework activation." + "persistence_sha256": "b9ce8d7ca4d19fa01ddeb78c1dbf3cf6f18bc5bb94e3973e31ba3f0fe2847d6b", + "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback. New revision s1401 registers the frozen offline Memory archive on all three backends through the shared chain; s1300/s1400 resources and checksums remain unchanged, and archived history is preserved. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles. Managed Processing mode maintenance uses the canonical target lock, backup providers and durable evidence without changing pc_schema_revision or frozen migration resources. Its reviewed plan binds configuration and migration ID; one transaction publishes the mode and lease invalidation, and interrupted retries reuse the original recovery point. The Processing compatibility command delegates current managed Server storage and rejects complete unversioned historical Server storage before domain writes; unmanaged domain behavior is retained. The Atomic compatibility migration delegates managed databases and recognized complete Server baselines to the shared full Server chain; legacy decision files remain limited to unmanaged conversion. Registered Atomic current repair now shares the Processing maintenance transaction, canonical target lock, reviewed backup and durable evidence; exact managed layout and authority validation precede repair, and full Server validation precedes commit. It preserves revision and immutable authority, repairs SQLite FTS helpers, and resumes from the original recovery point. Vector/Code profiles and complete native lifecycle acceptance remain incomplete." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/persistence/migrations/atomic_projection.py b/src/powercontext/builtin/persistence/migrations/atomic_projection.py new file mode 100644 index 0000000000..13108742dd --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/atomic_projection.py @@ -0,0 +1,164 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Registered current-row repair without changing managed schema or authority.""" + +from __future__ import annotations + +import hashlib +from datetime import datetime +from pathlib import Path + +from sqlalchemy import Connection, select, tuple_ +from typing_extensions import override + +from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex +from powercontext.builtin.persistence.tables import ( + ARTIFACT_HEADS_TABLE, + ARTIFACT_LINEAGE_ARTIFACTS_TABLE, + ARTIFACT_LINEAGE_SOURCES_TABLE, + ARTIFACT_TAGS_TABLE, + ARTIFACTS_TABLE, +) +from powercontext.builtin.runtime import atomic_memory_rebuild + +from .bundle import sqlite_inventory +from .connections import BackendIdentity +from .maintenance import ManagedMaintenanceRunner +from .models import MigrationError, digest +from .mysql_schema import mysql_inventory +from .pinned import finish, pinned +from .server_readiness import check_server_layout + + +def _sqlite_helper_digest(connection, batch_size): + snapshot = hashlib.sha256() + after = None + while True: + query = "SELECT rowid,* FROM pc_atomic_memory_current_fts" + parameters = {"batch": batch_size} + if after is not None: + query += " WHERE rowid > :after" + parameters["after"] = after + rows = connection.exec_driver_sql(query + " ORDER BY rowid LIMIT :batch", parameters).all() + if not rows: + return snapshot.hexdigest() + for row in rows: + snapshot.update(digest(tuple(row)).encode()) + after = rows[-1][0] + + +class AtomicProjectionMigrationRunner(ManagedMaintenanceRunner): + operation = "atomic_projection" + affected_objects: tuple[str, ...] = ("pc_atomic_memory_current", "pc_atomic_memory_current_fts") + + def __init__(self, *args, **kwargs): + super().__init__( + *args, + implementation_checksum=digest({ + "owner": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "rebuild": hashlib.sha256(Path(atomic_memory_rebuild.__file__).read_bytes()).hexdigest(), + }), + **kwargs, + ) + self.affected_objects = ( + ("pc_atomic_memory_current", "pc_atomic_memory_current_fts") + if isinstance(self.connections.config, SQLiteConfig) + else ("pc_atomic_memory_current",) + ) + + def _index(self, connection): + return SQLiteAtomicMemoryIndex() if connection.dialect.name == "sqlite" else OceanBaseAtomicMemoryIndex() + + @override + def _state(self, connection: Connection, identity: BackendIdentity) -> tuple[str, bool, bool]: + check_server_layout(connection) + index = self._index(connection) + try: + report = finish( + atomic_memory_rebuild.verify_atomic_memory_current( + pinned(connection), index, batch_size=self.batch_size + ) + ) + except Exception as error: + raise MigrationError("invalid_authority", "Repair requires valid Atomic Memory authority.") from error + required = not report.ready + if isinstance(index, SQLiteAtomicMemoryIndex): + required |= not finish(index.fts_matches_current(pinned(connection))) + try: + self.bundle.domains.validate( + connection, self.bundle.head, backend=identity.product, configuration=self.configuration + ) + except MigrationError as error: + # Only the current projection/head or helper mismatch can be + # admitted here. Full validation must pass before repair commits. + if error.code != "projection_verification_failed" or not required: + raise + fingerprint = digest({ + "schema": sqlite_inventory(connection) if identity.product == "sqlite" else mysql_inventory(connection), + "rows": self._rows_digest(connection, index.table), + }) + return fingerprint, required, not required + + def _rows_digest(self, connection, current): + snapshot = hashlib.sha256() + for table in ( + ARTIFACTS_TABLE, + ARTIFACT_HEADS_TABLE, + ARTIFACT_TAGS_TABLE, + ARTIFACT_LINEAGE_SOURCES_TABLE, + ARTIFACT_LINEAGE_ARTIFACTS_TABLE, + current, + ): + snapshot.update(table.name.encode()) + keys = tuple(table.primary_key.columns) + statement = select(table).order_by(*keys).limit(self.batch_size) + if "family" in table.c: + statement = statement.where(table.c.family == "atomic-memory") + after = None + while True: + query = statement if after is None else statement.where(tuple_(*keys) > after) + rows = connection.execute(query).mappings().all() + if not rows: + break + for row in rows: + snapshot.update( + digest([ + {"bytes": value.hex()} + if isinstance(value, bytes) + else value.isoformat() + if isinstance(value, datetime) + else value + for value in row.values() + ]).encode() + ) + after = tuple(rows[-1][key.name] for key in keys) + if connection.dialect.name == "sqlite": + snapshot.update(_sqlite_helper_digest(connection, self.batch_size).encode()) + return snapshot.hexdigest() + + @override + def _apply_operation(self, connection: Connection, guard) -> None: + index = self._index(connection) + report = finish( + atomic_memory_rebuild.rebuild_atomic_memory_current( + pinned(connection), index, batch_size=self.batch_size, guard=guard + ) + ) + if not report.ready: + raise MigrationError("projection_verification_failed", "Atomic current repair failed validation.") + if isinstance(index, SQLiteAtomicMemoryIndex): + finish(index.rebuild_fts(pinned(connection))) diff --git a/src/powercontext/builtin/persistence/migrations/deployment.py b/src/powercontext/builtin/persistence/migrations/deployment.py index 7aa7999bff..55a78c9c08 100644 --- a/src/powercontext/builtin/persistence/migrations/deployment.py +++ b/src/powercontext/builtin/persistence/migrations/deployment.py @@ -28,6 +28,7 @@ from powercontext.builtin.runtime.config import BuiltinConfig from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest +from .atomic_projection import AtomicProjectionMigrationRunner from .bundle import MigrationBundle from .connections import MaintenanceConnections from .models import MigrationError @@ -55,6 +56,7 @@ def production_bundle() -> MigrationBundle: | MySQLMigrationRunner | TopicMemoryRecoveryMigrationRunner | ProcessingConfigurationMigrationRunner + | AtomicProjectionMigrationRunner ) @@ -79,21 +81,24 @@ def deployment_runner( # noqa: C901 -- Ordered backend admission checks. evidence_dir: Path | None = None, lock_coordination: Literal["single-host"] | None = None, acknowledge_previous_run: str | None = None, - bundle: Literal["server", "artifacts", "topic-memory-recovery", "processing"] = "server", + bundle: Literal["server", "artifacts", "topic-memory-recovery", "processing", "atomic-projection"] = "server", migration_id: str = "rfc1515", batch_size: int = 100, ) -> MigrationRunner: """Use the real configured identity without opening a business profile.""" database = settings.database configuration = _migration_configuration(settings) - if bundle in {"server", "processing"}: + if bundle in {"server", "processing", "atomic-projection"}: if settings.inference.embedding_model is not None or settings.code.enabled: raise MigrationError( "unsupported_upgrade_configuration", "The registered 1.3.0 Server path covers the no-vector, Code-disabled configuration.", ) configuration["embedding_enabled"] = False - selected = server_bundle() if bundle in {"server", "processing"} else production_bundle() + selected = server_bundle() if bundle in {"server", "processing", "atomic-projection"} else production_bundle() + maintenance_runner = ( + AtomicProjectionMigrationRunner if bundle == "atomic-projection" else ProcessingConfigurationMigrationRunner + ) if bundle == "topic-memory-recovery": if not isinstance(database, (SQLiteConfig, SeekDBConfig, OceanBaseConfig)): raise MigrationError("unsupported_backend", "The database has no Topic recovery migration adapter.") @@ -125,8 +130,8 @@ def deployment_runner( # noqa: C901 -- Ordered backend admission checks. lock_coordination=lock_coordination, acknowledge_previous_run=acknowledge_previous_run, ) - if bundle == "processing": - return ProcessingConfigurationMigrationRunner( + if bundle in {"processing", "atomic-projection"}: + return maintenance_runner( connections, selected, configuration=configuration, @@ -149,9 +154,9 @@ def deployment_runner( # noqa: C901 -- Ordered backend admission checks. ) # Match SQLite's URL normalization before the runner expands standalone Path inputs. target = Path(os.path.abspath(url.database)) - if bundle == "processing": + if bundle in {"processing", "atomic-projection"}: directory = evidence_dir or target.with_name(target.name + ".pc-migration-state") - return ProcessingConfigurationMigrationRunner( + return maintenance_runner( MaintenanceConnections(database), selected, configuration=configuration, diff --git a/src/powercontext/builtin/persistence/migrations/maintenance.py b/src/powercontext/builtin/persistence/migrations/maintenance.py new file mode 100644 index 0000000000..d21b3d243c --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/maintenance.py @@ -0,0 +1,276 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared locked, backed-up transactions for registered Server maintenance.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Literal +from uuid import uuid4 + +from sqlalchemy import Connection +from sqlalchemy.engine import make_url + +from powercontext.builtin.persistence.sqlite import SQLiteConfig + +from .backup import BackupContext, ForkBackupProvider, SQLiteBackupProvider +from .bundle import MigrationBundle +from .connections import BackendIdentity, MaintenanceConnections +from .models import MigrationError, MigrationPlan, MigrationResult, digest +from .mysql import _Evidence, _Journal +from .server_readiness import check_server_schema +from .sqlite import SQLiteMigrationRunner + + +class ManagedMaintenanceRunner: + """Own the common transaction, recovery point and external maintenance journal.""" + + operation: str + affected_objects: tuple[str, ...] + + def __init__( + self, + connections: MaintenanceConnections, + bundle: MigrationBundle, + *, + configuration: dict[str, Any], + evidence_directory: Path, + implementation_checksum: str, + migration_id: str = "rfc1515", + batch_size: int = 100, + ) -> None: + self.connections = connections + self.bundle = bundle + self.configuration = json.loads(json.dumps(configuration)) + self.directory = evidence_directory.expanduser().resolve() + self.migration_id = migration_id + self.batch_size = batch_size + self.configuration_digest = digest({"configuration": configuration, "migration_id": migration_id}) + self.checksum = digest({ + "bundle": bundle.checksum, + "operation": self.operation, + "lifecycle": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "implementation": implementation_checksum, + }) + + def _journal(self, identity: BackendIdentity) -> _Journal: + return _Journal(self.directory / self.operation, identity) + + def _inspect(self, connection: Connection | None, identity: BackendIdentity, **options) -> MigrationPlan: + self.bundle.assert_unchanged() + if connection is None: + raise MigrationError("migration_required", "Initialize or upgrade the full Server bundle first.") + self._check_previous_maintenance(identity) + fingerprint, required, completed = self._state(connection, identity) + evidence = self._journal(identity).read() + active = evidence is not None and evidence.state != "complete" + if active and ( + evidence.bundle_checksum != self.checksum + or evidence.configuration_digest != self.configuration_digest + or evidence.source_revision != self.bundle.head + ): + raise MigrationError( + "recovery_required", "Resume with the original registered maintenance operation and configuration." + ) + policy = options.get("backup_policy", "auto") + if policy not in {"auto", "manual", "skip"}: + raise MigrationError("invalid_backup_policy", "Choose auto, manual or skip.") + if active and ( + evidence.backup_policy != policy + or (fingerprint != evidence.source_fingerprint and (required or not completed)) + ): + raise MigrationError( + "recovery_required", "Keep the original registered maintenance recovery point and backup policy." + ) + available = isinstance(self.connections.config, SQLiteConfig) + coordination = self.connections.coordination(identity) + recovering = active or "pending_run_id" in coordination + fields = { + "database_id": identity.database_id, + "source_revision": self.bundle.head, + "target_revision": self.bundle.head, + "schema_fingerprint": fingerprint, + "bundle_checksum": self.checksum, + "configuration_digest": self.configuration_digest, + "state": "recovery_required" if recovering else "migration_required" if required else "ready", + "revisions": (), + "domain_steps": (self.operation,) if required or active else (), + "domain_validators": ("server_upgrade",), + "backup_policy": policy, + "backup_available": available, + "backup_reason": () + if available + else ("Native Fork recovery acceptance is unavailable; choose manual or skip.",), + "affected_objects": self.affected_objects, + "shared_database": options.get("shared_database", False), + "service": options.get("service"), + "coordination": coordination, + "readiness_scope": "server", + } + return MigrationPlan.model_validate({"plan_id": digest(fields), **fields}) + + def plan(self, **options) -> MigrationPlan: + return self.connections.run(lambda connection, identity, guard: self._inspect(connection, identity, **options)) + + def verify(self) -> MigrationResult: + plan = self.plan() + if plan.state != "ready": + raise MigrationError(plan.state, "Apply the registered maintenance operation first.") + return MigrationResult(revision=self.bundle.head, changed=False, readiness_scope="server") + + def _provider(self, connection: Connection, identity: BackendIdentity): + config = self.connections.config + if isinstance(config, SQLiteConfig): + return SQLiteBackupProvider(Path(str(make_url(config.url).database))) + return ForkBackupProvider( + connection, + database_name=identity.database_name, + expected_product="seekdb" if identity.product == "seekdb" else "oceanbase_ai", + directory=self.directory, + ) + + def apply( # noqa: C901 -- One transaction owns backup, domain maintenance and recovery evidence. + self, + *, + plan_id: str | None = None, + accepted: bool = False, + maintenance_confirmed: bool = False, + backup_policy: Literal["auto", "manual", "skip"] = "auto", + backup_confirmed: bool = False, + accept_no_backup: bool = False, + backup_ref: str | None = None, + **options, + ) -> MigrationResult: + initial = self.plan(backup_policy=backup_policy, **options) + pending = (initial.coordination or {}).get("pending_run_id") + acknowledged = self.connections.acknowledge_previous_run + if pending is not None and pending != acknowledged: + raise MigrationError("recovery_required", "Acknowledge the exact previous remote maintenance run.") + if pending is None and acknowledged is not None: + raise MigrationError("stale_ack", "There is no pending run matching this acknowledgment.") + if initial.state == "ready": + return self.verify() + if options.get("service") is not None: + raise MigrationError("service_unsupported", "Use full Server maintenance to manage a local service.") + SQLiteMigrationRunner._require_acceptance( + initial, plan_id, accepted, maintenance_confirmed, backup_confirmed, accept_no_backup + ) + + def execute(connection, identity, guard): + current = self._inspect(connection, identity, backup_policy=backup_policy, **options) + if current != initial: + raise MigrationError("plan_changed", "Maintenance state changed while acquiring the shared lock.") + if not current.domain_steps: + guard() + return MigrationResult(revision=self.bundle.head, changed=False, readiness_scope="server") + journal = self._journal(identity) + evidence = journal.read() + if evidence is None or evidence.state == "complete": + evidence = _Evidence( + state="preparing", + database_id=identity.database_id, + source_revision=self.bundle.head, + source_fingerprint=initial.schema_fingerprint, + bundle_checksum=self.checksum, + configuration_digest=self.configuration_digest, + last_revision=self.bundle.head, + maintenance_window_id=uuid4().hex, + backup_policy=backup_policy, + backup_state="pending", + ) + journal.write(evidence) + context = BackupContext( + database_id=identity.database_id, + source_revision=self.bundle.head, + bundle_checksum=self.checksum, + maintenance_window_id=evidence.maintenance_window_id, + objects=self.bundle.affected_objects, + writers_stopped=True, + ) + provider = self._provider(connection, identity) + # Backup must see committed state. The shared OS/engine lock remains held. + connection.commit() + if evidence.state == "preparing": + if backup_policy == "auto": + evidence.backup = provider.create_backup(context) + evidence.backup_state = "completed" + else: + evidence.backup_state = "user_confirmed" if backup_policy == "manual" else "skipped" + evidence.manual_ref = backup_ref + evidence.state = "active" + journal.write(evidence) + if evidence.backup is not None and provider.inspect_backup(evidence.backup).state != "completed": + raise MigrationError( + "backup_failed", "The original registered maintenance recovery point is unavailable." + ) + guard() + connection.exec_driver_sql( + "BEGIN IMMEDIATE" if connection.dialect.name == "sqlite" else "START TRANSACTION" + ) + locked = self._inspect(connection, identity, backup_policy=backup_policy, **options) + if locked.schema_fingerprint != initial.schema_fingerprint: + raise MigrationError("plan_changed", "Maintenance state changed before the write transaction.") + self._apply_operation(connection, guard) + guard() + check_server_schema(connection) + self.bundle.domains.validate( + connection, self.bundle.head, backend=identity.product, configuration=self.configuration + ) + connection.commit() + evidence.state = "complete" + journal.write(evidence) + backup_state = evidence.backup_state + if backup_state == "pending": + raise MigrationError("backup_failed", "The registered maintenance recovery point has not completed.") + return MigrationResult( + revision=self.bundle.head, + changed=True, + readiness_scope="server", + backup_state=backup_state, + backup_ref=evidence.backup.location if evidence.backup else evidence.manual_ref, + ) + + return self.connections.run(execute, writable=True) + + def _state(self, connection: Connection, identity: BackendIdentity) -> tuple[str, bool, bool]: + raise NotImplementedError + + def _apply_operation(self, connection: Connection, guard) -> None: + raise NotImplementedError + + def _check_previous_maintenance(self, identity: BackendIdentity) -> None: + for operation in ("processing_configuration", "atomic_projection"): + if operation != self.operation: + previous = _Journal(self.directory / operation, identity).read() + if previous is not None and previous.state != "complete": + raise MigrationError( + "recovery_required", "Complete the previous registered maintenance operation first." + ) + main_path = ( + self.directory / "maintenance.json" + if isinstance(self.connections.config, SQLiteConfig) + else self.directory / identity.database_id / "maintenance.json" + ) + if main_path.exists(): + try: + main = json.loads(main_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise MigrationError( + "recovery_required", "The original full Server evidence cannot be read." + ) from error + if not isinstance(main, dict) or main.get("state") != "complete": + raise MigrationError("recovery_required", "Complete the original full Server maintenance first.") diff --git a/src/powercontext/builtin/persistence/migrations/processing_configuration.py b/src/powercontext/builtin/persistence/migrations/processing_configuration.py index e3161e2459..ff233a318c 100644 --- a/src/powercontext/builtin/persistence/migrations/processing_configuration.py +++ b/src/powercontext/builtin/persistence/migrations/processing_configuration.py @@ -12,110 +12,62 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Registered offline Processing configuration changes on current Server storage. - -This operation changes scheduling ownership, not the Alembic revision. Every -Processing phase joins one transaction: a failed mode switch cannot leave a -partially published configuration or an independently committed lease fence. -""" +"""Registered offline Processing configuration changes on current Server storage.""" from __future__ import annotations import hashlib -import json from pathlib import Path -from typing import Any, Literal -from uuid import uuid4 from sqlalchemy import Connection, text -from sqlalchemy.engine import make_url +from typing_extensions import override from powercontext.builtin.persistence import processing_migration from powercontext.builtin.persistence.processing_migration import ProcessingSchemaNotReadyError -from powercontext.builtin.persistence.sqlite import SQLiteConfig -from .backup import BackupContext, ForkBackupProvider, SQLiteBackupProvider -from .bundle import MigrationBundle, sqlite_inventory -from .connections import BackendIdentity, MaintenanceConnections -from .models import MigrationError, MigrationPlan, MigrationResult, digest -from .mysql import _Evidence, _Journal +from .bundle import sqlite_inventory +from .connections import BackendIdentity +from .maintenance import ManagedMaintenanceRunner +from .models import MigrationError, digest from .mysql_schema import mysql_inventory from .pinned import finish, pinned from .server_readiness import check_server_schema -from .sqlite import SQLiteMigrationRunner -OPERATION = "processing_configuration" -_OBJECTS = ( - "pc_artifact_processing_schema", - "pc_artifact_processing_leases", - "pc_artifact_processing_auto_wave_targets", -) +class ProcessingConfigurationMigrationRunner(ManagedMaintenanceRunner): + """Publish mode changes and lease fencing in the same shared transaction.""" -def _marker(connection: Connection) -> dict[str, Any]: - return dict( - connection - .execute( - text( - "SELECT schema_version,source_version,phase,migration_id,config_manifest " - "FROM pc_artifact_processing_schema WHERE singleton=1" - ) - ) - .mappings() - .one() + operation = "processing_configuration" + affected_objects = ( + "pc_artifact_processing_schema", + "pc_artifact_processing_leases", + "pc_artifact_processing_auto_wave_targets", ) + def __init__(self, *args, **kwargs): + super().__init__( + *args, + implementation_checksum=digest({ + "owner": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "processing": hashlib.sha256(Path(processing_migration.__file__).read_bytes()).hexdigest(), + }), + **kwargs, + ) -class ProcessingConfigurationMigrationRunner: - """Share target locks, declarations, backup providers and durable evidence.""" - - def __init__( - self, - connections: MaintenanceConnections, - bundle: MigrationBundle, - *, - configuration: dict[str, Any], - evidence_directory: Path, - migration_id: str = "rfc1515", - batch_size: int = 100, - ) -> None: - self.connections = connections - self.bundle = bundle - self.configuration = json.loads(json.dumps(configuration)) - self.directory = evidence_directory.expanduser().resolve() - self.migration_id = migration_id - self.batch_size = batch_size - self.configuration_digest = digest({"configuration": configuration, "migration_id": migration_id}) - self.checksum = digest({ - "bundle": bundle.checksum, - "operation": OPERATION, - "implementation": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), - "processing": hashlib.sha256(Path(processing_migration.__file__).read_bytes()).hexdigest(), - }) - - def _journal(self, identity: BackendIdentity) -> _Journal: - return _Journal(self.directory / OPERATION, identity) - - def _inspect(self, connection: Connection | None, identity: BackendIdentity, **options) -> MigrationPlan: - self.bundle.assert_unchanged() - if connection is None: - raise MigrationError("migration_required", "Initialize or upgrade the full Server bundle first.") + @override + def _state(self, connection: Connection, identity: BackendIdentity) -> tuple[str, bool, bool]: check_server_schema(connection) - marker = _marker(connection) - fingerprint = digest({ - "schema": sqlite_inventory(connection) if identity.product == "sqlite" else mysql_inventory(connection), - "processing": marker, - }) - evidence = self._journal(identity).read() - active = evidence is not None and evidence.state != "complete" - if active and ( - evidence.bundle_checksum != self.checksum - or evidence.configuration_digest != self.configuration_digest - or evidence.source_revision != self.bundle.head - ): - raise MigrationError( - "recovery_required", "Resume with the original Processing operation and configuration." + marker = dict( + connection + .execute( + text( + "SELECT schema_version,source_version,phase,migration_id,config_manifest " + "FROM pc_artifact_processing_schema WHERE singleton=1" + ) ) + .mappings() + .one() + ) required = finish( processing_migration.plan_processing_migration( pinned(connection), config_manifest=self.configuration["processing_manifest"] @@ -125,185 +77,23 @@ def _inspect(self, connection: Connection | None, identity: BackendIdentity, **o raise MigrationError( "migration_id_reused", "A Processing configuration switch requires a new migration ID." ) - policy = options.get("backup_policy", "auto") - if policy not in {"auto", "manual", "skip"}: - raise MigrationError("invalid_backup_policy", "Choose auto, manual or skip.") - if active and ( - evidence.backup_policy != policy - or ( - fingerprint != evidence.source_fingerprint and (required or marker["migration_id"] != self.migration_id) - ) - ): - raise MigrationError("recovery_required", "Keep the original Processing recovery point and backup policy.") - main_path = ( - self.directory / "maintenance.json" - if isinstance(self.connections.config, SQLiteConfig) - else self.directory / identity.database_id / "maintenance.json" - ) - if main_path.exists(): - try: - main = json.loads(main_path.read_text(encoding="utf-8")) - except (OSError, ValueError) as error: - raise MigrationError( - "recovery_required", "The original full Server evidence cannot be read." - ) from error - if not isinstance(main, dict) or main.get("state") != "complete": - raise MigrationError("recovery_required", "Complete the original full Server maintenance first.") - available = isinstance(self.connections.config, SQLiteConfig) - coordination = self.connections.coordination(identity) - recovering = active or "pending_run_id" in coordination - fields = { - "database_id": identity.database_id, - "source_revision": self.bundle.head, - "target_revision": self.bundle.head, - "schema_fingerprint": fingerprint, - "bundle_checksum": self.checksum, - "configuration_digest": self.configuration_digest, - "state": "recovery_required" if recovering else "migration_required" if required else "ready", - "revisions": (), - "domain_steps": (OPERATION,) if required or active else (), - "domain_validators": ("server_upgrade",), - "backup_policy": policy, - "backup_available": available, - "backup_reason": () - if available - else ("Native Fork recovery acceptance is unavailable; choose manual or skip.",), - "affected_objects": _OBJECTS, - "shared_database": options.get("shared_database", False), - "service": options.get("service"), - "coordination": coordination, - "readiness_scope": "server", - } - return MigrationPlan.model_validate({"plan_id": digest(fields), **fields}) - - def plan(self, **options) -> MigrationPlan: - return self.connections.run(lambda connection, identity, guard: self._inspect(connection, identity, **options)) - - def verify(self) -> MigrationResult: - plan = self.plan() - if plan.state != "ready": - raise MigrationError(plan.state, "Apply the registered Processing configuration operation first.") - return MigrationResult(revision=self.bundle.head, changed=False, readiness_scope="server") - - def _provider(self, connection: Connection, identity: BackendIdentity): - config = self.connections.config - if isinstance(config, SQLiteConfig): - return SQLiteBackupProvider(Path(str(make_url(config.url).database))) - return ForkBackupProvider( - connection, - database_name=identity.database_name, - expected_product="seekdb" if identity.product == "seekdb" else "oceanbase_ai", - directory=self.directory, - ) - - def apply( # noqa: C901 -- One transaction owns backup, configuration transition and recovery evidence. - self, - *, - plan_id: str | None = None, - accepted: bool = False, - maintenance_confirmed: bool = False, - backup_policy: Literal["auto", "manual", "skip"] = "auto", - backup_confirmed: bool = False, - accept_no_backup: bool = False, - backup_ref: str | None = None, - **options, - ) -> MigrationResult: - initial = self.plan(backup_policy=backup_policy, **options) - pending = (initial.coordination or {}).get("pending_run_id") - acknowledged = self.connections.acknowledge_previous_run - if pending is not None and pending != acknowledged: - raise MigrationError("recovery_required", "Acknowledge the exact previous remote maintenance run.") - if pending is None and acknowledged is not None: - raise MigrationError("stale_ack", "There is no pending run matching this acknowledgment.") - if initial.state == "ready": - return self.verify() - if options.get("service") is not None: - raise MigrationError("service_unsupported", "Use full Server maintenance to manage a local service.") - SQLiteMigrationRunner._require_acceptance( - initial, plan_id, accepted, maintenance_confirmed, backup_confirmed, accept_no_backup - ) - - def execute(connection, identity, guard): # noqa: C901 -- Keep the locked transaction and its recovery evidence together. - current = self._inspect(connection, identity, backup_policy=backup_policy, **options) - if current != initial: - raise MigrationError("plan_changed", "Processing state changed while acquiring the shared lock.") - if not current.domain_steps: - guard() - return MigrationResult(revision=self.bundle.head, changed=False, readiness_scope="server") - journal = self._journal(identity) - evidence = journal.read() - if evidence is None or evidence.state == "complete": - evidence = _Evidence( - state="preparing", - database_id=identity.database_id, - source_revision=self.bundle.head, - source_fingerprint=initial.schema_fingerprint, - bundle_checksum=self.checksum, - configuration_digest=self.configuration_digest, - last_revision=self.bundle.head, - maintenance_window_id=uuid4().hex, - backup_policy=backup_policy, - backup_state="pending", + fingerprint = digest({ + "schema": sqlite_inventory(connection) if identity.product == "sqlite" else mysql_inventory(connection), + "processing": marker, + }) + return fingerprint, required, marker["migration_id"] == self.migration_id + + @override + def _apply_operation(self, connection: Connection, guard) -> None: + try: + while not finish( + processing_migration.apply_processing_migration( + pinned(connection), + config_manifest=self.configuration["processing_manifest"], + migration_id=self.migration_id, + batch_size=self.batch_size, ) - journal.write(evidence) - context = BackupContext( - database_id=identity.database_id, - source_revision=self.bundle.head, - bundle_checksum=self.checksum, - maintenance_window_id=evidence.maintenance_window_id, - objects=self.bundle.affected_objects, - writers_stopped=True, - ) - provider = self._provider(connection, identity) - # Backup must see committed state. The shared OS/engine lock remains held. - connection.commit() - if evidence.state == "preparing": - if backup_policy == "auto": - evidence.backup = provider.create_backup(context) - evidence.backup_state = "completed" - else: - evidence.backup_state = "user_confirmed" if backup_policy == "manual" else "skipped" - evidence.manual_ref = backup_ref - evidence.state = "active" - journal.write(evidence) - if evidence.backup is not None and provider.inspect_backup(evidence.backup).state != "completed": - raise MigrationError("backup_failed", "The original Processing recovery point is unavailable.") - guard() - connection.exec_driver_sql( - "BEGIN IMMEDIATE" if connection.dialect.name == "sqlite" else "START TRANSACTION" - ) - locked = self._inspect(connection, identity, backup_policy=backup_policy, **options) - if locked.schema_fingerprint != initial.schema_fingerprint: - raise MigrationError("plan_changed", "Processing state changed before the write transaction.") - try: - while not finish( - processing_migration.apply_processing_migration( - pinned(connection), - config_manifest=self.configuration["processing_manifest"], - migration_id=self.migration_id, - batch_size=self.batch_size, - ) - ).complete: - guard() - except ProcessingSchemaNotReadyError as error: - raise MigrationError("incompatible_processing_configuration", str(error)) from error - guard() - check_server_schema(connection) - self.bundle.domains.validate( - connection, self.bundle.head, backend=identity.product, configuration=self.configuration - ) - connection.commit() - evidence.state = "complete" - journal.write(evidence) - backup_state = evidence.backup_state - if backup_state == "pending": - raise MigrationError("backup_failed", "The Processing recovery point has not completed.") - return MigrationResult( - revision=self.bundle.head, - changed=True, - readiness_scope="server", - backup_state=backup_state, - backup_ref=evidence.backup.location if evidence.backup else evidence.manual_ref, - ) - - return self.connections.run(execute, writable=True) + ).complete: + guard() + except ProcessingSchemaNotReadyError as error: + raise MigrationError("incompatible_processing_configuration", str(error)) from error diff --git a/src/powercontext/builtin/persistence/migrations/server_readiness.py b/src/powercontext/builtin/persistence/migrations/server_readiness.py index b7c778d0fe..0b9d4e7424 100644 --- a/src/powercontext/builtin/persistence/migrations/server_readiness.py +++ b/src/powercontext/builtin/persistence/migrations/server_readiness.py @@ -21,6 +21,13 @@ def check_server_schema(connection): + bundle = check_server_layout(connection) + backend = "sqlite" if connection.dialect.name == "sqlite" else "seekdb" + bundle.domains.validate(connection, bundle.head, backend=backend, configuration={}) + + +def check_server_layout(connection): + """Validate the exact managed layout before a registered domain repair.""" from .deployment import server_bundle from .mysql_schema import MySQLSchema, mysql_inventory @@ -40,12 +47,10 @@ def check_server_schema(connection): ) if connection.exec_driver_sql("PRAGMA foreign_key_check").first(): raise MigrationError("verification_failed", "A live foreign key is invalid.") - backend = "sqlite" else: schema = MySQLSchema(bundle) schema.verify(mysql_inventory(connection), bundle.head) - backend = "seekdb" - bundle.domains.validate(connection, bundle.head, backend=backend, configuration={}) + return bundle def prepare_server_storage(settings, *, injected_embedding=False): diff --git a/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py b/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py index f11d9298e0..da9007df74 100644 --- a/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py @@ -281,6 +281,17 @@ async def rebuild_fts(self, connection: AsyncConnection, /) -> None: f"INSERT INTO {_FTS_TABLE} ({names}) SELECT {values} FROM pc_atomic_memory_current" # noqa: S608 ) + async def fts_matches_current(self, connection: AsyncConnection, /) -> bool: + """Compare all helper values and multiplicities, including stable identity tokens.""" + names = ", ".join(_FTS_COLUMNS) + values = ", ".join(_FTS_IDENTITY_SQL if name == "identity_token" else name for name in _FTS_COLUMNS) + expected = f"SELECT {values}, COUNT(*) FROM pc_atomic_memory_current GROUP BY {values}" # noqa: S608 + actual = f"SELECT {names}, COUNT(*) FROM {_FTS_TABLE} GROUP BY {names}" # noqa: S608 + for left, right in ((expected, actual), (actual, expected)): + if (await connection.exec_driver_sql(f"SELECT 1 FROM ({left} EXCEPT {right}) LIMIT 1")).first(): # noqa: S608 + return False + return True + async def search( self, connection: AsyncConnection, scope_id: str, request: AtomicMemorySearchRequest, / ) -> AtomicMemorySearchChannels: diff --git a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py index 716480b6e3..a02c6d36e8 100644 --- a/src/powercontext/builtin/runtime/atomic_memory_rebuild.py +++ b/src/powercontext/builtin/runtime/atomic_memory_rebuild.py @@ -17,6 +17,7 @@ from __future__ import annotations import json +from collections.abc import Callable from hashlib import sha256 from time import perf_counter @@ -236,6 +237,37 @@ async def _initialize_rebuild(connection: AsyncConnection, index: AtomicMemoryIn await index.reconfigure_vector_column(connection) +async def rebuild_atomic_memory_current( + connection: AsyncConnection, + index: AtomicMemoryIndex, + *, + batch_size: int = 100, + guard: Callable[[], None] = lambda: None, +) -> AtomicMemoryProjectionRebuildReport: + """Repair registered no-vector rows inside the maintenance owner's transaction. + + The caller validates the existing schema, owns backup/recovery and commits + only after full Server validation. No initialization or external I/O occurs. + """ + if index.capabilities.embedding_profile is not None: + raise AtomicMemoryIndexError("embedding-profile", "Registered repair requires a no-vector profile") + await _require_authority(connection) + artifacts = ArtifactRepository((AtomicMemory,)) + publisher = AtomicMemoryProjectionPublisher(index) + after = None + while batch := await _identity_batch(connection, after, batch_size): + guard() + for scope_id, artifact_id in batch: + record = await _record(connection, artifacts, scope_id, artifact_id) + if record.state.state is AtomicMemoryStateValue.ACTIVE: + await publisher.publish(connection, scope_id, record, await publisher.prepare(record.artifact.content)) + else: + await publisher.remove(connection, scope_id, artifact_id) + after = scope_id, artifact_id + await connection.execute(index.table.delete().where(_obsolete_current(index))) + return await verify_atomic_memory_current(connection, index, batch_size=batch_size) + + async def rebuild_atomic_memory_projection( database: AsyncDatabase, index: AtomicMemoryIndex, @@ -325,4 +357,9 @@ async def rebuild_atomic_memory_projection( ) -__all__ = ["AtomicMemoryProjectionRebuildReport", "rebuild_atomic_memory_projection", "verify_atomic_memory_current"] +__all__ = [ + "AtomicMemoryProjectionRebuildReport", + "rebuild_atomic_memory_current", + "rebuild_atomic_memory_projection", + "verify_atomic_memory_current", +] diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index ec99e836c9..036675259b 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -436,13 +436,69 @@ def atomic_memory_migrate( @app.command("atomic-memory-rebuild-projection") def atomic_memory_rebuild_projection( env_file: Annotated[Path | None, typer.Option(help="Load the deployment's settings.")] = None, + action: Annotated[ + Literal["plan", "apply", "verify"], typer.Option(help="Managed projection maintenance action.") + ] = "apply", maintenance_confirmed: Annotated[ bool, typer.Option(help="Confirm every API, host and Worker is stopped and input writes are paused.") ] = False, batch_size: Annotated[int, typer.Option(min=1, max=1000, help="Identities read per maintenance batch.")] = 100, + evidence_dir: database_migration.EvidenceDir = None, + lock_coordination: database_migration.LockCoordination = None, + backup: Annotated[ + database_migration.BackupPolicy | None, typer.Option(help="Managed storage backup policy.") + ] = None, + plan_id: Annotated[str | None, typer.Option(help="Exact managed projection plan to accept.")] = None, + yes: Annotated[bool, typer.Option("--yes", help="Accept the managed projection plan.")] = False, + backup_confirmed: Annotated[bool, typer.Option(help="Declare a manual backup exists.")] = False, + accept_no_backup: Annotated[bool, typer.Option(help="Accept the no-backup risk.")] = False, + backup_ref: Annotated[str | None, typer.Option(help="Operator manual backup reference.")] = None, + acknowledge_previous_run: Annotated[ + str | None, typer.Option(help="OceanBase token confirming the previous remote maintenance ended.") + ] = None, ) -> None: """Rebuild active Atomic Memory search rows from existing authority.""" + env_file = resolve_server_environment_file(env_file, discover=True) + with database_migration._operator_errors(), server_settings_context(env_file=env_file) as settings: + managed = _managed_storage(settings.database) + if managed: + if action == "plan": + database_migration.plan( + env_file=env_file, + evidence_dir=evidence_dir, + lock_coordination=lock_coordination, + backup=backup or "auto", + bundle="atomic-projection", + batch_size=batch_size, + ) + elif action == "verify": + database_migration.verify( + env_file=env_file, + evidence_dir=evidence_dir, + lock_coordination=lock_coordination, + bundle="atomic-projection", + batch_size=batch_size, + ) + else: + database_migration.apply( + env_file=env_file, + evidence_dir=evidence_dir, + lock_coordination=lock_coordination, + bundle="atomic-projection", + batch_size=batch_size, + backup=backup, + plan_id=plan_id, + yes=yes, + maintenance_confirmed=maintenance_confirmed, + backup_confirmed=backup_confirmed, + accept_no_backup=accept_no_backup, + backup_ref=backup_ref, + acknowledge_previous_run=acknowledge_previous_run, + ) + return + if action != "apply": + raise typer.BadParameter("plan and verify require managed Server storage") # noqa: TRY003 if not maintenance_confirmed: raise typer.BadParameter("projection rebuild requires --maintenance-confirmed after stopping every writer") # noqa: TRY003 with server_settings_context(env_file=env_file) as settings: diff --git a/src/powercontext/server/database_migration.py b/src/powercontext/server/database_migration.py index 295b0f2a80..3242833b3a 100644 --- a/src/powercontext/server/database_migration.py +++ b/src/powercontext/server/database_migration.py @@ -40,7 +40,7 @@ BackupPolicy = Literal["auto", "manual", "skip"] Bundle = Annotated[ - Literal["server", "artifacts", "topic-memory-recovery", "processing"], + Literal["server", "artifacts", "topic-memory-recovery", "processing", "atomic-projection"], typer.Option(help="Select the registered domain bundle; readiness covers only that bundle."), ] EnvFile = Annotated[Path | None, typer.Option(help="Read the same environment file as the deployed Server.")] @@ -245,7 +245,7 @@ def _inspect( evidence_dir: Path | None, lock_coordination: Literal["single-host"] | None, verify: bool, - bundle: Literal["server", "artifacts", "topic-memory-recovery", "processing"] = "server", + bundle: Literal["server", "artifacts", "topic-memory-recovery", "processing", "atomic-projection"] = "server", migration_id: str = "rfc1515", batch_size: int = 100, ) -> None: diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index 40be1e215d..9b71ced493 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -36,6 +36,7 @@ from sqlalchemy.pool import NullPool from typer.testing import CliRunner +from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemoryContent from powercontext.builtin.persistence.migrations import mysql as mysql_migrations from powercontext.builtin.persistence.migrations.backup import BackupCapabilities, BackupProvider from powercontext.builtin.persistence.migrations.connections import BackendIdentity, MaintenanceConnections @@ -455,6 +456,7 @@ def test_mysql_atomic_compatibility_command_uses_shared_server_upgrade( "--lock-coordination", "single-host", ] + def invoke(options): # Pytest live SQL logging suspends capture during emit. Keep its # stream switching outside Typer's independent stdout isolation. @@ -500,6 +502,71 @@ def inspect_history(connection, identity, guard): assert backend.connections().run(inspect_history) == ("s1401", b'{"history":"unchanged"}') +def test_mysql_registered_projection_repair_preserves_authority_and_revision(backend: _BackendTarget) -> None: + from powercontext.builtin.persistence.migrations.atomic_projection import AtomicProjectionMigrationRunner + + full = _full_server_runner(backend) + plan = full.plan() + full.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + content = ( + AtomicMemoryContent(kind="fact", text="Preserve immutable nebula evidence.") + .model_dump_json(by_alias=True) + .encode() + ) + + def seed(connection, identity, guard): + connection.exec_driver_sql( + "INSERT INTO pc_scopes (scope_id,title,summary,scope_id_search,title_search,summary_search,version) " + "VALUES ('scope','title','summary','scope','title','summary',1)" + ) + connection.execute( + text( + "INSERT INTO pc_artifacts (scope_id,family,artifact_id,revision,content) " + "VALUES ('scope','atomic-memory','fact',1,:content)" + ), + {"content": content}, + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_heads (scope_id,family,artifact_id,revision) " + "VALUES ('scope','atomic-memory','fact',1)" + ) + connection.commit() + + backend.connections().run(seed, writable=True) + runner = AtomicProjectionMigrationRunner( + backend.connections(), + full.bundle, + configuration=full.configuration, + evidence_directory=backend.evidence_directory, + batch_size=1, + ) + plan = runner.plan(backup_policy="manual") + assert plan.state == "migration_required" + result = runner.apply( + plan_id=plan.plan_id, + accepted=True, + maintenance_confirmed=True, + backup_policy="manual", + backup_confirmed=True, + backup_ref="operator://projection-recovery", + ) + assert result.backup_state == "user_confirmed" and result.backup_ref == "operator://projection-recovery" + assert runner.verify().state == "ready" + assert full.verify().state == "ready" + assert not runner.apply().changed + + def inspect_content(connection, identity, guard): + return ( + connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar_one(), + connection.exec_driver_sql("SELECT content FROM pc_artifacts WHERE artifact_id='fact'").scalar_one(), + connection.exec_driver_sql( + "SELECT text FROM pc_atomic_memory_current WHERE artifact_id='fact'" + ).scalar_one(), + ) + + assert backend.connections().run(inspect_content) == ("s1401", content, "Preserve immutable nebula evidence.") + + def _load_full_server_baseline(backend: _BackendTarget) -> None: runner = _full_server_runner(backend) diff --git a/tests/test_managed_atomic_projection.py b/tests/test_managed_atomic_projection.py new file mode 100644 index 0000000000..bbee60a965 --- /dev/null +++ b/tests/test_managed_atomic_projection.py @@ -0,0 +1,225 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Managed projection repair must preserve authority and shared maintenance ownership.""" + +import asyncio +import json +import os +import sqlite3 +import subprocess +import sys +from contextlib import closing +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemoryContent +from powercontext.builtin.persistence.migrations.deployment import deployment_runner +from powercontext.builtin.persistence.migrations.locking import local_migration_lock +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts +from powercontext.cli.app import create_cli +from powercontext.server.cli import app +from powercontext.server.settings import ServerSettings + + +@pytest.fixture +def damaged(tmp_path, monkeypatch): + path = tmp_path / "managed.db" + settings = ServerSettings(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{path}")) + runner = deployment_runner(settings) + plan = runner.plan() + runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + with closing(sqlite3.connect(path)) as connection, connection: + connection.execute( + "INSERT INTO pc_scopes (scope_id,title,summary,scope_id_search,title_search,summary_search,version) " + "VALUES ('scope','title','summary','scope','title','summary',1)" + ) + connection.execute( + "INSERT INTO pc_artifacts (scope_id,family,artifact_id,revision,content) " + "VALUES ('scope','atomic-memory','fact',1,?)", + ( + AtomicMemoryContent(kind="fact", text="Preserve immutable nebula evidence.") + .model_dump_json(by_alias=True) + .encode(), + ), + ) + connection.execute( + "INSERT INTO pc_artifact_heads (scope_id,family,artifact_id,revision) " + "VALUES ('scope','atomic-memory','fact',1)" + ) + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_KIND", "sqlite") + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_URL", f"sqlite+aiosqlite:///{path}") + return path, settings + + +def authority(path): + with closing(sqlite3.connect(path)) as connection: + return tuple( + (table, tuple(connection.execute(f"SELECT * FROM {table} ORDER BY 1,2,3").fetchall())) # noqa: S608 -- Fixed fixture tables. + for table in ("pc_artifacts", "pc_artifact_heads", "pc_artifact_tags", "pc_access_owners") + ) + + +@pytest.mark.parametrize("damage", ["missing_current", "helper_only", "helper_identity"]) +def test_managed_repair_uses_shared_plan_backup_and_lock_without_changing_authority(damaged, damage): + path, settings = damaged + cli = create_cli([app]) + arguments = ["server", "atomic-memory-rebuild-projection"] + if damage != "missing_current": + initial = CliRunner().invoke(cli, [*arguments, "--action", "plan"]) + assert initial.exit_code == 0, initial.output + first_plan = json.loads(initial.output) + repaired = CliRunner().invoke( + cli, + [ + *arguments, + "--maintenance-confirmed", + "--plan-id", + first_plan["plan_id"], + "--backup", + "auto", + "--yes", + ], + ) + assert repaired.exit_code == 0, repaired.output + with closing(sqlite3.connect(path)) as connection, connection: + if damage == "helper_only": + connection.execute("DELETE FROM pc_atomic_memory_current_fts") + else: + connection.execute("UPDATE pc_atomic_memory_current_fts SET identity_token='broken'") + original_authority = authority(path) + original = path.read_bytes() + inspected = CliRunner().invoke(cli, [*arguments, "--action", "plan"]) + assert inspected.exit_code == 0, inspected.output + plan = json.loads(inspected.output) + assert plan["state"] == "migration_required" and plan["readiness_scope"] == "server" + assert path.read_bytes() == original + denied = CliRunner().invoke(cli, [*arguments, "--maintenance-confirmed"]) + assert denied.exit_code != 0 and path.read_bytes() == original + accepted = [ + *arguments, + "--maintenance-confirmed", + "--plan-id", + plan["plan_id"], + "--backup", + "auto", + "--yes", + ] + with local_migration_lock(path): + locked = CliRunner().invoke(cli, accepted) + assert locked.exit_code != 0 and "migration_locked" in locked.output + assert path.read_bytes() == original + repaired = CliRunner().invoke(cli, accepted) + assert repaired.exit_code == 0, repaired.output + result = json.loads(repaired.output) + assert result["backup_state"] == "completed" and Path(result["backup_ref"]).is_file() + assert authority(Path(result["backup_ref"])) == original_authority == authority(path) + assert deployment_runner(settings).verify().state == "ready" + verified = CliRunner().invoke(cli, [*arguments, "--action", "verify"]) + assert verified.exit_code == 0, verified.output + after = path.read_bytes() + repeated = CliRunner().invoke(cli, [*arguments, "--maintenance-confirmed"]) + assert repeated.exit_code == 0, repeated.output + assert json.loads(repeated.output)["changed"] is False and path.read_bytes() == after + + async def search(): + async with open_builtin_contexts(BuiltinConfig(database=settings.database)) as contexts: + result = await contexts.atomic_memory.for_scope("scope").search("nebula", mode="text") + assert [hit.hit.artifact_ref.artifact_id for hit in result.hits] == ["fact"] + + asyncio.run(search()) + + +def test_managed_repair_rejects_broken_authority_before_backup_or_writes(damaged): + path, _settings = damaged + with closing(sqlite3.connect(path)) as connection, connection: + connection.execute("DELETE FROM pc_artifact_heads WHERE artifact_id='fact'") + original = path.read_bytes() + result = CliRunner().invoke(create_cli([app]), ["server", "atomic-memory-rebuild-projection", "--action", "plan"]) + assert result.exit_code != 0 and path.read_bytes() == original + assert json.loads(result.output)["error"] == "invalid_authority" + assert not list(path.parent.glob("*.pc-migration-state/atomic_projection/*/maintenance.json")) + + +def test_projection_repair_rejects_a_processing_configuration_switch(damaged, monkeypatch): + path, _settings = damaged + original = path.read_bytes() + monkeypatch.setenv("POWERCONTEXT_SERVER_RUNTIME_ARTIFACT_PROCESSING_SUPERVISOR_MODE", "dedicated") + result = CliRunner().invoke(create_cli([app]), ["server", "atomic-memory-rebuild-projection", "--action", "plan"]) + assert result.exit_code != 0, result.output + assert json.loads(result.output)["error"] == "incompatible_processing_configuration" + assert path.read_bytes() == original + + +def test_process_exit_during_projection_repair_rolls_back_and_reuses_original_backup(damaged): + path, settings = damaged + runner = deployment_runner(settings, bundle="atomic-projection", batch_size=1) + plan = runner.plan() + original_authority = authority(path) + environment = {key: value for key, value in os.environ.items() if not key.startswith("POWERCONTEXT_SERVER_")} + environment.update( + POWERCONTEXT_SERVER_DATABASE_KIND="sqlite", + POWERCONTEXT_SERVER_DATABASE_URL=f"sqlite+aiosqlite:///{path}", + ) + interrupted = subprocess.run( + [ + sys.executable, + "-c", + """ +import os +from sqlalchemy import Engine, event +from powercontext.cli.app import main +@event.listens_for(Engine, "after_cursor_execute") +def interrupt(connection, cursor, statement, parameters, context, executemany): + if statement.startswith("INSERT INTO pc_atomic_memory_current "): + os._exit(73) +main() +""", + "server", + "db-migrate", + "apply", + "--bundle", + "atomic-projection", + "--batch-size", + "1", + "--plan-id", + plan.plan_id, + "--backup", + "auto", + "--yes", + "--maintenance-confirmed", + ], + env=environment, + capture_output=True, + text=True, + timeout=30, + ) + assert interrupted.returncode == 73, interrupted.stdout + interrupted.stderr + journal = next(path.parent.glob("*.pc-migration-state/atomic_projection/*/maintenance.json")) + original = json.loads(journal.read_text()) + assert original["state"] == "active" + with closing(sqlite3.connect(path)) as connection: + assert connection.execute("SELECT COUNT(*) FROM pc_atomic_memory_current").fetchone() == (0,) + recovery = runner.plan() + assert recovery.state == "recovery_required" + result = runner.apply(plan_id=recovery.plan_id, accepted=True, maintenance_confirmed=True) + finished = json.loads(journal.read_text()) + assert finished["state"] == "complete" + assert finished["maintenance_window_id"] == original["maintenance_window_id"] + assert result.backup_ref == original["backup"]["location"] + assert authority(path) == original_authority + assert deployment_runner(settings).verify().state == "ready" From 0a7a7495d59f8d706cd361f9f4937b76b9edb69c Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 04:11:31 +0800 Subject: [PATCH 62/94] test(kimi): retain bounded Server startup failure evidence Record the failed Uvicorn thread state and basename/function/line frames before shutdown, then retain whether the thread exited. Upload these diagnostics from both real Kimi workflows. Keep the 15-second startup budget and all host assertions. Constraint: Capture no source lines, frame locals, prompts, credentials or request bodies. Scope-risk: Local Python 3.12 blocks in stdlib socketpair even without PowerContext; this does not establish the CI runner cause or fix it. Tested: Real pinned Kimi CLI 2.1.1 on Windows Python 3.14: 2 passed. Python 3.12 failure reproducer retains socketpair startup frames and post-shutdown liveness. Scoped native hooks (Ty run separately due the Linux .venv link), focused Windows Ty and full Linux Ty passed. --- .github/workflows/kimi-code-acceptance.yml | 2 ++ tests/e2e/test_kimi_code_host.py | 32 +++++++++++++++++++--- tests/e2e/test_kimi_code_workflows.py | 5 +++- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/.github/workflows/kimi-code-acceptance.yml b/.github/workflows/kimi-code-acceptance.yml index 8084f33f4a..8d70880bf8 100644 --- a/.github/workflows/kimi-code-acceptance.yml +++ b/.github/workflows/kimi-code-acceptance.yml @@ -135,6 +135,8 @@ jobs: reports.mkdir(exist_ok=True) for evidence in (temporary / "kimi-acceptance").glob("*/evidence.json"): shutil.copy2(evidence, reports / (evidence.parent.name + ".json")) + for evidence in (temporary / "kimi-acceptance").glob("*/server-startup.json"): + shutil.copy2(evidence, reports / (evidence.parent.name + "-server-startup.json")) PY - name: Upload controlled acceptance evidence diff --git a/tests/e2e/test_kimi_code_host.py b/tests/e2e/test_kimi_code_host.py index 55818d6728..db574e9ed9 100644 --- a/tests/e2e/test_kimi_code_host.py +++ b/tests/e2e/test_kimi_code_host.py @@ -21,6 +21,7 @@ import shutil import socket import subprocess +import sys import threading import time from collections.abc import Callable, Iterator @@ -233,22 +234,45 @@ def log_message(self, format: str, *args: Any) -> None: # noqa: A002 - stdlib s @contextmanager -def serving(app): +def serving(app, *, startup_diagnostic: Path | None = None): with socket.socket() as probe: probe.bind(("127.0.0.1", 0)) port = probe.getsockname()[1] server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="error")) thread = threading.Thread(target=server.run, daemon=True) thread.start() + failure = None try: - deadline = time.monotonic() + 15 + started = time.monotonic() + deadline = started + 15 while not server.started and thread.is_alive() and time.monotonic() < deadline: time.sleep(0.05) - assert server.started + if not server.started: + frames = [] + frame = sys._current_frames().get(thread.ident) + while frame is not None: + frames.append({ + "file": Path(frame.f_code.co_filename).name, + "function": frame.f_code.co_name, + "line": frame.f_lineno, + }) + frame = frame.f_back + failure = { + "startup_seconds": time.monotonic() - started, + "thread_alive_before_shutdown": thread.is_alive(), + "server_started": server.started, + "frames": frames, + } + if startup_diagnostic is not None: + startup_diagnostic.write_text(json.dumps(failure, indent=2) + "\n", encoding="utf-8") + assert server.started, json.dumps(failure) yield server, f"http://127.0.0.1:{port}", thread finally: server.should_exit = True thread.join(timeout=10) + if failure is not None and startup_diagnostic is not None: + failure["thread_alive_after_shutdown"] = thread.is_alive() + startup_diagnostic.write_text(json.dumps(failure, indent=2) + "\n", encoding="utf-8") @pytest.fixture @@ -293,7 +317,7 @@ def test_real_kimi_memory_capture_recall_and_fail_open(tmp_path: Path, monkeypat scheduler_path=tmp_path / "scheduler", ) with ( - serving(app) as (server, endpoint, server_thread), + serving(app, startup_diagnostic=tmp_path / "server-startup.json") as (server, endpoint, server_thread), httpx.Client( base_url=endpoint, headers={"Authorization": "Bearer kimi-synthetic-server-token"}, diff --git a/tests/e2e/test_kimi_code_workflows.py b/tests/e2e/test_kimi_code_workflows.py index 0a80064ec9..908e26f9cc 100644 --- a/tests/e2e/test_kimi_code_workflows.py +++ b/tests/e2e/test_kimi_code_workflows.py @@ -112,7 +112,10 @@ def choose(body, scope_id, messages): gateway.choose_tool = choose try: - with serving(app) as (_, endpoint, _), httpx.Client(base_url=endpoint, timeout=10) as client: + with ( + serving(app, startup_diagnostic=tmp_path / "server-startup.json") as (_, endpoint, _), + httpx.Client(base_url=endpoint, timeout=10) as client, + ): scope = client.post( "/v1/scopes", json={"title": "Kimi workflows", "summary": "Native acceptance", "idempotency_key": "kimi-workflows"}, From 1c24549e38cc9f1fb968bdc758f0980053c9a1a7 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 04:27:37 +0800 Subject: [PATCH 63/94] fix(migrations): prevent Topic bundle writes to managed storage Reject pc_schema_revision in the partial Topic recovery bundle before reading maintenance evidence or creating a backup. Managed databases, including unknown revisions, must use the full Server chain or registered maintenance. Preserve unmanaged conversion and recovery behavior. Constraint: No frozen resource, revision or authoritative data is changed by this admission rule. Scope-risk: Native refusal cases use isolated real engines; they do not establish restart recovery or installed upgrade acceptance. Tested: New SQLite regressions failed before the guard. Windows Topic suite 26 passed; Linux Topic and release-gate selection 85 passed; actual seekDB/OceanBase refusal 2 passed without skips under live SQL logging. Scoped hooks, full Linux Ty, persistence digest and embedded workflow Python syntax passed. --- .github/workflows/e2e-harness.yml | 5 ++-- docs/en/docs/operate/database-migrations.md | 3 ++ docs/zh/docs/operate/database-migrations.md | 3 ++ release/upgrade-support.json | 4 +-- .../migrations/topic_memory_recovery.py | 5 ++++ .../test_mysql_migration_runner.py | 30 +++++++++++++++++++ .../test_topic_memory_recovery_migration.py | 15 ++++++++++ 7 files changed, 61 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 29801a650e..f239e3edc4 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -188,6 +188,7 @@ jobs: tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_registered_processing_mode_switch_preserves_revision_and_history \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_atomic_compatibility_command_uses_shared_server_upgrade \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_registered_projection_repair_preserves_authority_and_revision \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_topic_bundle_refuses_managed_server_storage \ -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-recovery.xml uv run --locked python - <<'PY' from pathlib import Path @@ -195,8 +196,8 @@ jobs: suites = ElementTree.parse(Path(".powercontext/recall-effort-ci/full-server-recovery.xml")).getroot() cases = list(suites.iter("testcase")) - if len(cases) != 7 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): - raise SystemExit("All seven native full Server recovery and maintenance scenarios must pass without skips.") + if len(cases) != 8 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): + raise SystemExit("All eight native full Server recovery and maintenance scenarios must pass without skips.") PY - name: Collect the database log diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 11e9bd1e19..11ad21b60b 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -106,6 +106,9 @@ the pre-recovery budget layout and the attempts-table layout from implementation and retains terminal blocks and accepted receipts. It verifies only its registered Topic objects in an existing Server database, including shared processing-lease fencing; other business tables remain untouched. Its output still reports `server_ready=false`. +This partial bundle is restricted to unmanaged storage. Any database containing `pc_schema_revision` is rejected before +backup or writes, including an unknown revision. Use `--bundle server` or a registered managed maintenance operation. + Stop all APIs, Workers, schedulers and SDK writers first, and suspend automatic restarts. Review and apply one plan: ```bash diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index ecb0735141..5a4142c673 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -92,6 +92,9 @@ apply 只修复派生行，完整 Server 校验通过后才提交。权威数据 登记的 Topic 对象和共享处理 lease 的 fencing，不改变其他业务表；输出仍报告 `server_ready=false`。 默认 `--bundle server` 的完整升级路径见上文。 +此部分 bundle 仅适用于未纳入统一框架的存储。只要数据库包含 `pc_schema_revision`，即使 revision 未知， +也会在备份和写入前拒绝。请使用 `--bundle server` 或已登记的受管维护操作。 + 先停止全部 API、Worker、调度和 SDK 写入者，并暂停自动重启。审阅并执行一次计划： ```bash diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 94e3b7df5f..40d3f50f40 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "b9ce8d7ca4d19fa01ddeb78c1dbf3cf6f18bc5bb94e3973e31ba3f0fe2847d6b", - "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback. New revision s1401 registers the frozen offline Memory archive on all three backends through the shared chain; s1300/s1400 resources and checksums remain unchanged, and archived history is preserved. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles. Managed Processing mode maintenance uses the canonical target lock, backup providers and durable evidence without changing pc_schema_revision or frozen migration resources. Its reviewed plan binds configuration and migration ID; one transaction publishes the mode and lease invalidation, and interrupted retries reuse the original recovery point. The Processing compatibility command delegates current managed Server storage and rejects complete unversioned historical Server storage before domain writes; unmanaged domain behavior is retained. The Atomic compatibility migration delegates managed databases and recognized complete Server baselines to the shared full Server chain; legacy decision files remain limited to unmanaged conversion. Registered Atomic current repair now shares the Processing maintenance transaction, canonical target lock, reviewed backup and durable evidence; exact managed layout and authority validation precede repair, and full Server validation precedes commit. It preserves revision and immutable authority, repairs SQLite FTS helpers, and resumes from the original recovery point. Vector/Code profiles and complete native lifecycle acceptance remain incomplete." + "persistence_sha256": "f7687306a1c9f79dd07d01e14f0c030d6d8433512ebb4d8fe971e6dfd076858a", + "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback. New revision s1401 registers the frozen offline Memory archive on all three backends through the shared chain; s1300/s1400 resources and checksums remain unchanged, and archived history is preserved. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles. Managed Processing mode maintenance uses the canonical target lock, backup providers and durable evidence without changing pc_schema_revision or frozen migration resources. Its reviewed plan binds configuration and migration ID; one transaction publishes the mode and lease invalidation, and interrupted retries reuse the original recovery point. The Processing compatibility command delegates current managed Server storage and rejects complete unversioned historical Server storage before domain writes; unmanaged domain behavior is retained. The Atomic compatibility migration delegates managed databases and recognized complete Server baselines to the shared full Server chain; legacy decision files remain limited to unmanaged conversion. Registered Atomic current repair now shares the Processing maintenance transaction, canonical target lock, reviewed backup and durable evidence; exact managed layout and authority validation precede repair, and full Server validation precedes commit. It preserves revision and immutable authority, repairs SQLite FTS helpers, and resumes from the original recovery point. Vector/Code profiles and complete native lifecycle acceptance remain incomplete. The partial Topic recovery bundle rejects pc_schema_revision before backup or writes so it cannot bypass the managed Server chain, including on unknown revisions." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/persistence/migrations/topic_memory_recovery.py b/src/powercontext/builtin/persistence/migrations/topic_memory_recovery.py index 15feb5e41b..eb0aa69e11 100644 --- a/src/powercontext/builtin/persistence/migrations/topic_memory_recovery.py +++ b/src/powercontext/builtin/persistence/migrations/topic_memory_recovery.py @@ -357,6 +357,11 @@ def _provider(self, connection: Connection, identity: BackendIdentity) -> Backup def _inspect(self, connection: Connection | None, identity: BackendIdentity, **options) -> MigrationPlan: if _resource_checksum() != self.checksum: raise MigrationError("plan_changed", "Topic recovery migration resources changed after planning.") + if connection is not None and inspect(connection).has_table("pc_schema_revision"): + raise MigrationError( + "unsupported_target", + "Managed Server storage requires the full Server bundle or registered maintenance.", + ) evidence = self._evidence(identity) active = evidence is not None and evidence.get("state") != "complete" if evidence and ( diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index 9b71ced493..dba2905282 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -47,6 +47,7 @@ from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.oceanbase import profile as oceanbase_profile from powercontext.builtin.persistence.seekdb import SeekDBConfig +from powercontext.builtin.persistence.topic_memory_recovery_schema import recovery_statements from powercontext.builtin.runtime.config import BuiltinConfig, RuntimeConfig from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest from powercontext.cli.app import create_cli @@ -263,6 +264,35 @@ def inspect(connection: Connection | None, _identity: BackendIdentity, _guard: C return backend.connections().run(inspect) +def test_mysql_topic_bundle_refuses_managed_server_storage(backend: _BackendTarget) -> None: + from powercontext.builtin.persistence.migrations.topic_memory_recovery import TopicMemoryRecoveryMigrationRunner + + def seed(connection, _identity, _guard): + connection.exec_driver_sql("CREATE TABLE pc_schema_revision (version_num VARCHAR(32) PRIMARY KEY NOT NULL)") + connection.exec_driver_sql("INSERT INTO pc_schema_revision VALUES ('unknown-future')") + for statement in recovery_statements("budget_v0", "mysql"): + connection.exec_driver_sql(statement) + connection.commit() + + backend.connections().run(seed, writable=True) + + def snapshot(connection, _identity, _guard): + assert connection is not None + return mysql_inventory(connection) + + before = backend.connections().run(snapshot) + runner = TopicMemoryRecoveryMigrationRunner( + backend.config, + evidence_directory=backend.evidence_directory, + lock_coordination="single-host" if isinstance(backend.config, OceanBaseConfig) else None, + ) + with pytest.raises(MigrationError, match="unsupported_target"): + runner.plan() + after = backend.connections().run(snapshot) + assert after == before + assert not list(backend.evidence_directory.rglob("topic-memory-recovery.json")) + + def test_mysql_maintenance_commits_rolls_back_and_reopens_for_ddl(backend: _BackendTarget) -> None: def create(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: assert connection is not None diff --git a/tests/e2e/test_topic_memory_recovery_migration.py b/tests/e2e/test_topic_memory_recovery_migration.py index b9bfe6a834..4cfb6c6c2c 100644 --- a/tests/e2e/test_topic_memory_recovery_migration.py +++ b/tests/e2e/test_topic_memory_recovery_migration.py @@ -127,6 +127,21 @@ def _apply(runner): return runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True) +@pytest.mark.parametrize("revision", ["s1401", "unknown-future"]) +def test_topic_bundle_refuses_managed_server_storage_before_backup_or_writes(tmp_path, revision): + path = tmp_path / "managed-topic.db" + config, _ = _legacy(path) + with closing(sqlite3.connect(path)) as connection, connection: + connection.execute("CREATE TABLE pc_schema_revision (version_num VARCHAR(32) PRIMARY KEY NOT NULL)") + connection.execute("INSERT INTO pc_schema_revision VALUES (?)", (revision,)) + original = path.read_bytes() + runner = TopicMemoryRecoveryMigrationRunner(config) + with pytest.raises(MigrationError, match="unsupported_target"): + _apply(runner) + assert path.read_bytes() == original + assert not path.with_name(path.name + ".pc-migration-state").exists() + + @pytest.fixture def atomic_maintenance_database(tmp_path, monkeypatch): path = tmp_path / "atomic-maintenance.db" From be53c1b7efab02c4eb18cce5183cc7fd11be7b36 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 04:40:34 +0800 Subject: [PATCH 64/94] fix(server): avoid identity schema DDL on managed storage Use the shared managed-table guard when initializing Server identity after profile admission. Preserve concurrent unmanaged schema creation and transactional identity value creation and rotation. Use a portable SQLite URL in the offline identity reset CLI fixture. Constraint: Full Server profile admission still validates the managed bundle. No frozen migration, revision or authoritative identity value is rewritten. Scope-risk: Real seekDB and OceanBase cases prove stable identity readback and rotation without startup DDL; they do not prove crash recovery or installed historical service lifecycle. Tested: The new SQLite managed initialization regression failed before the fix. Windows identity and CLI selection 15 passed; Linux identity, CLI and release-policy selection 132 passed; real seekDB/OceanBase identity scenarios 2 passed without skips under live SQL logging. Scoped hooks, full Linux Ty, persistence digest and embedded workflow Python syntax passed. --- .github/workflows/e2e-harness.yml | 5 +-- docs/en/docs/operate/database-migrations.md | 3 ++ docs/zh/docs/operate/database-migrations.md | 3 ++ release/upgrade-support.json | 4 +-- src/powercontext/server/identity.py | 10 ++++-- .../test_mysql_migration_runner.py | 33 +++++++++++++++++++ tests/test_cli.py | 4 ++- tests/test_server_identity.py | 22 +++++++++++++ 8 files changed, 77 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index f239e3edc4..60a97721b8 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -189,6 +189,7 @@ jobs: tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_atomic_compatibility_command_uses_shared_server_upgrade \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_registered_projection_repair_preserves_authority_and_revision \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_topic_bundle_refuses_managed_server_storage \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_managed_identity_initialization_requires_no_ddl \ -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-recovery.xml uv run --locked python - <<'PY' from pathlib import Path @@ -196,8 +197,8 @@ jobs: suites = ElementTree.parse(Path(".powercontext/recall-effort-ci/full-server-recovery.xml")).getroot() cases = list(suites.iter("testcase")) - if len(cases) != 8 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): - raise SystemExit("All eight native full Server recovery and maintenance scenarios must pass without skips.") + if len(cases) != 9 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): + raise SystemExit("All nine native full Server recovery and maintenance scenarios must pass without skips.") PY - name: Collect the database log diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 11ad21b60b..49c2c166b4 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -26,6 +26,9 @@ Revision `s1401` registers `pc_memory_artifact_archive` in the managed target sc Ordinary Server startup checks readiness without repairing an existing database. Empty local persistent SQLite/seekDB storage initializes through the same revision chain; OceanBase requires an explicit maintenance job first. In-memory SQLite remains transient runtime storage. A changed schema or inconsistent projection requires maintenance, not restart-based repair. +Server identity initialization checks registered tables without issuing DDL on managed storage. Creating the initial +identity value and explicitly rotating a stopped clone's identity retain their existing transactional behavior. + Existing fresh deployments outside the no-vector, Code-disabled profile retain domain bootstrap and startup checks; they are not automatically adopted into this chain or covered by its upgrade support. Changing configuration cannot bypass admission or implicitly add vector/Code objects to managed storage. `--bundle artifacts` retains the partial Artifact prototype chain starting at `p0001`; `--bundle topic-memory-recovery` covers only registered Topic recovery layouts. Both report `server_ready=false`. Neither establishes full Server readiness, and their revisions cannot be stamped as full Server revisions. diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 5a4142c673..f5756f4cda 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -24,6 +24,9 @@ Candidate 旧表保留为 `pc_retained_dream_candidate_heads` 和 `pc_retained_d 普通 Server 启动先做只读就绪检查，不修补已有库。空的本地持久 SQLite/seekDB 可通过同一版本链初始化；OceanBase 必须先执行显式维护命令初始化。内存 SQLite 仍为临时运行时存储。迁移后发现结构或投影被篡改，必须停写维护，不能靠重启修复。 +Server identity 初始化在受管库上只检查已登记的表，不执行 DDL。首次创建 identity 值以及显式轮换已停止副本的 +identity，仍使用原有事务语义。 + 无向量、Code 关闭之外的既有新建部署仍使用原领域初始化和启动检查，不会自动纳入此版本链；这些配置不属于已登记升级范围。受管数据库不能通过切换配置绕过检查或隐式增加向量/Code 表。 `--bundle artifacts` 保留 `p0001` 开始的局部 Artifact 原型链；`--bundle topic-memory-recovery` 只处理登记的 Topic 恢复布局。两者均报告 `server_ready=false`，不得用于证明完整 Server 就绪，也不能把其版本号直接 stamp 为完整 Server 版本。 diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 40d3f50f40..5df9d19237 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "f7687306a1c9f79dd07d01e14f0c030d6d8433512ebb4d8fe971e6dfd076858a", - "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback. New revision s1401 registers the frozen offline Memory archive on all three backends through the shared chain; s1300/s1400 resources and checksums remain unchanged, and archived history is preserved. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles. Managed Processing mode maintenance uses the canonical target lock, backup providers and durable evidence without changing pc_schema_revision or frozen migration resources. Its reviewed plan binds configuration and migration ID; one transaction publishes the mode and lease invalidation, and interrupted retries reuse the original recovery point. The Processing compatibility command delegates current managed Server storage and rejects complete unversioned historical Server storage before domain writes; unmanaged domain behavior is retained. The Atomic compatibility migration delegates managed databases and recognized complete Server baselines to the shared full Server chain; legacy decision files remain limited to unmanaged conversion. Registered Atomic current repair now shares the Processing maintenance transaction, canonical target lock, reviewed backup and durable evidence; exact managed layout and authority validation precede repair, and full Server validation precedes commit. It preserves revision and immutable authority, repairs SQLite FTS helpers, and resumes from the original recovery point. Vector/Code profiles and complete native lifecycle acceptance remain incomplete. The partial Topic recovery bundle rejects pc_schema_revision before backup or writes so it cannot bypass the managed Server chain, including on unknown revisions." + "persistence_sha256": "53741f45c16f15237657b31508fea472993d954f07200fe78a09e709d945ed7e", + "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback. New revision s1401 registers the frozen offline Memory archive on all three backends through the shared chain; s1300/s1400 resources and checksums remain unchanged, and archived history is preserved. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles. Managed Processing mode maintenance uses the canonical target lock, backup providers and durable evidence without changing pc_schema_revision or frozen migration resources. Its reviewed plan binds configuration and migration ID; one transaction publishes the mode and lease invalidation, and interrupted retries reuse the original recovery point. The Processing compatibility command delegates current managed Server storage and rejects complete unversioned historical Server storage before domain writes; unmanaged domain behavior is retained. The Atomic compatibility migration delegates managed databases and recognized complete Server baselines to the shared full Server chain; legacy decision files remain limited to unmanaged conversion. Registered Atomic current repair now shares the Processing maintenance transaction, canonical target lock, reviewed backup and durable evidence; exact managed layout and authority validation precede repair, and full Server validation precedes commit. It preserves revision and immutable authority, repairs SQLite FTS helpers, and resumes from the original recovery point. Vector/Code profiles and complete native lifecycle acceptance remain incomplete. The partial Topic recovery bundle rejects pc_schema_revision before backup or writes so it cannot bypass the managed Server chain, including on unknown revisions. Managed Server identity initialization checks the existing identity table through the shared schema guard without startup DDL; full profile admission remains responsible for bundle readiness. Identity row creation and explicit rotation retain their transactional behavior." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/server/identity.py b/src/powercontext/server/identity.py index f98bcc50f1..c351bce867 100644 --- a/src/powercontext/server/identity.py +++ b/src/powercontext/server/identity.py @@ -22,12 +22,13 @@ from typing import TypeVar from uuid import uuid4 -from sqlalchemy import CheckConstraint, Column, Integer, MetaData, String, Table, insert, select, update +from sqlalchemy import CheckConstraint, Column, Integer, MetaData, String, Table, insert, inspect, select, update from sqlalchemy.exc import IntegrityError, OperationalError from sqlalchemy.schema import CreateTable from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile +from powercontext.builtin.persistence.schema import create_tables from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile, is_sqlite_lock_error from powercontext.builtin.runtime.config import DatabaseConfig @@ -54,12 +55,17 @@ def __init__(self, database: AsyncDatabase) -> None: self._database = database async def initialize(self) -> None: - """Create the identity schema safely across concurrent initializers.""" + """Check managed tables or safely initialize an unmanaged identity schema.""" await _retry_initialization(self._initialize) async def _initialize(self) -> None: async with self._database.transaction() as connection: + if await connection.run_sync(lambda sync: inspect(sync).has_table("pc_schema_revision")): + # Managed profiles validate the full bundle before admission. + # This shared check refuses missing tables without startup DDL. + await create_tables(connection, (SERVER_IDENTITY_TABLE,)) + return await connection.execute(CreateTable(SERVER_IDENTITY_TABLE, if_not_exists=True)) async def load_or_create(self) -> str: diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index dba2905282..ce1de204d9 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -293,6 +293,39 @@ def snapshot(connection, _identity, _guard): assert not list(backend.evidence_directory.rglob("topic-memory-recovery.json")) +def test_mysql_managed_identity_initialization_requires_no_ddl(backend: _BackendTarget) -> None: + from powercontext.builtin.persistence.oceanbase import OceanBaseProfile + from powercontext.builtin.persistence.seekdb import SeekDBProfile + from powercontext.server.identity import ServerIdentityRepository + + _load_full_server_baseline(backend) + _apply(_full_server_runner(backend), policy="manual") + + async def scenario(): + opened = ( + OceanBaseProfile.open(backend.config, tables=()) + if isinstance(backend.config, OceanBaseConfig) + else SeekDBProfile.open(backend.config, tables=()) + ) + async with opened as profile: + + @event.listens_for(profile.database.engine.sync_engine, "before_cursor_execute") + def reject_ddl(_connection, _cursor, statement, _parameters, _context, _executemany): + assert not statement.lstrip().upper().startswith(("CREATE ", "ALTER ", "DROP ")), ( + "Managed identity initialization must not issue DDL" + ) + + repository = ServerIdentityRepository(profile.database) + await repository.initialize() + original = await repository.load_or_create() + assert await repository.load_or_create() == original + rotated = await repository.rotate() + assert rotated != original and await repository.load_or_create() == rotated + + asyncio.run(scenario()) + assert _full_server_runner(backend).verify().state == "ready" + + def test_mysql_maintenance_commits_rolls_back_and_reopens_for_ddl(backend: _BackendTarget) -> None: def create(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: assert connection is not None diff --git a/tests/test_cli.py b/tests/test_cli.py index 463db822bb..f6d0598529 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -670,7 +670,9 @@ def test_cli_reports_broken_command_providers(monkeypatch, name: str, value: str def test_server_identity_reset_requires_offline_confirmation_and_rotates(tmp_path) -> None: database = tmp_path / "runtime.db" environment = tmp_path / "server.env" - environment.write_text(f"POWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite:///{database}\n") + environment.write_text( + f"POWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite:///{database.as_posix()}\n", encoding="utf-8" + ) cli = create_cli([server_app]) refused = CliRunner().invoke(cli, ["server", "identity-reset", "--env-file", str(environment)]) diff --git a/tests/test_server_identity.py b/tests/test_server_identity.py index 845b27a3b1..b225d781f0 100644 --- a/tests/test_server_identity.py +++ b/tests/test_server_identity.py @@ -24,12 +24,34 @@ from sqlalchemy.exc import OperationalError from sqlalchemy.schema import CreateTable +from powercontext.builtin.persistence.migrations.deployment import deployment_runner from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.server.identity import ( SERVER_IDENTITY_TABLE, ServerIdentityRepository, open_server_identity_repository, ) +from powercontext.server.settings import ServerSettings + + +def test_managed_server_identity_initialization_requires_no_ddl(tmp_path) -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'managed.db'}") + runner = deployment_runner(ServerSettings(database=config)) + plan = runner.plan() + runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + + async def scenario() -> None: + async with SQLiteProfile.open_readonly(config) as profile: + + @event.listens_for(profile.database.engine.sync_engine, "before_cursor_execute") + def reject_ddl(_connection, _cursor, statement, _parameters, _context, _executemany): + assert not statement.lstrip().upper().startswith(("CREATE ", "ALTER ", "DROP ")), ( + "Managed identity initialization must not issue DDL" + ) + + await ServerIdentityRepository(profile.database).initialize() + + asyncio.run(scenario()) def test_server_identity_singleton_key_is_not_auto_incremented_by_mysql_profiles() -> None: From f8c9487ccbb34cfa8ed80d2684b6c5f5da8a9ba1 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 04:47:20 +0800 Subject: [PATCH 65/94] docs(migrations): inventory startup and maintenance ownership Map profile admission, compatibility helpers, projections, Topic recovery, Processing configuration, Server identity, Memory archives and optional objects to their managed and unmanaged owners. Distinguish read-only historical inspection from runtime admission, and identify Code's embedded tables separately from its local cache. Constraint: The inventory does not register optional configurations or claim successful release acceptance. Scope-risk: Documentation describes inspected entry points; native crash recovery and complete installed service lifecycle remain unproved. Tested: Inspected current runtime composition, profiles, shared table creation, CLI delegation, domain migration resources and Code storage initialization. Scoped document hooks and git diff --check passed. --- docs/en/docs/operate/database-migrations.md | 29 +++++++++++++++++++++ docs/zh/docs/operate/database-migrations.md | 26 ++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 49c2c166b4..8ea620c725 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -50,6 +50,35 @@ conversion for unmanaged legacy Memory layouts. Decision files for that conversi chain. The compatibility command returns the shared Server plan/result for managed storage; an already-ready apply is a read-only no-op. See [Migrate to Atomic Memory](atomic-memory-migration.md). +## Initialization and maintenance ownership + +The following inventory identifies the owners of schema changes and projection repair. "Managed" means that the +database contains `pc_schema_revision`, including an unknown revision. Passing a partial domain check does not admit +that database for Server use. Full profile admission checks the registered Server layout and domain state first. + +| Entry point or helper | Managed storage | Unmanaged storage | +| --- | --- | --- | +| `SQLiteProfile.open`, `SeekDBProfile.open`, `OceanBaseProfile.open`; `schema.create_tables` | Full profile readiness checks precede use; selected tables must already exist | Retain selected-table creation and domain admission | +| Runtime composition: Skill distribution, tag families, Scope search, revision metadata and Artifact head columns | Skip compatibility DDL and backfills; the registered Server chain owns these objects | Retain the existing compatibility helpers | +| Runtime Atomic Memory authority/current indexes and SQLite FTS helpers | Validate current data; repair through `db-migrate --bundle atomic-projection` after layout admission | Retain domain initialization and offline projection repair | +| Runtime Experience/Skill searchable text and FTS indexes | The registered Server domain step builds and validates projections; startup does not rebuild them | Retain domain initialization and projection recovery | +| Runtime Topic Memory indexes, recovery tables and retrieval shape | Initialize with `validate_only=True`; partial Topic recovery rejects managed storage | Retain domain initialization; registered partial recovery covers only its declared layouts | +| Processing schema and ownership configuration | Validate at startup; change configuration through `db-migrate --bundle processing` | Retain domain bootstrap and `processing-migrate` | +| `ServerIdentityRepository.initialize` | Check the identity table without DDL; identity value creation and explicit rotation are ordinary transactions | Retain concurrent-safe identity table initialization | +| Offline Memory archive | The Server chain registers its structure at `s1401`; runtime admission checks it | Legacy offline Memory conversion owns its archive and conversion receipts | +| `atomic-memory-migrate` and `processing-migrate` | Delegate to the shared Server or registered maintenance plan | Retain their domain-only behavior; recognized historical Server storage must use the full upgrade first | +| Optional vector and Code objects | No registered configuration transition; reject before adding objects | Retain existing feature bootstrap without claiming registered upgrade support | + +These helpers remain available to their owning runtime or maintenance operations; calling a low-level helper directly +does not replace profile admission, a reviewed maintenance plan, or backup and recovery requirements. Future changes to +managed objects belong in the registered revision/domain steps and validators, rather than another startup helper. +Read-only SQLite inspection and dedicated maintenance connections can inspect historical storage without business +initialization; successfully opening an inspection connection does not establish Server readiness. + +Code's embedded seekDB store adds `pc_code_generations`, `pc_code_nodes`, `pc_code_edges` and `ix_code_search` to the +deployment database. Other Code deployments use a local SQLite cache. These are derived indexes owned by the Code +service; their existence does not register a Code-enabled Server baseline or permit implicit managed-schema expansion. + ## Processing configuration maintenance `--bundle processing` registers an offline ownership-configuration operation on a complete, current Server database. diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index f5756f4cda..3447989ba6 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -40,6 +40,32 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 该转换的决策文件不能修改已登记的 Server 版本链。兼容命令在受管库上返回共享 Server 计划或结果， 已就绪时 apply 保持只读，不重复迁移。参见[迁移到 Atomic Memory](atomic-memory-migration.md)。 +## 初始化与维护职责 + +下表列出结构变更与投影修复的执行入口。含有 `pc_schema_revision` 的数据库均视为受管库，包括未知 revision； +局部领域检查通过不代表整个 Server 可以使用该库。Profile 准入先验证已登记的整库结构与领域状态。 + +| 入口或辅助函数 | 受管数据库 | 未受管数据库 | +| --- | --- | --- | +| `SQLiteProfile.open`、`SeekDBProfile.open`、`OceanBaseProfile.open`；`schema.create_tables` | 使用前先做整库就绪检查；所选表必须已经存在 | 保留所选表创建和领域准入 | +| Runtime composition：Skill 分发、标签 Family、Scope 搜索、revision 元数据和 Artifact Head 列 | 跳过兼容性 DDL 与回填，由已登记的 Server 版本链负责 | 保留既有兼容辅助函数 | +| Runtime Atomic Memory 权威记录、current 索引和 SQLite FTS 辅助对象 | 校验 current 数据；结构准入后通过 `db-migrate --bundle atomic-projection` 修复 | 保留领域初始化与离线投影修复 | +| Runtime Experience/Skill 搜索文本与 FTS 索引 | 由已登记的 Server 领域步骤构建和校验；启动不重建 | 保留领域初始化与投影恢复 | +| Runtime Topic Memory 索引、恢复表和检索配置 | 使用 `validate_only=True` 初始化；局部 Topic 恢复拒绝受管库 | 保留领域初始化；局部恢复只覆盖其登记的布局 | +| Processing 结构与所有权配置 | 启动时校验；配置变更走 `db-migrate --bundle processing` | 保留领域 bootstrap 和 `processing-migrate` | +| `ServerIdentityRepository.initialize` | 检查 identity 表，不执行 DDL；创建 identity 值和显式轮换仍使用普通事务 | 保留支持并发的 identity 表初始化 | +| 离线 Memory 归档 | Server 版本链在 `s1401` 登记结构，运行时准入检查该结构 | 旧 Memory 离线转换负责其归档和转换回执 | +| `atomic-memory-migrate`、`processing-migrate` | 转发到共享 Server 或已登记的维护计划 | 保留领域独立行为；识别为历史完整 Server 的库必须先完成整库升级 | +| 可选向量与 Code 对象 | 尚无已登记的配置转换，添加对象前拒绝 | 保留既有功能初始化，不宣称已登记升级支持 | + +这些辅助函数仍供所属 runtime 或维护操作使用；直接调用底层函数不能替代 Profile 准入、已确认的维护计划和 +备份恢复要求。未来受管对象的变化应加入已登记的 revision、领域步骤与校验器，不另建启动迁移入口。 +只读 SQLite 检查入口与独立维护连接可以在业务初始化之前检查历史存储；检查连接打开成功不代表 Server 已就绪。 + +Code 的嵌入式 seekDB 存储在部署数据库中添加 `pc_code_generations`、`pc_code_nodes`、`pc_code_edges` 和 +`ix_code_search`；其他 Code 部署使用本地 SQLite 缓存。这些派生索引由 Code 服务负责，其存在不代表已经登记 +Code 启用的 Server 基线，也不允许隐式扩展受管结构。 + ## Processing 配置维护 `--bundle processing` 在结构完整、已到当前 revision 的 Server 数据库上登记离线所有权配置操作。 From 4fe13093e72b5e8e5ad8611fa1a4b04c515872eb Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 05:05:15 +0800 Subject: [PATCH 66/94] test(service): cover native upgrades from installed 1.3.0 Install the fixed released source wheel separately in each native CI job. Start it through an isolated real native registration, populate a Scope over HTTP, then migrate through the public CLI and verify automatic or explicit target activation according to the original running state. Preserve source wheel hash verification and distinguish SQLite lifecycle acceptance from backend recovery. Constraint: No runtime migration branch, support-matrix expansion or readiness bypass is introduced. Scope-risk: Local evidence covers Windows Task Scheduler and SQLite with actual installed source and target distributions. Linux systemd and macOS LaunchAgent cases still require their resulting-Head CI executions; native engine crash recovery and complete installed domain acceptance remain separate. Tested: Windows actual 1.3.0-to-target running/stopped lifecycle 2 passed, 14 SQLAlchemy deprecation warnings; existing Windows native lifecycle control 1 passed. Source authentication and schedule fixture mistakes failed before correction, with direct launcher diagnostics retained. Scoped hooks, full Linux Ty and git diff --check passed. --- .github/workflows/native-service.yml | 9 ++ docs/en/docs/operate/database-migrations.md | 8 ++ docs/zh/docs/operate/database-migrations.md | 6 + .../native/test_personal_service_lifecycle.py | 103 ++++++++++++++++++ 4 files changed, 126 insertions(+) diff --git a/.github/workflows/native-service.yml b/.github/workflows/native-service.yml index b4fae40996..69e1055663 100644 --- a/.github/workflows/native-service.yml +++ b/.github/workflows/native-service.yml @@ -50,6 +50,9 @@ jobs: wheel=$(find dist -name '*.whl' -print -quit) uv venv .native-venv uv pip install --python .native-venv/bin/python "powercontext[cli,server] @ file://$PWD/$wheel" pytest + uv venv .native-source-venv + uv pip install --python .native-source-venv/bin/python "powercontext[cli,server] @ https://files.pythonhosted.org/packages/30/bd/463f998f8794bc7f8ff588d47157ca0e62dd03a9f3ae466e2eb34fb66045/powercontext-1.3.0-py3-none-any.whl#sha256=8da7983763f3a6ab80b413c770a4973a46c557a54e1a1fa675099e58d47305e5" + echo "POWERCONTEXT_NATIVE_UPGRADE_SOURCE_PYTHON=$PWD/.native-source-venv/bin/python" >> "$GITHUB_ENV" - name: Exercise the real systemd user lifecycle run: >- @@ -105,6 +108,9 @@ jobs: # Finish bytecode compilation during installation, before launchd # runs the cold-start lifecycle under its bounded startup deadline. uv pip install --compile-bytecode --python .native-venv/bin/python "powercontext[cli,server] @ file://$PWD/$wheel" pytest + uv venv .native-source-venv + uv pip install --compile-bytecode --python .native-source-venv/bin/python "powercontext[cli,server] @ https://files.pythonhosted.org/packages/30/bd/463f998f8794bc7f8ff588d47157ca0e62dd03a9f3ae466e2eb34fb66045/powercontext-1.3.0-py3-none-any.whl#sha256=8da7983763f3a6ab80b413c770a4973a46c557a54e1a1fa675099e58d47305e5" + echo "POWERCONTEXT_NATIVE_UPGRADE_SOURCE_PYTHON=$PWD/.native-source-venv/bin/python" >> "$GITHUB_ENV" - name: Exercise the real LaunchAgent lifecycle run: >- @@ -157,6 +163,9 @@ jobs: $wheelUri = "file:///" + $wheel.Replace('\', '/') uv venv .native-venv uv pip install --python .native-venv\Scripts\python.exe "powercontext[cli,server] @ $wheelUri" pytest + uv venv .native-source-venv + uv pip install --python .native-source-venv\Scripts\python.exe "powercontext[cli,server] @ https://files.pythonhosted.org/packages/30/bd/463f998f8794bc7f8ff588d47157ca0e62dd03a9f3ae466e2eb34fb66045/powercontext-1.3.0-py3-none-any.whl#sha256=8da7983763f3a6ab80b413c770a4973a46c557a54e1a1fa675099e58d47305e5" + "POWERCONTEXT_NATIVE_UPGRADE_SOURCE_PYTHON=$PWD\.native-source-venv\Scripts\python.exe" >> $env:GITHUB_ENV - name: Verify binary cursor persistence and restart compatibility shell: pwsh diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 8ea620c725..704fbe3056 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -414,6 +414,14 @@ that it stays stopped until an explicit start, and completes service maintenance Each operating system requires its own native run. A frozen baseline fixture is distinct from upgrading a database populated by an installed historical release. +The suite also installs the actual 1.3.0 wheel in a separate environment, starts it through the native manager and +populates a Scope through HTTP. Both running and stopped service cases use the public maintenance command, verify the +backup and retained data, and confirm that registration switches to the target executable while preserving the intended +running state. The CI setup verifies the source wheel's SHA-256 and supplies +`POWERCONTEXT_NATIVE_UPGRADE_SOURCE_PYTHON`; local native runs need the same installed-source interpreter. These cases +cover SQLite service switching. Backend crash recovery and the full installed-release domain suite remain separate +acceptance requirements. + For manual deployment, stop the old service first. After a supported complete migration succeeds, ensure the launch definition points to the new environment before starting it. For foreground deployment, exit the old process and use the existing command from the new environment: diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 3447989ba6..05961281e7 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -312,6 +312,12 @@ powercontext service restart 升级已停止服务的冻结 SQLite 基线，确认升级后仍保持停止并可显式启动；同时验证数据库已就绪时运行中服务的维护恢复。 每个操作系统须分别执行原生测试。冻结基线夹具与实际历史版本安装并写入数据库后的升级验收是不同的证据。 +套件还在独立环境中安装真实 1.3.0 wheel，通过原生管理器启动，并用 HTTP 创建 Scope。运行中和已停止两种服务状态 +均通过公开维护命令升级，验证备份与旧数据保留，并确认注册切换到目标解释器且保持预期运行状态。 +CI 安装时校验来源 wheel 的 SHA-256，并设置 `POWERCONTEXT_NATIVE_UPGRADE_SOURCE_PYTHON`； +本地运行原生测试时也需提供该已安装来源版本的解释器。这些用例覆盖 SQLite 服务切换， +后端崩溃恢复与完整安装版本领域验收仍需分别验证。 + 手动管理时，先停旧服务，在已支持的完整迁移成功后确认服务启动定义指向新版，再启动。前台部署则退出旧进程，使用新版环境的现有入口： ```bash diff --git a/tests/native/test_personal_service_lifecycle.py b/tests/native/test_personal_service_lifecycle.py index 99836ccab7..663a0c1564 100644 --- a/tests/native/test_personal_service_lifecycle.py +++ b/tests/native/test_personal_service_lifecycle.py @@ -31,6 +31,7 @@ from pathlib import Path from typing import Any +import httpx import pytest from alembic import command from sqlalchemy import create_engine @@ -267,6 +268,108 @@ def test_native_public_migration_of_ready_storage_restores_a_running_service( _cleanup(adapter) +@pytest.mark.parametrize("originally_running", [True, False], ids=["running", "stopped"]) +def test_native_public_migration_switches_an_installed_130_service( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, originally_running: bool +) -> None: + """Upgrade storage populated by the released service through its actual native manager.""" + + source = Path(os.environ["POWERCONTEXT_NATIVE_UPGRADE_SOURCE_PYTHON"]) + source_version = _run( + str(source), "-I", "-c", 'from importlib.metadata import version; print(version("powercontext"))' + ).stdout.strip() + assert source_version == "1.3.0" + source_launcher = source.with_name("pythonw.exe") if os.name == "nt" else source + assert source_launcher.is_file() + adapter = _native_adapter(suffix=f"installed-upgrade-{'running' if originally_running else 'stopped'}") + controller = ServiceController(adapter) + environment = _environment_file(tmp_path) + with environment.open("a", encoding="utf-8") as output: + output.write( + "POWERCONTEXT_SERVER_AUTH_TOKEN=native-upgrade-fixture-token\nPOWERCONTEXT_SERVER_ACCESS_MODE=enforced\n" + ) + port = _service_port(tmp_path) + assert port is not None + definition = ServiceDefinition( + ownership=OWNERSHIP_MARKER, + definition_version=DEFINITION_VERSION, + package_version=source_version, + python_executable=str(source_launcher.absolute()), + endpoint=f"http://127.0.0.1:{port}", + data_dir=str(tmp_path / "data"), + env_file=load_protected_environment_file(environment).identity, + ) + try: + adapter.write(adapter.render(definition)) + adapter.update_suspended() + # Starting a valid older executable intentionally leaves a stale-version + # status until migration switches the registration to this environment. + try: + controller.start() + except ServiceError as error: + pytest.fail(f"{error}\n\n{_failure_diagnostics(adapter, tmp_path)}") + assert adapter.manager_state() is ManagerState.ACTIVE, _failure_diagnostics(adapter, tmp_path) + with httpx.Client( + base_url=definition.endpoint, + headers={"Authorization": "Bearer native-upgrade-fixture-token"}, + timeout=30, + trust_env=False, + ) as client: + created = client.post( + "/v1/scopes", + json={ + "title": "Installed service upgrade", + "summary": "Preserve data across the native executable switch.", + "idempotency_key": "native-installed-upgrade", + }, + ) + assert created.is_success, created.text + scope_id = created.json()["scope_id"] + if not originally_running: + controller.stop() + # Select this isolated registration; the manager and migration remain real. + monkeypatch.setattr(database_migration, "ServiceController", lambda: controller) + cli = create_cli([server_app]) + reviewed = _native_migration_command(cli, environment, "plan") + assert reviewed["state"] == "migration_required" + assert reviewed["service"]["originally_running"] is originally_running + assert reviewed["service"]["definition"]["python_executable"] == definition.python_executable + assert reviewed["service"]["target_definition"]["python_executable"] == service_python_executable() + result = _native_migration_command( + cli, + environment, + "apply", + "--backup", + "auto", + "--plan-id", + reviewed["plan_id"], + "--yes", + "--maintenance-confirmed", + ) + assert result["server_ready"] is True + assert result["backup_state"] == "completed" + assert result["backup_ref"] and Path(result["backup_ref"]).is_file() + installed = adapter.inspect().definition + assert installed is not None + assert installed.python_executable == service_python_executable() + assert installed.package_version == version("powercontext") + if not originally_running: + assert adapter.manager_state() in {ManagerState.INACTIVE, ManagerState.FAILED} + assert controller.maintenance_path.exists() + assert controller.start().ok, _failure_diagnostics(adapter, tmp_path) + assert controller.status().ok, _failure_diagnostics(adapter, tmp_path) + assert not controller.maintenance_path.exists() + retained = client.get(f"/v1/scopes/{scope_id}") + assert retained.is_success, retained.text + assert retained.json()["title"] == "Installed service upgrade" + assert client.get("/health/ready").status_code == 200 + finally: + _capture_native_failure(adapter, tmp_path) + with suppress(Exception): + controller.uninstall() + _cleanup(adapter) + + def _native_migration_command(cli, environment: Path, action: str, *options: str) -> dict[str, Any]: result = CliRunner().invoke( cli, ["server", "db-migrate", action, "--env-file", str(environment), "--manage-service", *options] From 66962ec09c3f67175efb06f23e50c26d6d256240 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 05:53:07 +0800 Subject: [PATCH 67/94] test(migrations): recover after an OceanBase client exits Terminate a real migration subprocess immediately after successful registered DDL. Require explicit old-run acknowledgement after its remote session ends, then preserve the original recovery point, historical bytes and idempotency. Constraint: the database engine stays running; manual backup declarations do not prove restoration or engine crash recovery. Scope-risk: native full-Server maintenance regression coverage only. Tested: real OceanBase client-exit case (1 passed); complete registered native recovery/maintenance selection (10 passed, zero skips, 614.61 seconds); full Linux Python 3.11 Ty; scoped pre-commit hooks. Refs: #2084, #2154 --- .../test_mysql_migration_runner.py | 75 +++++++++++++++++++ .../crash_after_registered_ddl.py | 62 +++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 tests/fixtures/database_migrations/crash_after_registered_ddl.py diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index ce1de204d9..aaee862041 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -20,6 +20,9 @@ import json import logging import os +import subprocess +import sys +import time from collections.abc import Callable, Iterator from contextlib import contextmanager from dataclasses import dataclass @@ -721,6 +724,78 @@ def inspect_history(connection: Connection | None, _identity: BackendIdentity, _ assert not _apply(_full_server_runner(backend), policy="manual").changed +@pytest.mark.parametrize("backend", ("oceanbase",), indirect=True) +def test_oceanbase_full_server_client_exit_requires_acknowledgement_and_preserves_recovery( + backend: _BackendTarget, +) -> None: + """A dead remote client requires acknowledgement; the database engine stays running.""" + assert isinstance(backend.config, OceanBaseConfig) + _load_full_server_baseline(backend) + root = Path(__file__).resolve().parents[3] + environment = { + **os.environ, + "PYTHONPATH": os.pathsep.join(filter(None, (str(root), os.environ.get("PYTHONPATH")))), + "POWERCONTEXT_MIGRATION_CRASH_TARGET": backend.config.url.get_secret_value(), + } + with (backend.evidence_directory / "crashed-client.log").open("w", encoding="utf-8") as log: + crashed = subprocess.run( + [ + sys.executable, + str(root / "tests/fixtures/database_migrations/crash_after_registered_ddl.py"), + str(backend.evidence_directory), + ], + env=environment, + stdout=log, + stderr=subprocess.STDOUT, + timeout=180, + check=False, + ) + assert crashed.returncode == 73, "The migration client did not exit at the registered DDL boundary" + marker = json.loads((backend.evidence_directory / "client-exit.json").read_text(encoding="utf-8")) + journals = list(backend.evidence_directory.rglob("maintenance.json")) + assert len(journals) == 1 + interrupted = json.loads(journals[0].read_text(encoding="utf-8")) + assert interrupted["state"] == "active" and interrupted["manual_ref"] == "operator://manual-backup" + + def client_active(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]): + assert connection is not None + return connection.execute( + text("SELECT ID FROM information_schema.PROCESSLIST WHERE ID=:connection_id"), + {"connection_id": marker["connection_id"]}, + ).first() + + deadline = time.monotonic() + 15 + while backend.connections().run(client_active) is not None: + assert time.monotonic() < deadline, "The terminated client's remote session did not end" + time.sleep(0.2) + recovery = _full_server_runner(backend).plan(backup_policy="manual") + assert recovery.state == "recovery_required" and recovery.source_revision == "s1300" + token = (recovery.coordination or {}).get("pending_run_id") + assert token + with pytest.raises(MigrationError, match="recovery_required"): + _apply(_full_server_runner(backend), policy="manual") + # The completed DDL and vanished remote session establish this test's known + # interruption. An arbitrary remote failure cannot be acknowledged blindly. + resumed = _full_server_runner(backend, acknowledge_previous_run=token) + result = _apply(resumed, policy="manual") + assert result.state == "ready" and result.readiness_scope == "server" and result.revision == "s1401" + assert result.backup_state == "user_confirmed" and result.backup_ref == interrupted["manual_ref"] + assert resumed.verify().state == "ready" + completed = json.loads(journals[0].read_text(encoding="utf-8")) + assert completed["state"] == "complete" + for key in ("maintenance_window_id", "source_fingerprint", "manual_ref"): + assert completed[key] == interrupted[key] + + def history(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]): + assert connection is not None + return connection.exec_driver_sql( + "SELECT content FROM pc_artifacts WHERE scope_id='historical' AND artifact_id='preserved'" + ).scalar_one() + + assert backend.connections().run(history) == b'{"history":"unchanged"}' + assert not _apply(_full_server_runner(backend), policy="manual").changed + + def _interrupt_after_committed_citation_ddl(backend: _BackendTarget) -> None: _load_baseline(backend) _interrupt_after_ddl(backend.runner(), "ALTER TABLE pc_artifacts ADD COLUMN memory_citations") diff --git a/tests/fixtures/database_migrations/crash_after_registered_ddl.py b/tests/fixtures/database_migrations/crash_after_registered_ddl.py new file mode 100644 index 0000000000..49915fdf24 --- /dev/null +++ b/tests/fixtures/database_migrations/crash_after_registered_ddl.py @@ -0,0 +1,62 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Terminate a migration client after the remote backend completes registered DDL.""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +from pydantic import SecretStr +from sqlalchemy import Engine, event + +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from tests.builtin.persistence.test_mysql_migration_runner import _BackendTarget, _full_server_runner + + +def main() -> None: + evidence = Path(sys.argv[1]) + backend = _BackendTarget( + OceanBaseConfig(url=SecretStr(os.environ["POWERCONTEXT_MIGRATION_CRASH_TARGET"])), evidence + ) + + @event.listens_for(Engine, "after_cursor_execute") + def exit_after_ddl(connection, _cursor, statement, _parameters, _context, _executemany): + if statement.lstrip().startswith("CREATE TABLE pc_retained_dream_candidate_versions"): + connection_id = connection.exec_driver_sql("SELECT CONNECTION_ID()").scalar_one() + with (evidence / "client-exit.json").open("w", encoding="utf-8") as marker: + json.dump({"connection_id": connection_id}, marker) + marker.flush() + os.fsync(marker.fileno()) + # Bypass Python cleanup and leave the remote interruption unacknowledged. + os._exit(73) + + runner = _full_server_runner(backend) + plan = runner.plan(backup_policy="manual") + runner.apply( + plan_id=plan.plan_id, + accepted=True, + maintenance_confirmed=True, + backup_policy="manual", + backup_confirmed=True, + backup_ref="operator://manual-backup", + ) + raise AssertionError("The registered DDL exit point was not reached") # noqa: TRY003 + + +if __name__ == "__main__": + main() From d64d6b54ce84b22bacbb712e8f3d813f57315a32 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 05:53:08 +0800 Subject: [PATCH 68/94] ci(migrations): budget the full native Server acceptance suite Split OceanBase recovery and maintenance into two 600-second groups while requiring all ten cases without skips. Increase the seekDB aggregate step to 15 minutes, retaining each isolated runner case's 180-second deadline. Constraint: no case, assertion or native per-case deadline is removed. Scope-risk: native CI scheduling budget; runtime sources are unchanged. Tested: complete OceanBase selection (10 passed in 614.61 seconds); failed CI logs show 20/21 seekDB cases passed before the 10-minute aggregate deadline; workflow YAML hooks and embedded Python syntax passed. Refs: #2084, #2154 --- .github/workflows/e2e-harness.yml | 24 +++++++++++++++++++----- .github/workflows/master.yml | 4 +++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 60a97721b8..162ff95e89 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -185,20 +185,34 @@ jobs: -Dpowercontext_test -e 'CREATE DATABASE pc_migration_probe_server_recovery' timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -q -rs \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_full_server_interruption_preserves_history_and_original_maintenance \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_oceanbase_full_server_client_exit_requires_acknowledgement_and_preserves_recovery \ + -o faulthandler_timeout=180 --durations=10 \ + -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-recovery.xml + + - name: Maintain the registered Server chain on real OceanBase without skips + env: + POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- + mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 + run: | + timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -q -rs \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_registered_processing_mode_switch_preserves_revision_and_history \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_atomic_compatibility_command_uses_shared_server_upgrade \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_registered_projection_repair_preserves_authority_and_revision \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_topic_bundle_refuses_managed_server_storage \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_managed_identity_initialization_requires_no_ddl \ - -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-recovery.xml + -o faulthandler_timeout=180 --durations=10 \ + -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-maintenance.xml uv run --locked python - <<'PY' from pathlib import Path from xml.etree import ElementTree - suites = ElementTree.parse(Path(".powercontext/recall-effort-ci/full-server-recovery.xml")).getroot() - cases = list(suites.iter("testcase")) - if len(cases) != 9 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): - raise SystemExit("All nine native full Server recovery and maintenance scenarios must pass without skips.") + reports = ( + Path(".powercontext/recall-effort-ci/full-server-recovery.xml"), + Path(".powercontext/recall-effort-ci/full-server-maintenance.xml"), + ) + cases = [case for report in reports for case in ElementTree.parse(report).iter("testcase")] + if len(cases) != 10 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): + raise SystemExit("All ten native full Server recovery and maintenance scenarios must pass without skips.") PY - name: Collect the database log diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 15678bc55f..c63ae8711d 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -345,7 +345,9 @@ jobs: run: make code-seekdb-test - name: Test unified migrations with embedded seekdb - timeout-minutes: 10 + # The isolated full-Server cases exceed ten minutes in aggregate; + # each runner case retains its separate 180-second deadline below. + timeout-minutes: 15 env: POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" POWERCONTEXT_TEST_MIGRATION_SQL_LOG: "1" From 793861b7b1a6074bd6e4335f2d987ed781e65d2c Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 05:53:08 +0800 Subject: [PATCH 69/94] test(service): retry installed upgrades after backup failure Block the real backup directory for actual installed 1.3.0 native services in both original running states. Require failure to preserve the stopped writer, old registration and source schema, then recover storage and finish upgrading. Constraint: no mocked backup or native manager; the source release and target are separate installed environments. Scope-risk: native service acceptance matrix grows by two cases per platform. Tested: actual Windows Task Scheduler backup-failure/retry variants (2 passed, zero skips); full Linux Python 3.11 Ty and scoped Windows-native hooks passed. Linux/macOS executions require their new-Head native CI results. Refs: #2084, #2154 --- .../native/test_personal_service_lifecycle.py | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/native/test_personal_service_lifecycle.py b/tests/native/test_personal_service_lifecycle.py index 663a0c1564..ca9e23812a 100644 --- a/tests/native/test_personal_service_lifecycle.py +++ b/tests/native/test_personal_service_lifecycle.py @@ -269,8 +269,9 @@ def test_native_public_migration_of_ready_storage_restores_a_running_service( @pytest.mark.parametrize("originally_running", [True, False], ids=["running", "stopped"]) +@pytest.mark.parametrize("backup_blocked", [False, True], ids=["healthy-backup", "backup-retry"]) def test_native_public_migration_switches_an_installed_130_service( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, originally_running: bool + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, originally_running: bool, backup_blocked: bool ) -> None: """Upgrade storage populated by the released service through its actual native manager.""" @@ -335,6 +336,43 @@ def test_native_public_migration_switches_an_installed_130_service( assert reviewed["service"]["originally_running"] is originally_running assert reviewed["service"]["definition"]["python_executable"] == definition.python_executable assert reviewed["service"]["target_definition"]["python_executable"] == service_python_executable() + if backup_blocked: + database = tmp_path / "data" / "powercontext.db" + blocked_directory = database.with_name(database.name + ".pc-migration-backups") + blocked_directory.write_text("Backup storage is unavailable.", encoding="utf-8") + failed = CliRunner().invoke( + cli, + [ + "server", + "db-migrate", + "apply", + "--env-file", + str(environment), + "--manage-service", + "--backup", + "auto", + "--plan-id", + reviewed["plan_id"], + "--yes", + "--maintenance-confirmed", + ], + ) + assert failed.exit_code == 1, failed.output + assert json.loads(failed.output)["error"] == "backup_failed" + assert adapter.manager_state() in {ManagerState.INACTIVE, ManagerState.FAILED} + assert controller.maintenance_path.exists() + assert adapter.inspect().definition == definition + with sqlite3.connect(database.as_uri() + "?mode=ro", uri=True) as unchanged: + assert unchanged.execute( + "SELECT title FROM pc_scopes WHERE scope_id=?", (scope_id,) + ).fetchone() == ("Installed service upgrade",) + assert ( + unchanged.execute("SELECT name FROM sqlite_master WHERE name='pc_schema_revision'").fetchall() + == [] + ) + blocked_directory.unlink() + reviewed = _native_migration_command(cli, environment, "plan") + assert reviewed["service"]["originally_running"] is originally_running result = _native_migration_command( cli, environment, From 3c992051ffd5d565b4d92b8a93192d08895a45d0 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 06:29:54 +0800 Subject: [PATCH 70/94] fix(migrations): preserve separate Code caches through Server upgrades Admit no-vector SQLite and OceanBase Code configurations whose complete installed 1.3.0 inventories match the existing registered baseline. Keep embedded Code and vector storage subject to their own configuration profiles. Require installed-source Code indexing and target HTTP queries with unchanged fingerprints, symbols and cache files in both new release-gate cases. Constraint: Frozen migration revisions and resources remain unchanged. Scope-risk: Vector and embedded Code registration and full failure recovery remain incomplete; release policy stays upgrade-required. Tested: Focused migration/release/maintenance suites: 96 passed; native OceanBase Code upgrade and full SQLite migration selection: 15 passed; full Python 3.11 Ty and scoped Windows hooks passed. --- .../setup-upgrade-oceanbase/action.yml | 2 +- docs/en/docs/operate/database-migrations.md | 14 +-- docs/zh/docs/operate/database-migrations.md | 13 +-- release/upgrade-support.json | 14 ++- scripts/upgrade_release_gate.py | 15 +++ .../persistence/migrations/deployment.py | 6 +- .../migrations/server_readiness.py | 6 +- .../test_full_server_migrations.py | 57 +++++++++++ .../test_mysql_migration_runner.py | 34 ++++++- tests/release_upgrades/test_full_server.py | 98 ++++++++++++++++++- tests/test_upgrade_release_gate.py | 3 + 11 files changed, 241 insertions(+), 21 deletions(-) diff --git a/.github/actions/setup-upgrade-oceanbase/action.yml b/.github/actions/setup-upgrade-oceanbase/action.yml index 949b649610..4c6adca444 100644 --- a/.github/actions/setup-upgrade-oceanbase/action.yml +++ b/.github/actions/setup-upgrade-oceanbase/action.yml @@ -49,7 +49,7 @@ runs: docker exec upgrade-oceanbase \ obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ -Dpc_upgrade_test \ - -e "CREATE DATABASE pc_upgrade_fresh; SET GLOBAL recyclebin=OFF; SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode'; SHOW VARIABLES LIKE 'recyclebin';" + -e "CREATE DATABASE pc_upgrade_fresh; CREATE DATABASE pc_upgrade_code_test; CREATE DATABASE pc_upgrade_code_fresh; SET GLOBAL recyclebin=OFF; SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode'; SHOW VARIABLES LIKE 'recyclebin';" exit 0 fi if [ "$(docker inspect --format '{{.State.Running}}' upgrade-oceanbase)" != true ]; then diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 704fbe3056..ea1b3ee10f 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -14,7 +14,7 @@ The design follows [RFC #1771](https://github.com/oceanbase/powercontext/pull/17 `powercontext server db-migrate status/plan/apply/verify` default to `--bundle server` and use the configured persistent SQLite, embedded seekDB, or OceanBase MySQL database. Explicit environment files are authoritative; arbitrary bundle paths are not accepted. -The full Server path registers the actual 1.3.0 no-vector, Code-disabled configuration as `s1300 → s1400 → s1401`. It covers the complete schema, Candidate renaming and retained history, Dream task formats, Processing ownership, full-text projections, and existing Source references. Unknown layouts, extra objects, vector/Code configurations, and unregistered sources fail before writes. Older Memory collection conversion is outside this path. +The full Server path registers the actual 1.3.0 no-vector configuration as `s1300 → s1400 → s1401`. Code may be enabled on SQLite and OceanBase: its local cache is separate from the Server database and remains unchanged by database maintenance. It covers the complete schema, Candidate renaming and retained history, Dream task formats, Processing ownership, full-text projections, and existing Source references. Unknown layouts, extra objects, vector configurations, embedded seekDB Code storage, and unregistered sources fail before writes. Older Memory collection conversion is outside this path. Only successful schema, domain, projection, and task validation returns `state=ready`, `readiness_scope=server`, and `server_ready=true`. Publication also requires installed-artifact acceptance on all three backends. An implemented path is not evidence that a release passed acceptance; consult the installation guide and release gate for support status. @@ -29,7 +29,7 @@ Ordinary Server startup checks readiness without repairing an existing database. Server identity initialization checks registered tables without issuing DDL on managed storage. Creating the initial identity value and explicitly rotating a stopped clone's identity retain their existing transactional behavior. -Existing fresh deployments outside the no-vector, Code-disabled profile retain domain bootstrap and startup checks; they are not automatically adopted into this chain or covered by its upgrade support. Changing configuration cannot bypass admission or implicitly add vector/Code objects to managed storage. +Existing fresh deployments with vectors or embedded seekDB Code storage retain domain bootstrap and startup checks; they are not automatically adopted into this chain or covered by its upgrade support. Changing configuration cannot bypass admission or implicitly add vector/Code objects to managed storage. The release gate includes separate SQLite and OceanBase Code-enabled cases that build an index with installed 1.3.0, preserve the cache files, and query its original fingerprint and symbols after the upgrade. `--bundle artifacts` retains the partial Artifact prototype chain starting at `p0001`; `--bundle topic-memory-recovery` covers only registered Topic recovery layouts. Both report `server_ready=false`. Neither establishes full Server readiness, and their revisions cannot be stamped as full Server revisions. @@ -67,7 +67,8 @@ that database for Server use. Full profile admission checks the registered Serve | `ServerIdentityRepository.initialize` | Check the identity table without DDL; identity value creation and explicit rotation are ordinary transactions | Retain concurrent-safe identity table initialization | | Offline Memory archive | The Server chain registers its structure at `s1401`; runtime admission checks it | Legacy offline Memory conversion owns its archive and conversion receipts | | `atomic-memory-migrate` and `processing-migrate` | Delegate to the shared Server or registered maintenance plan | Retain their domain-only behavior; recognized historical Server storage must use the full upgrade first | -| Optional vector and Code objects | No registered configuration transition; reject before adding objects | Retain existing feature bootstrap without claiming registered upgrade support | +| Vector and embedded seekDB Code objects | No registered configuration transition; reject before adding objects | Retain existing feature bootstrap without claiming registered upgrade support | +| Separate Code cache on SQLite/OceanBase | Preserve existing cache files; main database follows the shared chain | Code service owns local cache creation and indexing | These helpers remain available to their owning runtime or maintenance operations; calling a low-level helper directly does not replace profile admission, a reviewed maintenance plan, or backup and recovery requirements. Future changes to @@ -77,7 +78,8 @@ initialization; successfully opening an inspection connection does not establish Code's embedded seekDB store adds `pc_code_generations`, `pc_code_nodes`, `pc_code_edges` and `ix_code_search` to the deployment database. Other Code deployments use a local SQLite cache. These are derived indexes owned by the Code -service; their existence does not register a Code-enabled Server baseline or permit implicit managed-schema expansion. +service; their existence does not register an embedded Code Server baseline or permit implicit managed-schema expansion. +Code caches on SQLite/OceanBase do not change the main database layout. ## Processing configuration maintenance @@ -109,7 +111,7 @@ An already matching configuration is a read-only no-op. ## Atomic Memory projection maintenance `--bundle atomic-projection` repairs the current Atomic Memory rows and SQLite FTS helper on a current managed -Server database in the registered no-vector, Code-disabled configuration. It preserves the Server revision, +Server database in the registered no-vector configuration, including Code with a separate cache on SQLite/OceanBase. It preserves the Server revision, immutable content/history, lifecycle, formal tags, permissions and Source/Processing state. ```bash @@ -128,7 +130,7 @@ OceanBase requires the same fixed-host coordination and previous-run acknowledgm Inspection and repair scan the relevant Atomic history, heads, tags, lineage and current projection. Keyset batches bound row buffering; this is an offline whole-projection operation. It does not register vector-profile changes or -optional Code configurations. The compatibility command `server atomic-memory-rebuild-projection` delegates managed +embedded seekDB Code configurations. The compatibility command `server atomic-memory-rebuild-projection` delegates managed databases to this bundle and supports `--action plan/apply/verify` with the same confirmation and backup options. ## Topic Memory recovery bundle diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 05961281e7..12d64f5471 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -12,7 +12,7 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 `powercontext server db-migrate status/plan/apply/verify` 默认选择 `--bundle server`，使用部署配置中的持久 SQLite、嵌入式 seekDB 或 OceanBase MySQL 模式数据库。维护命令以显式环境文件为准，不接受任意测试 bundle 路径。 -完整 Server 路径登记了真实 1.3.0 的无向量、Code 关闭配置，版本链为 `s1300 → s1400 → s1401`。它覆盖完整数据库结构、Candidate 改名与历史保留、Dream 任务格式、Processing 所有权声明、全文检索投影和既有 Source 引用。未知结构、额外对象、不同向量/Code 配置及未登记来源均在写入前拒绝。更早的 Memory 集合转换不属于此路径。 +完整 Server 路径登记了真实 1.3.0 的无向量配置，版本链为 `s1300 → s1400 → s1401`。SQLite、OceanBase 可以启用 Code：其本地缓存独立于 Server 数据库，数据库维护保留缓存文件。此路径覆盖完整数据库结构、Candidate 改名与历史保留、Dream 任务格式、Processing 所有权声明、全文检索投影和既有 Source 引用。未知结构、额外对象、向量配置、嵌入式 seekDB Code 存储及未登记来源均在写入前拒绝。更早的 Memory 集合转换不属于此路径。 只有结构、领域数据、搜索投影和任务校验全部通过，才返回 `state=ready`、`readiness_scope=server`、`server_ready=true`。发布还必须通过真实安装包在三个后端上的完整验收；实现路径存在不等于某次发布已通过验收，支持状态以安装指南和发布门禁为准。 @@ -27,7 +27,7 @@ Candidate 旧表保留为 `pc_retained_dream_candidate_heads` 和 `pc_retained_d Server identity 初始化在受管库上只检查已登记的表，不执行 DDL。首次创建 identity 值以及显式轮换已停止副本的 identity，仍使用原有事务语义。 -无向量、Code 关闭之外的既有新建部署仍使用原领域初始化和启动检查，不会自动纳入此版本链；这些配置不属于已登记升级范围。受管数据库不能通过切换配置绕过检查或隐式增加向量/Code 表。 +启用向量或嵌入式 seekDB Code 存储的既有新建部署仍使用原领域初始化和启动检查，不会自动纳入此版本链；这些配置不属于已登记升级范围。受管数据库不能通过切换配置绕过检查或隐式增加向量/Code 表。发布闸门分别验收 SQLite、OceanBase 的 Code 启用配置：用实际安装的 1.3.0 创建索引，保留缓存文件，并在升级后查询原指纹和符号。 `--bundle artifacts` 保留 `p0001` 开始的局部 Artifact 原型链；`--bundle topic-memory-recovery` 只处理登记的 Topic 恢复布局。两者均报告 `server_ready=false`，不得用于证明完整 Server 就绪，也不能把其版本号直接 stamp 为完整 Server 版本。 @@ -56,7 +56,8 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 | `ServerIdentityRepository.initialize` | 检查 identity 表，不执行 DDL；创建 identity 值和显式轮换仍使用普通事务 | 保留支持并发的 identity 表初始化 | | 离线 Memory 归档 | Server 版本链在 `s1401` 登记结构，运行时准入检查该结构 | 旧 Memory 离线转换负责其归档和转换回执 | | `atomic-memory-migrate`、`processing-migrate` | 转发到共享 Server 或已登记的维护计划 | 保留领域独立行为；识别为历史完整 Server 的库必须先完成整库升级 | -| 可选向量与 Code 对象 | 尚无已登记的配置转换，添加对象前拒绝 | 保留既有功能初始化，不宣称已登记升级支持 | +| 向量与嵌入式 seekDB Code 对象 | 尚无已登记的配置转换，添加对象前拒绝 | 保留既有功能初始化，不宣称已登记升级支持 | +| SQLite、OceanBase 的独立 Code 缓存 | 保留缓存文件，主数据库走共享版本链 | Code 服务负责本地缓存创建和索引构建 | 这些辅助函数仍供所属 runtime 或维护操作使用；直接调用底层函数不能替代 Profile 准入、已确认的维护计划和 备份恢复要求。未来受管对象的变化应加入已登记的 revision、领域步骤与校验器，不另建启动迁移入口。 @@ -64,7 +65,7 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 Code 的嵌入式 seekDB 存储在部署数据库中添加 `pc_code_generations`、`pc_code_nodes`、`pc_code_edges` 和 `ix_code_search`；其他 Code 部署使用本地 SQLite 缓存。这些派生索引由 Code 服务负责，其存在不代表已经登记 -Code 启用的 Server 基线，也不允许隐式扩展受管结构。 +嵌入式 Code 的 Server 基线，也不允许隐式扩展受管结构。SQLite、OceanBase 的 Code 缓存不改变主库结构。 ## Processing 配置维护 @@ -93,7 +94,7 @@ powercontext server db-migrate verify --bundle processing --env-file .env \ ## Atomic Memory 投影维护 `--bundle atomic-projection` 在当前受管 Server 数据库上修复 Atomic Memory current 行和 SQLite FTS -辅助索引，适用于已登记的无向量、Code 关闭配置。它保留 Server revision、不可变正文与历史、生命周期、 +辅助索引，适用于已登记的无向量配置，包括 SQLite、OceanBase 使用独立缓存的 Code 配置。它保留 Server revision、不可变正文与历史、生命周期、 正式标签、权限和 Source/Processing 状态。 ```bash @@ -110,7 +111,7 @@ apply 只修复派生行，完整 Server 校验通过后才提交。权威数据 并在报告上次运行未结束时提供准确的确认 token。 检查和修复扫描相关 Atomic 历史、head、标签、lineage 及 current 投影，通过键集分页限制行缓冲； -这是停服执行的全投影操作，不登记向量 profile 切换或可选 Code 配置。 +这是停服执行的全投影操作，不登记向量 profile 切换或嵌入式 seekDB Code 配置。 兼容命令 `server atomic-memory-rebuild-projection` 在受管数据库上转发到此 bundle， 支持 `--action plan/apply/verify` 和同样的确认、备份选项。 diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 5df9d19237..1f3086e1b9 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "53741f45c16f15237657b31508fea472993d954f07200fe78a09e709d945ed7e", - "rationale": "Register the complete 1.3.0 no-vector, Code-disabled Server chain on all three backends, preserving history, existing Source references and pending intents with projection rebuilds and read-only startup admission. Review master 464cd152: the Atomic Memory snapshot budget changes admission and buffering, preserving the snapshot digest and frozen schema/revision resources. SQLite legacy tag maintenance creates the final table name directly to preserve canonical DDL, tag rows and transactional rollback. New revision s1401 registers the frozen offline Memory archive on all three backends through the shared chain; s1300/s1400 resources and checksums remain unchanged, and archived history is preserved. Publication requires installed-release upgrade, repeat-apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional vector/Code and older Memory layouts need separately registered profiles. Managed Processing mode maintenance uses the canonical target lock, backup providers and durable evidence without changing pc_schema_revision or frozen migration resources. Its reviewed plan binds configuration and migration ID; one transaction publishes the mode and lease invalidation, and interrupted retries reuse the original recovery point. The Processing compatibility command delegates current managed Server storage and rejects complete unversioned historical Server storage before domain writes; unmanaged domain behavior is retained. The Atomic compatibility migration delegates managed databases and recognized complete Server baselines to the shared full Server chain; legacy decision files remain limited to unmanaged conversion. Registered Atomic current repair now shares the Processing maintenance transaction, canonical target lock, reviewed backup and durable evidence; exact managed layout and authority validation precede repair, and full Server validation precedes commit. It preserves revision and immutable authority, repairs SQLite FTS helpers, and resumes from the original recovery point. Vector/Code profiles and complete native lifecycle acceptance remain incomplete. The partial Topic recovery bundle rejects pc_schema_revision before backup or writes so it cannot bypass the managed Server chain, including on unknown revisions. Managed Server identity initialization checks the existing identity table through the shared schema guard without startup DDL; full profile admission remains responsible for bundle readiness. Identity row creation and explicit rotation retain their transactional behavior." + "persistence_sha256": "e7312f2ea747900b9eb99fbcce68618e783baf50fd4e2f7f4ffa7d0b88ceac85", + "rationale": "The shared s1300 -> s1400 -> s1401 chain registers the complete installed 1.3.0 no-vector Server database on SQLite, seekDB and OceanBase. SQLite and OceanBase may enable Code because its local cache is separate from this database; exact installed source inventories match the existing no-vector baselines. Admission preserves that ownership boundary instead of rejecting the flag. The installed-release matrix includes both Code-enabled backends and requires the source-built index to retain its fingerprint, symbols and immutable cache files through target Server queries. Vector profiles and embedded seekDB Code storage still require their own registered transitions. Frozen migration schemas, domain resources and released revision checksums remain unchanged. The chain preserves historical bytes, Source references, Candidate history and pending intents, fences old workers, and validates projections, Dream tasks, Processing ownership and live readiness. Registered maintenance shares the target lock, backup providers and durable recovery evidence; managed startup and identity admission do not perform schema changes. Processing mode changes and Atomic current projection repairs preserve revision and immutable authority, and resume from the original recovery point. Publication requires complete installed-release acceptance, repeated apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional profile and complete native failure recovery acceptance remain incomplete." }, "cases": [], "automatic_sources": { @@ -25,6 +25,16 @@ "backend": "oceanbase", "configuration": "full-server-no-vector", "suite": "tests/release_upgrades/test_full_server.py" + }, + { + "backend": "sqlite", + "configuration": "full-server-code", + "suite": "tests/release_upgrades/test_full_server.py" + }, + { + "backend": "oceanbase", + "configuration": "full-server-code", + "suite": "tests/release_upgrades/test_full_server.py" } ] } diff --git a/scripts/upgrade_release_gate.py b/scripts/upgrade_release_gate.py index 77303d4d3d..aa1bb9685d 100644 --- a/scripts/upgrade_release_gate.py +++ b/scripts/upgrade_release_gate.py @@ -323,6 +323,8 @@ def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dic } with case_workspace(case["backend"], output) as directory: extras = "cli,server,seekdb" if case["backend"] == "seekdb" else "cli,server" + if case["configuration"] == "full-server-code": + extras += ",code" interpreters = {} artifact = case["source_artifact"] packages = { @@ -385,6 +387,19 @@ def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dic ) validate_junit(report) backend = validate_backend_evidence(output / "backend.json", case["backend"]) + require(backend.get("configuration") == case["configuration"], "Configuration evidence differs from the case") + if case["configuration"] == "full-server-code": + code = backend.get("code") + if not isinstance(code, dict): + raise GateError("Installed Code acceptance evidence required") # noqa: TRY003 -- operator diagnostic + require( + re.fullmatch(r"[0-9a-f]{64}", str(code.get("fingerprint", ""))) + and all( + code.get(field) is True + for field in ("immutable_cache_files_preserved", "symbol_ids_preserved", "server_query_ready") + ), + "The installed Code index must remain queryable with its original immutable identity", + ) return {"case": case, "installed": metadata, "database": backend, "status": "passed"} diff --git a/src/powercontext/builtin/persistence/migrations/deployment.py b/src/powercontext/builtin/persistence/migrations/deployment.py index 55a78c9c08..df61330ca6 100644 --- a/src/powercontext/builtin/persistence/migrations/deployment.py +++ b/src/powercontext/builtin/persistence/migrations/deployment.py @@ -89,10 +89,12 @@ def deployment_runner( # noqa: C901 -- Ordered backend admission checks. database = settings.database configuration = _migration_configuration(settings) if bundle in {"server", "processing", "atomic-projection"}: - if settings.inference.embedding_model is not None or settings.code.enabled: + if settings.inference.embedding_model is not None or ( + settings.code.enabled and isinstance(database, SeekDBConfig) + ): raise MigrationError( "unsupported_upgrade_configuration", - "The registered 1.3.0 Server path covers the no-vector, Code-disabled configuration.", + "The registered 1.3.0 Server path requires no vector provider or embedded Code storage.", ) configuration["embedding_enabled"] = False selected = server_bundle() if bundle in {"server", "processing", "atomic-projection"} else production_bundle() diff --git a/src/powercontext/builtin/persistence/migrations/server_readiness.py b/src/powercontext/builtin/persistence/migrations/server_readiness.py index 0b9d4e7424..7d133d18c4 100644 --- a/src/powercontext/builtin/persistence/migrations/server_readiness.py +++ b/src/powercontext/builtin/persistence/migrations/server_readiness.py @@ -59,7 +59,11 @@ def prepare_server_storage(settings, *, injected_embedding=False): database = settings.database if database.kind == "sqlite" and database.is_in_memory: return - if injected_embedding or settings.inference.embedding_model is not None or settings.code.enabled: + if ( + injected_embedding + or settings.inference.embedding_model is not None + or (settings.code.enabled and database.kind == "seekdb") + ): _check_unmanaged_optional_profile(database) return if database.kind == "oceanbase": diff --git a/tests/builtin/persistence/test_full_server_migrations.py b/tests/builtin/persistence/test_full_server_migrations.py index 8b1db6857d..0c9f1bac9c 100644 --- a/tests/builtin/persistence/test_full_server_migrations.py +++ b/tests/builtin/persistence/test_full_server_migrations.py @@ -237,6 +237,63 @@ def test_optional_configuration_cannot_silently_extend_managed_database(source): assert dump(source.database) == before +def test_code_with_separate_cache_upgrades_and_reopens_server(source, tmp_path): + from fastapi.testclient import TestClient + + from powercontext.builtin.code import CodeConfig + from powercontext.builtin.persistence.migrations.deployment import deployment_runner + from powercontext.server.factory import create_server_app + from powercontext.server.settings import McpConfig, ServerSettings + + settings = ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{source.database}"), + code=CodeConfig(enabled=True, cache_dir=tmp_path / "code-cache"), + mcp=McpConfig(enabled=False), + ) + runner = deployment_runner(settings) + assert runner.plan().source_revision == "s1300" + apply(runner) + assert runner.verify().readiness_scope == "server" + app = create_server_app(settings=settings, scheduler_path=tmp_path / "scheduler.db") + with TestClient(app) as client: + assert client.get("/health/ready").status_code == 200 + response = client.get("/v1/scopes") + assert response.status_code == 200 + assert any(scope["scope_id"] == "scope" for scope in response.json()["items"]) + code = client.post("/v1/scopes/scope/code/query", json={"operation": {"kind": "status"}}) + assert code.status_code == 200 + assert code.json()["status"] == "disabled" + assert code.json()["reason"] == "code_repository_unconfigured" + assert not apply(deployment_runner(settings)).changed + with closing(sqlite3.connect(source.database)) as connection: + assert connection.execute("SELECT content FROM pc_artifacts WHERE artifact_id='historical'").fetchall() == [ + (b'{"history":"unchanged"}',) + ] + + +@pytest.mark.parametrize("bundle", ["server", "processing", "atomic-projection"]) +@pytest.mark.parametrize("configuration", ["vector", "embedded-code"]) +def test_unregistered_optional_storage_is_rejected_before_opening_database(tmp_path, bundle, configuration): + from powercontext.builtin.code import CodeConfig + from powercontext.builtin.persistence.migrations.deployment import deployment_runner + from powercontext.builtin.persistence.seekdb import SeekDBConfig + from powercontext.builtin.runtime.config import InferenceConfig + from powercontext.server.settings import ServerSettings + + if configuration == "vector": + settings = ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'server.db'}"), + inference=InferenceConfig( + embedding_model="openai:fixture", embedding_profile_id="fixture", embedding_dimension=3 + ), + ) + else: + settings = ServerSettings(database=SeekDBConfig(path=tmp_path / "embedded"), code=CodeConfig(enabled=True)) + with pytest.raises(MigrationError, match="unsupported_upgrade_configuration"): + deployment_runner(settings, bundle=bundle) + assert list(tmp_path.iterdir()) == [] + + def test_captured_oceanbase_baseline_preserves_ordered_audit_cursor(): import copy diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index aaee862041..c95e9d21fb 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -27,7 +27,7 @@ from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path -from typing import Literal +from typing import Literal, cast from unittest.mock import Mock from uuid import uuid4 @@ -460,6 +460,38 @@ def _full_server_runner( ) +@pytest.mark.parametrize("backend", ("oceanbase",), indirect=True) +def test_oceanbase_server_upgrade_with_separate_code_cache_preserves_history( + backend: _BackendTarget, tmp_path: Path +) -> None: + from powercontext.builtin.code import CodeConfig + from powercontext.builtin.persistence.migrations.server_readiness import prepare_server_storage + + _load_full_server_baseline(backend) + settings = ServerSettings( + database=backend.config, + code=CodeConfig(enabled=True, cache_dir=tmp_path / "code-cache"), + ) + runner = cast( + MySQLMigrationRunner, + deployment_runner(settings, evidence_dir=backend.evidence_directory, lock_coordination="single-host"), + ) + assert runner.plan(backup_policy="manual").source_revision == "s1300" + result = _apply(runner, policy="manual") + assert result.readiness_scope == "server" and result.revision == "s1401" + assert runner.verify().state == "ready" + prepare_server_storage(settings) + assert not _apply(runner, policy="manual").changed + + def history(connection, identity, guard): + guard() + return connection.exec_driver_sql( + "SELECT content FROM pc_artifacts WHERE scope_id='historical' AND artifact_id='preserved'" + ).scalar_one() + + assert backend.connections().run(history) == b'{"history":"unchanged"}' + + def test_mysql_registered_processing_mode_switch_preserves_revision_and_history(backend: _BackendTarget) -> None: _load_full_server_baseline(backend) _apply(_full_server_runner(backend), policy="manual") diff --git a/tests/release_upgrades/test_full_server.py b/tests/release_upgrades/test_full_server.py index 12f1aa5138..53a7be6fe3 100644 --- a/tests/release_upgrades/test_full_server.py +++ b/tests/release_upgrades/test_full_server.py @@ -229,6 +229,15 @@ def environment_for(case, *, fresh=False): "NO_PROXY": "localhost,127.0.0.1", "no_proxy": "localhost,127.0.0.1", }) + if case["configuration"] == "full-server-code": + environment["POWERCONTEXT_SERVER_CODE_ENABLED"] = "true" + environment["POWERCONTEXT_SERVER_CODE_CACHE_DIR"] = str( + case["work"] / ("fresh-code-cache" if fresh else "code-cache") + ) + scope = case.get("code_scope") + environment["POWERCONTEXT_SERVER_CODE_REPOSITORIES"] = json.dumps( + {} if fresh or scope is None else {scope: {"root": str(case["work"] / "repository")}} + ) database = case["work"] / ("fresh" if fresh else "database") backend = case["backend"] environment["POWERCONTEXT_SERVER_DATABASE_KIND"] = backend @@ -243,6 +252,9 @@ def environment_for(case, *, fresh=False): expected = "/pc_upgrade_fresh" if fresh else "/pc_upgrade_test" assert parsed.scheme == "mysql+aoceanbase" and parsed.hostname in {"127.0.0.1", "localhost"} assert parsed.path == expected, "Only dedicated disposable OceanBase databases may be used" + if case["configuration"] == "full-server-code": + parsed = parsed._replace(path="/pc_upgrade_code_fresh" if fresh else "/pc_upgrade_code_test") + url = parsed.geturl() environment["POWERCONTEXT_SERVER_DATABASE_URL"] = url environment["PC_UPGRADE_OCEANBASE_URL"] = url return environment @@ -374,6 +386,11 @@ def seed(client, *, candidate_prefix): def case(model_gateway): # Missing gate inputs are errors, never skips or an implicit local database. assert os.environ["PC_UPGRADE_SOURCE_VERSION"] == "1.3.0", "Add a reviewed adapter for another source release" + configuration = os.environ["PC_UPGRADE_CONFIGURATION"] + assert configuration in {"full-server-no-vector", "full-server-code"}, "Unregistered acceptance configuration" + assert configuration != "full-server-code" or os.environ["PC_UPGRADE_BACKEND"] != "seekdb", ( + "Embedded Code storage requires its own registered migration" + ) return { "source": Path(os.environ["PC_UPGRADE_SOURCE_PYTHON"]), "target": Path(os.environ["PC_UPGRADE_TARGET_PYTHON"]), @@ -381,9 +398,55 @@ def case(model_gateway): "work": Path(os.environ["PC_UPGRADE_WORK"]), "report": Path(os.environ["PC_UPGRADE_REPORT_DIR"]), "model": model_gateway, + "configuration": configuration, + } + + +def code_cache_snapshot(case): + directory = case["work"] / "code-cache" + return { + path.relative_to(directory).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(directory.rglob("*")) + if path.is_file() and path.suffix in {".json", ".sqlite"} } +def historical_code(case, scope): + repository = case["work"] / "repository" + repository.mkdir() + (repository / "evidence.py").write_text("def upgrade_code_evidence():\n return 2084\n", encoding="utf-8") + for arguments in (("init",), ("add", ".")): + command( + ["git", "-C", str(repository), *arguments], + environment_for(case), + case["work"], + case["report"] / f"code-git-{arguments[0]}.log", + ) + case["code_scope"] = scope + indexed = json.loads( + command( + [console(case["source"]), "code", "index", "--scope", scope], + environment_for(case), + case["work"], + case["report"] / "source-code-index.log", + ) + ) + with server( + case["source"], environment_for(case), case["work"], case["report"] / "source-code-server.log" + ) as client: + result = post( + client, + f"/v1/scopes/{scope}/code/query", + {"operation": {"kind": "symbols", "query": "upgrade_code_evidence"}}, + ) + assert result["status"] == "ready" and result["fingerprint"] == indexed["fingerprint"] + symbols = [item for item in result["items"] if item["name"] == "upgrade_code_evidence"] + assert symbols and all(item["path"] == "evidence.py" for item in symbols) + cache = code_cache_snapshot(case) + assert any(name.endswith("/graph.sqlite") for name in cache) + return {"fingerprint": result["fingerprint"], "symbols": symbols, "cache_sha256": cache} + + @pytest.fixture(scope="session") def historical(case): if case["backend"] == "oceanbase": @@ -391,6 +454,8 @@ def historical(case): assert not snapshot(case, "target", "fresh-empty", fresh=True)["tables"] with server(case["source"], environment_for(case), case["work"], case["report"] / "source-server.log") as client: value = seed(client, candidate_prefix="/v1/artifact-candidates") + if case["configuration"] == "full-server-code": + value["code"] = historical_code(case, value["scope"]) value["task"] = task_state(case, "source", "seed", value["scope"]) value["schema"] = snapshot(case, "source", "source") assert {"pc_artifact_candidate_heads", "pc_artifact_candidate_versions", "pc_scopes", "pc_artifacts"} <= set( @@ -491,11 +556,38 @@ def migration(operation, *options, label=""): @pytest.fixture -def target(case, migrated): +def target(case, migrated, preserved_code): with server(case["target"], environment_for(case), case["work"], case["report"] / "target-server.log") as client: yield client +@pytest.fixture(scope="session") +def preserved_code(case, migrated): + if case["configuration"] != "full-server-code": + return None + historical = migrated["historical"] + source = historical["code"] + assert code_cache_snapshot(case) == source["cache_sha256"] + with server( + case["target"], environment_for(case), case["work"], case["report"] / "target-code-server.log" + ) as client: + result = post( + client, + f"/v1/scopes/{historical['scope']}/code/query", + {"operation": {"kind": "symbols", "query": "upgrade_code_evidence"}}, + ) + assert result["status"] == "ready" and result["fingerprint"] == source["fingerprint"] + symbols = [item for item in result["items"] if item["name"] == "upgrade_code_evidence"] + assert symbols == source["symbols"] + assert code_cache_snapshot(case) == source["cache_sha256"] + return { + "fingerprint": source["fingerprint"], + "immutable_cache_files_preserved": True, + "symbol_ids_preserved": True, + "server_query_ready": True, + } + + @pytest.fixture(scope="session") def processed_background(case, migrated): historical = migrated["historical"] @@ -665,7 +757,7 @@ def normalize(sql): } -def test_fresh_upgrade_equivalence(case, migrated, processed_background): +def test_fresh_upgrade_equivalence(case, migrated, processed_background, preserved_code): if case["backend"] == "oceanbase": environment = environment_for(case, fresh=True) env_file = case["work"] / "fresh.env" @@ -731,6 +823,8 @@ def test_fresh_upgrade_equivalence(case, migrated, processed_background): "target_schema": digest(actual), "fresh_schema": digest(expected), "background_processing": processed_background, + "configuration": case["configuration"], + "code": preserved_code, } if case["backend"] == "oceanbase": evidence.update(tenant_mode=inventory["tenant_mode"], proxy_version=inventory["proxy_version"]) diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py index 290db8df0f..618e1b3d12 100644 --- a/tests/test_upgrade_release_gate.py +++ b/tests/test_upgrade_release_gate.py @@ -387,6 +387,9 @@ def test_repository_baseline_covers_all_three_databases(): assert selection["strategy"] == "fixed-version" assert selection["source_version"] == "1.3.0" assert {profile["backend"] for profile in selection["profiles"]} == {"sqlite", "seekdb", "oceanbase"} + assert { + profile["backend"] for profile in selection["profiles"] if profile["configuration"] == "full-server-code" + } == {"sqlite", "oceanbase"} @pytest.mark.parametrize("workflow_name,job_name", [("master", "upgrade-release-gate"), ("release", "release-build")]) From cb40660ac56c31dd98c296f8552f2dc2bf397a9e Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 06:44:57 +0800 Subject: [PATCH 71/94] test(upgrades): accept installed Code configuration and query status Supply Code settings as a JSON object so installed 1.3.0 receives a strict boolean. Assert the documented query status and reserve isolated local OceanBase Code database names for repeatable acceptance. Constraint: preserve old Code fingerprints, symbol identities and cache bytes. Scope-risk: installed upgrade acceptance only; runtime payload is unchanged. Tested: installed 1.3.0 to 3c992051 Code upgrade, SQLite 11 passed and actual OceanBase 11 passed, zero skips; scoped prek hooks; full Linux Python 3.11 Ty; git diff --check. --- tests/release_upgrades/test_full_server.py | 27 +++++++++++++--------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/release_upgrades/test_full_server.py b/tests/release_upgrades/test_full_server.py index 53a7be6fe3..b7b8e99f90 100644 --- a/tests/release_upgrades/test_full_server.py +++ b/tests/release_upgrades/test_full_server.py @@ -230,14 +230,12 @@ def environment_for(case, *, fresh=False): "no_proxy": "localhost,127.0.0.1", }) if case["configuration"] == "full-server-code": - environment["POWERCONTEXT_SERVER_CODE_ENABLED"] = "true" - environment["POWERCONTEXT_SERVER_CODE_CACHE_DIR"] = str( - case["work"] / ("fresh-code-cache" if fresh else "code-cache") - ) scope = case.get("code_scope") - environment["POWERCONTEXT_SERVER_CODE_REPOSITORIES"] = json.dumps( - {} if fresh or scope is None else {scope: {"root": str(case["work"] / "repository")}} - ) + environment["POWERCONTEXT_SERVER_CODE"] = json.dumps({ + "enabled": True, + "cache_dir": str(case["work"] / ("fresh-code-cache" if fresh else "code-cache")), + "repositories": {} if fresh or scope is None else {scope: {"root": str(case["work"] / "repository")}}, + }) database = case["work"] / ("fresh" if fresh else "database") backend = case["backend"] environment["POWERCONTEXT_SERVER_DATABASE_KIND"] = backend @@ -251,10 +249,17 @@ def environment_for(case, *, fresh=False): parsed = urlsplit(url) expected = "/pc_upgrade_fresh" if fresh else "/pc_upgrade_test" assert parsed.scheme == "mysql+aoceanbase" and parsed.hostname in {"127.0.0.1", "localhost"} - assert parsed.path == expected, "Only dedicated disposable OceanBase databases may be used" if case["configuration"] == "full-server-code": - parsed = parsed._replace(path="/pc_upgrade_code_fresh" if fresh else "/pc_upgrade_code_test") + code_path = "/pc_upgrade_code_fresh" if fresh else "/pc_upgrade_code_test" + if parsed.path == expected: + parsed = parsed._replace(path=code_path) + else: + assert re.fullmatch(re.escape(code_path) + r"_[0-9a-f]{8}", parsed.path), ( + "Only dedicated disposable Code databases may be used" + ) url = parsed.geturl() + else: + assert parsed.path == expected, "Only dedicated disposable OceanBase databases may be used" environment["POWERCONTEXT_SERVER_DATABASE_URL"] = url environment["PC_UPGRADE_OCEANBASE_URL"] = url return environment @@ -439,7 +444,7 @@ def historical_code(case, scope): f"/v1/scopes/{scope}/code/query", {"operation": {"kind": "symbols", "query": "upgrade_code_evidence"}}, ) - assert result["status"] == "ready" and result["fingerprint"] == indexed["fingerprint"] + assert result["status"] == "ok" and result["fingerprint"] == indexed["fingerprint"] symbols = [item for item in result["items"] if item["name"] == "upgrade_code_evidence"] assert symbols and all(item["path"] == "evidence.py" for item in symbols) cache = code_cache_snapshot(case) @@ -576,7 +581,7 @@ def preserved_code(case, migrated): f"/v1/scopes/{historical['scope']}/code/query", {"operation": {"kind": "symbols", "query": "upgrade_code_evidence"}}, ) - assert result["status"] == "ready" and result["fingerprint"] == source["fingerprint"] + assert result["status"] == "ok" and result["fingerprint"] == source["fingerprint"] symbols = [item for item in result["items"] if item["name"] == "upgrade_code_evidence"] assert symbols == source["symbols"] assert code_cache_snapshot(case) == source["cache_sha256"] From f22862e4b34d00a292610f884f2d707818d88929 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 07:00:42 +0800 Subject: [PATCH 72/94] test(code): initialize Server storage before SDK fixture writes Bootstrap each shared Code fixture through the Server lifespan before the SDK writes Scope and Artifact data. Separate Code caches now use registered Server admission, so an unregistered SDK-only database is not a Server fixture. Constraint: retain strict schema admission and every existing behavior assertion. Scope-risk: three test setups only; runtime and migration resources are unchanged. Tested: all three failures reproduced before setup correction; related Linux Code suites 62 passed and 3 skipped; scoped Ruff, formatting and Python 3.11 Ty passed. --- tests/builtin/test_native_code.py | 6 +++++- tests/e2e/test_native_code_history.py | 9 +++++++++ tests/e2e/test_native_code_multilanguage.py | 6 ++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/builtin/test_native_code.py b/tests/builtin/test_native_code.py index 299ec69270..0872d6a49a 100644 --- a/tests/builtin/test_native_code.py +++ b/tests/builtin/test_native_code.py @@ -759,7 +759,8 @@ def test_http_client_mcp_and_prepared_code_share_freshness(repository, tmp_path) settings=ServerSettings( database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'http.db'}"), code=code, - ) + ), + scheduler_path=tmp_path / "bootstrap-scheduler.db", ) def http_client(headers=None, timeout=None, auth=None, **_): @@ -774,6 +775,9 @@ def http_client(headers=None, timeout=None, auth=None, **_): async def exercise(): nonlocal app + # Initialize Server-owned storage before seeding it through the SDK. + async with app.router.lifespan_context(app): + pass database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'http.db'}") async with open_builtin_runtime( BuiltinConfig(database=database), scheduler_path=tmp_path / "scheduler.db" diff --git a/tests/e2e/test_native_code_history.py b/tests/e2e/test_native_code_history.py index b39231baf8..65f9a4ea2e 100644 --- a/tests/e2e/test_native_code_history.py +++ b/tests/e2e/test_native_code_history.py @@ -58,6 +58,15 @@ def test_http_code_prepare_preserves_approved_experience_when_memory_is_empty(tm lesson = "Budget preparation must preserve approved historical evidence." async def exercise(): + bootstrap = create_server_app( + settings=ServerSettings( + database=database, mcp=McpConfig(enabled=False), metrics=MetricsConfig(enabled=False) + ), + scheduler_path=tmp_path / "bootstrap-scheduler.db", + ) + # Initialize Server-owned storage before seeding it through the SDK. + async with bootstrap.router.lifespan_context(bootstrap): + pass async with open_builtin_runtime( BuiltinConfig(database=database), scheduler_path=tmp_path / "runtime-scheduler.db" ) as runtime: diff --git a/tests/e2e/test_native_code_multilanguage.py b/tests/e2e/test_native_code_multilanguage.py index 45fecd8bb6..a29dac15ba 100644 --- a/tests/e2e/test_native_code_multilanguage.py +++ b/tests/e2e/test_native_code_multilanguage.py @@ -51,6 +51,12 @@ def test_prepare_uses_each_language_and_drops_stale_code(tmp_path): database = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'database.db'}") async def exercise(): + bootstrap = create_server_app( + settings=ServerSettings(database=database), scheduler_path=tmp_path / "bootstrap-scheduler.db" + ) + # Initialize Server-owned storage before seeding it through the SDK. + async with bootstrap.router.lifespan_context(bootstrap): + pass async with open_builtin_runtime( BuiltinConfig(database=database), scheduler_path=tmp_path / "scope.db" ) as runtime: From 9f20986acf1e3719a7c9927b8708e84e195ebb9d Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 07:03:35 +0800 Subject: [PATCH 73/94] test(service): recover installed upgrades after switch and startup failures Exercise an installed 1.3.0 service with a real native manager through a port conflict after migration and an injected filesystem ENOSPC at the registration replacement boundary. Retain stopped service recovery evidence, verify the ready database without writes, and recover through explicit start. Constraint: keep migrations, registration, probes and service execution real. Scope-risk: acceptance only; ENOSPC is injected and the port listener is real. Tested: Windows Task Scheduler six-case installed-source matrix 6 passed with zero skips; scoped hooks and Python 3.11 Ty passed; full Ty passed with the cross-host fixture directory supplied as a module search path. --- .../native/test_personal_service_lifecycle.py | 104 ++++++++++++++---- 1 file changed, 83 insertions(+), 21 deletions(-) diff --git a/tests/native/test_personal_service_lifecycle.py b/tests/native/test_personal_service_lifecycle.py index ca9e23812a..821d61dfab 100644 --- a/tests/native/test_personal_service_lifecycle.py +++ b/tests/native/test_personal_service_lifecycle.py @@ -14,6 +14,7 @@ from __future__ import annotations +import errno import json import os import plistlib @@ -25,8 +26,8 @@ import time import uuid import xml.etree.ElementTree as ET -from collections.abc import Callable -from contextlib import closing, suppress +from collections.abc import Callable, Iterator +from contextlib import closing, contextmanager, suppress from importlib.metadata import version from pathlib import Path from typing import Any @@ -268,10 +269,13 @@ def test_native_public_migration_of_ready_storage_restores_a_running_service( _cleanup(adapter) -@pytest.mark.parametrize("originally_running", [True, False], ids=["running", "stopped"]) -@pytest.mark.parametrize("backup_blocked", [False, True], ids=["healthy-backup", "backup-retry"]) +@pytest.mark.parametrize( + ("originally_running", "failure"), + [(True, None), (False, None), (True, "backup"), (False, "backup"), (True, "startup"), (True, "switch")], + ids=["running", "stopped", "backup-retry-running", "backup-retry-stopped", "startup-retry", "switch-retry"], +) def test_native_public_migration_switches_an_installed_130_service( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, originally_running: bool, backup_blocked: bool + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, originally_running: bool, failure: str | None ) -> None: """Upgrade storage populated by the released service through its actual native manager.""" @@ -336,8 +340,8 @@ def test_native_public_migration_switches_an_installed_130_service( assert reviewed["service"]["originally_running"] is originally_running assert reviewed["service"]["definition"]["python_executable"] == definition.python_executable assert reviewed["service"]["target_definition"]["python_executable"] == service_python_executable() - if backup_blocked: - database = tmp_path / "data" / "powercontext.db" + database = tmp_path / "data" / "powercontext.db" + if failure == "backup": blocked_directory = database.with_name(database.name + ".pc-migration-backups") blocked_directory.write_text("Backup storage is unavailable.", encoding="utf-8") failed = CliRunner().invoke( @@ -373,20 +377,50 @@ def test_native_public_migration_switches_an_installed_130_service( blocked_directory.unlink() reviewed = _native_migration_command(cli, environment, "plan") assert reviewed["service"]["originally_running"] is originally_running - result = _native_migration_command( - cli, - environment, - "apply", - "--backup", - "auto", - "--plan-id", - reviewed["plan_id"], - "--yes", - "--maintenance-confirmed", - ) - assert result["server_ready"] is True - assert result["backup_state"] == "completed" - assert result["backup_ref"] and Path(result["backup_ref"]).is_file() + with _native_upgrade_fault(adapter, port, failure, monkeypatch): + applied = CliRunner().invoke( + cli, + [ + "server", + "db-migrate", + "apply", + "--env-file", + str(environment), + "--manage-service", + "--backup", + "auto", + "--plan-id", + reviewed["plan_id"], + "--yes", + "--maintenance-confirmed", + ], + ) + result = json.loads(applied.output) + if failure in {"startup", "switch"}: + assert applied.exit_code == 1, applied.output + assert result["error"] == "service_start_failed" + assert result["database_state"] == "ready" and result["revision"] == "s1401" + assert adapter.manager_state() in {ManagerState.INACTIVE, ManagerState.FAILED} + phase = "switching" if failure == "switch" else "start_failed" + assert json.loads(controller.maintenance_path.read_text())["phase"] == phase + if failure == "switch": + assert adapter.inspect().definition == definition + else: + with socket.create_connection(("127.0.0.1", port), timeout=2): + pass + else: + assert applied.exit_code == 0, f"{applied.output}\n{applied.exception!r}" + assert result["server_ready"] is True + assert result["backup_state"] == "completed" + assert result["backup_ref"] and Path(result["backup_ref"]).is_file() + if failure in {"startup", "switch"}: + before = database.read_bytes() + with server_settings_context(env_file=environment) as settings: + assert deployment_runner(settings).verify().state == "ready" + assert database.read_bytes() == before + # Recovery starts the verified registration without another + # migration apply or backup. + assert controller.start().ok, _failure_diagnostics(adapter, tmp_path) installed = adapter.inspect().definition assert installed is not None assert installed.python_executable == service_python_executable() @@ -408,6 +442,34 @@ def test_native_public_migration_switches_an_installed_130_service( _cleanup(adapter) +@contextmanager +def _native_upgrade_fault( + adapter: NativeServiceAdapter, port: int, failure: str | None, monkeypatch: pytest.MonkeyPatch +) -> Iterator[None]: + with closing(socket.socket()) as listener, monkeypatch.context() as fault: + if failure == "startup": + native_start = adapter.start + + def occupy_then_start(*, reload_definition: bool) -> None: + # Bind after migration and stopped checks, then let the actual + # native manager attempt to start the registered executable. + listener.bind(("127.0.0.1", port)) + listener.listen(16) + native_start(reload_definition=reload_definition) + + fault.setattr(adapter, "start", occupy_then_start) + elif failure == "switch": + native_replace = os.replace + + def fail_definition_replace(source: str | os.PathLike[str], destination: str | os.PathLike[str]) -> None: + if Path(destination) == adapter.artifact_path: + raise OSError(errno.ENOSPC, "Test registration storage is full") + native_replace(source, destination) + + fault.setattr(os, "replace", fail_definition_replace) + yield + + def _native_migration_command(cli, environment: Path, action: str, *options: str) -> dict[str, Any]: result = CliRunner().invoke( cli, ["server", "db-migrate", action, "--env-file", str(environment), "--manage-service", *options] From 772ab5e9616122928db558bea12532c934b304fc Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 07:31:47 +0800 Subject: [PATCH 74/94] test(migrations): verify recovery after an OceanBase engine crash Kill the explicitly owned observer after registered DDL and reopen its original data directory, then verify acknowledgement, recovery identity, historical data and idempotency. Add a separate CI gate that requires an executed test with zero skips and document native service failure recovery. Constraint: Preserve frozen migration resources and distinguish source fixtures from installed-release and backup restoration acceptance. Scope-risk: Opt-in destructive test is bound to an exact disposable container ID and its loopback port; runtime behavior is unchanged. Tested: Native engine recovery 1 passed, existing client-exit recovery 1 passed, documentation and CI selection 116 passed/4 skipped, scoped hooks and full Python 3.11 Ty passed. --- .github/workflows/master.yml | 59 +++++ docs/en/docs/operate/database-migrations.md | 13 ++ docs/zh/docs/operate/database-migrations.md | 10 + .../crash_after_registered_ddl.py | 20 +- .../native/test_oceanbase_engine_recovery.py | 211 ++++++++++++++++++ 5 files changed, 309 insertions(+), 4 deletions(-) create mode 100644 tests/native/test_oceanbase_engine_recovery.py diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index c63ae8711d..8e244a6a68 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -66,6 +66,65 @@ jobs: if-no-files-found: error retention-days: 30 + oceanbase-engine-recovery: + name: OceanBase engine crash recovery + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').tests) }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + env: + POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_engine_crash?charset=utf8mb4 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Set up the environment + uses: ./.github/actions/setup-python-env + - name: Start an isolated OceanBase engine + uses: ./.github/actions/setup-upgrade-oceanbase + - name: Bind the destructive probe to this job's owned container + run: | + docker exec upgrade-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ + -e 'CREATE DATABASE pc_migration_probe_engine_crash;' + container_id=$(docker inspect --format '{{.Id}}' upgrade-oceanbase) + [[ "$container_id" =~ ^[0-9a-f]{64}$ ]] + echo "POWERCONTEXT_TEST_MIGRATION_OCEANBASE_ENGINE_CRASH_ID=$container_id" >> "$GITHUB_ENV" + - name: Verify migration recovery after observer SIGKILL + run: | + uv run pytest -q tests/native/test_oceanbase_engine_recovery.py \ + --junitxml "$RUNNER_TEMP/oceanbase-engine-recovery/junit.xml" \ + --basetemp "$RUNNER_TEMP/oceanbase-engine-recovery/cases" + uv run python - <<'PY' + import os + from pathlib import Path + from xml.etree import ElementTree + path = Path(os.environ["RUNNER_TEMP"]) / "oceanbase-engine-recovery/junit.xml" + suites = list(ElementTree.parse(path).getroot().iter("testsuite")) + assert sum(int(s.get("tests", 0)) for s in suites) == 1 + assert all(int(s.get(key, 0)) == 0 for s in suites for key in ("failures", "errors", "skipped")) + PY + - name: Report recovery failures + if: failure() + run: >- + uv run python .github/scripts/report_pytest_failures.py + "$RUNNER_TEMP/oceanbase-engine-recovery/junit.xml" + - name: Preserve diagnostics and stop the owned engine + if: always() + run: | + mkdir -p "$RUNNER_TEMP/oceanbase-engine-recovery" + docker logs upgrade-oceanbase > "$RUNNER_TEMP/oceanbase-engine-recovery/container.log" 2>&1 || true + docker stop --timeout 30 upgrade-oceanbase || true + - name: Save engine recovery evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: oceanbase-engine-recovery-${{ github.sha }} + path: ${{ runner.temp }}/oceanbase-engine-recovery/ + if-no-files-found: error + retention-days: 7 + quality: timeout-minutes: 20 runs-on: ubuntu-latest diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index ea1b3ee10f..e9848df013 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -424,6 +424,10 @@ running state. The CI setup verifies the source wheel's SHA-256 and supplies cover SQLite service switching. Backend crash recovery and the full installed-release domain suite remain separate acceptance requirements. +Failure cases cover unavailable backup storage, an actual port conflict when the target service starts, and injected +`ENOSPC` when replacing its registration file. Startup/switch failures retain the stopped service and recovery record +after database verification; an explicit service start completes recovery without another migration apply. + For manual deployment, stop the old service first. After a supported complete migration succeeds, ensure the launch definition points to the new environment before starting it. For foreground deployment, exit the old process and use the existing command from the new environment: @@ -465,6 +469,15 @@ or finding an unacknowledged previous run blocks continuation. Unavailable native file locking is rejected, including runtime fallback to a soft lock; it does not reconnect or silently weaken coordination and continue. These locks do not stop business writers. +The separate Linux OceanBase engine-recovery test requires +`POWERCONTEXT_TEST_MIGRATION_OCEANBASE_ENGINE_CRASH_ID` with the exact disposable container ID and +`POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL` with its matching loopback port and dedicated `pc_migration_probe_*` database. +It sends `SIGKILL` to observer after registered DDL, then restarts observer in the same directory and verifies the original +recovery record, required acknowledgement, historical bytes, final revision and repeated apply. It does not restart the +slim image's bootstrap script, which unfolds a snapshot and truncates logs. CI requires one executed test and zero skips. +The test uses frozen historical resources and an operator backup declaration; it does not establish installed-release +acceptance or verified backup restoration. + seekdb/OceanBase DDL may commit implicitly, so one Python transaction cannot make the entire upgrade atomic. Recovery recognizes only the registered sequence: `p0001` creates the four frozen tables and two indexes; `p0002` adds the two nullable `MEDIUMBLOB` citation columns; `p0003` removes the tag-family CHECK. Actual schema, data checks, and persisted diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 12d64f5471..563c6eaaa6 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -319,6 +319,9 @@ CI 安装时校验来源 wheel 的 SHA-256，并设置 `POWERCONTEXT_NATIVE_UPGR 本地运行原生测试时也需提供该已安装来源版本的解释器。这些用例覆盖 SQLite 服务切换， 后端崩溃恢复与完整安装版本领域验收仍需分别验证。 +失败场景覆盖备份存储不可用、目标服务启动时真实的端口冲突，以及替换服务注册文件时注入的 `ENOSPC`。 +启动或切换失败后，保留已停止的服务和恢复记录；数据库验证已经完成时，通过显式启动服务恢复，无需再次执行迁移 apply。 + 手动管理时，先停旧服务，在已支持的完整迁移成功后确认服务启动定义指向新版，再启动。前台部署则退出旧进程，使用新版环境的现有入口： ```bash @@ -335,6 +338,13 @@ powercontext server run --env-file deployment.env --role all ## 存储层与恢复边界 +独立的 Linux OceanBase 引擎恢复测试要求通过 `POWERCONTEXT_TEST_MIGRATION_OCEANBASE_ENGINE_CRASH_ID` +提供隔离容器的完整 ID，并通过 `POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL` 提供与其匹配的回环端口及专用 +`pc_migration_probe_*` 数据库。测试在已登记 DDL 完成后向 observer 发送 `SIGKILL`，从原目录重启 observer， +验证原恢复记录、必需的确认 token、历史数据字节、最终 revision 和重复执行。它不重新执行 slim 镜像的启动脚本， +因为该脚本会展开预置快照并截断日志。CI 要求这一个测试实际执行且零跳过。 +测试使用冻结历史资源和操作员备份声明，不能替代安装版本升级验收，也不代表已验证备份可恢复。 + 维护执行器复用配置，使用专用连接承载 Alembic；冻结历史 SQL 不依赖当前 Repository 或应用 metadata。seekdb 从启动引擎前到关闭引擎后持有规范化目录的锁。OceanBase 在每条 DDL 前后验证操作系统锁、主机绑定、持久运行记录及维护连接。正常结束时先关闭迁移主连接，才清理运行记录并释放文件锁；引擎随后关闭。执行中断或关闭连接失败均保留记录；后续执行需要准确的旧运行 token，以及 DBA 确认远端执行已结束的人工声明。PC 不从本机进程退出或本地锁可获取推断远端执行结束。锁文件被替换、连接丢失或旧运行未确认时阻止继续。平台不支持原生文件锁时拒绝执行，包括运行时降级为软锁的情况，不重连或静默弱化互斥继续迁移。这些锁不阻止业务写入者。 seekdb／OceanBase DDL 可能隐式提交，不能用 Python 事务承诺整个迁移原子化。局部 Artifact 原型只识别已登记顺序：`p0001` 创建冻结四表及两个索引；`p0002` 添加两个可空的 `MEDIUMBLOB` 引用列；`p0003` 删除标签 family 的 CHECK；`p0004` 添加指向精确 revision 的 Artifact handle。继续执行前须用实际结构、数据检查和持久维护证据证明已完成的已知前缀；revision 的后置条件通过后才推进 Alembic 版本。未知对象、结构或中间状态拒绝执行，不乐观 stamp。 diff --git a/tests/fixtures/database_migrations/crash_after_registered_ddl.py b/tests/fixtures/database_migrations/crash_after_registered_ddl.py index 49915fdf24..303f6c9f9b 100644 --- a/tests/fixtures/database_migrations/crash_after_registered_ddl.py +++ b/tests/fixtures/database_migrations/crash_after_registered_ddl.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Terminate a migration client after the remote backend completes registered DDL.""" +"""Interrupt a migration client or its engine after registered DDL completes.""" from __future__ import annotations import json import os import sys +import time from pathlib import Path from pydantic import SecretStr @@ -38,12 +39,23 @@ def main() -> None: def exit_after_ddl(connection, _cursor, statement, _parameters, _context, _executemany): if statement.lstrip().startswith("CREATE TABLE pc_retained_dream_candidate_versions"): connection_id = connection.exec_driver_sql("SELECT CONNECTION_ID()").scalar_one() - with (evidence / "client-exit.json").open("w", encoding="utf-8") as marker: + wait_for_engine = os.environ.get("POWERCONTEXT_MIGRATION_WAIT_FOR_ENGINE_CRASH") == "1" + marker_name = "engine-crash-ready.json" if wait_for_engine else "client-exit.json" + with (evidence / marker_name).open("w", encoding="utf-8") as marker: json.dump({"connection_id": connection_id}, marker) marker.flush() os.fsync(marker.fileno()) - # Bypass Python cleanup and leave the remote interruption unacknowledged. - os._exit(73) + if not wait_for_engine: + # Bypass cleanup and leave the remote interruption unacknowledged. + os._exit(73) + deadline = time.monotonic() + 180 + while not (evidence / "engine-stopped").exists(): + if time.monotonic() >= deadline: + raise TimeoutError("The parent did not confirm engine termination") # noqa: TRY003 + time.sleep(0.05) + # Require a real connection failure after the parent kills observer. + connection.exec_driver_sql("SELECT 1") + raise AssertionError("The migration connection survived engine termination") # noqa: TRY003 runner = _full_server_runner(backend) plan = runner.plan(backup_policy="manual") diff --git a/tests/native/test_oceanbase_engine_recovery.py b/tests/native/test_oceanbase_engine_recovery.py new file mode 100644 index 0000000000..af1ebe5b1d --- /dev/null +++ b/tests/native/test_oceanbase_engine_recovery.py @@ -0,0 +1,211 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Actual observer crash/reopen acceptance in an explicitly owned disposable container.""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import time +from collections.abc import Iterator +from pathlib import Path +from uuid import uuid4 + +import pytest +from pydantic import SecretStr +from sqlalchemy.engine import make_url + +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from tests.builtin.persistence.test_mysql_migration_runner import ( + _apply, + _BackendTarget, + _full_server_runner, + _load_full_server_baseline, +) + +pytestmark = pytest.mark.skipif( + not os.environ.get("POWERCONTEXT_TEST_MIGRATION_OCEANBASE_ENGINE_CRASH_ID"), + reason="requires the exact ID and loopback URL of a disposable OceanBase container owned by this test run", +) + + +def _docker(container: str, operation: str, *arguments: str, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["docker", operation, container, *arguments], # noqa: S607 + capture_output=True, + text=True, + timeout=30, + check=check, + ) + + +@pytest.fixture +def engine_target(tmp_path: Path) -> Iterator[tuple[str, _BackendTarget]]: + container = os.environ["POWERCONTEXT_TEST_MIGRATION_OCEANBASE_ENGINE_CRASH_ID"] + assert re.fullmatch(r"[0-9a-f]{64}", container), "Supply the exact owned container ID" + (info,) = json.loads(_docker(container, "inspect").stdout) + assert info["Id"] == container and info["State"]["Running"] + config = OceanBaseConfig(url=SecretStr(os.environ["POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL"])) + url = make_url(config.url.get_secret_value()) + assert url.host in {"127.0.0.1", "localhost"} and url.database + assert url.database.startswith("pc_migration_probe_") + assert not any(key.lower() in {"db", "database"} for key in url.query), ( + "The crash target must not override its selected database" + ) + (binding,) = info["NetworkSettings"]["Ports"]["2881/tcp"] + assert binding["HostIp"] == "127.0.0.1" and int(binding["HostPort"]) == (url.port or 2881) + (tmp_path / "container.json").write_text( + json.dumps({"id": container, "image": info["Config"]["Image"], "image_id": info["Image"], "port": url.port}), + encoding="utf-8", + ) + evidence = tmp_path / "evidence" + management = _BackendTarget(config, evidence) + database = f"pc_migration_probe_crash_{uuid4().hex}" + + def create(connection, _identity, _guard): + assert connection is not None + connection.exec_driver_sql(f"CREATE DATABASE `{database}`") + + management.connections().run(create) + child = config.model_copy( + update={"url": SecretStr(url.set(database=database).render_as_string(hide_password=False))} + ) + try: + yield container, _BackendTarget(child, evidence) + finally: + # Keep the owned database and journal for diagnosis, including on failure. + result = _docker(container, "logs", check=False) + (tmp_path / "oceanbase.log").write_text(result.stdout + result.stderr, encoding="utf-8") + + +def test_oceanbase_observer_crash_resumes_original_migration_and_preserves_history( + engine_target: tuple[str, _BackendTarget], tmp_path: Path +) -> None: + container, backend = engine_target + assert isinstance(backend.config, OceanBaseConfig) + _load_full_server_baseline(backend) + evidence = backend.evidence_directory + root = Path(__file__).resolve().parents[2] + environment = { + **os.environ, + "PYTHONPATH": os.pathsep.join(filter(None, (str(root), os.environ.get("PYTHONPATH")))), + "POWERCONTEXT_MIGRATION_CRASH_TARGET": backend.config.url.get_secret_value(), + "POWERCONTEXT_MIGRATION_WAIT_FOR_ENGINE_CRASH": "1", + } + with (tmp_path / "migration-client.log").open("w", encoding="utf-8") as log: + child = subprocess.Popen( + [ + sys.executable, + str(root / "tests/fixtures/database_migrations/crash_after_registered_ddl.py"), + str(evidence), + ], + env=environment, + stdout=log, + stderr=subprocess.STDOUT, + ) + try: + deadline = time.monotonic() + 180 + while not (evidence / "engine-crash-ready.json").exists(): + assert child.poll() is None, "Migration exited before reaching the engine crash boundary" + assert time.monotonic() < deadline, "Migration did not reach the registered DDL" + time.sleep(0.1) + (journal,) = evidence.rglob("maintenance.json") + interrupted = json.loads(journal.read_text()) + assert interrupted["state"] == "active" + (pid,) = _docker(container, "exec", "pgrep", "-x", "observer").stdout.split() + assert pid.isdigit() + cwd = _docker(container, "exec", "readlink", f"/proc/{pid}/cwd").stdout.strip() + executable = _docker(container, "exec", "readlink", f"/proc/{pid}/exe").stdout.strip() + expected = _docker(container, "exec", "readlink", "-f", "/root/demo/bin/observer").stdout.strip() + assert cwd == "/root/demo" and executable == expected + _docker(container, "exec", "kill", "-KILL", pid) + assert _docker(container, "exec", "readlink", f"/proc/{pid}/exe", check=False).returncode != 0 + (evidence / "engine-stopped").write_text("Owned observer terminated with SIGKILL.\n", encoding="utf-8") + assert child.wait(timeout=120) != 0 + finally: + if child.poll() is None: + child.terminate() + child.wait(timeout=30) + # The slim image's entrypoint unfolds a snapshot and truncates logs. Restart + # only observer so this test recovers the same data and commit logs. + subprocess.run( + ["docker", "exec", "--workdir", cwd, "--env", "LD_LIBRARY_PATH=/root/demo/lib", container, "bin/observer"], # noqa: S607 + capture_output=True, + text=True, + timeout=30, + check=True, + ) + deadline = time.monotonic() + 240 + + def ping(connection, _identity, _guard): + assert connection is not None + return connection.exec_driver_sql("SELECT 1").scalar_one() + + while True: + try: + assert backend.connections().run(ping) == 1 + break + except Exception: + if time.monotonic() >= deadline: + raise + time.sleep(1) + (new_pid,) = _docker(container, "exec", "pgrep", "-x", "observer").stdout.split() + assert new_pid != pid + recovery = _full_server_runner(backend).plan(backup_policy="manual") + assert recovery.state == "recovery_required" and recovery.source_revision == "s1300" + token = (recovery.coordination or {}).get("pending_run_id") + assert token + with pytest.raises(MigrationError, match="recovery_required"): + _apply(_full_server_runner(backend), policy="manual") + resumed = _full_server_runner(backend, acknowledge_previous_run=token) + result = _apply(resumed, policy="manual") + assert result.state == "ready" and result.revision == "s1401" and result.readiness_scope == "server" + assert result.backup_state == "user_confirmed" and result.backup_ref == interrupted["manual_ref"] + assert resumed.verify().state == "ready" + completed = json.loads(journal.read_text()) + assert completed["state"] == "complete" + for key in ("maintenance_window_id", "source_fingerprint", "manual_ref"): + assert completed[key] == interrupted[key] + + def history(connection, _identity, _guard): + assert connection is not None + return ( + connection.exec_driver_sql( + "SELECT content FROM pc_artifacts WHERE scope_id='historical' AND artifact_id='preserved'" + ).scalar_one(), + connection.exec_driver_sql("SELECT title,summary FROM pc_scopes WHERE scope_id='historical'").one(), + ) + + assert backend.connections().run(history) == ( + b'{"history":"unchanged"}', + ("retained title", "retained summary"), + ) + assert not _apply(_full_server_runner(backend), policy="manual").changed + (tmp_path / "engine-recovery.json").write_text( + json.dumps({ + "previous_pid": pid, + "pid": new_pid, + "revision": result.revision, + "history_preserved": True, + "recovery_identity_preserved": True, + "repeat_unchanged": True, + "backup": "operator declaration, restoration not tested", + }), + encoding="utf-8", + ) From 017abf5e0764f39d9c326edf240683480bbc6799 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 07:42:43 +0800 Subject: [PATCH 75/94] fix(ci): create the engine recovery evidence parent before pytest Pytest creates its basetemp directory without missing parents. Initialize the evidence directory before invoking the native engine test; keep the one-test and zero-skip gate intact. Tested: workflow shell syntax and embedded Python validation; git diff --check. CI failure evidence retained under job 120151229234. --- .github/workflows/master.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 8e244a6a68..bdf72462dd 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -93,6 +93,7 @@ jobs: echo "POWERCONTEXT_TEST_MIGRATION_OCEANBASE_ENGINE_CRASH_ID=$container_id" >> "$GITHUB_ENV" - name: Verify migration recovery after observer SIGKILL run: | + mkdir -p "$RUNNER_TEMP/oceanbase-engine-recovery" uv run pytest -q tests/native/test_oceanbase_engine_recovery.py \ --junitxml "$RUNNER_TEMP/oceanbase-engine-recovery/junit.xml" \ --basetemp "$RUNNER_TEMP/oceanbase-engine-recovery/cases" From 5563b6585446c620d1021ab4894e396ddab47365 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 07:47:43 +0800 Subject: [PATCH 76/94] test(service): bind the Unix startup conflict after TIME_WAIT The listener used by the startup failure fixture must reuse the stopped service port on Unix. Without SO_REUSEADDR, a closed accepted connection can make the fixture fail before the native start is exercised. Windows retains its exclusive binding behavior. Tested: Linux socket reproduction fails with EADDRINUSE before the change and listens after it; Ruff check and format passed. Linux/macOS native acceptance remains required. --- tests/native/test_personal_service_lifecycle.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/native/test_personal_service_lifecycle.py b/tests/native/test_personal_service_lifecycle.py index 821d61dfab..4f4c1b0f87 100644 --- a/tests/native/test_personal_service_lifecycle.py +++ b/tests/native/test_personal_service_lifecycle.py @@ -453,6 +453,10 @@ def _native_upgrade_fault( def occupy_then_start(*, reload_definition: bool) -> None: # Bind after migration and stopped checks, then let the actual # native manager attempt to start the registered executable. + if sys.platform != "win32": + # Reuse the stopped server's port even while its accepted + # connections remain in TIME_WAIT. + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) listener.bind(("127.0.0.1", port)) listener.listen(16) native_start(reload_definition=reload_definition) From 35ed35cb2fbf2dc7ce6ae856b24f8c7718f18f1f Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 08:03:18 +0800 Subject: [PATCH 77/94] feat(migrations): register the embedded seekDB Code schema Freeze the three Code tables captured from installed 1.3.0 and extend the existing complete Server chain with their exact schema. Preserve graph rows on upgrade, initialize them for fresh Code-enabled storage, and validate them at maintenance and runtime admission. Include the extension in backup scope and reject implicit managed-schema expansion through the standalone Code service. Add the sixth installed-release profile with immutable cache, graph-row, fingerprint and symbol checks. Initialize the HTTP/CLI integration fixture through the real Server before SDK writes and document the registered boundary in both locales. Constraint: All existing s1300/s1400/s1401 revision checksums remain unchanged. Vector configurations and enabling Code on already managed storage without its tables still require explicit transitions. Scope-risk: Managed seekDB Code initialization now validates rather than creating missing tables or indexes; unmanaged standalone Code behavior is preserved. Tested: Real seekDB migration/admission 3 passed, actual HTTP/CLI sync/restart 1 passed, related framework/release/CI suites 132 passed, full Linux Python 3.11 Ty and scoped hooks passed. Installed-release acceptance remains a separate gate. --- docs/en/docs/operate/database-migrations.md | 10 +- docs/zh/docs/operate/database-migrations.md | 12 +- release/upgrade-support.json | 9 +- src/powercontext/builtin/code/seekdb.py | 5 + .../persistence/migrations/deployment.py | 8 +- .../builtin/persistence/migrations/mysql.py | 7 +- .../persistence/migrations/mysql_schema.py | 39 +- .../migrations/server_readiness.py | 14 +- .../migrations/server_resources/manifest.json | 5 +- .../profiles/seekdb-code.json | 507 ++++++++++++++++++ .../test_full_server_migrations.py | 20 +- .../test_mysql_migration_runner.py | 76 +++ tests/e2e/test_native_code_seekdb.py | 9 +- tests/release_upgrades/README.md | 5 +- tests/release_upgrades/test_full_server.py | 11 +- tests/test_upgrade_release_gate.py | 2 +- 16 files changed, 692 insertions(+), 47 deletions(-) create mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/profiles/seekdb-code.json diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index e9848df013..5370cced35 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -14,7 +14,7 @@ The design follows [RFC #1771](https://github.com/oceanbase/powercontext/pull/17 `powercontext server db-migrate status/plan/apply/verify` default to `--bundle server` and use the configured persistent SQLite, embedded seekDB, or OceanBase MySQL database. Explicit environment files are authoritative; arbitrary bundle paths are not accepted. -The full Server path registers the actual 1.3.0 no-vector configuration as `s1300 → s1400 → s1401`. Code may be enabled on SQLite and OceanBase: its local cache is separate from the Server database and remains unchanged by database maintenance. It covers the complete schema, Candidate renaming and retained history, Dream task formats, Processing ownership, full-text projections, and existing Source references. Unknown layouts, extra objects, vector configurations, embedded seekDB Code storage, and unregistered sources fail before writes. Older Memory collection conversion is outside this path. +The full Server path registers the actual 1.3.0 no-vector configuration as `s1300 → s1400 → s1401`, with Code enabled or disabled on all three backends. SQLite and OceanBase keep Code in a separate local cache. Embedded seekDB Code adds three tables through a frozen schema extension captured from installed 1.3.0; maintenance preserves existing graph rows and creates the registered tables for a new Code-enabled database. The path covers the complete schema, Candidate renaming and retained history, Dream task formats, Processing ownership, full-text projections, and existing Source references. Unknown layouts, extra objects, vector configurations and unregistered sources fail before writes. Older Memory collection conversion is outside this path. Only successful schema, domain, projection, and task validation returns `state=ready`, `readiness_scope=server`, and `server_ready=true`. Publication also requires installed-artifact acceptance on all three backends. An implemented path is not evidence that a release passed acceptance; consult the installation guide and release gate for support status. @@ -29,7 +29,7 @@ Ordinary Server startup checks readiness without repairing an existing database. Server identity initialization checks registered tables without issuing DDL on managed storage. Creating the initial identity value and explicitly rotating a stopped clone's identity retain their existing transactional behavior. -Existing fresh deployments with vectors or embedded seekDB Code storage retain domain bootstrap and startup checks; they are not automatically adopted into this chain or covered by its upgrade support. Changing configuration cannot bypass admission or implicitly add vector/Code objects to managed storage. The release gate includes separate SQLite and OceanBase Code-enabled cases that build an index with installed 1.3.0, preserve the cache files, and query its original fingerprint and symbols after the upgrade. +Existing fresh deployments with vectors retain domain bootstrap and startup checks; they are not automatically adopted into this chain or covered by its upgrade support. Changing configuration cannot bypass admission or implicitly add vector/Code objects to managed storage. Enabling embedded Code on an already managed database without its tables requires a separately registered configuration transition; startup and the standalone Code CLI reject it. The release gate includes separate Code-enabled cases on all three backends: build an index with installed 1.3.0, preserve its cache files and embedded graph rows, and query the original fingerprint and symbols after the upgrade. `--bundle artifacts` retains the partial Artifact prototype chain starting at `p0001`; `--bundle topic-memory-recovery` covers only registered Topic recovery layouts. Both report `server_ready=false`. Neither establishes full Server readiness, and their revisions cannot be stamped as full Server revisions. @@ -67,7 +67,8 @@ that database for Server use. Full profile admission checks the registered Serve | `ServerIdentityRepository.initialize` | Check the identity table without DDL; identity value creation and explicit rotation are ordinary transactions | Retain concurrent-safe identity table initialization | | Offline Memory archive | The Server chain registers its structure at `s1401`; runtime admission checks it | Legacy offline Memory conversion owns its archive and conversion receipts | | `atomic-memory-migrate` and `processing-migrate` | Delegate to the shared Server or registered maintenance plan | Retain their domain-only behavior; recognized historical Server storage must use the full upgrade first | -| Vector and embedded seekDB Code objects | No registered configuration transition; reject before adding objects | Retain existing feature bootstrap without claiming registered upgrade support | +| Vector objects | No registered configuration transition; reject before adding objects | Retain existing feature bootstrap without claiming registered upgrade support | +| Embedded seekDB Code objects | The registered Code profile creates and validates the three tables and full-text index; Code startup never repairs missing managed objects | Recognize the complete released Code-enabled layout for explicit migration; retain standalone unmanaged Code bootstrap | | Separate Code cache on SQLite/OceanBase | Preserve existing cache files; main database follows the shared chain | Code service owns local cache creation and indexing | These helpers remain available to their owning runtime or maintenance operations; calling a low-level helper directly @@ -78,7 +79,8 @@ initialization; successfully opening an inspection connection does not establish Code's embedded seekDB store adds `pc_code_generations`, `pc_code_nodes`, `pc_code_edges` and `ix_code_search` to the deployment database. Other Code deployments use a local SQLite cache. These are derived indexes owned by the Code -service; their existence does not register an embedded Code Server baseline or permit implicit managed-schema expansion. +service; the frozen extension registers their schema for maintenance and full Server admission. A partial or changed +extension fails validation. Its existence does not permit implicit managed-schema expansion. Code caches on SQLite/OceanBase do not change the main database layout. ## Processing configuration maintenance diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 563c6eaaa6..57e8b7836a 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -12,7 +12,7 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 `powercontext server db-migrate status/plan/apply/verify` 默认选择 `--bundle server`，使用部署配置中的持久 SQLite、嵌入式 seekDB 或 OceanBase MySQL 模式数据库。维护命令以显式环境文件为准，不接受任意测试 bundle 路径。 -完整 Server 路径登记了真实 1.3.0 的无向量配置，版本链为 `s1300 → s1400 → s1401`。SQLite、OceanBase 可以启用 Code：其本地缓存独立于 Server 数据库，数据库维护保留缓存文件。此路径覆盖完整数据库结构、Candidate 改名与历史保留、Dream 任务格式、Processing 所有权声明、全文检索投影和既有 Source 引用。未知结构、额外对象、向量配置、嵌入式 seekDB Code 存储及未登记来源均在写入前拒绝。更早的 Memory 集合转换不属于此路径。 +完整 Server 路径登记了真实 1.3.0 的无向量配置，版本链为 `s1300 → s1400 → s1401`，三个后端均可启用或关闭 Code。SQLite、OceanBase 的 Code 本地缓存独立于 Server 数据库；嵌入式 seekDB Code 的三张表通过实际安装的 1.3.0 采集并冻结为结构扩展。维护保留已有图数据，为新建的 Code 启用数据库创建已登记的表。此路径覆盖完整数据库结构、Candidate 改名与历史保留、Dream 任务格式、Processing 所有权声明、全文检索投影和既有 Source 引用。未知结构、额外对象、向量配置及未登记来源均在写入前拒绝。更早的 Memory 集合转换不属于此路径。 只有结构、领域数据、搜索投影和任务校验全部通过，才返回 `state=ready`、`readiness_scope=server`、`server_ready=true`。发布还必须通过真实安装包在三个后端上的完整验收；实现路径存在不等于某次发布已通过验收，支持状态以安装指南和发布门禁为准。 @@ -27,7 +27,7 @@ Candidate 旧表保留为 `pc_retained_dream_candidate_heads` 和 `pc_retained_d Server identity 初始化在受管库上只检查已登记的表，不执行 DDL。首次创建 identity 值以及显式轮换已停止副本的 identity，仍使用原有事务语义。 -启用向量或嵌入式 seekDB Code 存储的既有新建部署仍使用原领域初始化和启动检查，不会自动纳入此版本链；这些配置不属于已登记升级范围。受管数据库不能通过切换配置绕过检查或隐式增加向量/Code 表。发布闸门分别验收 SQLite、OceanBase 的 Code 启用配置：用实际安装的 1.3.0 创建索引，保留缓存文件，并在升级后查询原指纹和符号。 +启用向量的既有新建部署仍使用原领域初始化和启动检查，不会自动纳入此版本链；这些配置不属于已登记升级范围。受管数据库不能通过切换配置绕过检查或隐式增加向量/Code 表。已经受管但没有 Code 表的数据库若要启用嵌入式 Code，需要另行登记配置转换，启动入口与独立 Code CLI 均拒绝隐式扩展。发布闸门分别验收三个后端的 Code 启用配置：用实际安装的 1.3.0 创建索引，保留缓存文件与嵌入式图数据，并在升级后查询原指纹和符号。 `--bundle artifacts` 保留 `p0001` 开始的局部 Artifact 原型链；`--bundle topic-memory-recovery` 只处理登记的 Topic 恢复布局。两者均报告 `server_ready=false`，不得用于证明完整 Server 就绪，也不能把其版本号直接 stamp 为完整 Server 版本。 @@ -56,7 +56,8 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 | `ServerIdentityRepository.initialize` | 检查 identity 表，不执行 DDL；创建 identity 值和显式轮换仍使用普通事务 | 保留支持并发的 identity 表初始化 | | 离线 Memory 归档 | Server 版本链在 `s1401` 登记结构，运行时准入检查该结构 | 旧 Memory 离线转换负责其归档和转换回执 | | `atomic-memory-migrate`、`processing-migrate` | 转发到共享 Server 或已登记的维护计划 | 保留领域独立行为；识别为历史完整 Server 的库必须先完成整库升级 | -| 向量与嵌入式 seekDB Code 对象 | 尚无已登记的配置转换，添加对象前拒绝 | 保留既有功能初始化，不宣称已登记升级支持 | +| 向量对象 | 尚无已登记的配置转换，添加对象前拒绝 | 保留既有功能初始化，不宣称已登记升级支持 | +| 嵌入式 seekDB Code 对象 | 已登记 Code profile 创建并验证三张表及全文索引；Code 启动不修补缺失的受管对象 | 完整的已发布 Code 启用结构可显式迁移；独立未受管 Code 保留既有初始化 | | SQLite、OceanBase 的独立 Code 缓存 | 保留缓存文件，主数据库走共享版本链 | Code 服务负责本地缓存创建和索引构建 | 这些辅助函数仍供所属 runtime 或维护操作使用；直接调用底层函数不能替代 Profile 准入、已确认的维护计划和 @@ -64,8 +65,9 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 只读 SQLite 检查入口与独立维护连接可以在业务初始化之前检查历史存储；检查连接打开成功不代表 Server 已就绪。 Code 的嵌入式 seekDB 存储在部署数据库中添加 `pc_code_generations`、`pc_code_nodes`、`pc_code_edges` 和 -`ix_code_search`；其他 Code 部署使用本地 SQLite 缓存。这些派生索引由 Code 服务负责，其存在不代表已经登记 -嵌入式 Code 的 Server 基线，也不允许隐式扩展受管结构。SQLite、OceanBase 的 Code 缓存不改变主库结构。 +`ix_code_search`；其他 Code 部署使用本地 SQLite 缓存。这些派生索引的数据由 Code 服务负责， +冻结扩展为维护与完整 Server 准入登记其结构。部分缺失或发生变化的扩展会被拒绝，不能隐式扩展受管结构。 +SQLite、OceanBase 的 Code 缓存不改变主库结构。 ## Processing 配置维护 diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 1f3086e1b9..c3730c3ceb 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "e7312f2ea747900b9eb99fbcce68618e783baf50fd4e2f7f4ffa7d0b88ceac85", - "rationale": "The shared s1300 -> s1400 -> s1401 chain registers the complete installed 1.3.0 no-vector Server database on SQLite, seekDB and OceanBase. SQLite and OceanBase may enable Code because its local cache is separate from this database; exact installed source inventories match the existing no-vector baselines. Admission preserves that ownership boundary instead of rejecting the flag. The installed-release matrix includes both Code-enabled backends and requires the source-built index to retain its fingerprint, symbols and immutable cache files through target Server queries. Vector profiles and embedded seekDB Code storage still require their own registered transitions. Frozen migration schemas, domain resources and released revision checksums remain unchanged. The chain preserves historical bytes, Source references, Candidate history and pending intents, fences old workers, and validates projections, Dream tasks, Processing ownership and live readiness. Registered maintenance shares the target lock, backup providers and durable recovery evidence; managed startup and identity admission do not perform schema changes. Processing mode changes and Atomic current projection repairs preserve revision and immutable authority, and resume from the original recovery point. Publication requires complete installed-release acceptance, repeated apply, old-startup rejection, background Worker execution and fresh-schema equivalence. Optional profile and complete native failure recovery acceptance remain incomplete." + "persistence_sha256": "120c6d0ebd99b90ee594603d3fed45d416e7b1af6a54d5b2344cd87a61f6de59", + "rationale": "The shared s1300 -> s1400 -> s1401 chain covers installed 1.3.0 no-vector Server storage on SQLite, seekDB and OceanBase with Code disabled or enabled. SQLite/OceanBase Code caches remain separate. The seekDB Code extension freezes three exact tables and their indexes captured from the installed source wheel. The same core chain preserves these rows and initializes this extension for a new Code-enabled database; all existing revision checksums remain unchanged. Complete source/target and DDL-prefix validation include the extension. Startup and standalone Code initialization cannot add or repair managed Code objects. Existing managed storage without Code tables cannot enable the feature by changing a flag. Code-enabled installed acceptance on all three backends checks original fingerprints, symbols and immutable cache files; embedded acceptance additionally compares exact graph-table row digests and fresh/upgraded schemas. Strict Server admission remains intact; Server test fixtures initialize through a Server before SDK writes. Target locks, original recovery points, acknowledgements, historical authority and backup policies remain shared. Vector configurations and complete recovery/restore acceptance remain unfinished; release policy stays upgrade-required." }, "cases": [], "automatic_sources": { @@ -35,6 +35,11 @@ "backend": "oceanbase", "configuration": "full-server-code", "suite": "tests/release_upgrades/test_full_server.py" + }, + { + "backend": "seekdb", + "configuration": "full-server-code", + "suite": "tests/release_upgrades/test_full_server.py" } ] } diff --git a/src/powercontext/builtin/code/seekdb.py b/src/powercontext/builtin/code/seekdb.py index 298e031e29..f7a1dce8f8 100644 --- a/src/powercontext/builtin/code/seekdb.py +++ b/src/powercontext/builtin/code/seekdb.py @@ -203,6 +203,7 @@ def initialize(self, deadline: float) -> None: with self.connection(deadline) as connection, connection.cursor() as cursor: cursor.execute("SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE()") existing = {row[0] for row in cursor.fetchall()} + managed = "pc_schema_revision" in existing # Even IF NOT EXISTS acquires seekdb's DDL lock. Reopening a complete # index must not wait for unrelated internal schema initialization. for table, statement in zip( @@ -210,12 +211,16 @@ def initialize(self, deadline: float) -> None: ): check_deadline(deadline) if table not in existing: + if managed: + raise CodeError("code_storage_unavailable") cursor.execute(statement) cursor.execute( "SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema = DATABASE() " "AND table_name = 'pc_code_nodes' AND index_name = 'ix_code_search'" ) if cursor.fetchone()[0] == 0: + if managed: + raise CodeError("code_storage_unavailable") try: cursor.execute( "CREATE FULLTEXT INDEX ix_code_search ON pc_code_nodes (searchable_text) WITH PARSER SPACE" diff --git a/src/powercontext/builtin/persistence/migrations/deployment.py b/src/powercontext/builtin/persistence/migrations/deployment.py index df61330ca6..33e79c9bcf 100644 --- a/src/powercontext/builtin/persistence/migrations/deployment.py +++ b/src/powercontext/builtin/persistence/migrations/deployment.py @@ -89,14 +89,14 @@ def deployment_runner( # noqa: C901 -- Ordered backend admission checks. database = settings.database configuration = _migration_configuration(settings) if bundle in {"server", "processing", "atomic-projection"}: - if settings.inference.embedding_model is not None or ( - settings.code.enabled and isinstance(database, SeekDBConfig) - ): + if settings.inference.embedding_model is not None: raise MigrationError( "unsupported_upgrade_configuration", - "The registered 1.3.0 Server path requires no vector provider or embedded Code storage.", + "The registered 1.3.0 Server path requires no vector provider.", ) configuration["embedding_enabled"] = False + if settings.code.enabled and isinstance(database, SeekDBConfig): + configuration["embedded_code_enabled"] = True selected = server_bundle() if bundle in {"server", "processing", "atomic-projection"} else production_bundle() maintenance_runner = ( AtomicProjectionMigrationRunner if bundle == "atomic-projection" else ProcessingConfigurationMigrationRunner diff --git a/src/powercontext/builtin/persistence/migrations/mysql.py b/src/powercontext/builtin/persistence/migrations/mysql.py index 9179530e3f..ad2bfefa0e 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql.py +++ b/src/powercontext/builtin/persistence/migrations/mysql.py @@ -155,9 +155,12 @@ def __init__( ) -> None: self.connections = connections self.bundle = bundle - self.schema = MySQLSchema(bundle) self.evidence_directory = evidence_directory.expanduser().resolve() self.configuration = json.loads(json.dumps(configuration or {})) + embedded_code = self.configuration.get("embedded_code_enabled", False) + if embedded_code and connections.config.kind != "seekdb": + raise MigrationError("unsupported_upgrade_configuration", "Embedded Code storage requires seekDB.") + self.schema = MySQLSchema(bundle, embedded_code=embedded_code) self.configuration_digest = digest(self.configuration) if connections.config.kind == "oceanbase": if ( @@ -693,7 +696,7 @@ def _backup_context(self, database_id: str, revision: str | None, window: str) - source_revision=revision, bundle_checksum=self.checksum, maintenance_window_id=window, - objects=self.bundle.affected_objects, + objects=self.schema.affected_objects, writers_stopped=True, ) diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py index 1afc04b294..80feb49f6a 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql_schema.py +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -140,14 +140,19 @@ def read_revision( class MySQLSchema: """Validate source, intermediate and post-DDL shapes from frozen resources.""" - def __init__(self, bundle: MigrationBundle) -> None: # noqa: C901 -- Keep ordered admission checks with their operation. + def __init__(self, bundle: MigrationBundle, *, embedded_code: bool = False) -> None: # noqa: C901 -- Keep ordered admission checks with their operation. self.bundle = bundle self.revisions = bundle.revisions manifest = json.loads((bundle.snapshot_directory / "manifest.json").read_text()) self.optional_table_options = {} + self.affected_objects = bundle.affected_objects if "mysql_state_layouts" in manifest: self._load_state_layouts(manifest["mysql_state_layouts"]) + if embedded_code: + self._extend_embedded_code(manifest) return + if embedded_code: + raise MigrationError("invalid_bundle", "Embedded Code needs a registered complete Server layout.") self.bundle = bundle if not bundle.mysql_layouts: raise MigrationError("invalid_bundle", "MySQL layout steps are not registered in this bundle.") @@ -252,6 +257,38 @@ def execute(self, connection: Connection, statement: str, configuration: dict[st else: connection.exec_driver_sql(statement, execution_options={"no_parameters": True}) + def _extend_embedded_code(self, manifest: dict[str, Any]) -> None: + resource = manifest.get("mysql_schema_extensions", {}).get("seekdb_code") + if resource not in self.bundle.resources: + raise MigrationError("invalid_bundle", "Embedded Code has no frozen schema registration.") + profile = json.loads((self.bundle.snapshot_directory / resource).read_text(encoding="utf-8")) + revision = profile["source_revision"] + statements = profile["statements"] + tables = profile["tables"] + if ( + profile.get("format") != "powercontext.mysql-schema-extension.v1" + or profile.get("backend") != "seekdb" + or revision != self.revisions[0] + or not statements + or dict(_parse_table(sql) for sql in statements) != tables + or any(set(tables) & set(state["tables"]) for state in self._schemas.values()) + ): + raise MigrationError("invalid_bundle", "The embedded Code schema registration is inconsistent.") + # Preserve the released Code tables through every core transition. A new + # database creates the same registered extension in the baseline revision. + state = copy.deepcopy(self._schemas[revision]) + for statement in statements: + name, table = _parse_table(statement) + state["tables"][name] = table + self._prefixes[revision].append(copy.deepcopy(state)) + self._statements[revision] += tuple(statements) + self._schemas[revision] = state + for later in self.revisions[1:]: + for prefix in self._prefixes[later]: + prefix["tables"].update(copy.deepcopy(tables)) + self._schemas[later]["tables"].update(copy.deepcopy(tables)) + self.affected_objects = (*self.affected_objects, *tables) + @staticmethod def fingerprint(inventory: dict[str, Any]) -> str: return digest(inventory) diff --git a/src/powercontext/builtin/persistence/migrations/server_readiness.py b/src/powercontext/builtin/persistence/migrations/server_readiness.py index 7d133d18c4..8e6639ebaa 100644 --- a/src/powercontext/builtin/persistence/migrations/server_readiness.py +++ b/src/powercontext/builtin/persistence/migrations/server_readiness.py @@ -48,8 +48,12 @@ def check_server_layout(connection): if connection.exec_driver_sql("PRAGMA foreign_key_check").first(): raise MigrationError("verification_failed", "A live foreign key is invalid.") else: - schema = MySQLSchema(bundle) - schema.verify(mysql_inventory(connection), bundle.head) + inventory = mysql_inventory(connection) + embedded_code = connection.engine.url.drivername == "mysql+aseekdb" and any( + name.startswith("pc_code_") for name in inventory["tables"] + ) + schema = MySQLSchema(bundle, embedded_code=embedded_code) + schema.verify(inventory, bundle.head) return bundle @@ -59,11 +63,7 @@ def prepare_server_storage(settings, *, injected_embedding=False): database = settings.database if database.kind == "sqlite" and database.is_in_memory: return - if ( - injected_embedding - or settings.inference.embedding_model is not None - or (settings.code.enabled and database.kind == "seekdb") - ): + if injected_embedding or settings.inference.embedding_model is not None: _check_unmanaged_optional_profile(database) return if database.kind == "oceanbase": diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json index bc932749bb..bbd70edf70 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/server_resources/manifest.json @@ -172,5 +172,8 @@ "schemas/mysql-s1400.json", "schemas/mysql-s1401.json" ], - "mysql_history": "mysql" + "mysql_history": "mysql", + "mysql_schema_extensions": { + "seekdb_code": "profiles/seekdb-code.json" + } } diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/profiles/seekdb-code.json b/src/powercontext/builtin/persistence/migrations/server_resources/profiles/seekdb-code.json new file mode 100644 index 0000000000..f9a1b838b6 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/profiles/seekdb-code.json @@ -0,0 +1,507 @@ +{ + "format": "powercontext.mysql-schema-extension.v1", + "source_release": "1.3.0", + "source_revision": "s1300", + "backend": "seekdb", + "statements": [ + "CREATE TABLE `pc_code_generations` (\n `id` char(32) COLLATE utf8mb4_bin NOT NULL,\n `binding` char(64) COLLATE utf8mb4_bin NOT NULL,\n `content_hash` char(64) COLLATE utf8mb4_bin NOT NULL,\n `node_count` bigint(20) NOT NULL,\n `edge_count` bigint(20) NOT NULL,\n `payload_bytes` bigint(20) NOT NULL,\n PRIMARY KEY (`id`),\n KEY `ix_code_binding` (`binding`) BLOCK_SIZE 16384 LOCAL\n) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_bin ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0", + "CREATE TABLE `pc_code_nodes` (\n `generation_id` char(32) COLLATE utf8mb4_bin NOT NULL,\n `id` char(64) COLLATE utf8mb4_bin NOT NULL,\n `kind` varchar(32) COLLATE utf8mb4_bin NOT NULL,\n `path` text COLLATE utf8mb4_bin NOT NULL,\n `file_id` char(64) COLLATE utf8mb4_bin DEFAULT NULL,\n `parent_id` char(64) COLLATE utf8mb4_bin DEFAULT NULL,\n `name` text COLLATE utf8mb4_bin NOT NULL,\n `qualified_name` text COLLATE utf8mb4_bin NOT NULL,\n `start_line` bigint(20) NOT NULL,\n `end_line` bigint(20) NOT NULL,\n `payload` longtext COLLATE utf8mb4_bin NOT NULL,\n `searchable_text` longtext COLLATE utf8mb4_bin NOT NULL,\n PRIMARY KEY (`generation_id`, `id`),\n KEY `ix_code_path` (`generation_id`, `path`(190), `start_line`) BLOCK_SIZE 16384 LOCAL,\n KEY `ix_code_name` (`generation_id`, `name`(190)) BLOCK_SIZE 16384 LOCAL,\n KEY `ix_code_qualified` (`generation_id`, `qualified_name`(190)) BLOCK_SIZE 16384 LOCAL,\n KEY `ix_code_parent` (`generation_id`, `parent_id`) BLOCK_SIZE 16384 LOCAL,\n FULLTEXT KEY `ix_code_search` (`searchable_text`) WITH PARSER space PARSER_PROPERTIES=(min_token_size=3,max_token_size=84) BLOCK_SIZE 16384\n) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_bin ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0", + "CREATE TABLE `pc_code_edges` (\n `generation_id` char(32) COLLATE utf8mb4_bin NOT NULL,\n `ordinal` bigint(20) NOT NULL,\n `source_id` char(64) COLLATE utf8mb4_bin NOT NULL,\n `target_id` char(64) COLLATE utf8mb4_bin NOT NULL,\n `kind` varchar(32) COLLATE utf8mb4_bin NOT NULL,\n `resolution` varchar(32) COLLATE utf8mb4_bin NOT NULL,\n `payload` longtext COLLATE utf8mb4_bin NOT NULL,\n PRIMARY KEY (`generation_id`, `ordinal`),\n KEY `ix_code_source` (`generation_id`, `source_id`, `kind`) BLOCK_SIZE 16384 LOCAL,\n KEY `ix_code_target` (`generation_id`, `target_id`, `kind`) BLOCK_SIZE 16384 LOCAL\n) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_bin ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0" + ], + "tables": { + "pc_code_generations": { + "columns": { + "id": { + "type": [ + "char", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "binding": { + "type": [ + "char", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "content_hash": { + "type": [ + "char", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "node_count": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "edge_count": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "payload_bytes": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + } + }, + "primary_key": [ + [ + "id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": { + "ix_code_binding": { + "columns": [ + [ + "binding", + null, + "asc" + ] + ], + "unique": false + } + } + }, + "pc_code_nodes": { + "columns": { + "generation_id": { + "type": [ + "char", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "id": { + "type": [ + "char", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "kind": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "path": { + "type": [ + "text", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "file_id": { + "type": [ + "char", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "parent_id": { + "type": [ + "char", + [ + "64" + ] + ], + "nullable": true, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "name": { + "type": [ + "text", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "qualified_name": { + "type": [ + "text", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "start_line": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "end_line": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "payload": { + "type": [ + "longtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "searchable_text": { + "type": [ + "longtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "generation_id", + null, + "asc" + ], + [ + "id", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": { + "ix_code_path": { + "columns": [ + [ + "generation_id", + null, + "asc" + ], + [ + "path", + 190, + "asc" + ], + [ + "start_line", + null, + "asc" + ] + ], + "unique": false + }, + "ix_code_name": { + "columns": [ + [ + "generation_id", + null, + "asc" + ], + [ + "name", + 190, + "asc" + ] + ], + "unique": false + }, + "ix_code_qualified": { + "columns": [ + [ + "generation_id", + null, + "asc" + ], + [ + "qualified_name", + 190, + "asc" + ] + ], + "unique": false + }, + "ix_code_parent": { + "columns": [ + [ + "generation_id", + null, + "asc" + ], + [ + "parent_id", + null, + "asc" + ] + ], + "unique": false + }, + "ix_code_search": { + "columns": [ + [ + "searchable_text", + null, + "asc" + ] + ], + "unique": false, + "kind": "fulltext", + "parser": "space", + "parser_properties": { + "min_token_size": 3, + "max_token_size": 84 + } + } + } + }, + "pc_code_edges": { + "columns": { + "generation_id": { + "type": [ + "char", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "ordinal": { + "type": [ + "bigint", + [] + ], + "nullable": false, + "default": null, + "charset": null, + "collation": null, + "extra": [] + }, + "source_id": { + "type": [ + "char", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "target_id": { + "type": [ + "char", + [ + "64" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "kind": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "resolution": { + "type": [ + "varchar", + [ + "32" + ] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + }, + "payload": { + "type": [ + "longtext", + [] + ], + "nullable": false, + "default": null, + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "extra": [] + } + }, + "primary_key": [ + [ + "generation_id", + null, + "asc" + ], + [ + "ordinal", + null, + "asc" + ] + ], + "foreign_keys": [], + "checks": {}, + "indexes": { + "ix_code_source": { + "columns": [ + [ + "generation_id", + null, + "asc" + ], + [ + "source_id", + null, + "asc" + ], + [ + "kind", + null, + "asc" + ] + ], + "unique": false + }, + "ix_code_target": { + "columns": [ + [ + "generation_id", + null, + "asc" + ], + [ + "target_id", + null, + "asc" + ], + [ + "kind", + null, + "asc" + ] + ], + "unique": false + } + } + } + }, + "provenance": { + "source_capture_sha256": "c7a2e0896b701a51e7a4998f69fb8fc53f0f33691150fb4d7cfa2bb66f214ffc", + "source_engine": "5.7.25-OceanBase seekdb-v1.4.0.0", + "source_wheel_sha256": "8da7983763f3a6ab80b413c770a4973a46c557a54e1a1fa675099e58d47305e5", + "source_module": "/tmp/pc2084-release-130/lib/python3.11/site-packages/powercontext/__init__.py", + "scope": "schema captured from installed 1.3.0; upgrade and populated Code reuse require separate acceptance" + } +} diff --git a/tests/builtin/persistence/test_full_server_migrations.py b/tests/builtin/persistence/test_full_server_migrations.py index 0c9f1bac9c..afee2a391d 100644 --- a/tests/builtin/persistence/test_full_server_migrations.py +++ b/tests/builtin/persistence/test_full_server_migrations.py @@ -272,23 +272,17 @@ def test_code_with_separate_cache_upgrades_and_reopens_server(source, tmp_path): @pytest.mark.parametrize("bundle", ["server", "processing", "atomic-projection"]) -@pytest.mark.parametrize("configuration", ["vector", "embedded-code"]) -def test_unregistered_optional_storage_is_rejected_before_opening_database(tmp_path, bundle, configuration): - from powercontext.builtin.code import CodeConfig +def test_unregistered_vector_storage_is_rejected_before_opening_database(tmp_path, bundle): from powercontext.builtin.persistence.migrations.deployment import deployment_runner - from powercontext.builtin.persistence.seekdb import SeekDBConfig from powercontext.builtin.runtime.config import InferenceConfig from powercontext.server.settings import ServerSettings - if configuration == "vector": - settings = ServerSettings( - database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'server.db'}"), - inference=InferenceConfig( - embedding_model="openai:fixture", embedding_profile_id="fixture", embedding_dimension=3 - ), - ) - else: - settings = ServerSettings(database=SeekDBConfig(path=tmp_path / "embedded"), code=CodeConfig(enabled=True)) + settings = ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'server.db'}"), + inference=InferenceConfig( + embedding_model="openai:fixture", embedding_profile_id="fixture", embedding_dimension=3 + ), + ) with pytest.raises(MigrationError, match="unsupported_upgrade_configuration"): deployment_runner(settings, bundle=bundle) assert list(tmp_path.iterdir()) == [] diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index c95e9d21fb..dbff63dd2e 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -492,6 +492,82 @@ def history(connection, identity, guard): assert backend.connections().run(history) == b'{"history":"unchanged"}' +@pytest.mark.parametrize("backend", ("seekdb",), indirect=True) +@pytest.mark.parametrize("origin", ("empty", "released")) +def test_embedded_code_migration_preserves_registered_tables_and_reopens_server( + backend: _BackendTarget, tmp_path: Path, origin: str +) -> None: + from powercontext.builtin.code import CodeConfig + from powercontext.builtin.persistence.dream_schema import check_dream_schema + from powercontext.builtin.persistence.migrations.server_readiness import prepare_server_storage + + bundle = server_bundle() + profile = json.loads((bundle.snapshot_directory / "profiles/seekdb-code.json").read_text()) + names = tuple(profile["tables"]) + if origin == "released": + _load_full_server_baseline(backend) + + def populate(connection, _identity, _guard): + for statement in profile["statements"]: + connection.exec_driver_sql(statement, execution_options={"no_parameters": True}) + connection.exec_driver_sql( + "INSERT INTO pc_code_generations VALUES (%s,%s,%s,0,0,0)", + ("a" * 32, "b" * 64, "c" * 64), + ) + connection.commit() + + backend.connections().run(populate) + settings = ServerSettings( + database=backend.config, + code=CodeConfig(enabled=True, cache_dir=tmp_path / "code-cache"), + ) + runner = cast(MySQLMigrationRunner, deployment_runner(settings, evidence_dir=backend.evidence_directory)) + plan = runner.plan(backup_policy="manual") + assert plan.source_revision == ("s1300" if origin == "released" else None) + result = _apply(runner, policy="manual") + assert result.state == "ready" and result.revision == bundle.head and result.readiness_scope == "server" + assert runner.verify().state == "ready" + prepare_server_storage(settings) + + def read(connection, _identity, _guard): + check_dream_schema(connection) + inventory = mysql_inventory(connection) + assert {name: inventory["tables"][name] for name in names} == profile["tables"] + return connection.exec_driver_sql("SELECT * FROM pc_code_generations").all() + + assert backend.connections().run(read) == ( + [("a" * 32, "b" * 64, "c" * 64, 0, 0, 0)] if origin == "released" else [] + ) + assert not _apply(runner, policy="manual").changed + + def damage(connection, _identity, _guard): + connection.exec_driver_sql("DROP INDEX ix_code_name ON pc_code_nodes") + + backend.connections().run(damage) + with pytest.raises(MigrationError): + prepare_server_storage(settings) + with pytest.raises(MigrationError): + backend.connections().run(read) + + +@pytest.mark.parametrize("backend", ("seekdb",), indirect=True) +def test_code_service_cannot_add_unregistered_tables_to_managed_storage( + backend: _BackendTarget, tmp_path: Path +) -> None: + from powercontext.builtin.code import CodeConfig, CodeError + from powercontext.builtin.code.configuration import open_code_service + + _apply(_full_server_runner(backend), policy="manual") + + async def open_service(): + async with open_code_service(CodeConfig(enabled=True, cache_dir=tmp_path / "code-cache"), backend.config): + pytest.fail("Code cannot implicitly expand the managed Server schema") + + with pytest.raises(CodeError, match="code_storage_unavailable"): + asyncio.run(open_service()) + assert _full_server_runner(backend).verify().state == "ready" + + def test_mysql_registered_processing_mode_switch_preserves_revision_and_history(backend: _BackendTarget) -> None: _load_full_server_baseline(backend) _apply(_full_server_runner(backend), policy="manual") diff --git a/tests/e2e/test_native_code_seekdb.py b/tests/e2e/test_native_code_seekdb.py index 4aa874f491..ab432a5fb0 100644 --- a/tests/e2e/test_native_code_seekdb.py +++ b/tests/e2e/test_native_code_seekdb.py @@ -511,11 +511,16 @@ def test_process_exit_before_publication_recovers_without_partial_results(reposi assert cursor.fetchone()[0] == 2 -def test_http_prepare_online_cli_sync_and_restart(tmp_path, backend): - database, _ = backend +def test_http_prepare_online_cli_sync_and_restart(tmp_path): + database = SeekDBConfig(path=tmp_path / "server-database") root = make_repository(tmp_path) async def exercise(): + bootstrap = create_server_app( + settings=ServerSettings(database=database, code=CodeConfig(enabled=True, cache_dir=tmp_path / "cache")) + ) + async with bootstrap.router.lifespan_context(bootstrap): + pass async with open_builtin_runtime( BuiltinConfig(database=database), scheduler_path=tmp_path / "scheduler.db" ) as runtime: diff --git a/tests/release_upgrades/README.md b/tests/release_upgrades/README.md index 71caf38861..dd479dd0d7 100644 --- a/tests/release_upgrades/README.md +++ b/tests/release_upgrades/README.md @@ -19,7 +19,10 @@ source wheels, invalid artifact metadata, catalog outages, and a target at or be Evidence records the catalog digest, exact source artifact, target artifact digest, and expanded matrix. The resolver requires universal `py3-none-any` wheels; a distribution format change must extend it explicitly. -All three profiles use `test_full_server.py`. It starts the installed 1.3.0 Server, writes data through HTTP, +All six profiles use `test_full_server.py`: Code enabled or disabled on SQLite, seekDB and OceanBase. +Code cases preserve immutable cache files and original fingerprint/symbol identities. Embedded seekDB additionally +preserves the exact graph-table row digests through maintenance; its frozen extension creates the same schema for fresh storage. +The suite starts the installed 1.3.0 Server, writes data through HTTP, stops it, seeds a pending processing intent using the installed SDK, and runs the target wheel's public `server db-migrate status/plan/apply/verify` commands. Verification must report `state=ready` and `server_ready=true`. The target Server must then start, preserve data and permissions, accept new writes, and support retrieval. diff --git a/tests/release_upgrades/test_full_server.py b/tests/release_upgrades/test_full_server.py index b7b8e99f90..884debc2c5 100644 --- a/tests/release_upgrades/test_full_server.py +++ b/tests/release_upgrades/test_full_server.py @@ -393,9 +393,6 @@ def case(model_gateway): assert os.environ["PC_UPGRADE_SOURCE_VERSION"] == "1.3.0", "Add a reviewed adapter for another source release" configuration = os.environ["PC_UPGRADE_CONFIGURATION"] assert configuration in {"full-server-no-vector", "full-server-code"}, "Unregistered acceptance configuration" - assert configuration != "full-server-code" or os.environ["PC_UPGRADE_BACKEND"] != "seekdb", ( - "Embedded Code storage requires its own registered migration" - ) return { "source": Path(os.environ["PC_UPGRADE_SOURCE_PYTHON"]), "target": Path(os.environ["PC_UPGRADE_TARGET_PYTHON"]), @@ -448,7 +445,8 @@ def historical_code(case, scope): symbols = [item for item in result["items"] if item["name"] == "upgrade_code_evidence"] assert symbols and all(item["path"] == "evidence.py" for item in symbols) cache = code_cache_snapshot(case) - assert any(name.endswith("/graph.sqlite") for name in cache) + graph_file = "/graph.json" if case["backend"] == "seekdb" else "/graph.sqlite" + assert any(name.endswith(graph_file) for name in cache) return {"fingerprint": result["fingerprint"], "symbols": symbols, "cache_sha256": cache} @@ -573,6 +571,11 @@ def preserved_code(case, migrated): historical = migrated["historical"] source = historical["code"] assert code_cache_snapshot(case) == source["cache_sha256"] + if case["backend"] == "seekdb": + names = ("pc_code_generations", "pc_code_nodes", "pc_code_edges") + before = historical["schema"]["data_sha256"] + after = migrated["schema"]["data_sha256"] + assert {name: before[name] for name in names} == {name: after[name] for name in names} with server( case["target"], environment_for(case), case["work"], case["report"] / "target-code-server.log" ) as client: diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py index 618e1b3d12..dc02f2f256 100644 --- a/tests/test_upgrade_release_gate.py +++ b/tests/test_upgrade_release_gate.py @@ -389,7 +389,7 @@ def test_repository_baseline_covers_all_three_databases(): assert {profile["backend"] for profile in selection["profiles"]} == {"sqlite", "seekdb", "oceanbase"} assert { profile["backend"] for profile in selection["profiles"] if profile["configuration"] == "full-server-code" - } == {"sqlite", "oceanbase"} + } == {"sqlite", "seekdb", "oceanbase"} @pytest.mark.parametrize("workflow_name,job_name", [("master", "upgrade-release-gate"), ("release", "release-build")]) From a60065b8a8d078156402d358bdafebc175976388 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 08:29:34 +0800 Subject: [PATCH 78/94] test(migrations): exercise installed SQLite backup restoration Use the automatic recovery point for both installed SQLite profiles, restore it to an independent database, compare the complete source schema and row digests, and verify source readback followed by another target upgrade. Preserve separate Code caches and the original backup checksum. Record the bounded SQLite proof separately from remote-backend restoration. Constraint: Existing revision resources and production runtime are unchanged. Scope-risk: The installed acceptance suite performs extra SQLite lifecycle work. Tested: Installed 1.3.0 and 1.4.1.dev194+g35ed35cb2; both SQLite profiles 11 passed, zero skips; missing-backup red case failed as expected. Tested: Upgrade gate/backup suites 75 passed, 2 existing platform skips; scoped prek and full Linux Python 3.11 Ty passed. --- docs/en/docs/operate/database-migrations.md | 6 ++ docs/zh/docs/operate/database-migrations.md | 5 + tests/release_upgrades/README.md | 11 +- tests/release_upgrades/test_full_server.py | 109 ++++++++++++++++++-- 4 files changed, 120 insertions(+), 11 deletions(-) diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 5370cced35..bfb89fc3ec 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -350,6 +350,12 @@ initialization reports `not_required`. Unsupported, failed, or incomplete auto b It returns `backup_unsupported` or a backup failure category and never silently skips. Changing policy requires a new plan confirmation. +The installed-release SQLite acceptance suite restores the generated automatic backup to a separate database. +It compares the complete source schema and row digests, reads historical records with the installed source Server, +then upgrades the restored copy and verifies target reads, retrieval and new writes. Code-enabled cases preserve +their separate immutable caches as well. This exercise is specific to SQLite; it does not verify seekDB or +OceanBase backup restoration. Keep the original recovery point until your own restored deployment is verified. + A separate `BackupProvider` reuses backend settings and maintenance connections. SQLite uses Online Backup API to create an independent file, including committed WAL, and checks integrity. seekdb and OceanBase executors reuse `ForkBackupProvider`, preferring native `FORK DATABASE`. The current executor rejects automatic `FORK TABLE`: enabling diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 57e8b7836a..4d83e38343 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -269,6 +269,11 @@ powercontext server db-migrate verify --env-file oceanbase.env \ 自动备份报告 `completed`，手动备份报告 `user_confirmed`，跳过报告 `skipped`；真正空库初始化报告 `not_required`。自动备份不支持、失败或未完成时停在迁移写入之前，返回 `backup_unsupported` 或备份失败类别，不能自动改成跳过。选择其他策略后需要重新确认计划。 +SQLite 安装版本验收套件将自动备份还原到独立数据库，比较完整来源结构和逐表数据摘要， +使用已安装的旧版 Server 读回历史记录，再升级还原副本并验证新版读取、检索和新写入。 +启用 Code 的用例还保留独立的不可变缓存。这项演练仅覆盖 SQLite，不证明 seekDB 或 OceanBase 备份可以还原。 +在确认自己的还原部署可用前，应保留原始恢复点。 + 独立 `BackupProvider` 复用后端配置和维护连接。SQLite 使用 Online Backup API 创建独立文件，包含已提交 WAL，并检查文件完整性。seekdb 和 OceanBase 执行器复用 `ForkBackupProvider`，优先采用原生 `FORK DATABASE`。当前执行器拒绝自动 `FORK TABLE`；启用前须单独登记同库备份对象，并完成受影响依赖、跨表一致性和恢复路径的验收。对于需要备份的升级，实际引擎和结构缺少通过验收的恢复证据时，`--backup auto` 在迁移写入前返回 `backup_unsupported`。用户手动备份与明确接受风险的不备份路径不依赖 Fork 支持。PC 不替用户声明已完成验收，也不自动改为物理备份或其他策略。 | 产品 | `FORK TABLE` 起始版本 | `FORK DATABASE` 起始版本 | diff --git a/tests/release_upgrades/README.md b/tests/release_upgrades/README.md index dd479dd0d7..cb01d55ab4 100644 --- a/tests/release_upgrades/README.md +++ b/tests/release_upgrades/README.md @@ -26,8 +26,13 @@ The suite starts the installed 1.3.0 Server, writes data through HTTP, stops it, seeds a pending processing intent using the installed SDK, and runs the target wheel's public `server db-migrate status/plan/apply/verify` commands. Verification must report `state=ready` and `server_ready=true`. The target Server must then start, preserve data and permissions, accept new writes, and support retrieval. -Partial migration success, missing framework/baseline support, and startup rejection all fail. These disposable -fixtures explicitly accept no backup; this is not a production upgrade procedure. +Partial migration success, missing framework/baseline support, and startup rejection all fail. SQLite profiles use +automatic backup and restore that recovery point into a separate database. The suite compares the complete +restored source schema and row digests, reads historical Memory, Candidates, grants and cursors with installed +1.3.0, upgrades the restored database again, then verifies target reads, FTS and new writes. Code-enabled SQLite +also preserves its separate immutable cache files. The original backup checksum must remain unchanged. +seekDB and OceanBase fixtures explicitly accept no backup; these cases do not establish backup restoration. +This is not a production upgrade procedure. `cases` remains available for additional exact source versions/configurations. Each case needs `source_version`, `backend`, `configuration`, and `suite`; overlapping cases fail. Optional features that create persistent objects @@ -62,7 +67,7 @@ Provide these behavioral tests (additional tests are allowed): | `test_source_release_database` | Start the actual source Server to create its complete database. Populate representative Memory history, references, permissions, Candidates, tasks and cursors through that version's public interfaces; verify the data before shutdown. Do not create a minimal schema by hand. | | `test_legacy_startup_is_rejected` | Target startup on the old database must fail without changing schema or row digests. | | `test_repeated_migration` | A second plan/apply/verify returns unchanged and preserves every table row digest. | -| `test_public_migration_commands` | Stop writers, run the target artifact's public maintenance commands, and validate full readiness. A limited bundle verification does not pass. | +| `test_public_migration_commands` | Stop writers, run the target artifact's public maintenance commands, and validate full readiness. SQLite additionally restores the actual automatic recovery point, reads it with the source Server, upgrades it and verifies target use. A limited bundle verification does not pass. | | `test_target_startup` | Start the target Server against the migrated database and verify readiness. | | `test_read_write_and_retrieval` | Read the old records, write new ones, and retrieve both through public interfaces. | | `test_authorization` | Verify permitted and denied access and preservation of relevant grants. | diff --git a/tests/release_upgrades/test_full_server.py b/tests/release_upgrades/test_full_server.py index 884debc2c5..8a84280ea9 100644 --- a/tests/release_upgrades/test_full_server.py +++ b/tests/release_upgrades/test_full_server.py @@ -25,6 +25,7 @@ import os import re import shlex +import shutil import signal import socket import subprocess @@ -509,19 +510,20 @@ def migration(operation, *options, label=""): return result migration("status") - plan = migration("plan", "--backup", "skip") + backup_policy = "auto" if case["backend"] == "sqlite" else "skip" + plan = migration("plan", "--backup", backup_policy) assert plan.get("plan_id"), "The public plan must identify exactly what will be applied" - # These are disposable fixtures; all Server processes are stopped. This - # explicit no-backup choice is never a recommendation for production. - migration( + # SQLite exercises its supported automatic recovery point. The other + # disposable backend fixtures explicitly accept no backup. + applied = migration( "apply", "--backup", - "skip", + backup_policy, "--plan-id", plan["plan_id"], "--yes", "--maintenance-confirmed", - "--accept-no-backup", + *(["--accept-no-backup"] if backup_policy == "skip" else []), ) verified = migration("verify") assert verified.get("state") == "ready" and verified.get("server_ready") is True, ( @@ -550,6 +552,7 @@ def migration(operation, *options, label=""): # same scope by publishing new Artifacts. Compare exact old counters here. processing = task_state(case, "target", "verify", historical["scope"]) return { + "applied": applied, "verified": verified, "repeated": repeated, "schema": snapshot(case, "target", "migrated"), @@ -558,6 +561,95 @@ def migration(operation, *options, label=""): } +@pytest.fixture(scope="session") +def restored_backup(case, migrated): + if case["backend"] != "sqlite": + return None + backup = Path(migrated["applied"].get("backup_ref") or "") + assert backup.is_file(), "SQLite installed-release acceptance requires an actual automatic backup" + before = hashlib.sha256(backup.read_bytes()).hexdigest() + restored = { + **case, + "work": case["work"] / "restored", + "report": case["report"] / "restored", + } + restored["work"].mkdir() + restored["report"].mkdir() + shutil.copyfile(backup, restored["work"] / "database") + historical = migrated["historical"] + if case["configuration"] == "full-server-code": + # Code caches are separate from the database recovery point. The + # operator must preserve those immutable files as well. + for name in ("code-cache", "repository"): + shutil.copytree(case["work"] / name, restored["work"] / name) + assert code_cache_snapshot(restored) == historical["code"]["cache_sha256"] + source = snapshot(restored, "source", "restored-source") + assert canonical_schema(source) == canonical_schema(historical["schema"]) + assert source["data_sha256"] == historical["schema"]["data_sha256"] + with server( + restored["source"], environment_for(restored), restored["work"], restored["report"] / "source-server.log" + ) as client: + records = post(client, "/v1/memory/entries/list", {"scope_id": historical["scope"]})["entries"] + assert {digest(item) for item in records} == {digest(item) for item in historical["memory"]} + candidate = post( + client, + "/v1/artifact-candidates/get", + {"scope_id": historical["scope"], "candidate_id": historical["candidate"]["candidate_id"]}, + ) + assert candidate == historical["candidate"] + assert post(client, "/v1/access/bindings/create", historical["grant"]) == historical["binding"] + continuation = client.get("/v1/scopes", params={"limit": 1, "cursor": historical["cursor"]}) + assert continuation.status_code == 200 + assert continuation.json()["items"] == historical["continuation"]["items"] + environment = environment_for(restored) + + def migration(operation, *options): + return json.loads( + command( + [console(restored["target"]), "server", "db-migrate", operation, *options], + environment, + restored["work"], + restored["report"] / f"migration-{operation}.log", + ) + ) + + plan = migration("plan", "--backup", "auto") + assert plan["source_revision"] == "s1300" + applied = migration("apply", "--backup", "auto", "--plan-id", plan["plan_id"], "--yes", "--maintenance-confirmed") + assert applied["changed"] is True and applied["backup_state"] == "completed" + assert migration("verify")["server_ready"] is True + with server(restored["target"], environment, restored["work"], restored["report"] / "target-server.log") as client: + records = post(client, "/v1/memory/entries/list", {"scope_id": historical["scope"]})["entries"] + assert {digest(item) for item in records} == {digest(item) for item in historical["memory"]} + found = post( + client, "/v1/memory/search", {"scope_id": historical["scope"], "query": "Upgrade fixture", "mode": "fts"} + )["hits"] + assert {item["memory"]["text"] for item in found} >= {item["text"] for item in historical["memory"]} + candidate = post( + client, + "/v1/candidates/get", + {"scope_id": historical["scope"], "candidate_id": historical["candidate"]["candidate_id"]}, + ) + assert candidate["proposal"] == historical["candidate"]["proposal"] + assert candidate["version"] == historical["candidate"]["version"] + text = "A restored backup accepts writes after its registered upgrade." + written = post(client, "/v1/memory/remember", {"scope_id": historical["scope"], "kind": "fact", "text": text})[ + "records" + ] + assert written and written[0]["text"] == text + assert hashlib.sha256(backup.read_bytes()).hexdigest() == before + if case["configuration"] == "full-server-code": + assert code_cache_snapshot(restored) == historical["code"]["cache_sha256"] + return { + "method": "sqlite_online", + "source_schema_and_rows_preserved": True, + "source_server_readback": True, + "target_upgrade_and_read_write": True, + "backup_sha256_unchanged": before, + "separate_code_cache_preserved": case["configuration"] == "full-server-code", + } + + @pytest.fixture def target(case, migrated, preserved_code): with server(case["target"], environment_for(case), case["work"], case["report"] / "target-server.log") as client: @@ -659,7 +751,7 @@ def test_repeated_migration(migrated): assert migrated["repeated"]["changed"] is False -def test_public_migration_commands(migrated): +def test_public_migration_commands(migrated, restored_backup): assert migrated["verified"]["server_ready"] is True @@ -765,7 +857,7 @@ def normalize(sql): } -def test_fresh_upgrade_equivalence(case, migrated, processed_background, preserved_code): +def test_fresh_upgrade_equivalence(case, migrated, processed_background, preserved_code, restored_backup): if case["backend"] == "oceanbase": environment = environment_for(case, fresh=True) env_file = case["work"] / "fresh.env" @@ -824,6 +916,7 @@ def test_fresh_upgrade_equivalence(case, migrated, processed_background, preserv "Upgraded and fresh database objects differ; no blanket table/constraint exclusions allowed" ) evidence = { + "backup_restore": restored_backup, "backend": case["backend"], "engine_version": inventory["engine_version"], "driver_version": inventory["driver_version"], From e645a2429be757afe7f44c88514377e92f3cf90e Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 08:39:02 +0800 Subject: [PATCH 79/94] fix(migrations): reflect native vector index definitions explicitly Record VECTOR KEY columns and algorithm parameters as an index instead of a synthetic column. Reject duplicate, unknown and malformed parameter options. Preserve the registered no-vector layouts and fail-closed optional-profile policy. Constraint: No historical revision resources or checksums change; vector upgrade profiles remain disabled. Scope-risk: Normalized inventories of previously unsupported vector layouts change to reflect actual native indexes. Tested: Six regression cases failed before the correction. Full schema suite including real seekDB reflection/recovery: 58 passed, zero skips. Tested: Server baseline, full migration and upgrade-gate suites: 110 passed, zero skips. Full Linux Python 3.11 Ty and scoped hooks passed. --- release/upgrade-support.json | 4 +- .../persistence/migrations/mysql_schema.py | 30 +++++++++ .../test_mysql_migration_schema.py | 62 +++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/release/upgrade-support.json b/release/upgrade-support.json index c3730c3ceb..6b7a890e61 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "120c6d0ebd99b90ee594603d3fed45d416e7b1af6a54d5b2344cd87a61f6de59", - "rationale": "The shared s1300 -> s1400 -> s1401 chain covers installed 1.3.0 no-vector Server storage on SQLite, seekDB and OceanBase with Code disabled or enabled. SQLite/OceanBase Code caches remain separate. The seekDB Code extension freezes three exact tables and their indexes captured from the installed source wheel. The same core chain preserves these rows and initializes this extension for a new Code-enabled database; all existing revision checksums remain unchanged. Complete source/target and DDL-prefix validation include the extension. Startup and standalone Code initialization cannot add or repair managed Code objects. Existing managed storage without Code tables cannot enable the feature by changing a flag. Code-enabled installed acceptance on all three backends checks original fingerprints, symbols and immutable cache files; embedded acceptance additionally compares exact graph-table row digests and fresh/upgraded schemas. Strict Server admission remains intact; Server test fixtures initialize through a Server before SDK writes. Target locks, original recovery points, acknowledgements, historical authority and backup policies remain shared. Vector configurations and complete recovery/restore acceptance remain unfinished; release policy stays upgrade-required." + "persistence_sha256": "3694fdad13add880be1f87506ceced65e247c7b407ccc0fd09b7db64ae3c901f", + "rationale": "The shared s1300 -> s1400 -> s1401 chain covers installed 1.3.0 no-vector Server storage on SQLite, seekDB and OceanBase with Code disabled or enabled. SQLite/OceanBase Code caches remain separate. The seekDB Code extension freezes three exact tables and their indexes captured from the installed source wheel. The same core chain preserves these rows and initializes this extension for a new Code-enabled database; all existing revision checksums remain unchanged. Complete source/target and DDL-prefix validation include the extension. Startup and standalone Code initialization cannot add or repair managed Code objects. Existing managed storage without Code tables cannot enable the feature by changing a flag. Code-enabled installed acceptance on all three backends checks original fingerprints, symbols and immutable cache files; embedded acceptance additionally compares exact graph-table row digests and fresh/upgraded schemas. Strict Server admission remains intact; Server test fixtures initialize through a Server before SDK writes. Target locks, original recovery points, acknowledgements, historical authority and backup policies remain shared. Vector configurations and complete recovery/restore acceptance remain unfinished; release policy stays upgrade-required. Native VECTOR KEY reflection now records ordered key columns and the explicit HNSW algorithm, distance, library and construction/search parameters instead of treating the clause as a column. Duplicate, unknown or malformed vector index options fail reflection. This parser correction does not register or enable vector upgrade configurations, alter schema, or change existing revision checksums." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py index 80feb49f6a..66d3fa2a25 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql_schema.py +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -741,6 +741,33 @@ def _fulltext_definition(part: list[str]) -> tuple[str, dict[str, Any]]: } +def _vector_definition(part: list[str]) -> tuple[str, dict[str, Any]]: + columns, end = _group(part, 3) + if part[end : end + 1] != ["with"]: + raise MigrationError("incompatible_schema", "A VECTOR index is missing its parameter declaration.") + arguments, end = _group(part, end + 1) + parameters: dict[str, str | int] = {} + for option in _parts(arguments): + if len(option) != 3 or option[1] != "=" or option[0] in parameters: + raise MigrationError("incompatible_schema", "A VECTOR index parameter is invalid or duplicated.") + name, _, value = option + if name in {"distance", "type", "lib"} and re.fullmatch(r"[a-z][a-z_0-9]*", value): + parameters[name] = value + elif name in {"m", "ef_construction", "ef_search"} and value.isdigit() and int(value) > 0: + parameters[name] = int(value) + else: + raise MigrationError("incompatible_schema", "An unsupported VECTOR index parameter was discovered.") + if not {"distance", "type"} <= parameters.keys(): + raise MigrationError("incompatible_schema", "A VECTOR index has no complete algorithm declaration.") + _physical(part[end:]) + return part[2], { + "columns": _key_columns(columns), + "unique": False, + "kind": "vector", + "parameters": parameters, + } + + def _table_clause( table: dict[str, Any], part: list[str], defaults: dict[str, str | None], database: str | None ) -> None: @@ -765,6 +792,9 @@ def _table_clause( elif part[0] == "fulltext": index_name, definition = _fulltext_definition(part) _put_named(table["indexes"], index_name, definition) + elif part[0] == "vector" and part[1:2] in (["key"], ["index"]): + index_name, definition = _vector_definition(part) + _put_named(table["indexes"], index_name, definition) elif constraint is None: _put_named(table["columns"], part[0], _column(part, defaults)) else: diff --git a/tests/builtin/persistence/test_mysql_migration_schema.py b/tests/builtin/persistence/test_mysql_migration_schema.py index b938216016..1fd98788e1 100644 --- a/tests/builtin/persistence/test_mysql_migration_schema.py +++ b/tests/builtin/persistence/test_mysql_migration_schema.py @@ -566,6 +566,68 @@ def test_version_table_is_validated_before_reading_and_is_not_a_business_table(s read_revision(cast(Connection, database), _inventory(database), revisions=schema.bundle.revisions) +_VECTOR_TABLE = ( + "CREATE TABLE pc_vector_projection (" + "id BIGINT NOT NULL, embedding VECTOR(3) NOT NULL, PRIMARY KEY (id), " + "VECTOR KEY ix_embedding (embedding) WITH " + "(DISTANCE=L2, TYPE=HNSW, LIB=VSAG, M=16, EF_CONSTRUCTION=200, EF_SEARCH=64) BLOCK_SIZE 16384" + ") DEFAULT CHARSET=utf8mb4" +) + + +def test_vector_index_reflection_preserves_parameters_without_inventing_a_column() -> None: + # VECTOR KEY comes from the actual installed 1.3.0 optional-profile captures. + table = _inventory(_ReflectedDatabase({"pc_vector_projection": _VECTOR_TABLE}))["tables"]["pc_vector_projection"] + assert set(table["columns"]) == {"id", "embedding"} + assert table["columns"]["embedding"]["type"] == ["vector", ["3"]] + assert table["indexes"]["ix_embedding"] == { + "columns": [["embedding", None, "asc"]], + "unique": False, + "kind": "vector", + "parameters": { + "distance": "l2", + "type": "hnsw", + "lib": "vsag", + "m": 16, + "ef_construction": 200, + "ef_search": 64, + }, + } + changed = _VECTOR_TABLE.replace("EF_SEARCH=64", "EF_SEARCH=128") + actual = _inventory(_ReflectedDatabase({"pc_vector_projection": changed}))["tables"]["pc_vector_projection"] + assert actual["columns"] == table["columns"] + assert actual["indexes"] != table["indexes"] + + +@pytest.mark.parametrize( + "replacement", + ["M=16, M=32", "M=-1", "M=0", "M=anything", "UNKNOWN_OPTION=16"], +) +def test_unprovable_vector_index_options_fail_schema_reflection(replacement: str) -> None: + definition = _VECTOR_TABLE.replace("M=16", replacement) + with pytest.raises(MigrationError, match="VECTOR"): + _inventory(_ReflectedDatabase({"pc_vector_projection": definition})) + + +@pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") +def test_real_seekdb_vector_index_is_reflected_as_an_index(short_tmp_path: Path) -> None: + def inspect_vector(connection, _identity, _guard): + assert connection is not None + connection.exec_driver_sql(_VECTOR_TABLE) + connection.commit() + table = mysql_inventory(connection)["tables"]["pc_vector_projection"] + assert set(table["columns"]) == {"id", "embedding"} + index = table["indexes"]["ix_embedding"] + assert index["kind"] == "vector" + assert index["columns"] == [["embedding", None, "asc"]] + assert index["parameters"]["distance"] == "l2" + assert index["parameters"]["m"] == 16 + assert index["parameters"]["ef_construction"] == 200 + assert index["parameters"]["ef_search"] == 64 + + MaintenanceConnections(SeekDBConfig(path=short_tmp_path / "seekdb")).run(inspect_vector, writable=True) + + @pytest.mark.parametrize("violation", ["foreign_key", "check"]) def test_portable_integrity_queries_reject_preexisting_invalid_rows(schema: MySQLSchema, violation: str) -> None: # Execute the portable existence queries on real SQLite, without claiming From 5f769d6c76c32977bb9676a39284ef9d9bf000c9 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 09:42:57 +0800 Subject: [PATCH 80/94] feat(migrations): register exact Server vector profiles Select frozen SQLite and native vector layouts from the historical embedding profile and preserve embeddings, Topic retrieval shape and existing authority. Register SQLite helper backfill and transactional triggers in maintenance; managed startup rejects profile drift and missing helpers without repair. Constraint: Keep existing revision resources and no-vector checksums unchanged. Scope-risk: Installed vector release acceptance remains required; no full release support is declared and partial processing/projection bundles remain rejected. Tested: SQLite/full-Server/baseline suites 56 passed; actual seekDB/OceanBase vector migrations 4 passed; migration CLI/release suites 141 passed (2 existing async teardown warnings); affected Server E2E 62 passed/28 backend skips. Tested: Full Python 3.11 Ty, scoped prek hooks and git diff --check passed. --- docs/en/docs/operate/database-migrations.md | 8 +- docs/zh/docs/operate/database-migrations.md | 8 +- release/upgrade-support.json | 4 +- .../builtin/persistence/migrations/bundle.py | 24 +- .../persistence/migrations/deployment.py | 37 +- .../persistence/migrations/mysql_schema.py | 37 ++ .../migrations/server_readiness.py | 25 +- .../server_resources/profiles/vector.json | 270 +++++++++++++ .../server_resources/server_vector_domain.py | 379 ++++++++++++++++++ .../builtin/persistence/migrations/sqlite.py | 21 +- .../persistence/migrations/vector_profile.py | 143 +++++++ src/powercontext/server/factory.py | 6 +- .../test_full_server_migrations.py | 4 +- .../test_mysql_migration_runner.py | 65 +++ .../test_server_vector_migrations.py | 225 +++++++++++ 15 files changed, 1229 insertions(+), 27 deletions(-) create mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/profiles/vector.json create mode 100644 src/powercontext/builtin/persistence/migrations/server_resources/server_vector_domain.py create mode 100644 src/powercontext/builtin/persistence/migrations/vector_profile.py create mode 100644 tests/builtin/persistence/test_server_vector_migrations.py diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index bfb89fc3ec..ced2b9d4d8 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -14,7 +14,7 @@ The design follows [RFC #1771](https://github.com/oceanbase/powercontext/pull/17 `powercontext server db-migrate status/plan/apply/verify` default to `--bundle server` and use the configured persistent SQLite, embedded seekDB, or OceanBase MySQL database. Explicit environment files are authoritative; arbitrary bundle paths are not accepted. -The full Server path registers the actual 1.3.0 no-vector configuration as `s1300 → s1400 → s1401`, with Code enabled or disabled on all three backends. SQLite and OceanBase keep Code in a separate local cache. Embedded seekDB Code adds three tables through a frozen schema extension captured from installed 1.3.0; maintenance preserves existing graph rows and creates the registered tables for a new Code-enabled database. The path covers the complete schema, Candidate renaming and retained history, Dream task formats, Processing ownership, full-text projections, and existing Source references. Unknown layouts, extra objects, vector configurations and unregistered sources fail before writes. Older Memory collection conversion is outside this path. +The full Server path registers actual 1.3.0 storage as `s1300 → s1400 → s1401`, with Code enabled or disabled on all three backends and a separate profile for unit-normalized L2 vectors. SQLite and OceanBase keep Code in a separate local cache. Embedded seekDB Code adds three tables through a frozen schema extension captured from installed 1.3.0; maintenance preserves existing graph rows and creates the registered tables for a new Code-enabled database. The path covers the complete schema, Candidate renaming and retained history, Dream task formats, Processing ownership, search projections, and existing Source references. Unknown layouts, extra objects and unregistered sources fail before writes. Older Memory collection conversion is outside this path. Only successful schema, domain, projection, and task validation returns `state=ready`, `readiness_scope=server`, and `server_ready=true`. Publication also requires installed-artifact acceptance on all three backends. An implemented path is not evidence that a release passed acceptance; consult the installation guide and release gate for support status. @@ -29,7 +29,9 @@ Ordinary Server startup checks readiness without repairing an existing database. Server identity initialization checks registered tables without issuing DDL on managed storage. Creating the initial identity value and explicitly rotating a stopped clone's identity retain their existing transactional behavior. -Existing fresh deployments with vectors retain domain bootstrap and startup checks; they are not automatically adopted into this chain or covered by its upgrade support. Changing configuration cannot bypass admission or implicitly add vector/Code objects to managed storage. Enabling embedded Code on an already managed database without its tables requires a separately registered configuration transition; startup and the standalone Code CLI reject it. The release gate includes separate Code-enabled cases on all three backends: build an index with installed 1.3.0, preserve its cache files and embedded graph rows, and query the original fingerprint and symbols after the upgrade. +Vector maintenance requires the exact historical embedding profile: profile ID, model, dimension, distance and normalization. It preserves historical embeddings and Topic retrieval shape without calling an embedding provider. The registered SQLite transition adds and backfills Atomic Memory's vec0 helper and transactional triggers; native backends retain the registered vector columns and HNSW parameters. `pc_server_storage_profile` stores the complete embedding configuration, not another revision or recovery journal. Generic storage readers validate this declaration, and Server startup uses the configured or injected model's actual profile. Missing managed helpers and profile changes are rejected before startup repair or writes. + +Empty persistent SQLite/seekDB Server storage with vectors initializes through the selected profile of the same chain. Existing unmanaged storage requires explicit maintenance and an exact registered source layout. Changing configuration cannot bypass admission, enable vectors on no-vector managed storage, or implicitly add vector/Code objects. Enabling embedded Code on an already managed database without its tables requires a separately registered configuration transition; startup and the standalone Code CLI reject it. The release gate includes separate Code-enabled cases on all three backends: build an index with installed 1.3.0, preserve its cache files and embedded graph rows, and query the original fingerprint and symbols after the upgrade. Vector release acceptance must independently cover the matching installed configuration; a successful no-vector case does not establish vector support. `--bundle artifacts` retains the partial Artifact prototype chain starting at `p0001`; `--bundle topic-memory-recovery` covers only registered Topic recovery layouts. Both report `server_ready=false`. Neither establishes full Server readiness, and their revisions cannot be stamped as full Server revisions. @@ -67,7 +69,7 @@ that database for Server use. Full profile admission checks the registered Serve | `ServerIdentityRepository.initialize` | Check the identity table without DDL; identity value creation and explicit rotation are ordinary transactions | Retain concurrent-safe identity table initialization | | Offline Memory archive | The Server chain registers its structure at `s1401`; runtime admission checks it | Legacy offline Memory conversion owns its archive and conversion receipts | | `atomic-memory-migrate` and `processing-migrate` | Delegate to the shared Server or registered maintenance plan | Retain their domain-only behavior; recognized historical Server storage must use the full upgrade first | -| Vector objects | No registered configuration transition; reject before adding objects | Retain existing feature bootstrap without claiming registered upgrade support | +| Vector objects | The selected Server profile owns schema, helper backfill and exact configuration admission; changing embedding spaces requires a separate transition | A complete released vector layout can be explicitly migrated; standalone unmanaged SDK storage retains domain initialization | | Embedded seekDB Code objects | The registered Code profile creates and validates the three tables and full-text index; Code startup never repairs missing managed objects | Recognize the complete released Code-enabled layout for explicit migration; retain standalone unmanaged Code bootstrap | | Separate Code cache on SQLite/OceanBase | Preserve existing cache files; main database follows the shared chain | Code service owns local cache creation and indexing | diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 4d83e38343..a26cd2e58d 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -12,7 +12,7 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 `powercontext server db-migrate status/plan/apply/verify` 默认选择 `--bundle server`，使用部署配置中的持久 SQLite、嵌入式 seekDB 或 OceanBase MySQL 模式数据库。维护命令以显式环境文件为准，不接受任意测试 bundle 路径。 -完整 Server 路径登记了真实 1.3.0 的无向量配置，版本链为 `s1300 → s1400 → s1401`，三个后端均可启用或关闭 Code。SQLite、OceanBase 的 Code 本地缓存独立于 Server 数据库；嵌入式 seekDB Code 的三张表通过实际安装的 1.3.0 采集并冻结为结构扩展。维护保留已有图数据，为新建的 Code 启用数据库创建已登记的表。此路径覆盖完整数据库结构、Candidate 改名与历史保留、Dream 任务格式、Processing 所有权声明、全文检索投影和既有 Source 引用。未知结构、额外对象、向量配置及未登记来源均在写入前拒绝。更早的 Memory 集合转换不属于此路径。 +完整 Server 路径登记了真实 1.3.0 的存储结构，版本链为 `s1300 → s1400 → s1401`，三个后端均可启用或关闭 Code，并为单位归一化的 L2 向量登记独立 profile。SQLite、OceanBase 的 Code 本地缓存独立于 Server 数据库；嵌入式 seekDB Code 的三张表通过实际安装的 1.3.0 采集并冻结为结构扩展。维护保留已有图数据，为新建的 Code 启用数据库创建已登记的表。此路径覆盖完整数据库结构、Candidate 改名与历史保留、Dream 任务格式、Processing 所有权声明、检索投影和既有 Source 引用。未知结构、额外对象及未登记来源均在写入前拒绝。更早的 Memory 集合转换不属于此路径。 只有结构、领域数据、搜索投影和任务校验全部通过，才返回 `state=ready`、`readiness_scope=server`、`server_ready=true`。发布还必须通过真实安装包在三个后端上的完整验收；实现路径存在不等于某次发布已通过验收，支持状态以安装指南和发布门禁为准。 @@ -27,7 +27,9 @@ Candidate 旧表保留为 `pc_retained_dream_candidate_heads` 和 `pc_retained_d Server identity 初始化在受管库上只检查已登记的表，不执行 DDL。首次创建 identity 值以及显式轮换已停止副本的 identity，仍使用原有事务语义。 -启用向量的既有新建部署仍使用原领域初始化和启动检查，不会自动纳入此版本链；这些配置不属于已登记升级范围。受管数据库不能通过切换配置绕过检查或隐式增加向量/Code 表。已经受管但没有 Code 表的数据库若要启用嵌入式 Code，需要另行登记配置转换，启动入口与独立 Code CLI 均拒绝隐式扩展。发布闸门分别验收三个后端的 Code 启用配置：用实际安装的 1.3.0 创建索引，保留缓存文件与嵌入式图数据，并在升级后查询原指纹和符号。 +向量维护要求声明与历史数据完全一致的 embedding profile：profile ID、模型、维度、距离和归一化方式。迁移保留历史向量及 Topic 检索配置，不调用 embedding provider。SQLite 的登记步骤添加并回填 Atomic Memory 的 vec0 辅助索引和事务触发器；原生后端保留登记的向量列与 HNSW 参数。`pc_server_storage_profile` 保存完整的 embedding 配置，不是另一张 revision 表或恢复日志。通用存储读入口校验该声明，Server 启动使用配置或注入模型的实际 profile。受管辅助对象缺失或 profile 变化会在启动修补或写入之前被拒绝。 + +空的持久 SQLite/seekDB Server 启用向量时，通过同一版本链所选的 profile 初始化。已有未受管库必须显式维护，且来源结构必须与登记布局完全一致。切换配置不能绕过准入检查，不能给受管无向量库直接启用向量，也不能隐式增加向量或 Code 对象。已经受管但没有 Code 表的数据库若要启用嵌入式 Code，需要另行登记配置转换，启动入口与独立 Code CLI 均拒绝隐式扩展。发布闸门分别验收三个后端的 Code 启用配置：用实际安装的 1.3.0 创建索引，保留缓存文件与嵌入式图数据，并在升级后查询原指纹和符号。向量发布验收必须独立覆盖相应的实际安装配置；无向量配置通过不能证明向量支持。 `--bundle artifacts` 保留 `p0001` 开始的局部 Artifact 原型链；`--bundle topic-memory-recovery` 只处理登记的 Topic 恢复布局。两者均报告 `server_ready=false`，不得用于证明完整 Server 就绪，也不能把其版本号直接 stamp 为完整 Server 版本。 @@ -56,7 +58,7 @@ MySQL 兼容后端的结构反射支持 CHECK 表达式中的二元 `+`、`-`、 | `ServerIdentityRepository.initialize` | 检查 identity 表，不执行 DDL；创建 identity 值和显式轮换仍使用普通事务 | 保留支持并发的 identity 表初始化 | | 离线 Memory 归档 | Server 版本链在 `s1401` 登记结构，运行时准入检查该结构 | 旧 Memory 离线转换负责其归档和转换回执 | | `atomic-memory-migrate`、`processing-migrate` | 转发到共享 Server 或已登记的维护计划 | 保留领域独立行为；识别为历史完整 Server 的库必须先完成整库升级 | -| 向量对象 | 尚无已登记的配置转换，添加对象前拒绝 | 保留既有功能初始化，不宣称已登记升级支持 | +| 向量对象 | 所选 Server profile 负责结构、辅助索引回填及精确配置准入；切换 embedding 空间需要独立转换 | 完整的已发布向量结构可显式迁移；独立未受管 SDK 存储保留领域初始化 | | 嵌入式 seekDB Code 对象 | 已登记 Code profile 创建并验证三张表及全文索引；Code 启动不修补缺失的受管对象 | 完整的已发布 Code 启用结构可显式迁移；独立未受管 Code 保留既有初始化 | | SQLite、OceanBase 的独立 Code 缓存 | 保留缓存文件，主数据库走共享版本链 | Code 服务负责本地缓存创建和索引构建 | diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 6b7a890e61..2be951b6b7 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "3694fdad13add880be1f87506ceced65e247c7b407ccc0fd09b7db64ae3c901f", - "rationale": "The shared s1300 -> s1400 -> s1401 chain covers installed 1.3.0 no-vector Server storage on SQLite, seekDB and OceanBase with Code disabled or enabled. SQLite/OceanBase Code caches remain separate. The seekDB Code extension freezes three exact tables and their indexes captured from the installed source wheel. The same core chain preserves these rows and initializes this extension for a new Code-enabled database; all existing revision checksums remain unchanged. Complete source/target and DDL-prefix validation include the extension. Startup and standalone Code initialization cannot add or repair managed Code objects. Existing managed storage without Code tables cannot enable the feature by changing a flag. Code-enabled installed acceptance on all three backends checks original fingerprints, symbols and immutable cache files; embedded acceptance additionally compares exact graph-table row digests and fresh/upgraded schemas. Strict Server admission remains intact; Server test fixtures initialize through a Server before SDK writes. Target locks, original recovery points, acknowledgements, historical authority and backup policies remain shared. Vector configurations and complete recovery/restore acceptance remain unfinished; release policy stays upgrade-required. Native VECTOR KEY reflection now records ordered key columns and the explicit HNSW algorithm, distance, library and construction/search parameters instead of treating the clause as a column. Duplicate, unknown or malformed vector index options fail reflection. This parser correction does not register or enable vector upgrade configurations, alter schema, or change existing revision checksums." + "persistence_sha256": "0c8246b49cc05493e86bd011219c81306cca37d0ef3babad651d434d18cafce5", + "rationale": "The shared s1300 -> s1400 -> s1401 chain registers installed 1.3.0 no-vector Server storage on SQLite, seekDB and OceanBase with Code disabled or enabled. SQLite/OceanBase Code caches remain separate; embedded seekDB Code uses the three frozen released tables. A selected vector profile binds the frozen schema and domain contract to the exact historical profile ID, model, dimension, distance and normalization. SQLite maintenance registers and backfills the Atomic vec0 helper and transactional triggers; native layouts preserve vector column dimensions and explicit HNSW parameters, including ordered key columns, algorithm, distance, library and construction/search options. Unknown index options remain rejected. The vector-only configuration declaration is not a second revision or recovery journal. Full layout and domain checks precede managed use; configured or injected model drift and missing helpers cannot trigger startup repair. No embedding provider is called during migration and historical embeddings, Topic shape, authority and evidence are preserved. Existing no-vector revision resources and all three checksums remain unchanged; selected vector profiles have separately bound checksums. Target locks, original recovery points, acknowledgements and backup policies remain shared. The declared installed-release gate still contains the six no-vector/Code cases; vector installed-artifact matrix and complete recovery/restore acceptance remain unfinished. Source tests and schema captures do not establish that release acceptance passed. Policy remains upgrade-required." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/persistence/migrations/bundle.py b/src/powercontext/builtin/persistence/migrations/bundle.py index 126d49d92c..f3cb808288 100644 --- a/src/powercontext/builtin/persistence/migrations/bundle.py +++ b/src/powercontext/builtin/persistence/migrations/bundle.py @@ -28,8 +28,11 @@ from alembic.script import ScriptDirectory from sqlalchemy import Connection +from powercontext.builtin.artifacts.memory import EmbeddingProfile + from .domains import DomainRegistry from .models import MigrationError, digest +from .vector_profile import ServerVectorProfile ALEMBIC_CONTEXT_LOCK = threading.RLock() @@ -42,7 +45,7 @@ class MigrationBundle: partially applied DDL cannot be mistaken for a supported baseline. """ - def __init__(self, directory: Path) -> None: # noqa: C901 -- Keep ordered admission checks with their operation. + def __init__(self, directory: Path, *, embedding_profile: EmbeddingProfile | None = None) -> None: # noqa: C901 -- Keep ordered admission checks with their operation. self.directory = directory.resolve() # Execute exactly the reviewed bytes, even if the installed resources # change during a long backup. Alembic reloads scripts on every command. @@ -69,6 +72,23 @@ def __init__(self, directory: Path) -> None: # noqa: C901 -- Keep ordered admis self.revisions = tuple(item.revision for item in revisions) self.head = self.revisions[-1] manifest = json.loads(resources["manifest.json"]) + self.vector_profile = ( + ServerVectorProfile(self.snapshot_directory, embedding_profile) if embedding_profile is not None else None + ) + if self.vector_profile is not None: + layouts = self.vector_profile.sqlite_layouts(self.snapshot_directory) + manifest["sqlite_fingerprints"] = {revision: [digest(rows)] for revision, rows in layouts.items()} + manifest["sqlite_baselines"] = {digest(layouts["s1300"]): "s1300"} + manifest["affected_objects"] = list( + dict.fromkeys([ + *manifest["affected_objects"], + *(row[1] for row in self.vector_profile.layout["sqlite"]["target"]), + ]) + ) + for step in manifest["domain_steps"]: + if step["id"] == "server_upgrade": + step["module"] = "server_vector_domain.py" + step["resources"].append(ServerVectorProfile.resource) mysql_layouts = manifest.get("mysql_layouts", []) if ( not isinstance(mysql_layouts, list) @@ -105,6 +125,8 @@ def __init__(self, directory: Path) -> None: # noqa: C901 -- Keep ordered admis domain_inputs = self.domains.checksum_inputs(item.revision, self.resources) if domain_inputs: inputs["domain_contract"] = digest(domain_inputs) + if self.vector_profile is not None: + inputs["server_vector_profile"] = self.vector_profile.checksum self.checksums[item.revision] = digest(inputs) self.checksum = digest(self.resources) diff --git a/src/powercontext/builtin/persistence/migrations/deployment.py b/src/powercontext/builtin/persistence/migrations/deployment.py index 33e79c9bcf..9c194fa86a 100644 --- a/src/powercontext/builtin/persistence/migrations/deployment.py +++ b/src/powercontext/builtin/persistence/migrations/deployment.py @@ -22,6 +22,7 @@ from sqlalchemy.engine import make_url +from powercontext.builtin.artifacts.memory import EmbeddingProfile from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.seekdb import SeekDBConfig from powercontext.builtin.persistence.sqlite import SQLiteConfig @@ -41,9 +42,9 @@ from powercontext.server.settings import ServerSettings -def server_bundle() -> MigrationBundle: +def server_bundle(*, embedding_profile: EmbeddingProfile | None = None) -> MigrationBundle: """The complete, shared Server revision chain.""" - return MigrationBundle(Path(__file__).parent / "server_resources") + return MigrationBundle(Path(__file__).parent / "server_resources", embedding_profile=embedding_profile) def production_bundle() -> MigrationBundle: @@ -75,6 +76,22 @@ def _migration_configuration(settings: ServerSettings) -> dict[str, Any]: } +def configured_embedding_profile(settings: ServerSettings) -> EmbeddingProfile | None: + inference = settings.inference + if inference.embedding_model is None: + return None + if inference.embedding_profile_id is None or inference.embedding_dimension is None: + raise MigrationError( + "embedding_profile_mismatch", "Embedding configuration needs its exact profile and dimension." + ) + return EmbeddingProfile( + profile_id=inference.embedding_profile_id, + model=inference.embedding_model, + dimension=inference.embedding_dimension, + normalization=inference.embedding_normalization, + ) + + def deployment_runner( # noqa: C901 -- Ordered backend admission checks. settings: ServerSettings, *, @@ -84,20 +101,28 @@ def deployment_runner( # noqa: C901 -- Ordered backend admission checks. bundle: Literal["server", "artifacts", "topic-memory-recovery", "processing", "atomic-projection"] = "server", migration_id: str = "rfc1515", batch_size: int = 100, + embedding_profile: EmbeddingProfile | None = None, ) -> MigrationRunner: """Use the real configured identity without opening a business profile.""" database = settings.database configuration = _migration_configuration(settings) + profile = embedding_profile or configured_embedding_profile(settings) if bundle in {"server", "processing", "atomic-projection"}: - if settings.inference.embedding_model is not None: + if profile is not None and bundle != "server": raise MigrationError( "unsupported_upgrade_configuration", - "The registered 1.3.0 Server path requires no vector provider.", + "This partial maintenance bundle has no registered vector configuration path.", ) - configuration["embedding_enabled"] = False + configuration["embedding_enabled"] = profile is not None + if profile is not None: + configuration["embedding_profile"] = profile.model_dump(mode="json") if settings.code.enabled and isinstance(database, SeekDBConfig): configuration["embedded_code_enabled"] = True - selected = server_bundle() if bundle in {"server", "processing", "atomic-projection"} else production_bundle() + selected = ( + server_bundle(embedding_profile=profile) + if bundle in {"server", "processing", "atomic-projection"} + else production_bundle() + ) maintenance_runner = ( AtomicProjectionMigrationRunner if bundle == "atomic-projection" else ProcessingConfigurationMigrationRunner ) diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py index 66d3fa2a25..e6b52dfeb3 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql_schema.py +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -150,6 +150,8 @@ def __init__(self, bundle: MigrationBundle, *, embedded_code: bool = False) -> N self._load_state_layouts(manifest["mysql_state_layouts"]) if embedded_code: self._extend_embedded_code(manifest) + if bundle.vector_profile is not None: + self._extend_vectors() return if embedded_code: raise MigrationError("invalid_bundle", "Embedded Code needs a registered complete Server layout.") @@ -293,6 +295,41 @@ def _extend_embedded_code(self, manifest: dict[str, Any]) -> None: def fingerprint(inventory: dict[str, Any]) -> str: return digest(inventory) + def _extend_vectors(self) -> None: + profile = self.bundle.vector_profile + if profile is None: + raise MigrationError("invalid_bundle", "Vector schema selection needs an exact profile.") + definitions = profile.mysql_tables() + self.optional_table_options.update(profile.layout["mysql_optional_table_options"]) + tables = dict(_parse_table(sql) for sql in definitions.values()) + atomic_name = "pc_atomic_memory_current" + atomic = tables.pop(atomic_name) + source, target, *later = self.revisions + for revision in self.revisions: + for state in [*self._prefixes[revision], self._schemas[revision]]: + if atomic_name in state["tables"]: + state["tables"][atomic_name] = copy.deepcopy(atomic) + if revision != source: + state["tables"].update(copy.deepcopy(tables)) + self._statements[source] = tuple( + definitions[atomic_name] if sql.startswith(f"CREATE TABLE `{atomic_name}` (") else sql + for sql in self._statements[source] + ) + source_state = self._schemas[source] + for name, table in tables.items(): + source_state["tables"][name] = copy.deepcopy(table) + self._prefixes[source].append(copy.deepcopy(source_state)) + self._statements[source] += (definitions[name],) + declaration_sql = profile.layout["mysql_profile_table"] + declaration_name, declaration = _parse_table(declaration_sql) + self._schemas[target]["tables"][declaration_name] = copy.deepcopy(declaration) + self._prefixes[target].append(copy.deepcopy(self._schemas[target])) + self._statements[target] += (declaration_sql,) + for revision in later: + for state in [*self._prefixes[revision], self._schemas[revision]]: + state["tables"][declaration_name] = copy.deepcopy(declaration) + self.affected_objects = tuple(dict.fromkeys([*self.affected_objects, *tables, declaration_name])) + def _matches(self, inventory: dict[str, Any], expected: dict[str, Any]) -> bool: # Captured backends may expose an explicit default that another backend # omits. Only registered values may be absent; retain it in fingerprints. diff --git a/src/powercontext/builtin/persistence/migrations/server_readiness.py b/src/powercontext/builtin/persistence/migrations/server_readiness.py index 8e6639ebaa..7f109f9df3 100644 --- a/src/powercontext/builtin/persistence/migrations/server_readiness.py +++ b/src/powercontext/builtin/persistence/migrations/server_readiness.py @@ -23,15 +23,23 @@ def check_server_schema(connection): bundle = check_server_layout(connection) backend = "sqlite" if connection.dialect.name == "sqlite" else "seekdb" - bundle.domains.validate(connection, bundle.head, backend=backend, configuration={}) + configuration = ( + {"embedding_profile": bundle.vector_profile.profile.model_dump(mode="json")} + if bundle.vector_profile is not None + else {} + ) + bundle.domains.validate(connection, bundle.head, backend=backend, configuration=configuration) def check_server_layout(connection): """Validate the exact managed layout before a registered domain repair.""" from .deployment import server_bundle from .mysql_schema import MySQLSchema, mysql_inventory + from .vector_profile import load_sqlite_vector_extension, read_embedding_profile - bundle = server_bundle() + bundle = server_bundle(embedding_profile=read_embedding_profile(connection)) + if connection.dialect.name == "sqlite" and bundle.vector_profile is not None: + load_sqlite_vector_extension(connection) revisions = connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalars().all() if revisions != [bundle.head]: raise MigrationError("migration_required", "The full Server revision chain is incomplete.") @@ -57,13 +65,14 @@ def check_server_layout(connection): return bundle -def prepare_server_storage(settings, *, injected_embedding=False): - from .deployment import deployment_runner +def prepare_server_storage(settings, *, injected_embedding=False, embedding_profile=None): + from .deployment import configured_embedding_profile, deployment_runner database = settings.database if database.kind == "sqlite" and database.is_in_memory: return - if injected_embedding or settings.inference.embedding_model is not None: + profile = embedding_profile or configured_embedding_profile(settings) + if injected_embedding and profile is None: _check_unmanaged_optional_profile(database) return if database.kind == "oceanbase": @@ -74,11 +83,15 @@ def verify(connection, identity, guard): raise MigrationError( "migration_required", "Initialize the remote database with an explicit db-migrate job." ) + bundle = check_server_layout(connection) + stored = bundle.vector_profile.profile if bundle.vector_profile is not None else None + if stored != profile: + raise MigrationError("embedding_profile_mismatch", "Server configuration differs from managed storage.") check_server_schema(connection) MaintenanceConnections(database).run(verify) return - runner = deployment_runner(settings) + runner = deployment_runner(settings, embedding_profile=profile) plan = runner.plan() if plan.state == "uninitialized" and database.kind in {"sqlite", "seekdb"}: runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/profiles/vector.json b/src/powercontext/builtin/persistence/migrations/server_resources/profiles/vector.json new file mode 100644 index 0000000000..ecce7ee1f8 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/profiles/vector.json @@ -0,0 +1,270 @@ +{ + "format": "powercontext.server-vector-profile.v1", + "source_release": "1.3.0", + "source_capture_sha256": { + "optional-release130-sqlite-vec3-20261023": "069f2153ac095fd00bfadbbb02b055d9bac54e4a8fd0f1c51dfa2ff88abccb92", + "optional-release130-oceanbase-vec3-e942bce93f": "b62b07ce28537fbdfcaeea9711bb6de1e3131794d0b76984c771527972ca876d" + }, + "sqlite": { + "source": [ + [ + "table", + "pc_topic_memory_chunk_vec", + "pc_topic_memory_chunk_vec", + "CREATE VIRTUAL TABLE pc_topic_memory_chunk_vec USING vec0(scope_id TEXT partition key, embedding float[__DIMENSION__])" + ], + [ + "table", + "pc_topic_memory_chunk_vec_chunks", + "pc_topic_memory_chunk_vec_chunks", + "CREATE TABLE \"pc_topic_memory_chunk_vec_chunks\"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,sequence_id integer,partition00,validity BLOB NOT NULL, rowids BLOB NOT NULL)" + ], + [ + "table", + "pc_topic_memory_chunk_vec_info", + "pc_topic_memory_chunk_vec_info", + "CREATE TABLE \"pc_topic_memory_chunk_vec_info\" (key text primary key, value any)" + ], + [ + "table", + "pc_topic_memory_chunk_vec_rowids", + "pc_topic_memory_chunk_vec_rowids", + "CREATE TABLE \"pc_topic_memory_chunk_vec_rowids\"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)" + ], + [ + "table", + "pc_topic_memory_chunk_vec_vector_chunks00", + "pc_topic_memory_chunk_vec_vector_chunks00", + "CREATE TABLE \"pc_topic_memory_chunk_vec_vector_chunks00\"(rowid PRIMARY KEY,vectors BLOB NOT NULL)" + ], + [ + "table", + "pc_topic_memory_topic_vec", + "pc_topic_memory_topic_vec", + "CREATE VIRTUAL TABLE pc_topic_memory_topic_vec USING vec0(scope_id TEXT partition key, embedding float[__DIMENSION__])" + ], + [ + "table", + "pc_topic_memory_topic_vec_chunks", + "pc_topic_memory_topic_vec_chunks", + "CREATE TABLE \"pc_topic_memory_topic_vec_chunks\"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,sequence_id integer,partition00,validity BLOB NOT NULL, rowids BLOB NOT NULL)" + ], + [ + "table", + "pc_topic_memory_topic_vec_info", + "pc_topic_memory_topic_vec_info", + "CREATE TABLE \"pc_topic_memory_topic_vec_info\" (key text primary key, value any)" + ], + [ + "table", + "pc_topic_memory_topic_vec_rowids", + "pc_topic_memory_topic_vec_rowids", + "CREATE TABLE \"pc_topic_memory_topic_vec_rowids\"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)" + ], + [ + "table", + "pc_topic_memory_topic_vec_vector_chunks00", + "pc_topic_memory_topic_vec_vector_chunks00", + "CREATE TABLE \"pc_topic_memory_topic_vec_vector_chunks00\"(rowid PRIMARY KEY,vectors BLOB NOT NULL)" + ], + [ + "table", + "pc_topic_memory_vector_chunks", + "pc_topic_memory_vector_chunks", + "CREATE TABLE pc_topic_memory_vector_chunks (\n\tvector_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, \n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tchunk_ordinal INTEGER NOT NULL, \n\tprofile_fingerprint VARCHAR(64) NOT NULL, \n\tCONSTRAINT uq_pc_topic_memory_vector_chunks_active UNIQUE (scope_id, artifact_id, chunk_ordinal)\n)" + ], + [ + "table", + "pc_topic_memory_vector_topics", + "pc_topic_memory_vector_topics", + "CREATE TABLE pc_topic_memory_vector_topics (\n\tvector_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, \n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tprofile_fingerprint VARCHAR(64) NOT NULL, \n\tCONSTRAINT uq_pc_topic_memory_vector_topics_active UNIQUE (scope_id, artifact_id)\n)" + ] + ], + "target": [ + [ + "table", + "pc_topic_memory_chunk_vec", + "pc_topic_memory_chunk_vec", + "CREATE VIRTUAL TABLE pc_topic_memory_chunk_vec USING vec0(scope_id TEXT partition key, embedding float[__DIMENSION__])" + ], + [ + "table", + "pc_topic_memory_chunk_vec_chunks", + "pc_topic_memory_chunk_vec_chunks", + "CREATE TABLE \"pc_topic_memory_chunk_vec_chunks\"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,sequence_id integer,partition00,validity BLOB NOT NULL, rowids BLOB NOT NULL)" + ], + [ + "table", + "pc_topic_memory_chunk_vec_info", + "pc_topic_memory_chunk_vec_info", + "CREATE TABLE \"pc_topic_memory_chunk_vec_info\" (key text primary key, value any)" + ], + [ + "table", + "pc_topic_memory_chunk_vec_rowids", + "pc_topic_memory_chunk_vec_rowids", + "CREATE TABLE \"pc_topic_memory_chunk_vec_rowids\"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)" + ], + [ + "table", + "pc_topic_memory_chunk_vec_vector_chunks00", + "pc_topic_memory_chunk_vec_vector_chunks00", + "CREATE TABLE \"pc_topic_memory_chunk_vec_vector_chunks00\"(rowid PRIMARY KEY,vectors BLOB NOT NULL)" + ], + [ + "table", + "pc_topic_memory_topic_vec", + "pc_topic_memory_topic_vec", + "CREATE VIRTUAL TABLE pc_topic_memory_topic_vec USING vec0(scope_id TEXT partition key, embedding float[__DIMENSION__])" + ], + [ + "table", + "pc_topic_memory_topic_vec_chunks", + "pc_topic_memory_topic_vec_chunks", + "CREATE TABLE \"pc_topic_memory_topic_vec_chunks\"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,sequence_id integer,partition00,validity BLOB NOT NULL, rowids BLOB NOT NULL)" + ], + [ + "table", + "pc_topic_memory_topic_vec_info", + "pc_topic_memory_topic_vec_info", + "CREATE TABLE \"pc_topic_memory_topic_vec_info\" (key text primary key, value any)" + ], + [ + "table", + "pc_topic_memory_topic_vec_rowids", + "pc_topic_memory_topic_vec_rowids", + "CREATE TABLE \"pc_topic_memory_topic_vec_rowids\"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)" + ], + [ + "table", + "pc_topic_memory_topic_vec_vector_chunks00", + "pc_topic_memory_topic_vec_vector_chunks00", + "CREATE TABLE \"pc_topic_memory_topic_vec_vector_chunks00\"(rowid PRIMARY KEY,vectors BLOB NOT NULL)" + ], + [ + "table", + "pc_topic_memory_vector_chunks", + "pc_topic_memory_vector_chunks", + "CREATE TABLE pc_topic_memory_vector_chunks (\n\tvector_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, \n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tchunk_ordinal INTEGER NOT NULL, \n\tprofile_fingerprint VARCHAR(64) NOT NULL, \n\tCONSTRAINT uq_pc_topic_memory_vector_chunks_active UNIQUE (scope_id, artifact_id, chunk_ordinal)\n)" + ], + [ + "table", + "pc_topic_memory_vector_topics", + "pc_topic_memory_vector_topics", + "CREATE TABLE pc_topic_memory_vector_topics (\n\tvector_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, \n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tprofile_fingerprint VARCHAR(64) NOT NULL, \n\tCONSTRAINT uq_pc_topic_memory_vector_topics_active UNIQUE (scope_id, artifact_id)\n)" + ], + [ + "table", + "pc_atomic_memory_current_vec", + "pc_atomic_memory_current_vec", + "CREATE VIRTUAL TABLE pc_atomic_memory_current_vec USING vec0(identity TEXT PRIMARY KEY, scope_id TEXT PARTITION KEY, kind TEXT, embedding float[__DIMENSION__], +artifact_id TEXT, +revision INTEGER, +state_version INTEGER, +text TEXT, chunk_size=128 /* __ATOMIC_FINGERPRINT__ */)" + ], + [ + "table", + "pc_atomic_memory_current_vec_auxiliary", + "pc_atomic_memory_current_vec_auxiliary", + "CREATE TABLE \"pc_atomic_memory_current_vec_auxiliary\"( rowid integer PRIMARY KEY , value00, value01, value02, value03)" + ], + [ + "table", + "pc_atomic_memory_current_vec_chunks", + "pc_atomic_memory_current_vec_chunks", + "CREATE TABLE \"pc_atomic_memory_current_vec_chunks\"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,sequence_id integer,partition00,validity BLOB NOT NULL, rowids BLOB NOT NULL)" + ], + [ + "table", + "pc_atomic_memory_current_vec_info", + "pc_atomic_memory_current_vec_info", + "CREATE TABLE \"pc_atomic_memory_current_vec_info\" (key text primary key, value any)" + ], + [ + "table", + "pc_atomic_memory_current_vec_metadatachunks00", + "pc_atomic_memory_current_vec_metadatachunks00", + "CREATE TABLE \"pc_atomic_memory_current_vec_metadatachunks00\"(rowid PRIMARY KEY, data BLOB NOT NULL)" + ], + [ + "table", + "pc_atomic_memory_current_vec_metadatatext00", + "pc_atomic_memory_current_vec_metadatatext00", + "CREATE TABLE \"pc_atomic_memory_current_vec_metadatatext00\"(rowid PRIMARY KEY, data TEXT)" + ], + [ + "table", + "pc_atomic_memory_current_vec_rowids", + "pc_atomic_memory_current_vec_rowids", + "CREATE TABLE \"pc_atomic_memory_current_vec_rowids\"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id TEXT UNIQUE NOT NULL,chunk_id INTEGER,chunk_offset INTEGER)" + ], + [ + "table", + "pc_atomic_memory_current_vec_vector_chunks00", + "pc_atomic_memory_current_vec_vector_chunks00", + "CREATE TABLE \"pc_atomic_memory_current_vec_vector_chunks00\"(rowid PRIMARY KEY,vectors BLOB NOT NULL)" + ], + [ + "trigger", + "pc_atomic_memory_vec_delete", + "pc_atomic_memory_current", + "CREATE TRIGGER pc_atomic_memory_vec_delete AFTER DELETE ON pc_atomic_memory_current BEGIN DELETE FROM pc_atomic_memory_current_vec WHERE identity = ('i' || lower(hex(old.scope_id)) || 'x' || lower(hex(old.artifact_id))); END" + ], + [ + "trigger", + "pc_atomic_memory_vec_insert", + "pc_atomic_memory_current", + "CREATE TRIGGER pc_atomic_memory_vec_insert AFTER INSERT ON pc_atomic_memory_current BEGIN INSERT INTO pc_atomic_memory_current_vec (identity, scope_id, kind, embedding, artifact_id, revision, state_version, text) SELECT 'i' || lower(hex(new.scope_id)) || 'x' || lower(hex(new.artifact_id)), new.scope_id, new.kind, new.embedding, new.artifact_id, new.revision, new.state_version, new.text WHERE NOT (new.embedding IS NULL OR new.embedding_input_hash IS NULL OR new.profile_fingerprint IS NULL OR new.profile_fingerprint <> '__ATOMIC_FINGERPRINT__' OR new.projection_format <> 'powercontext.atomic-memory.current.v1' OR LENGTH(new.embedding) <> __VECTOR_BYTES__); END" + ], + [ + "trigger", + "pc_atomic_memory_vec_update", + "pc_atomic_memory_current", + "CREATE TRIGGER pc_atomic_memory_vec_update AFTER UPDATE OF scope_id, artifact_id, kind, embedding, revision, state_version, text, embedding_input_hash, profile_fingerprint, projection_format ON pc_atomic_memory_current WHEN old.scope_id IS NOT new.scope_id OR old.artifact_id IS NOT new.artifact_id OR old.kind IS NOT new.kind OR old.embedding IS NOT new.embedding OR old.revision IS NOT new.revision OR old.state_version IS NOT new.state_version OR old.text IS NOT new.text OR old.embedding_input_hash IS NOT new.embedding_input_hash OR old.profile_fingerprint IS NOT new.profile_fingerprint OR old.projection_format IS NOT new.projection_format BEGIN DELETE FROM pc_atomic_memory_current_vec WHERE identity = ('i' || lower(hex(old.scope_id)) || 'x' || lower(hex(old.artifact_id))); INSERT INTO pc_atomic_memory_current_vec (identity, scope_id, kind, embedding, artifact_id, revision, state_version, text) SELECT 'i' || lower(hex(new.scope_id)) || 'x' || lower(hex(new.artifact_id)), new.scope_id, new.kind, new.embedding, new.artifact_id, new.revision, new.state_version, new.text WHERE NOT (new.embedding IS NULL OR new.embedding_input_hash IS NULL OR new.profile_fingerprint IS NULL OR new.profile_fingerprint <> '__ATOMIC_FINGERPRINT__' OR new.projection_format <> 'powercontext.atomic-memory.current.v1' OR LENGTH(new.embedding) <> __VECTOR_BYTES__); END" + ], + [ + "table", + "pc_server_storage_profile", + "pc_server_storage_profile", + "CREATE TABLE pc_server_storage_profile (singleton INTEGER NOT NULL PRIMARY KEY CHECK (singleton=1), embedding_profile TEXT NOT NULL)" + ] + ], + "initialize": [ + "CREATE VIRTUAL TABLE pc_topic_memory_chunk_vec USING vec0(scope_id TEXT partition key, embedding float[__DIMENSION__])", + "CREATE VIRTUAL TABLE pc_topic_memory_topic_vec USING vec0(scope_id TEXT partition key, embedding float[__DIMENSION__])", + "CREATE TABLE pc_topic_memory_vector_chunks (\n\tvector_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, \n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tchunk_ordinal INTEGER NOT NULL, \n\tprofile_fingerprint VARCHAR(64) NOT NULL, \n\tCONSTRAINT uq_pc_topic_memory_vector_chunks_active UNIQUE (scope_id, artifact_id, chunk_ordinal)\n)", + "CREATE TABLE pc_topic_memory_vector_topics (\n\tvector_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, \n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tprofile_fingerprint VARCHAR(64) NOT NULL, \n\tCONSTRAINT uq_pc_topic_memory_vector_topics_active UNIQUE (scope_id, artifact_id)\n)" + ], + "upgrade": [ + "CREATE VIRTUAL TABLE pc_atomic_memory_current_vec USING vec0(identity TEXT PRIMARY KEY, scope_id TEXT PARTITION KEY, kind TEXT, embedding float[__DIMENSION__], +artifact_id TEXT, +revision INTEGER, +state_version INTEGER, +text TEXT, chunk_size=128 /* __ATOMIC_FINGERPRINT__ */)", + "CREATE TRIGGER pc_atomic_memory_vec_delete AFTER DELETE ON pc_atomic_memory_current BEGIN DELETE FROM pc_atomic_memory_current_vec WHERE identity = ('i' || lower(hex(old.scope_id)) || 'x' || lower(hex(old.artifact_id))); END", + "CREATE TRIGGER pc_atomic_memory_vec_insert AFTER INSERT ON pc_atomic_memory_current BEGIN INSERT INTO pc_atomic_memory_current_vec (identity, scope_id, kind, embedding, artifact_id, revision, state_version, text) SELECT 'i' || lower(hex(new.scope_id)) || 'x' || lower(hex(new.artifact_id)), new.scope_id, new.kind, new.embedding, new.artifact_id, new.revision, new.state_version, new.text WHERE NOT (new.embedding IS NULL OR new.embedding_input_hash IS NULL OR new.profile_fingerprint IS NULL OR new.profile_fingerprint <> '__ATOMIC_FINGERPRINT__' OR new.projection_format <> 'powercontext.atomic-memory.current.v1' OR LENGTH(new.embedding) <> __VECTOR_BYTES__); END", + "CREATE TRIGGER pc_atomic_memory_vec_update AFTER UPDATE OF scope_id, artifact_id, kind, embedding, revision, state_version, text, embedding_input_hash, profile_fingerprint, projection_format ON pc_atomic_memory_current WHEN old.scope_id IS NOT new.scope_id OR old.artifact_id IS NOT new.artifact_id OR old.kind IS NOT new.kind OR old.embedding IS NOT new.embedding OR old.revision IS NOT new.revision OR old.state_version IS NOT new.state_version OR old.text IS NOT new.text OR old.embedding_input_hash IS NOT new.embedding_input_hash OR old.profile_fingerprint IS NOT new.profile_fingerprint OR old.projection_format IS NOT new.projection_format BEGIN DELETE FROM pc_atomic_memory_current_vec WHERE identity = ('i' || lower(hex(old.scope_id)) || 'x' || lower(hex(old.artifact_id))); INSERT INTO pc_atomic_memory_current_vec (identity, scope_id, kind, embedding, artifact_id, revision, state_version, text) SELECT 'i' || lower(hex(new.scope_id)) || 'x' || lower(hex(new.artifact_id)), new.scope_id, new.kind, new.embedding, new.artifact_id, new.revision, new.state_version, new.text WHERE NOT (new.embedding IS NULL OR new.embedding_input_hash IS NULL OR new.profile_fingerprint IS NULL OR new.profile_fingerprint <> '__ATOMIC_FINGERPRINT__' OR new.projection_format <> 'powercontext.atomic-memory.current.v1' OR LENGTH(new.embedding) <> __VECTOR_BYTES__); END", + "CREATE TABLE pc_server_storage_profile (singleton INTEGER NOT NULL PRIMARY KEY CHECK (singleton=1), embedding_profile TEXT NOT NULL)" + ], + "rebuilt_tables": [ + "pc_candidate_heads", + "pc_candidate_versions", + "pc_profile_policies", + "pc_artifacts", + "pc_artifact_processing_intents", + "pc_dream_runs", + "pc_topic_memory_work_budgets" + ] + }, + "mysql": { + "pc_atomic_memory_current": "CREATE TABLE `pc_atomic_memory_current` (\n `scope_id` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `artifact_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `revision` bigint(20) NOT NULL,\n `state_version` bigint(20) NOT NULL,\n `content_hash` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `projection_format` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `kind` mediumtext NOT NULL,\n `text` mediumtext NOT NULL,\n `searchable_text` mediumtext NOT NULL,\n `tag_keys` mediumtext NOT NULL,\n `embedding` VECTOR(__DIMENSION__) DEFAULT NULL,\n `profile_fingerprint` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n `embedding_input_hash` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,\n PRIMARY KEY (`scope_id`, `artifact_id`),\n FULLTEXT KEY `ix_pc_atomic_memory_current_fts` (`searchable_text`) WITH PARSER space PARSER_PROPERTIES=(min_token_size=3,max_token_size=84) BLOCK_SIZE 16384,\n VECTOR KEY `ix_pc_atomic_memory_current_embedding` (`embedding`) WITH (DISTANCE=L2,TYPE=HNSW, LIB=VSAG, M=16, EF_CONSTRUCTION=200, EF_SEARCH=64) BLOCK_SIZE 16384,\n CONSTRAINT `ck_pc_atomic_memory_current_revision` CHECK ((`revision` > 0)),\n CONSTRAINT `ck_pc_atomic_memory_current_state_version` CHECK ((`state_version` >= 0)),\n CONSTRAINT `ck_pc_atomic_memory_current_embedding_metadata` CHECK ((((`embedding` is null) and (`profile_fingerprint` is null) and (`embedding_input_hash` is null)) or ((`embedding` is not null) and (`profile_fingerprint` is not null) and (`embedding_input_hash` is not null))))\n) DEFAULT CHARSET=utf8mb4", + "pc_topic_memory_vector_chunks": "CREATE TABLE `pc_topic_memory_vector_chunks` (\n `vector_id` bigint(20) NOT NULL AUTO_INCREMENT,\n `scope_id` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `artifact_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `revision` bigint(20) NOT NULL,\n `chunk_ordinal` bigint(20) NOT NULL,\n `profile_fingerprint` varchar(64) NOT NULL,\n `embedding` VECTOR(__DIMENSION__) NOT NULL,\n PRIMARY KEY (`vector_id`),\n UNIQUE KEY `uq_pc_topic_memory_vector_chunks_active` (`scope_id`, `artifact_id`, `chunk_ordinal`) BLOCK_SIZE 16384 LOCAL,\n VECTOR KEY `ix_pc_topic_memory_vector_chunks_embedding` (`embedding`) WITH (DISTANCE=L2,TYPE=HNSW, LIB=VSAG, M=16, EF_CONSTRUCTION=200, EF_SEARCH=64) BLOCK_SIZE 16384\n) DEFAULT CHARSET=utf8mb4", + "pc_topic_memory_vector_topics": "CREATE TABLE `pc_topic_memory_vector_topics` (\n `vector_id` bigint(20) NOT NULL AUTO_INCREMENT,\n `scope_id` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `artifact_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n `revision` bigint(20) NOT NULL,\n `profile_fingerprint` varchar(64) NOT NULL,\n `embedding` VECTOR(__DIMENSION__) NOT NULL,\n PRIMARY KEY (`vector_id`),\n UNIQUE KEY `uq_pc_topic_memory_vector_topics_active` (`scope_id`, `artifact_id`) BLOCK_SIZE 16384 LOCAL,\n VECTOR KEY `ix_pc_topic_memory_vector_topics_embedding` (`embedding`) WITH (DISTANCE=L2,TYPE=HNSW, LIB=VSAG, M=16, EF_CONSTRUCTION=200, EF_SEARCH=64) BLOCK_SIZE 16384\n) DEFAULT CHARSET=utf8mb4" + }, + "mysql_profile_table": "CREATE TABLE pc_server_storage_profile (singleton BIGINT NOT NULL, embedding_profile MEDIUMTEXT NOT NULL, PRIMARY KEY (singleton), CONSTRAINT ck_pc_server_storage_profile_singleton CHECK (singleton=1))", + "mysql_optional_table_options": { + "pc_topic_memory_vector_topics": { + "auto_increment_mode": [ + "order" + ] + }, + "pc_topic_memory_vector_chunks": { + "auto_increment_mode": [ + "order" + ] + } + } +} diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/server_vector_domain.py b/src/powercontext/builtin/persistence/migrations/server_resources/server_vector_domain.py new file mode 100644 index 0000000000..cbc6c587e0 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/server_resources/server_vector_domain.py @@ -0,0 +1,379 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Registered Server vector maintenance and live readiness requirements.""" + +import hashlib +import json +import runpy +from pathlib import Path + +from sqlalchemy import inspect, text + +from powercontext.builtin.persistence.experience_index import ( + APPROVED_ARTIFACT_PROJECTION_VERSION, + rebuild_experience_projections, + rebuild_skill_projections, +) +from powercontext.builtin.persistence.lexical_projection import ( + LEXICAL_ANALYZER_VERSION, + record_lexical_projection_version, + refresh_lexical_projections, +) +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.pinned import finish, pinned + +assert_atomic_memory_migration_ready = runpy.run_path(str(Path(__file__).parent / "atomic_memory_readiness_1300.py"))[ + "assert_atomic_memory_migration_ready" +] + + +def dream(): + return runpy.run_path(str(Path(__file__).parent / "dream/upgrade.py")) + + +def preflight(connection, *, configuration): + names = set(inspect(connection).get_table_names()) + if not names or names == {"pc_schema_revision"}: + return + _validate_vector_configuration(connection, configuration) + # Real 1.3.0 publishes Atomic Memory already. An old schema with pre-1.3 + # collection data needs its separately declared conversion, never a stamp. + finish(assert_atomic_memory_migration_ready(pinned(connection))) + if "pc_artifact_candidate_versions" in names: + _validate_intent_bindings(connection) + module = dream() + try: + module["validate_dream_tasks"](connection) + module["validate_dream_processing_manifest"]( + connection, target_manifest=configuration["processing_manifest"] + ) + except module["DreamUpgradeError"] as error: + raise MigrationError( + error.reason, "The legacy Dream task or Processing configuration needs maintenance." + ) from error + + +def apply(connection, *, configuration): + _record_profile(connection, configuration) + # Dream's schema revision preserves the legacy manifest; this is the only + # permitted ownership transformation, not an unconditional target overwrite. + old = json.loads( + connection.exec_driver_sql( + "SELECT config_manifest FROM pc_artifact_processing_schema WHERE singleton=1" + ).scalar_one() + ) + if "handoff.dream.v1" not in old["bindings"]: + upgraded = dream()["validate_dream_processing_manifest"]( + connection, target_manifest=configuration["processing_manifest"] + ) + connection.execute( + text("UPDATE pc_artifact_processing_schema SET config_manifest=:value WHERE singleton=1"), + {"value": json.dumps(upgraded, sort_keys=True, separators=(",", ":"))}, + ) + connection.exec_driver_sql( + "INSERT INTO pc_access_relationship_heads (name,revision) SELECT 'authorization',0 WHERE NOT EXISTS (SELECT 1 FROM pc_access_relationship_heads WHERE name='authorization')" + ) + async_connection = pinned(connection) + from powercontext.builtin.persistence.atomic_memory_index_schema import atomic_memory_current_table + + finish(refresh_lexical_projections(async_connection, atomic_memory_current_table())) + finish(rebuild_experience_projections(async_connection)) + finish(rebuild_skill_projections(async_connection)) + finish( + record_lexical_projection_version( + async_connection, "pc_artifact_heads", version=APPROVED_ARTIFACT_PROJECTION_VERSION + ) + ) + if connection.dialect.name == "sqlite": + from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex + + finish(SQLiteAtomicMemoryIndex().rebuild_fts(async_connection)) + connection.exec_driver_sql("DELETE FROM pc_artifact_fts") + connection.exec_driver_sql( + "INSERT INTO pc_artifact_fts (scope_id,family,artifact_id,revision,searchable_text) SELECT scope_id,family,artifact_id,revision,searchable_text FROM pc_artifact_heads WHERE family IN ('experience','skill') AND searchable_text IS NOT NULL" + ) + # Both FTS tables are registered schema; this initializer only refreshes + # their rows and existing marker when the analyzer has changed. + from powercontext.builtin.persistence.sqlite.topic_memory_index import SQLiteTopicMemoryFTSIndex + from powercontext.builtin.persistence.topic_memory import TopicMemoryRepository + + finish(SQLiteTopicMemoryFTSIndex().initialize(async_connection)) + _backfill_atomic_vectors(connection, configuration) + else: + from powercontext.builtin.persistence.oceanbase.topic_memory_index import OceanBaseTopicMemoryFTSIndex + + finish(OceanBaseTopicMemoryFTSIndex().initialize(async_connection)) + _ensure_shape(connection, configuration) + from powercontext.builtin.persistence.topic_memory import TopicMemoryRepository + + finish( + TopicMemoryRepository(index=_topic_index(connection, configuration)).initialize( + async_connection, validate_only=True + ) + ) + + +def verify(connection, *, configuration): + dream()["verify_dream_storage"](connection) + validate(connection, configuration=configuration) + + +def validate(connection, *, configuration): + _validate_vector_configuration(connection, configuration) + _require_declaration(connection, configuration) + finish(assert_atomic_memory_migration_ready(pinned(connection))) + expected = { + "pc_atomic_memory_current": LEXICAL_ANALYZER_VERSION, + "pc_topic_memory_active_topics": LEXICAL_ANALYZER_VERSION, + "pc_topic_memory_active_chunks": LEXICAL_ANALYZER_VERSION, + "pc_artifact_heads": APPROVED_ARTIFACT_PROJECTION_VERSION, + } + actual = dict( + connection + .exec_driver_sql("SELECT projection, analyzer_version FROM pc_lexical_projection_versions") + .tuples() + .all() + ) + if actual != expected: + raise MigrationError("projection_migration_required", "The registered search projections are not current.") + frozen = dream() + manifest = frozen["_processing_manifest"](connection, frozen["_schema"](), upgraded=True) + _validate_intent_bindings(connection) + target = configuration.get("processing_manifest") + if target and any(manifest[key] != target[key] for key in ("bindings", "mode", "capabilities")): + raise MigrationError( + "incompatible_processing_configuration", "Processing ownership differs from the migrated declaration." + ) + from powercontext.builtin.persistence.dream import DreamRepository + + for row in connection.exec_driver_sql( + "SELECT scope_id,run_id,status,operation,generation,request_generation,principal_key,idempotency_key FROM pc_dream_runs" + ).mappings(): + record = finish(DreamRepository().get(pinned(connection), row["scope_id"], row["run_id"])) + if ( + any(getattr(record.run, key) != row[key] for key in ("scope_id", "run_id", "status", "operation")) + or record.generation != row["generation"] + or record.request_generation != row["request_generation"] + or hashlib.sha256(record.principal_id.encode()).hexdigest() != row["principal_key"] + or (record.request is not None and record.request.idempotency_key != row["idempotency_key"]) + ): + raise MigrationError("unsupported_task_format", "A Dream payload disagrees with its durable task identity.") + from powercontext.builtin.persistence.processing_intents import ArtifactProcessingIntentRepository + + for scope, binding in connection.exec_driver_sql( + "SELECT scope_id,binding_name FROM pc_artifact_processing_intents" + ): + finish(ArtifactProcessingIntentRepository().load(pinned(connection), scope, binding)) + # Check exact current projection identities without comparing mutable heads + # to retained pre-upgrade Candidate snapshots. + if connection.exec_driver_sql( + "SELECT 1 FROM pc_atomic_memory_current p WHERE NOT EXISTS (SELECT 1 FROM pc_artifact_heads h WHERE h.scope_id=p.scope_id AND h.family='atomic-memory' AND h.artifact_id=p.artifact_id AND h.revision=p.revision) LIMIT 1" + ).first(): + raise MigrationError( + "projection_verification_failed", "An Atomic Memory projection references a missing current head." + ) + + from powercontext.builtin.persistence.topic_memory import TopicMemoryRepository + + if connection.dialect.name == "sqlite": + _validate_sqlite_fts(connection) + _validate_atomic_helper(connection, configuration) + finish( + TopicMemoryRepository(index=_topic_index(connection, configuration)).initialize( + pinned(connection), validate_only=True + ) + ) + + +def _validate_sqlite_fts(connection): + # Read-only multiset comparison also catches duplicate/stale helper rows. + projections = ( + ( + "pc_atomic_memory_current", + "pc_atomic_memory_current_fts", + "scope_id,artifact_id,revision,state_version,content_hash,projection_format,kind,text,searchable_text,tag_keys", + "", + ), + ( + "pc_artifact_heads", + "pc_artifact_fts", + "scope_id,family,artifact_id,revision,searchable_text", + " WHERE family IN ('experience','skill') AND searchable_text IS NOT NULL", + ), + ( + "pc_topic_memory_active_topics", + "pc_topic_memory_topic_fts", + "scope_id,artifact_id,revision,searchable_text", + "", + ), + ( + "pc_topic_memory_active_chunks", + "pc_topic_memory_chunk_fts", + "scope_id,artifact_id,revision,chunk_ordinal,searchable_text", + "", + ), + ) + for source, helper, columns, predicate in projections: + expected = f"SELECT {columns},COUNT(*) FROM {source}{predicate} GROUP BY {columns}" # noqa: S608 -- Only frozen schema names and projections are interpolated. + actual = f"SELECT {columns},COUNT(*) FROM {helper} GROUP BY {columns}" # noqa: S608 -- Only frozen schema names and projections are interpolated. + for left, right in ((expected, actual), (actual, expected)): + if connection.exec_driver_sql(f"SELECT 1 FROM ({left} EXCEPT {right}) LIMIT 1").first(): # noqa: S608 -- Only frozen schema names and projections are interpolated. + raise MigrationError( + "projection_verification_failed", "A full-text helper differs from its authoritative current rows." + ) + + +def _validate_intent_bindings(connection): + manifest = json.loads( + connection.exec_driver_sql( + "SELECT config_manifest FROM pc_artifact_processing_schema WHERE singleton=1" + ).scalar_one() + ) + for (binding,) in connection.exec_driver_sql("SELECT DISTINCT binding_name FROM pc_artifact_processing_intents"): + if binding not in manifest["bindings"]: + raise MigrationError("unsupported_task_format", "A pending Processing intent has no registered binding.") + + +def _profile(configuration): + from powercontext.builtin.artifacts.memory import EmbeddingProfile + + try: + profile = EmbeddingProfile.model_validate(configuration["embedding_profile"]) + except (ValueError, KeyError) as error: + raise MigrationError("embedding_profile_mismatch", "The exact vector profile must be declared.") from error + if profile.dimension < 1 or profile.distance != "l2" or profile.normalization != "unit": + raise MigrationError( + "unsupported_upgrade_configuration", "Vectors require a positive unit-normalized L2 profile." + ) + return profile + + +def _validate_vector_configuration(connection, configuration): + from powercontext.builtin.persistence.atomic_memory_index import atomic_memory_profile_fingerprint + from powercontext.builtin.persistence.topic_memory_index import topic_memory_embedding_profile_fingerprint + + profile = _profile(configuration) + atomic = atomic_memory_profile_fingerprint(profile) + topic = topic_memory_embedding_profile_fingerprint(profile) + row = connection.exec_driver_sql("SELECT shape,profile_fingerprint FROM pc_topic_memory_retrieval_shape").first() + if row is not None and tuple(row) != ("hybrid", topic): + raise MigrationError( + "embedding_profile_mismatch", "The historical Topic retrieval space differs from configuration." + ) + if row is None and connection.exec_driver_sql("SELECT 1 FROM pc_topic_memory_active_topics LIMIT 1").first(): + raise MigrationError("embedding_profile_mismatch", "Historical Topics have no declared retrieval space.") + invalid = "embedding IS NOT NULL AND (profile_fingerprint IS NULL OR profile_fingerprint<>:profile)" + if connection.dialect.name == "sqlite": + invalid += " OR (embedding IS NOT NULL AND length(embedding)<>:length)" + if connection.execute( + text("SELECT 1 FROM pc_atomic_memory_current WHERE " + invalid + " LIMIT 1"), # noqa: S608 -- Only two fixed backend predicates are concatenated. + {"profile": atomic, "length": profile.dimension * 4}, + ).first(): + raise MigrationError( + "embedding_profile_mismatch", "Historical Atomic Memory vectors differ from configuration." + ) + for name in ("pc_topic_memory_vector_topics", "pc_topic_memory_vector_chunks"): + if connection.execute( + text(f"SELECT 1 FROM {name} WHERE profile_fingerprint<>:profile LIMIT 1"), # noqa: S608 -- Both table names are frozen above. + {"profile": topic}, + ).first(): + raise MigrationError("embedding_profile_mismatch", "Historical Topic vectors differ from configuration.") + + +def _require_declaration(connection, configuration): + from powercontext.builtin.persistence.migrations.vector_profile import read_embedding_profile + + if read_embedding_profile(connection) != _profile(configuration): + raise MigrationError("embedding_profile_mismatch", "The managed vector configuration declaration differs.") + + +def _record_profile(connection, configuration): + row = connection.exec_driver_sql("SELECT embedding_profile FROM pc_server_storage_profile").first() + if row is None: + connection.execute( + text("INSERT INTO pc_server_storage_profile (singleton,embedding_profile) VALUES (1,:profile)"), + {"profile": _profile(configuration).model_dump_json()}, + ) + _require_declaration(connection, configuration) + + +def _ensure_shape(connection, configuration): + from powercontext.builtin.persistence.topic_memory_index import topic_memory_embedding_profile_fingerprint + + if connection.exec_driver_sql("SELECT 1 FROM pc_topic_memory_retrieval_shape").first() is None: + if connection.exec_driver_sql("SELECT 1 FROM pc_topic_memory_active_topics LIMIT 1").first(): + raise MigrationError("embedding_profile_mismatch", "Historical Topics have no declared retrieval space.") + connection.execute( + text( + "INSERT INTO pc_topic_memory_retrieval_shape (singleton,shape,profile_fingerprint) VALUES (1,'hybrid',:profile)" + ), + {"profile": topic_memory_embedding_profile_fingerprint(_profile(configuration))}, + ) + + +def _topic_index(connection, configuration): + from powercontext.builtin.persistence.topic_memory_index import CompositeTopicMemoryIndex + + if connection.dialect.name == "sqlite": + from powercontext.builtin.persistence.sqlite.topic_memory_index import ( + SQLiteTopicMemoryFTSIndex, + SQLiteTopicMemoryVectorIndex, + ) + + return CompositeTopicMemoryIndex( + SQLiteTopicMemoryFTSIndex(), SQLiteTopicMemoryVectorIndex(_profile(configuration)) + ) + from powercontext.builtin.persistence.oceanbase.topic_memory_index import ( + OceanBaseTopicMemoryFTSIndex, + OceanBaseTopicMemoryVectorIndex, + ) + + return CompositeTopicMemoryIndex( + OceanBaseTopicMemoryFTSIndex(), OceanBaseTopicMemoryVectorIndex(_profile(configuration)) + ) + + +def _backfill_atomic_vectors(connection, configuration): + from powercontext.builtin.persistence.atomic_memory_index import atomic_memory_profile_fingerprint + + profile = _profile(configuration) + connection.exec_driver_sql("DELETE FROM pc_atomic_memory_current_vec") + connection.execute( + text( + "INSERT INTO pc_atomic_memory_current_vec (identity,scope_id,kind,embedding,artifact_id,revision,state_version,text) " + "SELECT 'i'||lower(hex(scope_id))||'x'||lower(hex(artifact_id)),scope_id,kind,embedding,artifact_id,revision,state_version,text " + "FROM pc_atomic_memory_current WHERE embedding IS NOT NULL AND embedding_input_hash IS NOT NULL " + "AND profile_fingerprint=:profile AND projection_format='powercontext.atomic-memory.current.v1' AND length(embedding)=:length" + ), + {"profile": atomic_memory_profile_fingerprint(profile), "length": profile.dimension * 4}, + ) + + +def _validate_atomic_helper(connection, configuration): + from powercontext.builtin.persistence.atomic_memory_index import atomic_memory_profile_fingerprint + + profile = _profile(configuration) + expected = ( + "SELECT 'i'||lower(hex(scope_id))||'x'||lower(hex(artifact_id)),scope_id,kind,embedding,artifact_id,revision,state_version,text " + "FROM pc_atomic_memory_current WHERE embedding IS NOT NULL AND embedding_input_hash IS NOT NULL " + "AND profile_fingerprint=:profile AND projection_format='powercontext.atomic-memory.current.v1' AND length(embedding)=:length" + ) + actual = "SELECT identity,scope_id,kind,embedding,artifact_id,revision,state_version,text FROM pc_atomic_memory_current_vec" + values = {"profile": atomic_memory_profile_fingerprint(profile), "length": profile.dimension * 4} + for left, right in ((expected, actual), (actual, expected)): + if connection.execute(text(f"SELECT 1 FROM ({left} EXCEPT {right}) LIMIT 1"), values).first(): # noqa: S608 -- Both complete projections are frozen above. + raise MigrationError( + "projection_verification_failed", "The Atomic vector helper differs from current rows." + ) diff --git a/src/powercontext/builtin/persistence/migrations/sqlite.py b/src/powercontext/builtin/persistence/migrations/sqlite.py index 951a0f3c51..213b6796dc 100644 --- a/src/powercontext/builtin/persistence/migrations/sqlite.py +++ b/src/powercontext/builtin/persistence/migrations/sqlite.py @@ -252,9 +252,20 @@ def _check_file(self) -> None: @contextmanager def _connect(self, *, read_only: bool) -> Iterator[Connection]: uri = self.database.as_uri() + ("?mode=ro" if read_only else "?mode=rwc") - engine = create_engine( - "sqlite://", creator=lambda: sqlite3.connect(uri, uri=True, isolation_level=None, timeout=0) - ) + + def connect() -> sqlite3.Connection: + driver = sqlite3.connect(uri, uri=True, isolation_level=None, timeout=0) + if self.bundle.vector_profile is not None: + import sqlite_vec + + driver.enable_load_extension(True) + try: + sqlite_vec.load(driver) + finally: + driver.enable_load_extension(False) + return driver + + engine = create_engine("sqlite://", creator=connect) try: with engine.connect() as connection: if read_only: @@ -535,12 +546,16 @@ def _upgrade(self, connection: Connection) -> None: self._verify_schema(connection, current) with ALEMBIC_CONTEXT_LOCK: command.upgrade(self._config(connection), revision) + if self.bundle.vector_profile is not None: + self.bundle.vector_profile.sqlite_upgrade(connection, revision) self.bundle.domains.apply(connection, revision, backend="sqlite", configuration=self.configuration) self._verify_schema(connection, revision) connection.commit() current = revision def _verify_schema(self, connection: Connection, revision: str | None) -> None: + if self.bundle.vector_profile is not None: + self.bundle.vector_profile.verify_declaration(connection, required=False) excluded = _CONTROL_NAMES | frozenset(self.bundle.retained_objects) inventory = sqlite_inventory(connection) self._verify_retained(inventory, revision) diff --git a/src/powercontext/builtin/persistence/migrations/vector_profile.py b/src/powercontext/builtin/persistence/migrations/vector_profile.py new file mode 100644 index 0000000000..83b86ae874 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/vector_profile.py @@ -0,0 +1,143 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Render the registered vector layout without invoking runtime initializers.""" + +from __future__ import annotations + +import copy +import json +import sqlite3 +from pathlib import Path +from typing import Any + +from sqlalchemy import Connection, inspect +from sqlalchemy.engine import AdaptedConnection +from sqlalchemy.exc import OperationalError + +from powercontext.builtin.artifacts.memory import EmbeddingProfile +from powercontext.builtin.persistence.atomic_memory_index import atomic_memory_profile_fingerprint + +from .models import MigrationError, digest + + +class ServerVectorProfile: + """Bind frozen schema templates to one exact embedding space.""" + + resource = "profiles/vector.json" + + def __init__(self, directory: Path, profile: EmbeddingProfile): + if profile.dimension < 1 or profile.distance != "l2" or profile.normalization != "unit": + raise MigrationError( + "unsupported_upgrade_configuration", "Server vectors require a positive unit-normalized L2 profile." + ) + self.profile = profile.model_copy(deep=True) + source = (directory / self.resource).read_text(encoding="utf-8") + self.layout: dict[str, Any] = json.loads( + source + .replace("__DIMENSION__", str(profile.dimension)) + .replace("__VECTOR_BYTES__", str(profile.dimension * 4)) + .replace("__ATOMIC_FINGERPRINT__", atomic_memory_profile_fingerprint(profile)) + ) + if self.layout.get("format") != "powercontext.server-vector-profile.v1": + raise MigrationError("invalid_bundle", "The Server vector schema registration is invalid.") + self.checksum = digest({ + "template": json.loads(source), + "profile": profile.model_dump(mode="json"), + }) + + def sqlite_layouts(self, directory: Path) -> dict[str, list[list[Any]]]: + core = json.loads((directory / "schemas/sqlite.json").read_text(encoding="utf-8")) + archive = json.loads((directory / "schemas/sqlite-s1401.json").read_text(encoding="utf-8"))["create"] + layouts = { + "s1300": core["source"] + self.layout["sqlite"]["source"], + "s1400": core["target"] + self.layout["sqlite"]["target"], + } + # The frozen s1400 table rebuild uses ALTER TABLE RENAME, whose stored + # definition quotes the final name. Match those exact reviewed bytes. + for row in layouts["s1400"]: + if row[0] == "table" and row[1] in self.layout["sqlite"]["rebuilt_tables"]: + row[3] = row[3].replace(f"CREATE TABLE {row[1]} ", f'CREATE TABLE "{row[1]}" ', 1) + layouts["s1401"] = layouts["s1400"] + [ + ["table", "pc_memory_artifact_archive", "pc_memory_artifact_archive", archive] + ] + for entries in layouts.values(): + entries.sort(key=lambda row: (row[0], row[1])) + return layouts + + def sqlite_upgrade(self, connection: Connection, revision: str) -> None: + operations = {"s1300": "initialize", "s1400": "upgrade"} + if revision in operations: + for statement in self.layout["sqlite"][operations[revision]]: + connection.exec_driver_sql(statement) + + def verify_declaration(self, connection: Connection, *, required: bool) -> None: + stored = read_embedding_profile(connection) + if stored is None: + if required: + raise MigrationError("embedding_profile_mismatch", "The managed vector profile declaration is missing.") + elif stored != self.profile: + raise MigrationError( + "embedding_profile_mismatch", "The configured embedding profile differs from the managed database." + ) + + def mysql_tables(self) -> dict[str, str]: + return copy.deepcopy(self.layout["mysql"]) + + +def read_embedding_profile(connection: Connection) -> EmbeddingProfile | None: + """Read a managed configuration declaration; never infer it from vector bytes.""" + if "pc_server_storage_profile" not in inspect(connection).get_table_names(): + return None + try: + rows = connection.exec_driver_sql("SELECT singleton,embedding_profile FROM pc_server_storage_profile").all() + if len(rows) != 1 or rows[0][0] != 1: + raise ValueError("The profile declaration must contain one singleton") # noqa: TRY301, TRY003 + return EmbeddingProfile.model_validate_json(rows[0][1]) + except (ValueError, TypeError) as error: + raise MigrationError( + "embedding_profile_mismatch", "The managed vector profile declaration is invalid." + ) from error + + +def load_sqlite_vector_extension(connection: Connection) -> None: + """Allow generic storage readers to validate a managed vector database.""" + try: + connection.exec_driver_sql("SELECT vec_version()") + except OperationalError as error: + if not isinstance(error.orig, sqlite3.Error) or "no such function: vec_version" not in str(error.orig): + raise + else: + return + import sqlite_vec + + driver = connection.connection.dbapi_connection + if isinstance(driver, AdaptedConnection): + + async def load(raw): + await raw.enable_load_extension(True) + try: + await raw.load_extension(sqlite_vec.loadable_path()) + finally: + await raw.enable_load_extension(False) + + driver.run_async(load) + elif isinstance(driver, sqlite3.Connection): + driver.enable_load_extension(True) + try: + sqlite_vec.load(driver) + finally: + driver.enable_load_extension(False) + else: + raise MigrationError("unsupported_backend", "The managed SQLite vector profile needs a supported driver.") diff --git a/src/powercontext/server/factory.py b/src/powercontext/server/factory.py index 8becbe498f..a58a725eb9 100644 --- a/src/powercontext/server/factory.py +++ b/src/powercontext/server/factory.py @@ -182,7 +182,11 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: _log_in_memory_database_warning() from powercontext.builtin.persistence.migrations.server_readiness import prepare_server_storage - await asyncio.to_thread(prepare_server_storage, resolved, injected_embedding=embedding_model is not None) + await asyncio.to_thread( + prepare_server_storage, + resolved, + embedding_profile=None if embedding_model is None else embedding_model.profile, + ) configured_cursor_secret = ( None if resolved.cursor_signing_secret is None else resolved.cursor_signing_secret.get_secret_value() ) diff --git a/tests/builtin/persistence/test_full_server_migrations.py b/tests/builtin/persistence/test_full_server_migrations.py index afee2a391d..458ebf7f4b 100644 --- a/tests/builtin/persistence/test_full_server_migrations.py +++ b/tests/builtin/persistence/test_full_server_migrations.py @@ -271,8 +271,8 @@ def test_code_with_separate_cache_upgrades_and_reopens_server(source, tmp_path): ] -@pytest.mark.parametrize("bundle", ["server", "processing", "atomic-projection"]) -def test_unregistered_vector_storage_is_rejected_before_opening_database(tmp_path, bundle): +@pytest.mark.parametrize("bundle", ["processing", "atomic-projection"]) +def test_partial_vector_maintenance_is_rejected_before_opening_database(tmp_path, bundle): from powercontext.builtin.persistence.migrations.deployment import deployment_runner from powercontext.builtin.runtime.config import InferenceConfig from powercontext.server.settings import ServerSettings diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index dbff63dd2e..2a0d0fdc83 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -460,6 +460,71 @@ def _full_server_runner( ) +@pytest.mark.parametrize( + ("origin", "dimension", "code"), + [("released", 3, False), ("empty", 4, True)], +) +def test_mysql_vector_upgrade_preserves_space_and_registered_indexes( + backend: _BackendTarget, tmp_path: Path, origin: str, dimension: int, code: bool +) -> None: + from powercontext.builtin.artifacts.memory import EmbeddingProfile + from powercontext.builtin.code import CodeConfig + from powercontext.builtin.persistence.migrations.server_readiness import check_server_schema, prepare_server_storage + from powercontext.builtin.persistence.topic_memory_index import topic_memory_embedding_profile_fingerprint + from powercontext.builtin.runtime.config import InferenceConfig + + profile = EmbeddingProfile(profile_id="native-vectors", model="fixture", dimension=dimension) + bundle = server_bundle(embedding_profile=profile) + settings = ServerSettings( + database=backend.config, + code=CodeConfig(enabled=code, cache_dir=tmp_path / "code-cache"), + inference=InferenceConfig( + embedding_model=profile.model, + embedding_profile_id=profile.profile_id, + embedding_dimension=dimension, + ), + ) + runner = cast( + MySQLMigrationRunner, + deployment_runner( + settings, + evidence_dir=backend.evidence_directory, + lock_coordination="single-host" if isinstance(backend.config, OceanBaseConfig) else None, + ), + ) + if origin == "released": + + def initialize_source(connection, _identity, _guard): + baseline = runner.schema.recovery(mysql_inventory(connection), None) + for statement in baseline.pending_statements: + runner.schema.execute(connection, statement, runner.configuration) + connection.execute( + text("INSERT INTO pc_topic_memory_retrieval_shape VALUES (1,'hybrid',:profile)"), + {"profile": topic_memory_embedding_profile_fingerprint(profile)}, + ) + connection.commit() + + backend.connections().run(initialize_source, writable=True) + assert runner.plan(backup_policy="manual").source_revision == ("s1300" if origin == "released" else None) + assert _apply(runner, policy="manual").revision == bundle.head + assert not _apply(runner, policy="manual").changed + assert runner.verify().state == "ready" + prepare_server_storage(settings) + + def verify(connection, _identity, _guard): + check_server_schema(connection) + inventory = mysql_inventory(connection) + for name in ("pc_atomic_memory_current", "pc_topic_memory_vector_topics", "pc_topic_memory_vector_chunks"): + table = inventory["tables"][name] + assert table["columns"]["embedding"]["type"] == ["vector", [str(dimension)]] + assert any(index.get("kind") == "vector" for index in table["indexes"].values()) + assert json.loads( + connection.exec_driver_sql("SELECT embedding_profile FROM pc_server_storage_profile").scalar_one() + ) == profile.model_dump(mode="json") + + backend.connections().run(verify) + + @pytest.mark.parametrize("backend", ("oceanbase",), indirect=True) def test_oceanbase_server_upgrade_with_separate_code_cache_preserves_history( backend: _BackendTarget, tmp_path: Path diff --git a/tests/builtin/persistence/test_server_vector_migrations.py b/tests/builtin/persistence/test_server_vector_migrations.py new file mode 100644 index 0000000000..73bef2c89b --- /dev/null +++ b/tests/builtin/persistence/test_server_vector_migrations.py @@ -0,0 +1,225 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Explicit vector maintenance preserves the configured embedding space.""" + +import asyncio +import json +from contextlib import closing + +import pytest +import sqlite_vec +from alembic import command +from fastapi.testclient import TestClient +from sqlalchemy import create_engine + +from powercontext.builtin.artifacts.memory import EmbeddingProfile +from powercontext.builtin.inference.models import EmbeddingResult +from powercontext.builtin.persistence.migrations.bundle import sqlite_inventory +from powercontext.builtin.persistence.migrations.deployment import server_bundle +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.server_readiness import check_server_schema +from powercontext.builtin.persistence.migrations.sqlite import SQLiteMigrationRunner +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.topic_memory_index import topic_memory_embedding_profile_fingerprint +from powercontext.builtin.runtime.config import BuiltinConfig +from powercontext.builtin.runtime.processing_registry import canonical_processing_manifest +from powercontext.server.factory import create_server_app +from powercontext.server.settings import McpConfig, ServerSettings + + +class ControlledEmbedding: + def __init__(self, embedding): + self.profile = embedding + + async def embed(self, texts, /): + vector = (1.0,) + (0.0,) * (self.profile.dimension - 1) + return EmbeddingResult(vectors=tuple(vector for _ in texts)) + + +def connect(database): + import sqlite3 + + connection = sqlite3.connect(database) + connection.enable_load_extension(True) + try: + sqlite_vec.load(connection) + finally: + connection.enable_load_extension(False) + return connection + + +def profile(dimension): + return EmbeddingProfile(profile_id="migration-vectors", model="fixture", dimension=dimension) + + +def configuration(embedding): + return { + "processing_manifest": canonical_processing_manifest(BuiltinConfig()), + "embedding_enabled": True, + "embedding_profile": embedding.model_dump(mode="json"), + } + + +def apply(runner): + plan = runner.plan() + return runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + + +def source(database, embedding): + bundle = server_bundle() + frozen = json.loads((bundle.directory / "profiles/vector.json").read_text()) + engine = create_engine("sqlite://", creator=lambda: connect(database)) + with engine.connect() as connection: + connection.exec_driver_sql("BEGIN IMMEDIATE") + config = bundle.config(connection) + config.attributes["configuration"] = configuration(embedding) + command.upgrade(config, "s1300") + connection.exec_driver_sql("DROP TABLE pc_schema_revision") + for statement in frozen["sqlite"]["initialize"]: + connection.exec_driver_sql(statement.replace("__DIMENSION__", str(embedding.dimension))) + connection.exec_driver_sql( + "INSERT INTO pc_topic_memory_retrieval_shape VALUES (1,'hybrid',?)", + (topic_memory_embedding_profile_fingerprint(embedding),), + ) + connection.exec_driver_sql( + "INSERT INTO pc_scopes (scope_id,title,summary,scope_id_search,title_search,summary_search,version)" + " VALUES ('scope','title','summary','scope','title','summary',1)" + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifacts (scope_id,family,artifact_id,revision,content)" + " VALUES ('scope','custom-family','historical',1,?)", + (b'{"history":"unchanged"}',), + ) + connection.commit() + engine.dispose() + + +@pytest.mark.parametrize("dimension", [3, 4]) +def test_vector_upgrade_preserves_history_shape_and_matches_fresh_storage(tmp_path, dimension): + embedding = profile(dimension) + database = tmp_path / "old.db" + source(database, embedding) + bundle = server_bundle(embedding_profile=embedding) + runner = SQLiteMigrationRunner(database, bundle, configuration=configuration(embedding)) + assert runner.plan().source_revision == "s1300" + assert apply(runner).backup_state == "completed" + assert not apply(runner).changed + assert runner.verify().state == "ready" + with closing(connect(database)) as connection: + assert connection.execute("SELECT content FROM pc_artifacts WHERE artifact_id='historical'").fetchone() == ( + b'{"history":"unchanged"}', + ) + assert connection.execute( + "SELECT shape,profile_fingerprint FROM pc_topic_memory_retrieval_shape" + ).fetchone() == ( + "hybrid", + topic_memory_embedding_profile_fingerprint(embedding), + ) + assert json.loads( + connection.execute("SELECT embedding_profile FROM pc_server_storage_profile").fetchone()[0] + ) == embedding.model_dump(mode="json") + + fresh = tmp_path / "fresh.db" + apply(SQLiteMigrationRunner(fresh, bundle, configuration=configuration(embedding))) + inventories = [] + for path in (database, fresh): + engine = create_engine("sqlite://", creator=lambda path=path: connect(path)) + with engine.connect() as connection: + check_server_schema(connection) + inventories.append(sqlite_inventory(connection, exclude=frozenset(bundle.retained_objects))) + engine.dispose() + assert inventories[0] == inventories[1] + settings = ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{database}"), mcp=McpConfig(enabled=False) + ) + app = create_server_app( + settings=settings, embedding_model=ControlledEmbedding(embedding), scheduler_path=tmp_path / "scheduler" + ) + with TestClient(app) as client: + remembered = client.post( + "/v1/memory/remember", + json={"scope_id": "scope", "kind": "fact", "text": "New vectors remain searchable after maintenance."}, + ) + assert remembered.status_code == 200 + searched = client.post( + "/v1/memory/search", json={"scope_id": "scope", "query": "New vectors", "mode": "vector"} + ) + assert searched.status_code == 200 + assert searched.json()["hits"][0]["memory"]["text"] == "New vectors remain searchable after maintenance." + + +def test_vector_configuration_change_is_rejected_without_rebuilding_helpers(tmp_path): + database = tmp_path / "managed.db" + embedding = profile(3) + bundle = server_bundle(embedding_profile=embedding) + apply(SQLiteMigrationRunner(database, bundle, configuration=configuration(embedding))) + before = database.read_bytes() + other = embedding.model_copy(update={"model": "different-model"}) + runner = SQLiteMigrationRunner(database, server_bundle(embedding_profile=other), configuration=configuration(other)) + with pytest.raises(MigrationError, match="embedding_profile_mismatch"): + runner.plan() + assert database.read_bytes() == before + + +def test_missing_managed_vector_helper_is_rejected_without_startup_repair(tmp_path): + database = tmp_path / "managed.db" + embedding = profile(3) + bundle = server_bundle(embedding_profile=embedding) + apply(SQLiteMigrationRunner(database, bundle, configuration=configuration(embedding))) + with closing(connect(database)) as connection, connection: + connection.execute("DROP TRIGGER pc_atomic_memory_vec_insert") + before = database.read_bytes() + engine = create_engine("sqlite://", creator=lambda: connect(database)) + try: + with engine.connect() as connection, pytest.raises(MigrationError, match="incompatible_schema"): + check_server_schema(connection) + finally: + engine.dispose() + assert database.read_bytes() == before + + +def test_generic_storage_reader_admits_managed_vectors_without_explicit_extension_flag(tmp_path): + database = tmp_path / "managed.db" + embedding = profile(3) + runner = SQLiteMigrationRunner( + database, server_bundle(embedding_profile=embedding), configuration=configuration(embedding) + ) + apply(runner) + + async def read(): + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{database}") + async with SQLiteProfile.open(config, tables=()) as storage, storage.database.transaction() as connection: + assert ( + await connection.exec_driver_sql("SELECT COUNT(*) FROM pc_atomic_memory_current_vec") + ).scalar_one() == 0 + + asyncio.run(read()) + assert not apply(runner).changed + + +def test_configured_vector_plan_is_read_only_for_a_missing_database(tmp_path): + from powercontext.builtin.persistence.migrations.deployment import deployment_runner + from powercontext.builtin.runtime.config import InferenceConfig + from powercontext.server.settings import ServerSettings + + settings = ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'server.db'}"), + inference=InferenceConfig( + embedding_model="openai:fixture", embedding_profile_id="fixture", embedding_dimension=3 + ), + ) + plan = deployment_runner(settings).plan() + assert plan.state == "uninitialized" and plan.readiness_scope == "server" + assert list(tmp_path.iterdir()) == [] From a8643b88847bff0aeacf208a454fdd294cc343bb Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 11:28:15 +0800 Subject: [PATCH 81/94] test(upgrades): require installed vector profiles across backends Require all twelve SQLite, seekDB and OceanBase combinations with Code and vector storage enabled or disabled. Seed historical Atomic and Topic embeddings through an installed 1.3.0 Server, compare exact embedding digests without re-embedding, and verify semantic reads and fresh-storage equivalence through the installed target. Cover profile drift, missing SQLite helpers and actual SQLite backup restoration without relaxing runtime admission or release policy. Constraint: preserve frozen revision checksums and require zero skipped upgrade phases. Scope-risk: native failures remain release-blocking; no complete acceptance claim. Tested: pytest tests/test_upgrade_release_gate.py -q (62 passed); pytest tests/test_upgrade_publication.py -q (8 passed); scoped Ruff check/format, scoped prek, Python 3.11 Ty, git diff --check. --- .../setup-upgrade-oceanbase/action.yml | 2 +- .github/workflows/master.yml | 2 +- .github/workflows/release.yml | 2 +- docs/en/docs/operate/database-migrations.md | 2 +- docs/zh/docs/operate/database-migrations.md | 2 +- release/upgrade-support.json | 32 ++- scripts/upgrade_database_probe.py | 88 +++++--- scripts/upgrade_release_gate.py | 23 +- tests/release_upgrades/README.md | 12 +- tests/release_upgrades/test_full_server.py | 208 ++++++++++++++++-- tests/test_upgrade_release_gate.py | 13 +- 11 files changed, 328 insertions(+), 58 deletions(-) diff --git a/.github/actions/setup-upgrade-oceanbase/action.yml b/.github/actions/setup-upgrade-oceanbase/action.yml index 4c6adca444..4851ab98ca 100644 --- a/.github/actions/setup-upgrade-oceanbase/action.yml +++ b/.github/actions/setup-upgrade-oceanbase/action.yml @@ -49,7 +49,7 @@ runs: docker exec upgrade-oceanbase \ obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ -Dpc_upgrade_test \ - -e "CREATE DATABASE pc_upgrade_fresh; CREATE DATABASE pc_upgrade_code_test; CREATE DATABASE pc_upgrade_code_fresh; SET GLOBAL recyclebin=OFF; SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode'; SHOW VARIABLES LIKE 'recyclebin';" + -e "CREATE DATABASE pc_upgrade_fresh; CREATE DATABASE pc_upgrade_code_test; CREATE DATABASE pc_upgrade_code_fresh; CREATE DATABASE pc_upgrade_vector_test; CREATE DATABASE pc_upgrade_vector_fresh; CREATE DATABASE pc_upgrade_vector_code_test; CREATE DATABASE pc_upgrade_vector_code_fresh; SET GLOBAL recyclebin=OFF; SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode'; SHOW VARIABLES LIKE 'recyclebin';" exit 0 fi if [ "$(docker inspect --format '{{.State.Running}}' upgrade-oceanbase)" != true ]; then diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index bdf72462dd..5605e17073 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -22,7 +22,7 @@ jobs: upgrade-release-gate: name: Upgrade release gate runs-on: ubuntu-24.04 - timeout-minutes: 90 + timeout-minutes: 180 env: PC_UPGRADE_OCEANBASE_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_upgrade_test?charset=utf8mb4 PC_UPGRADE_OCEANBASE_FRESH_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_upgrade_fresh?charset=utf8mb4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 792b68c168..2e3cc49050 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ concurrency: jobs: release-build: runs-on: ubuntu-24.04 - timeout-minutes: 90 + timeout-minutes: 180 env: PC_UPGRADE_OCEANBASE_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_upgrade_test?charset=utf8mb4 PC_UPGRADE_OCEANBASE_FRESH_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_upgrade_fresh?charset=utf8mb4 diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index ced2b9d4d8..fb7098242a 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -31,7 +31,7 @@ identity value and explicitly rotating a stopped clone's identity retain their e Vector maintenance requires the exact historical embedding profile: profile ID, model, dimension, distance and normalization. It preserves historical embeddings and Topic retrieval shape without calling an embedding provider. The registered SQLite transition adds and backfills Atomic Memory's vec0 helper and transactional triggers; native backends retain the registered vector columns and HNSW parameters. `pc_server_storage_profile` stores the complete embedding configuration, not another revision or recovery journal. Generic storage readers validate this declaration, and Server startup uses the configured or injected model's actual profile. Missing managed helpers and profile changes are rejected before startup repair or writes. -Empty persistent SQLite/seekDB Server storage with vectors initializes through the selected profile of the same chain. Existing unmanaged storage requires explicit maintenance and an exact registered source layout. Changing configuration cannot bypass admission, enable vectors on no-vector managed storage, or implicitly add vector/Code objects. Enabling embedded Code on an already managed database without its tables requires a separately registered configuration transition; startup and the standalone Code CLI reject it. The release gate includes separate Code-enabled cases on all three backends: build an index with installed 1.3.0, preserve its cache files and embedded graph rows, and query the original fingerprint and symbols after the upgrade. Vector release acceptance must independently cover the matching installed configuration; a successful no-vector case does not establish vector support. +Empty persistent SQLite/seekDB Server storage with vectors initializes through the selected profile of the same chain. Existing unmanaged storage requires explicit maintenance and an exact registered source layout. Changing configuration cannot bypass admission, enable vectors on no-vector managed storage, or implicitly add vector/Code objects. Enabling embedded Code on an already managed database without its tables requires a separately registered configuration transition; startup and the standalone Code CLI reject it. The release gate requires all twelve combinations of backend, vectors enabled or disabled, and Code enabled or disabled. Code cases build an index with installed 1.3.0, preserve its cache files and embedded graph rows, and query the original fingerprint and symbols after the upgrade. Vector cases preserve nonempty historical Atomic and Topic embeddings, verify semantic reads and new writes through the installed target, reject profile drift, and compare fresh/upgraded storage. SQLite also tests missing-helper rejection and backup restoration. A successful no-vector case does not establish vector support; every declared case must pass for the actual target artifact. `--bundle artifacts` retains the partial Artifact prototype chain starting at `p0001`; `--bundle topic-memory-recovery` covers only registered Topic recovery layouts. Both report `server_ready=false`. Neither establishes full Server readiness, and their revisions cannot be stamped as full Server revisions. diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index a26cd2e58d..1f4aa07934 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -29,7 +29,7 @@ identity，仍使用原有事务语义。 向量维护要求声明与历史数据完全一致的 embedding profile：profile ID、模型、维度、距离和归一化方式。迁移保留历史向量及 Topic 检索配置，不调用 embedding provider。SQLite 的登记步骤添加并回填 Atomic Memory 的 vec0 辅助索引和事务触发器；原生后端保留登记的向量列与 HNSW 参数。`pc_server_storage_profile` 保存完整的 embedding 配置，不是另一张 revision 表或恢复日志。通用存储读入口校验该声明，Server 启动使用配置或注入模型的实际 profile。受管辅助对象缺失或 profile 变化会在启动修补或写入之前被拒绝。 -空的持久 SQLite/seekDB Server 启用向量时，通过同一版本链所选的 profile 初始化。已有未受管库必须显式维护，且来源结构必须与登记布局完全一致。切换配置不能绕过准入检查，不能给受管无向量库直接启用向量，也不能隐式增加向量或 Code 对象。已经受管但没有 Code 表的数据库若要启用嵌入式 Code，需要另行登记配置转换，启动入口与独立 Code CLI 均拒绝隐式扩展。发布闸门分别验收三个后端的 Code 启用配置：用实际安装的 1.3.0 创建索引，保留缓存文件与嵌入式图数据，并在升级后查询原指纹和符号。向量发布验收必须独立覆盖相应的实际安装配置；无向量配置通过不能证明向量支持。 +空的持久 SQLite/seekDB Server 启用向量时，通过同一版本链所选的 profile 初始化。已有未受管库必须显式维护，且来源结构必须与登记布局完全一致。切换配置不能绕过准入检查，不能给受管无向量库直接启用向量，也不能隐式增加向量或 Code 对象。已经受管但没有 Code 表的数据库若要启用嵌入式 Code，需要另行登记配置转换，启动入口与独立 Code CLI 均拒绝隐式扩展。发布闸门要求三个后端、向量开关及 Code 开关组成的十二种配置全部通过。Code 配置用实际安装的 1.3.0 创建索引，保留缓存文件与嵌入式图数据，并在升级后查询原指纹和符号。向量配置保留非空的历史 Atomic 与 Topic 向量，通过实际安装的目标包验证语义读取和新写入，拒绝 profile 漂移，并对比新建库与升级库。SQLite 还验证辅助对象缺失时的拒绝行为及备份还原。无向量配置通过不能证明向量支持，实际目标安装包必须通过每个登记配置的验收。 `--bundle artifacts` 保留 `p0001` 开始的局部 Artifact 原型链；`--bundle topic-memory-recovery` 只处理登记的 Topic 恢复布局。两者均报告 `server_ready=false`，不得用于证明完整 Server 就绪，也不能把其版本号直接 stamp 为完整 Server 版本。 diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 2be951b6b7..1f53cd0686 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -4,7 +4,7 @@ "mode": "upgrade-required", "schema_review": { "persistence_sha256": "0c8246b49cc05493e86bd011219c81306cca37d0ef3babad651d434d18cafce5", - "rationale": "The shared s1300 -> s1400 -> s1401 chain registers installed 1.3.0 no-vector Server storage on SQLite, seekDB and OceanBase with Code disabled or enabled. SQLite/OceanBase Code caches remain separate; embedded seekDB Code uses the three frozen released tables. A selected vector profile binds the frozen schema and domain contract to the exact historical profile ID, model, dimension, distance and normalization. SQLite maintenance registers and backfills the Atomic vec0 helper and transactional triggers; native layouts preserve vector column dimensions and explicit HNSW parameters, including ordered key columns, algorithm, distance, library and construction/search options. Unknown index options remain rejected. The vector-only configuration declaration is not a second revision or recovery journal. Full layout and domain checks precede managed use; configured or injected model drift and missing helpers cannot trigger startup repair. No embedding provider is called during migration and historical embeddings, Topic shape, authority and evidence are preserved. Existing no-vector revision resources and all three checksums remain unchanged; selected vector profiles have separately bound checksums. Target locks, original recovery points, acknowledgements and backup policies remain shared. The declared installed-release gate still contains the six no-vector/Code cases; vector installed-artifact matrix and complete recovery/restore acceptance remain unfinished. Source tests and schema captures do not establish that release acceptance passed. Policy remains upgrade-required." + "rationale": "The shared s1300 -> s1400 -> s1401 chain registers installed 1.3.0 no-vector Server storage on SQLite, seekDB and OceanBase with Code disabled or enabled. SQLite/OceanBase Code caches remain separate; embedded seekDB Code uses the three frozen released tables. A selected vector profile binds the frozen schema and domain contract to the exact historical profile ID, model, dimension, distance and normalization. SQLite maintenance registers and backfills the Atomic vec0 helper and transactional triggers; native layouts preserve vector column dimensions and explicit HNSW parameters, including ordered key columns, algorithm, distance, library and construction/search options. Unknown index options remain rejected. The vector-only configuration declaration is not a second revision or recovery journal. Full layout and domain checks precede managed use; configured or injected model drift and missing helpers cannot trigger startup repair. No embedding provider is called during migration and historical embeddings, Topic shape, authority and evidence are preserved. Existing no-vector revision resources and all three checksums remain unchanged; selected vector profiles have separately bound checksums. Target locks, original recovery points, acknowledgements and backup policies remain shared. The installed-release gate requires all twelve backend/configuration combinations, with Code enabled or disabled and vectors enabled or disabled. Vector cases seed nonempty Atomic and Topic vectors through the actual source Server, preserve exact embedding digests and retrieval shape, and verify historical/new semantic reads, configuration-drift rejection, SQLite missing-helper rejection and fresh-storage equivalence using the independently installed target. Maintenance must not call an embedding provider. SQLite vector cases also restore the automatic backup with the old Server before re-upgrading. Complete current-artifact and recovery/restore acceptance remain required. Source tests and schema captures do not establish that release acceptance passed. Policy remains upgrade-required." }, "cases": [], "automatic_sources": { @@ -40,6 +40,36 @@ "backend": "seekdb", "configuration": "full-server-code", "suite": "tests/release_upgrades/test_full_server.py" + }, + { + "backend": "sqlite", + "configuration": "full-server-vector", + "suite": "tests/release_upgrades/test_full_server.py" + }, + { + "backend": "seekdb", + "configuration": "full-server-vector", + "suite": "tests/release_upgrades/test_full_server.py" + }, + { + "backend": "oceanbase", + "configuration": "full-server-vector", + "suite": "tests/release_upgrades/test_full_server.py" + }, + { + "backend": "sqlite", + "configuration": "full-server-vector-code", + "suite": "tests/release_upgrades/test_full_server.py" + }, + { + "backend": "seekdb", + "configuration": "full-server-vector-code", + "suite": "tests/release_upgrades/test_full_server.py" + }, + { + "backend": "oceanbase", + "configuration": "full-server-vector-code", + "suite": "tests/release_upgrades/test_full_server.py" } ] } diff --git a/scripts/upgrade_database_probe.py b/scripts/upgrade_database_probe.py index 1fb738d5a1..e591d16b3e 100644 --- a/scripts/upgrade_database_probe.py +++ b/scripts/upgrade_database_probe.py @@ -26,6 +26,7 @@ import json import os import sqlite3 +from collections.abc import Collection from importlib import import_module from importlib.metadata import version from pathlib import Path @@ -48,33 +49,68 @@ def encode(value): return hashlib.sha256("\n".join(hashes).encode()).hexdigest() -async def inspect(backend: str, database: Path, *, add_revision: bool, canonical: bool = False) -> dict[str, Any]: - if backend == "sqlite": - with sqlite3.connect(database) as connection: - if add_revision: - connection.execute(REVISION_DDL) - objects = { - plural: dict( - connection.execute("SELECT name, sql FROM sqlite_schema WHERE type=? ORDER BY name", (kind,)) - ) - for kind, plural in ( - ("table", "tables"), - ("index", "indexes"), - ("view", "views"), - ("trigger", "triggers"), - ) - } - data = { - name: row_digest(connection.execute('SELECT * FROM "' + name.replace('"', '""') + '"')) # noqa: S608 -- Reflected identifier is quoted. - for name in objects["tables"] - } - return { - "data_sha256": data, - "engine_version": sqlite3.sqlite_version, - "driver_version": version("aiosqlite"), - **objects, +def vector_snapshot(cursor: Any, tables: Collection[str], *, sqlite: bool) -> dict[str, Any]: + if not os.environ.get("POWERCONTEXT_SERVER_INFERENCE_EMBEDDING_MODEL"): + return {} + result = {} + for table in ("pc_atomic_memory_current", "pc_topic_memory_vector_topics", "pc_topic_memory_vector_chunks"): + if table not in tables: + continue + if table == "pc_atomic_memory_current": + query = ( + "SELECT scope_id,artifact_id,embedding,embedding_input_hash,profile_fingerprint " + "FROM pc_atomic_memory_current WHERE embedding IS NOT NULL" + ) + elif sqlite: + helper = { + "pc_topic_memory_vector_topics": "pc_topic_memory_topic_vec", + "pc_topic_memory_vector_chunks": "pc_topic_memory_chunk_vec", + }[table] + query = f"SELECT m.*,v.embedding FROM {table} m JOIN {helper} v ON m.vector_id=v.rowid" # noqa: S608 -- Both names come from the fixed fixture table mapping. + else: + query = f"SELECT * FROM {table} WHERE embedding IS NOT NULL" # noqa: S608 -- The three fixture table names are fixed above. + cursor.execute(query) + rows = cursor.fetchall() + result[table] = {"rows": len(rows), "sha256": row_digest(rows)} + return result + + +def sqlite_snapshot(database: Path, *, add_revision: bool) -> dict[str, Any]: + with sqlite3.connect(database) as connection: + if os.environ.get("POWERCONTEXT_SERVER_INFERENCE_EMBEDDING_MODEL"): + connection.enable_load_extension(True) + try: + import_module("sqlite_vec").load(connection) + finally: + connection.enable_load_extension(False) + if add_revision: + connection.execute(REVISION_DDL) + objects = { + plural: dict(connection.execute("SELECT name, sql FROM sqlite_schema WHERE type=? ORDER BY name", (kind,))) + for kind, plural in ( + ("table", "tables"), + ("index", "indexes"), + ("view", "views"), + ("trigger", "triggers"), + ) } + data = { + name: row_digest(connection.execute('SELECT * FROM "' + name.replace('"', '""') + '"')) # noqa: S608 -- Reflected identifier is quoted. + for name in objects["tables"] + } + vectors = vector_snapshot(connection.cursor(), objects["tables"], sqlite=True) + return { + "data_sha256": data, + "vectors": vectors, + "engine_version": sqlite3.sqlite_version, + "driver_version": version("aiosqlite"), + **objects, + } + +async def inspect(backend: str, database: Path, *, add_revision: bool, canonical: bool = False) -> dict[str, Any]: + if backend == "sqlite": + return sqlite_snapshot(database, add_revision=add_revision) # Use the driver's public API, not a current-checkout Profile that could run # startup guards or mutate an old schema before the target CLI is tested. pymysql = import_module("pymysql") @@ -115,6 +151,7 @@ async def inspect(backend: str, database: Path, *, add_revision: bool, canonical tables[name] = cursor.fetchone()[1] cursor.execute("SELECT * FROM `" + name.replace("`", "``") + "`") # noqa: S608 -- Escaped identifiers reflected from the isolated fixture. data[name] = row_digest(cursor.fetchall()) + vectors = vector_snapshot(cursor, names, sqlite=False) cursor.execute("SHOW FULL TABLES WHERE Table_type = 'VIEW'") views = {} for name in sorted(row[0] for row in cursor.fetchall()): @@ -130,6 +167,7 @@ async def inspect(backend: str, database: Path, *, add_revision: bool, canonical return { "logical_tables": logical, "data_sha256": data, + "vectors": vectors, "triggers": triggers, "engine_version": engine, "driver_version": version("pymysql"), diff --git a/scripts/upgrade_release_gate.py b/scripts/upgrade_release_gate.py index aa1bb9685d..de6be9c72e 100644 --- a/scripts/upgrade_release_gate.py +++ b/scripts/upgrade_release_gate.py @@ -323,7 +323,7 @@ def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dic } with case_workspace(case["backend"], output) as directory: extras = "cli,server,seekdb" if case["backend"] == "seekdb" else "cli,server" - if case["configuration"] == "full-server-code": + if case["configuration"] in {"full-server-code", "full-server-vector-code"}: extras += ",code" interpreters = {} artifact = case["source_artifact"] @@ -388,7 +388,7 @@ def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dic validate_junit(report) backend = validate_backend_evidence(output / "backend.json", case["backend"]) require(backend.get("configuration") == case["configuration"], "Configuration evidence differs from the case") - if case["configuration"] == "full-server-code": + if case["configuration"] in {"full-server-code", "full-server-vector-code"}: code = backend.get("code") if not isinstance(code, dict): raise GateError("Installed Code acceptance evidence required") # noqa: TRY003 -- operator diagnostic @@ -400,6 +400,25 @@ def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dic ), "The installed Code index must remain queryable with its original immutable identity", ) + if case["configuration"] in {"full-server-vector", "full-server-vector-code"}: + vectors = backend.get("vectors") + if not isinstance(vectors, dict): + raise GateError("Installed vector acceptance evidence required") # noqa: TRY003 -- operator diagnostic + require( + vectors.get("dimension") == (4 if case["configuration"].endswith("-code") else 3) + and vectors.get("maintenance_embedding_requests") == 0 + and all( + vectors.get(field) is True + for field in ( + "historical_atomic_and_topic_embeddings_preserved", + "historical_and_new_vector_reads", + "fresh_vector_read", + "profile_drift_rejected", + ) + ) + and (case["backend"] != "sqlite" or vectors.get("missing_sqlite_helper_rejected") is True), + "Installed vector acceptance must preserve and retrieve the original embedding space", + ) return {"case": case, "installed": metadata, "database": backend, "status": "passed"} diff --git a/tests/release_upgrades/README.md b/tests/release_upgrades/README.md index cb01d55ab4..fc35fd6548 100644 --- a/tests/release_upgrades/README.md +++ b/tests/release_upgrades/README.md @@ -19,7 +19,14 @@ source wheels, invalid artifact metadata, catalog outages, and a target at or be Evidence records the catalog digest, exact source artifact, target artifact digest, and expanded matrix. The resolver requires universal `py3-none-any` wheels; a distribution format change must extend it explicitly. -All six profiles use `test_full_server.py`: Code enabled or disabled on SQLite, seekDB and OceanBase. +All twelve profiles use `test_full_server.py`: Code enabled or disabled, with or without vectors, on SQLite, +seekDB and OceanBase. Vector cases use exact unit-normalized L2 profiles: dimension three without Code and +dimension four with Code. The local controlled embedding endpoint serves actual installed source/target processes; +it does not replace either runtime or its persistence. Each source Server writes nonempty Atomic Memory and Topic +vectors. Maintenance must preserve the exact historical embedding digests without calling that endpoint. Target +acceptance checks historical and new vector retrieval, Topic content, configured profile drift without writes, +and fresh-storage equivalence. SQLite additionally rejects a missing vector helper without repairing the clone. +These assertions establish deterministic execution and space preservation, not embedding model quality. Code cases preserve immutable cache files and original fingerprint/symbol identities. Embedded seekDB additionally preserves the exact graph-table row digests through maintenance; its frozen extension creates the same schema for fresh storage. The suite starts the installed 1.3.0 Server, writes data through HTTP, @@ -93,7 +100,8 @@ they follow actual installed operations and compare persisted data. The profile actual background Memory processing with controlled model responses. Both installed releases declare the Memory capability and use the same processing ownership; the Worker check changes only the execution interval after migration. The separate SDK fixture tests the old pending intent's format and acknowledgment. Controlled responses establish -execution and evidence preservation, not real-model quality. Vector storage needs separately reviewed profiles. +execution and evidence preservation, not real-model quality. Vector storage uses its separately registered profiles; +passing a no-vector case cannot establish vector acceptance. SQLite inventories include indexes, views and triggers; MySQL-family table DDL includes indexes and constraints, and views are collected separately. MySQL comparison parses columns, indexes (including parser properties), constraints, and CHECK expressions, ignoring only physical formatting and backend-generated constraint names. Views and triggers are collected independently. SQLite normalizes whitespace and identifier quoting. Structural differences fail rather than being silently excluded. diff --git a/tests/release_upgrades/test_full_server.py b/tests/release_upgrades/test_full_server.py index 8a84280ea9..1b30626e30 100644 --- a/tests/release_upgrades/test_full_server.py +++ b/tests/release_upgrades/test_full_server.py @@ -67,6 +67,7 @@ def objects(value): @pytest.fixture(scope="session") def model_gateway(): stages = [] + embeddings = [] class Handler(BaseHTTPRequestHandler): def log_message(self, format, *args): # noqa: A002 -- stdlib signature @@ -74,8 +75,33 @@ def log_message(self, format, *args): # noqa: A002 -- stdlib signature def do_POST(self): body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) - payload = next(obj for obj in objects(body["messages"]) if "evidence" in obj) - if "proposal" in payload: + if self.path.endswith("/embeddings"): + model = re.fullmatch(r"upgrade-vector-([34])", body["model"]) + assert model is not None, "Unregistered controlled embedding model" + dimension = int(model[1]) + inputs = body["input"] + if isinstance(inputs, str): + inputs = [inputs] + embeddings.append(dimension) + response = json.dumps({ + "object": "list", + "model": body["model"], + "data": [ + {"object": "embedding", "index": index, "embedding": [1.0, *([0.0] * (dimension - 1))]} + for index in range(len(inputs)) + ], + "usage": {"prompt_tokens": len(inputs), "total_tokens": len(inputs)}, + }).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(response))) + self.end_headers() + self.wfile.write(response) + return + payload = next((obj for obj in objects(body["messages"]) if "evidence" in obj), None) + if payload is None: + value = {"ready": True} + elif "proposal" in payload: stages.append("reconcile") proposal = payload["proposal"] duplicate = next((item for item in payload["related"] if item["text"] == proposal["text"]), None) @@ -129,7 +155,7 @@ def do_POST(self): thread = threading.Thread(target=gateway.serve_forever, daemon=True) thread.start() try: - yield {"url": f"http://127.0.0.1:{gateway.server_port}/v1", "stages": stages} + yield {"url": f"http://127.0.0.1:{gateway.server_port}/v1", "stages": stages, "embeddings": embeddings} finally: gateway.shutdown() gateway.server_close() @@ -230,7 +256,16 @@ def environment_for(case, *, fresh=False): "NO_PROXY": "localhost,127.0.0.1", "no_proxy": "localhost,127.0.0.1", }) - if case["configuration"] == "full-server-code": + if case["vector_dimension"]: + dimension = case["vector_dimension"] + environment.update({ + "POWERCONTEXT_SERVER_INFERENCE_EMBEDDING_MODEL": f"openai:upgrade-vector-{dimension}", + "POWERCONTEXT_SERVER_INFERENCE_EMBEDDING_BASE_URL": case["model"]["url"], + "POWERCONTEXT_SERVER_INFERENCE_EMBEDDING_PROFILE_ID": f"upgrade-vector-{dimension}-unit", + "POWERCONTEXT_SERVER_INFERENCE_EMBEDDING_DIMENSION": str(dimension), + "POWERCONTEXT_SERVER_INFERENCE_EMBEDDING_NORMALIZATION": "unit", + }) + if case["code_enabled"]: scope = case.get("code_scope") environment["POWERCONTEXT_SERVER_CODE"] = json.dumps({ "enabled": True, @@ -250,13 +285,18 @@ def environment_for(case, *, fresh=False): parsed = urlsplit(url) expected = "/pc_upgrade_fresh" if fresh else "/pc_upgrade_test" assert parsed.scheme == "mysql+aoceanbase" and parsed.hostname in {"127.0.0.1", "localhost"} - if case["configuration"] == "full-server-code": - code_path = "/pc_upgrade_code_fresh" if fresh else "/pc_upgrade_code_test" + if case["configuration"] != "full-server-no-vector": + suffix = { + "full-server-code": "code", + "full-server-vector": "vector", + "full-server-vector-code": "vector_code", + }[case["configuration"]] + profile_path = f"/pc_upgrade_{suffix}_{'fresh' if fresh else 'test'}" if parsed.path == expected: - parsed = parsed._replace(path=code_path) + parsed = parsed._replace(path=profile_path) else: - assert re.fullmatch(re.escape(code_path) + r"_[0-9a-f]{8}", parsed.path), ( - "Only dedicated disposable Code databases may be used" + assert re.fullmatch(re.escape(profile_path) + r"_[0-9a-f]{8}", parsed.path), ( + "Only dedicated disposable profile databases may be used" ) url = parsed.geturl() else: @@ -393,7 +433,13 @@ def case(model_gateway): # Missing gate inputs are errors, never skips or an implicit local database. assert os.environ["PC_UPGRADE_SOURCE_VERSION"] == "1.3.0", "Add a reviewed adapter for another source release" configuration = os.environ["PC_UPGRADE_CONFIGURATION"] - assert configuration in {"full-server-no-vector", "full-server-code"}, "Unregistered acceptance configuration" + assert configuration in { + "full-server-no-vector", + "full-server-code", + "full-server-vector", + "full-server-vector-code", + }, "Unregistered acceptance configuration" + code_enabled = configuration.endswith("-code") return { "source": Path(os.environ["PC_UPGRADE_SOURCE_PYTHON"]), "target": Path(os.environ["PC_UPGRADE_TARGET_PYTHON"]), @@ -402,6 +448,8 @@ def case(model_gateway): "report": Path(os.environ["PC_UPGRADE_REPORT_DIR"]), "model": model_gateway, "configuration": configuration, + "code_enabled": code_enabled, + "vector_dimension": {"full-server-vector": 3, "full-server-vector-code": 4}.get(configuration, 0), } @@ -458,13 +506,35 @@ def historical(case): assert not snapshot(case, "target", "fresh-empty", fresh=True)["tables"] with server(case["source"], environment_for(case), case["work"], case["report"] / "source-server.log") as client: value = seed(client, candidate_prefix="/v1/artifact-candidates") - if case["configuration"] == "full-server-code": + if case["vector_dimension"]: + value["topic_content"] = { + "title": "Upgrade topic evidence", + "summary": "Historical semantic retrieval", + "detail": "# Upgrade evidence\nHistorical Topic vectors must remain usable after maintenance.", + } + value["topic"] = post( + client, + f"/v1/scopes/{value['scope']}/artifacts", + { + "family": "topic-memory", + "content": value["topic_content"], + }, + ) + assert_vector_reads(client, value) + if case["code_enabled"]: value["code"] = historical_code(case, value["scope"]) value["task"] = task_state(case, "source", "seed", value["scope"]) value["schema"] = snapshot(case, "source", "source") assert {"pc_artifact_candidate_heads", "pc_artifact_candidate_versions", "pc_scopes", "pc_artifacts"} <= set( value["schema"]["tables"] ) + if case["vector_dimension"]: + assert set(value["schema"]["vectors"]) == { + "pc_atomic_memory_current", + "pc_topic_memory_vector_topics", + "pc_topic_memory_vector_chunks", + } + assert all(item["rows"] > 0 for item in value["schema"]["vectors"].values()) (case["report"] / "source-fixture.json").write_text(json.dumps(value, indent=2) + "\n") return value @@ -487,6 +557,7 @@ def rejected_legacy_startup(case, historical): @pytest.fixture(scope="session") def migrated(case, historical, rejected_legacy_startup): + embedding_requests = len(case["model"]["embeddings"]) environment = environment_for(case) env_file = case["work"] / "deployment.env" env_file.write_text( @@ -551,7 +622,7 @@ def migration(operation, *options, label=""): # Capture pending work before later behavioral tests legitimately dirty the # same scope by publishing new Artifacts. Compare exact old counters here. processing = task_state(case, "target", "verify", historical["scope"]) - return { + result = { "applied": applied, "verified": verified, "repeated": repeated, @@ -559,6 +630,10 @@ def migration(operation, *options, label=""): "historical": historical, "processing": processing, } + if case["vector_dimension"]: + assert result["schema"]["vectors"] == historical["schema"]["vectors"] + assert len(case["model"]["embeddings"]) == embedding_requests, "Maintenance must not re-embed history" + return result @pytest.fixture(scope="session") @@ -577,7 +652,7 @@ def restored_backup(case, migrated): restored["report"].mkdir() shutil.copyfile(backup, restored["work"] / "database") historical = migrated["historical"] - if case["configuration"] == "full-server-code": + if case["code_enabled"]: # Code caches are separate from the database recovery point. The # operator must preserve those immutable files as well. for name in ("code-cache", "repository"): @@ -601,6 +676,8 @@ def restored_backup(case, migrated): continuation = client.get("/v1/scopes", params={"limit": 1, "cursor": historical["cursor"]}) assert continuation.status_code == 200 assert continuation.json()["items"] == historical["continuation"]["items"] + if case["vector_dimension"]: + assert_vector_reads(client, historical) environment = environment_for(restored) def migration(operation, *options): @@ -637,8 +714,10 @@ def migration(operation, *options): "records" ] assert written and written[0]["text"] == text + if case["vector_dimension"]: + assert_vector_reads(client, historical, new_text=text) assert hashlib.sha256(backup.read_bytes()).hexdigest() == before - if case["configuration"] == "full-server-code": + if case["code_enabled"]: assert code_cache_snapshot(restored) == historical["code"]["cache_sha256"] return { "method": "sqlite_online", @@ -646,19 +725,73 @@ def migration(operation, *options): "source_server_readback": True, "target_upgrade_and_read_write": True, "backup_sha256_unchanged": before, - "separate_code_cache_preserved": case["configuration"] == "full-server-code", + "separate_code_cache_preserved": case["code_enabled"], } @pytest.fixture -def target(case, migrated, preserved_code): +def target(case, migrated, preserved_code, vector_admission): with server(case["target"], environment_for(case), case["work"], case["report"] / "target-server.log") as client: yield client +@pytest.fixture(scope="session") +def vector_admission(case, migrated): + if not case["vector_dimension"]: + return None + before = snapshot(case, "target", "before-profile-drift") + environment = environment_for(case) + environment["POWERCONTEXT_SERVER_INFERENCE_EMBEDDING_PROFILE_ID"] = "different-embedding-space" + log = case["report"] / "profile-drift-server.log" + with ( + pytest.raises(AssertionError, match="Server exited before readiness"), + server(case["target"], environment, case["work"], log), + ): + pytest.fail("Managed storage admitted a different embedding profile") + assert "embedding_profile_mismatch" in log.read_text() + after = snapshot(case, "target", "after-profile-drift") + assert canonical_schema(before) == canonical_schema(after) + assert before["data_sha256"] == after["data_sha256"] + if case["backend"] == "sqlite": + clone = {**case, "work": case["work"] / "missing-helper", "report": case["report"] / "missing-helper"} + clone["work"].mkdir() + clone["report"].mkdir() + command( + [ + str(case["target"]), + "-I", + "-c", + "import sqlite3,sys\n" + "from contextlib import closing\n" + "with closing(sqlite3.connect(sys.argv[1])) as source, " + "closing(sqlite3.connect(sys.argv[2])) as target:\n" + " source.backup(target)\n" + " target.execute('DROP TRIGGER pc_atomic_memory_vec_insert')\n" + " target.commit()\n", + str(case["work"] / "database"), + str(clone["work"] / "database"), + ], + environment_for(case), + case["work"], + clone["report"] / "remove-helper.log", + ) + before = snapshot(clone, "target", "before-missing-helper-startup") + log = clone["report"] / "server.log" + with ( + pytest.raises(AssertionError, match="Server exited before readiness"), + server(clone["target"], environment_for(clone), clone["work"], log), + ): + pytest.fail("Managed startup repaired a missing vector helper") + assert "incompatible_schema" in log.read_text() + after = snapshot(clone, "target", "after-missing-helper-startup") + assert canonical_schema(before) == canonical_schema(after) + assert before["data_sha256"] == after["data_sha256"] + return {"profile_drift_rejected": True, "missing_sqlite_helper_rejected": case["backend"] == "sqlite"} + + @pytest.fixture(scope="session") def preserved_code(case, migrated): - if case["configuration"] != "full-server-code": + if not case["code_enabled"]: return None historical = migrated["historical"] source = historical["code"] @@ -761,7 +894,25 @@ def test_target_startup(target): assert response.json()["status"] in {"ready", "degraded"} # No inference provider in this explicit-write profile. -def test_read_write_and_retrieval(target, historical): +def assert_vector_reads(client, historical, *, new_text=None): + scope = historical["scope"] + hits = post(client, "/v1/memory/search", {"scope_id": scope, "query": "Upgrade fixture", "mode": "vector"})["hits"] + expected = {item["text"] for item in historical["memory"]} + if new_text is not None: + expected.add(new_text) + assert {item["memory"]["text"] for item in hits} >= expected + topics = post( + client, + f"/v1/scopes/{scope}/artifacts/topic-memory/search", + {"query": "Upgrade evidence", "mode": "hybrid", "include_scores": True}, + )["results"] + topic = next(item for item in topics if item["artifact_id"] == historical["topic"]["artifact_id"]) + assert topic["content"] == historical["topic_content"] + assert topic["revision"] == historical["topic"]["revision"] + assert {"topic_vector", "detail_vector"} <= topic["scores"]["channels"].keys() + + +def test_read_write_and_retrieval(case, target, historical): scope = historical["scope"] records = post(target, "/v1/memory/entries/list", {"scope_id": scope})["entries"] assert {digest(item) for item in records} >= {digest(item) for item in historical["memory"]} @@ -771,6 +922,8 @@ def test_read_write_and_retrieval(target, historical): found = post(target, "/v1/memory/search", {"scope_id": scope, "query": "Upgrade fixture", "mode": "fts"})["hits"] texts = {item["memory"]["text"] for item in found} assert texts >= {text, *(item["text"] for item in historical["memory"])} + if case["vector_dimension"]: + assert_vector_reads(target, historical, new_text=text) def test_authorization(target, historical): @@ -857,7 +1010,9 @@ def normalize(sql): } -def test_fresh_upgrade_equivalence(case, migrated, processed_background, preserved_code, restored_backup): +def test_fresh_upgrade_equivalence( + case, migrated, processed_background, preserved_code, restored_backup, vector_admission +): if case["backend"] == "oceanbase": environment = environment_for(case, fresh=True) env_file = case["work"] / "fresh.env" @@ -909,6 +1064,11 @@ def test_fresh_upgrade_equivalence(case, migrated, processed_background, preserv client, "/v1/memory/search", {"scope_id": fresh["scope"], "query": "Upgrade fixture", "mode": "fts"} )["hits"] assert {item["memory"]["text"] for item in hits} >= {item["text"] for item in fresh["memory"]} + if case["vector_dimension"]: + hits = post( + client, "/v1/memory/search", {"scope_id": fresh["scope"], "query": "Upgrade fixture", "mode": "vector"} + )["hits"] + assert {item["memory"]["text"] for item in hits} >= {item["text"] for item in fresh["memory"]} inventory = snapshot(case, "target", "fresh", fresh=True) actual = canonical_schema(migrated["schema"]) expected = canonical_schema(inventory) @@ -926,6 +1086,16 @@ def test_fresh_upgrade_equivalence(case, migrated, processed_background, preserv "background_processing": processed_background, "configuration": case["configuration"], "code": preserved_code, + "vectors": None + if not case["vector_dimension"] + else { + "dimension": case["vector_dimension"], + "historical_atomic_and_topic_embeddings_preserved": True, + "historical_and_new_vector_reads": True, + "maintenance_embedding_requests": 0, + "fresh_vector_read": True, + **vector_admission, + }, } if case["backend"] == "oceanbase": evidence.update(tenant_mode=inventory["tenant_mode"], proxy_version=inventory["proxy_version"]) diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py index dc02f2f256..e6d28dd3c2 100644 --- a/tests/test_upgrade_release_gate.py +++ b/tests/test_upgrade_release_gate.py @@ -381,15 +381,20 @@ def test_publishing_jobs_depend_on_nonoptional_preupload_upgrade_gate(): assert workflow["jobs"][name]["needs"] == "release-build" -def test_repository_baseline_covers_all_three_databases(): +@pytest.mark.parametrize( + "configuration", ["full-server-no-vector", "full-server-code", "full-server-vector", "full-server-vector-code"] +) +def test_repository_baseline_covers_all_three_databases(configuration): policy = gate.load_policy(Path(__file__).parents[1]) selection = policy["automatic_sources"] assert selection["strategy"] == "fixed-version" assert selection["source_version"] == "1.3.0" assert {profile["backend"] for profile in selection["profiles"]} == {"sqlite", "seekdb", "oceanbase"} - assert { - profile["backend"] for profile in selection["profiles"] if profile["configuration"] == "full-server-code" - } == {"sqlite", "seekdb", "oceanbase"} + assert {profile["backend"] for profile in selection["profiles"] if profile["configuration"] == configuration} == { + "sqlite", + "seekdb", + "oceanbase", + } @pytest.mark.parametrize("workflow_name,job_name", [("master", "upgrade-release-gate"), ("release", "release-build")]) From 747dac3e695f91bd31ab18d98b01590ee30a57f9 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 23 Oct 2026 11:34:12 +0800 Subject: [PATCH 82/94] fix(migrations): reject incomplete projections before admission --- release/upgrade-support.json | 4 +- .../server_resources/server_domain.py | 26 +++++++------ .../server_resources/server_vector_domain.py | 29 ++++++++------ .../test_mysql_migration_runner.py | 3 ++ .../test_server_vector_migrations.py | 39 +++++++++++++++++++ tests/test_managed_atomic_projection.py | 18 +++++++++ 6 files changed, 93 insertions(+), 26 deletions(-) diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 1f53cd0686..3e33daa7bf 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "0c8246b49cc05493e86bd011219c81306cca37d0ef3babad651d434d18cafce5", - "rationale": "The shared s1300 -> s1400 -> s1401 chain registers installed 1.3.0 no-vector Server storage on SQLite, seekDB and OceanBase with Code disabled or enabled. SQLite/OceanBase Code caches remain separate; embedded seekDB Code uses the three frozen released tables. A selected vector profile binds the frozen schema and domain contract to the exact historical profile ID, model, dimension, distance and normalization. SQLite maintenance registers and backfills the Atomic vec0 helper and transactional triggers; native layouts preserve vector column dimensions and explicit HNSW parameters, including ordered key columns, algorithm, distance, library and construction/search options. Unknown index options remain rejected. The vector-only configuration declaration is not a second revision or recovery journal. Full layout and domain checks precede managed use; configured or injected model drift and missing helpers cannot trigger startup repair. No embedding provider is called during migration and historical embeddings, Topic shape, authority and evidence are preserved. Existing no-vector revision resources and all three checksums remain unchanged; selected vector profiles have separately bound checksums. Target locks, original recovery points, acknowledgements and backup policies remain shared. The installed-release gate requires all twelve backend/configuration combinations, with Code enabled or disabled and vectors enabled or disabled. Vector cases seed nonempty Atomic and Topic vectors through the actual source Server, preserve exact embedding digests and retrieval shape, and verify historical/new semantic reads, configuration-drift rejection, SQLite missing-helper rejection and fresh-storage equivalence using the independently installed target. Maintenance must not call an embedding provider. SQLite vector cases also restore the automatic backup with the old Server before re-upgrading. Complete current-artifact and recovery/restore acceptance remain required. Source tests and schema captures do not establish that release acceptance passed. Policy remains upgrade-required." + "persistence_sha256": "abafa3a241dcb8f2a88cc6844604e78f56538400c6c9f4db5cf49b25d0f8b2a6", + "rationale": "The shared s1300 -> s1400 -> s1401 chain registers installed 1.3.0 no-vector Server storage on SQLite, seekDB and OceanBase with Code disabled or enabled. SQLite/OceanBase Code caches remain separate; embedded seekDB Code uses the three frozen released tables. A selected vector profile binds the frozen schema and domain contract to the exact historical profile ID, model, dimension, distance and normalization. SQLite maintenance registers and backfills the Atomic vec0 helper and transactional triggers; native layouts preserve vector column dimensions and explicit HNSW parameters, including ordered key columns, algorithm, distance, library and construction/search options. Unknown index options remain rejected. The vector-only configuration declaration is not a second revision or recovery journal. Full layout and domain checks precede managed use; configured or injected model drift and missing helpers cannot trigger startup repair. No embedding provider is called during migration and historical embeddings, Topic shape, authority and evidence are preserved. The schema revision chain and registered layouts are unchanged. Full Server readiness validates Atomic Memory projections against authority and checks complete SQLite FTS identities for both vector and no-vector profiles before admission. Validation is read-only and damaged projections require explicit maintenance. Reviewed plans bind the draft validation resources; selected vector profiles have separately bound checksums. Target locks, original recovery points, acknowledgements and backup policies remain shared. The installed-release gate requires all twelve backend/configuration combinations, with Code enabled or disabled and vectors enabled or disabled. Vector cases seed nonempty Atomic and Topic vectors through the actual source Server, preserve exact embedding digests and retrieval shape, and verify historical/new semantic reads, configuration-drift rejection, SQLite missing-helper rejection and fresh-storage equivalence using the independently installed target. Maintenance must not call an embedding provider. SQLite vector cases also restore the automatic backup with the old Server before re-upgrading. Complete current-artifact and recovery/restore acceptance remain required. Source tests and schema captures do not establish that release acceptance passed. Policy remains upgrade-required." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/server_domain.py b/src/powercontext/builtin/persistence/migrations/server_resources/server_domain.py index 515405a689..c7c95a53f1 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/server_domain.py +++ b/src/powercontext/builtin/persistence/migrations/server_resources/server_domain.py @@ -168,13 +168,15 @@ def validate(connection, *, configuration): "SELECT scope_id,binding_name FROM pc_artifact_processing_intents" ): finish(ArtifactProcessingIntentRepository().load(pinned(connection), scope, binding)) - # Check exact current projection identities without comparing mutable heads - # to retained pre-upgrade Candidate snapshots. - if connection.exec_driver_sql( - "SELECT 1 FROM pc_atomic_memory_current p WHERE NOT EXISTS (SELECT 1 FROM pc_artifact_heads h WHERE h.scope_id=p.scope_id AND h.family='atomic-memory' AND h.artifact_id=p.artifact_id AND h.revision=p.revision) LIMIT 1" - ).first(): + from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex + from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex + from powercontext.builtin.runtime.atomic_memory_rebuild import verify_atomic_memory_current + + atomic_index = SQLiteAtomicMemoryIndex() if connection.dialect.name == "sqlite" else OceanBaseAtomicMemoryIndex() + report = finish(verify_atomic_memory_current(pinned(connection), atomic_index)) + if not report.ready: raise MigrationError( - "projection_verification_failed", "An Atomic Memory projection references a missing current head." + "projection_verification_failed", "Atomic Memory projections differ from their authoritative records." ) from powercontext.builtin.persistence.topic_memory import TopicMemoryRepository @@ -192,14 +194,14 @@ def validate(connection, *, configuration): def _validate_sqlite_fts(connection): + from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex + + if not finish(SQLiteAtomicMemoryIndex().fts_matches_current(pinned(connection))): + raise MigrationError( + "projection_verification_failed", "The Atomic Memory full-text helper differs from its current rows." + ) # Read-only multiset comparison also catches duplicate/stale helper rows. projections = ( - ( - "pc_atomic_memory_current", - "pc_atomic_memory_current_fts", - "scope_id,artifact_id,revision,state_version,content_hash,projection_format,kind,text,searchable_text,tag_keys", - "", - ), ( "pc_artifact_heads", "pc_artifact_fts", diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/server_vector_domain.py b/src/powercontext/builtin/persistence/migrations/server_resources/server_vector_domain.py index cbc6c587e0..0c6982da37 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/server_vector_domain.py +++ b/src/powercontext/builtin/persistence/migrations/server_resources/server_vector_domain.py @@ -176,13 +176,18 @@ def validate(connection, *, configuration): "SELECT scope_id,binding_name FROM pc_artifact_processing_intents" ): finish(ArtifactProcessingIntentRepository().load(pinned(connection), scope, binding)) - # Check exact current projection identities without comparing mutable heads - # to retained pre-upgrade Candidate snapshots. - if connection.exec_driver_sql( - "SELECT 1 FROM pc_atomic_memory_current p WHERE NOT EXISTS (SELECT 1 FROM pc_artifact_heads h WHERE h.scope_id=p.scope_id AND h.family='atomic-memory' AND h.artifact_id=p.artifact_id AND h.revision=p.revision) LIMIT 1" - ).first(): + from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex + from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex + from powercontext.builtin.runtime.atomic_memory_rebuild import verify_atomic_memory_current + + profile = _profile(configuration) + atomic_index = ( + SQLiteAtomicMemoryIndex(profile) if connection.dialect.name == "sqlite" else OceanBaseAtomicMemoryIndex(profile) + ) + report = finish(verify_atomic_memory_current(pinned(connection), atomic_index)) + if not report.ready: raise MigrationError( - "projection_verification_failed", "An Atomic Memory projection references a missing current head." + "projection_verification_failed", "Atomic Memory projections differ from their authoritative records." ) from powercontext.builtin.persistence.topic_memory import TopicMemoryRepository @@ -198,14 +203,14 @@ def validate(connection, *, configuration): def _validate_sqlite_fts(connection): + from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex + + if not finish(SQLiteAtomicMemoryIndex().fts_matches_current(pinned(connection))): + raise MigrationError( + "projection_verification_failed", "The Atomic Memory full-text helper differs from its current rows." + ) # Read-only multiset comparison also catches duplicate/stale helper rows. projections = ( - ( - "pc_atomic_memory_current", - "pc_atomic_memory_current_fts", - "scope_id,artifact_id,revision,state_version,content_hash,projection_format,kind,text,searchable_text,tag_keys", - "", - ), ( "pc_artifact_heads", "pc_artifact_fts", diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index 2a0d0fdc83..62a94ae39f 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -772,6 +772,9 @@ def seed(connection, identity, guard): connection.commit() backend.connections().run(seed, writable=True) + for inspect_state in (full.plan, full.verify): + with pytest.raises(MigrationError, match="projection_verification_failed"): + inspect_state() runner = AtomicProjectionMigrationRunner( backend.connections(), full.bundle, diff --git a/tests/builtin/persistence/test_server_vector_migrations.py b/tests/builtin/persistence/test_server_vector_migrations.py index 73bef2c89b..68b4ca5df3 100644 --- a/tests/builtin/persistence/test_server_vector_migrations.py +++ b/tests/builtin/persistence/test_server_vector_migrations.py @@ -223,3 +223,42 @@ def test_configured_vector_plan_is_read_only_for_a_missing_database(tmp_path): plan = deployment_runner(settings).plan() assert plan.state == "uninitialized" and plan.readiness_scope == "server" assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize("damage", ["missing_current", "helper_identity"]) +def test_vector_readiness_rejects_incomplete_atomic_projections(tmp_path, damage): + database = tmp_path / "managed.db" + embedding = profile(3) + source(database, embedding) + runner = SQLiteMigrationRunner( + database, server_bundle(embedding_profile=embedding), configuration=configuration(embedding) + ) + apply(runner) + settings = ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{database}"), mcp=McpConfig(enabled=False) + ) + app = create_server_app( + settings=settings, embedding_model=ControlledEmbedding(embedding), scheduler_path=tmp_path / "scheduler" + ) + with TestClient(app) as client: + remembered = client.post( + "/v1/memory/remember", + json={"scope_id": "scope", "kind": "fact", "text": "Keep this vector-backed memory searchable."}, + ) + assert remembered.status_code == 200 + assert runner.verify().state == "ready" + with closing(connect(database)) as connection, connection: + if damage == "missing_current": + connection.execute("DELETE FROM pc_atomic_memory_current") + else: + connection.execute("UPDATE pc_atomic_memory_current_fts SET identity_token='broken'") + original = database.read_bytes() + for inspect_state in (runner.plan, runner.verify): + with pytest.raises(MigrationError, match="projection_verification_failed"): + inspect_state() + app = create_server_app( + settings=settings, embedding_model=ControlledEmbedding(embedding), scheduler_path=tmp_path / "scheduler" + ) + with pytest.raises(MigrationError, match="projection_verification_failed"), TestClient(app): + pytest.fail("Incomplete projections must block vector Server startup") + assert database.read_bytes() == original diff --git a/tests/test_managed_atomic_projection.py b/tests/test_managed_atomic_projection.py index bbee60a965..547eda8cbf 100644 --- a/tests/test_managed_atomic_projection.py +++ b/tests/test_managed_atomic_projection.py @@ -24,15 +24,18 @@ from pathlib import Path import pytest +from fastapi.testclient import TestClient from typer.testing import CliRunner from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemoryContent from powercontext.builtin.persistence.migrations.deployment import deployment_runner from powercontext.builtin.persistence.migrations.locking import local_migration_lock +from powercontext.builtin.persistence.migrations.models import MigrationError from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts from powercontext.cli.app import create_cli from powercontext.server.cli import app +from powercontext.server.factory import create_server_app from powercontext.server.settings import ServerSettings @@ -103,6 +106,21 @@ def test_managed_repair_uses_shared_plan_backup_and_lock_without_changing_author connection.execute("UPDATE pc_atomic_memory_current_fts SET identity_token='broken'") original_authority = authority(path) original = path.read_bytes() + for operation in ("status", "plan", "verify"): + rejected = CliRunner().invoke(cli, ["server", "db-migrate", operation]) + assert rejected.exit_code != 0, rejected.output + assert json.loads(rejected.output)["error"] == "projection_verification_failed" + + async def reject_runtime(): + with pytest.raises(MigrationError, match="projection_verification_failed"): + async with open_builtin_contexts(BuiltinConfig(database=settings.database)): + pytest.fail("Incomplete projections must block runtime admission") + + asyncio.run(reject_runtime()) + server = create_server_app(settings=settings, scheduler_path=path.parent / "scheduler.db") + with pytest.raises(MigrationError, match="projection_verification_failed"), TestClient(server): + pytest.fail("Incomplete projections must block Server startup") + assert path.read_bytes() == original inspected = CliRunner().invoke(cli, [*arguments, "--action", "plan"]) assert inspected.exit_code == 0, inspected.output plan = json.loads(inspected.output) From 3ad35d628de6618f12cfbf1d7fbcef7773b98280 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 14:47:52 +0800 Subject: [PATCH 83/94] test(server): initialize managed authorization fixtures before startup Prepare the complete registered Server schema before injected access stores write fixture tables, selecting the exact injected embedding profile. Initialize the remote vector Server through the same explicit maintenance path. Allow the OceanBase clean migration suite sixty minutes and its job one hundred minutes. The preceding run passed forty-eight cases before the forty-five-minute step budget expired; per-node process deadlines and all assertions remain. Constraint: Managed startup continues to reject partial or unknown schemas. Scope-risk: Test fixture setup and CI budgets only; no runtime admission change. Tested: Reproduced unknown_baseline before the fix; affected SQLite tests 53 passed; actual OceanBase 3 passed with zero skips; scoped Ruff, format, Python 3.11 Ty and YAML hooks passed. Full Ty reports the existing unrelated fixture verify.py import of task. --- .github/workflows/e2e-harness.yml | 4 ++-- tests/e2e/server_storage.py | 9 +++++++- ...ic_memory_search_authorization_snapshot.py | 22 +++++++++++++------ tests/e2e/test_runtime_server.py | 19 +++++++++++----- ...est_topic_search_authorization_snapshot.py | 22 +++++++++++++------ 5 files changed, 53 insertions(+), 23 deletions(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 162ff95e89..d3671f1383 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -242,7 +242,7 @@ jobs: ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.suite != 'recall-effort') && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').database) }} runs-on: ubuntu-latest - timeout-minutes: 70 + timeout-minutes: 100 env: OCEANBASE_IMAGE: ghcr.io/oceanbase/oceanbase-ce:4.3.5.6-106000012026040916 POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- @@ -330,7 +330,7 @@ jobs: tests/e2e/test_background_metrics.py - name: Run clean migration evidence, grants and vectors without skips - timeout-minutes: 45 + timeout-minutes: 60 run: | uv run --locked python scripts/run_atomic_memory_clean_migration.py \ oceanbase --output "$MIGRATION_REPORTS/clean-migration" diff --git a/tests/e2e/server_storage.py b/tests/e2e/server_storage.py index 351b5fa6f2..8245d11fde 100644 --- a/tests/e2e/server_storage.py +++ b/tests/e2e/server_storage.py @@ -16,15 +16,22 @@ from pathlib import Path +from powercontext.builtin.artifacts.memory import EmbeddingProfile from powercontext.builtin.persistence.migrations.deployment import deployment_runner from powercontext.server.settings import ServerSettings -def prepare_server_database(settings: ServerSettings, *, evidence_dir: Path | None = None) -> None: +def prepare_server_database( + settings: ServerSettings, + *, + evidence_dir: Path | None = None, + embedding_profile: EmbeddingProfile | None = None, +) -> None: runner = deployment_runner( settings, evidence_dir=evidence_dir, lock_coordination="single-host" if settings.database.kind == "oceanbase" else None, + embedding_profile=embedding_profile, ) plan = runner.plan(backup_policy="skip") assert plan.state in {"uninitialized", "ready"} diff --git a/tests/e2e/test_atomic_memory_search_authorization_snapshot.py b/tests/e2e/test_atomic_memory_search_authorization_snapshot.py index 8439ce2174..1e8bf46d0f 100644 --- a/tests/e2e/test_atomic_memory_search_authorization_snapshot.py +++ b/tests/e2e/test_atomic_memory_search_authorization_snapshot.py @@ -65,6 +65,7 @@ from powercontext.server.authz.service import AccessControlService from powercontext.server.factory import create_server_app from powercontext.server.settings import AccessControlConfig, McpConfig, MetricsConfig, ServerSettings +from tests.e2e.server_storage import prepare_server_database ADMIN = PrincipalRef(type="service", id="admin") VIEWER = PrincipalRef(type="user", id="viewer") @@ -131,16 +132,23 @@ async def _isolated_database(backend: str, tmp_path: Path) -> AsyncIterator[Data @asynccontextmanager async def _server(database: DatabaseConfig, provider: str, tmp_path: Path, embedding: _Embedding): + settings = ServerSettings( + database=database, + runtime=RuntimeConfig(artifact_processing_families=()), + access=AccessControlConfig(mode="enforced", deployment_id=DEPLOYMENT_ID), + metrics=MetricsConfig(enabled=False), + mcp=McpConfig(enabled=False), + ) + await asyncio.to_thread( + prepare_server_database, + settings, + evidence_dir=tmp_path / "migration-evidence", + embedding_profile=embedding.profile, + ) open_access = open_builtin_access_control if provider == "builtin" else open_casbin_access_control async with open_access(database, bootstrap_administrators=(ADMIN,), deployment_id=DEPLOYMENT_ID) as access: app = create_server_app( - settings=ServerSettings( - database=database, - runtime=RuntimeConfig(artifact_processing_families=()), - access=AccessControlConfig(mode="enforced", deployment_id=DEPLOYMENT_ID), - metrics=MetricsConfig(enabled=False), - mcp=McpConfig(enabled=False), - ), + settings=settings, scheduler_path=tmp_path / "scheduler.db", access_control=access, authentication_provider=_Authentication(), diff --git a/tests/e2e/test_runtime_server.py b/tests/e2e/test_runtime_server.py index 9f7dccd889..99cfca932a 100644 --- a/tests/e2e/test_runtime_server.py +++ b/tests/e2e/test_runtime_server.py @@ -202,10 +202,17 @@ def _server_settings( ) -def _initialize_remote_server_storage(settings: ServerSettings, tmp_path: Path) -> None: +def _initialize_remote_server_storage( + settings: ServerSettings, tmp_path: Path, *, embedding_profile: EmbeddingProfile | None = None +) -> None: if not isinstance(settings.database, OceanBaseConfig): return - runner = deployment_runner(settings, evidence_dir=tmp_path / "migration-evidence", lock_coordination="single-host") + runner = deployment_runner( + settings, + evidence_dir=tmp_path / "migration-evidence", + lock_coordination="single-host", + embedding_profile=embedding_profile, + ) plan = runner.plan(backup_policy="skip") result = runner.apply( plan_id=plan.plan_id, @@ -821,12 +828,12 @@ async def scenario() -> None: @pytest.mark.parametrize("database_kind", ["sqlite", "oceanbase"]) def test_server_databases_share_vector_and_hybrid_search_behavior( database: SQLiteConfig | OceanBaseConfig, + tmp_path: Path, ) -> None: + settings = ServerSettings(database=database, mcp=McpConfig(enabled=False)) + _initialize_remote_server_storage(settings, tmp_path, embedding_profile=EMBEDDING_PROFILE) app = create_server_app( - settings=ServerSettings( - database=database, - mcp=McpConfig(enabled=False), - ), + settings=settings, candidate_pipeline=atomic_memory_pipeline(ContentCandidatePipeline()), embedding_model=KeywordEmbeddingModel(), ) diff --git a/tests/e2e/test_topic_search_authorization_snapshot.py b/tests/e2e/test_topic_search_authorization_snapshot.py index 63bd9d35fa..87f0445b9e 100644 --- a/tests/e2e/test_topic_search_authorization_snapshot.py +++ b/tests/e2e/test_topic_search_authorization_snapshot.py @@ -51,6 +51,7 @@ ) from powercontext.server.factory import create_server_app from powercontext.server.settings import AccessControlConfig, McpConfig, MetricsConfig, ServerSettings +from tests.e2e.server_storage import prepare_server_database ADMIN = PrincipalRef(type="service", id="admin") VIEWER = PrincipalRef(type="user", id="viewer") @@ -159,6 +160,19 @@ async def _server( tmp_path, provider, embedding, *, database=None, external_policy=None, access_factory=None, access_database=None ): database = database or SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'topics.db'}") + settings = ServerSettings( + database=database, + runtime=RuntimeConfig(artifact_processing_families=()), + access=AccessControlConfig(mode="enforced", deployment_id="topic-snapshot"), + metrics=MetricsConfig(enabled=False), + mcp=McpConfig(enabled=False), + ) + await asyncio.to_thread( + prepare_server_database, + settings, + evidence_dir=tmp_path / "migration-evidence", + embedding_profile=embedding.profile, + ) opener = open_casbin_access_control if provider == "casbin" else open_builtin_access_control async def evaluate(request): @@ -200,13 +214,7 @@ def allowed(evaluation): if access_factory is not None: access = await access_factory(access) app = create_server_app( - settings=ServerSettings( - database=database, - runtime=RuntimeConfig(artifact_processing_families=()), - access=AccessControlConfig(mode="enforced", deployment_id="topic-snapshot"), - metrics=MetricsConfig(enabled=False), - mcp=McpConfig(enabled=False), - ), + settings=settings, scheduler_path=tmp_path / "scheduler.db", authentication_provider=_Authentication(), access_control=access, From 04a33ffc4e64fd9f00f17b1b933dc932bef008d1 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 15:07:33 +0800 Subject: [PATCH 84/94] fix(runtime): validate managed SQLite without reserving the writer Keep managed admission checks in an explicit read snapshot. Unmanaged bootstrap retains BEGIN IMMEDIATE and rechecks schema registration after obtaining it. Regression coverage pauses real startup validation while an existing WAL runtime commits a Scope, including the Topic worker startup path. Constraint: Preserve processing, Atomic Memory and Topic readiness checks. Scope-risk: Read snapshots retain SQLite journal-mode locking semantics; no arbitrary-scale startup latency claim. Tested: Managed startup regression fails twice before the fix and passes afterward; 52 related and 69 embedding/profile/migration cases pass. Scoped Ruff, format and Ty Python 3.11 pass. --- .../builtin/runtime/composition.py | 13 +++- tests/test_managed_startup_concurrency.py | 73 +++++++++++++++++++ 2 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 tests/test_managed_startup_concurrency.py diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index ea9ab6f550..8fc5a090c5 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -1134,11 +1134,16 @@ async def open_builtin_contexts( # noqa: C901 -- Backend admission and resource load_vector_extension=embedding_model is not None, ) as profile: async with profile.database.transaction() as connection: - # This block is SQLite-only. Reserve SQLite's single-writer slot - # before startup schema reads, so concurrent Runtime openings cannot - # both establish a read snapshot and fail while upgrading to a write. - await connection.exec_driver_sql("BEGIN IMMEDIATE") managed = await connection.run_sync(lambda sync: inspect(sync).has_table("pc_schema_revision")) + if managed: + # Managed startup only validates; keep one read snapshot + # without reserving SQLite's writer slot. + await connection.exec_driver_sql("BEGIN") + else: + # Unmanaged bootstrap may write. Recheck after taking the + # writer slot in case a concurrent initializer registered it. + await connection.exec_driver_sql("BEGIN IMMEDIATE") + managed = await connection.run_sync(lambda sync: inspect(sync).has_table("pc_schema_revision")) if managed: await assert_processing_schema_ready(connection, canonical_processing_manifest(config)) await atomic_index.validate_current(connection) diff --git a/tests/test_managed_startup_concurrency.py b/tests/test_managed_startup_concurrency.py new file mode 100644 index 0000000000..0fb3b8bc4c --- /dev/null +++ b/tests/test_managed_startup_concurrency.py @@ -0,0 +1,73 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Managed startup validation must allow concurrent SQLite WAL writes.""" + +import asyncio + +import pytest + +from powercontext.builtin.persistence.migrations.deployment import deployment_runner +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.topic_memory import TopicMemoryRepository +from powercontext.builtin.runtime import BuiltinConfig, composition, open_builtin_contexts +from powercontext.builtin.scope import ScopeDraft +from powercontext.server.settings import ServerSettings + + +@pytest.mark.parametrize("topic_worker", [False, True]) +def test_managed_runtime_startup_allows_an_existing_runtime_to_commit(tmp_path, monkeypatch, topic_worker): + database = SQLiteConfig( + url=f"sqlite+aiosqlite:///{tmp_path / 'managed.db'}", busy_timeout_ms=250, journal_mode="WAL" + ) + runner = deployment_runner(ServerSettings(database=database)) + plan = runner.plan() + runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + + async def scenario(): + validating = asyncio.Event() + resume = asyncio.Event() + config = BuiltinConfig(database=database) + + class PausedTopicValidation(TopicMemoryRepository): + async def initialize(self, connection, /, **options): + await super().initialize(connection, **options) + validating.set() + await resume.wait() + + async def open_runtime(): + async with open_builtin_contexts(config, _topic_memory_worker=topic_worker): + pass + + async with open_builtin_contexts(config) as running: + monkeypatch.setattr(composition, "TopicMemoryRepository", PausedTopicValidation) + opening = asyncio.create_task(open_runtime()) + try: + await asyncio.wait_for(validating.wait(), timeout=10) + created = await asyncio.wait_for( + running.scopes.create( + ScopeDraft( + title="Concurrent capture", + summary="Committed while another runtime validates its managed storage.", + idempotency_key="concurrent-startup", + ) + ), + timeout=2, + ) + assert created.title == "Concurrent capture" + finally: + resume.set() + await opening + + asyncio.run(scenario()) From 23323a459d87aee845b3c1d95226e7982673c7f5 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 23 Oct 2026 16:32:20 +0800 Subject: [PATCH 85/94] fix(migrations): pin SQLite startup validation snapshot --- release/upgrade-support.json | 4 +- .../builtin/persistence/dream_schema.py | 5 + tests/e2e/test_atomic_artifact_search_http.py | 21 ++-- tests/e2e/test_managed_startup_snapshot.py | 104 ++++++++++++++++++ 4 files changed, 125 insertions(+), 9 deletions(-) create mode 100644 tests/e2e/test_managed_startup_snapshot.py diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 3e33daa7bf..fd507814ae 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "abafa3a241dcb8f2a88cc6844604e78f56538400c6c9f4db5cf49b25d0f8b2a6", - "rationale": "The shared s1300 -> s1400 -> s1401 chain registers installed 1.3.0 no-vector Server storage on SQLite, seekDB and OceanBase with Code disabled or enabled. SQLite/OceanBase Code caches remain separate; embedded seekDB Code uses the three frozen released tables. A selected vector profile binds the frozen schema and domain contract to the exact historical profile ID, model, dimension, distance and normalization. SQLite maintenance registers and backfills the Atomic vec0 helper and transactional triggers; native layouts preserve vector column dimensions and explicit HNSW parameters, including ordered key columns, algorithm, distance, library and construction/search options. Unknown index options remain rejected. The vector-only configuration declaration is not a second revision or recovery journal. Full layout and domain checks precede managed use; configured or injected model drift and missing helpers cannot trigger startup repair. No embedding provider is called during migration and historical embeddings, Topic shape, authority and evidence are preserved. The schema revision chain and registered layouts are unchanged. Full Server readiness validates Atomic Memory projections against authority and checks complete SQLite FTS identities for both vector and no-vector profiles before admission. Validation is read-only and damaged projections require explicit maintenance. Reviewed plans bind the draft validation resources; selected vector profiles have separately bound checksums. Target locks, original recovery points, acknowledgements and backup policies remain shared. The installed-release gate requires all twelve backend/configuration combinations, with Code enabled or disabled and vectors enabled or disabled. Vector cases seed nonempty Atomic and Topic vectors through the actual source Server, preserve exact embedding digests and retrieval shape, and verify historical/new semantic reads, configuration-drift rejection, SQLite missing-helper rejection and fresh-storage equivalence using the independently installed target. Maintenance must not call an embedding provider. SQLite vector cases also restore the automatic backup with the old Server before re-upgrading. Complete current-artifact and recovery/restore acceptance remain required. Source tests and schema captures do not establish that release acceptance passed. Policy remains upgrade-required." + "persistence_sha256": "403945de6a34fbc53e1a78a30dfd7c17bf5ebc2364d4484f4eea47e5e58caa49", + "rationale": "The shared s1300 -> s1400 -> s1401 chain registers installed 1.3.0 no-vector Server storage on SQLite, seekDB and OceanBase with Code disabled or enabled. SQLite/OceanBase Code caches remain separate; embedded seekDB Code uses the three frozen released tables. A selected vector profile binds the frozen schema and domain contract to the exact historical profile ID, model, dimension, distance and normalization. SQLite maintenance registers and backfills the Atomic vec0 helper and transactional triggers; native layouts preserve vector column dimensions and explicit HNSW parameters, including ordered key columns, algorithm, distance, library and construction/search options. Unknown index options remain rejected. The vector-only configuration declaration is not a second revision or recovery journal. Full layout and domain checks precede managed use; configured or injected model drift and missing helpers cannot trigger startup repair. No embedding provider is called during migration and historical embeddings, Topic shape, authority and evidence are preserved. The schema revision chain and registered layouts are unchanged. Full Server readiness validates Atomic Memory projections against authority and checks complete SQLite FTS identities for both vector and no-vector profiles before admission. Validation is read-only and damaged projections require explicit maintenance. SQLite startup preflight reads authority and projections in one explicit read transaction so concurrent WAL writes cannot produce a mixed-revision validation result; no schema or stored data format changes are introduced. Reviewed plans bind the draft validation resources; selected vector profiles have separately bound checksums. Target locks, original recovery points, acknowledgements and backup policies remain shared. The installed-release gate requires all twelve backend/configuration combinations, with Code enabled or disabled and vectors enabled or disabled. Vector cases seed nonempty Atomic and Topic vectors through the actual source Server, preserve exact embedding digests and retrieval shape, and verify historical/new semantic reads, configuration-drift rejection, SQLite missing-helper rejection and fresh-storage equivalence using the independently installed target. Maintenance must not call an embedding provider. SQLite vector cases also restore the automatic backup with the old Server before re-upgrading. Complete current-artifact and recovery/restore acceptance remain required. Source tests and schema captures do not establish that release acceptance passed. Policy remains upgrade-required." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/persistence/dream_schema.py b/src/powercontext/builtin/persistence/dream_schema.py index e96644dabe..6ebbdbb161 100644 --- a/src/powercontext/builtin/persistence/dream_schema.py +++ b/src/powercontext/builtin/persistence/dream_schema.py @@ -39,6 +39,11 @@ def __init__(self, reason: str, detail: str) -> None: async def assert_dream_schema_ready(engine: AsyncEngine) -> None: """Check before create_all, domain initializers, or workers; never execute DDL.""" async with engine.connect() as connection: + if connection.dialect.name == "sqlite": + # SQLite's legacy transaction mode does not begin a transaction for + # SELECTs. Pin one read snapshot before checking authority against + # projections, while still allowing concurrent WAL writers. + await connection.exec_driver_sql("BEGIN") await connection.run_sync(check_dream_schema) diff --git a/tests/e2e/test_atomic_artifact_search_http.py b/tests/e2e/test_atomic_artifact_search_http.py index 2f913f1ae7..f935fbdf11 100644 --- a/tests/e2e/test_atomic_artifact_search_http.py +++ b/tests/e2e/test_atomic_artifact_search_http.py @@ -93,15 +93,22 @@ def test_supported_long_query_preserves_all_terms_in_both_http_searches( migration_backend, query, mode, sqlite_expression_limit ): async def scenario(): + settings = ServerSettings( + database=migration_backend.config, + runtime=RuntimeConfig(artifact_processing_families=()), + metrics=MetricsConfig(enabled=False), + mcp=McpConfig(enabled=False), + ) + embedding = _Embedding() + prepare_server_database( + settings, + evidence_dir=migration_backend.directory / "migration-evidence", + embedding_profile=embedding.profile, + ) app = create_server_app( - settings=ServerSettings( - database=migration_backend.config, - runtime=RuntimeConfig(artifact_processing_families=()), - metrics=MetricsConfig(enabled=False), - mcp=McpConfig(enabled=False), - ), + settings=settings, scheduler_path=migration_backend.directory / "scheduler.db", - embedding_model=_Embedding(), + embedding_model=embedding, ) async with ( app.router.lifespan_context(app), diff --git a/tests/e2e/test_managed_startup_snapshot.py b/tests/e2e/test_managed_startup_snapshot.py new file mode 100644 index 0000000000..f3e4da2b19 --- /dev/null +++ b/tests/e2e/test_managed_startup_snapshot.py @@ -0,0 +1,104 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Healthy concurrent revisions must not cause managed startup to reject storage.""" + +import asyncio + +import aiosqlite +import pytest +from fastapi.testclient import TestClient + +from powercontext.builtin.persistence.migrations.deployment import deployment_runner +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime import BuiltinConfig, atomic_memory_rebuild, open_builtin_contexts +from powercontext.server.factory import create_server_app +from powercontext.server.settings import AccessControlConfig, BearerAuthConfig, McpConfig, ServerSettings +from tests.builtin.persistence.test_server_vector_migrations import ControlledEmbedding, profile + + +@pytest.mark.parametrize("entry", ["sdk", "server"]) +@pytest.mark.parametrize("vector", [False, True]) +def test_healthy_concurrent_update_does_not_block_managed_startup(tmp_path, monkeypatch, entry, vector): + embedding = ControlledEmbedding(profile(3)) if vector else None + settings = ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'managed.db'}", journal_mode="WAL"), + mcp=McpConfig(enabled=False), + access=AccessControlConfig(mode="disabled"), + auth=BearerAuthConfig(enabled=False), + ) + app = create_server_app(settings=settings, scheduler_path=tmp_path / "scheduler.db", embedding_model=embedding) + with TestClient(app) as client: + scope_response = client.post( + "/v1/scopes", + json={ + "title": "Concurrent startup", + "summary": "Concurrent update", + "idempotency_key": "concurrent-startup", + }, + ) + assert scope_response.status_code == 201, scope_response.text + scope = scope_response.json()["scope_id"] + response = client.post( + "/v1/memory/remember", json={"scope_id": scope, "kind": "fact", "text": "Before concurrent revision."} + ) + assert response.status_code == 200, response.text + artifact = response.json()["records"][0]["artifact"] + path = f"/v1/scopes/{scope}/artifacts/atomic-memory/{artifact['artifact_id']}" + etag = client.get(path).headers["ETag"] + original = atomic_memory_rebuild._record + updated = False + + async def interleaved_record(*args, **kwargs): + nonlocal updated + record = await original(*args, **kwargs) + # Schedule a real HTTP update between authoritative and projection + # reads in async startup. The synchronous migration runner already + # owns a snapshot and is not the boundary under test. + driver = args[0].connection.connection.driver_connection + if not updated and isinstance(driver, aiosqlite.Connection): + updated = True + replacement = client.put( + path, + headers={"If-Match": etag}, + json={"content": {"kind": "fact", "text": "After concurrent revision."}}, + ) + assert replacement.status_code == 200, replacement.text + return record + + monkeypatch.setattr(atomic_memory_rebuild, "_record", interleaved_record) + + async def start(): + async with open_builtin_contexts(BuiltinConfig(database=settings.database), embedding_model=embedding): + pass + + try: + if entry == "sdk": + asyncio.run(start()) + else: + with TestClient( + create_server_app( + settings=settings, embedding_model=embedding, scheduler_path=tmp_path / "second-scheduler.db" + ) + ): + pass + finally: + monkeypatch.setattr(atomic_memory_rebuild, "_record", original) + assert ( + deployment_runner(settings, embedding_profile=None if embedding is None else embedding.profile) + .verify() + .state + == "ready" + ) + assert client.get(path).json()["content"]["text"] == "After concurrent revision." From 1f6c89ce4c7e50c74878b746cc101c918ad2e8c3 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 23 Oct 2026 17:24:28 +0800 Subject: [PATCH 86/94] test(migrations): initialize long-query storage before event loop --- tests/e2e/test_atomic_artifact_search_http.py | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/e2e/test_atomic_artifact_search_http.py b/tests/e2e/test_atomic_artifact_search_http.py index f935fbdf11..5b90442ec4 100644 --- a/tests/e2e/test_atomic_artifact_search_http.py +++ b/tests/e2e/test_atomic_artifact_search_http.py @@ -92,19 +92,20 @@ async def with_limit(connection): def test_supported_long_query_preserves_all_terms_in_both_http_searches( migration_backend, query, mode, sqlite_expression_limit ): + settings = ServerSettings( + database=migration_backend.config, + runtime=RuntimeConfig(artifact_processing_families=()), + metrics=MetricsConfig(enabled=False), + mcp=McpConfig(enabled=False), + ) + embedding = _Embedding() + prepare_server_database( + settings, + evidence_dir=migration_backend.directory / "migration-evidence", + embedding_profile=embedding.profile, + ) + async def scenario(): - settings = ServerSettings( - database=migration_backend.config, - runtime=RuntimeConfig(artifact_processing_families=()), - metrics=MetricsConfig(enabled=False), - mcp=McpConfig(enabled=False), - ) - embedding = _Embedding() - prepare_server_database( - settings, - evidence_dir=migration_backend.directory / "migration-evidence", - embedding_profile=embedding.profile, - ) app = create_server_app( settings=settings, scheduler_path=migration_backend.directory / "scheduler.db", From 0b1d6f31188d8b09e4b7a467ec04309c1e6c7040 Mon Sep 17 00:00:00 2001 From: "Xin.Zh" Date: Fri, 23 Oct 2026 22:03:10 +0800 Subject: [PATCH 87/94] ci(oceanbase): budget isolated long-query acceptance (#2211) Allow complete Server setup within the aggregate long-query CI step. Eight healthy isolated cases can exceed the former ten-minute limit while each remains inside the existing three-minute process deadline. Constraint: Keep all assertions, the 180-second case limit and the 100-minute job limit. Scope-risk: CI aggregate execution budget only; runtime and migration resources are unchanged. Tested: Eight real OceanBase cases passed without skips in 665.9 seconds; YAML structure and git diff --check pass; Ty and other validators pass in the existing environment, while the hook wrapper reports JSON normalizations without a tracked JSON diff. --- .github/workflows/e2e-harness.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index d3671f1383..85b0a26a9e 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -336,7 +336,8 @@ jobs: oceanbase --output "$MIGRATION_REPORTS/clean-migration" - name: Test long-query retrieval and admission without skips - timeout-minutes: 10 + # Eight isolated cases include full Server setup and retain 180s deadlines. + timeout-minutes: 30 run: | uv run --locked python scripts/run_atomic_memory_clean_migration.py \ oceanbase --output "$MIGRATION_REPORTS/long-query" --tests \ From c3d642da774cdcdef49ff7d57009eefd7c6ced28 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sat, 24 Oct 2026 13:25:26 +0800 Subject: [PATCH 88/94] fix(migrations): register native portable receipt boolean --- release/upgrade-support.json | 4 +- .../server_resources/schemas/mysql-s1402.json | 8 +- .../test_mysql_migration_runner.py | 49 +++++++- .../persistence/test_mysql_portable_layout.py | 118 ++++++++++++++++++ tests/test_bub_compose_startup.py | 9 +- 5 files changed, 179 insertions(+), 9 deletions(-) create mode 100644 tests/builtin/persistence/test_mysql_portable_layout.py diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 9d4a365884..d6c7aabdf4 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.4.0", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "bf8900ac8131e2db663c6a1deaacff4641e8d9bc4722d7693095a1ebd072e41c", - "rationale": "The shared s1300 -> s1400 -> s1401 -> s1402 chain registers installed 1.3.0 no-vector Server storage on SQLite, seekDB and OceanBase with Code disabled or enabled. SQLite/OceanBase Code caches remain separate; embedded seekDB Code uses the three frozen released tables. A selected vector profile binds the frozen schema and domain contract to the exact historical profile ID, model, dimension, distance and normalization. SQLite maintenance registers and backfills the Atomic vec0 helper and transactional triggers; native layouts preserve vector column dimensions and explicit HNSW parameters, including ordered key columns, algorithm, distance, library and construction/search options. Unknown index options remain rejected. The vector-only configuration declaration is not a second revision or recovery journal. Full layout and domain checks precede managed use; configured or injected model drift and missing helpers cannot trigger startup repair. No embedding provider is called during migration and historical embeddings, Topic shape, authority and evidence are preserved. Full Server readiness validates Atomic Memory projections against authority and checks complete SQLite FTS identities for both vector and no-vector profiles before admission. Validation is read-only and damaged projections require explicit maintenance. SQLite startup preflight reads authority and projections in one explicit read transaction so concurrent WAL writes cannot produce a mixed-revision validation result; no schema or stored data format changes are introduced. Reviewed plans bind the draft validation resources; selected vector profiles have separately bound checksums. Target locks, original recovery points, acknowledgements and backup policies remain shared. The installed-release gate requires all twelve backend/configuration combinations, with Code enabled or disabled and vectors enabled or disabled. Vector cases seed nonempty Atomic and Topic vectors through the actual source Server, preserve exact embedding digests and retrieval shape, and verify historical/new semantic reads, configuration-drift rejection, SQLite missing-helper rejection and fresh-storage equivalence using the independently installed target. Maintenance must not call an embedding provider. SQLite vector cases also restore the automatic backup with the old Server before re-upgrading. Complete current-artifact and recovery/restore acceptance remain required. Source tests and schema captures do not establish that release acceptance passed. Policy remains upgrade-required. Revision s1402 adapts the storage already present on master: portable restore receipts and native microsecond precision for Artifact creation, policy updates, Topic publications and tag assignments. Earlier revision resources and checksums remain unchanged. Pending archive restores reject ordinary managed startup and readiness; only the explicit archive recovery entry can open a registered layout to finish projections, and it still checks the configured embedding profile. Interrupted first-revision MySQL initialization defers domain preflight until the executor has completed the registered baseline statements and seed data; missing maintenance evidence and unregistered DDL remain rejected." + "persistence_sha256": "56be4a21d721f2c8025c5b4a8b4bef34a91a72028f0eab01d1d30c01b87a0c31", + "rationale": "The shared s1300 -> s1400 -> s1401 -> s1402 chain registers installed 1.3.0 no-vector Server storage on SQLite, seekDB and OceanBase with Code disabled or enabled. SQLite/OceanBase Code caches remain separate; embedded seekDB Code uses the three frozen released tables. A selected vector profile binds the frozen schema and domain contract to the exact historical profile ID, model, dimension, distance and normalization. SQLite maintenance registers and backfills the Atomic vec0 helper and transactional triggers; native layouts preserve vector column dimensions and explicit HNSW parameters, including ordered key columns, algorithm, distance, library and construction/search options. Unknown index options remain rejected. The vector-only configuration declaration is not a second revision or recovery journal. Full layout and domain checks precede managed use; configured or injected model drift and missing helpers cannot trigger startup repair. No embedding provider is called during migration and historical embeddings, Topic shape, authority and evidence are preserved. Full Server readiness validates Atomic Memory projections against authority and checks complete SQLite FTS identities for both vector and no-vector profiles before admission. Validation is read-only and damaged projections require explicit maintenance. SQLite startup preflight reads authority and projections in one explicit read transaction so concurrent WAL writes cannot produce a mixed-revision validation result; no schema or stored data format changes are introduced. Reviewed plans bind the draft validation resources; selected vector profiles have separately bound checksums. Target locks, original recovery points, acknowledgements and backup policies remain shared. The installed-release gate requires all twelve backend/configuration combinations, with Code enabled or disabled and vectors enabled or disabled. Vector cases seed nonempty Atomic and Topic vectors through the actual source Server, preserve exact embedding digests and retrieval shape, and verify historical/new semantic reads, configuration-drift rejection, SQLite missing-helper rejection and fresh-storage equivalence using the independently installed target. Maintenance must not call an embedding provider. SQLite vector cases also restore the automatic backup with the old Server before re-upgrading. Complete current-artifact and recovery/restore acceptance remain required. Source tests and schema captures do not establish that release acceptance passed. Policy remains upgrade-required. Revision s1402 adapts the storage already present on master: portable restore receipts and native microsecond precision for Artifact creation, policy updates, Topic publications and tag assignments. Earlier revision resources and checksums remain unchanged. Pending archive restores reject ordinary managed startup and readiness; only the explicit archive recovery entry can open a registered layout to finish projections, and it still checks the configured embedding profile. Interrupted first-revision MySQL initialization defers domain preflight until the executor has completed the registered baseline statements and seed data; missing maintenance evidence and unregistered DDL remain rejected. The s1402 portable receipt boolean is explicitly declared and registered as TINYINT(1), matching native MySQL-compatible reflection without relaxing schema matching; earlier revision resources are unchanged." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1402.json b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1402.json index 8cd9b32951..cdc2492aff 100644 --- a/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1402.json +++ b/src/powercontext/builtin/persistence/migrations/server_resources/schemas/mysql-s1402.json @@ -3,7 +3,7 @@ "revisions": { "s1402": { "statements": [ - "CREATE TABLE pc_portable_restore_receipts (\n\tbundle_id VARCHAR(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tformat_version INTEGER NOT NULL, \n\ttotal_digest VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trecord_count BIGINT NOT NULL, \n\tstatus VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tinserted BIGINT NOT NULL, \n\talready_present BIGINT NOT NULL, \n\tprojections_ready BOOL NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (bundle_id), \n\tCONSTRAINT ck_pc_portable_restore_receipts_status CHECK (status IN ('authoritative_restored', 'ready')), \n\tCONSTRAINT ck_pc_portable_restore_receipts_record_count CHECK (record_count >= 0), \n\tCONSTRAINT ck_pc_portable_restore_receipts_inserted CHECK (inserted >= 0), \n\tCONSTRAINT ck_pc_portable_restore_receipts_existing CHECK (already_present >= 0)\n)", + "CREATE TABLE pc_portable_restore_receipts (\n\tbundle_id VARCHAR(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tformat_version INTEGER NOT NULL, \n\ttotal_digest VARCHAR(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trecord_count BIGINT NOT NULL, \n\tstatus VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tinserted BIGINT NOT NULL, \n\talready_present BIGINT NOT NULL, \n\tprojections_ready TINYINT(1) NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (bundle_id), \n\tCONSTRAINT ck_pc_portable_restore_receipts_status CHECK (status IN ('authoritative_restored', 'ready')), \n\tCONSTRAINT ck_pc_portable_restore_receipts_record_count CHECK (record_count >= 0), \n\tCONSTRAINT ck_pc_portable_restore_receipts_inserted CHECK (inserted >= 0), \n\tCONSTRAINT ck_pc_portable_restore_receipts_existing CHECK (already_present >= 0)\n)", "ALTER TABLE `pc_artifacts` MODIFY COLUMN `created_at` DATETIME(6) NULL", "ALTER TABLE `pc_profile_policies` MODIFY COLUMN `updated_at` DATETIME(6) NOT NULL", "ALTER TABLE `pc_topic_memory_revision_publications` MODIFY COLUMN `published_at` DATETIME(6) NOT NULL", @@ -98,8 +98,10 @@ }, "projections_ready": { "type": [ - "bool", - [] + "tinyint", + [ + "1" + ] ], "nullable": false, "default": null, diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index 8a03d79fc9..d361734fcf 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -817,14 +817,18 @@ def test_mysql_full_server_registers_portable_storage_and_retains_timestamp_prec if origin == "released": _load_full_server_baseline(backend) runner = _full_server_runner(backend) - assert _apply(runner, policy="manual").state == "ready" + try: + assert _apply(runner, policy="manual").state == "ready" + finally: + _capture_portable_schema(backend) assert runner.verify().state == "ready" # Native DATETIME stores UTC values without timezone metadata. timestamp = datetime(2026, 10, 24, 12, 34, 56, 123456, tzinfo=UTC).replace(tzinfo=None) def verify(connection, _identity, _guard): inventory = mysql_inventory(connection) - assert "pc_portable_restore_receipts" in inventory["tables"] + receipt = inventory["tables"]["pc_portable_restore_receipts"] + assert receipt["columns"]["projections_ready"]["type"] == ["tinyint", ["1"]] for table, column in ( ("pc_artifacts", "created_at"), ("pc_profile_policies", "updated_at"), @@ -839,7 +843,21 @@ def verify(connection, _identity, _guard): ), {"content": b'{"history":"retained"}', "created_at": timestamp}, ) + connection.execute( + text( + "INSERT INTO pc_portable_restore_receipts " + "(bundle_id,format_version,total_digest,record_count,status,inserted,already_present,projections_ready,updated_at) " + "VALUES ('receipt',1,'sha256:fixture',0,'ready',0,0,1,:updated_at)" + ), + {"updated_at": timestamp}, + ) connection.commit() + assert ( + connection.exec_driver_sql( + "SELECT projections_ready FROM pc_portable_restore_receipts WHERE bundle_id='receipt'" + ).scalar_one() + == 1 + ) assert ( connection.exec_driver_sql( "SELECT created_at FROM pc_artifacts WHERE scope_id='precision' AND artifact_id='retained'" @@ -851,6 +869,33 @@ def verify(connection, _identity, _guard): assert not _apply(runner, policy="manual").changed +def _capture_portable_schema(backend: _BackendTarget) -> None: + """Keep content-free native reflection even when a new DDL prefix is rejected.""" + + def capture(connection, _identity, _guard): + names = {row[0] for row in connection.exec_driver_sql("SHOW FULL TABLES")} + tables = {} + for name in ( + "pc_portable_restore_receipts", + "pc_artifacts", + "pc_profile_policies", + "pc_topic_memory_revision_publications", + "pc_artifact_tags", + ): + if name in names: + tables[name] = connection.exec_driver_sql(f"SHOW CREATE TABLE `{name}`").one()[1] + return {"backend_version": connection.exec_driver_sql("SELECT VERSION()").scalar_one(), "tables": tables} + + try: + diagnostic = backend.connections().run(capture) + except Exception as error: + diagnostic = {"capture_error_type": type(error).__name__} + backend.evidence_directory.mkdir(parents=True, exist_ok=True) + (backend.evidence_directory / "portable-schema.json").write_text( + json.dumps(diagnostic, ensure_ascii=False, indent=2), encoding="utf-8" + ) + + def test_mysql_full_server_resumes_interrupted_initialization_before_manifest_seed(backend: _BackendTarget) -> None: runner = _full_server_runner(backend) _interrupt_after_ddl(runner, "CREATE TABLE `pc_artifact_candidate_versions`", policy="auto") diff --git a/tests/builtin/persistence/test_mysql_portable_layout.py b/tests/builtin/persistence/test_mysql_portable_layout.py new file mode 100644 index 0000000000..debe53a35f --- /dev/null +++ b/tests/builtin/persistence/test_mysql_portable_layout.py @@ -0,0 +1,118 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Strict portable layout recognition using native Boolean reflection evidence. + +The receipt SHOW CREATE below is modeled, not a claimed native capture. Its +Boolean spelling is anchored to the separately captured seekDB 1.4 table. +""" + +import json +from pathlib import Path + +import pytest + +from powercontext.builtin.persistence.migrations.deployment import server_bundle +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.mysql_schema import MySQLSchema +from tests.builtin.persistence.test_mysql_migration_schema import _inventory, _ReflectedDatabase + +_RECEIPT = "pc_portable_restore_receipts" +_RECEIPT_SHOW_CREATE = """CREATE TABLE `pc_portable_restore_receipts` ( + `bundle_id` varchar(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `format_version` int(11) NOT NULL, + `total_digest` varchar(71) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `record_count` bigint(20) NOT NULL, + `status` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `inserted` bigint(20) NOT NULL, + `already_present` bigint(20) NOT NULL, + `projections_ready` tinyint(1) NOT NULL, + `updated_at` datetime NOT NULL, + PRIMARY KEY (`bundle_id`), + CONSTRAINT `ck_pc_portable_restore_receipts_status` CHECK ((`status` in ('authoritative_restored','ready'))), + CONSTRAINT `ck_pc_portable_restore_receipts_record_count` CHECK ((`record_count` >= 0)), + CONSTRAINT `ck_pc_portable_restore_receipts_inserted` CHECK ((`inserted` >= 0)), + CONSTRAINT `ck_pc_portable_restore_receipts_existing` CHECK ((`already_present` >= 0)) +) ORGANIZATION INDEX DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC +COMPRESSION = 'zstd_1.3.8' BLOCK_SIZE = 16384 TABLET_SIZE = 134217728 PCTFREE = 0""" +_PRECISION_COLUMNS = ( + ("pc_artifacts", "created_at"), + ("pc_profile_policies", "updated_at"), + ("pc_topic_memory_revision_publications", "published_at"), + ("pc_artifact_tags", "assigned_at"), +) + + +def historical_inventory(bundle): + inventory = {"tables": {}, "views": [], "triggers": []} + for revision in ("s1300", "s1400", "s1401"): + fragment = json.loads((bundle.directory / f"schemas/mysql-{revision}.json").read_text()) + for change in fragment["revisions"][revision]["changes"]: + for name, table in change.items(): + if table is None: + del inventory["tables"][name] + else: + inventory["tables"][name] = table + return inventory + + +def receipt_inventory(): + return _inventory(_ReflectedDatabase({_RECEIPT: _RECEIPT_SHOW_CREATE}))["tables"][_RECEIPT] + + +def test_native_boolean_reflection_allows_only_registered_portable_recovery_prefixes(): + fixture = Path(__file__).parents[2] / "fixtures/database_migrations/schemas/seekdb_v1_3_0_checks.json" + captured = json.loads(fixture.read_text())["tables"]["pc_model_usage_daily"] + reflected = _inventory(_ReflectedDatabase({"pc_model_usage_daily": captured["show_create_table"]})) + native_boolean = reflected["tables"]["pc_model_usage_daily"]["columns"]["input_complete"] + receipt = receipt_inventory() + assert receipt["columns"]["projections_ready"] == native_boolean + + bundle = server_bundle() + schema = MySQLSchema(bundle) + inventory = historical_inventory(bundle) + assert schema.identify(inventory) == "s1401" + inventory["tables"][_RECEIPT] = receipt + recovery = schema.recovery(inventory, "s1401") + assert recovery.revision == "s1402" + assert recovery.pending_statements == tuple( + f"ALTER TABLE `{table}` MODIFY COLUMN `{column}` DATETIME(6) " + + ("NULL" if table == "pc_artifacts" else "NOT NULL") + for table, column in _PRECISION_COLUMNS + ) + for position, (table, column) in enumerate(_PRECISION_COLUMNS, 1): + inventory["tables"][table]["columns"][column]["type"] = ["datetime", ["6"]] + assert schema.recovery(inventory, "s1401").pending_statements == recovery.pending_statements[position:] + assert schema.identify(inventory) == "s1402" + schema.verify(inventory, "s1402") + + +@pytest.mark.parametrize("damage", ["boolean_width", "timestamp_precision", "missing_constraint"]) +def test_portable_target_rejects_unregistered_types_and_constraints(damage): + bundle = server_bundle() + schema = MySQLSchema(bundle) + inventory = historical_inventory(bundle) + inventory["tables"][_RECEIPT] = receipt_inventory() + for table, column in _PRECISION_COLUMNS: + inventory["tables"][table]["columns"][column]["type"] = ["datetime", ["6"]] + if damage == "boolean_width": + inventory["tables"][_RECEIPT]["columns"]["projections_ready"]["type"] = ["tinyint", ["2"]] + elif damage == "timestamp_precision": + inventory["tables"]["pc_artifacts"]["columns"]["created_at"]["type"] = ["datetime", ["3"]] + else: + del inventory["tables"][_RECEIPT]["checks"]["ck_pc_portable_restore_receipts_record_count"] + with pytest.raises(MigrationError, match="incompatible_schema"): + schema.verify(inventory, "s1402") + with pytest.raises(MigrationError, match="incompatible_schema"): + schema.recovery(inventory, "s1401") diff --git a/tests/test_bub_compose_startup.py b/tests/test_bub_compose_startup.py index d2b955fb8f..58dc7cadde 100644 --- a/tests/test_bub_compose_startup.py +++ b/tests/test_bub_compose_startup.py @@ -32,7 +32,9 @@ def _run(tmp_path, monkeypatch, *, vector=False, code=False, failed_action=""): events = tmp_path / "events.jsonl" # Substitute only process boundaries: execute the real shell launcher and # its inline maintenance program without Docker or a database service. - executable = f"#!{sys.executable}\n" + r""" + executable = ( + f"#!{sys.executable}\n" + + r""" import json import os import subprocess @@ -64,6 +66,7 @@ def record(action): assert args[args.index("--plan-id") + 1] == "reviewed-plan" print(json.dumps({"plan_id": "reviewed-plan", "state": "ready", "server_ready": True})) """ + ) for name in ("docker", "powercontext"): path = commands / name path.write_text(executable, encoding="utf-8") @@ -75,7 +78,9 @@ def record(action): monkeypatch.setenv("POWERCONTEXT_E2E_DATABASE", "oceanbase") monkeypatch.setenv("POWERCONTEXT_E2E_OUTPUT", str(tmp_path / "output")) monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_KIND", "oceanbase") - monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_URL", "mysql+aoceanbase://test@database.invalid:2881/test?charset=utf8mb4") + monkeypatch.setenv( + "POWERCONTEXT_SERVER_DATABASE_URL", "mysql+aoceanbase://test@database.invalid:2881/test?charset=utf8mb4" + ) monkeypatch.setenv("POWERCONTEXT_SERVER_CODE", json.dumps({"enabled": code})) monkeypatch.setenv("BUB_TEST_EVENTS", str(events)) monkeypatch.setenv("BUB_TEST_FAILED_ACTION", failed_action) From 135838e8957caa991f625394f067f5e0e49e208e Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sat, 24 Oct 2026 13:32:41 +0800 Subject: [PATCH 89/94] fix(desktop): qualify current Server protocol --- desktop/README.md | 2 +- desktop/README.zh.md | 2 +- desktop/VALIDATION.md | 35 ++++++++++++++++--- .../src/connections/compatibility.json | 19 ++++++++++ 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/desktop/README.md b/desktop/README.md index a661c77d41..ae4a6d96a9 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -119,7 +119,7 @@ Installed boundary checks exercise an 8192-byte Unicode note, reject over-budget After upgrading from an earlier preview, open Connections and select your saved connection. Profiles matching the bundled API contract appear under “Available for this Desktop build”; compare their tested Server builds with your deployment before selecting one, saving, checking and explicitly using the connection again. New connections start with no compatibility selection. -Historical profiles appear as disabled records. A saved incompatible or unavailable selection remains visible with an explanation until explicitly replaced; it is never silently upgraded. For the qualified current Server sources, explicitly choose `sqlite-atomic-d9a78567-v1` and check the connection before selecting a Scope; that record's contract digest matches this build's bundled contract, while `sqlite-atomic-caf2a35a-v1`, `sqlite-atomic-683e2db0-v1` and `sqlite-atomic-cab054db-v1` qualify their corresponding Server source deployments and do not match it. The recorded Server source and wheel identities must match the deployment you intend to use. The independently qualified `ci-fixtures` build is for disposable acceptance tests. Historical qualification evidence and remaining requirements are documented in the [validation guide](VALIDATION.md). +Historical profiles appear as disabled records. A saved incompatible or unavailable selection remains visible with an explanation until explicitly replaced; it is never silently upgraded. For the qualified current Server sources, explicitly choose `sqlite-atomic-8f10c560-v1` and check the connection before selecting a Scope; that record's contract digest matches this build's bundled contract, while `sqlite-atomic-caf2a35a-v1`, `sqlite-atomic-683e2db0-v1`, `sqlite-atomic-cab054db-v1` and `sqlite-atomic-d9a78567-v1` qualify their corresponding Server source deployments and do not match it. The recorded Server source and wheel identities must match the deployment you intend to use. The independently qualified `ci-fixtures` build is for disposable acceptance tests. Historical qualification evidence and remaining requirements are documented in the [validation guide](VALIDATION.md). ## Build on macOS diff --git a/desktop/README.zh.md b/desktop/README.zh.md index 40fd44df3e..b2bc710428 100644 --- a/desktop/README.zh.md +++ b/desktop/README.zh.md @@ -102,7 +102,7 @@ Windows GitHub Actions 会构建未签名安装包，安装到临时中文路径 从早期预览版升级后，请打开「连接」并选中已保存的连接。与当前 API 契约匹配的配置显示在「此 Desktop 构建可用」中；先核对已测试的 Server 构建与实际部署，再明确选择、保存、检查并使用该连接。新连接默认不选择兼容配置。 -历史配置显示为不可选择的记录。已保存的不匹配或不可用配置仍会显示并说明原因，须明确替换，不会静默升级。对于已资格化的当前 Server 源码，明确选择 `sqlite-atomic-d9a78567-v1` 并在选取范围前检查连接；该记录的契约摘要与本构建的打包契约匹配，而 `sqlite-atomic-caf2a35a-v1`、`sqlite-atomic-683e2db0-v1` 与 `sqlite-atomic-cab054db-v1` 只资格化各自对应 Server 源码的部署，并不匹配打包契约。已记录的 Server 源码和 wheel 身份必须与实际部署一致。独立验证的 `ci-fixtures` 构建仅用于临时验收测试。历史资格证据和剩余要求见[验证说明](VALIDATION.md)。 +历史配置显示为不可选择的记录。已保存的不匹配或不可用配置仍会显示并说明原因，须明确替换，不会静默升级。对于已资格化的当前 Server 源码，明确选择 `sqlite-atomic-8f10c560-v1` 并在选取范围前检查连接；该记录的契约摘要与本构建的打包契约匹配，而 `sqlite-atomic-caf2a35a-v1`、`sqlite-atomic-683e2db0-v1`、`sqlite-atomic-cab054db-v1` 与 `sqlite-atomic-d9a78567-v1` 只资格化各自对应 Server 源码的部署，并不匹配打包契约。已记录的 Server 源码和 wheel 身份必须与实际部署一致。独立验证的 `ci-fixtures` 构建仅用于临时验收测试。历史资格证据和剩余要求见[验证说明](VALIDATION.md)。 ## 在 macOS 上构建 diff --git a/desktop/VALIDATION.md b/desktop/VALIDATION.md index 5ec28c8e48..07956389d3 100644 --- a/desktop/VALIDATION.md +++ b/desktop/VALIDATION.md @@ -22,7 +22,7 @@ cargo test --locked --manifest-path desktop/src-tauri/Cargo.toml ## Compatibility scope -[compatibility.json](src-tauri/src/connections/compatibility.json) preserves the recorded Server sources, normalized OpenAPI contract digests and qualification-wheel digests for the historical profiles below. The current contract is qualified by `sqlite-atomic-d9a78567-v1` below; ordinary builds require explicitly selecting it and checking the connection before business operations become available. The `sqlite-atomic-caf2a35a-v1`, `sqlite-atomic-683e2db0-v1` and `sqlite-atomic-cab054db-v1` records qualify their corresponding Server source deployments and do not match this bundled contract. Historical records are disabled in the selector, and saved unavailable selections remain visible until explicitly replaced. CI builds have their own artifact identities and do not change product qualification. +[compatibility.json](src-tauri/src/connections/compatibility.json) preserves the recorded Server sources, normalized OpenAPI contract digests and qualification-wheel digests for the historical profiles below. The current contract is qualified by `sqlite-atomic-8f10c560-v1` below; ordinary builds require explicitly selecting it and checking the connection before business operations become available. The `sqlite-atomic-caf2a35a-v1`, `sqlite-atomic-683e2db0-v1`, `sqlite-atomic-cab054db-v1` and `sqlite-atomic-d9a78567-v1` records qualify their corresponding Server source deployments and do not match this bundled contract. Historical records are disabled in the selector, and saved unavailable selections remain visible until explicitly replaced. CI builds have their own artifact identities and do not change product qualification. The real Server harness covers SQLite with anonymous loopback, static Bearer, HTTPS with explicit CA/base path, and injected-provider/enforced access. It exercises save/search/exact reads, identity changes, binding revocation, ambiguous writes without replay and Scope pagination. Installed UI checks cover repeated launches preserving the existing editor, late committed-save responses preserving a new Scope draft, explicit connection/Scope selection, clipboard copies, connection isolation, draft cancellation, byte limits and independent Server survival after Desktop exit. @@ -30,9 +30,36 @@ Selecting a compatibility profile does not attest the remote binary identity. Ex ### Current Server protocol qualification +#### Server 8f10c560 protocol qualification + +The bundled contract is qualified by `sqlite-atomic-8f10c560-v1`. It identifies Server sources +at `8f10c560776b32b717c3511fad9da349982a5748` and normalized OpenAPI digest +`5878999ce9e239d6642e4da858074b281ed465629432c578b3584eec47cbce6c`. +The qualification wheel is `powercontext-1.4.1.dev242+g8f10c5607-py3-none-any.whl`, +SHA-256 `c771a39de6d6797efc33d406619aa02eeb7d7731f27147b3287079616709db97`. +All 583 packaged runtime files across `powercontext/` and `powercontext_service_bootstrap/` +match that Git commit byte for byte. The OpenAPI contract, project configuration and lock also match. +The sorted runtime source-manifest digest is +`d9f578a88dc9a2e0fffb3b75b9c4fba8bc245720857333c6bd6ad590e488b17c` +(one `wheel/path sha256` entry per line, with a final newline). + +The existing real-Server harness passed against the isolated wheel and SQLite in all four modes: +anonymous loopback, static Bearer, HTTPS with an explicit CA and base path, and an injected +provider with enforced access. The ten recorded operations cover Scope selection and pagination, +Atomic Memory save/search/exact revision reads, identity changes, same-identity revocation, +committed writes with lost responses without replay, and independent Server survival after the +client exits. + +Qualification ran on macOS 15.7.3 ARM64 with Python 3.12.13 and Rust 1.95.0, from a clean worktree +at `8f10c560776b32b717c3511fad9da349982a5748`. The native `server_probe` was built with the unchanged +product modules and Cargo lock. Credentials used session-only storage. This establishes the stated +Server protocol compatibility on that host; it does not establish installed GUI workflows, native +credential persistence, Intel or Windows execution, signing, notarization, or release qualification. +Historical records retain their exact source, contract, wheel and platform evidence. + #### Server d9a78567 protocol qualification -The bundled contract is qualified by `sqlite-atomic-d9a78567-v1`. The record identifies Server +The corresponding historical contract is qualified by `sqlite-atomic-d9a78567-v1`. The record identifies Server sources at `d9a785675dedb72cd72a386b346e590f4c067ac0` and normalized OpenAPI digest `a6758fd35296abefb55ab86b152584a369d5f2deedfee7a3f9822e16ad1edc25`. The qualification wheel is `powercontext-1.4.1.dev69+ga200d6094.d20261022-py3-none-any.whl`, @@ -139,8 +166,8 @@ matching `sqlite-atomic-caf2a35a-v1`. The observed app is identified by the exec this record does not identify its exact build source revision. This is historical evidence, not a procedure or native UI qualification for the current bundled contract. -The current `sqlite-atomic-d9a78567-v1` record has non-GUI protocol evidence on Linux x86_64 only. -Native macOS save/search/exact-read and clipboard acceptance against that record remains unverified. +The current `sqlite-atomic-8f10c560-v1` record has native protocol evidence on macOS ARM64. +Installed macOS GUI save/search/exact-read and clipboard acceptance against that record remains unverified. The normal release app (without `ci-fixtures`) was operated on macOS 26.5.2 ARM64 against the wheel recorded under [Server caf2a35a protocol qualification](#server-caf2a35a-protocol-qualification) diff --git a/desktop/src-tauri/src/connections/compatibility.json b/desktop/src-tauri/src/connections/compatibility.json index 9132847dfb..5e0738421a 100644 --- a/desktop/src-tauri/src/connections/compatibility.json +++ b/desktop/src-tauri/src/connections/compatibility.json @@ -151,5 +151,24 @@ "search_memory" ], "evidence": "desktop/VALIDATION.md#server-d9a78567-protocol-qualification" + }, + { + "id": "sqlite-atomic-8f10c560-v1", + "serverCommit": "8f10c560776b32b717c3511fad9da349982a5748", + "contractSha256": "5878999ce9e239d6642e4da858074b281ed465629432c578b3584eec47cbce6c", + "artifactSha256": "c771a39de6d6797efc33d406619aa02eeb7d7731f27147b3287079616709db97", + "operations": [ + "get_access_principal", + "get_artifact_revision", + "get_capabilities", + "get_default_scope", + "get_liveness", + "get_readiness", + "get_scope", + "list_scopes", + "remember_memory", + "search_memory" + ], + "evidence": "desktop/VALIDATION.md#server-8f10c560-protocol-qualification" } ] From d12e791cf912a87870ca10307a33fe8d491cb245 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sat, 24 Oct 2026 15:54:13 +0800 Subject: [PATCH 90/94] fix(ci): stabilize isolated migration validation --- .github/workflows/master.yml | 8 ++++---- scripts/run_atomic_memory_clean_migration.py | 5 ++++- tests/builtin/runtime/test_processing_composition.py | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 95886945d1..58ae89a5f3 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -384,7 +384,7 @@ jobs: code-seekdb: needs: changes if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').database) }} - timeout-minutes: 20 + timeout-minutes: 30 runs-on: ubuntu-latest strategy: fail-fast: false @@ -405,9 +405,9 @@ jobs: run: make code-seekdb-test - name: Test unified migrations with embedded seekdb - # The isolated full-Server cases exceed ten minutes in aggregate; - # each runner case retains its separate 180-second deadline below. - timeout-minutes: 15 + # Full-Server cases run in separate processes and exceed fifteen minutes + # in aggregate; each runner keeps its 180-second deadline below. + timeout-minutes: 25 env: POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" POWERCONTEXT_TEST_MIGRATION_SQL_LOG: "1" diff --git a/scripts/run_atomic_memory_clean_migration.py b/scripts/run_atomic_memory_clean_migration.py index fcb93a48f4..d293f393bf 100644 --- a/scripts/run_atomic_memory_clean_migration.py +++ b/scripts/run_atomic_memory_clean_migration.py @@ -57,7 +57,10 @@ def run_node(node: str, directory: Path) -> bool: "-vv", "-ra", "-o", - "faulthandler_timeout=120", + # The CPython watchdog walks other threads without the GIL and can + # itself segfault (python/cpython#116008). Keep native fault dumps; + # GNU timeout above enforces the per-process deadline independently. + "faulthandler_timeout=0", "--durations=20", "-o", f"cache_dir={directory / 'pytest-cache'}", diff --git a/tests/builtin/runtime/test_processing_composition.py b/tests/builtin/runtime/test_processing_composition.py index bfc558c6e5..514ebf5587 100644 --- a/tests/builtin/runtime/test_processing_composition.py +++ b/tests/builtin/runtime/test_processing_composition.py @@ -188,8 +188,8 @@ async def scenario() -> None: # Exercise the OceanBase role's real startup gate against a real SQLite # transaction. A failed gate must run before dialect-specific indexes. @asynccontextmanager - async def local_profile(_config, *, tables): - async with SQLiteProfile.open(database, tables=tables) as profile: + async def local_profile(_config, *, tables, _archive_recovery=False): + async with SQLiteProfile.open(database, tables=tables, _archive_recovery=_archive_recovery) as profile: yield profile monkeypatch.setattr(composition.OceanBaseProfile, "open", local_profile) From 9b457be2d21f3fca8d683451856239498d5efc32 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sat, 24 Oct 2026 17:58:19 +0800 Subject: [PATCH 91/94] fix(migrations): guard background startup and upgrade cleanup --- scripts/upgrade_process.py | 139 +++++++++++++++ scripts/upgrade_release_gate.py | 52 +++++- src/powercontext/server/cli.py | 3 + tests/e2e/test_background_metrics.py | 72 +++++--- .../e2e/test_background_startup_admission.py | 111 ++++++++++++ tests/release_upgrades/README.md | 10 ++ tests/test_background_metrics.py | 2 + tests/test_metrics_merge.py | 5 +- tests/test_processing_security.py | 3 +- tests/test_upgrade_release_gate.py | 161 ++++++++++++++++++ 10 files changed, 525 insertions(+), 33 deletions(-) create mode 100644 scripts/upgrade_process.py create mode 100644 tests/e2e/test_background_startup_admission.py diff --git a/scripts/upgrade_process.py b/scripts/upgrade_process.py new file mode 100644 index 0000000000..949f97b3e4 --- /dev/null +++ b/scripts/upgrade_process.py @@ -0,0 +1,139 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Own one Linux upgrade command and reap its descendants, including new sessions.""" + +import argparse +import ctypes +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + + +def own_descendants(expected_parent: int) -> None: + if sys.platform != "linux": + raise RuntimeError("Installed-release acceptance requires Linux") # noqa: TRY003 + # A separate, single-threaded supervisor cannot adopt unrelated gate/test + # children. setsid() does not escape Linux subreaper ancestry. + libc = ctypes.CDLL(None, use_errno=True) + if libc.prctl(36, 1, 0, 0, 0) != 0: # PR_SET_CHILD_SUBREAPER + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + if libc.prctl(1, signal.SIGTERM, 0, 0, 0) != 0: # PR_SET_PDEATHSIG + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + if os.getppid() != expected_parent: + raise RuntimeError("The gate exited before process ownership was established") # noqa: TRY003 + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + + +def children() -> list[int]: + return [int(value) for value in Path(f"/proc/self/task/{os.getpid()}/children").read_text().split()] + + +def reap_finished(exclude: int | None = None) -> set[int]: + reaped = set() + for pid in children(): + if pid != exclude and os.waitpid(pid, os.WNOHANG)[0]: + reaped.add(pid) + return reaped + + +def wait_command(process: subprocess.Popen[bytes], timeout: float) -> int: + deadline = time.monotonic() + timeout + while (result := process.poll()) is None: + # A live command may wait for a double-forked daemon to disappear. + # Reap already-exited adoptees without disturbing any live descendants. + reap_finished(exclude=process.pid) + if time.monotonic() >= deadline: + raise subprocess.TimeoutExpired(process.args, timeout) + time.sleep(0.02) + return result + + +def reap_descendants(shutdown_timeout: float) -> None: + deadline = time.monotonic() + shutdown_timeout + notified = {} + while True: + for pid in reap_finished(): + notified.pop(pid, None) + owned = children() + if not owned: + return + requested = signal.SIGTERM if time.monotonic() < deadline else signal.SIGKILL + for pid in owned: + if notified.get(pid) != requested: + # These are our direct children, never a global PID scan. No + # wait/reap happens between ownership inspection and signalling, + # so even an exited child's PID cannot be reused here. + os.kill(pid, requested) + notified[pid] = requested + # Killing a parent can adopt another detached generation. Keep owning + # and reaping it before returning, rather than leaving it to init. As + # with Popen.wait(), SIGKILL still waits for kernel-level exit/reaping; + # an uninterruptible kernel wait must not admit the next case early. + time.sleep(0.02) + + +def supervise(command: list[str], timeout: float, shutdown_timeout: float, expected_parent: int) -> int: + def interrupted(_signal, _frame): + raise KeyboardInterrupt + + signal.signal(signal.SIGTERM, interrupted) + own_descendants(expected_parent) + process = None + try: + process = subprocess.Popen(command) # noqa: S603 -- fixed gate command, no shell + try: + result = wait_command(process, timeout) + except subprocess.TimeoutExpired: + print(f"Upgrade command exceeded {timeout:g} seconds; closing owned processes", file=sys.stderr) + result = 124 + except KeyboardInterrupt: + result = 130 + finally: + # A repeated cancellation must not interrupt ownership cleanup. + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + if process is not None and process.poll() is None: + # Let pytest unwind fixtures and let the Server close SQL sessions + # and seekDB normally before forcing termination. + process.send_signal(signal.SIGINT) + try: + wait_command(process, shutdown_timeout) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + reap_descendants(shutdown_timeout) + return result if result >= 0 else 128 - result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--parent-pid", type=int, required=True) + parser.add_argument("--timeout", type=float, required=True) + parser.add_argument("--shutdown-timeout", type=float, default=30) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command or args.timeout <= 0 or args.shutdown_timeout <= 0: + parser.error("A command and positive time limits are required") + return supervise(command, args.timeout, args.shutdown_timeout, args.parent_pid) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/upgrade_release_gate.py b/scripts/upgrade_release_gate.py index de6be9c72e..7044273311 100644 --- a/scripts/upgrade_release_gate.py +++ b/scripts/upgrade_release_gate.py @@ -22,6 +22,7 @@ import os import re import shutil +import signal import subprocess import sys import tempfile @@ -233,6 +234,51 @@ def run(command: list[str], *, environment: dict[str, str], cwd: Path, log: Path require(result.returncode == 0, f"Command failed; inspect {log.name}") +def run_owned( + command: list[str], + *, + environment: dict[str, str], + cwd: Path, + log: Path, + timeout: float = 1800, + shutdown_timeout: float = 30, +) -> None: + require(sys.platform == "linux", "Installed-release acceptance requires Linux process ownership and cleanup") + supervised = [ + sys.executable, + str(Path(__file__).with_name("upgrade_process.py")), + "--parent-pid", + str(os.getpid()), + "--timeout", + str(timeout), + "--shutdown-timeout", + str(shutdown_timeout), + "--", + *command, + ] + # The supervisor enforces the deadline and reaps every owned descendant, + # including detached native engines, before workspace cleanup can run. + with ( + log.open("w", encoding="utf-8") as output, + subprocess.Popen( # noqa: S603 -- fixed gate supervisor and reviewed command, no shell + supervised, env=environment, cwd=cwd, stdout=output, stderr=subprocess.STDOUT, start_new_session=True + ) as process, + ): + try: + returncode = process.wait() + except BaseException: + # Repeated interrupts must not release the workspace before the + # owner has finished closing descendants. + previous = signal.signal(signal.SIGINT, signal.SIG_IGN) + try: + process.terminate() + process.wait() + finally: + signal.signal(signal.SIGINT, previous) + raise + require(returncode == 0, f"Command failed; inspect {log.name}") + + def installed(python: Path, environment: dict[str, str], cwd: Path) -> dict[str, str]: result = subprocess.run( # noqa: S603 -- isolated environment interpreter [ @@ -333,14 +379,14 @@ def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dic } for role, package in packages.items(): venv = directory / role - run( + run_owned( ["uv", "venv", "--python", sys.executable, str(venv)], environment=environment, cwd=directory, log=output / f"{role}-venv.log", ) python = venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python") - run( + run_owned( ["uv", "pip", "install", "--python", str(python), package], environment=environment, cwd=directory, @@ -363,7 +409,7 @@ def run_case(root: Path, wheel: Path, case: dict[str, Any], output: Path) -> dic "PC_UPGRADE_EVIDENCE": str(output / "backend.json"), }) report = output / "junit.xml" - run( + run_owned( [ sys.executable, "-m", diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index 036675259b..782ff5afa3 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -757,6 +757,9 @@ async def _run_background_async(settings: ServerSettings, tracing: Any) -> None: inference=settings.inference, external_skills=settings.external_skills, ) + from powercontext.builtin.persistence.migrations.server_readiness import prepare_server_storage + + await asyncio.to_thread(prepare_server_storage, settings) metrics = ServerMetrics(role="background") if settings.metrics.enabled else None stopped = asyncio.Event() loop = asyncio.get_running_loop() diff --git a/tests/e2e/test_background_metrics.py b/tests/e2e/test_background_metrics.py index 4399592917..d4f8201d22 100644 --- a/tests/e2e/test_background_metrics.py +++ b/tests/e2e/test_background_metrics.py @@ -19,20 +19,17 @@ import pytest +from powercontext.builtin.persistence.migrations.deployment import deployment_runner from powercontext.builtin.persistence.oceanbase import OceanBaseConfig -from powercontext.builtin.runtime.composition import open_builtin_runtime -from powercontext.builtin.runtime.config import BuiltinConfig, RuntimeConfig -from powercontext.server.authz import PrincipalRef -from powercontext.server.authz.composition import open_builtin_access_control +from powercontext.builtin.runtime.config import RuntimeConfig +from powercontext.server.settings import ServerSettings +from tests.e2e.atomic_memory_migration_backend import migration_profile -async def _initialize_background_database(config: BuiltinConfig) -> None: - principal = PrincipalRef(type="service", id="server-token", description="PowerContext static bearer") - async with ( - open_builtin_access_control(config.database, bootstrap_administrators=(principal,)), - open_builtin_runtime(config), - ): - pass +async def _database_objects(config: OceanBaseConfig) -> tuple[tuple[str, str], ...]: + async with migration_profile(config) as profile, profile.database.transaction() as connection: + rows = await connection.exec_driver_sql("SHOW FULL TABLES") + return tuple(sorted((row[0], row[1]) for row in rows)) @pytest.mark.parametrize("migration_backend", ["oceanbase"], indirect=True) @@ -47,15 +44,6 @@ def test_background_cli_serves_only_authorized_metrics_and_stops_cleanly(tmp_pat import httpx assert isinstance(migration_backend.config, OceanBaseConfig) - # Complete cold OceanBase schema/index creation before timing the listener lifecycle. - asyncio.run( - _initialize_background_database( - BuiltinConfig( - database=migration_backend.config, - runtime=RuntimeConfig(artifact_processing_role="background"), - ) - ) - ) with socket.socket() as candidate: candidate.bind(("127.0.0.1", 0)) port = candidate.getsockname()[1] @@ -68,19 +56,47 @@ def test_background_cli_serves_only_authorized_metrics_and_stops_cleanly(tmp_pat "POWERCONTEXT_SERVER_METRICS_BACKGROUND_PORT": str(port), "POWERCONTEXT_SERVER_METRICS_ENABLED": "true", }) + command = [ + sys.executable, + "-c", + "from powercontext.server.cli import app; app()", + "run", + "--role", + "background", + "--no-env-file", + ] + before = asyncio.run(_database_objects(migration_backend.config)) + assert before == () + rejected = subprocess.run(command, env=environment, capture_output=True, text=True, timeout=60, check=False) + assert rejected.returncode != 0, rejected.stdout + rejected.stderr + assert "migration_required" in rejected.stdout + rejected.stderr + assert asyncio.run(_database_objects(migration_backend.config)) == before + + # Explicit initialization must finish before either split Server role can serve. + runner = deployment_runner( + ServerSettings( + database=migration_backend.config, + runtime=RuntimeConfig(artifact_processing_role="background"), + ), + evidence_dir=tmp_path / "migration-evidence", + lock_coordination="single-host", + ) + plan = runner.plan(backup_policy="skip") + initialized = runner.apply( + plan_id=plan.plan_id, + accepted=True, + maintenance_confirmed=True, + backup_policy="skip", + accept_no_backup=True, + ) + assert initialized.state == "ready" + assert initialized.readiness_scope == "server" reports = os.environ.get("MIGRATION_REPORTS") log_path = Path(reports) / "background-metrics" / "background.log" if reports else tmp_path / "background.log" log_path.parent.mkdir(parents=True, exist_ok=True) with log_path.open("w") as output: process = subprocess.Popen( - [ - os.path.join(os.path.dirname(sys.executable), "powercontext"), - "server", - "run", - "--role", - "background", - "--no-env-file", - ], + command, env=environment, stdout=output, stderr=subprocess.STDOUT, diff --git a/tests/e2e/test_background_startup_admission.py b/tests/e2e/test_background_startup_admission.py new file mode 100644 index 0000000000..fdbd6ff1d0 --- /dev/null +++ b/tests/e2e/test_background_startup_admission.py @@ -0,0 +1,111 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Local admission regressions; native split-role acceptance lives in test_background_metrics.""" + +import asyncio + +import httpx +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from typer.testing import CliRunner + +from powercontext.builtin.persistence.migrations.connections import MaintenanceConnections +from powercontext.builtin.persistence.migrations.deployment import deployment_runner +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.server import cli +from powercontext.server.factory import create_server_app +from powercontext.server.settings import McpConfig, ServerSettings +from powercontext.server.tracing import ServerTracing + + +@pytest.mark.parametrize("access_mode", ["disabled", "enforced"]) +def test_background_cli_rejects_uninitialized_remote_storage_before_business_writes(tmp_path, monkeypatch, access_mode): + path = tmp_path / "empty.db" + path.touch() + before = path.read_bytes() + + def inspect_local_database(_self, operation, *, writable=False): + # Exercise the real remote admission decision with an actual empty SQL + # inventory. This substitutes only the network connection, not native OB. + assert writable is False + engine = create_engine(f"sqlite:///{path}") + try: + with engine.connect() as connection: + return operation(connection, None, lambda: None) + finally: + engine.dispose() + + def unexpected_business_profile(*args, **kwargs): + pytest.fail("uninitialized remote storage reached a business profile") + + monkeypatch.setattr(MaintenanceConnections, "run", inspect_local_database) + monkeypatch.setattr(cli, "open_builtin_access_control", unexpected_business_profile) + monkeypatch.setattr(cli, "open_builtin_runtime", unexpected_business_profile) + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_KIND", "oceanbase") + monkeypatch.setenv( + "POWERCONTEXT_SERVER_DATABASE_URL", "mysql+aoceanbase://test@localhost:2881/test?charset=utf8mb4" + ) + monkeypatch.setenv("POWERCONTEXT_SERVER_ACCESS_MODE", access_mode) + if access_mode == "enforced": + monkeypatch.setenv("POWERCONTEXT_SERVER_AUTH_TOKEN", "isolated-background-test-token") + result = CliRunner().invoke(cli.app, ["run", "--role", "background", "--no-env-file"]) + + assert isinstance(result.exception, MigrationError), result.output + assert result.exception.code == "migration_required" + assert path.read_bytes() == before + + +def test_managed_storage_remains_usable_by_background_runner_and_api(tmp_path, monkeypatch): + settings = ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'managed.db'}"), + mcp=McpConfig(enabled=False), + ) + runner = deployment_runner(settings) + plan = runner.plan() + assert runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True).state == "ready" + + with TestClient(create_server_app(settings=settings, scheduler_path=tmp_path / "scheduler.db")) as client: + created = client.post( + "/v1/scopes", + json={ + "title": "Preserved scope", + "summary": "Created before background startup", + "idempotency_key": "scope", + }, + ) + assert created.status_code == 201, created.text + scope_id = created.json()["scope_id"] + + async def serve(app, configured, stopped): + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://testserver") as client: + assert (await client.get("/health/live")).status_code == 200 + metrics = await client.get("/metrics") + assert metrics.status_code == 200 + assert 'powercontext_server_readiness_state{state="ready"} 1.0' in metrics.text + stopped.set() + + monkeypatch.setattr(cli, "serve_background_metrics", serve) + # SQLite supports the all role. Invoke the runner with a real local runtime + # to check its admitted lifecycle; the native OB test covers split-role CLI. + asyncio.run(cli._run_background_async(settings, ServerTracing.context_only(role="background"))) + + with TestClient(create_server_app(settings=settings, scheduler_path=tmp_path / "scheduler.db")) as client: + response = client.get(f"/v1/scopes/{scope_id}") + assert response.status_code == 200, response.text + assert response.json()["title"] == "Preserved scope" + assert client.get("/health/ready").status_code == 200 + assert runner.verify().state == "ready" diff --git a/tests/release_upgrades/README.md b/tests/release_upgrades/README.md index fc35fd6548..50a44add81 100644 --- a/tests/release_upgrades/README.md +++ b/tests/release_upgrades/README.md @@ -140,3 +140,13 @@ Acceptance uses a private copy of those files. Only a passing run creates `valid in its evidence directory. CI uploads that directory; local `make publish` calls the runner with `--release --publish` and uploads its exact returned paths, never a fresh glob of `dist/`. Failed acceptance produces no publication set. Managed persistence hashes normalize CRLF/LF checkout line endings; artifact SHA-256s remain byte-exact. + + +Installed-release execution requires Linux. Each gate command runs under a separate +Linux child subreaper, which owns detached Server and embedded-engine descendants. +On timeout it first interrupts the command so fixtures can close the Server and +seekDB normally, then terminates and reaps any remaining owned descendants before +collecting native logs, deleting the workspace, or starting the next case. Normal +success and failure also clean up leftover descendants. If the gate is terminated, +the owner receives a Linux parent-death signal and completes the same cleanup. +Policy and documentation checks remain available on other platforms. diff --git a/tests/test_background_metrics.py b/tests/test_background_metrics.py index 133677d847..ec3cc4560b 100644 --- a/tests/test_background_metrics.py +++ b/tests/test_background_metrics.py @@ -25,6 +25,7 @@ from pydantic import SecretStr, ValidationError from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.runtime.config import RuntimeConfig from powercontext.server.authz import AccessAction from powercontext.server.authz.errors import AccessDeniedError, AccessUnavailableError @@ -166,6 +167,7 @@ def test_background_runner_closes_runtime_and_removes_signal_handlers(monkeypatc async def scenario(): _, settings = _app(enabled=outcome != "disabled") + settings = settings.model_copy(update={"database": SQLiteConfig()}) events = [] signals = [] loop = asyncio.get_running_loop() diff --git a/tests/test_metrics_merge.py b/tests/test_metrics_merge.py index 66a5a37c69..2ae8ce7122 100644 --- a/tests/test_metrics_merge.py +++ b/tests/test_metrics_merge.py @@ -27,6 +27,7 @@ from prometheus_client.parser import text_string_to_metric_families from pydantic import SecretStr +from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.runtime import BuiltinRuntime from powercontext.builtin.runtime.observations import DurationObservations from powercontext.builtin.runtime.processing_contracts import ArtifactProcessingModelUsageDelta @@ -185,7 +186,9 @@ async def serve(app, settings, stopped): monkeypatch.setattr("powercontext.server.cli.open_builtin_runtime", open_runtime) monkeypatch.setattr("powercontext.server.cli.serve_background_metrics", serve) - await _run_background_async(ServerSettings(), ServerTracing.context_only(role="background")) + await _run_background_async( + ServerSettings(database=SQLiteConfig()), ServerTracing.context_only(role="background") + ) assert len(rendered) == 2 for samples in rendered: assert _value(samples, "service_role_info", role="background") == 1 diff --git a/tests/test_processing_security.py b/tests/test_processing_security.py index 18ffeb1a22..8033a80bf2 100644 --- a/tests/test_processing_security.py +++ b/tests/test_processing_security.py @@ -116,9 +116,10 @@ async def scenario(): def test_background_cli_preserves_deployment_identity_when_building_the_worker_spec(monkeypatch): async def scenario(): settings = ServerSettings( + database=SQLiteConfig(), access=AccessControlConfig( mode="enforced", background_principal_id="background-worker", deployment_id="test" - ) + ), ) observed = {} stop_callbacks = [] diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py index e6d28dd3c2..7064a19ddc 100644 --- a/tests/test_upgrade_release_gate.py +++ b/tests/test_upgrade_release_gate.py @@ -16,6 +16,11 @@ import json import os +import shutil +import signal +import subprocess +import sys +import time import zipfile from pathlib import Path from xml.etree import ElementTree @@ -539,3 +544,159 @@ def test_artifact_hashes_do_not_normalize_bytes(tmp_path): before = gate.sha256(artifact) artifact.write_bytes(b"payload\r\n") assert gate.sha256(artifact) != before + + +_PROCESS_FIXTURE = """ +import os, signal, subprocess, sys, time +from pathlib import Path + +role, mode, control, evidence = sys.argv[1:] +control, evidence = Path(control), Path(evidence) +child = None +if mode == "forced-timeout": + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) +else: + def stop(_signal, _frame): + raise SystemExit(0) + signal.signal(signal.SIGTERM, stop) +try: + ready = evidence / (role + ".ready") + ready.write_text(str(os.getpid())) + ready.replace(evidence / (role + ".pid")) + if role != "engine": + next_role = "server" if role == "root" else "engine" + child = subprocess.Popen( + [sys.executable, __file__, next_role, mode, str(control), str(evidence)], + start_new_session=True, + ) + if role == "root" and mode in {"success", "failure"}: + while not (evidence / "engine.pid").exists(): + time.sleep(0.02) + # Deliberately leave both detached generations for the command owner. + os._exit(0 if mode == "success" else 7) + while control.exists(): + time.sleep(0.02) +finally: + if child is not None: + child.terminate() + try: + child.wait(timeout=2) + except subprocess.TimeoutExpired: + child.kill() + child.wait() + (evidence / (role + ".closed")).write_text("closed") +""" + + +@pytest.mark.skipif(sys.platform != "linux", reason="The upgrade supervisor uses Linux subreaper ownership") +@pytest.mark.parametrize("mode", ["success", "failure", "timeout", "forced-timeout"]) +def test_upgrade_command_reaps_detached_descendants_before_workspace_cleanup(tmp_path, mode): + fixture = tmp_path / "process_fixture.py" + fixture.write_text(_PROCESS_FIXTURE) + control = tmp_path / "keep-running" + control.touch() + report = tmp_path / "evidence" + report.mkdir() + # This child belongs to the test, not the isolated command supervisor. + sentinel = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + try: + with gate.case_workspace("sqlite", report) as work: + arguments = [sys.executable, str(fixture), "root", mode, str(control), str(report)] + options = { + "environment": dict(os.environ), + "cwd": work, + "log": report / "command.log", + "timeout": 5 if mode.endswith("timeout") else 20, + "shutdown_timeout": 0.5, + } + if mode == "success": + gate.run_owned(arguments, **options) + else: + with pytest.raises(gate.GateError, match="Command failed"): + gate.run_owned(arguments, **options) + assert sentinel.poll() is None + for role in ("root", "server", "engine"): + pid = int((report / f"{role}.pid").read_text()) + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + if mode != "forced-timeout": + assert (report / "server.closed").is_file() + assert (report / "engine.closed").is_file() + if mode.endswith("timeout"): + assert "exceeded 5 seconds" in (report / "command.log").read_text() + assert not work.exists() + finally: + # Also let controlled fixture processes exit if the regression fails. + # Never signal a stale PID obtained from a fixture file. + control.unlink(missing_ok=True) + sentinel.terminate() + sentinel.wait(timeout=5) + + +@pytest.mark.skipif(sys.platform != "linux", reason="The upgrade supervisor uses Linux subreaper ownership") +@pytest.mark.parametrize("cancellation", ["parent-term", "repeat-interrupt"]) +def test_upgrade_command_owner_finishes_cleanup_after_gate_cancellation(tmp_path, cancellation): + fixture = tmp_path / "process_fixture.py" + fixture.write_text(_PROCESS_FIXTURE) + control = tmp_path / "keep-running" + control.touch() + report = tmp_path / "evidence" + report.mkdir() + owner_script = tmp_path / "owner.py" + owner_script.write_text( + "import os,sys\nfrom pathlib import Path\n" + f"sys.path[:0] = {[str(Path(gate.__file__).parent), str(Path(matrix.__file__).parent)]!r}\n" + "import upgrade_release_gate as gate\n" + f"report = Path({str(report)!r})\n" + "with gate.case_workspace('sqlite', report) as work:\n" + " (report / 'workspace').write_text(str(work))\n" + f" gate.run_owned({[sys.executable, str(fixture), 'root', 'forced-timeout', str(control), str(report)]!r}, " + "environment=dict(os.environ), cwd=work, log=report / 'command.log', timeout=60, shutdown_timeout=0.5)\n" + ) + sentinel = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + with (report / "owner.log").open("w") as log: + owner = subprocess.Popen([sys.executable, str(owner_script)], stdout=log, stderr=subprocess.STDOUT) + pids = [] + try: + deadline = time.monotonic() + 15 + while not (report / "engine.pid").exists(): + assert owner.poll() is None, (report / "owner.log").read_text() + assert time.monotonic() < deadline, "Controlled processes did not become ready" + time.sleep(0.02) + pids = [int((report / f"{role}.pid").read_text()) for role in ("root", "server", "engine")] + work = Path((report / "workspace").read_text()) + owner.send_signal(signal.SIGTERM if cancellation == "parent-term" else signal.SIGINT) + if cancellation == "repeat-interrupt": + time.sleep(0.1) + assert owner.poll() is None, "Cancellation did not wait for owned processes" + owner.send_signal(signal.SIGINT) + assert owner.wait(timeout=10) != 0 + if cancellation == "parent-term": + # The dead gate cannot await its owner; PDEATHSIG initiates cleanup. + deadline = time.monotonic() + 10 + while any(_process_exists(pid) for pid in pids): + assert time.monotonic() < deadline, "Detached descendants survived gate termination" + time.sleep(0.02) + else: + # The second interrupt must not delete the workspace early. + assert not any(_process_exists(pid) for pid in pids) + assert not work.exists() + assert sentinel.poll() is None + finally: + control.unlink(missing_ok=True) + if owner.poll() is None: + owner.terminate() + owner.wait(timeout=5) + sentinel.terminate() + sentinel.wait(timeout=5) + if cancellation == "parent-term" and pids and not any(_process_exists(pid) for pid in pids): + shutil.rmtree(Path((report / "workspace").read_text()), ignore_errors=True) + + +def _process_exists(pid): + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + return True From d3490d60902c7450288e523296e3e2db6b08514d Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 25 Oct 2026 17:29:14 +0800 Subject: [PATCH 92/94] test(service): drain startup-conflict probe connections --- .../native/test_personal_service_lifecycle.py | 31 +++++++++---- tests/test_native_service_diagnostics.py | 44 ++++++++++++++++++- 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/tests/native/test_personal_service_lifecycle.py b/tests/native/test_personal_service_lifecycle.py index 60fcb622ec..87bae12cf0 100644 --- a/tests/native/test_personal_service_lifecycle.py +++ b/tests/native/test_personal_service_lifecycle.py @@ -27,9 +27,11 @@ import uuid import xml.etree.ElementTree as ET from collections.abc import Callable, Iterator -from contextlib import closing, contextmanager, suppress +from contextlib import ExitStack, closing, contextmanager, suppress from importlib.metadata import version from pathlib import Path +from socketserver import BaseRequestHandler, TCPServer +from threading import Thread from typing import Any import httpx @@ -446,19 +448,14 @@ def test_native_public_migration_switches_an_installed_130_service( def _native_upgrade_fault( adapter: NativeServiceAdapter, port: int, failure: str | None, monkeypatch: pytest.MonkeyPatch ) -> Iterator[None]: - with closing(socket.socket()) as listener, monkeypatch.context() as fault: + with ExitStack() as resources, monkeypatch.context() as fault: if failure == "startup": native_start = adapter.start def occupy_then_start(*, reload_definition: bool) -> None: # Bind after migration and stopped checks, then let the actual # native manager attempt to start the registered executable. - if sys.platform != "win32": - # Reuse the stopped server's port even while its accepted - # connections remain in TIME_WAIT. - listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - listener.bind(("127.0.0.1", port)) - listener.listen(16) + resources.enter_context(_occupy_service_port(port)) native_start(reload_definition=reload_definition) fault.setattr(adapter, "start", occupy_then_start) @@ -474,6 +471,24 @@ def fail_definition_replace(source: str | os.PathLike[str], destination: str | o yield +@contextmanager +def _occupy_service_port(port: int) -> Iterator[None]: + # Accept and close probe connections so repeated liveness checks cannot fill + # the backlog and turn a live port conflict into a connection timeout. + with TCPServer(("127.0.0.1", port), BaseRequestHandler, bind_and_activate=False) as listener: + # Reuse the stopped server's port while its connections remain in TIME_WAIT. + listener.allow_reuse_address = sys.platform != "win32" + listener.server_bind() + listener.server_activate() + thread = Thread(target=listener.serve_forever, daemon=True) + thread.start() + try: + yield + finally: + listener.shutdown() + thread.join() + + def _native_migration_command(cli, environment: Path, action: str, *options: str) -> dict[str, Any]: result = CliRunner().invoke( cli, ["server", "db-migrate", action, "--env-file", str(environment), "--manage-service", *options] diff --git a/tests/test_native_service_diagnostics.py b/tests/test_native_service_diagnostics.py index 09c47995a8..871b491341 100644 --- a/tests/test_native_service_diagnostics.py +++ b/tests/test_native_service_diagnostics.py @@ -14,15 +14,20 @@ from __future__ import annotations +import errno import json +import socket import subprocess +import sys +from contextlib import nullcontext from pathlib import Path from unittest.mock import Mock import pytest from powercontext.service.adapters.launchd import LaunchdUserAdapter -from powercontext.service.model import ServiceError, SupportState +from powercontext.service.model import ProbeState, ServiceError, SupportState +from powercontext.service.probe import probe_server from tests.native import test_personal_service_lifecycle as lifecycle @@ -64,3 +69,40 @@ def cleanup(adapter: LaunchdUserAdapter) -> None: assert snapshot == {"retry_token_present": True, "retry_state": '{"attempts":[1]}'} assert "state = running\npid = 123" in (logs / "launchd-before-cleanup.log").read_text(encoding="utf-8") assert (logs / "launchd-process-sample.log").read_text(encoding="utf-8") == "Service process stack sample" + + +@pytest.mark.parametrize("startup_raises", [False, True], ids=["normal-exit", "failed-start"]) +def test_native_startup_fault_drains_probes_and_releases_port( + monkeypatch: pytest.MonkeyPatch, startup_raises: bool +) -> None: + port = lifecycle._unused_loopback_port() + address = ("127.0.0.1", port) + adapter = Mock() + + def start(*, reload_definition: bool) -> None: + # The native manager boundary is modeled; port ownership and probes are real. + with socket.socket() as contender: + with pytest.raises(OSError) as conflict: + contender.bind(address) + assert conflict.value.errno == errno.EADDRINUSE + if startup_raises: + raise ServiceError("fixture start failed") # noqa: TRY003 + + adapter.start.side_effect = start + expected = pytest.raises(ServiceError, match="fixture start failed") if startup_raises else nullcontext() + with expected, lifecycle._native_upgrade_fault(adapter, port, "startup", monkeypatch): + adapter.start(reload_definition=False) + # A controller and retried native launches can probe more often than + # the listener backlog; every probe must still see a port conflict. + for _ in range(40): + assert probe_server(f"http://127.0.0.1:{port}").state is ProbeState.CONFLICT + with socket.create_connection(address, timeout=2): + pass + + # Exiting the fault, including a failed adapter.start(), releases the endpoint + # so the subsequent native recovery can bind it normally. + with socket.socket() as recovered: + if sys.platform != "win32": + recovered.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + recovered.bind(address) + recovered.listen() From 1df6c186ab6dae2aa643b3f0346c4a8ff4db3eb3 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 25 Oct 2026 21:03:35 +0800 Subject: [PATCH 93/94] ci(e2e): budget OceanBase acceptance phases --- .github/workflows/e2e-harness.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 06041cfc21..68a4c22760 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -386,7 +386,8 @@ jobs: acceptance: needs: validate runs-on: ubuntu-latest - timeout-minutes: 40 + # OceanBase regressions can take 32 minutes before the separate replay scenarios. + timeout-minutes: ${{ matrix.database == 'oceanbase' && 60 || 40 }} strategy: fail-fast: false matrix: @@ -405,6 +406,7 @@ jobs: version: ${{ env.UV_VERSION }} - name: Verify OceanBase Atomic Memory and HTTP regressions + timeout-minutes: 40 if: matrix.database == 'oceanbase' env: COMPOSE_PROJECT_NAME: powercontext-e2e-oceanbase @@ -489,6 +491,7 @@ jobs: PY - name: Run deterministic scenarios + timeout-minutes: ${{ matrix.database == 'oceanbase' && 15 || 40 }} id: acceptance_scenarios env: POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }} From b8ee3658c8b728a829fa19974dd832a32d1d80f4 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 25 Oct 2026 21:34:03 +0800 Subject: [PATCH 94/94] ci(upgrades): size recovery storage and retain diagnostics --- .github/workflows/e2e-harness.yml | 40 +++++++++++++++- scripts/upgrade_release_gate.py | 62 ++++++++++++++++-------- tests/test_upgrade_release_gate.py | 77 ++++++++++++++++++++++++++---- 3 files changed, 148 insertions(+), 31 deletions(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 68a4c22760..79b83bc7ac 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -98,10 +98,37 @@ jobs: if docker exec recall-effort-oceanbase \ obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ -Dpowercontext_test -e 'SELECT 1' >/dev/null 2>&1; then + # The slim image's default 1 GB maximum is too small for the + # indexes created across the full Server recovery scenarios. + docker exec recall-effort-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@sys \ + -e "ALTER SYSTEM SET datafile_maxsize='8G'; ALTER SYSTEM SET datafile_next='512M'; ALTER SYSTEM SET datafile_size='4G';" + datafile_ready=false + for config_attempt in $(seq 1 30); do + docker exec recall-effort-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@sys --batch --skip-column-names \ + -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next');" \ + > "$RUNNER_TEMP/recall-effort-datafile.tsv" + if awk ' + $5 == "datafile_size" && $7 == "4G" { size = 1 } + $5 == "datafile_maxsize" && $7 == "8G" { maximum = 1 } + $5 == "datafile_next" && $7 == "512M" { increment = 1 } + END { exit !(size && maximum && increment) } + ' "$RUNNER_TEMP/recall-effort-datafile.tsv"; then + datafile_ready=true + break + fi + sleep 2 + done + cat "$RUNNER_TEMP/recall-effort-datafile.tsv" + if [ "$datafile_ready" != true ]; then + echo 'OceanBase datafile configuration did not become effective within one minute.' >&2 + exit 1 + fi docker exec recall-effort-oceanbase \ obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ -Dpowercontext_test \ - -e "SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode';" + -e "SET GLOBAL recyclebin=OFF; SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode'; SHOW GLOBAL VARIABLES LIKE 'recyclebin';" exit 0 fi if [ "$(docker inspect --format '{{.State.Running}}' recall-effort-oceanbase)" != true ]; then @@ -221,6 +248,17 @@ jobs: run: | mkdir -p .powercontext/recall-effort-ci docker logs recall-effort-oceanbase > .powercontext/recall-effort-ci/oceanbase.log 2>&1 || true + for logfile in observer.log observer.log.wf; do + docker exec recall-effort-oceanbase tail -n 20000 "/root/demo/log/$logfile" \ + > ".powercontext/recall-effort-ci/$logfile" 2>&1 || true + done + df -h / > .powercontext/recall-effort-ci/runner-disk.txt 2>&1 || true + docker exec recall-effort-oceanbase df -h /root/demo \ + > .powercontext/recall-effort-ci/container-disk.txt 2>&1 || true + docker exec recall-effort-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@sys \ + -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next'); SELECT data_disk_allocated, data_disk_capacity, data_disk_in_use FROM oceanbase.GV\$OB_SERVERS;" \ + > .powercontext/recall-effort-ci/database-disk.txt 2>&1 || true - name: Save validation reports if: always() diff --git a/scripts/upgrade_release_gate.py b/scripts/upgrade_release_gate.py index 7044273311..4f0821d7e0 100644 --- a/scripts/upgrade_release_gate.py +++ b/scripts/upgrade_release_gate.py @@ -312,27 +312,49 @@ def _preserve_seekdb_logs(directory: Path, output: Path) -> None: remaining = 16 * 1024 * 1024 files = [] + # Preserve the existing revision journal before native logs can use the budget. + paths = [ + path + for name in ("database", "fresh") + for path in sorted((directory / f"{name}.pc-migration-state").glob("*/maintenance.json")) + ] + paths += [path for name in ("database", "fresh") for path in sorted((directory / name).rglob("*.log*"))] with zipfile.ZipFile(output / "seekdb-native-logs.zip", "w", compression=zipfile.ZIP_DEFLATED) as archive: - for name in ("database", "fresh"): - for path in sorted((directory / name).rglob("*.log*")): - if remaining == 0: - break - if path.is_symlink() or not path.is_file(): - continue - with path.open("rb") as source: - size = source.seek(0, os.SEEK_END) - length = min(size, 4 * 1024 * 1024, remaining) - source.seek(size - length) - content = source.read(length) - relative = path.relative_to(directory).as_posix() - archive.writestr(relative, content) - remaining -= len(content) - files.append({ - "path": relative, - "size_bytes": size, - "captured_bytes": len(content), - "truncated": len(content) < size, - }) + for path in paths: + if remaining == 0: + break + if path.is_symlink() or not path.is_file(): + continue + relative = path.relative_to(directory).as_posix() + segments = [] + captured = 0 + with path.open("rb") as source: + size = source.seek(0, os.SEEK_END) + length = min(size, 4 * 1024 * 1024, remaining) + if length < size: + head = length // 2 + ranges = [ + (f"head/{relative}", 0, head), + (f"tail/{relative}", size - (length - head), length - head), + ] + else: + ranges = [(relative, 0, length)] + for entry, offset, count in ranges: + if count == 0 and size: + continue + source.seek(offset) + content = source.read(count) + archive.writestr(entry, content) + remaining -= len(content) + captured += len(content) + segments.append({"path": entry, "offset_bytes": offset, "captured_bytes": len(content)}) + files.append({ + "path": relative, + "size_bytes": size, + "captured_bytes": captured, + "truncated": captured < size, + "segments": segments, + }) usage = shutil.disk_usage(directory) manifest = { "files": files, diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py index 7064a19ddc..14a21e5730 100644 --- a/tests/test_upgrade_release_gate.py +++ b/tests/test_upgrade_release_gate.py @@ -474,6 +474,9 @@ def execute(_root, _wheel, case, _output): def test_failed_seekdb_acceptance_keeps_native_logs_before_workspace_cleanup(tmp_path): output = tmp_path / "evidence" output.mkdir() + journal = b'{"state":"active","last_revision":"s1300","pending_revision":"s1400"}' + maintenance = "database.pc-migration-state/disposable-database/maintenance.json" + fresh_maintenance = "fresh.pc-migration-state/disposable-fresh/maintenance.json" with pytest.raises(gate.GateError, match="fixture migration failed"), gate.case_workspace("seekdb", output) as work: native = work / "database/log" @@ -483,34 +486,88 @@ def test_failed_seekdb_acceptance_keeps_native_logs_before_workspace_cleanup(tmp fresh = work / "fresh/log" fresh.mkdir(parents=True) (fresh / "observer.log.wf").write_bytes(b"fresh native diagnostic\n") + for relative in (maintenance, fresh_maintenance): + path = work / relative + path.parent.mkdir(parents=True) + path.write_bytes(journal) + (path.parent / "backup.data").write_bytes(b"backup payload must not be collected") raise gate.GateError("fixture migration failed") # noqa: TRY003 -- synthetic acceptance failure assert not work.exists() with zipfile.ZipFile(output / "seekdb-native-logs.zip") as archive: - assert set(archive.namelist()) == {"database/log/observer.log", "fresh/log/observer.log.wf"} + assert set(archive.namelist()) == { + "database/log/observer.log", + "fresh/log/observer.log.wf", + maintenance, + fresh_maintenance, + } assert archive.read("database/log/observer.log") == b"native recovery diagnostic\n" assert archive.read("fresh/log/observer.log.wf") == b"fresh native diagnostic\n" + assert archive.read(maintenance) == journal + assert archive.read(fresh_maintenance) == journal manifest = json.loads((output / "seekdb-native-logs.json").read_text()) - assert len(manifest["files"]) == 2 + assert len(manifest["files"]) == 4 assert not any(item["truncated"] for item in manifest["files"]) -def test_seekdb_diagnostics_keep_a_bounded_tail_of_large_native_logs(tmp_path): +def test_seekdb_diagnostics_failure_preserves_acceptance_failure(tmp_path, capsys): + with ( + pytest.raises(gate.GateError, match="fixture migration failed"), + gate.case_workspace("seekdb", tmp_path / "missing-output") as work, + ): + raise gate.GateError("fixture migration failed") # noqa: TRY003 -- synthetic acceptance failure + + assert not work.exists() + assert "seekDB native log collection failed: FileNotFoundError" in capsys.readouterr().err + + +def test_seekdb_diagnostics_keep_bounded_head_and_tail_of_large_native_logs(tmp_path): output = tmp_path / "evidence" output.mkdir() - size = 5 * 1024 * 1024 + content = b"initial DDL diagnostic\n" + b"x" * (5 * 1024 * 1024) + b"final wait diagnostic\n" with gate.case_workspace("seekdb", output) as work: native = work / "database/log" native.mkdir(parents=True) - (native / "observer.log").write_bytes(b"x" * size + b"final diagnostic") + (native / "observer.log").write_bytes(content) with zipfile.ZipFile(output / "seekdb-native-logs.zip") as archive: - tail = archive.read("database/log/observer.log") - assert len(tail) <= 4 * 1024 * 1024 - assert tail.endswith(b"final diagnostic") + head = archive.read("head/database/log/observer.log") + tail = archive.read("tail/database/log/observer.log") + assert len(head) + len(tail) <= 4 * 1024 * 1024 + assert head.startswith(b"initial DDL diagnostic\n") + assert tail.endswith(b"final wait diagnostic\n") + manifest = json.loads((output / "seekdb-native-logs.json").read_text()) + record = manifest["files"][0] + assert record["truncated"] is True + assert record["size_bytes"] == len(content) + assert record["captured_bytes"] == len(head) + len(tail) + for segment in record["segments"]: + start = segment["offset_bytes"] + assert archive.read(segment["path"]) == content[start : start + segment["captured_bytes"]] + + +def test_seekdb_diagnostics_share_one_budget_with_maintenance_evidence(tmp_path): + output = tmp_path / "evidence" + output.mkdir() + maintenance = "database.pc-migration-state/disposable-database/maintenance.json" + journal = b'{"state":"active","pending_revision":"s1400"}' + with gate.case_workspace("seekdb", output) as work: + path = work / maintenance + path.parent.mkdir(parents=True) + path.write_bytes(journal) + for store in ("database", "fresh"): + native = work / store / "log" + native.mkdir(parents=True) + for number in range(3): + (native / f"observer.log.{number}").write_bytes(b"x" * (5 * 1024 * 1024)) + manifest = json.loads((output / "seekdb-native-logs.json").read_text()) - assert manifest["files"][0]["truncated"] is True - assert manifest["files"][0]["size_bytes"] == size + len(b"final diagnostic") + with zipfile.ZipFile(output / "seekdb-native-logs.zip") as archive: + assert archive.read(maintenance) == journal + captured = sum(entry.file_size for entry in archive.infolist()) + assert captured <= 16 * 1024 * 1024 + assert captured == sum(record["captured_bytes"] for record in manifest["files"]) + assert manifest["budget_exhausted"] is True @pytest.mark.parametrize(