From c1ca71bfc8051afc0dfddb10a950bc560cf8b4a0 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Sun, 23 Aug 2026 21:34:14 +0800 Subject: [PATCH 01/32] fix(transport): enforce loopback-only plaintext HTTP policy for Server and Client The ready-to-run Server and the primary Python Client did not enforce the network safety policy already applied by the Agent integrations, allowing two unsafe transports: - ServerSettings accepted a non-loopback bind (e.g. 0.0.0.0) while bearer authentication was disabled, silently exposing an unauthenticated Server. - ClientSettings / PowerContextClient accepted a non-loopback http:// URL and would send a bearer token over plaintext. Introduce a single shared transport-policy module (powercontext.transport) and consume it everywhere: - ClientSettings rejects non-loopback plaintext http:// Server URLs. - PowerContextClient refuses to send a bearer token over an unencrypted non-loopback connection. - ServerSettings rejects an unauthenticated non-loopback bind unless the operator opts in via POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK (for TLS-terminated / controlled-network deployments). - The `server run` CLI applies the same guard, since model_copy bypasses the settings validator on --host overrides. Add a cross-surface contract test suite and update the EN/ZH configuration reference. Closes #1319 --- docs/en/docs/reference/configuration.md | 11 ++- docs/zh/docs/reference/configuration.md | 9 +- src/powercontext/client/client.py | 3 + src/powercontext/client/settings.py | 4 + src/powercontext/server/cli.py | 17 +++- src/powercontext/server/settings.py | 34 +++++++ src/powercontext/transport.py | 58 ++++++++++++ tests/test_cli.py | 19 ++++ tests/test_server.py | 1 + tests/test_transport.py | 115 ++++++++++++++++++++++++ 10 files changed, 267 insertions(+), 4 deletions(-) create mode 100644 src/powercontext/transport.py create mode 100644 tests/test_transport.py diff --git a/docs/en/docs/reference/configuration.md b/docs/en/docs/reference/configuration.md index 0acc30e8b2..54199756e3 100644 --- a/docs/en/docs/reference/configuration.md +++ b/docs/en/docs/reference/configuration.md @@ -35,6 +35,7 @@ Server settings use the `POWERCONTEXT_SERVER_` prefix. | `POWERCONTEXT_SERVER_MCP_PATH` | `/mcp` | MCP path | | `POWERCONTEXT_SERVER_AUTH_ENABLED` | `false` | Require one static bearer token for HTTP and MCP | | `POWERCONTEXT_SERVER_AUTH_TOKEN` | unset | Static bearer token; required when authentication is enabled | +| `POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK` | `false` | Opt in to a non-loopback bind while authentication is disabled | | `POWERCONTEXT_SERVER_LOGGING_LEVEL` | `INFO` | Operational log level | | `POWERCONTEXT_SERVER_LOGGING_FORMAT` | `console` | `console` or structured `json` output | | `POWERCONTEXT_SERVER_LOGGING_ACCESS` | `true` | Log external HTTP and logical MCP request completion | @@ -53,8 +54,14 @@ Server settings use the `POWERCONTEXT_SERVER_` prefix. | `POWERCONTEXT_SERVER_EXTERNAL_SKILLS` | unset | JSON object containing the host identity and explicit Codex Skill roots | Static bearer authentication is disabled by default. When enabled, API and MCP requests must include -`Authorization: Bearer `; the liveness and readiness endpoints remain public. Plain HTTP should remain on a -loopback address. Use TLS before exposing an authenticated Server over a network. +`Authorization: Bearer `; the liveness and readiness endpoints remain public. Plain HTTP is trusted only on a +loopback address (`127.0.0.1`, `localhost`, `::1`). The Server refuses to start when it binds to a non-loopback address +while authentication is disabled; either enable authentication, keep the bind on loopback, or, when TLS is terminated +upstream or the network is otherwise controlled, set `POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true` to +opt in explicitly. Use TLS before exposing an authenticated Server over a network. + +The Python Client and CLI apply the matching rule for outbound requests: an unencrypted `http://` Server URL is accepted +only for loopback hosts, and the Client refuses to send a bearer token over an unencrypted non-loopback connection. Example with a controlled SQLite path and scheduled extraction: diff --git a/docs/zh/docs/reference/configuration.md b/docs/zh/docs/reference/configuration.md index 7a8c173988..68c3b20721 100644 --- a/docs/zh/docs/reference/configuration.md +++ b/docs/zh/docs/reference/configuration.md @@ -35,6 +35,7 @@ Server 配置使用 `POWERCONTEXT_SERVER_` 前缀。 | `POWERCONTEXT_SERVER_MCP_PATH` | `/mcp` | MCP 路径 | | `POWERCONTEXT_SERVER_AUTH_ENABLED` | `false` | HTTP 和 MCP 是否要求一个静态 Bearer token | | `POWERCONTEXT_SERVER_AUTH_TOKEN` | 未设置 | 静态 Bearer token；启用鉴权时必须设置 | +| `POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK` | `false` | 在鉴权关闭时显式允许绑定非 loopback 地址 | | `POWERCONTEXT_SERVER_LOGGING_LEVEL` | `INFO` | operational log 级别 | | `POWERCONTEXT_SERVER_LOGGING_FORMAT` | `console` | `console` 或结构化 `json` 输出 | | `POWERCONTEXT_SERVER_LOGGING_ACCESS` | `true` | 记录外部 HTTP 和逻辑 MCP request completion | @@ -53,7 +54,13 @@ Server 配置使用 `POWERCONTEXT_SERVER_` 前缀。 | `POWERCONTEXT_SERVER_EXTERNAL_SKILLS` | 未设置 | 包含 host identity 和显式 Codex Skill roots 的 JSON object | 静态 Bearer 鉴权默认关闭。启用后，API 和 MCP 请求必须携带 `Authorization: Bearer `；liveness 和 -readiness endpoint 仍然公开。明文 HTTP 应只用于 loopback 地址；通过网络暴露启用鉴权的 Server 前必须配置 TLS。 +readiness endpoint 仍然公开。明文 HTTP 仅在 loopback 地址（`127.0.0.1`、`localhost`、`::1`）上受信任。当 Server 绑定到 +非 loopback 地址且鉴权关闭时会拒绝启动；此时应启用鉴权、改回绑定 loopback，或在 TLS 由上游终止或网络本身受控的场景下， +显式设置 `POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true` 主动选择接受。通过网络暴露启用鉴权的 +Server 前必须配置 TLS。 + +Python Client 和 CLI 对出站请求应用相同规则：明文 `http://` 的 Server URL 仅接受 loopback 主机，并且 Client 拒绝 +通过明文的非 loopback 连接发送 Bearer token。 指定 SQLite 路径并启用定时提取的示例： diff --git a/src/powercontext/client/client.py b/src/powercontext/client/client.py index 2cbefe7c16..ff7b51ceaf 100644 --- a/src/powercontext/client/client.py +++ b/src/powercontext/client/client.py @@ -144,6 +144,7 @@ UPDATE_HANDOFF_REPORT_WORKSTREAM, Operation, ) +from powercontext.transport import is_plaintext_non_loopback REQUEST_ID_HEADER = "X-PowerContext-Request-ID" _RequestT = TypeVar("_RequestT") @@ -162,6 +163,8 @@ def __init__( http_client: httpx.AsyncClient | None = None, ) -> None: self._base_url = base_url.rstrip("/") + if token and is_plaintext_non_loopback(self._base_url): + raise ValueError("refusing to send a bearer token over unencrypted non-loopback HTTP") # noqa: TRY003 self._headers = {"Authorization": f"Bearer {token}"} if token else None self._owned_http_client: httpx.AsyncClient | None = None if http_client is None: diff --git a/src/powercontext/client/settings.py b/src/powercontext/client/settings.py index 580ceabd04..934353d43a 100644 --- a/src/powercontext/client/settings.py +++ b/src/powercontext/client/settings.py @@ -5,6 +5,8 @@ from pydantic import Field, HttpUrl, SecretStr, TypeAdapter, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict +from powercontext.transport import is_plaintext_non_loopback + _HTTP_URL = TypeAdapter(HttpUrl) @@ -32,6 +34,8 @@ def validate_server_url(cls, value: str) -> str: raise ValueError("PowerContext Server URL must not contain credentials") # noqa: TRY003 if parsed.query or parsed.fragment: raise ValueError("PowerContext Server URL must not contain a query or fragment") # noqa: TRY003 + if is_plaintext_non_loopback(normalized): + raise ValueError("Unencrypted PowerContext Server URLs must be loopback addresses") # noqa: TRY003 return normalized diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index 875ca7ac4f..6ac1bc1ddd 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -8,7 +8,11 @@ from powercontext.server.factory import create_server_app from powercontext.server.logging import configure_server_logging -from powercontext.server.settings import HttpConfig, ServerSettings +from powercontext.server.settings import ( + HttpConfig, + ServerSettings, + is_unauthenticated_non_loopback_bind, +) from powercontext.server.tracing import configure_server_tracing HELP_OPTION_NAMES = ("-h", "--help") @@ -39,6 +43,17 @@ def run( port=environment.http.port if port is None else port, ) settings = environment.model_copy(update={"http": http}) + if is_unauthenticated_non_loopback_bind( + host=settings.http.host, + auth_enabled=settings.auth.enabled, + allow_unauthenticated_non_loopback=settings.allow_unauthenticated_non_loopback, + ): + raise typer.BadParameter( # noqa: TRY003 + "refusing to bind an unauthenticated Server to a non-loopback address; " + "enable authentication, keep the bind on loopback, or set " + "POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true to opt in", + param_hint="--host", + ) configure_server_logging(settings.logging) tracing = configure_server_tracing(settings.tracing) try: diff --git a/src/powercontext/server/settings.py b/src/powercontext/server/settings.py index 99e0a0ad4e..dd15f7b704 100644 --- a/src/powercontext/server/settings.py +++ b/src/powercontext/server/settings.py @@ -17,12 +17,34 @@ normalize_database_discriminator, ) from powercontext.paths import default_database_path, sqlite_url +from powercontext.transport import is_loopback_host + +_UNSAFE_BIND_MESSAGE = ( + "A non-loopback bind requires bearer authentication; " + "set allow_unauthenticated_non_loopback to opt in when TLS is " + "terminated upstream or the network is otherwise controlled" +) def _default_database() -> SQLiteConfig: return SQLiteConfig(url=sqlite_url(default_database_path())) +def is_unauthenticated_non_loopback_bind( + *, + host: str, + auth_enabled: bool, + allow_unauthenticated_non_loopback: bool, +) -> bool: + """Return whether a bind exposes an unauthenticated Server off loopback.""" + + return ( + not is_loopback_host(host) + and not auth_enabled + and not allow_unauthenticated_non_loopback + ) + + class HttpConfig(BaseModel): """HTTP listener configuration.""" @@ -130,6 +152,7 @@ class ServerSettings(BaseSettings): http: HttpConfig = Field(default_factory=HttpConfig) mcp: McpConfig = Field(default_factory=McpConfig) auth: BearerAuthConfig = Field(default_factory=BearerAuthConfig) + allow_unauthenticated_non_loopback: bool = False dashboard: DashboardConfig = Field(default_factory=DashboardConfig) logging: ServerLoggingConfig = Field(default_factory=ServerLoggingConfig) metrics: MetricsConfig = Field(default_factory=MetricsConfig) @@ -151,6 +174,16 @@ def require_authentication_for_dashboard(self) -> ServerSettings: raise ValueError("Dashboard requires Server bearer authentication") # noqa: TRY003 return self + @model_validator(mode="after") + def reject_unauthenticated_non_loopback_bind(self) -> ServerSettings: + if is_unauthenticated_non_loopback_bind( + host=self.http.host, + auth_enabled=self.auth.enabled, + allow_unauthenticated_non_loopback=self.allow_unauthenticated_non_loopback, + ): + raise ValueError(_UNSAFE_BIND_MESSAGE) + return self + __all__ = [ "BearerAuthConfig", @@ -163,4 +196,5 @@ def require_authentication_for_dashboard(self) -> ServerSettings: "ServerLoggingConfig", "ServerSettings", "TracingConfig", + "is_unauthenticated_non_loopback_bind", ] diff --git a/src/powercontext/transport.py b/src/powercontext/transport.py new file mode 100644 index 0000000000..b27bb35016 --- /dev/null +++ b/src/powercontext/transport.py @@ -0,0 +1,58 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared network transport safety policy for PowerContext surfaces. + +Every surface that opens or configures an HTTP connection to a Server -- the +Python Client, the CLI, and the Agent integrations -- follows the same rule: +plaintext HTTP is only trusted on a loopback address. Bearer credentials must +never leave the machine over an unencrypted connection, and an unauthenticated +Server must not bind to a routable address without an explicit opt-in. +""" + +from __future__ import annotations + +from urllib.parse import urlsplit + +#: Hosts for which unencrypted HTTP is considered safe. +LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) + + +def is_loopback_host(host: str | None) -> bool: + """Return whether ``host`` names a loopback interface. + + Accepts bare hostnames (``localhost``), IPv4 literals (``127.0.0.1``), and + IPv6 literals with or without brackets (``::1`` / ``[::1]``). + """ + + if not host: + return False + normalized = host.strip().lower() + if normalized.startswith("[") and normalized.endswith("]"): + normalized = normalized[1:-1] + return normalized in LOOPBACK_HOSTS + + +def is_plaintext_non_loopback(url: str) -> bool: + """Return whether ``url`` sends plaintext HTTP to a non-loopback host.""" + + parsed = urlsplit(url) + return parsed.scheme == "http" and not is_loopback_host(parsed.hostname) + + +__all__ = [ + "LOOPBACK_HOSTS", + "is_loopback_host", + "is_plaintext_non_loopback", +] diff --git a/tests/test_cli.py b/tests/test_cli.py index 514a9b516e..a10394586d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -198,6 +198,7 @@ def test_server_command_layers_partial_cli_overrides_over_environment_settings( monkeypatch.setattr("powercontext.server.cli._run_server", run_server) monkeypatch.setattr("powercontext.server.cli.configure_server_logging", lambda _config: None) monkeypatch.setattr("powercontext.server.cli.configure_server_tracing", lambda _config: tracing) + monkeypatch.setenv("POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK", "true") for name, value in environment.items(): monkeypatch.setenv(name, value) @@ -213,6 +214,24 @@ def test_server_command_layers_partial_cli_overrides_over_environment_settings( tracing.shutdown.assert_called_once_with() +def test_server_command_rejects_an_unauthenticated_non_loopback_host_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + run_server = Mock() + tracing = Mock() + monkeypatch.setattr("powercontext.server.cli._run_server", run_server) + monkeypatch.setattr("powercontext.server.cli.configure_server_logging", lambda _config: None) + monkeypatch.setattr("powercontext.server.cli.configure_server_tracing", lambda _config: tracing) + + result = CliRunner().invoke( + create_cli([server_app]), + ["server", "run", "--host", "0.0.0.0"], # noqa: S104 - exercises the non-loopback guard. + ) + + assert result.exit_code != 0 + run_server.assert_not_called() + + def test_server_command_does_not_load_client_settings(monkeypatch: pytest.MonkeyPatch) -> None: run_server = Mock() tracing = Mock() diff --git a/tests/test_server.py b/tests/test_server.py index 95e5d910e5..ede28d57d3 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -29,6 +29,7 @@ async def incubate(self, _sources: tuple[Source, ...], /) -> tuple[ExperienceCan def test_settings_load_server_environment(monkeypatch) -> None: monkeypatch.setenv("POWERCONTEXT_SERVER_HTTP_HOST", "127.0.0.2") monkeypatch.setenv("POWERCONTEXT_SERVER_HTTP_PORT", "9000") + monkeypatch.setenv("POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK", "true") monkeypatch.setenv( "POWERCONTEXT_SERVER_DATABASE_URL", "sqlite+aiosqlite:////var/lib/powercontext/test.db", diff --git a/tests/test_transport.py b/tests/test_transport.py new file mode 100644 index 0000000000..053156ab20 --- /dev/null +++ b/tests/test_transport.py @@ -0,0 +1,115 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared network transport-policy contract across Client, CLI, and Server.""" + +from __future__ import annotations + +import httpx +import pytest +from pydantic import SecretStr, ValidationError + +from powercontext.client import PowerContextClient +from powercontext.client.settings import ClientSettings +from powercontext.server.settings import BearerAuthConfig, HttpConfig, ServerSettings +from powercontext.transport import is_loopback_host, is_plaintext_non_loopback + +_ALL_INTERFACES = "0.0.0.0" # noqa: S104 - a non-loopback bind used to exercise the policy. + + +@pytest.mark.parametrize( + "host", + ["127.0.0.1", "localhost", "LOCALHOST", "::1", "[::1]"], +) +def test_loopback_hosts_are_recognized(host: str) -> None: + assert is_loopback_host(host) + + +@pytest.mark.parametrize( + "host", + [_ALL_INTERFACES, "memory.example", "192.168.1.10", "::", "", None], +) +def test_non_loopback_hosts_are_rejected(host: str | None) -> None: + assert not is_loopback_host(host) + + +def test_plaintext_non_loopback_detects_only_remote_http() -> None: + assert is_plaintext_non_loopback("http://memory.example") + assert not is_plaintext_non_loopback("http://127.0.0.1:8000") + assert not is_plaintext_non_loopback("https://memory.example") + + +@pytest.mark.parametrize( + "server_url", + ["http://memory.example", "http://192.168.1.10:8000"], +) +def test_client_settings_reject_non_loopback_plaintext_urls(server_url: str) -> None: + with pytest.raises(ValidationError): + ClientSettings(server_url=server_url) + + +@pytest.mark.parametrize( + "server_url", + ["http://127.0.0.1:8000", "http://localhost:8000", "https://memory.example"], +) +def test_client_settings_accept_loopback_or_tls_urls(server_url: str) -> None: + assert ClientSettings(server_url=server_url).server_url.startswith(("http://", "https://")) + + +def test_client_refuses_a_bearer_token_over_non_loopback_plaintext() -> None: + transport = httpx.MockTransport(lambda request: httpx.Response(200, json={"status": "ok"})) + with httpx.Client(transport=transport): # noqa: SIM117 - guard runs before any request. + with pytest.raises(ValueError, match="bearer token"): + PowerContextClient("http://memory.example", token="probe-token") # noqa: S106 - test credential. + + +def test_client_allows_a_bearer_token_over_loopback_plaintext() -> None: + client = PowerContextClient("http://127.0.0.1:8000", token="probe-token") # noqa: S106 - test credential. + assert client is not None + + +def test_client_allows_an_unauthenticated_non_loopback_transport() -> None: + async def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"status": "ok"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(respond)) + client = PowerContextClient("http://testserver", http_client=http_client) + assert client is not None + + +def test_server_rejects_an_unauthenticated_non_loopback_bind() -> None: + with pytest.raises(ValidationError): + ServerSettings( + http={"host": _ALL_INTERFACES}, + auth={"enabled": False}, + mcp={"enabled": False}, + dashboard={"enabled": False}, + ) + + +def test_server_allows_a_non_loopback_bind_with_authentication() -> None: + settings = ServerSettings( + http=HttpConfig(host=_ALL_INTERFACES), + auth=BearerAuthConfig(enabled=True, token=SecretStr("server-secret")), + ) + assert settings.http.host == _ALL_INTERFACES + + +def test_server_allows_a_non_loopback_bind_with_an_explicit_opt_in() -> None: + settings = ServerSettings( + http=HttpConfig(host=_ALL_INTERFACES), + auth=BearerAuthConfig(enabled=False), + allow_unauthenticated_non_loopback=True, + ) + assert settings.allow_unauthenticated_non_loopback is True From b1547e467b895ad0cfefa34adb3f72d79d03a3ea Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Mon, 24 Aug 2026 23:06:22 +0800 Subject: [PATCH 02/32] test(transport): pin the Codex plugin to the shared loopback policy Load the Codex plugin's settings module by path and assert its private loopback host set and _http_base_url behavior match the shared transport contract, so the isolated plugin copy cannot silently drift. Tighten the configuration docs to note the plaintext-URL rule applies to configured Server URLs. --- docs/en/docs/reference/configuration.md | 5 ++- docs/zh/docs/reference/configuration.md | 2 +- tests/test_transport.py | 58 ++++++++++++++++++++++--- 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/docs/en/docs/reference/configuration.md b/docs/en/docs/reference/configuration.md index 53a2ef0d58..5c54743b16 100644 --- a/docs/en/docs/reference/configuration.md +++ b/docs/en/docs/reference/configuration.md @@ -62,8 +62,9 @@ while authentication is disabled; either enable authentication, keep the bind on upstream or the network is otherwise controlled, set `POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true` to opt in explicitly. Use TLS before exposing an authenticated Server over a network. -The Python Client and CLI apply the matching rule for outbound requests: an unencrypted `http://` Server URL is accepted -only for loopback hosts, and the Client refuses to send a bearer token over an unencrypted non-loopback connection. +The Python Client and CLI apply the matching rule for outbound requests: a configured unencrypted `http://` Server URL is +accepted only for loopback hosts, and the Client refuses to send a bearer token over an unencrypted non-loopback +connection. The Dashboard is enabled by default and shares the Server listener and port with the HTTP API and MCP. With no scopes configured, the page shows an empty state. Dashboard initialization failures are logged with their direct cause and do diff --git a/docs/zh/docs/reference/configuration.md b/docs/zh/docs/reference/configuration.md index f6ff403ed5..bf13750b62 100644 --- a/docs/zh/docs/reference/configuration.md +++ b/docs/zh/docs/reference/configuration.md @@ -61,7 +61,7 @@ readiness endpoint 仍然公开。明文 HTTP 仅在 loopback 地址（`127.0.0. 显式设置 `POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true` 主动选择接受。通过网络暴露启用鉴权的 Server 前必须配置 TLS。 -Python Client 和 CLI 对出站请求应用相同规则：明文 `http://` 的 Server URL 仅接受 loopback 主机，并且 Client 拒绝 +Python Client 和 CLI 对出站请求应用相同规则：配置的明文 `http://` Server URL 仅接受 loopback 主机，并且 Client 拒绝 通过明文的非 loopback 连接发送 Bearer token。 Dashboard 默认启用，并与 HTTP API、MCP 共用监听地址和端口。默认未配置 scope，页面会显示空状态；Dashboard diff --git a/tests/test_transport.py b/tests/test_transport.py index 053156ab20..8a3de091b5 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -12,10 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared network transport-policy contract across Client, CLI, and Server.""" +"""Shared network transport-policy contract across Client, CLI, Server, and the Codex plugin.""" from __future__ import annotations +import importlib.util +from pathlib import Path +from types import ModuleType + import httpx import pytest from pydantic import SecretStr, ValidationError @@ -23,10 +27,33 @@ from powercontext.client import PowerContextClient from powercontext.client.settings import ClientSettings from powercontext.server.settings import BearerAuthConfig, HttpConfig, ServerSettings -from powercontext.transport import is_loopback_host, is_plaintext_non_loopback +from powercontext.transport import LOOPBACK_HOSTS, is_loopback_host, is_plaintext_non_loopback _ALL_INTERFACES = "0.0.0.0" # noqa: S104 - a non-loopback bind used to exercise the policy. +# The Codex plugin ships as an isolated package (it only depends on pydantic-settings) and cannot +# import powercontext, so it keeps its own copy of the loopback policy. Load it by path to pin that +# copy to the shared contract and catch drift the two implementations could otherwise hide. +_CODEX_SETTINGS_PATH = ( + Path(__file__).resolve().parent.parent + / "integrations" + / "codex" + / "plugins" + / "powercontext" + / "settings.py" +) + + +def _load_codex_settings() -> ModuleType: + spec = importlib.util.spec_from_file_location("codex_plugin_settings", _CODEX_SETTINGS_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_CODEX_SETTINGS = _load_codex_settings() + @pytest.mark.parametrize( "host", @@ -91,10 +118,8 @@ async def respond(request: httpx.Request) -> httpx.Response: def test_server_rejects_an_unauthenticated_non_loopback_bind() -> None: with pytest.raises(ValidationError): ServerSettings( - http={"host": _ALL_INTERFACES}, - auth={"enabled": False}, - mcp={"enabled": False}, - dashboard={"enabled": False}, + http=HttpConfig(host=_ALL_INTERFACES), + auth=BearerAuthConfig(enabled=False), ) @@ -113,3 +138,24 @@ def test_server_allows_a_non_loopback_bind_with_an_explicit_opt_in() -> None: allow_unauthenticated_non_loopback=True, ) assert settings.allow_unauthenticated_non_loopback is True + + +def test_codex_plugin_shares_the_loopback_host_set() -> None: + assert _CODEX_SETTINGS._LOOPBACK_HOSTS == LOOPBACK_HOSTS + + +@pytest.mark.parametrize( + "host", + ["127.0.0.1", "localhost", "[::1]", _ALL_INTERFACES, "memory.example", "192.168.1.10"], +) +def test_codex_plugin_matches_the_shared_plaintext_policy(host: str) -> None: + """The Codex plugin's own loopback check must agree with the shared transport contract.""" + + base_url = f"http://{host}:8000" + transport_rejects = is_plaintext_non_loopback(base_url) + try: + _CODEX_SETTINGS._http_base_url(f"{base_url}/mcp") + codex_rejects = False + except ValueError: + codex_rejects = True + assert codex_rejects == transport_rejects From be13bed72701808f09db98d8dad1e19a2e473d91 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Tue, 25 Aug 2026 16:23:59 +0800 Subject: [PATCH 03/32] fix(transport): treat the whole 127.0.0.0/8 block as loopback The loopback check only matched the three named hosts 127.0.0.1, ::1 and localhost, so plaintext HTTP to any other 127.0.0.0/8 address (e.g. 127.0.0.2) was wrongly rejected as non-loopback. Detect loopback via ipaddress.ip_address(...).is_loopback, which covers the whole IPv4 loopback block, and route the Client CLI through the shared helper. Both agent plugins ship isolated and cannot import powercontext, so each vendors its own copy of the check; parametrised drift-guard tests pin both copies to the shared contract. Surface ServerSettings validation errors as a typer.BadParameter instead of a raw traceback. --- .../powercontext/claude_code_settings.py | 17 +++- .../codex/plugins/powercontext/settings.py | 17 +++- src/powercontext/cli/system.py | 4 +- src/powercontext/server/cli.py | 6 +- src/powercontext/server/settings.py | 6 +- src/powercontext/transport.py | 17 +++- tests/test_server.py | 1 - tests/test_transport.py | 83 ++++++++++++------- 8 files changed, 107 insertions(+), 44 deletions(-) diff --git a/integrations/claude-code/plugins/powercontext/claude_code_settings.py b/integrations/claude-code/plugins/powercontext/claude_code_settings.py index 69fa2cde4f..28a24724ae 100644 --- a/integrations/claude-code/plugins/powercontext/claude_code_settings.py +++ b/integrations/claude-code/plugins/powercontext/claude_code_settings.py @@ -16,15 +16,30 @@ from __future__ import annotations +import ipaddress import os from dataclasses import dataclass from urllib.parse import urlsplit, urlunsplit +# Kept in lockstep with powercontext.transport.LOOPBACK_HOSTS; the plugin ships +# isolated and cannot import powercontext. _LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) _TRUE_VALUES = frozenset({"1", "true", "yes", "on"}) _FALSE_VALUES = frozenset({"0", "false", "no", "off"}) +def _is_loopback_host(host: str) -> bool: + """Mirror ``powercontext.transport.is_loopback_host`` for the isolated plugin.""" + + normalized = host.strip().lower() + if normalized in _LOOPBACK_HOSTS: + return True + try: + return ipaddress.ip_address(normalized).is_loopback + except ValueError: + return False + + @dataclass(frozen=True, slots=True) class ClaudeCodePluginSettings: """Configuration loaded once by a plugin entry point.""" @@ -145,7 +160,7 @@ def _http_base_url(value: str) -> str: raise ValueError("PowerContext Server URL must use HTTP or HTTPS") # noqa: TRY003 if parsed.query or parsed.fragment: raise ValueError("PowerContext Server URL must not contain a query or fragment") # noqa: TRY003 - if parsed.scheme == "http" and parsed.hostname.lower() not in _LOOPBACK_HOSTS: + if parsed.scheme == "http" and not _is_loopback_host(parsed.hostname): raise ValueError("unencrypted PowerContext URLs must be loopback addresses") # noqa: TRY003 path = parsed.path.rstrip("/") if path.endswith("/mcp"): diff --git a/integrations/codex/plugins/powercontext/settings.py b/integrations/codex/plugins/powercontext/settings.py index 797d12b330..ab754950ee 100644 --- a/integrations/codex/plugins/powercontext/settings.py +++ b/integrations/codex/plugins/powercontext/settings.py @@ -16,6 +16,7 @@ from __future__ import annotations +import ipaddress from pathlib import Path from typing import Any, Literal from urllib.parse import urlsplit, urlunsplit @@ -26,10 +27,24 @@ from typing_extensions import override _MCP_CONFIGURATION_PATH = Path(__file__).with_name(".mcp.json") +# Kept in lockstep with powercontext.transport.LOOPBACK_HOSTS; the plugin ships +# isolated and cannot import powercontext, so tests pin the two copies together. _LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) _AUTHORIZATION_ENVIRONMENT = {"Authorization": "POWERCONTEXT_CODEX_AUTHORIZATION"} +def _is_loopback_host(host: str) -> bool: + """Mirror ``powercontext.transport.is_loopback_host`` for the isolated plugin.""" + + normalized = host.strip().lower() + if normalized in _LOOPBACK_HOSTS: + return True + try: + return ipaddress.ip_address(normalized).is_loopback + except ValueError: + return False + + class _McpEndpoint(BaseModel): model_config = ConfigDict(extra="forbid") @@ -156,7 +171,7 @@ def _http_base_url(mcp_url: str) -> str: raise ValueError("PowerContext MCP URL must use HTTP or HTTPS") # noqa: TRY003 if parsed.query or parsed.fragment: raise ValueError("PowerContext MCP URL must not contain a query or fragment") # noqa: TRY003 - if parsed.scheme == "http" and parsed.hostname.lower() not in _LOOPBACK_HOSTS: + if parsed.scheme == "http" and not _is_loopback_host(parsed.hostname): raise ValueError("unencrypted PowerContext MCP URLs must be loopback addresses") # noqa: TRY003 mcp_path = parsed.path.rstrip("/") if not mcp_path.endswith("/mcp"): diff --git a/src/powercontext/cli/system.py b/src/powercontext/cli/system.py index 08f0249de6..5c82a29bff 100644 --- a/src/powercontext/cli/system.py +++ b/src/powercontext/cli/system.py @@ -36,6 +36,7 @@ from powercontext.http import HealthResponse, ReadinessResponse, ReadinessStatus from powercontext.paths import powercontext_data_dir +from powercontext.transport import is_loopback_host HELP_OPTION_NAMES = ("-h", "--help") DEFAULT_MARKETPLACE_SOURCE = "oceanbase/powercontext" @@ -43,7 +44,6 @@ PLUGIN_NAME = "powercontext" CLAUDE_MARKETPLACE_NAME = "powercontext" _GITHUB_REPOSITORY = re.compile(r"^[^/\s]+/[^/\s]+$") -_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) setup_app = typer.Typer( name="setup", @@ -1007,7 +1007,7 @@ def _normalize_claude_server_url(value: str) -> str: raise SetupError.claude_server_url_scheme() if parsed.query or parsed.fragment: raise SetupError.claude_server_url_suffix() - if parsed.scheme == "http" and parsed.hostname.lower() not in _LOOPBACK_HOSTS: + if parsed.scheme == "http" and not is_loopback_host(parsed.hostname): raise SetupError.claude_server_url_transport() path = parsed.path.rstrip("/") if path.endswith("/mcp"): diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index eaaaeed12a..5dba8c740b 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -19,6 +19,7 @@ from typing import Annotated, Any import typer +from pydantic import ValidationError from powercontext.server.factory import create_server_app from powercontext.server.logging import configure_server_logging @@ -51,7 +52,10 @@ def run( ) -> None: """Run the ASGI service in the foreground.""" - environment = ServerSettings() + try: + environment = ServerSettings() + except ValidationError as error: + raise typer.BadParameter(str(error)) from error http = HttpConfig( host=environment.http.host if host is None else host, port=environment.http.port if port is None else port, diff --git a/src/powercontext/server/settings.py b/src/powercontext/server/settings.py index 34b4219f02..f38b17fbab 100644 --- a/src/powercontext/server/settings.py +++ b/src/powercontext/server/settings.py @@ -52,11 +52,7 @@ def is_unauthenticated_non_loopback_bind( ) -> bool: """Return whether a bind exposes an unauthenticated Server off loopback.""" - return ( - not is_loopback_host(host) - and not auth_enabled - and not allow_unauthenticated_non_loopback - ) + return not is_loopback_host(host) and not auth_enabled and not allow_unauthenticated_non_loopback class HttpConfig(BaseModel): diff --git a/src/powercontext/transport.py b/src/powercontext/transport.py index b27bb35016..38a276b208 100644 --- a/src/powercontext/transport.py +++ b/src/powercontext/transport.py @@ -23,17 +23,21 @@ from __future__ import annotations +import ipaddress from urllib.parse import urlsplit -#: Hosts for which unencrypted HTTP is considered safe. +#: Named hosts for which unencrypted HTTP is always considered safe. IP literals +#: are additionally accepted whenever they fall inside a loopback range -- for +#: IPv4 that is the whole ``127.0.0.0/8`` block, not just ``127.0.0.1``. LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) def is_loopback_host(host: str | None) -> bool: """Return whether ``host`` names a loopback interface. - Accepts bare hostnames (``localhost``), IPv4 literals (``127.0.0.1``), and - IPv6 literals with or without brackets (``::1`` / ``[::1]``). + Accepts bare hostnames (``localhost``), IPv4 literals anywhere in the + ``127.0.0.0/8`` block (``127.0.0.1``, ``127.0.0.2``, ...), and IPv6 loopback + literals with or without brackets (``::1`` / ``[::1]``). """ if not host: @@ -41,7 +45,12 @@ def is_loopback_host(host: str | None) -> bool: normalized = host.strip().lower() if normalized.startswith("[") and normalized.endswith("]"): normalized = normalized[1:-1] - return normalized in LOOPBACK_HOSTS + if normalized in LOOPBACK_HOSTS: + return True + try: + return ipaddress.ip_address(normalized).is_loopback + except ValueError: + return False def is_plaintext_non_loopback(url: str) -> bool: diff --git a/tests/test_server.py b/tests/test_server.py index e9b1d54613..38109993db 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -90,7 +90,6 @@ def test_settings_load_server_environment(monkeypatch) -> None: monkeypatch.delenv("POWERCONTEXT_SERVER_DASHBOARD_SCOPES", raising=False) monkeypatch.setenv("POWERCONTEXT_SERVER_HTTP_HOST", "127.0.0.2") monkeypatch.setenv("POWERCONTEXT_SERVER_HTTP_PORT", "9000") - monkeypatch.setenv("POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK", "true") monkeypatch.setenv( "POWERCONTEXT_SERVER_DATABASE_URL", "sqlite+aiosqlite:////var/lib/powercontext/test.db", diff --git a/tests/test_transport.py b/tests/test_transport.py index 8a3de091b5..59966f6733 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared network transport-policy contract across Client, CLI, Server, and the Codex plugin.""" +"""Shared network transport-policy contract across Client, CLI, Server, and the agent plugins.""" from __future__ import annotations import importlib.util +import sys from pathlib import Path from types import ModuleType @@ -31,33 +32,47 @@ _ALL_INTERFACES = "0.0.0.0" # noqa: S104 - a non-loopback bind used to exercise the policy. -# The Codex plugin ships as an isolated package (it only depends on pydantic-settings) and cannot -# import powercontext, so it keeps its own copy of the loopback policy. Load it by path to pin that -# copy to the shared contract and catch drift the two implementations could otherwise hide. -_CODEX_SETTINGS_PATH = ( - Path(__file__).resolve().parent.parent - / "integrations" - / "codex" - / "plugins" - / "powercontext" - / "settings.py" -) +# Both agent plugins ship isolated (they do not depend on powercontext) and each vendors its own +# copy of the loopback policy. Load every copy by path and pin it to the shared contract so drift in +# any one implementation is caught. Loading is defensive: a moved path or a plugin import error +# skips that plugin's drift guard rather than failing collection for the whole module. +_INTEGRATIONS = Path(__file__).resolve().parent.parent / "integrations" +_VENDORED_PLUGIN_PATHS = { + "codex": _INTEGRATIONS / "codex" / "plugins" / "powercontext" / "settings.py", + "claude-code": _INTEGRATIONS / "claude-code" / "plugins" / "powercontext" / "claude_code_settings.py", +} -def _load_codex_settings() -> ModuleType: - spec = importlib.util.spec_from_file_location("codex_plugin_settings", _CODEX_SETTINGS_PATH) +def _load_plugin_module(name: str, path: Path) -> tuple[ModuleType | None, str]: + module_name = f"vendored_plugin_{name}".replace("-", "_") + spec = importlib.util.spec_from_file_location(module_name, path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -_CODEX_SETTINGS = _load_codex_settings() + # Register before executing: a slotted dataclass (the Claude Code plugin) resolves its own + # module via sys.modules during class creation and fails to import otherwise. + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception as error: # pragma: no cover - exercised only when a plugin is unavailable. + sys.modules.pop(module_name, None) + return None, repr(error) + return module, "" + + +_VENDORED_PLUGINS = {name: _load_plugin_module(name, path) for name, path in _VENDORED_PLUGIN_PATHS.items()} +_VENDORED_PLUGIN_PARAMS = [ + pytest.param( + module, + id=name, + marks=pytest.mark.skipif(module is None, reason=f"{name} plugin unavailable: {error}"), + ) + for name, (module, error) in _VENDORED_PLUGINS.items() +] @pytest.mark.parametrize( "host", - ["127.0.0.1", "localhost", "LOCALHOST", "::1", "[::1]"], + ["127.0.0.1", "127.0.0.2", "127.1.2.3", "localhost", "LOCALHOST", "::1", "[::1]"], ) def test_loopback_hosts_are_recognized(host: str) -> None: assert is_loopback_host(host) @@ -140,22 +155,32 @@ def test_server_allows_a_non_loopback_bind_with_an_explicit_opt_in() -> None: assert settings.allow_unauthenticated_non_loopback is True -def test_codex_plugin_shares_the_loopback_host_set() -> None: - assert _CODEX_SETTINGS._LOOPBACK_HOSTS == LOOPBACK_HOSTS +@pytest.mark.parametrize("plugin", _VENDORED_PLUGIN_PARAMS) +def test_vendored_plugin_shares_the_loopback_host_set(plugin: ModuleType) -> None: + assert plugin._LOOPBACK_HOSTS == LOOPBACK_HOSTS +@pytest.mark.parametrize("plugin", _VENDORED_PLUGIN_PARAMS) @pytest.mark.parametrize( "host", - ["127.0.0.1", "localhost", "[::1]", _ALL_INTERFACES, "memory.example", "192.168.1.10"], + [ + "127.0.0.1", + "127.0.0.2", + "localhost", + "[::1]", + _ALL_INTERFACES, + "memory.example", + "192.168.1.10", + ], ) -def test_codex_plugin_matches_the_shared_plaintext_policy(host: str) -> None: - """The Codex plugin's own loopback check must agree with the shared transport contract.""" +def test_vendored_plugin_matches_the_shared_plaintext_policy(plugin: ModuleType, host: str) -> None: + """Each plugin's vendored loopback check must agree with the shared transport contract.""" base_url = f"http://{host}:8000" transport_rejects = is_plaintext_non_loopback(base_url) try: - _CODEX_SETTINGS._http_base_url(f"{base_url}/mcp") - codex_rejects = False + plugin._http_base_url(f"{base_url}/mcp") + plugin_rejects = False except ValueError: - codex_rejects = True - assert codex_rejects == transport_rejects + plugin_rejects = True + assert plugin_rejects == transport_rejects From 67c1842a6c8925b5558bc7c5c4cb4ac17550afad Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Tue, 25 Aug 2026 16:24:13 +0800 Subject: [PATCH 04/32] fix(client): skip the plaintext-token guard for caller-supplied transports The bearer-token-over-plaintext guard is only meaningful for the transport the client opens itself, where base_url's scheme reflects what crosses the wire. A caller-supplied http_client owns its own transport (ASGI in-process, a Unix socket, a TLS-terminating proxy -- all carrying an http:// label), so the scheme is no longer a reliable signal and enforcing it only produced false positives, breaking the authenticated in-memory e2e flow. Enforce the guard only when the client creates the transport. --- src/powercontext/client/client.py | 7 ++++++- tests/test_transport.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/powercontext/client/client.py b/src/powercontext/client/client.py index 11b1b3e892..0b105cd4b2 100644 --- a/src/powercontext/client/client.py +++ b/src/powercontext/client/client.py @@ -188,7 +188,12 @@ def __init__( http_client: httpx.AsyncClient | None = None, ) -> None: self._base_url = base_url.rstrip("/") - if token and is_plaintext_non_loopback(self._base_url): + # The plaintext guard only holds meaning for the transport this facade opens itself: there + # ``base_url``'s scheme accurately reflects what crosses the wire. A caller-supplied + # ``http_client`` owns its own transport (ASGI in-process, a Unix socket, a TLS-terminating + # proxy -- all of which carry an ``http://`` label), so the scheme is no longer a reliable + # signal and enforcing it would only produce false positives. + if token and http_client is None and is_plaintext_non_loopback(self._base_url): raise ValueError("refusing to send a bearer token over unencrypted non-loopback HTTP") # noqa: TRY003 self._headers = {"Authorization": f"Bearer {token}"} if token else None self._owned_http_client: httpx.AsyncClient | None = None diff --git a/tests/test_transport.py b/tests/test_transport.py index 59966f6733..47adf6ced2 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -121,6 +121,16 @@ def test_client_allows_a_bearer_token_over_loopback_plaintext() -> None: assert client is not None +def test_client_allows_a_bearer_token_over_a_caller_supplied_transport() -> None: + # A caller-supplied http_client owns its transport (here an in-process ASGI/mock transport), so + # the base_url scheme is only a routing label and the plaintext-token guard must not fire. + http_client = httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"status": "ok"})), + ) + client = PowerContextClient("http://testserver", token="probe-token", http_client=http_client) # noqa: S106 - test credential. + assert client is not None + + def test_client_allows_an_unauthenticated_non_loopback_transport() -> None: async def respond(request: httpx.Request) -> httpx.Response: return httpx.Response(200, json={"status": "ok"}) From 8615e7d9e8bfc8f8f3488eceb892da2c3b821477 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Tue, 25 Aug 2026 16:24:13 +0800 Subject: [PATCH 05/32] fix(e2e): opt the acceptance harness into its non-loopback Server bind The Docker image binds POWERCONTEXT_SERVER_HTTP_HOST=0.0.0.0 so the port is reachable from the harness container, but the unauthenticated-non-loopback guard now refuses that bind and the Server fails to start. The Compose network is isolated, so opt in via POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK to restore startup. --- e2e/bub/compose.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/e2e/bub/compose.yaml b/e2e/bub/compose.yaml index f6a537e224..4510fdb183 100644 --- a/e2e/bub/compose.yaml +++ b/e2e/bub/compose.yaml @@ -39,6 +39,9 @@ services: - POWERCONTEXT_SERVER_INFERENCE_GENERATION_TIMEOUT_SECONDS=${POWERCONTEXT_SERVER_INFERENCE_GENERATION_TIMEOUT_SECONDS:-120} - POWERCONTEXT_SERVER_DATABASE_KIND=sqlite - POWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite:////data/powercontext.db + # The image binds 0.0.0.0 so the port is reachable from the harness; this Compose network is + # isolated, so opt the unauthenticated Server into the non-loopback bind it now guards against. + - POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true healthcheck: test: - CMD From fb4250a63645af28894526e0c943b6bb00144fe5 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Tue, 25 Aug 2026 21:05:19 +0800 Subject: [PATCH 06/32] fix(transport): harden the transport-safety policy per PR #1344 review - client: gate the plaintext-token guard on an explicit trust_transport_security opt-in; supplying an http_client is no longer treated as evidence the transport is safe, so a bearer token is still refused over unencrypted non-loopback HTTP. - docker: declare POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK in the image itself so the documented `docker run` starts out of the box, document the network-exposure trade-off, and drop the masking env from the e2e compose so it exercises the real image contract; add a Dockerfile smoke test. - server cli: merge --host/--port before validation so a safe override can repair an unsafe environment bind, and translate the actionable settings failures (unauthenticated non-loopback bind, missing bearer token) into friendly CLI errors recognised by exception identity rather than raw validation text. --- docker/Dockerfile | 6 +++ docker/README.md | 11 +++++ e2e/bub/compose.yaml | 3 -- src/powercontext/client/client.py | 18 +++++--- src/powercontext/server/cli.py | 70 ++++++++++++++++++++-------- src/powercontext/server/settings.py | 24 +++++++++- tests/e2e/test_statistics_flow.py | 1 + tests/test_cli.py | 49 +++++++++++++++++++- tests/test_docker_contract.py | 71 +++++++++++++++++++++++++++++ tests/test_transport.py | 29 ++++++++++-- 10 files changed, 246 insertions(+), 36 deletions(-) create mode 100644 tests/test_docker_contract.py diff --git a/docker/Dockerfile b/docker/Dockerfile index 0cc3c02102..5636128efa 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -49,6 +49,12 @@ ENV PATH="/app/.venv/bin:$PATH" \ POWERCONTEXT_SERVER_HTTP_HOST=0.0.0.0 \ POWERCONTEXT_SERVER_HTTP_PORT=8000 +# The container binds 0.0.0.0 so a published port is reachable, and its network namespace is the +# controlled boundary the unauthenticated-bind guard is designed for. Declare that opt-in here so the +# documented `docker run` starts out of the box; production deployments should still enable bearer +# auth or terminate TLS upstream (see docker/README.md). +ENV POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true + COPY --from=builder --chown=powercontext:powercontext /app/.venv /app/.venv USER powercontext diff --git a/docker/README.md b/docker/README.md index 8d0f899885..e805773924 100644 --- a/docker/README.md +++ b/docker/README.md @@ -30,6 +30,17 @@ requests share one refresh. Checks use the Runtime's credentials and never expos another database or inference provider with the same `POWERCONTEXT_SERVER_*` environment variables used by a regular Server installation. +## Network exposure + +PowerContext refuses to start an unauthenticated Server on a non-loopback address unless +`POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true` opts in. The image binds `0.0.0.0` so a published +port is reachable, and its network namespace is the controlled boundary that opt-in is meant for, so the image sets +it by default and the `docker run` above starts without extra configuration. Access is still governed by which ports +you publish (`--publish`) and the surrounding network. For an exposed deployment, put the Server behind a +TLS-terminating proxy and enable bearer authentication with +`POWERCONTEXT_SERVER_AUTH_ENABLED=true` and `POWERCONTEXT_SERVER_AUTH_TOKEN=...`; when authentication is enabled the +opt-in is no longer required. + The `Build Docker image` GitHub workflow builds downloadable Linux amd64 and arm64 image archives for pull requests, changes merged to `main`, and manual runs. Publishing a GitHub Release pushes a multi-platform image to Docker Hub. Repository configuration must provide `DOCKER_USERNAME` and `DOCKER_PASSWORD` secrets plus a `DOCKER_PUSH_BASE` diff --git a/e2e/bub/compose.yaml b/e2e/bub/compose.yaml index 4510fdb183..f6a537e224 100644 --- a/e2e/bub/compose.yaml +++ b/e2e/bub/compose.yaml @@ -39,9 +39,6 @@ services: - POWERCONTEXT_SERVER_INFERENCE_GENERATION_TIMEOUT_SECONDS=${POWERCONTEXT_SERVER_INFERENCE_GENERATION_TIMEOUT_SECONDS:-120} - POWERCONTEXT_SERVER_DATABASE_KIND=sqlite - POWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite:////data/powercontext.db - # The image binds 0.0.0.0 so the port is reachable from the harness; this Compose network is - # isolated, so opt the unauthenticated Server into the non-loopback bind it now guards against. - - POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true healthcheck: test: - CMD diff --git a/src/powercontext/client/client.py b/src/powercontext/client/client.py index 0b105cd4b2..25b75d4534 100644 --- a/src/powercontext/client/client.py +++ b/src/powercontext/client/client.py @@ -186,14 +186,20 @@ def __init__( token: str | None = None, timeout: float = 10.0, http_client: httpx.AsyncClient | None = None, + trust_transport_security: bool = False, ) -> None: self._base_url = base_url.rstrip("/") - # The plaintext guard only holds meaning for the transport this facade opens itself: there - # ``base_url``'s scheme accurately reflects what crosses the wire. A caller-supplied - # ``http_client`` owns its own transport (ASGI in-process, a Unix socket, a TLS-terminating - # proxy -- all of which carry an ``http://`` label), so the scheme is no longer a reliable - # signal and enforcing it would only produce false positives. - if token and http_client is None and is_plaintext_non_loopback(self._base_url): + # Plaintext HTTP only carries a bearer token safely on loopback. When this facade opens the + # transport itself, ``base_url``'s scheme accurately reflects what crosses the wire. A + # caller-supplied ``http_client`` *may* instead own a transport whose ``http://`` label is + # only a routing token -- an in-process ASGI app, a Unix socket, or a TLS-terminating proxy + # -- but a plain pooling ``httpx.AsyncClient`` (e.g. the shared client the LangGraph adapter + # installs) is exactly as exposed as one we would open ourselves. Supplying a transport is + # therefore not evidence of safety: the guard stays on for caller-supplied transports too, + # and a caller that knows its transport is secure must say so explicitly via + # ``trust_transport_security`` rather than have safety inferred from the argument being set. + transport_trusted = http_client is not None and trust_transport_security + if token and not transport_trusted and is_plaintext_non_loopback(self._base_url): raise ValueError("refusing to send a bearer token over unencrypted non-loopback HTTP") # noqa: TRY003 self._headers = {"Authorization": f"Bearer {token}"} if token else None self._owned_http_client: httpx.AsyncClient | None = None diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index 5dba8c740b..3783b2bafd 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -24,14 +24,31 @@ from powercontext.server.factory import create_server_app from powercontext.server.logging import configure_server_logging from powercontext.server.settings import ( - HttpConfig, + MissingBearerTokenError, ServerSettings, - is_unauthenticated_non_loopback_bind, + UnauthenticatedNonLoopbackBindError, ) from powercontext.server.tracing import configure_server_tracing HELP_OPTION_NAMES = ("-h", "--help") +# Shown when the merged bind fails the unauthenticated-non-loopback policy. It repeats the +# operator's concrete levers -- authenticate, stay on loopback, or opt in via the full env var -- +# instead of surfacing pydantic's internal validation dump (see ``_friendly_bad_parameter``). +_UNSAFE_BIND_CLI_MESSAGE = ( + "refusing to bind an unauthenticated Server to a non-loopback address; " + "enable authentication, keep the bind on loopback, or set " + "POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true to opt in" +) + +# Shown when authentication is enabled without a token; names the concrete env-var levers the +# operator can set instead of surfacing pydantic's internal validation dump. +_MISSING_BEARER_CLI_MESSAGE = ( + "authentication is enabled but no bearer token is configured; " + "set POWERCONTEXT_SERVER_AUTH_TOKEN=... or disable it with " + "POWERCONTEXT_SERVER_AUTH_ENABLED=false" +) + app = typer.Typer( name="server", context_settings={"help_option_names": HELP_OPTION_NAMES}, @@ -52,26 +69,20 @@ def run( ) -> None: """Run the ASGI service in the foreground.""" + # Layer the CLI overrides in *before* validation. ``ServerSettings`` enforces its bind policy on + # construction, so the command-line ``--host`` / ``--port`` must be merged with the environment + # first -- otherwise a safe ``--host 127.0.0.1`` could not repair an unsafe + # ``POWERCONTEXT_SERVER_HTTP_HOST=0.0.0.0`` (and vice versa), because the cross-field check would + # run against the environment value alone rather than the address we actually bind. + http_overrides: dict[str, Any] = {} + if host is not None: + http_overrides["host"] = host + if port is not None: + http_overrides["port"] = port try: - environment = ServerSettings() + settings = ServerSettings(http=http_overrides) if http_overrides else ServerSettings() except ValidationError as error: - raise typer.BadParameter(str(error)) from error - http = HttpConfig( - host=environment.http.host if host is None else host, - port=environment.http.port if port is None else port, - ) - settings = environment.model_copy(update={"http": http}) - if is_unauthenticated_non_loopback_bind( - host=settings.http.host, - auth_enabled=settings.auth.enabled, - allow_unauthenticated_non_loopback=settings.allow_unauthenticated_non_loopback, - ): - raise typer.BadParameter( # noqa: TRY003 - "refusing to bind an unauthenticated Server to a non-loopback address; " - "enable authentication, keep the bind on loopback, or set " - "POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true to opt in", - param_hint="--host", - ) + raise _friendly_bad_parameter(error) from error configure_server_logging(settings.logging) tracing = configure_server_tracing(settings.tracing) try: @@ -93,6 +104,25 @@ def run( tracing.shutdown() +def _friendly_bad_parameter(error: ValidationError) -> typer.BadParameter: + """Translate a settings ``ValidationError`` into an actionable CLI parameter error. + + ``ServerSettings`` enforces its policies at construction time, so a rejected ``--host`` / + environment combination arrives here wrapped in pydantic's generic validation report. The + policy failures an operator can act on directly are recognised by identity via pydantic's + ``ctx['error']`` -- not by matching the raw text -- and translated into a concrete lever. + Anything else falls back to pydantic's message unchanged. + """ + + for detail in error.errors(include_context=True): + cause = (detail.get("ctx") or {}).get("error") + if isinstance(cause, UnauthenticatedNonLoopbackBindError): + return typer.BadParameter(_UNSAFE_BIND_CLI_MESSAGE, param_hint="--host") + if isinstance(cause, MissingBearerTokenError): + return typer.BadParameter(_MISSING_BEARER_CLI_MESSAGE) + return typer.BadParameter(str(error)) + + def _run_server(application: Any, *, host: str, port: int) -> None: import uvicorn diff --git a/src/powercontext/server/settings.py b/src/powercontext/server/settings.py index f38b17fbab..060730d89a 100644 --- a/src/powercontext/server/settings.py +++ b/src/powercontext/server/settings.py @@ -40,6 +40,24 @@ ) +class UnauthenticatedNonLoopbackBindError(ValueError): + """Raised when a bind would expose an unauthenticated Server off loopback. + + A dedicated type lets callers that assemble the settings (e.g. the CLI) recognise this + policy failure by identity -- via pydantic's ``ctx['error']`` -- and translate it into an + actionable message, without matching against the raw validation text. + """ + + +class MissingBearerTokenError(ValueError): + """Raised when authentication is enabled but no bearer token is configured. + + Recognised by identity the same way as :class:`UnauthenticatedNonLoopbackBindError`, so the + CLI can point the operator at the concrete token / disable levers instead of surfacing + pydantic's raw validation report. + """ + + def _default_database() -> SQLiteConfig: return SQLiteConfig(url=sqlite_url(default_database_path())) @@ -86,7 +104,7 @@ class BearerAuthConfig(BaseModel): @model_validator(mode="after") def require_token_when_enabled(self) -> BearerAuthConfig: if self.enabled and (self.token is None or not self.token.get_secret_value()): - raise ValueError("Bearer token is required when authentication is enabled") # noqa: TRY003 + raise MissingBearerTokenError("Bearer token is required when authentication is enabled") # noqa: TRY003 return self @@ -197,7 +215,7 @@ def reject_unauthenticated_non_loopback_bind(self) -> ServerSettings: auth_enabled=self.auth.enabled, allow_unauthenticated_non_loopback=self.allow_unauthenticated_non_loopback, ): - raise ValueError(_UNSAFE_BIND_MESSAGE) + raise UnauthenticatedNonLoopbackBindError(_UNSAFE_BIND_MESSAGE) return self @@ -209,8 +227,10 @@ def reject_unauthenticated_non_loopback_bind(self) -> ServerSettings: "HttpConfig", "McpConfig", "MetricsConfig", + "MissingBearerTokenError", "ServerLoggingConfig", "ServerSettings", "TracingConfig", + "UnauthenticatedNonLoopbackBindError", "is_unauthenticated_non_loopback_bind", ] diff --git a/tests/e2e/test_statistics_flow.py b/tests/e2e/test_statistics_flow.py index 1f83eb452e..d3b5165491 100644 --- a/tests/e2e/test_statistics_flow.py +++ b/tests/e2e/test_statistics_flow.py @@ -94,6 +94,7 @@ def _client(app) -> tuple[httpx.AsyncClient, PowerContextClient]: "http://testserver", token=_AUTH_TOKEN, http_client=transport, + trust_transport_security=True, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 33445115a0..ce31808d5e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -242,8 +242,55 @@ def test_server_command_rejects_an_unauthenticated_non_loopback_host_override( ["server", "run", "--host", "0.0.0.0"], # noqa: S104 - exercises the non-loopback guard. ) - assert result.exit_code != 0 + assert result.exit_code == 2 # typer.BadParameter, not an unhandled traceback. run_server.assert_not_called() + # The operator gets the actionable opt-in lever (the full env var), not pydantic's internal dump. + assert "--host" in result.output + assert "POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true" in result.output + assert "pydantic" not in result.output + + +def test_server_command_reports_a_friendly_error_when_auth_lacks_a_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + run_server = Mock() + tracing = Mock() + monkeypatch.setattr("powercontext.server.cli._run_server", run_server) + monkeypatch.setattr("powercontext.server.cli.configure_server_logging", lambda _config: None) + monkeypatch.setattr("powercontext.server.cli.configure_server_tracing", lambda _config: tracing) + monkeypatch.setenv("POWERCONTEXT_SERVER_AUTH_ENABLED", "true") + monkeypatch.delenv("POWERCONTEXT_SERVER_AUTH_TOKEN", raising=False) + + result = CliRunner().invoke(create_cli([server_app]), ["server", "run"]) + + assert result.exit_code == 2 # typer.BadParameter, not an unhandled traceback. + run_server.assert_not_called() + # The operator gets the concrete token / disable levers, not pydantic's internal dump. + assert "POWERCONTEXT_SERVER_AUTH_TOKEN" in result.output + assert "POWERCONTEXT_SERVER_AUTH_ENABLED=false" in result.output + assert "pydantic" not in result.output + + +def test_server_command_lets_a_loopback_override_repair_an_unsafe_environment_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The bind policy must validate the address we actually bind, not the environment value alone: a + # safe ``--host 127.0.0.1`` has to repair an unsafe POWERCONTEXT_SERVER_HTTP_HOST=0.0.0.0. + run_server = Mock() + tracing = Mock() + monkeypatch.setattr("powercontext.server.cli._run_server", run_server) + monkeypatch.setattr("powercontext.server.cli.configure_server_logging", lambda _config: None) + monkeypatch.setattr("powercontext.server.cli.configure_server_tracing", lambda _config: tracing) + monkeypatch.setenv("POWERCONTEXT_SERVER_HTTP_HOST", "0.0.0.0") # noqa: S104 - unsafe env value the CLI repairs. + + result = CliRunner().invoke( + create_cli([server_app]), + ["server", "run", "--host", "127.0.0.1"], + ) + + assert result.exit_code == 0 + run_server.assert_called_once() + assert run_server.call_args.kwargs["host"] == "127.0.0.1" def test_server_command_does_not_load_client_settings(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_docker_contract.py b/tests/test_docker_contract.py new file mode 100644 index 0000000000..0df97467ac --- /dev/null +++ b/tests/test_docker_contract.py @@ -0,0 +1,71 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Smoke test that the documented container image starts under its own environment. + +The published image binds ``0.0.0.0`` with authentication disabled, so the unauthenticated-bind +guard would refuse to start unless the image also declares the opt-in. This test reads the real +``POWERCONTEXT_SERVER_*`` environment straight from ``docker/Dockerfile`` and drives the documented +``server run`` invocation (no CLI overrides, as ``docker run ... powercontext-server:local`` uses), +so a Dockerfile that drops the opt-in fails here instead of only in a Compose file that sets its own. +""" + +from __future__ import annotations + +import os +import re +from pathlib import Path +from unittest.mock import Mock + +import pytest +from typer.testing import CliRunner + +from powercontext.cli.app import create_cli +from powercontext.server.cli import app as server_app + +_DOCKERFILE = Path(__file__).resolve().parent.parent / "docker" / "Dockerfile" +_SERVER_ENV_PATTERN = re.compile(r"(POWERCONTEXT_SERVER_[A-Z0-9_]+)=([^\s\\]+)") + + +def _dockerfile_server_environment() -> dict[str, str]: + """Extract the image's ``POWERCONTEXT_SERVER_*`` ENV assignments from the Dockerfile.""" + + return dict(_SERVER_ENV_PATTERN.findall(_DOCKERFILE.read_text(encoding="utf-8"))) + + +def test_dockerfile_pins_the_bind_environment_under_test() -> None: + # Guard the test's own premise: if these keys move, the smoke test below stops meaning anything. + environment = _dockerfile_server_environment() + assert environment.get("POWERCONTEXT_SERVER_HTTP_HOST") == "0.0.0.0" # noqa: S104 - the image's documented bind. + + +def test_documented_container_invocation_starts_out_of_the_box(monkeypatch: pytest.MonkeyPatch) -> None: + run_server = Mock() + tracing = Mock() + monkeypatch.setattr("powercontext.server.cli._run_server", run_server) + monkeypatch.setattr("powercontext.server.cli.configure_server_logging", lambda _config: None) + monkeypatch.setattr("powercontext.server.cli.configure_server_tracing", lambda _config: tracing) + # Reproduce the container's environment exactly, with no ambient POWERCONTEXT_SERVER_* leaking in. + for name in list(os.environ): + if name.startswith("POWERCONTEXT_SERVER_"): + monkeypatch.delenv(name, raising=False) + for name, value in _dockerfile_server_environment().items(): + monkeypatch.setenv(name, value) + + # The documented `docker run` passes no CLI overrides; the image's CMD is `server run`. + result = CliRunner().invoke(create_cli([server_app]), ["server", "run"]) + + assert result.exit_code == 0, result.output + run_server.assert_called_once() + assert run_server.call_args.kwargs["host"] == "0.0.0.0" # noqa: S104 - the image's documented bind. diff --git a/tests/test_transport.py b/tests/test_transport.py index 47adf6ced2..c23e578535 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -121,16 +121,37 @@ def test_client_allows_a_bearer_token_over_loopback_plaintext() -> None: assert client is not None -def test_client_allows_a_bearer_token_over_a_caller_supplied_transport() -> None: - # A caller-supplied http_client owns its transport (here an in-process ASGI/mock transport), so - # the base_url scheme is only a routing label and the plaintext-token guard must not fire. +def test_client_allows_a_bearer_token_over_a_trusted_caller_supplied_transport() -> None: + # A caller-supplied http_client may own a transport whose ``http://`` label is only a routing + # token (here an in-process ASGI/mock transport). The caller must vouch for it explicitly with + # ``trust_transport_security`` before the plaintext-token guard stands down. http_client = httpx.AsyncClient( transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"status": "ok"})), ) - client = PowerContextClient("http://testserver", token="probe-token", http_client=http_client) # noqa: S106 - test credential. + client = PowerContextClient( + "http://testserver", + token="probe-token", # noqa: S106 - test credential. + http_client=http_client, + trust_transport_security=True, + ) assert client is not None +def test_client_refuses_a_bearer_token_over_an_untrusted_caller_supplied_transport() -> None: + # Supplying an http_client is not evidence of safety: a shared pooling client (as the LangGraph + # adapter installs) points at a real network. Without an explicit trust opt-in the guard must + # still refuse to leak the token over non-loopback plaintext. + http_client = httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"status": "ok"})), + ) + with pytest.raises(ValueError, match="bearer token"): + PowerContextClient( + "http://memory.example", + token="probe-token", # noqa: S106 - test credential. + http_client=http_client, + ) + + def test_client_allows_an_unauthenticated_non_loopback_transport() -> None: async def respond(request: httpx.Request) -> httpx.Response: return httpx.Response(200, json={"status": "ok"}) From f99181678cd6ffe5b49b1b77ffd5012fafab2549 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Tue, 25 Aug 2026 21:11:39 +0800 Subject: [PATCH 07/32] fix(server): satisfy ty and make CLI error assertions width-independent - cli: unpack the ServerSettings kwargs so the partial ``http`` mapping is not type-checked against the full HttpConfig annotation (ty invalid-argument-type). - tests: pin a wide COLUMNS in the friendly-error tests so the rich panel does not hard-wrap the asserted --host / env-var tokens on a narrow CI terminal. --- src/powercontext/server/cli.py | 6 +++++- tests/test_cli.py | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index 3783b2bafd..8fc0235ead 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -79,8 +79,12 @@ def run( http_overrides["host"] = host if port is not None: http_overrides["port"] = port + # Pass ``http`` as a partial mapping so ``nested_model_default_partial_update`` merges it with the + # environment; unpack from a kwargs dict so the partial dict is not type-checked against the full + # ``HttpConfig`` the field annotation advertises. + settings_kwargs: dict[str, Any] = {"http": http_overrides} if http_overrides else {} try: - settings = ServerSettings(http=http_overrides) if http_overrides else ServerSettings() + settings = ServerSettings(**settings_kwargs) except ValidationError as error: raise _friendly_bad_parameter(error) from error configure_server_logging(settings.logging) diff --git a/tests/test_cli.py b/tests/test_cli.py index ce31808d5e..d92c2e6186 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -236,6 +236,9 @@ def test_server_command_rejects_an_unauthenticated_non_loopback_host_override( monkeypatch.setattr("powercontext.server.cli._run_server", run_server) monkeypatch.setattr("powercontext.server.cli.configure_server_logging", lambda _config: None) monkeypatch.setattr("powercontext.server.cli.configure_server_tracing", lambda _config: tracing) + # Keep the error panel on one line so the asserted tokens are not hard-wrapped by the terminal + # width (rich breaks on hyphens and force-splits long words on a narrow CI terminal). + monkeypatch.setenv("COLUMNS", "1000") result = CliRunner().invoke( create_cli([server_app]), @@ -260,6 +263,8 @@ def test_server_command_reports_a_friendly_error_when_auth_lacks_a_token( monkeypatch.setattr("powercontext.server.cli.configure_server_tracing", lambda _config: tracing) monkeypatch.setenv("POWERCONTEXT_SERVER_AUTH_ENABLED", "true") monkeypatch.delenv("POWERCONTEXT_SERVER_AUTH_TOKEN", raising=False) + # Keep the error panel on one line so the asserted env-var tokens are not hard-wrapped (see above). + monkeypatch.setenv("COLUMNS", "1000") result = CliRunner().invoke(create_cli([server_app]), ["server", "run"]) From 9d6fb2cf439243d7dc8524f4b0fbd15a21a896a5 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Tue, 25 Aug 2026 21:21:07 +0800 Subject: [PATCH 08/32] fix(pi): align the plugin loopback check with the shared transport contract Pi's isLoopback only accepted localhost, 127.0.0.1 and [::1], so its production resolveConfig rejected http://127.0.0.2:8000 as "must use HTTPS outside loopback" while the shared policy and the Python plugins already trust the whole 127.0.0.0/8 block. Rewrite it to mirror is_loopback_host (strip brackets, lowercase, accept 127.0.0.0/8, localhost and ::1). Add a Pi Vitest drift guard and a JSON fixture of loopback/non-loopback host vectors that both the Python drift guard and the Pi suite consume, so a future divergence from the shared contract fails in at least one language. --- .../pi/plugins/powercontext/src/config.ts | 17 ++++++- .../tests/transport-policy.spec.ts | 45 +++++++++++++++++++ .../fixtures/transport_loopback_vectors.json | 5 +++ tests/test_transport.py | 44 +++++++++--------- 4 files changed, 86 insertions(+), 25 deletions(-) create mode 100644 integrations/pi/plugins/powercontext/tests/transport-policy.spec.ts create mode 100644 tests/fixtures/transport_loopback_vectors.json diff --git a/integrations/pi/plugins/powercontext/src/config.ts b/integrations/pi/plugins/powercontext/src/config.ts index 7579c9d093..aa4eb952a3 100644 --- a/integrations/pi/plugins/powercontext/src/config.ts +++ b/integrations/pi/plugins/powercontext/src/config.ts @@ -67,8 +67,23 @@ function envInteger( return value } +function isIpv4Loopback(host: string): boolean { + const octets = host.split('.') + if (octets.length !== 4) return false + if (!octets.every((octet) => /^\d{1,3}$/.test(octet) && Number(octet) <= 255)) return false + // The whole 127.0.0.0/8 block is loopback, not just 127.0.0.1. + return octets[0] === '127' +} + function isLoopback(hostname: string): boolean { - return hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '[::1]' + // Mirror the shared `is_loopback_host` transport contract (src/powercontext/transport.py): trim, + // lowercase, drop the IPv6 brackets that URL.hostname keeps, then accept `localhost`, the whole + // IPv4 127.0.0.0/8 block, and IPv6 ::1. Drift here is pinned by transport-policy.spec.ts, which + // shares its host vectors with the Python drift guard. + const normalized = hostname.trim().toLowerCase().replace(/^\[/, '').replace(/\]$/, '') + if (!normalized) return false + if (normalized === 'localhost' || normalized === '::1') return true + return isIpv4Loopback(normalized) } function normalizeBaseUrl(value: string): string { diff --git a/integrations/pi/plugins/powercontext/tests/transport-policy.spec.ts b/integrations/pi/plugins/powercontext/tests/transport-policy.spec.ts new file mode 100644 index 0000000000..90c2cb0ec0 --- /dev/null +++ b/integrations/pi/plugins/powercontext/tests/transport-policy.spec.ts @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' +import { resolveConfig } from '../src/config.ts' + +// Single source of truth shared with the Python transport drift guard (tests/test_transport.py). +// Both suites drive the SAME host vectors through their production entry points, so a plugin that +// drifts from the shared 127.0.0.0/8 loopback contract fails in at least one language. +const vectorsPath = fileURLToPath( + new URL('../../../../../tests/fixtures/transport_loopback_vectors.json', import.meta.url), +) +const vectors = JSON.parse(readFileSync(vectorsPath, 'utf8')) as { + loopback: string[] + non_loopback: string[] +} + +describe('Pi transport policy', () => { + it.each(vectors.loopback)('trusts a plaintext loopback bind for %s', (host) => { + // WHATWG URL lowercases the host, so the normalized baseUrl echoes the lowercased authority. + const config = resolveConfig({ POWERCONTEXT_PI_BASE_URL: `http://${host}:8000` }) + expect(config.baseUrl).toBe(`http://${host.toLowerCase()}:8000`) + }) + + it.each(vectors.non_loopback)('refuses a plaintext non-loopback bind for %s', (host) => { + expect(() => resolveConfig({ POWERCONTEXT_PI_BASE_URL: `http://${host}:8000` })).toThrow( + /must use HTTPS outside loopback/, + ) + }) +}) diff --git a/tests/fixtures/transport_loopback_vectors.json b/tests/fixtures/transport_loopback_vectors.json new file mode 100644 index 0000000000..169566943d --- /dev/null +++ b/tests/fixtures/transport_loopback_vectors.json @@ -0,0 +1,5 @@ +{ + "_comment": "Shared transport-policy host vectors -- the single source of truth for the loopback contract. Consumed by the Python drift guard (tests/test_transport.py) and the Pi plugin's Vitest drift guard (integrations/pi/plugins/powercontext/tests/transport-policy.spec.ts). Every entry is a valid URL authority so both languages can build http://:8000: loopback hosts must be trusted for plaintext, non_loopback hosts must be refused plaintext. Keep this aligned with src/powercontext/transport.py.", + "loopback": ["127.0.0.1", "127.0.0.2", "127.1.2.3", "localhost", "LOCALHOST", "[::1]"], + "non_loopback": ["0.0.0.0", "memory.example", "192.168.1.10"] +} diff --git a/tests/test_transport.py b/tests/test_transport.py index c23e578535..2001722279 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -17,6 +17,7 @@ from __future__ import annotations import importlib.util +import json import sys from pathlib import Path from types import ModuleType @@ -32,10 +33,20 @@ _ALL_INTERFACES = "0.0.0.0" # noqa: S104 - a non-loopback bind used to exercise the policy. -# Both agent plugins ship isolated (they do not depend on powercontext) and each vendors its own -# copy of the loopback policy. Load every copy by path and pin it to the shared contract so drift in -# any one implementation is caught. Loading is defensive: a moved path or a plugin import error -# skips that plugin's drift guard rather than failing collection for the whole module. +# Loopback host vectors shared with the TypeScript plugins' own drift guards (e.g. the Pi plugin's +# transport-policy.spec.ts). Keeping one JSON source of truth means a plugin that drifts from the +# 127.0.0.0/8 loopback contract fails here or in that plugin's suite -- never silently in both. +_LOOPBACK_VECTORS = json.loads( + (Path(__file__).resolve().parent / "fixtures" / "transport_loopback_vectors.json").read_text(encoding="utf-8"), +) +_SHARED_LOOPBACK_HOSTS: list[str] = _LOOPBACK_VECTORS["loopback"] +_SHARED_NON_LOOPBACK_HOSTS: list[str] = _LOOPBACK_VECTORS["non_loopback"] + +# The Python plugins (Codex, Claude Code) ship isolated (they do not depend on powercontext) and +# each vendors its own copy of the loopback policy. Load every copy by path and pin it to the shared +# contract so drift in any one implementation is caught. Loading is defensive: a moved path or a +# plugin import error skips that plugin's drift guard rather than failing collection for the whole +# module. The TypeScript plugins are pinned by their own Vitest suites against the same vectors. _INTEGRATIONS = Path(__file__).resolve().parent.parent / "integrations" _VENDORED_PLUGIN_PATHS = { "codex": _INTEGRATIONS / "codex" / "plugins" / "powercontext" / "settings.py", @@ -70,18 +81,14 @@ def _load_plugin_module(name: str, path: Path) -> tuple[ModuleType | None, str]: ] -@pytest.mark.parametrize( - "host", - ["127.0.0.1", "127.0.0.2", "127.1.2.3", "localhost", "LOCALHOST", "::1", "[::1]"], -) +# The shared vectors carry only valid URL authorities so both languages can build them into a URL; +# `"::1"` (unbracketed) plus the falsy inputs are Python-only edge cases the shared set cannot express. +@pytest.mark.parametrize("host", [*_SHARED_LOOPBACK_HOSTS, "::1"]) def test_loopback_hosts_are_recognized(host: str) -> None: assert is_loopback_host(host) -@pytest.mark.parametrize( - "host", - [_ALL_INTERFACES, "memory.example", "192.168.1.10", "::", "", None], -) +@pytest.mark.parametrize("host", [*_SHARED_NON_LOOPBACK_HOSTS, "::", "", None]) def test_non_loopback_hosts_are_rejected(host: str | None) -> None: assert not is_loopback_host(host) @@ -192,18 +199,7 @@ def test_vendored_plugin_shares_the_loopback_host_set(plugin: ModuleType) -> Non @pytest.mark.parametrize("plugin", _VENDORED_PLUGIN_PARAMS) -@pytest.mark.parametrize( - "host", - [ - "127.0.0.1", - "127.0.0.2", - "localhost", - "[::1]", - _ALL_INTERFACES, - "memory.example", - "192.168.1.10", - ], -) +@pytest.mark.parametrize("host", [*_SHARED_LOOPBACK_HOSTS, *_SHARED_NON_LOOPBACK_HOSTS]) def test_vendored_plugin_matches_the_shared_plaintext_policy(plugin: ModuleType, host: str) -> None: """Each plugin's vendored loopback check must agree with the shared transport contract.""" From 5d255df9b5a054b0976d19316b8994966c5af06d Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Tue, 25 Aug 2026 21:29:22 +0800 Subject: [PATCH 09/32] fix(test): canonicalise the shared vectors JSON and de-flake the CLI error assertions pretty-format-json rewrites the fixture with one array element per line; apply that formatting so the quality hook passes in CI. The friendly-error CLI tests asserted tokens (--host, the opt-in env var) against typer's rich error panel. Under GitHub Actions typer forces force_terminal=True and the runner's TERM=dumb pins rich to 80 columns while ignoring COLUMNS, so the panel hyphen-breaks --host and force-splits the long env var. Replace the COLUMNS override with a fixture that neutralises the forced terminal and sets an explicit wide width, making the assertions independent of the terminal environment. --- .../fixtures/transport_loopback_vectors.json | 15 +++++++++++-- tests/test_cli.py | 22 ++++++++++++++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/tests/fixtures/transport_loopback_vectors.json b/tests/fixtures/transport_loopback_vectors.json index 169566943d..25e643fbcb 100644 --- a/tests/fixtures/transport_loopback_vectors.json +++ b/tests/fixtures/transport_loopback_vectors.json @@ -1,5 +1,16 @@ { "_comment": "Shared transport-policy host vectors -- the single source of truth for the loopback contract. Consumed by the Python drift guard (tests/test_transport.py) and the Pi plugin's Vitest drift guard (integrations/pi/plugins/powercontext/tests/transport-policy.spec.ts). Every entry is a valid URL authority so both languages can build http://:8000: loopback hosts must be trusted for plaintext, non_loopback hosts must be refused plaintext. Keep this aligned with src/powercontext/transport.py.", - "loopback": ["127.0.0.1", "127.0.0.2", "127.1.2.3", "localhost", "LOCALHOST", "[::1]"], - "non_loopback": ["0.0.0.0", "memory.example", "192.168.1.10"] + "loopback": [ + "127.0.0.1", + "127.0.0.2", + "127.1.2.3", + "localhost", + "LOCALHOST", + "[::1]" + ], + "non_loopback": [ + "0.0.0.0", + "memory.example", + "192.168.1.10" + ] } diff --git a/tests/test_cli.py b/tests/test_cli.py index d92c2e6186..0b37f68585 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -228,17 +228,30 @@ def test_server_command_layers_partial_cli_overrides_over_environment_settings( tracing.shutdown.assert_called_once_with() +@pytest.fixture +def _wide_error_panel(monkeypatch: pytest.MonkeyPatch) -> None: + """Render typer's rich error panel wide enough that asserted tokens are never wrapped. + + Under GitHub Actions typer forces ``force_terminal=True``; combined with the runner's + ``TERM=dumb`` that pins rich's console to 80 columns and ignores ``COLUMNS`` (the dumb-terminal + branch returns before honouring it), which hyphen-breaks ``--host`` and force-splits the long + opt-in env var. Neutralise the forced terminal so the captured pipe reads as non-terminal (hence + non-dumb) and set an explicit width, so the panel stays on wide lines in any environment. + """ + + monkeypatch.setattr("typer.rich_utils.FORCE_TERMINAL", None) + monkeypatch.setattr("typer.rich_utils.MAX_WIDTH", 10_000) + + def test_server_command_rejects_an_unauthenticated_non_loopback_host_override( monkeypatch: pytest.MonkeyPatch, + _wide_error_panel: None, ) -> None: run_server = Mock() tracing = Mock() monkeypatch.setattr("powercontext.server.cli._run_server", run_server) monkeypatch.setattr("powercontext.server.cli.configure_server_logging", lambda _config: None) monkeypatch.setattr("powercontext.server.cli.configure_server_tracing", lambda _config: tracing) - # Keep the error panel on one line so the asserted tokens are not hard-wrapped by the terminal - # width (rich breaks on hyphens and force-splits long words on a narrow CI terminal). - monkeypatch.setenv("COLUMNS", "1000") result = CliRunner().invoke( create_cli([server_app]), @@ -255,6 +268,7 @@ def test_server_command_rejects_an_unauthenticated_non_loopback_host_override( def test_server_command_reports_a_friendly_error_when_auth_lacks_a_token( monkeypatch: pytest.MonkeyPatch, + _wide_error_panel: None, ) -> None: run_server = Mock() tracing = Mock() @@ -263,8 +277,6 @@ def test_server_command_reports_a_friendly_error_when_auth_lacks_a_token( monkeypatch.setattr("powercontext.server.cli.configure_server_tracing", lambda _config: tracing) monkeypatch.setenv("POWERCONTEXT_SERVER_AUTH_ENABLED", "true") monkeypatch.delenv("POWERCONTEXT_SERVER_AUTH_TOKEN", raising=False) - # Keep the error panel on one line so the asserted env-var tokens are not hard-wrapped (see above). - monkeypatch.setenv("COLUMNS", "1000") result = CliRunner().invoke(create_cli([server_app]), ["server", "run"]) From b33dcbe00ea1c15cd11e11aa69df3661b28ed50f Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Wed, 26 Aug 2026 14:10:44 +0800 Subject: [PATCH 10/32] fix(client): refuse all non-loopback plaintext unless the caller vouches for the transport The loopback guard only tripped when a bearer token was present, so an unauthenticated client would still ship Memory content -- carried in the request body -- over unencrypted non-loopback HTTP. Broaden the guard to reject *any* plaintext non-loopback request, and let a caller that knows its transport is secure opt in explicitly with `trust_transport_security=True`; supplying an `http_client` is not itself evidence of safety. Thread that opt-in through the LangGraph adapter's `shared_http_client` / `open_client`: a plain pooling client stays untrusted, while in-process ASGI harnesses declare trust. Update the in-process test harnesses accordingly. Also stop the remote-access guides from documenting a bare `server run --host 0.0.0.0`, which the bind policy now rejects; show the authenticated bind and the explicit unauthenticated opt-in instead, and add a command-level contract test so a copy-pasteable command that fails to start fails in CI rather than only for a reader. --- .../remote-access-implementation.md | 18 ++- .../remote-access-implementation.md | 16 ++- .../src/powercontext_langgraph/client.py | 24 +++- src/powercontext/client/client.py | 24 ++-- src/powercontext/server/handoff_picker.py | 6 +- tests/e2e/test_candidate_review.py | 6 +- tests/e2e/test_dsh_http_chain.py | 2 +- tests/e2e/test_external_skill_registry.py | 4 +- tests/e2e/test_reviewed_generation.py | 4 +- tests/e2e/test_runtime_server.py | 16 +-- .../test_scheduled_experience_incubation.py | 4 +- tests/e2e/test_server_cli.py | 4 +- tests/langgraph_adapter/test_recall.py | 4 +- tests/langgraph_adapter/test_tools.py | 2 +- tests/test_docs_remote_access_contract.py | 126 ++++++++++++++++++ tests/test_server_tracing.py | 2 +- tests/test_transport.py | 34 +++-- 17 files changed, 242 insertions(+), 54 deletions(-) create mode 100644 tests/test_docs_remote_access_contract.py diff --git a/docs/en/development/remote-access-implementation.md b/docs/en/development/remote-access-implementation.md index 71257f8bf0..e847fadc9d 100644 --- a/docs/en/development/remote-access-implementation.md +++ b/docs/en/development/remote-access-implementation.md @@ -19,12 +19,26 @@ uv run powercontext server run ``` The default listener is `127.0.0.1:8000`. SQLite data is stored in `powercontext.db`. Command options can override the -listener: +listener, but binding to a routable address is refused for an *unauthenticated* Server: enable bearer authentication +(recommended -- terminate TLS at a reverse proxy), or opt in explicitly when TLS is terminated upstream or the network +is otherwise controlled. ```bash -uv run powercontext server run --host 0.0.0.0 --port 8080 +# Recommended: authenticate the Server, then bind a routable address (put TLS in front in production). +POWERCONTEXT_SERVER_AUTH_ENABLED=true \ +POWERCONTEXT_SERVER_AUTH_TOKEN="replace-with-a-strong-token" \ + uv run powercontext server run --host 0.0.0.0 --port 8080 ``` +```bash +# Or, with TLS terminated upstream / a controlled network, opt in to an unauthenticated bind. +POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true \ + uv run powercontext server run --host 0.0.0.0 --port 8080 +``` + +Running `--host 0.0.0.0` without one of the above exits with an error rather than silently exposing an +unauthenticated Server. + The process opens its configured database, creates a scope-bound Builtin runtime, and closes owned database, inference, and scheduler resources during shutdown. diff --git a/docs/zh/development/remote-access-implementation.md b/docs/zh/development/remote-access-implementation.md index 4ca94f3eb4..d3f1b867e7 100644 --- a/docs/zh/development/remote-access-implementation.md +++ b/docs/zh/development/remote-access-implementation.md @@ -17,12 +17,24 @@ uv add "powercontext[cli,server]" uv run powercontext server run ``` -默认监听 `127.0.0.1:8000`，SQLite 数据保存在 `powercontext.db`。命令参数可以覆盖监听地址： +默认监听 `127.0.0.1:8000`，SQLite 数据保存在 `powercontext.db`。命令参数可以覆盖监听地址，但**未认证**的 Server 会拒绝 +绑定到可路由地址：请启用 bearer 认证（推荐——在反向代理处终止 TLS），或在 TLS 已由上游终止 / 网络受控时显式 opt-in。 ```bash -uv run powercontext server run --host 0.0.0.0 --port 8080 +# 推荐：先为 Server 启用认证，再绑定可路由地址（生产环境在前面加 TLS）。 +POWERCONTEXT_SERVER_AUTH_ENABLED=true \ +POWERCONTEXT_SERVER_AUTH_TOKEN="replace-with-a-strong-token" \ + uv run powercontext server run --host 0.0.0.0 --port 8080 ``` +```bash +# 或者，在 TLS 由上游终止 / 网络受控的前提下，显式 opt-in 到未认证绑定。 +POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true \ + uv run powercontext server run --host 0.0.0.0 --port 8080 +``` + +不带上述任一设置直接 `--host 0.0.0.0` 会以报错退出，而不会静默地暴露一个未认证的 Server。 + 进程会打开配置的 database，创建按 scope 隔离的 Builtin runtime，并在关闭时释放其持有的 database、inference 和 scheduler 资源。 diff --git a/integrations/langgraph/src/powercontext_langgraph/client.py b/integrations/langgraph/src/powercontext_langgraph/client.py index b8d925a666..dbcce69b16 100644 --- a/integrations/langgraph/src/powercontext_langgraph/client.py +++ b/integrations/langgraph/src/powercontext_langgraph/client.py @@ -30,8 +30,11 @@ from .settings import PowerContextLangGraphSettings # A shared HTTP client lets a long-running deployment reuse one connection pool across nodes and tools, and lets tests -# route requests to an in-process ASGI app. When set, per-operation clients borrow it and never close it. -_SHARED_HTTP_CLIENT: ContextVar[httpx.AsyncClient | None] = ContextVar( +# route requests to an in-process ASGI app. When set, per-operation clients borrow it and never close it. The bool is the +# caller's explicit vouch that the transport is secure despite an ``http://`` base URL (an in-process ASGI app, a Unix +# socket, or a TLS-terminating proxy); a plain pooling client is *not* trusted by default, so the loopback guard still +# refuses to send its requests over non-loopback plaintext. +_SHARED_HTTP_CLIENT: ContextVar[tuple[httpx.AsyncClient, bool] | None] = ContextVar( "powercontext_langgraph_http_client", default=None ) @@ -84,19 +87,30 @@ def open_client(config: ResolvedConfig) -> PowerContextClient: shared = _SHARED_HTTP_CLIENT.get() if shared is not None: - return PowerContextClient(config.base_url, token=config.token, http_client=shared) + client, trust_transport_security = shared + return PowerContextClient( + config.base_url, + token=config.token, + http_client=client, + trust_transport_security=trust_transport_security, + ) return PowerContextClient(config.base_url, token=config.token, timeout=config.timeout) @contextmanager -def shared_http_client(client: httpx.AsyncClient) -> Iterator[None]: +def shared_http_client(client: httpx.AsyncClient, *, trust_transport_security: bool = False) -> Iterator[None]: """Install a shared HTTP client for the duration of the block. Operations that borrow it use its timeout configuration, overriding the resolved ``PowerContextScope(timeout=...)`` / ``POWERCONTEXT_LANGGRAPH_TIMEOUT`` value. + + A plain pooling client keeps the default transport-safety guard, so it still refuses to send + requests over non-loopback plaintext. Pass ``trust_transport_security=True`` only when the + installed client's transport is secure despite an ``http://`` base URL -- an in-process ASGI app, + a Unix-domain socket, or a proxy that terminates TLS. """ - token = _SHARED_HTTP_CLIENT.set(client) + token = _SHARED_HTTP_CLIENT.set((client, trust_transport_security)) try: yield finally: diff --git a/src/powercontext/client/client.py b/src/powercontext/client/client.py index 25b75d4534..5108e19cf6 100644 --- a/src/powercontext/client/client.py +++ b/src/powercontext/client/client.py @@ -189,18 +189,20 @@ def __init__( trust_transport_security: bool = False, ) -> None: self._base_url = base_url.rstrip("/") - # Plaintext HTTP only carries a bearer token safely on loopback. When this facade opens the - # transport itself, ``base_url``'s scheme accurately reflects what crosses the wire. A - # caller-supplied ``http_client`` *may* instead own a transport whose ``http://`` label is - # only a routing token -- an in-process ASGI app, a Unix socket, or a TLS-terminating proxy - # -- but a plain pooling ``httpx.AsyncClient`` (e.g. the shared client the LangGraph adapter - # installs) is exactly as exposed as one we would open ourselves. Supplying a transport is - # therefore not evidence of safety: the guard stays on for caller-supplied transports too, - # and a caller that knows its transport is secure must say so explicitly via - # ``trust_transport_security`` rather than have safety inferred from the argument being set. + # Plaintext HTTP is only trusted on loopback -- for *any* request, not just an authenticated + # one. The request body itself carries Memory content, so a missing bearer token does not make + # an unencrypted non-loopback request safe. When this facade opens the transport itself, + # ``base_url``'s scheme accurately reflects what crosses the wire. A caller-supplied + # ``http_client`` *may* instead own a transport whose ``http://`` label is only a routing + # token -- an in-process ASGI app, a Unix socket, or a TLS-terminating proxy -- but a plain + # pooling ``httpx.AsyncClient`` (e.g. the shared client the LangGraph adapter installs) is + # exactly as exposed as one we would open ourselves. Supplying a transport is therefore not + # evidence of safety: the guard stays on for caller-supplied transports too, and a caller that + # knows its transport is secure must say so explicitly via ``trust_transport_security`` rather + # than have safety inferred from the argument being set. transport_trusted = http_client is not None and trust_transport_security - if token and not transport_trusted and is_plaintext_non_loopback(self._base_url): - raise ValueError("refusing to send a bearer token over unencrypted non-loopback HTTP") # noqa: TRY003 + if not transport_trusted and is_plaintext_non_loopback(self._base_url): + raise ValueError("refusing to send requests over unencrypted non-loopback HTTP") # noqa: TRY003 self._headers = {"Authorization": f"Bearer {token}"} if token else None self._owned_http_client: httpx.AsyncClient | None = None if http_client is None: diff --git a/src/powercontext/server/handoff_picker.py b/src/powercontext/server/handoff_picker.py index 11f52bcdac..9cebcbd028 100644 --- a/src/powercontext/server/handoff_picker.py +++ b/src/powercontext/server/handoff_picker.py @@ -85,7 +85,11 @@ class HandoffWorkstreamSelection(BaseModel): def register_handoff_workstream_picker(server: FastMCP, http_client: httpx.AsyncClient) -> None: """Register the Report-backed Workstream picker on an existing MCP server.""" - picker = _HandoffWorkstreamPicker(PowerContextClient("http://fastapi", http_client=http_client)) + # ``http_client`` is the Server's own in-process ASGI transport; ``http://fastapi`` is only a + # routing label, so vouch for it explicitly rather than have the loopback guard reject it. + picker = _HandoffWorkstreamPicker( + PowerContextClient("http://fastapi", http_client=http_client, trust_transport_security=True) + ) server.tool( picker.select, name="select_handoff_workstream", diff --git a/tests/e2e/test_candidate_review.py b/tests/e2e/test_candidate_review.py index b5d1248a4d..49ab5eac7b 100644 --- a/tests/e2e/test_candidate_review.py +++ b/tests/e2e/test_candidate_review.py @@ -104,7 +104,7 @@ async def scenario() -> None: base_url="http://testserver", ) as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) capabilities = await client.get_capabilities() captured = await client.capture_content_source( CaptureContentSourceRequest( @@ -261,7 +261,9 @@ async def __aenter__(self): transport=httpx.ASGITransport(app=app), base_url="http://testserver", ) - self._client = PowerContextClient("http://testserver", http_client=self._transport) + self._client = PowerContextClient( + "http://testserver", http_client=self._transport, trust_transport_security=True + ) return self async def __aexit__(self, *_args) -> None: diff --git a/tests/e2e/test_dsh_http_chain.py b/tests/e2e/test_dsh_http_chain.py index b83e84b1b5..ebf8a4fa17 100644 --- a/tests/e2e/test_dsh_http_chain.py +++ b/tests/e2e/test_dsh_http_chain.py @@ -51,7 +51,7 @@ async def scenario() -> None: base_url="http://testserver", ) as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) live = await client.get_liveness() ready = await client.get_readiness() remembered = await client.remember_memory( diff --git a/tests/e2e/test_external_skill_registry.py b/tests/e2e/test_external_skill_registry.py index ef17807063..f8e6a1aeae 100644 --- a/tests/e2e/test_external_skill_registry.py +++ b/tests/e2e/test_external_skill_registry.py @@ -70,7 +70,7 @@ async def scenario() -> None: base_url="http://testserver", ) as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) capabilities = await client.get_capabilities() assert capabilities.external_skill_registry is True assert capabilities.managed_skill_generation is False @@ -131,7 +131,7 @@ async def scenario() -> None: transport=httpx.ASGITransport(app=app), base_url="http://testserver", ) as transport: - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) registration = ( await client.scan_external_skills(ScanExternalSkillsRequest(scope_id="project:example")) ).registrations[0] diff --git a/tests/e2e/test_reviewed_generation.py b/tests/e2e/test_reviewed_generation.py index cfed444261..d886a864f6 100644 --- a/tests/e2e/test_reviewed_generation.py +++ b/tests/e2e/test_reviewed_generation.py @@ -69,7 +69,7 @@ async def scenario() -> None: transport=httpx.ASGITransport(app=app), base_url="http://testserver", ) as transport: - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) captured = await client.capture_content_source( CaptureContentSourceRequest( scope_id="project", @@ -120,7 +120,7 @@ async def scenario() -> None: transport=httpx.ASGITransport(app=app), base_url="http://testserver", ) as transport: - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) captured = await client.capture_content_source( CaptureContentSourceRequest( scope_id="project", diff --git a/tests/e2e/test_runtime_server.py b/tests/e2e/test_runtime_server.py index 61a6de486c..c72a8ce496 100644 --- a/tests/e2e/test_runtime_server.py +++ b/tests/e2e/test_runtime_server.py @@ -188,7 +188,7 @@ async def scenario() -> None: base_url="http://testserver", ) as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) readiness = await client.get_readiness() capabilities = await client.get_capabilities() captured = await client.capture_content_source( @@ -261,7 +261,7 @@ async def scenario() -> None: base_url="http://testserver", ) as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) await client.remember_memory(RememberMemoryRequest(scope_id=writer_scope, kind="fact", text=memory_text)) leaked = await client.list_memory_entries(ListMemoryEntriesRequest(scope_id=reader_scope)) await client.remember_memory( @@ -298,7 +298,7 @@ async def scenario() -> None: base_url="http://testserver", ) as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) readiness = await client.get_readiness() captured = await client.capture_content_source( CaptureContentSourceRequest( @@ -334,7 +334,7 @@ async def scenario() -> None: base_url="http://testserver", ) as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) capabilities = await client.get_capabilities() captured = await client.capture_content_source( CaptureContentSourceRequest( @@ -424,7 +424,7 @@ async def scenario() -> None: base_url="http://testserver", ) as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) project = await client.create_handoff_report_project( CreateHandoffReportProjectRequest(project_key="work-continuity", title="Work continuity") ) @@ -616,7 +616,7 @@ async def scenario() -> None: base_url="http://testserver", ) as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) capabilities = await client.get_capabilities() await client.capture_content_source( CaptureContentSourceRequest( @@ -669,7 +669,7 @@ async def scenario() -> None: base_url="http://testserver", ) as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) remembered = await client.remember_memory( RememberMemoryRequest( scope_id="project:powercontext", @@ -789,7 +789,7 @@ async def stale_revision() -> None: base_url="http://testserver", ) as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) remembered = await client.remember_memory( RememberMemoryRequest(scope_id="project", kind="decision", text="first") ) diff --git a/tests/e2e/test_scheduled_experience_incubation.py b/tests/e2e/test_scheduled_experience_incubation.py index 27fb61cba3..9e7cff2ff0 100644 --- a/tests/e2e/test_scheduled_experience_incubation.py +++ b/tests/e2e/test_scheduled_experience_incubation.py @@ -93,7 +93,7 @@ async def scenario() -> None: base_url="http://testserver", ) as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) captured = await client.capture_content_source( CaptureContentSourceRequest( scope_id=scope_id, @@ -123,7 +123,7 @@ async def scenario() -> None: base_url="http://testserver", ) as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) await asyncio.sleep(0.08) candidates = await _pending_experience(client, scope_id) assert len(candidates) == 1 diff --git a/tests/e2e/test_server_cli.py b/tests/e2e/test_server_cli.py index 72a160279b..131faa7d9e 100644 --- a/tests/e2e/test_server_cli.py +++ b/tests/e2e/test_server_cli.py @@ -48,7 +48,9 @@ async def __aenter__(self) -> Self: transport=httpx.ASGITransport(app=server_app), base_url="http://testserver", ) - self._sdk = PowerContextClient("http://testserver", http_client=self._http_client) + self._sdk = PowerContextClient( + "http://testserver", http_client=self._http_client, trust_transport_security=True + ) return self async def __aexit__( diff --git a/tests/langgraph_adapter/test_recall.py b/tests/langgraph_adapter/test_recall.py index d8a004cce3..31b1b73f40 100644 --- a/tests/langgraph_adapter/test_recall.py +++ b/tests/langgraph_adapter/test_recall.py @@ -95,10 +95,10 @@ async def driver() -> None: app.router.lifespan_context(app), httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://testserver") as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) from powercontext_langgraph.client import shared_http_client - with shared_http_client(transport): + with shared_http_client(transport, trust_transport_security=True): await scenario(client) asyncio.run(driver()) diff --git a/tests/langgraph_adapter/test_tools.py b/tests/langgraph_adapter/test_tools.py index 3d305b2dca..a1752814bd 100644 --- a/tests/langgraph_adapter/test_tools.py +++ b/tests/langgraph_adapter/test_tools.py @@ -64,7 +64,7 @@ async def driver() -> None: app.router.lifespan_context(app), httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://testserver") as transport, ): - with shared_http_client(transport): + with shared_http_client(transport, trust_transport_security=True): await scenario() asyncio.run(driver()) diff --git a/tests/test_docs_remote_access_contract.py b/tests/test_docs_remote_access_contract.py new file mode 100644 index 0000000000..6a0df04cb1 --- /dev/null +++ b/tests/test_docs_remote_access_contract.py @@ -0,0 +1,126 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Command-level contract for the documented ``server run`` invocations. + +``check-docs`` only builds the Markdown, so a copy-pasteable command that the bind policy now +rejects (e.g. ``server run --host 0.0.0.0`` with neither authentication nor the opt-in) would ship +green. This test extracts every ``powercontext server run`` command from the remote-access guides -- +their inline ``POWERCONTEXT_SERVER_*`` env and their ``--host`` / ``--port`` -- and drives each one +through the real CLI, so a documented command that fails to start fails here instead of only for a +reader who runs it. +""" + +from __future__ import annotations + +import os +import re +import shlex +from pathlib import Path +from unittest.mock import Mock + +import pytest +from typer.testing import CliRunner + +from powercontext.cli.app import create_cli +from powercontext.server.cli import app as server_app + +_DOCS_ROOT = Path(__file__).resolve().parent.parent / "docs" +_REMOTE_ACCESS_DOCS = ( + _DOCS_ROOT / "en" / "development" / "remote-access-implementation.md", + _DOCS_ROOT / "zh" / "development" / "remote-access-implementation.md", +) +_BASH_BLOCK = re.compile(r"```bash\n(.*?)```", re.DOTALL) +_ENV_ASSIGNMENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=") + + +class _DocumentedServerRun: + """A single documented ``server run`` command: its inline env and its CLI arguments.""" + + def __init__(self, *, doc: Path, env: dict[str, str], args: list[str], host: str, port: str | None) -> None: + self.doc = doc + self.env = env + self.args = args + self.host = host + self.port = port + + def __repr__(self) -> str: # Readable pytest ids. + return f"{self.doc.parent.parent.name}:{self.host}:{self.port or 'default'}" + + +def _parse_server_run_commands(doc: Path) -> list[_DocumentedServerRun]: + commands: list[_DocumentedServerRun] = [] + for block in _BASH_BLOCK.findall(doc.read_text(encoding="utf-8")): + # Rejoin shell line-continuations so an env-prefixed command spread over several lines is one + # logical command, then read each command line. + for line in block.replace("\\\n", " ").splitlines(): + if "powercontext server run" not in line: + continue + tokens = shlex.split(line, comments=True) + env: dict[str, str] = {} + index = 0 + while index < len(tokens) and _ENV_ASSIGNMENT.match(tokens[index]): + key, _, value = tokens[index].partition("=") + env[key] = value + index += 1 + rest = tokens[index:] + run_at = _index_of_server_run(rest) + args = rest[run_at + 2 :] + host = _option_value(args, "--host") or "127.0.0.1" + port = _option_value(args, "--port") + commands.append(_DocumentedServerRun(doc=doc, env=env, args=["server", "run", *args], host=host, port=port)) + return commands + + +def _index_of_server_run(tokens: list[str]) -> int: + for i in range(len(tokens) - 1): + if tokens[i] == "server" and tokens[i + 1] == "run": + return i + raise AssertionError(f"expected a 'server run' command in {tokens!r}") # noqa: TRY003 + + +def _option_value(args: list[str], name: str) -> str | None: + for i, token in enumerate(args): + if token == name and i + 1 < len(args): + return args[i + 1] + return None + + +_DOCUMENTED_COMMANDS = [command for doc in _REMOTE_ACCESS_DOCS for command in _parse_server_run_commands(doc)] + + +def test_docs_expose_a_non_loopback_bind_command() -> None: + # Guard the test's premise: if the guides stop documenting a routable bind, the policy coverage + # below silently evaporates. + assert any(command.host == "0.0.0.0" for command in _DOCUMENTED_COMMANDS) # noqa: S104 - documented bind. + + +@pytest.mark.parametrize("command", _DOCUMENTED_COMMANDS, ids=repr) +def test_documented_server_run_command_starts(command: _DocumentedServerRun, monkeypatch: pytest.MonkeyPatch) -> None: + run_server = Mock() + monkeypatch.setattr("powercontext.server.cli._run_server", run_server) + monkeypatch.setattr("powercontext.server.cli.configure_server_logging", lambda _config: None) + monkeypatch.setattr("powercontext.server.cli.configure_server_tracing", lambda _config: Mock()) + # Reproduce only the env the doc block sets, with no ambient POWERCONTEXT_SERVER_* leaking in. + for name in list(os.environ): + if name.startswith("POWERCONTEXT_SERVER_"): + monkeypatch.delenv(name, raising=False) + for name, value in command.env.items(): + monkeypatch.setenv(name, value) + + result = CliRunner().invoke(create_cli([server_app]), command.args) + + assert result.exit_code == 0, result.output + run_server.assert_called_once() + assert run_server.call_args.kwargs["host"] == command.host diff --git a/tests/test_server_tracing.py b/tests/test_server_tracing.py index 27c2983429..b2e7ee6d27 100644 --- a/tests/test_server_tracing.py +++ b/tests/test_server_tracing.py @@ -109,7 +109,7 @@ def respond(request: httpx.Request) -> httpx.Response: async def request_capabilities() -> None: async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: - client = PowerContextClient("http://testserver", http_client=http_client) + client = PowerContextClient("http://testserver", http_client=http_client, trust_transport_security=True) await client.get_capabilities() asyncio.run(request_capabilities()) diff --git a/tests/test_transport.py b/tests/test_transport.py index 2001722279..93fa6a2c22 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -116,10 +116,19 @@ def test_client_settings_accept_loopback_or_tls_urls(server_url: str) -> None: assert ClientSettings(server_url=server_url).server_url.startswith(("http://", "https://")) +def test_client_refuses_requests_over_non_loopback_plaintext() -> None: + # The public constructor is a transport surface in its own right: even without a bearer token the + # request body carries Memory content, so a plaintext non-loopback base URL must be refused. This + # is the ``capture_content_source`` probe from the #1319 acceptance -- the facade must not open + # such a transport itself. + with pytest.raises(ValueError, match="non-loopback"): + PowerContextClient("http://memory.example") + + def test_client_refuses_a_bearer_token_over_non_loopback_plaintext() -> None: transport = httpx.MockTransport(lambda request: httpx.Response(200, json={"status": "ok"})) with httpx.Client(transport=transport): # noqa: SIM117 - guard runs before any request. - with pytest.raises(ValueError, match="bearer token"): + with pytest.raises(ValueError, match="non-loopback"): PowerContextClient("http://memory.example", token="probe-token") # noqa: S106 - test credential. @@ -128,10 +137,11 @@ def test_client_allows_a_bearer_token_over_loopback_plaintext() -> None: assert client is not None -def test_client_allows_a_bearer_token_over_a_trusted_caller_supplied_transport() -> None: +def test_client_allows_a_trusted_caller_supplied_transport() -> None: # A caller-supplied http_client may own a transport whose ``http://`` label is only a routing # token (here an in-process ASGI/mock transport). The caller must vouch for it explicitly with - # ``trust_transport_security`` before the plaintext-token guard stands down. + # ``trust_transport_security`` before the plaintext guard stands down; this covers the + # authenticated e2e flow. http_client = httpx.AsyncClient( transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"status": "ok"})), ) @@ -151,7 +161,7 @@ def test_client_refuses_a_bearer_token_over_an_untrusted_caller_supplied_transpo http_client = httpx.AsyncClient( transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"status": "ok"})), ) - with pytest.raises(ValueError, match="bearer token"): + with pytest.raises(ValueError, match="non-loopback"): PowerContextClient( "http://memory.example", token="probe-token", # noqa: S106 - test credential. @@ -159,13 +169,15 @@ def test_client_refuses_a_bearer_token_over_an_untrusted_caller_supplied_transpo ) -def test_client_allows_an_unauthenticated_non_loopback_transport() -> None: - async def respond(request: httpx.Request) -> httpx.Response: - return httpx.Response(200, json={"status": "ok"}) - - http_client = httpx.AsyncClient(transport=httpx.MockTransport(respond)) - client = PowerContextClient("http://testserver", http_client=http_client) - assert client is not None +def test_client_refuses_an_unauthenticated_untrusted_non_loopback_transport() -> None: + # The guard is not gated on a token: an untrusted caller-supplied transport to a non-loopback + # plaintext host must be refused even with no credentials, because the Memory content in the body + # still crosses the wire in the clear. + http_client = httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"status": "ok"})) + ) + with pytest.raises(ValueError, match="non-loopback"): + PowerContextClient("http://memory.example", http_client=http_client) def test_server_rejects_an_unauthenticated_non_loopback_bind() -> None: From bc6009adfb877b8ffaccd4c74437b2696b0abe19 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Wed, 26 Aug 2026 22:35:10 +0800 Subject: [PATCH 11/32] fix(bub): let the operator vouch for a controlled non-loopback transport The hardened client guard now refuses plaintext non-loopback URLs unconditionally, which broke the e2e harness where agent containers reach the server over the private Compose bridge via host-gateway. Add an explicit POWERCONTEXT_BUB_TRUST_TRANSPORT_SECURITY opt-in (default false) that routes through the client's endorsed bypass: a caller-supplied httpx transport plus trust_transport_security=True. Enable it only in the e2e Compose harness, mirror the server-side ALLOW_UNAUTHENTICATED_NON_LOOPBACK opt-in shape, document the switch, and cover both the refusal default and the trusted construction. --- e2e/bub/README.md | 2 + e2e/bub/compose.yaml | 2 + e2e/bub/tests/test_bub_transport_trust.py | 91 +++++++++++++++++++ e2e/bub/uv.lock | 2 + integrations/bub/README.md | 1 + integrations/bub/pyproject.toml | 1 + .../bub/src/powercontext_bub/plugin.py | 28 +++++- 7 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 e2e/bub/tests/test_bub_transport_trust.py diff --git a/e2e/bub/README.md b/e2e/bub/README.md index 610084344b..3a10cdac76 100644 --- a/e2e/bub/README.md +++ b/e2e/bub/README.md @@ -90,6 +90,8 @@ Against an existing PowerContext Server, run the default `acceptance` category: ```bash export POWERCONTEXT_CLIENT_SERVER_URL=http://127.0.0.1:8000 export POWERCONTEXT_BUB_BASE_URL=http://host-gateway:8000 +# host-gateway is plaintext non-loopback; the harness network is private, so vouch for it explicitly. +export POWERCONTEXT_BUB_TRUST_TRANSPORT_SECURITY=true make harness-acceptance ``` diff --git a/e2e/bub/compose.yaml b/e2e/bub/compose.yaml index f6a537e224..289d668094 100644 --- a/e2e/bub/compose.yaml +++ b/e2e/bub/compose.yaml @@ -76,6 +76,8 @@ services: - POWERCONTEXT_CLIENT_TIMEOUT=150 - POWERCONTEXT_BUB_BASE_URL=http://host-gateway:8000 - POWERCONTEXT_BUB_TIMEOUT=150 + # The agent-to-server hop stays on the private Compose bridge, so the harness vouches for it. + - POWERCONTEXT_BUB_TRUST_TRANSPORT_SECURITY=true volumes: - ${POWERCONTEXT_E2E_OUTPUT}:/evidence - ${POWERCONTEXT_E2E_CODEX_AUTH_MOUNT:-/dev/null}:/codex/auth.json:ro diff --git a/e2e/bub/tests/test_bub_transport_trust.py b/e2e/bub/tests/test_bub_transport_trust.py new file mode 100644 index 0000000000..5bc8f7fd41 --- /dev/null +++ b/e2e/bub/tests/test_bub_transport_trust.py @@ -0,0 +1,91 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import httpx +import pytest +from powercontext_bub import plugin as plugin_module +from powercontext_bub.plugin import PowerContextPlugin, PowerContextSettings + + +def _plugin_with(settings: PowerContextSettings, monkeypatch, tmp_path: Path) -> PowerContextPlugin: + monkeypatch.setattr(plugin_module, "ensure_config", lambda _: settings) + return PowerContextPlugin(SimpleNamespace(workspace=tmp_path)) + + +def test_plugin_refuses_a_plaintext_non_loopback_server_by_default(monkeypatch, tmp_path: Path) -> None: + settings = PowerContextSettings(base_url="http://host-gateway:8000", scope_id="test:scope") + plugin = _plugin_with(settings, monkeypatch, tmp_path) + + async def prepare() -> None: + async with plugin._client(): + pass + + with pytest.raises(ValueError, match="non-loopback"): + asyncio.run(prepare()) + + +def test_trusting_the_transport_supplies_the_client_an_explicit_vouched_transport( + monkeypatch, + tmp_path: Path, +) -> None: + constructions: list[dict[str, Any]] = [] + + class RecordingClient: + def __init__( + self, + base_url: str, + *, + http_client: httpx.AsyncClient | None = None, + trust_transport_security: bool = False, + timeout: float | None = None, + ) -> None: + constructions.append({ + "base_url": base_url, + "http_client": http_client, + "trust_transport_security": trust_transport_security, + "timeout": timeout, + }) + + async def __aenter__(self) -> RecordingClient: + return self + + async def __aexit__(self, *exc_info: object) -> None: + del exc_info + + settings = PowerContextSettings( + base_url="http://host-gateway:8000", + scope_id="test:scope", + trust_transport_security=True, + ) + monkeypatch.setattr(plugin_module, "PowerContextClient", RecordingClient) + plugin = _plugin_with(settings, monkeypatch, tmp_path) + + async def open_client() -> None: + async with plugin._client(): + pass + + asyncio.run(open_client()) + + assert len(constructions) == 1 + construction = constructions[0] + assert construction["base_url"] == "http://host-gateway:8000" + assert isinstance(construction["http_client"], httpx.AsyncClient) + assert construction["trust_transport_security"] is True diff --git a/e2e/bub/uv.lock b/e2e/bub/uv.lock index 1dbccb5b3d..2a3e6b068a 100644 --- a/e2e/bub/uv.lock +++ b/e2e/bub/uv.lock @@ -1685,6 +1685,7 @@ version = "0.0.1" source = { editable = "../../integrations/bub" } dependencies = [ { name = "bub" }, + { name = "httpx" }, { name = "powercontext", extra = ["client"] }, { name = "pydantic-settings" }, ] @@ -1692,6 +1693,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "bub", specifier = ">=0.4.0,<0.5.0" }, + { name = "httpx", specifier = ">=0.28,<1" }, { name = "powercontext", extras = ["client"], specifier = ">=0.0.1" }, { name = "pydantic-settings", specifier = ">=2.7,<3" }, ] diff --git a/integrations/bub/README.md b/integrations/bub/README.md index d7a7ca8699..88cf5805d2 100644 --- a/integrations/bub/README.md +++ b/integrations/bub/README.md @@ -38,6 +38,7 @@ validated by Pydantic before the plugin starts. | `POWERCONTEXT_BUB_CAPTURE_CHECKPOINT_EVERY` | `5` | Flush Memory after this many captured events | | `POWERCONTEXT_BUB_CAPTURE_MAX_BYTES` | `8192` | Maximum UTF-8 bytes stored for one captured event | | `POWERCONTEXT_BUB_CAPTURE_LOG` | unset | Optional JSONL evidence path; records metadata but not event content | +| `POWERCONTEXT_BUB_TRUST_TRANSPORT_SECURITY` | `false` | Vouch for a plaintext non-loopback `BASE_URL` on an operator-controlled network (for example a private Compose bridge); otherwise such URLs are refused | Captured tool arguments redact values under credential-like keys. Known credential environment values are also removed from serialized event content. Keep the PowerContext scope and optional capture log protected because normal diff --git a/integrations/bub/pyproject.toml b/integrations/bub/pyproject.toml index 8c91506844..8c25ce2956 100644 --- a/integrations/bub/pyproject.toml +++ b/integrations/bub/pyproject.toml @@ -19,6 +19,7 @@ description = "Bub integration for PowerContext durable memory." requires-python = ">=3.12,<4.0" dependencies = [ "bub>=0.4.0,<0.5.0", + "httpx>=0.28,<1", "powercontext[client]>=0.0.1", "pydantic-settings>=2.7,<3", ] diff --git a/integrations/bub/src/powercontext_bub/plugin.py b/integrations/bub/src/powercontext_bub/plugin.py index 692458217c..03e4120429 100644 --- a/integrations/bub/src/powercontext_bub/plugin.py +++ b/integrations/bub/src/powercontext_bub/plugin.py @@ -20,11 +20,14 @@ import hashlib import json import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from dataclasses import replace from datetime import UTC, datetime from pathlib import Path from typing import Any +import httpx from bub import Settings, config, ensure_config, hookimpl from bub.hooks.interception import LlmCallRequest, LlmCallResult, ToolCall, ToolCallResult from bub.turn import TurnState @@ -65,6 +68,7 @@ class PowerContextSettings(Settings): capture_checkpoint_every: int = Field(default=5, ge=1, le=100) capture_max_bytes: int = Field(default=8192, ge=512, le=32768) capture_log: Path | None = None + trust_transport_security: bool = False class PowerContextPlugin: @@ -167,6 +171,24 @@ async def save_state(self, session_id: str, state: TurnState, message: Any, mode async with self._capture_lock: await self._flush_captured_sources(state, final=True) + @asynccontextmanager + async def _client(self) -> AsyncIterator[PowerContextClient]: + if self.settings.trust_transport_security: + # The operator vouched for the network (e.g. a private Compose bridge), and the + # client only honours that vouch for a caller-supplied transport. + async with ( + httpx.AsyncClient(timeout=self.settings.timeout) as transport, + PowerContextClient( + self.base_url, + http_client=transport, + trust_transport_security=True, + ) as client, + ): + yield client + return + async with PowerContextClient(self.base_url, timeout=self.settings.timeout) as client: + yield client + async def _prepare_context(self, query: str, state: TurnState) -> str | None: request = PrepareContextRequest( scope_id=self.scope_id, @@ -174,7 +196,7 @@ async def _prepare_context(self, query: str, state: TurnState) -> str | None: max_bytes=self.settings.max_bytes, ) try: - async with PowerContextClient(self.base_url, timeout=self.settings.timeout) as client: + async with self._client() as client: prepared = await client.prepare_context(request) except CLIENT_ERRORS as exc: state[STATE_KEY]["prepare_error"] = type(exc).__name__ @@ -223,7 +245,7 @@ async def _capture_event( }, ) try: - async with PowerContextClient(self.base_url, timeout=self.settings.timeout) as client: + async with self._client() as client: response = await client.capture_content_source(request) except CLIENT_ERRORS as exc: self._write_capture_record( @@ -254,7 +276,7 @@ async def _flush_captured_sources(self, state: TurnState, *, final: bool) -> Non return try: - async with PowerContextClient(self.base_url, timeout=self.settings.timeout) as client: + async with self._client() as client: response = await client.flush_memory(FlushMemoryRequest(scope_id=self.scope_id)) except CLIENT_ERRORS as exc: self._write_capture_record( From a40dd274e63ae52ba2a20aa2697e6125d5c90922 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Wed, 26 Aug 2026 22:44:40 +0800 Subject: [PATCH 12/32] fix(langchain): align the merged integrations with the transport-trust policy The LangChain middleware and pydantic-ai tests merged from master were written before the client started refusing all plaintext non-loopback URLs, so their in-process ASGI transports now hit the guard. Port the langgraph-style trust plumbing to the LangChain shared HTTP client and let the affected tests vouch for their ASGI transports with trust_transport_security=True. --- .../src/powercontext_langchain/client.py | 28 +++++++++++++++---- tests/e2e/test_pydantic_ai_chain.py | 6 ++-- tests/langchain_middleware/test_middleware.py | 4 +-- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/integrations/langchain/src/powercontext_langchain/client.py b/integrations/langchain/src/powercontext_langchain/client.py index b3e9fac2ef..cabc627d3e 100644 --- a/integrations/langchain/src/powercontext_langchain/client.py +++ b/integrations/langchain/src/powercontext_langchain/client.py @@ -29,7 +29,7 @@ from .scope import PowerContextScope, resolve_scope_id from .settings import PowerContextLangChainSettings -_SHARED_HTTP_CLIENT: ContextVar[httpx.AsyncClient | None] = ContextVar( +_SHARED_HTTP_CLIENT: ContextVar[tuple[httpx.AsyncClient, bool] | None] = ContextVar( "powercontext_langchain_http_client", default=None ) @@ -69,19 +69,35 @@ def _secret_value(secret: SecretStr | None) -> str | None: def open_client(config: ResolvedConfig) -> PowerContextClient: - """Return a client for one operation, borrowing a shared HTTP client when installed.""" + """Return a client for one operation, borrowing a shared HTTP client when installed. + + A borrowed client keeps its own timeout configuration, so ``config.timeout`` applies only + when no shared client is installed. + """ shared = _SHARED_HTTP_CLIENT.get() if shared is not None: - return PowerContextClient(config.base_url, token=config.token, http_client=shared) + client, trust_transport_security = shared + return PowerContextClient( + config.base_url, + token=config.token, + http_client=client, + trust_transport_security=trust_transport_security, + ) return PowerContextClient(config.base_url, token=config.token, timeout=config.timeout) @contextmanager -def shared_http_client(client: httpx.AsyncClient) -> Iterator[None]: - """Install a shared HTTP client for the duration of a block.""" +def shared_http_client(client: httpx.AsyncClient, *, trust_transport_security: bool = False) -> Iterator[None]: + """Install a shared HTTP client for the duration of a block. + + A plain pooling client keeps the default transport-safety guard, so it still refuses to send + requests over non-loopback plaintext. Pass ``trust_transport_security=True`` only when the + installed client's transport is secure despite an ``http://`` base URL -- an in-process ASGI app, + a Unix-domain socket, or a proxy that terminates TLS. + """ - token = _SHARED_HTTP_CLIENT.set(client) + token = _SHARED_HTTP_CLIENT.set((client, trust_transport_security)) try: yield finally: diff --git a/tests/e2e/test_pydantic_ai_chain.py b/tests/e2e/test_pydantic_ai_chain.py index ccbb674387..a5b7b58348 100644 --- a/tests/e2e/test_pydantic_ai_chain.py +++ b/tests/e2e/test_pydantic_ai_chain.py @@ -119,7 +119,7 @@ def __init__( timeout: float = 10, ) -> None: del timeout - super().__init__(base_url, token=token, http_client=transport) + super().__init__(base_url, token=token, http_client=transport, trust_transport_security=True) monkeypatch.setattr(toolset_module, "PowerContextClient", AsgiPowerContextClient) settings = PowerContextSettings( @@ -196,7 +196,7 @@ async def scenario() -> str: base_url="http://testserver", ) as transport, ): - seed_client = PowerContextClient("http://testserver", http_client=transport) + seed_client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) for index in range(20): await seed_client.capture_content_source( CaptureContentSourceRequest( @@ -216,7 +216,7 @@ def __init__( timeout: float = 10, ) -> None: del timeout - super().__init__(base_url, token=token, http_client=transport) + super().__init__(base_url, token=token, http_client=transport, trust_transport_security=True) monkeypatch.setattr(toolset_module, "PowerContextClient", AsgiPowerContextClient) capture_settings = PowerContextSettings( diff --git a/tests/langchain_middleware/test_middleware.py b/tests/langchain_middleware/test_middleware.py index 6ffd3d1fde..aade92392f 100644 --- a/tests/langchain_middleware/test_middleware.py +++ b/tests/langchain_middleware/test_middleware.py @@ -128,8 +128,8 @@ async def driver() -> None: app.router.lifespan_context(app), httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://testserver") as transport, ): - client = PowerContextClient("http://testserver", http_client=transport) - with shared_http_client(transport): + client = PowerContextClient("http://testserver", http_client=transport, trust_transport_security=True) + with shared_http_client(transport, trust_transport_security=True): await scenario(client) asyncio.run(driver()) From 142c2430f59c71d127ac894c1c7048e755f31fa1 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Wed, 26 Aug 2026 22:59:36 +0800 Subject: [PATCH 13/32] fix(bub): route the memory tools through the operator transport vouch The plugin hooks honoured POWERCONTEXT_BUB_TRUST_TRANSPORT_SECURITY, but the powercontext.search/remember/context tools still constructed bare clients, so the acceptance harness failed on the plaintext host-gateway hop. Hoist the vouched construction into a module-level open_client helper, carry the flag in the per-turn tool settings, and cover both tool paths in the trust tests. --- e2e/bub/tests/test_bub_transport_trust.py | 123 ++++++++++++++---- .../bub/src/powercontext_bub/plugin.py | 54 +++++--- .../bub/src/powercontext_bub/tools.py | 18 ++- 3 files changed, 147 insertions(+), 48 deletions(-) diff --git a/e2e/bub/tests/test_bub_transport_trust.py b/e2e/bub/tests/test_bub_transport_trust.py index 5bc8f7fd41..4be7aaf91b 100644 --- a/e2e/bub/tests/test_bub_transport_trust.py +++ b/e2e/bub/tests/test_bub_transport_trust.py @@ -17,12 +17,13 @@ import asyncio from pathlib import Path from types import SimpleNamespace -from typing import Any +from typing import Any, ClassVar import httpx import pytest from powercontext_bub import plugin as plugin_module -from powercontext_bub.plugin import PowerContextPlugin, PowerContextSettings +from powercontext_bub import tools as tools_module +from powercontext_bub.plugin import STATE_KEY, PowerContextPlugin, PowerContextSettings def _plugin_with(settings: PowerContextSettings, monkeypatch, tmp_path: Path) -> PowerContextPlugin: @@ -30,6 +31,10 @@ def _plugin_with(settings: PowerContextSettings, monkeypatch, tmp_path: Path) -> return PowerContextPlugin(SimpleNamespace(workspace=tmp_path)) +def _tool_settings(plugin: PowerContextPlugin) -> tools_module.ToolSettings: + return plugin.load_state(message=None, session_id="session-1")[STATE_KEY] + + def test_plugin_refuses_a_plaintext_non_loopback_server_by_default(monkeypatch, tmp_path: Path) -> None: settings = PowerContextSettings(base_url="http://host-gateway:8000", scope_id="test:scope") plugin = _plugin_with(settings, monkeypatch, tmp_path) @@ -42,34 +47,42 @@ async def prepare() -> None: asyncio.run(prepare()) +class RecordingClient: + constructions: ClassVar[list[dict[str, Any]]] = [] + + def __init__( + self, + base_url: str, + *, + http_client: httpx.AsyncClient | None = None, + trust_transport_security: bool = False, + timeout: float | None = None, + ) -> None: + RecordingClient.constructions.append({ + "base_url": base_url, + "http_client": http_client, + "trust_transport_security": trust_transport_security, + "timeout": timeout, + }) + + async def __aenter__(self) -> RecordingClient: + return self + + async def __aexit__(self, *exc_info: object) -> None: + del exc_info + + +@pytest.fixture +def constructions() -> Any: + RecordingClient.constructions.clear() + return RecordingClient.constructions + + def test_trusting_the_transport_supplies_the_client_an_explicit_vouched_transport( monkeypatch, tmp_path: Path, + constructions: list[dict[str, Any]], ) -> None: - constructions: list[dict[str, Any]] = [] - - class RecordingClient: - def __init__( - self, - base_url: str, - *, - http_client: httpx.AsyncClient | None = None, - trust_transport_security: bool = False, - timeout: float | None = None, - ) -> None: - constructions.append({ - "base_url": base_url, - "http_client": http_client, - "trust_transport_security": trust_transport_security, - "timeout": timeout, - }) - - async def __aenter__(self) -> RecordingClient: - return self - - async def __aexit__(self, *exc_info: object) -> None: - del exc_info - settings = PowerContextSettings( base_url="http://host-gateway:8000", scope_id="test:scope", @@ -89,3 +102,61 @@ async def open_client() -> None: assert construction["base_url"] == "http://host-gateway:8000" assert isinstance(construction["http_client"], httpx.AsyncClient) assert construction["trust_transport_security"] is True + + +def test_load_state_carries_the_transport_vouch_to_tool_settings(monkeypatch, tmp_path: Path) -> None: + refusing = _plugin_with( + PowerContextSettings(base_url="http://host-gateway:8000", scope_id="test:scope"), + monkeypatch, + tmp_path, + ) + assert _tool_settings(refusing)["trust_transport_security"] is False + + vouched = _plugin_with( + PowerContextSettings( + base_url="http://host-gateway:8000", + scope_id="test:scope", + trust_transport_security=True, + ), + monkeypatch, + tmp_path, + ) + assert _tool_settings(vouched)["trust_transport_security"] is True + + +def test_tools_refuse_a_plaintext_non_loopback_server_by_default(monkeypatch, tmp_path: Path) -> None: + settings = PowerContextSettings(base_url="http://host-gateway:8000", scope_id="test:scope") + tool_settings = _tool_settings(_plugin_with(settings, monkeypatch, tmp_path)) + + async def open_tool_client() -> None: + async with tools_module._client(tool_settings): + pass + + with pytest.raises(ValueError, match="non-loopback"): + asyncio.run(open_tool_client()) + + +def test_tools_honour_the_operator_transport_vouch( + monkeypatch, + tmp_path: Path, + constructions: list[dict[str, Any]], +) -> None: + settings = PowerContextSettings( + base_url="http://host-gateway:8000", + scope_id="test:scope", + trust_transport_security=True, + ) + tool_settings = _tool_settings(_plugin_with(settings, monkeypatch, tmp_path)) + monkeypatch.setattr(plugin_module, "PowerContextClient", RecordingClient) + + async def open_tool_client() -> None: + async with tools_module._client(tool_settings): + pass + + asyncio.run(open_tool_client()) + + assert len(constructions) == 1 + construction = constructions[0] + assert construction["base_url"] == "http://host-gateway:8000" + assert isinstance(construction["http_client"], httpx.AsyncClient) + assert construction["trust_transport_security"] is True diff --git a/integrations/bub/src/powercontext_bub/plugin.py b/integrations/bub/src/powercontext_bub/plugin.py index 04f6d53bb7..a963c0571b 100644 --- a/integrations/bub/src/powercontext_bub/plugin.py +++ b/integrations/bub/src/powercontext_bub/plugin.py @@ -20,7 +20,7 @@ import hashlib import json from collections.abc import AsyncIterator -from contextlib import asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import replace from datetime import UTC, datetime from pathlib import Path @@ -70,6 +70,34 @@ class PowerContextSettings(Settings): trust_transport_security: bool = False +def open_client( + base_url: str, + *, + timeout: float, + trust_transport_security: bool = False, +) -> AbstractAsyncContextManager[PowerContextClient]: + """Open a client, vouching for the transport only when the operator opted in.""" + + if trust_transport_security: + return _vouched_client(base_url, timeout) + return PowerContextClient(base_url, timeout=timeout) + + +@asynccontextmanager +async def _vouched_client(base_url: str, timeout_seconds: float) -> AsyncIterator[PowerContextClient]: + # The operator vouched for the network (e.g. a private Compose bridge), and the + # client only honours that vouch for a caller-supplied transport. + async with ( + httpx.AsyncClient(timeout=timeout_seconds) as transport, + PowerContextClient( + base_url, + http_client=transport, + trust_transport_security=True, + ) as client, + ): + yield client + + class PowerContextPlugin: """Bub hooks backed by the public PowerContext client.""" @@ -87,6 +115,7 @@ def load_state(self, message: Any, session_id: str) -> TurnState: "base_url": self.base_url, "scope_id": self.scope_id, "timeout": self.settings.timeout, + "trust_transport_security": self.settings.trust_transport_security, "capture_sequence": 0, "captured_events": 0, "captured_position": 0, @@ -170,23 +199,12 @@ async def save_state(self, session_id: str, state: TurnState, message: Any, mode async with self._capture_lock: await self._flush_captured_sources(state, final=True) - @asynccontextmanager - async def _client(self) -> AsyncIterator[PowerContextClient]: - if self.settings.trust_transport_security: - # The operator vouched for the network (e.g. a private Compose bridge), and the - # client only honours that vouch for a caller-supplied transport. - async with ( - httpx.AsyncClient(timeout=self.settings.timeout) as transport, - PowerContextClient( - self.base_url, - http_client=transport, - trust_transport_security=True, - ) as client, - ): - yield client - return - async with PowerContextClient(self.base_url, timeout=self.settings.timeout) as client: - yield client + def _client(self) -> AbstractAsyncContextManager[PowerContextClient]: + return open_client( + self.base_url, + timeout=self.settings.timeout, + trust_transport_security=self.settings.trust_transport_security, + ) async def _prepare_context(self, query: str, state: TurnState) -> str | None: request = PrepareContextRequest( diff --git a/integrations/bub/src/powercontext_bub/tools.py b/integrations/bub/src/powercontext_bub/tools.py index fa305a617f..556c688a22 100644 --- a/integrations/bub/src/powercontext_bub/tools.py +++ b/integrations/bub/src/powercontext_bub/tools.py @@ -17,6 +17,7 @@ from __future__ import annotations import json +from contextlib import AbstractAsyncContextManager from typing import Any, TypedDict, cast from bub import tool @@ -25,13 +26,14 @@ from powercontext.client import PowerContextClient from powercontext.http import PrepareContextRequest, RememberMemoryRequest, SearchMemoryRequest -from .plugin import STATE_KEY +from .plugin import STATE_KEY, open_client class ToolSettings(TypedDict): base_url: str scope_id: str timeout: float + trust_transport_security: bool class SearchInput(BaseModel): @@ -51,7 +53,7 @@ async def search_memory(param: SearchInput, *, context: Any) -> str: settings = _settings(context) request = SearchMemoryRequest(scope_id=settings["scope_id"], query=param.query, limit=param.limit) - async with PowerContextClient(settings["base_url"], timeout=settings["timeout"]) as client: + async with _client(settings) as client: response = await client.search_memory(request) if not response.hits: return "(no matching PowerContext memory)" @@ -81,7 +83,7 @@ async def remember_memory(param: RememberInput, *, context: Any) -> str: text=param.text, reason=param.reason, ) - async with PowerContextClient(settings["base_url"], timeout=settings["timeout"]) as client: + async with _client(settings) as client: response = await client.remember_memory(request) if response.entry is None: return "(PowerContext accepted the memory without an entry receipt)" @@ -94,7 +96,7 @@ async def prepare_context(query: str, *, context: Any) -> str: settings = _settings(context) request = PrepareContextRequest(scope_id=settings["scope_id"], query=query) - async with PowerContextClient(settings["base_url"], timeout=settings["timeout"]) as client: + async with _client(settings) as client: response = await client.prepare_context(request) return response.content or "(no relevant PowerContext context)" @@ -102,3 +104,11 @@ async def prepare_context(query: str, *, context: Any) -> str: def _settings(context: Any) -> ToolSettings: settings = context.state[STATE_KEY] return cast(ToolSettings, settings) + + +def _client(settings: ToolSettings) -> AbstractAsyncContextManager[PowerContextClient]: + return open_client( + settings["base_url"], + timeout=settings["timeout"], + trust_transport_security=settings["trust_transport_security"], + ) From be01f130de8a5cf94c3282eb656709ebbd4d9214 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Wed, 26 Aug 2026 23:28:16 +0800 Subject: [PATCH 14/32] docs(config): state that the client refuses all plaintext non-loopback requests The reference still described the pre-hardening guard, which only refused to attach a bearer token. Align both language variants with the shipped behaviour: every request over unencrypted non-loopback HTTP is refused, and callers whose http:// base URL merely labels a secure transport must supply their own http_client with an explicit trust_transport_security=True. --- docs/en/docs/reference/configuration.md | 6 ++++-- docs/zh/docs/reference/configuration.md | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/en/docs/reference/configuration.md b/docs/en/docs/reference/configuration.md index c60a3a46cc..4912b7b64b 100644 --- a/docs/en/docs/reference/configuration.md +++ b/docs/en/docs/reference/configuration.md @@ -64,8 +64,10 @@ upstream or the network is otherwise controlled, set `POWERCONTEXT_SERVER_ALLOW_ opt in explicitly. Use TLS before exposing an authenticated Server over a network. The Python Client and CLI apply the matching rule for outbound requests: a configured unencrypted `http://` Server URL is -accepted only for loopback hosts, and the Client refuses to send a bearer token over an unencrypted non-loopback -connection. +accepted only for loopback hosts, and the Client refuses to send any request -- authenticated or not -- over unencrypted +non-loopback HTTP. Code whose `http://` base URL is only a routing label for a transport that is secure in practice (an +in-process ASGI app, a Unix-domain socket, or a proxy that terminates TLS) must supply its own `http_client` and pass +`trust_transport_security=True` explicitly. The Dashboard is enabled by default and shares the Server listener and port with the HTTP API and MCP. With no scopes configured, the page shows an empty state. Dashboard initialization failures are logged with their direct cause and do diff --git a/docs/zh/docs/reference/configuration.md b/docs/zh/docs/reference/configuration.md index e20353d923..f3f255daa7 100644 --- a/docs/zh/docs/reference/configuration.md +++ b/docs/zh/docs/reference/configuration.md @@ -63,7 +63,9 @@ readiness endpoint 仍然公开。明文 HTTP 仅在 loopback 地址（`127.0.0. Server 前必须配置 TLS。 Python Client 和 CLI 对出站请求应用相同规则：配置的明文 `http://` Server URL 仅接受 loopback 主机，并且 Client 拒绝 -通过明文的非 loopback 连接发送 Bearer token。 +通过明文的非 loopback HTTP 发送任何请求——无论是否携带 Bearer token。当代码的 `http://` base URL 只是路由标签、 +实际传输是安全的（进程内 ASGI 应用、Unix domain socket、由代理终止 TLS）时，必须自行传入 `http_client` 并显式设置 +`trust_transport_security=True`。 Dashboard 默认启用，并与 HTTP API、MCP 共用监听地址和端口。默认未配置 scope，页面会显示空状态；Dashboard 初始化失败只记录包含直接原因的 warning，不影响 Server 的 HTTP API、MCP 和健康检查启动。 From e18883ebb98b74dfff750af44e1665b89f40bf55 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Thu, 27 Aug 2026 00:15:03 +0800 Subject: [PATCH 15/32] chore: retrigger CI after GitHub runner outage From aa42393a1c1d4cab9e75e538b61a44134db1a1c0 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Thu, 27 Aug 2026 02:18:53 +0800 Subject: [PATCH 16/32] docs: state the full 127.0.0.0/8 loopback range and fix a heading separator --- docs/en/docs/reference/configuration.md | 8 ++++---- docs/en/docs/reference/interfaces.md | 1 + docs/zh/docs/reference/configuration.md | 2 +- docs/zh/docs/reference/interfaces.md | 1 + 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/en/docs/reference/configuration.md b/docs/en/docs/reference/configuration.md index 4912b7b64b..cad8a58ac2 100644 --- a/docs/en/docs/reference/configuration.md +++ b/docs/en/docs/reference/configuration.md @@ -58,10 +58,10 @@ Server settings use the `POWERCONTEXT_SERVER_` prefix. Static bearer authentication is disabled by default. When enabled, API and MCP requests must include `Authorization: Bearer `; the liveness and readiness endpoints remain public. Plain HTTP is trusted only on a -loopback address (`127.0.0.1`, `localhost`, `::1`). The Server refuses to start when it binds to a non-loopback address -while authentication is disabled; either enable authentication, keep the bind on loopback, or, when TLS is terminated -upstream or the network is otherwise controlled, set `POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true` to -opt in explicitly. Use TLS before exposing an authenticated Server over a network. +loopback address (`localhost`, `::1`, or any address in `127.0.0.0/8`). The Server refuses to start when it binds to a +non-loopback address while authentication is disabled; either enable authentication, keep the bind on loopback, or, +when TLS is terminated upstream or the network is otherwise controlled, set +`POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true` to opt in explicitly. Use TLS before exposing an authenticated Server over a network. The Python Client and CLI apply the matching rule for outbound requests: a configured unencrypted `http://` Server URL is accepted only for loopback hosts, and the Client refuses to send any request -- authenticated or not -- over unencrypted diff --git a/docs/en/docs/reference/interfaces.md b/docs/en/docs/reference/interfaces.md index d16498d05f..59987d9def 100644 --- a/docs/en/docs/reference/interfaces.md +++ b/docs/en/docs/reference/interfaces.md @@ -80,6 +80,7 @@ Client and can automatically prepend bounded `PreparedContext`. Optional capture model and completed tool events, performs checkpoint Flush, and flushes remaining Sources after the run. MCP needs no adapter package but does not provide automatic context preparation, capture, or Flush. See [Configure Pydantic AI](../how-to/configure-pydantic-ai.md). + ## LangGraph adapter `powercontext-langgraph` connects a LangGraph graph to a running Server through the public Python Client. It supplies diff --git a/docs/zh/docs/reference/configuration.md b/docs/zh/docs/reference/configuration.md index f3f255daa7..9be49afb59 100644 --- a/docs/zh/docs/reference/configuration.md +++ b/docs/zh/docs/reference/configuration.md @@ -57,7 +57,7 @@ Server 配置使用 `POWERCONTEXT_SERVER_` 前缀。 | `POWERCONTEXT_SERVER_EXTERNAL_SKILLS` | 未设置 | 包含 host identity 和显式 Agent Skill targets 的 JSON object | 静态 Bearer 鉴权默认关闭。启用后，API 和 MCP 请求必须携带 `Authorization: Bearer `；liveness 和 -readiness endpoint 仍然公开。明文 HTTP 仅在 loopback 地址（`127.0.0.1`、`localhost`、`::1`）上受信任。当 Server 绑定到 +readiness endpoint 仍然公开。明文 HTTP 仅在 loopback 地址（`localhost`、`::1` 及 `127.0.0.0/8` 网段内的任意地址）上受信任。当 Server 绑定到 非 loopback 地址且鉴权关闭时会拒绝启动；此时应启用鉴权、改回绑定 loopback，或在 TLS 由上游终止或网络本身受控的场景下， 显式设置 `POWERCONTEXT_SERVER_ALLOW_UNAUTHENTICATED_NON_LOOPBACK=true` 主动选择接受。通过网络暴露启用鉴权的 Server 前必须配置 TLS。 diff --git a/docs/zh/docs/reference/interfaces.md b/docs/zh/docs/reference/interfaces.md index eaa08b4f35..a1ca194bfd 100644 --- a/docs/zh/docs/reference/interfaces.md +++ b/docs/zh/docs/reference/interfaces.md @@ -71,6 +71,7 @@ project-context skill 指导 DeepSeek Harness 何时检索、记忆、修订或 `PreparedContext`。可选 Capture 会保存经过清洗和限长的可见模型事件与已完成工具事件，执行 checkpoint Flush，并在 run 结束后 Flush 剩余 Source。MCP 不需要适配器包，但不提供自动 Context 准备、Capture 或 Flush。参见 [配置 Pydantic AI](../how-to/configure-pydantic-ai.md)。 + ## LangGraph 适配器 `powercontext-langgraph` 通过公开的 Python Client 把 LangGraph 图连接到运行中的 Server，提供三个组件： From ea1688ea75c5be9a98ff2d17a30fe106604c0f95 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Thu, 10 Sep 2026 22:35:51 +0800 Subject: [PATCH 17/32] feat: add local portable archive CLI --- docs/en/docs/how-to/portable-archive.md | 100 +++ docs/en/docs/index.md | 3 + docs/zh/docs/how-to/portable-archive.md | 92 +++ docs/zh/docs/index.md | 3 + pyproject.toml | 2 + .../builtin/portability/__init__.py | 17 + .../builtin/portability/bundle.py | 693 ++++++++++++++++++ .../builtin/runtime/application.py | 3 + .../builtin/runtime/composition.py | 1 + .../builtin/runtime/relational.py | 17 + src/powercontext/cli/archive.py | 102 +++ tests/builtin/portability/test_bundle.py | 392 ++++++++++ tests/e2e/test_portable_archive.py | 80 ++ tests/test_archive_cli.py | 48 ++ uv.lock | 14 +- zensical.toml | 2 + 16 files changed, 1568 insertions(+), 1 deletion(-) create mode 100644 docs/en/docs/how-to/portable-archive.md create mode 100644 docs/zh/docs/how-to/portable-archive.md create mode 100644 src/powercontext/builtin/portability/__init__.py create mode 100644 src/powercontext/builtin/portability/bundle.py create mode 100644 src/powercontext/cli/archive.py create mode 100644 tests/builtin/portability/test_bundle.py create mode 100644 tests/e2e/test_portable_archive.py create mode 100644 tests/test_archive_cli.py diff --git a/docs/en/docs/how-to/portable-archive.md b/docs/en/docs/how-to/portable-archive.md new file mode 100644 index 0000000000..016fa95f6a --- /dev/null +++ b/docs/en/docs/how-to/portable-archive.md @@ -0,0 +1,100 @@ +--- +title: Export and restore a portable archive +description: Create, validate, and restore a verified PowerContext logical bundle. +--- + +# Export and restore a portable archive + +A portable archive is a `.pcb` file for moving or recovering complete PowerContext scopes. It is a logical archive: +it preserves domain identities and immutable history rather than copying a SQLite file. Use it for a controlled local +backup, an offline transfer, or a future backend migration. + +This command is a local operator command. It opens the default SQLite database in the PowerContext user-data directory; +it does not call a remote Server API. Run it only on a machine whose local PowerContext data you control. + +## Prerequisites + +Install the CLI from the same PowerContext revision that created the local data. The archive command uses the persistent +SQLite database at `POWERCONTEXT_HOME/powercontext.db`; without `POWERCONTEXT_HOME`, PowerContext uses its operating +system user-data directory. + +```bash +uv tool install "powercontext[cli] @ git+https://github.com/oceanbase/powercontext.git@master" +export POWERCONTEXT_HOME="$HOME/.local/share/powercontext" +``` + +The archive contains content and metadata needed for recovery. Store it in a protected location with access controls +and retention appropriate for the underlying project data. The format deliberately omits credentials and configured +provider secrets, but it does not encrypt the project content. + +## Create and inspect an archive + +Pass every complete scope to include. A scope cannot be selected by prefix or partially by record type. + +```bash +powercontext archive export \ + --scope-id project:payments \ + --output ./backups/payments.pcb + +powercontext archive inspect ./backups/payments.pcb +``` + +`export` writes JSON containing the bundle ID, record count, and checksum. `inspect` verifies the ZIP structure, +per-record digests, total digest, record counts, and selected scopes without opening the target database. Keep the +reported checksum with the backup inventory if an external backup system needs an independent verification record. + +## Validate before a restore + +Always validate the archive against the destination first: + +```bash +powercontext archive restore ./backups/payments.pcb --dry-run +``` + +Dry-run performs no domain writes. It verifies checksums, required source and Artifact dependencies, and whether the +configured Runtime can restore the archive's source types and Artifact families. A validation failure exits with code +`2` and reports only a content-free reason; record bodies are not printed. + +To perform the write, repeat the command with explicit confirmation: + +```bash +powercontext archive restore ./backups/payments.pcb --yes +``` + +The result is JSON with `inserted`, `already_present`, and `projections_ready`. Treat the restore as ready for search +only when `projections_ready` is `true`. The Runtime rebuilds portable Memory and Experience search projections after +the authoritative rows have been restored. + +## What the bundle preserves + +Format version 1 carries the portable relational representation of these supported records: + +| Preserved | Not portable | +| --- | --- | +| Source journal heads and Source records | Search projections and indexes | +| Artifact Revisions, lineage, and heads | Source cursors and scheduler state | +| Memory entry versions and heads | External Skill registrations and host-local installation state | +| Candidate versions, decision heads, and their evidence references | Usage statistics, credentials, bearer tokens, and provider secrets | + +The archive has a versioned manifest (`format_version`, producer version, scopes, counts, exclusions, and total +checksum) plus NDJSON records. It is the authoritative round-trip format. CSV may be produced separately for bounded +analysis, but it cannot preserve immutable revisions, lineage, or evidence references and must not be used for restore. + +## Recovery and conflicts + +Restore is idempotent. Replaying the same archive recognizes identical records as `already_present`. A different +payload for an existing immutable identity never overwrites it: restore exits with code `3` and leaves the write +transaction rolled back. Resolve the conflicting target data or choose a clean destination, then run the same restore +command again. + +If the command is interrupted before completion, do not claim recovery succeeded. Authoritative records are +transactionally applied, so a failed write does not leave a successful-looking partial restore. Re-run the same archive +against the same destination after the previous command has stopped. + +## Current boundaries + +Export streams database rows to a temporary NDJSON file. Validation stores only record identities in a temporary +on-disk index, and restore replays dependency levels as streams, so aggregate archive payloads are not retained in +process memory. The local CLI always opens the default SQLite database under `POWERCONTEXT_HOME` (or the operating +system user-data directory); it is not a remote Server API and does not use Server authentication or non-SQLite +database settings. For a live production database, use the database provider's coordinated backup procedures. diff --git a/docs/en/docs/index.md b/docs/en/docs/index.md index dcdbc9fbc9..6c80f1eae9 100644 --- a/docs/en/docs/index.md +++ b/docs/en/docs/index.md @@ -48,6 +48,9 @@ overview: - title: Create a managed Skill description: Generate and review a managed Skill, then export one exact Revision to Codex. href: en/docs/how-to/create-and-export-skill/ + - title: Portable archive + description: Create a verified local logical backup, validate it, and restore complete scopes. + href: en/docs/how-to/portable-archive/ - title: Troubleshoot description: Diagnose connection, configuration, and integration problems. href: en/docs/how-to/troubleshoot/ diff --git a/docs/zh/docs/how-to/portable-archive.md b/docs/zh/docs/how-to/portable-archive.md new file mode 100644 index 0000000000..64f317e1f2 --- /dev/null +++ b/docs/zh/docs/how-to/portable-archive.md @@ -0,0 +1,92 @@ +--- +title: 导出和恢复可移植归档 +description: 创建、验证并恢复经过校验的 PowerContext 逻辑归档。 +--- + +# 导出和恢复可移植归档 + +可移植归档是用于迁移或恢复完整 PowerContext scope 的 `.pcb` 文件。它是逻辑归档：保存领域 identity 和不可变历史， +而不是复制 SQLite 文件。它适用于受控的本地备份、离线传输，以及未来的后端迁移。 + +这是一个本地运维命令。它直接打开 PowerContext 用户数据目录中的默认 SQLite 数据库，不调用远程 Server API。只应在 +你能够控制本机 PowerContext 数据的环境中运行。 + +## 前提条件 + +从创建本地数据的同一个 PowerContext revision 安装 CLI。归档命令使用 +`POWERCONTEXT_HOME/powercontext.db` 中持久化的 SQLite 数据库；未设置 `POWERCONTEXT_HOME` 时，PowerContext +会使用操作系统用户数据目录。 + +```bash +uv tool install "powercontext[cli] @ git+https://github.com/oceanbase/powercontext.git@master" +export POWERCONTEXT_HOME="$HOME/.local/share/powercontext" +``` + +归档包含恢复所需的内容和元数据。应按项目数据的访问控制和保留要求，将它放在受保护的位置。格式会排除凭据和已配置的 +provider secret，但不会加密项目内容。 + +## 创建并检查归档 + +传入每一个需要包含的完整 scope。不能用前缀选择 scope，也不能只导出某一种 record。 + +```bash +powercontext archive export \ + --scope-id project:payments \ + --output ./backups/payments.pcb + +powercontext archive inspect ./backups/payments.pcb +``` + +`export` 会输出 JSON，其中包括 bundle ID、record 数和 checksum。`inspect` 不打开目标数据库，只校验 ZIP 结构、 +单 record digest、总 digest、record 数和所选 scope。若外部备份系统需要独立校验记录，请把输出的 checksum 一并保存。 + +## 在恢复前验证 + +恢复前始终先针对目标环境验证： + +```bash +powercontext archive restore ./backups/payments.pcb --dry-run +``` + +dry-run 不会写入领域数据。它会校验 checksum、必需的 Source 和 Artifact 依赖，以及已配置 Runtime 是否支持归档中的 +source type 和 Artifact family。验证失败会以退出码 `2` 结束，只输出不含内容的原因，不会打印 record body。 + +要执行写恢复，必须显式确认： + +```bash +powercontext archive restore ./backups/payments.pcb --yes +``` + +结果为 JSON，包含 `inserted`、`already_present` 和 `projections_ready`。只有 `projections_ready` 为 `true` +时才能宣称搜索已就绪。Runtime 会在恢复权威数据后重建可移植的 Memory 和 Experience 搜索 projection。 + +## 归档保留的内容 + +格式版本 1 支持以下可移植关系记录： + +| 会保留 | 不可移植 | +| --- | --- | +| Source journal head 和 Source record | 搜索 projection 和 index | +| Artifact Revision、lineage 和 head | Source cursor 和 scheduler state | +| Memory entry version 和 head | External Skill registration 和 host-local installation state | +| Candidate version、decision head 及其 evidence reference | Usage statistic、credential、bearer token 和 provider secret | + +归档包含带版本的 manifest（`format_version`、producer version、scope、数量、排除项和总 checksum）以及 NDJSON record。 +它是权威的 round-trip 格式。CSV 可以单独用于有边界的分析，但无法保留不可变 revision、lineage 或 evidence reference， +不得用于恢复。 + +## 恢复和冲突处理 + +恢复是幂等的。重复回放同一归档时，内容相同的 record 会计入 `already_present`。已有不可变 identity 对应不同 payload +时，恢复绝不会覆盖它：命令以退出码 `3` 结束，并回滚本次写 transaction。解决目标数据冲突，或选择一个干净的目标后， +再次运行同一个恢复命令。 + +如果命令在完成前中断，不能宣称恢复已成功。权威 record 会在 transaction 中写入；失败的写入不会留下看似成功的部分 +恢复。先确认上一次命令已经停止，再对同一目标重新运行同一个归档。 + +## 当前边界 + +导出会把数据库 row 流式写到临时 NDJSON 文件。验证只在临时磁盘 index 中保存 record identity，恢复则按依赖层流式 +回放，因此归档的聚合 payload 不会常驻进程内存。本地 CLI 始终打开 `POWERCONTEXT_HOME`（或操作系统用户数据目录） +下的默认 SQLite 数据库；它不是远程 Server API，也不会读取 Server 认证或非 SQLite 数据库配置。生产数据库仍应使用 +数据库 provider 的协调备份流程。 diff --git a/docs/zh/docs/index.md b/docs/zh/docs/index.md index 0382bc5f4a..6a4243dad2 100644 --- a/docs/zh/docs/index.md +++ b/docs/zh/docs/index.md @@ -48,6 +48,9 @@ overview: - title: 创建 managed Skill description: 生成并审核 managed Skill，再将一个精确 Revision 导出给 Codex。 href: zh/docs/how-to/create-and-export-skill/ + - title: 可移植归档 + description: 创建经过校验的本地逻辑备份，验证后恢复完整 scope。 + href: zh/docs/how-to/portable-archive/ - title: 排查问题 description: 诊断连接、配置和集成问题。 href: zh/docs/how-to/troubleshoot/ diff --git a/pyproject.toml b/pyproject.toml index 89cd74e0ed..55744a31d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ tracing-otlp = [ ] cli = [ "platformdirs>=4,<5", + "powercontext[builtin]", "powercontext[client]", "typer>=0.16,<1", ] @@ -83,6 +84,7 @@ powercontext = "powercontext.cli.app:main" doctor = "powercontext.cli.system:doctor_app" server = "powercontext.server.cli:app" setup = "powercontext.cli.system:setup_app" +archive = "powercontext.cli.archive:archive_app" [project.urls] Homepage = "https://github.com/oceanbase/powercontext" diff --git a/src/powercontext/builtin/portability/__init__.py b/src/powercontext/builtin/portability/__init__.py new file mode 100644 index 0000000000..9265306519 --- /dev/null +++ b/src/powercontext/builtin/portability/__init__.py @@ -0,0 +1,17 @@ +"""Versioned logical export and restore for built-in relational data.""" + +from powercontext.builtin.portability.bundle import ( + BundleConflictError, + BundleFormatError, + BundleInspection, + BundleReceipt, + PortableBundleService, +) + +__all__ = [ + "BundleConflictError", + "BundleFormatError", + "BundleInspection", + "BundleReceipt", + "PortableBundleService", +] diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py new file mode 100644 index 0000000000..b9759d1e3c --- /dev/null +++ b/src/powercontext/builtin/portability/bundle.py @@ -0,0 +1,693 @@ +"""Portable, verified logical bundles for built-in relational storage. + +The archive deliberately contains domain rows only. Database files, search +indexes, trigger cursors, usage statistics, and host-local skill registrations +are deployment details and are not portable state. +""" + +# Error reason strings are the public inspection/validation contract. +# ruff: noqa: TRY003 + +from __future__ import annotations + +import base64 +import hashlib +import json +import re +import sqlite3 +import tempfile +import zipfile +from collections import Counter +from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import date, datetime +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any, Literal, cast +from uuid import uuid4 + +from sqlalchemy import insert, select +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.tables import ( + ARTIFACT_CANDIDATE_HEADS_TABLE, + ARTIFACT_CANDIDATE_VERSIONS_TABLE, + ARTIFACT_HEADS_TABLE, + ARTIFACT_LINEAGE_ARTIFACTS_TABLE, + ARTIFACT_LINEAGE_SOURCES_TABLE, + ARTIFACTS_TABLE, + MEMORY_ENTRY_HEADS_TABLE, + MEMORY_ENTRY_VERSIONS_TABLE, + SOURCE_JOURNAL_HEADS_TABLE, + SOURCES_TABLE, +) + +FORMAT_VERSION = 1 +_RECORDS_NAME = "records.ndjson" +_MANIFEST_NAME = "manifest.json" +_SHA256 = "sha256:" +_MAX_ARCHIVE_BYTES = 512 * 1024 * 1024 +_MAX_RECORDS_BYTES = 2 * 1024 * 1024 * 1024 +_MAX_RECORDS = 2_000_000 +_MAX_LINE_BYTES = 4 * 1024 * 1024 +_HEX_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") + +RecordType = Literal[ + "source_journal_head", + "source", + "artifact_revision", + "artifact_lineage_source", + "artifact_lineage_artifact", + "artifact_head", + "memory_entry_version", + "memory_entry_head", + "candidate_version", + "candidate_head", +] +ProjectionRebuilder = Callable[[tuple[str, ...]], Awaitable[None]] + + +class BundleFormatError(ValueError): + """The supplied archive is malformed, corrupt, or unsupported.""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + + +class BundleConflictError(ValueError): + """A target immutable identity exists with different canonical content.""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + + +@dataclass(frozen=True, slots=True) +class BundleInspection: + """Content-free facts established by parsing and verifying a bundle.""" + + bundle_id: str + scopes: tuple[str, ...] + record_count: int + records_by_type: Mapping[str, int] + total_digest: str + format_version: int + + +@dataclass(frozen=True, slots=True) +class BundleReceipt: + """Content-free result of an export or restore.""" + + bundle_id: str + record_count: int + total_digest: str + inserted: int = 0 + already_present: int = 0 + projections_ready: bool = False + + +@dataclass(frozen=True, slots=True) +class _Record: + record_type: RecordType + identity: Mapping[str, str | int] + payload: Mapping[str, Any] + digest: str + + +@dataclass(frozen=True, slots=True) +class _ParsedBundle: + inspection: BundleInspection + + +_TABLES: dict[RecordType, Any] = { + "source_journal_head": SOURCE_JOURNAL_HEADS_TABLE, + "source": SOURCES_TABLE, + "artifact_revision": ARTIFACTS_TABLE, + "artifact_lineage_source": ARTIFACT_LINEAGE_SOURCES_TABLE, + "artifact_lineage_artifact": ARTIFACT_LINEAGE_ARTIFACTS_TABLE, + "artifact_head": ARTIFACT_HEADS_TABLE, + "memory_entry_version": MEMORY_ENTRY_VERSIONS_TABLE, + "memory_entry_head": MEMORY_ENTRY_HEADS_TABLE, + "candidate_version": ARTIFACT_CANDIDATE_VERSIONS_TABLE, + "candidate_head": ARTIFACT_CANDIDATE_HEADS_TABLE, +} + +# Parent rows always precede their dependent rows. This matters for SQLite as +# well as for MySQL/OceanBase foreign-key enforcement. +_EXPORT_ORDER: tuple[RecordType, ...] = tuple(_TABLES) + + +class PortableBundleService: + """Export and restore complete scopes without depending on a DB dialect. + + The service is intentionally application-facing rather than a database-file + utility. Callers must perform authorization before calling :meth:`export`; + records are never enumerated before the scope arguments are validated. + """ + + def __init__( + self, + database: AsyncDatabase, + /, + *, + projection_rebuilder: ProjectionRebuilder | None = None, + supported_source_types: Iterable[str] | None = None, + supported_artifact_families: Iterable[str] | None = None, + ) -> None: + self._database = database + self._projection_rebuilder = projection_rebuilder + self._supported_source_types = None if supported_source_types is None else frozenset(supported_source_types) + self._supported_artifact_families = ( + None if supported_artifact_families is None else frozenset(supported_artifact_families) + ) + + async def export(self, scopes: Iterable[str], output: Path, /) -> BundleReceipt: + """Write a verified archive containing all portable records for scopes. + + A single read transaction provides a consistent snapshot on supported + relational backends. The resulting ZIP is written only after the + snapshot has been completely read, so a failed export never presents a + partially finalized archive at ``output``. + """ + + selected = _validate_scopes(scopes) + output.parent.mkdir(parents=True, exist_ok=True) + records_path = _temporary_path(output.parent, suffix=".records.ndjson") + archive_path = _temporary_path(output.parent, suffix=".pcb") + try: + with records_path.open("wb") as stream: + async with self._database.transaction() as connection: + count, records_by_type, digest = await self._stream_export(connection, selected, stream) + bundle_id = str(uuid4()) + manifest = { + "format_version": FORMAT_VERSION, + "bundle_id": bundle_id, + "producer": {"name": "powercontext", "version": _producer_version()}, + "scopes": list(selected), + "record_count": count, + "records_by_type": dict(sorted(records_by_type.items())), + "excluded_record_classes": [ + "search_projections", + "source_cursors", + "external_skill_registrations", + "usage_statistics", + ], + "total_digest": digest, + } + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: + archive.write(records_path, _RECORDS_NAME) + archive.writestr(_MANIFEST_NAME, _canonical_json(manifest)) + archive_path.replace(output) + return BundleReceipt(bundle_id=bundle_id, record_count=count, total_digest=digest) + finally: + records_path.unlink(missing_ok=True) + archive_path.unlink(missing_ok=True) + + async def inspect(self, source: Path, /) -> BundleInspection: + """Verify structure and checksums without touching the target database.""" + + return _parse_bundle(source).inspection + + async def validate( + self, source: Path, /, *, supported_source_types: Iterable[str] | None = None + ) -> BundleInspection: + """Verify bundle integrity, dependencies, and optional adapter support.""" + + parsed = _parse_bundle(source) + _validate_dependencies(source) + configured = self._supported_source_types if supported_source_types is None else set(supported_source_types) + if configured is not None: + supported = set(configured) + missing = sorted( + { + str(record.identity["source_type"]) + for record in _iter_records(source) + if record.record_type == "source" + } + - supported + ) + if missing: + raise BundleFormatError(f"target does not support source types: {', '.join(missing)}") + if self._supported_artifact_families is not None: + required_families = _required_artifact_families(_iter_records(source)) + missing_families = sorted(required_families - self._supported_artifact_families) + if missing_families: + raise BundleFormatError(f"target does not support artifact families: {', '.join(missing_families)}") + return parsed.inspection + + async def restore( + self, + source: Path, + /, + *, + supported_source_types: Iterable[str] | None = None, + ) -> BundleReceipt: + """Restore a validated bundle atomically, skipping identical records. + + Existing rows are compared using the same portable record encoding. A + difference at the same primary identity aborts the entire transaction; + no existing revision or head is overwritten. + """ + + parsed = _parse_bundle(source) + await self.validate(source, supported_source_types=supported_source_types) + async with self._database.transaction() as connection: + inserted, existing = await self._write_records(connection, source) + projections_ready = False + if self._projection_rebuilder is not None: + await self._projection_rebuilder(parsed.inspection.scopes) + projections_ready = True + return BundleReceipt( + bundle_id=parsed.inspection.bundle_id, + record_count=parsed.inspection.record_count, + total_digest=parsed.inspection.total_digest, + inserted=inserted, + already_present=existing, + projections_ready=projections_ready, + ) + + async def _write_records(self, connection: AsyncConnection, source: Path, /) -> tuple[int, int]: + """Replay records in dependency order inside the caller transaction.""" + inserted = 0 + existing = 0 + # A portable bundle may be produced by another implementation, so do + # not trust its physical order. Re-scan once per dependency level + # instead of materializing its records merely to sort them. + for record_type in _EXPORT_ORDER: + for record in _iter_records(source): + if record.record_type != record_type: + continue + table = _TABLES[record.record_type] + where = [table.c[key] == value for key, value in record.identity.items()] + row = (await connection.execute(select(table).where(*where))).mappings().one_or_none() + if row is not None: + present = _record_from_row(record.record_type, cast(Mapping[str, Any], dict(row))) + if present.digest != record.digest: + raise BundleConflictError( + f"immutable identity conflict: {record.record_type} {dict(record.identity)!r}" + ) + existing += 1 + continue + await connection.execute(insert(table).values(**_row_values(record))) + inserted += 1 + return inserted, existing + + async def _stream_export( + self, + connection: AsyncConnection, + scopes: tuple[str, ...], + stream: Any, + ) -> tuple[int, Counter[str], str]: + count = 0 + records_by_type: Counter[str] = Counter() + digest = _DigestAccumulator() + for record_type in _EXPORT_ORDER: + table = _TABLES[record_type] + statement = select(table).where(table.c.scope_id.in_(scopes)).order_by(*table.primary_key.columns) + rows = await connection.stream(statement) + async for row in rows.mappings(): + record = _record_from_row(record_type, cast(Mapping[str, Any], dict(row))) + stream.write(_canonical_json(_record_document(record)) + b"\n") + digest.add(record.digest) + count += 1 + records_by_type[record.record_type] += 1 + return count, records_by_type, digest.value() + + +def _record_from_row(record_type: RecordType, row: Mapping[str, Any]) -> _Record: + table = _TABLES[record_type] + identity = {column.name: _json_value(row[column.name]) for column in table.primary_key.columns} + payload = { + column.name: _json_value(row[column.name]) + for column in table.columns + if column.name not in identity and column.name not in {"searchable_text"} + } + canonical = {"record_type": record_type, "schema_version": 1, "identity": identity, "payload": payload} + return _Record(record_type=record_type, identity=identity, payload=payload, digest=_digest(canonical)) + + +def _record_document(record: _Record) -> dict[str, Any]: + return { + "record_type": record.record_type, + "schema_version": 1, + "identity": dict(record.identity), + "payload": dict(record.payload), + "digest": record.digest, + } + + +def _row_values(record: _Record) -> dict[str, Any]: + values = {key: _database_value(value) for key, value in record.identity.items()} + values.update({key: _database_value(value) for key, value in record.payload.items()}) + # Projections are intentionally excluded from logical bundles. Heads may + # use NULL searchable text; Memory projections are rebuilt by the runtime. + if record.record_type == "artifact_head": + values["searchable_text"] = None + elif record.record_type == "memory_entry_head": + values["searchable_text"] = str(record.payload["entry_content_hash"]) + return values + + +def _parse_bundle(source: Path) -> _ParsedBundle: + try: + with zipfile.ZipFile(source) as archive: + _validate_container(archive) + manifest = json.loads(archive.read(_MANIFEST_NAME)) + _validate_manifest(manifest) + except (OSError, json.JSONDecodeError, zipfile.BadZipFile) as error: + raise BundleFormatError("cannot read portable bundle") from error + count = 0 + counts: Counter[str] = Counter() + digest = _DigestAccumulator() + for record in _iter_records(source): + count += 1 + counts[record.record_type] += 1 + digest.add(record.digest) + expected_count = manifest["record_count"] + if count != expected_count: + raise BundleFormatError("manifest record count does not match records") + if digest.value() != manifest["total_digest"]: + raise BundleFormatError("bundle total digest does not match records") + sorted_counts = dict(sorted(counts.items())) + if sorted_counts != manifest["records_by_type"]: + raise BundleFormatError("manifest record classes do not match records") + inspection = BundleInspection( + bundle_id=manifest["bundle_id"], + scopes=tuple(manifest["scopes"]), + record_count=count, + records_by_type=sorted_counts, + total_digest=digest.value(), + format_version=manifest["format_version"], + ) + return _ParsedBundle(inspection=inspection) + + +def _iter_records(source: Path, /) -> Iterator[_Record]: + """Yield verified records without retaining the archive body in memory.""" + + try: + with zipfile.ZipFile(source) as archive, archive.open(_RECORDS_NAME) as stream: + count = 0 + for line in stream: + if len(line) > _MAX_LINE_BYTES: + raise BundleFormatError("record line exceeds size limit") + if not line.strip(): + continue + count += 1 + if count > _MAX_RECORDS: + raise BundleFormatError("record count exceeds limit") + yield _parse_record(json.loads(line)) + except (OSError, json.JSONDecodeError, zipfile.BadZipFile) as error: + raise BundleFormatError("cannot read portable bundle") from error + + +def _validate_container(archive: zipfile.ZipFile) -> None: + names = {item.filename for item in archive.infolist()} + if names != {_MANIFEST_NAME, _RECORDS_NAME}: + raise BundleFormatError("bundle must contain only manifest.json and records.ndjson") + if any( + item.is_dir() or item.file_size > _MAX_RECORDS_BYTES or item.compress_size > _MAX_ARCHIVE_BYTES + for item in archive.infolist() + ): + raise BundleFormatError("bundle exceeds resource limits") + if archive.getinfo(_RECORDS_NAME).file_size > _MAX_RECORDS_BYTES: + raise BundleFormatError("records exceed resource limit") + + +def _validate_manifest(manifest: object) -> None: + if not isinstance(manifest, dict): + raise BundleFormatError("manifest must be an object") + manifest = cast(dict[str, object], manifest) + required = {"format_version", "bundle_id", "scopes", "record_count", "records_by_type", "total_digest"} + if not required <= set(manifest): + raise BundleFormatError("manifest misses required fields") + if manifest["format_version"] != FORMAT_VERSION: + raise BundleFormatError("unsupported bundle format version") + if not isinstance(manifest["bundle_id"], str) or not isinstance(manifest["record_count"], int): + raise BundleFormatError("manifest has invalid identity or record count") + _validate_scopes(cast(Iterable[str], manifest["scopes"])) + if not isinstance(manifest["records_by_type"], dict) or not _valid_digest(manifest["total_digest"]): + raise BundleFormatError("manifest has invalid checksums") + + +def _parse_record(value: object) -> _Record: + if not isinstance(value, dict): + raise BundleFormatError("record must be an object") + value = cast(dict[str, object], value) + record_type = value.get("record_type") + if record_type not in _TABLES or value.get("schema_version") != 1: + raise BundleFormatError("unsupported record type or schema") + identity = value.get("identity") + payload = value.get("payload") + digest = value.get("digest") + if not isinstance(identity, dict) or not isinstance(payload, dict) or not isinstance(digest, str): + raise BundleFormatError("record has invalid fields") + identity = cast(dict[str, str | int], identity) + payload = cast(dict[str, Any], payload) + typed = cast(RecordType, record_type) + expected = {column.name for column in _TABLES[typed].primary_key.columns} + if set(identity) != expected or not _valid_digest(digest): + raise BundleFormatError("record has invalid identity or digest") + canonical = {"record_type": typed, "schema_version": 1, "identity": identity, "payload": payload} + if _digest(canonical) != digest: + raise BundleFormatError("record digest does not match content") + return _Record(record_type=typed, identity=identity, payload=payload, digest=digest) + + +def _validate_dependencies(source: Path, /) -> None: + """Check references with a temporary on-disk identity index. + + The index deliberately stores only canonical identities, never record + payloads. This permits dependency validation for large bundles without + retaining their content in the importing process. + """ + + with _identity_index(source) as contains: + for record in _iter_records(source): + _validate_record_dependencies(record, contains) + + +def _validate_record_dependencies( + record: _Record, + contains: Callable[[RecordType, Mapping[str, object]], bool], + /, +) -> None: + payload = record.payload + identity = record.identity + scope = identity["scope_id"] + if record.record_type == "source": + if not contains("source_journal_head", {"scope_id": scope}): + raise BundleFormatError("source has no journal head") + elif record.record_type in {"artifact_lineage_source"}: + if not contains( + "source", + {"scope_id": scope, "source_type": payload["source_type"], "source_id": payload["source_id"]}, + ): + raise BundleFormatError("artifact lineage references missing source") + elif record.record_type in {"artifact_lineage_artifact", "memory_entry_version"}: + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": payload.get("upstream_family", payload.get("family")), + "artifact_id": payload.get("upstream_artifact_id", identity.get("memory_artifact_id")), + "revision": payload.get("upstream_revision", payload.get("created_in_revision")), + }, + ): + raise BundleFormatError("record references missing artifact revision") + elif record.record_type == "artifact_head": + if not contains("artifact_revision", {**identity, "revision": payload["revision"]}): + raise BundleFormatError("artifact head references missing revision") + elif record.record_type == "memory_entry_head" and not contains( + "memory_entry_version", + { + "scope_id": scope, + "memory_artifact_id": identity["memory_artifact_id"], + "entry_version_id": payload["entry_version_id"], + }, + ): + raise BundleFormatError("memory head references missing entry version") + _validate_embedded_references((record,), contains) + + +@contextmanager +def _identity_index(source: Path, /) -> Iterator[Callable[[RecordType, Mapping[str, object]], bool]]: + index_path = _temporary_path(Path(tempfile.gettempdir()), suffix=".identities.sqlite3") + connection: sqlite3.Connection | None = None + try: + connection = sqlite3.connect(index_path) + connection.execute( + "CREATE TABLE identities (record_type TEXT NOT NULL, identity TEXT NOT NULL, " + "PRIMARY KEY (record_type, identity))" + ) + for record in _iter_records(source): + try: + connection.execute( + "INSERT INTO identities (record_type, identity) VALUES (?, ?)", + (record.record_type, _identity_key(record.identity)), + ) + except sqlite3.IntegrityError as error: + raise BundleFormatError("bundle contains duplicate immutable identity") from error + connection.commit() + + def contains(record_type: RecordType, identity: Mapping[str, object]) -> bool: + return ( + connection.execute( + "SELECT 1 FROM identities WHERE record_type = ? AND identity = ?", + (record_type, _identity_key(identity)), + ).fetchone() + is not None + ) + + yield contains + finally: + if connection is not None: + connection.close() + index_path.unlink(missing_ok=True) + + +def _validate_embedded_references( + records: Iterable[_Record], + contains: Callable[[RecordType, Mapping[str, object]], bool], +) -> None: + for record in records: + if record.record_type not in {"memory_entry_version", "candidate_version"}: + continue + scope_id = record.identity["scope_id"] + for reference in _reference_items(record.payload["source_refs"], "source"): + if not contains( + "source", + { + "scope_id": scope_id, + "source_type": reference["source_type"], + "source_id": reference["source_id"], + }, + ): + raise BundleFormatError("record references missing source") + for reference in _reference_items(record.payload["artifact_refs"], "artifact"): + if not contains( + "artifact_revision", + { + "scope_id": scope_id, + "family": reference["family"], + "artifact_id": reference["artifact_id"], + "revision": reference["revision"], + }, + ): + raise BundleFormatError("record references missing artifact revision") + + +def _reference_items(value: object, kind: Literal["source", "artifact"], /) -> tuple[Mapping[str, object], ...]: + encoded = cast(dict[str, object], value).get("base64") if isinstance(value, dict) else None + if not isinstance(value, dict) or set(value) != {"base64"} or not isinstance(encoded, str): + raise BundleFormatError(f"{kind} references are not encoded bytes") + try: + decoded = json.loads(base64.b64decode(encoded, validate=True)) + except (ValueError, json.JSONDecodeError) as error: + raise BundleFormatError(f"{kind} references are not valid JSON") from error + if not isinstance(decoded, list): + raise BundleFormatError(f"{kind} references must be an array") + fields = {"source_type", "source_id"} if kind == "source" else {"family", "artifact_id", "revision"} + references: list[Mapping[str, object]] = [] + for item in decoded: + if not isinstance(item, dict) or not fields <= set(item): + raise BundleFormatError(f"{kind} reference has invalid fields") + references.append(cast(Mapping[str, object], item)) + return tuple(references) + + +def _required_artifact_families(records: Iterable[_Record]) -> set[str]: + families: set[str] = set() + for record in records: + identity = record.identity + payload = record.payload + if record.record_type in { + "artifact_revision", + "artifact_lineage_source", + "artifact_lineage_artifact", + "artifact_head", + }: + families.add(str(identity["family"])) + if record.record_type in {"artifact_lineage_artifact", "candidate_version", "candidate_head"}: + for field in ("upstream_family", "target_family", "result_family"): + value = payload.get(field) + if value is not None: + families.add(str(value)) + if record.record_type in {"memory_entry_version", "memory_entry_head", "candidate_version", "candidate_head"}: + families.add(str(payload["family"])) + return families + + +def _validate_scopes(scopes: Iterable[str]) -> tuple[str, ...]: + selected = tuple(sorted(set(scopes))) + if not selected or any( + not isinstance(scope, str) or not scope.strip() or scope != scope.strip() for scope in selected + ): + raise ValueError("scopes must contain at least one non-empty trimmed scope ID") + return selected + + +def _identity_key(identity: Mapping[str, object], /) -> str: + return _canonical_json(identity).decode("utf-8") + + +class _DigestAccumulator: + """Incrementally hash a deterministic ordered record-digest sequence.""" + + def __init__(self) -> None: + self._hash = hashlib.sha256() + self._first = True + + def add(self, digest: str, /) -> None: + if not self._first: + self._hash.update(b"\n") + self._hash.update(digest.encode("ascii")) + self._first = False + + def value(self) -> str: + return _SHA256 + self._hash.hexdigest() + + +def _temporary_path(directory: Path, /, *, suffix: str) -> Path: + with tempfile.NamedTemporaryFile( + prefix=".powercontext-archive-", suffix=suffix, dir=directory, delete=False + ) as handle: + return Path(handle.name) + + +def _digest(value: object) -> str: + return _SHA256 + hashlib.sha256(_canonical_json(value)).hexdigest() + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8") + + +def _json_value(value: object) -> Any: + if isinstance(value, bytes | bytearray | memoryview): + return {"base64": base64.b64encode(bytes(value)).decode("ascii")} + if isinstance(value, datetime | date): + return value.isoformat() + return value + + +def _database_value(value: object) -> object: + encoded = cast(dict[str, object], value).get("base64") if isinstance(value, dict) else None + if isinstance(value, dict) and set(value) == {"base64"} and isinstance(encoded, str): + try: + return base64.b64decode(encoded, validate=True) + except ValueError as error: + raise BundleFormatError("invalid base64 column value") from error + return value + + +def _valid_digest(value: object) -> bool: + return isinstance(value, str) and _HEX_DIGEST.fullmatch(value) is not None + + +def _producer_version() -> str: + try: + return version("powercontext") + except PackageNotFoundError: + return "unknown" diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index 94eb7e8eef..7bc8b62fd4 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -176,6 +176,7 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler from powercontext.builtin.handoff_report.application import HandoffReportApplication + from powercontext.builtin.portability import PortableBundleService logger = logging.getLogger(__name__) @@ -1194,6 +1195,7 @@ def __init__( readiness: RuntimeReadinessChecks | None = None, clock: Clock | None = None, tracing: RuntimeTracing | None = None, + archive_service: PortableBundleService | None = None, ) -> None: if source_window_limit < 1: raise _RuntimeConfigurationError("source_window_limit") @@ -1237,6 +1239,7 @@ def __init__( self.review = ReviewApplication(self) self.skill = SkillApplication(self) self.statistics = StatisticsApplication(self) + self.archive = archive_service self.handoff_report: HandoffReportApplication | None = None self.processor = None if scope_ids is None else ScheduledSourceProcessor(self, scope_ids) self.experience_processor = ( diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index 7c623b068a..f43fcd34d0 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -281,6 +281,7 @@ async def open_builtin_runtime( recall_token_estimator=contexts.estimate_recall_tokens, readiness=RuntimeReadinessChecks(readiness_probes), tracing=tracing, + archive_service=contexts.portability, ) ) if config.handoff_report.enabled: diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 534a828f5b..9288a22855 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -77,6 +77,7 @@ from powercontext.builtin.persistence.sources import SourceRepository, StoredSource from powercontext.builtin.persistence.statistics import StatisticsRepository from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, SOURCE_JOURNAL_HEADS_TABLE +from powercontext.builtin.portability import PortableBundleService from powercontext.builtin.review.generation import ( GeneratedCandidateResult, GenerationCapabilityUnavailableError, @@ -305,6 +306,12 @@ def __init__( memory_artifact_id: str = "memory", ) -> None: self.database = database + self.portability = PortableBundleService( + database, + projection_rebuilder=self.rebuild_portable_projections, + supported_source_types=tuple(adapter.name for adapter in _SOURCE_ADAPTERS), + supported_artifact_families=(Handoff.family, Memory.family, Experience.family, Skill.family), + ) self.index = NoMemoryIndex() if index is None else index self.experience_index = NoExperienceIndex() if experience_index is None else experience_index self.repositories = _Repositories( @@ -458,6 +465,16 @@ async def handoff_scope_ids(self) -> tuple[str, ...]: ).scalars() return tuple(str(value) for value in values) + async def rebuild_portable_projections(self, scope_ids: tuple[str, ...], /) -> None: + """Rebuild target-local search projections after a logical restore.""" + + for scope_id in scope_ids: + services = self._services_for(scope_id) + _, catalog = services.sources() + await services.memory(catalog).rebuild_projections() + async with self.database.transaction() as connection: + await self.experience_index.initialize(connection) + async def incubate_experience(self, scope_id: str, limit: int, /) -> ExperienceIncubationResult: """Process one independent Task Outcome Source window for Review.""" diff --git a/src/powercontext/cli/archive.py b/src/powercontext/cli/archive.py new file mode 100644 index 0000000000..5da586d16a --- /dev/null +++ b/src/powercontext/cli/archive.py @@ -0,0 +1,102 @@ +"""Operator-facing portable bundle commands using a controlled built-in Runtime.""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import Awaitable, Callable +from pathlib import Path +from typing import Annotated, TypeVar + +import typer + +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.portability import ( + BundleConflictError, + BundleFormatError, + BundleInspection, + BundleReceipt, + PortableBundleService, +) +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_runtime +from powercontext.paths import default_database_path, default_scheduler_path, sqlite_url + +HELP_OPTION_NAMES = ("-h", "--help") +_ResultT = TypeVar("_ResultT", BundleInspection, BundleReceipt) + +archive_app = typer.Typer( + name="archive", + context_settings={"help_option_names": HELP_OPTION_NAMES}, + help="Create, inspect, validate, and restore portable logical bundles.", + no_args_is_help=True, +) + + +@archive_app.command("export") +def export_bundle( + scope_id: Annotated[ + list[str], typer.Option("--scope-id", min=1, help="Complete scope to export; repeat as needed.") + ], + output: Annotated[Path, typer.Option("--output", help="New portable bundle file to create.")], +) -> None: + """Create a verified logical bundle from complete scopes.""" + + _emit_archive_result(lambda archive: archive.export(scope_id, output)) + + +@archive_app.command("inspect") +def inspect_bundle( + source: Annotated[Path, typer.Argument(help="Portable bundle file to inspect.")], +) -> None: + """Verify archive structure and checksums without writing domain data.""" + + _emit_archive_result(lambda archive: archive.inspect(source)) + + +@archive_app.command("restore") +def restore_bundle( + source: Annotated[Path, typer.Argument(help="Portable bundle file to validate or restore.")], + dry_run: Annotated[bool, typer.Option("--dry-run", help="Validate only; never write domain data.")] = False, + yes: Annotated[bool, typer.Option("--yes", help="Confirm a write restore.")] = False, +) -> None: + """Validate or restore a portable bundle into the configured deployment.""" + + if dry_run: + _emit_archive_result(lambda archive: archive.validate(source)) + return + if not yes: + raise typer.BadParameter( # noqa: TRY003 + "restore writes immutable records; re-run with --yes to confirm", + param_hint="--yes", + ) + _emit_archive_result(lambda archive: archive.restore(source)) + + +async def _run_archive(operation: Callable[[PortableBundleService], Awaitable[_ResultT]], /) -> _ResultT: + config = BuiltinConfig(database=SQLiteConfig(url=sqlite_url(default_database_path()))) + async with open_builtin_runtime(config, scheduler_path=default_scheduler_path()) as runtime: + if runtime.archive is None: + raise RuntimeError("portable archive service is unavailable") # noqa: TRY003 + return await operation(runtime.archive) + + +def _emit_archive_result(operation: Callable[[PortableBundleService], Awaitable[_ResultT]], /) -> None: + """Render expected archive failures without leaking stack traces or records.""" + + try: + _emit(asyncio.run(_run_archive(operation))) + except BundleFormatError as error: + typer.echo(f"error: archive validation failed: {error}", err=True) + raise typer.Exit(code=2) from error + except BundleConflictError as error: + typer.echo(f"error: archive restore conflict: {error}", err=True) + raise typer.Exit(code=3) from error + + +def _emit(value: BundleInspection | BundleReceipt, /) -> None: + payload = ( + value.__dict__ + if hasattr(value, "__dict__") + else {field: getattr(value, field) for field in value.__dataclass_fields__} + ) + typer.echo(json.dumps(payload, default=list, sort_keys=True)) diff --git a/tests/builtin/portability/test_bundle.py b/tests/builtin/portability/test_bundle.py new file mode 100644 index 0000000000..6d9e10f15b --- /dev/null +++ b/tests/builtin/portability/test_bundle.py @@ -0,0 +1,392 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +"""Observable contracts for portable logical bundles.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import tracemalloc +import zipfile +from pathlib import Path + +import pytest +from sqlalchemy import delete, insert, select + +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import ( + ARTIFACT_CANDIDATE_HEADS_TABLE, + ARTIFACT_CANDIDATE_VERSIONS_TABLE, + ARTIFACT_HEADS_TABLE, + ARTIFACTS_TABLE, + BUILTIN_TABLES, + SOURCE_JOURNAL_HEADS_TABLE, + SOURCES_TABLE, +) +from powercontext.builtin.portability import BundleConflictError, BundleFormatError, PortableBundleService + + +def test_bundle_round_trips_authoritative_scope_data_and_excludes_projections(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=b'{"name":"source-1"}', + journal_position=1, + ) + ) + await connection.execute( + insert(ARTIFACTS_TABLE).values( + scope_id="project:one", + family="handoff", + artifact_id="handoff", + revision=1, + content=b'{"content":{"summary":"hello"}}', + ) + ) + await connection.execute( + insert(ARTIFACT_HEADS_TABLE).values( + scope_id="project:one", + family="handoff", + artifact_id="handoff", + revision=1, + searchable_text="not portable", + ) + ) + receipt = await PortableBundleService(source.database).export(["project:one"], archive) + assert receipt.record_count == 4 + + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + service = PortableBundleService(target.database) + inspection = await service.validate(archive, supported_source_types=("content",)) + assert inspection.scopes == ("project:one",) + restored = await service.restore(archive, supported_source_types=("content",)) + assert restored.inserted == 4 + repeated = await service.restore(archive, supported_source_types=("content",)) + assert repeated.already_present == 4 + async with target.database.transaction() as connection: + assert await connection.scalar(select(ARTIFACT_HEADS_TABLE.c.searchable_text)) is None + + asyncio.run(scenario()) + + +def test_restore_rejects_divergent_immutable_identity_without_overwrite(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=b"original", + journal_position=1, + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive) + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + async with target.database.transaction() as connection: + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=b"different", + journal_position=1, + ) + ) + service = PortableBundleService(target.database) + with pytest.raises(BundleConflictError): + await service.restore(archive) + async with target.database.transaction() as connection: + assert await connection.scalar(select(SOURCES_TABLE.c.payload)) == b"different" + + asyncio.run(scenario()) + + +def test_failed_restore_rolls_back_earlier_records_and_can_be_retried(tmp_path: Path) -> None: + """A later immutable conflict must not strand a partially restored scope.""" + + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=b"new source", + journal_position=1, + ) + ) + await connection.execute( + insert(ARTIFACTS_TABLE).values( + scope_id="project:one", + family="handoff", + artifact_id="handoff", + revision=1, + content=b"archive revision", + ) + ) + await connection.execute( + insert(ARTIFACT_HEADS_TABLE).values( + scope_id="project:one", family="handoff", artifact_id="handoff", revision=1 + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive) + + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + async with target.database.transaction() as connection: + await connection.execute( + insert(ARTIFACTS_TABLE).values( + scope_id="project:one", + family="handoff", + artifact_id="handoff", + revision=1, + content=b"conflicting revision", + ) + ) + service = PortableBundleService(target.database) + with pytest.raises(BundleConflictError): + await service.restore(archive) + + # Source rows precede revisions in the archive, yet none can survive + # the failed transaction. + async with target.database.transaction() as connection: + assert await connection.scalar(select(SOURCES_TABLE.c.source_id)) is None + await connection.execute(delete(ARTIFACTS_TABLE)) + + retried = await service.restore(archive) + assert retried.inserted == 4 + + asyncio.run(scenario()) + + +def test_restore_reports_projection_readiness_from_runtime_rebuilder(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await PortableBundleService(source.database).export(["project:one"], archive) + rebuilt: list[tuple[str, ...]] = [] + + async def rebuild(scopes: tuple[str, ...]) -> None: + rebuilt.append(scopes) + + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + receipt = await PortableBundleService(target.database, projection_rebuilder=rebuild).restore(archive) + assert rebuilt == [("project:one",)] + assert receipt.projections_ready is True + + asyncio.run(scenario()) + + +def test_restore_accepts_a_valid_bundle_with_records_in_a_different_physical_order(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "ordered.pcb" + reordered = tmp_path / "reordered.pcb" + database = tmp_path / "source.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{database}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=b"source", + journal_position=1, + ) + ) + await connection.execute( + insert(ARTIFACTS_TABLE).values( + scope_id="project:one", + family="handoff", + artifact_id="handoff", + revision=1, + content=b"artifact", + ) + ) + await connection.execute( + insert(ARTIFACT_HEADS_TABLE).values( + scope_id="project:one", family="handoff", artifact_id="handoff", revision=1 + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive) + + with zipfile.ZipFile(archive) as input_archive: + manifest = json.loads(input_archive.read("manifest.json")) + records = list(reversed(input_archive.read("records.ndjson").splitlines())) + digests = [json.loads(record)["digest"] for record in records] + manifest["total_digest"] = "sha256:" + hashlib.sha256("\n".join(digests).encode()).hexdigest() + with zipfile.ZipFile(reordered, "w") as output_archive: + output_archive.writestr("manifest.json", json.dumps(manifest)) + output_archive.writestr("records.ndjson", b"\n".join(records) + b"\n") + + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + receipt = await PortableBundleService(target.database).restore(reordered) + assert receipt.inserted == 4 + + asyncio.run(scenario()) + + +def test_restore_keeps_large_bundle_payloads_out_of_process_memory(tmp_path: Path) -> None: + """The import budget is independent of aggregate NDJSON payload size.""" + + async def scenario() -> None: + archive = tmp_path / "large.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + payload = b"x" * 8_192 + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1_500)) + await connection.execute( + insert(SOURCES_TABLE), + [ + { + "scope_id": "project:one", + "source_type": "content", + "source_id": f"source-{number}", + "payload": payload, + "journal_position": number, + } + for number in range(1, 1_501) + ], + ) + await PortableBundleService(source.database).export(["project:one"], archive) + + tracemalloc.start() + try: + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + receipt = await PortableBundleService(target.database).restore(archive) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert receipt.inserted == 1_501 + assert peak < 10 * 1024 * 1024 + + asyncio.run(scenario()) + + +def test_validate_rejects_an_artifact_family_the_target_runtime_cannot_restore(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await connection.execute( + insert(ARTIFACTS_TABLE).values( + scope_id="project:one", + family="unavailable-family", + artifact_id="record-1", + revision=1, + content=b"payload", + ) + ) + await connection.execute( + insert(ARTIFACT_HEADS_TABLE).values( + scope_id="project:one", + family="unavailable-family", + artifact_id="record-1", + revision=1, + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive) + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + with pytest.raises(BundleFormatError, match="artifact families"): + await PortableBundleService( + target.database, + supported_artifact_families=("handoff",), + ).validate(archive) + + asyncio.run(scenario()) + + +def test_validate_rejects_candidate_evidence_that_is_not_in_the_bundle(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + database = tmp_path / "source.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{database}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await connection.execute( + insert(ARTIFACT_CANDIDATE_VERSIONS_TABLE).values( + scope_id="project:one", + candidate_id="candidate-1", + version=1, + family="handoff", + proposal=b"{}", + source_refs=b'[{"source_type":"content","source_id":"missing"}]', + artifact_refs=b"[]", + target_family=None, + target_artifact_id=None, + target_revision=None, + reason=None, + ) + ) + await connection.execute( + insert(ARTIFACT_CANDIDATE_HEADS_TABLE).values( + scope_id="project:one", + candidate_id="candidate-1", + family="handoff", + version=1, + status="pending", + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive) + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + with pytest.raises(BundleFormatError, match="missing source"): + await PortableBundleService(target.database).validate(archive) + + asyncio.run(scenario()) diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py new file mode 100644 index 0000000000..6a948efbc9 --- /dev/null +++ b/tests/e2e/test_portable_archive.py @@ -0,0 +1,80 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +"""Portable archive acceptance through the public built-in Runtime.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +from powercontext.builtin.artifacts.memory import MemoryEntryInput +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime import ( + BuiltinConfig, + CaptureSource, + HandoffDraft, + HandoffSourceCitation, + HandoffStatement, + RememberMemoryRequest, + open_builtin_runtime, +) + + +def test_sqlite_portable_archive_restores_revisions_lineage_and_handoff_receipts(tmp_path: Path) -> None: + async def scenario() -> None: + scope_id = "project:portable" + archive_path = tmp_path / "portable.pcb" + source_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'source.db'}")) + target_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + + async with open_builtin_runtime(source_config) as source_runtime: + source = await source_runtime.sources.for_scope(scope_id).capture( + CaptureSource( + source_id="turn-1", + content="The archive must preserve the handoff citation.", + metadata={"origin": "portable-archive-e2e"}, + ) + ) + memory = await source_runtime.memory.for_scope(scope_id).remember( + RememberMemoryRequest( + entries=(MemoryEntryInput(kind="decision", text="Use the portable archive for migration."),) + ) + ) + assert memory.entry is not None + handoffs = source_runtime.handoff.for_scope(scope_id) + prepared = await handoffs.finalize( + HandoffDraft( + objective="Restore an exact logical handoff.", + state=( + HandoffStatement( + text="The source citation is durable.", + citations=(HandoffSourceCitation(source_ref=source.source_ref),), + ), + ), + disposition="continuable", + next_action=HandoffStatement( + text="Open the restored handoff.", + citations=(HandoffSourceCitation(source_ref=source.source_ref),), + ), + ) + ) + committed = await handoffs.commit(prepared) + assert source_runtime.archive is not None + exported = await source_runtime.archive.export([scope_id], archive_path) + + async with open_builtin_runtime(target_config) as target_runtime: + assert target_runtime.archive is not None + restored = await target_runtime.archive.restore(archive_path) + latest = await target_runtime.handoff.for_scope(scope_id).latest() + + assert restored.record_count == exported.record_count + assert restored.projections_ready is True + assert latest == committed + assert latest is not None + assert latest.revision == 1 + assert latest.lineage.sources == (source.source_ref,) + + asyncio.run(scenario()) diff --git a/tests/test_archive_cli.py b/tests/test_archive_cli.py new file mode 100644 index 0000000000..be1fec332e --- /dev/null +++ b/tests/test_archive_cli.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +from typer.testing import CliRunner + +from powercontext.cli.app import create_cli +from powercontext.cli.archive import archive_app + + +def test_archive_cli_exposes_a_safe_restore_confirmation_and_dry_run() -> None: + cli = create_cli([archive_app]) + runner = CliRunner() + + help_result = runner.invoke(cli, ["archive", "--help"]) + restore_help = runner.invoke(cli, ["archive", "restore", "--help"]) + + assert help_result.exit_code == 0 + assert all(command in help_result.output for command in ("export", "inspect", "restore")) + assert restore_help.exit_code == 0 + assert "--dry-run" in restore_help.output + assert "--yes" in restore_help.output + + +def test_archive_cli_can_export_inspect_and_dry_run_an_empty_scope(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "data")) + output = tmp_path / "scope.pcb" + cli = create_cli([archive_app]) + runner = CliRunner() + + exported = runner.invoke(cli, ["archive", "export", "--scope-id", "project:one", "--output", str(output)]) + inspected = runner.invoke(cli, ["archive", "inspect", str(output)]) + validated = runner.invoke(cli, ["archive", "restore", str(output), "--dry-run"]) + + assert exported.exit_code == 0, exported.output + assert output.is_file() + assert inspected.exit_code == 0, inspected.output + assert validated.exit_code == 0, validated.output + + +def test_archive_cli_reports_corrupt_or_missing_bundles_without_a_traceback() -> None: + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "inspect", "missing.pcb"]) + + assert result.exit_code == 2 + assert isinstance(result.exception, SystemExit) + assert "error: archive validation failed: cannot read portable bundle" in result.output + assert "Traceback" not in result.output diff --git a/uv.lock b/uv.lock index ab5b5a3c30..a64b2260a0 100644 --- a/uv.lock +++ b/uv.lock @@ -987,7 +987,7 @@ name = "importlib-metadata" version = "9.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp" }, + { name = "zipp", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } wheels = [ @@ -2074,10 +2074,16 @@ builtin = [ { name = "sqlite-vec" }, ] cli = [ + { name = "aiosqlite" }, + { name = "apscheduler" }, { name = "httpx", extra = ["socks"] }, { name = "opentelemetry-api" }, { name = "platformdirs" }, + { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, { name = "pydantic-settings" }, + { name = "pyobvector" }, + { name = "sqlalchemy", extra = ["asyncio"] }, + { name = "sqlite-vec" }, { name = "typer" }, ] client = [ @@ -2140,9 +2146,11 @@ dev = [ [package.metadata] requires-dist = [ { name = "aiosqlite", marker = "extra == 'builtin'", specifier = ">=0.22,<1" }, + { name = "aiosqlite", marker = "extra == 'cli'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'seekdb'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'server'", specifier = ">=0.22,<1" }, { name = "apscheduler", marker = "extra == 'builtin'", specifier = ">=3.11,<4" }, + { name = "apscheduler", marker = "extra == 'cli'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'seekdb'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'server'", specifier = ">=3.11,<4" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115,<1" }, @@ -2160,6 +2168,7 @@ requires-dist = [ { name = "prometheus-client", marker = "extra == 'server'", specifier = ">=0.21,<1" }, { name = "pydantic", specifier = ">=2.10,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'builtin'", specifier = ">=2.27.1,<3" }, + { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'cli'", specifier = ">=2.27.1,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'seekdb'", specifier = ">=2.27.1,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'server'", specifier = ">=2.27.1,<3" }, { name = "pydantic-settings", marker = "extra == 'builtin'", specifier = ">=2.7,<3" }, @@ -2169,13 +2178,16 @@ requires-dist = [ { name = "pydantic-settings", marker = "extra == 'server'", specifier = ">=2.7,<3" }, { name = "pylibseekdb", marker = "(sys_platform == 'darwin' and extra == 'seekdb') or (sys_platform == 'linux' and extra == 'seekdb')", specifier = ">=1.3.0.post4,<2" }, { name = "pyobvector", marker = "extra == 'builtin'", specifier = ">=0.2.28,<0.3" }, + { name = "pyobvector", marker = "extra == 'cli'", specifier = ">=0.2.28,<0.3" }, { name = "pyobvector", marker = "extra == 'seekdb'", specifier = ">=0.2.28,<0.3" }, { name = "pyobvector", marker = "extra == 'server'", specifier = ">=0.2.28,<0.3" }, { name = "rfc8785", specifier = ">=0.1.4,<1" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'builtin'", specifier = ">=2,<3" }, + { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'cli'", specifier = ">=2,<3" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'seekdb'", specifier = ">=2,<3" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'server'", specifier = ">=2,<3" }, { name = "sqlite-vec", marker = "extra == 'builtin'", specifier = ">=0.1.9,<0.2" }, + { name = "sqlite-vec", marker = "extra == 'cli'", specifier = ">=0.1.9,<0.2" }, { name = "sqlite-vec", marker = "extra == 'seekdb'", specifier = ">=0.1.9,<0.2" }, { name = "sqlite-vec", marker = "extra == 'server'", specifier = ">=0.1.9,<0.2" }, { name = "typer", marker = "extra == 'cli'", specifier = ">=0.16,<1" }, diff --git a/zensical.toml b/zensical.toml index ae3526af42..d1f2a17bde 100644 --- a/zensical.toml +++ b/zensical.toml @@ -22,6 +22,7 @@ nav = [ { "Review Candidates" = "en/docs/how-to/review-candidates.md" }, { "Create and review an Experience" = "en/docs/how-to/create-and-review-experience.md" }, { "Create and export a managed Skill" = "en/docs/how-to/create-and-export-skill.md" }, + { "Export and restore a portable archive" = "en/docs/how-to/portable-archive.md" }, { "Troubleshoot" = "en/docs/how-to/troubleshoot.md" }, ] }, { "Integrations" = [ @@ -97,6 +98,7 @@ nav = [ { "审核 Candidate" = "zh/docs/how-to/review-candidates.md" }, { "创建并审核 Experience" = "zh/docs/how-to/create-and-review-experience.md" }, { "创建并导出 managed Skill" = "zh/docs/how-to/create-and-export-skill.md" }, + { "导出和恢复可移植归档" = "zh/docs/how-to/portable-archive.md" }, { "排查问题" = "zh/docs/how-to/troubleshoot.md" }, ] }, { "集成方式" = [ From 3c0e6fb21ef0f8cd5cb41b7d503e8a315c9a69e5 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Thu, 10 Sep 2026 22:35:51 +0800 Subject: [PATCH 18/32] feat: add portable archive export and restore --- docs/en/docs/operate/index.md | 1 + docs/en/docs/operate/meta.json | 1 + docs/en/docs/operate/portable-archive.md | 145 ++ docs/en/docs/operate/troubleshoot.md | 2 +- docs/zh/docs/operate/index.md | 1 + docs/zh/docs/operate/meta.json | 1 + docs/zh/docs/operate/portable-archive.md | 134 ++ docs/zh/docs/operate/troubleshoot.md | 2 +- e2e/bub/uv.lock | 7 + pyproject.toml | 2 + .../builtin/persistence/tables.py | 22 + .../builtin/portability/__init__.py | 39 + .../builtin/portability/bundle.py | 1569 +++++++++++++++++ .../builtin/runtime/application.py | 3 + .../builtin/runtime/composition.py | 1 + .../builtin/runtime/relational.py | 17 + src/powercontext/cli/archive.py | 212 +++ tests/builtin/portability/test_bundle.py | 611 +++++++ tests/e2e/test_portable_archive.py | 118 ++ tests/e2e/test_portable_archive_oceanbase.py | 123 ++ tests/test_archive_cli.py | 71 + uv.lock | 48 +- 22 files changed, 3111 insertions(+), 19 deletions(-) create mode 100644 docs/en/docs/operate/portable-archive.md create mode 100644 docs/zh/docs/operate/portable-archive.md create mode 100644 src/powercontext/builtin/portability/__init__.py create mode 100644 src/powercontext/builtin/portability/bundle.py create mode 100644 src/powercontext/cli/archive.py create mode 100644 tests/builtin/portability/test_bundle.py create mode 100644 tests/e2e/test_portable_archive.py create mode 100644 tests/e2e/test_portable_archive_oceanbase.py create mode 100644 tests/test_archive_cli.py diff --git a/docs/en/docs/operate/index.md b/docs/en/docs/operate/index.md index b05f22c974..0fb2989175 100644 --- a/docs/en/docs/operate/index.md +++ b/docs/en/docs/operate/index.md @@ -11,6 +11,7 @@ Start with the operational task you need: | --- | --- | | Keep a personal Server running, deploy a container, or enable authentication | [Deploy the Server](deploy-server.md) | | Upgrade or switch background supervision mode | [Migrate Artifact processing state](artifact-processing-migration.md) | +| Export a local SQLite archive or restore one | [Portable archive](portable-archive.md) | | Inspect logs, metrics, and traces | [Observability](observability.md) | | Send traces to an analysis service | [Phoenix](trace-with-phoenix.md), [Langfuse](trace-with-langfuse.md) | | Diagnose failures and restore service or data | [Troubleshoot](troubleshoot.md) | diff --git a/docs/en/docs/operate/meta.json b/docs/en/docs/operate/meta.json index 5e04efad00..0b5bd68abd 100644 --- a/docs/en/docs/operate/meta.json +++ b/docs/en/docs/operate/meta.json @@ -5,6 +5,7 @@ "deploy-server", "connect-remote-server", "artifact-processing-migration", + "portable-archive", "observability", "trace-with-phoenix", "trace-with-langfuse", diff --git a/docs/en/docs/operate/portable-archive.md b/docs/en/docs/operate/portable-archive.md new file mode 100644 index 0000000000..aef202ff2b --- /dev/null +++ b/docs/en/docs/operate/portable-archive.md @@ -0,0 +1,145 @@ +--- +title: Export and restore a portable archive +description: Create, validate, and restore a verified PowerContext logical bundle. +--- + +# Export and restore a portable archive + +A portable archive is a `.pcb` file for moving or recovering complete PowerContext scopes. It is a logical archive: +it preserves domain identities and immutable history rather than copying a SQLite file. Use it for a controlled local +backup, an offline transfer, or a future backend migration. + +This is an offline operator command. It does not call a remote Server API. Without `--env-file` it opens the default +SQLite database; with `--env-file` it uses the same SQLite, SeekDB, or OceanBase settings as that deployment. Stop +PowerContext writers before restore. Export uses one database transaction as its consistent logical snapshot. + +## Prerequisites + +Install the CLI from the same PowerContext revision that created the local data. The archive command uses the persistent +SQLite database at `POWERCONTEXT_HOME/powercontext.db`; without `POWERCONTEXT_HOME`, PowerContext uses its operating +system user-data directory. + +```bash +uv tool install "powercontext[cli] @ git+https://github.com/oceanbase/powercontext.git@master" +export POWERCONTEXT_HOME="$HOME/.local/share/powercontext" +``` + +The archive contains content and metadata needed for recovery. Store it in a protected location with access controls +and retention appropriate for the underlying project data. The format deliberately omits credentials and configured +provider secrets, but it does not encrypt the project content. + +## Create and inspect an archive + +Pass every complete scope to include. A scope cannot be selected by prefix or partially by record type. + +```bash +powercontext archive export \ + --scope-id project:payments \ + --output ./backups/payments.pcb + +powercontext archive inspect ./backups/payments.pcb +``` + +Add `--no-compress` when the surrounding storage system already compresses objects. Export authorization occurs before +the first database query. For this offline command, authority is the operating-system and database permission needed to +read the configured deployment; it never reuses or exports Server bearer tokens. Progress is emitted as content-free +JSON on stderr while the final receipt remains JSON on stdout. + +`export` writes JSON containing the bundle ID, record count, and checksum. `inspect` verifies the ZIP structure, +per-record digests, total digest, record counts, and selected scopes without opening the target database. Keep the +reported checksum with the backup inventory if an external backup system needs an independent verification record. + +## Validate before a restore + +Always validate the archive against the destination first: + +```bash +powercontext archive restore ./backups/payments.pcb --dry-run +``` + +Dry-run performs no domain writes. It verifies checksums, required source and Artifact dependencies, and whether the +configured Runtime can restore the archive's source types and Artifact families. A validation failure exits with code +`2` and reports only a content-free reason; record bodies are not printed. A successful report includes +`already_present`, `conflicts`, required and unsupported Source types and Artifact families, and whether the target has +a projection rebuilder. Run against a different deployment with `--env-file ./target.env`. + +To perform the write, repeat the command with explicit confirmation: + +```bash +powercontext archive restore ./backups/payments.pcb --yes +``` + +The result is JSON with `inserted`, `already_present`, and `projections_ready`. Treat the restore as ready for search +only when `projections_ready` is `true`. The Runtime rebuilds portable Memory and Experience search projections after +the authoritative rows have been restored. + +## What the bundle preserves + +Format version 1 carries the portable relational representation of these supported records: + +| Preserved | Not portable | +| --- | --- | +| Scope identity, hierarchy, context/external references, and creation identity | Scope access bindings and host-local default selection | +| Source journal heads and Source records | Search projections and indexes | +| Artifact Revisions, lineage, cross-Scope publication provenance, and heads | Source cursors and scheduler state | +| Memory entry versions and heads | External Skill registrations and host-local installation state | +| Candidate versions, decision heads, and their evidence references | Audit events, usage facts, evaluation receipts, and restore receipts | +| Work contracts, task outcomes, Handoff boundaries and receipts stored as Sources | Credentials, bearer tokens, provider secrets, and host-local Skill installation state | + +The archive has a versioned manifest (`format_version`, producer version, scopes, counts, exclusions, and total +checksum) plus NDJSON records. It is the authoritative round-trip format. CSV may be produced separately for bounded +analysis, but it cannot preserve immutable revisions, lineage, or evidence references and must not be used for restore. + +## Recovery and conflicts + +Restore is idempotent. Replaying the same archive recognizes identical records as `already_present`. A different +payload for an existing immutable identity never overwrites it: restore exits with code `3` and leaves the write +transaction rolled back. Resolve the conflicting target data or choose a clean destination, then run the same restore +command again. + +If the command is interrupted before completion, do not claim recovery succeeded. Authoritative records are +transactionally applied, so a failed write does not leave a successful-looking partial restore. Re-run the same archive +against the same destination after the previous command has stopped. + +The target database stores a durable receipt keyed by `bundle_id`. `authoritative_restored` means the logical rows +committed but search rebuild has not completed; `ready` means projection rebuild completed. If projection rebuild +fails, re-run the same restore: identical rows are skipped and the receipt advances to `ready` only after verification. + +## SQLite to OceanBase migration + +Create separate environment files without placing credentials in the bundle: + +```bash +powercontext archive export --env-file ./sqlite.env \ + --scope-id project:payments --output ./payments.pcb +powercontext archive restore ./payments.pcb --env-file ./oceanbase.env --dry-run +powercontext archive restore ./payments.pcb --env-file ./oceanbase.env --yes +``` + +The environment files use `POWERCONTEXT_SERVER_DATABASE_KIND` and `POWERCONTEXT_SERVER_DATABASE_URL` as documented in +the Server configuration guide. Dry-run must report no unsupported families and zero conflicts before the write. + +## Format compatibility + +Format version `1` readers accept version `1` bundles from any PowerContext producer version. The producer version is +reported for diagnostics but does not replace the archive schema version. Writers produce only version `1`; there is no +format `0` downgrade. A reader rejects unknown archive or record schema versions before target writes. Keep the old +binary available until a restore drill succeeds when upgrading across a PowerContext major release. + +## Deployment-native backups + +Logical export is not a substitute for a crash-consistent deployment backup: + +- SQLite: stop all PowerContext writers before copying the database, or use the SQLite online backup API, for example + `sqlite3 powercontext.db ".backup './backups/powercontext.db'"`. Verify the copy with + `sqlite3 ./backups/powercontext.db "PRAGMA integrity_check"`, protect it like the source database, and test reopening it. +- OceanBase: enable tenant data backup and log archiving according to the deployed OceanBase version, retain the backup + destination and encryption material independently, and perform a restore drill into an isolated tenant. Use native + backup for point-in-time recovery and the portable bundle for logical cross-backend migration. + +## Current boundaries + +Export streams database rows to a temporary NDJSON file. Validation stores only record identities in a temporary +on-disk index, and restore replays dependency levels as streams, so aggregate archive payloads are not retained in +process memory. Scope metadata is bounded by the manifest scope list. The command is not a remote Server API and never +puts deployment configuration or database credentials into the archive. diff --git a/docs/en/docs/operate/troubleshoot.md b/docs/en/docs/operate/troubleshoot.md index 85c3606d88..48bb61ebdf 100644 --- a/docs/en/docs/operate/troubleshoot.md +++ b/docs/en/docs/operate/troubleshoot.md @@ -199,7 +199,7 @@ so the previous database remains available for recovery: ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_receipt_migration_review' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_portable_restore_receipts,pc_source_cursors,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_receipt_migration_review' \ -f ``` diff --git a/docs/zh/docs/operate/index.md b/docs/zh/docs/operate/index.md index ed6a6b4af0..af96d33734 100644 --- a/docs/zh/docs/operate/index.md +++ b/docs/zh/docs/operate/index.md @@ -11,6 +11,7 @@ description: 持续运行 Server，查找部署、监控与恢复步骤。 | --- | --- | | 持续运行个人 Server、部署容器或启用认证 | [部署 Server](deploy-server.md) | | 升级或切换后台监督模式 | [迁移 Artifact 处理状态](artifact-processing-migration.md) | +| 导出本地 SQLite 归档或恢复归档 | [可移植归档](portable-archive.md) | | 查看日志、指标和 Trace | [可观测性](observability.md) | | 将 Trace 发送到分析服务 | [Phoenix](trace-with-phoenix.md)、[Langfuse](trace-with-langfuse.md) | | 诊断失败并恢复服务或数据 | [诊断与恢复](troubleshoot.md) | diff --git a/docs/zh/docs/operate/meta.json b/docs/zh/docs/operate/meta.json index f8a7b06d97..a52b8ad8c2 100644 --- a/docs/zh/docs/operate/meta.json +++ b/docs/zh/docs/operate/meta.json @@ -5,6 +5,7 @@ "deploy-server", "connect-remote-server", "artifact-processing-migration", + "portable-archive", "observability", "trace-with-phoenix", "trace-with-langfuse", diff --git a/docs/zh/docs/operate/portable-archive.md b/docs/zh/docs/operate/portable-archive.md new file mode 100644 index 0000000000..7e54d7815f --- /dev/null +++ b/docs/zh/docs/operate/portable-archive.md @@ -0,0 +1,134 @@ +--- +title: 导出和恢复可移植归档 +description: 创建、验证并恢复经过校验的 PowerContext 逻辑归档。 +--- + +# 导出和恢复可移植归档 + +可移植归档是用于迁移或恢复完整 PowerContext scope 的 `.pcb` 文件。它是逻辑归档：保存领域 identity 和不可变历史， +而不是复制 SQLite 文件。它适用于受控的本地备份、离线传输，以及未来的后端迁移。 + +这是一个离线运维命令，不调用远程 Server API。不传 `--env-file` 时打开默认 SQLite 数据库；传入后使用该部署的 +SQLite、SeekDB 或 OceanBase 配置。恢复前应停止 PowerContext 写入进程；导出使用一个数据库 transaction 获取一致的 +逻辑 snapshot。 + +## 前提条件 + +从创建本地数据的同一个 PowerContext revision 安装 CLI。归档命令使用 +`POWERCONTEXT_HOME/powercontext.db` 中持久化的 SQLite 数据库；未设置 `POWERCONTEXT_HOME` 时，PowerContext +会使用操作系统用户数据目录。 + +```bash +uv tool install "powercontext[cli] @ git+https://github.com/oceanbase/powercontext.git@master" +export POWERCONTEXT_HOME="$HOME/.local/share/powercontext" +``` + +归档包含恢复所需的内容和元数据。应按项目数据的访问控制和保留要求，将它放在受保护的位置。格式会排除凭据和已配置的 +provider secret，但不会加密项目内容。 + +## 创建并检查归档 + +传入每一个需要包含的完整 scope。不能用前缀选择 scope，也不能只导出某一种 record。 + +```bash +powercontext archive export \ + --scope-id project:payments \ + --output ./backups/payments.pcb + +powercontext archive inspect ./backups/payments.pcb +``` + +外层存储已经压缩时可增加 `--no-compress`。导出授权一定发生在第一次数据库查询之前；离线命令以操作系统文件权限和数据库 +读取权限作为 operator authority，不复用或导出 Server bearer token。无内容的 JSON 进度写到 stderr，最终 receipt +仍以 JSON 写到 stdout。 + +`export` 会输出 JSON，其中包括 bundle ID、record 数和 checksum。`inspect` 不打开目标数据库，只校验 ZIP 结构、 +单 record digest、总 digest、record 数和所选 scope。若外部备份系统需要独立校验记录，请把输出的 checksum 一并保存。 + +## 在恢复前验证 + +恢复前始终先针对目标环境验证： + +```bash +powercontext archive restore ./backups/payments.pcb --dry-run +``` + +dry-run 不会写入领域数据。它会校验 checksum、必需的 Source 和 Artifact 依赖，以及已配置 Runtime 是否支持归档中的 +source type 和 Artifact family。验证失败会以退出码 `2` 结束，只输出不含内容的原因，不会打印 record body。成功报告 +包含 `already_present`、`conflicts`、所需和不支持的 Source type/Artifact family，以及目标是否支持 projection rebuild。 +可用 `--env-file ./target.env` 检查另一个部署。 + +要执行写恢复，必须显式确认： + +```bash +powercontext archive restore ./backups/payments.pcb --yes +``` + +结果为 JSON，包含 `inserted`、`already_present` 和 `projections_ready`。只有 `projections_ready` 为 `true` +时才能宣称搜索已就绪。Runtime 会在恢复权威数据后重建可移植的 Memory 和 Experience 搜索 projection。 + +## 归档保留的内容 + +格式版本 1 支持以下可移植关系记录： + +| 会保留 | 不可移植 | +| --- | --- | +| Scope identity、层级、context/external reference 和创建 identity | Scope access binding 和 host-local default selection | +| Source journal head 和 Source record | 搜索 projection 和 index | +| Artifact Revision、lineage、跨 Scope 发布来源和 head | Source cursor 和 scheduler state | +| Memory entry version 和 head | External Skill registration 和 host-local installation state | +| Candidate version、decision head 及其 evidence reference | Audit event、usage fact、evaluation receipt 和 restore receipt | +| 作为 Source 保存的 Work contract、task outcome、Handoff boundary 和 receipt | Credential、bearer token、provider secret 和 host-local Skill 安装状态 | + +归档包含带版本的 manifest（`format_version`、producer version、scope、数量、排除项和总 checksum）以及 NDJSON record。 +它是权威的 round-trip 格式。CSV 可以单独用于有边界的分析，但无法保留不可变 revision、lineage 或 evidence reference， +不得用于恢复。 + +## 恢复和冲突处理 + +恢复是幂等的。重复回放同一归档时，内容相同的 record 会计入 `already_present`。已有不可变 identity 对应不同 payload +时，恢复绝不会覆盖它：命令以退出码 `3` 结束，并回滚本次写 transaction。解决目标数据冲突，或选择一个干净的目标后， +再次运行同一个恢复命令。 + +如果命令在完成前中断，不能宣称恢复已成功。权威 record 会在 transaction 中写入；失败的写入不会留下看似成功的部分 +恢复。先确认上一次命令已经停止，再对同一目标重新运行同一个归档。 + +目标数据库会按 `bundle_id` 保存 durable receipt。`authoritative_restored` 表示逻辑数据已经提交但搜索尚未重建完成； +`ready` 表示 projection rebuild 已完成。若重建失败，重新恢复同一 bundle 会跳过相同记录，只有验证完成后才把 receipt +推进为 `ready`。 + +## 从 SQLite 迁移到 OceanBase + +分别创建环境文件，数据库凭据不会写入 bundle： + +```bash +powercontext archive export --env-file ./sqlite.env \ + --scope-id project:payments --output ./payments.pcb +powercontext archive restore ./payments.pcb --env-file ./oceanbase.env --dry-run +powercontext archive restore ./payments.pcb --env-file ./oceanbase.env --yes +``` + +环境文件使用 Server 配置文档中的 `POWERCONTEXT_SERVER_DATABASE_KIND` 和 +`POWERCONTEXT_SERVER_DATABASE_URL`。写入前，dry-run 应报告不存在不支持的 family，且冲突数为零。 + +## 格式兼容窗口 + +格式版本 `1` reader 接受任意 PowerContext producer version 生成的格式版本 `1` bundle。producer version 仅用于诊断， +不能替代 archive schema version。writer 只生成版本 `1`，不支持降级到格式 `0`。遇到未知 archive 或 record schema +version 时，reader 会在目标写入前拒绝。跨 PowerContext 大版本升级时，应保留旧二进制，直到恢复演练通过。 + +## 部署原生备份 + +逻辑导出不能替代 crash-consistent 部署备份： + +- SQLite：复制数据库前停止所有 PowerContext writer，或使用 SQLite online backup API，例如 + `sqlite3 powercontext.db ".backup './backups/powercontext.db'"`。随后执行 + `sqlite3 ./backups/powercontext.db "PRAGMA integrity_check"`，按源数据库等级保护备份，并测试重新打开。 +- OceanBase：根据实际部署版本启用 tenant data backup 和 log archiving，独立保管备份目标与加密材料，并在隔离 tenant + 进行恢复演练。point-in-time recovery 使用原生备份；逻辑跨后端迁移使用 portable bundle。 + +## 当前边界 + +导出会把数据库 row 流式写到临时 NDJSON 文件。验证只在临时磁盘 index 中保存 record identity，恢复则按依赖层流式 +回放，因此归档的聚合 payload 不会常驻进程内存；Scope metadata 由 manifest scope list 约束。命令不是远程 Server +API，也不会把部署配置或数据库凭据写入归档。 diff --git a/docs/zh/docs/operate/troubleshoot.md b/docs/zh/docs/operate/troubleshoot.md index 349d3018ef..c28a407f5b 100644 --- a/docs/zh/docs/operate/troubleshoot.md +++ b/docs/zh/docs/operate/troubleshoot.md @@ -192,7 +192,7 @@ collation，但不会包含数据库 URL 或凭据。 ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_receipt_migration_review' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_portable_restore_receipts,pc_source_cursors,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_receipt_migration_review' \ -f ``` diff --git a/e2e/bub/uv.lock b/e2e/bub/uv.lock index 06583e0a0e..9fd95caf63 100644 --- a/e2e/bub/uv.lock +++ b/e2e/bub/uv.lock @@ -1618,9 +1618,11 @@ client = [ [package.metadata] requires-dist = [ { name = "aiosqlite", marker = "extra == 'builtin'", specifier = ">=0.22,<1" }, + { name = "aiosqlite", marker = "extra == 'cli'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'seekdb'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'server'", specifier = ">=0.22,<1" }, { name = "apscheduler", marker = "extra == 'builtin'", specifier = ">=3.11,<4" }, + { name = "apscheduler", marker = "extra == 'cli'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'seekdb'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'server'", specifier = ">=3.11,<4" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115,<1" }, @@ -1631,6 +1633,7 @@ requires-dist = [ { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3,<1" }, { name = "jinja2", marker = "extra == 'server'", specifier = ">=3.1.6,<4" }, { name = "jsonschema", marker = "extra == 'builtin'", specifier = ">=4.23,<5" }, + { name = "jsonschema", marker = "extra == 'cli'", specifier = ">=4.23,<5" }, { name = "jsonschema", marker = "extra == 'seekdb'", specifier = ">=4.23,<5" }, { name = "jsonschema", marker = "extra == 'server'", specifier = ">=4.23,<5" }, { name = "opentelemetry-api", marker = "extra == 'cli'", specifier = ">=1.30,<2" }, @@ -1649,6 +1652,7 @@ requires-dist = [ { name = "pycasbin", marker = "extra == 'server'", specifier = ">=2.8,<3" }, { name = "pydantic", specifier = ">=2.10,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'builtin'", specifier = ">=2.27.1,<3" }, + { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'cli'", specifier = ">=2.27.1,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'seekdb'", specifier = ">=2.27.1,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'server'", specifier = ">=2.27.1,<3" }, { name = "pydantic-settings", marker = "extra == 'builtin'", specifier = ">=2.7,<3" }, @@ -1658,6 +1662,7 @@ requires-dist = [ { name = "pydantic-settings", marker = "extra == 'server'", specifier = ">=2.7,<3" }, { name = "pylibseekdb", marker = "(sys_platform == 'darwin' and extra == 'seekdb') or (sys_platform == 'linux' and extra == 'seekdb')", specifier = ">=1.3.0.post4,<2" }, { name = "pyobvector", marker = "extra == 'builtin'", specifier = ">=0.2.28,<0.3" }, + { name = "pyobvector", marker = "extra == 'cli'", specifier = ">=0.2.28,<0.3" }, { name = "pyobvector", marker = "extra == 'seekdb'", specifier = ">=0.2.28,<0.3" }, { name = "pyobvector", marker = "extra == 'server'", specifier = ">=0.2.28,<0.3" }, { name = "pyyaml", marker = "extra == 'builtin'", specifier = ">=6,<7" }, @@ -1668,9 +1673,11 @@ requires-dist = [ { name = "rfc8785", specifier = ">=0.1.4,<1" }, { name = "scalar-fastapi", marker = "extra == 'server'", specifier = ">=1.8.2,<2" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'builtin'", specifier = ">=2,<3" }, + { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'cli'", specifier = ">=2,<3" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'seekdb'", specifier = ">=2,<3" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'server'", specifier = ">=2,<3" }, { name = "sqlite-vec", marker = "extra == 'builtin'", specifier = ">=0.1.9,<0.2" }, + { name = "sqlite-vec", marker = "extra == 'cli'", specifier = ">=0.1.9,<0.2" }, { name = "sqlite-vec", marker = "extra == 'seekdb'", specifier = ">=0.1.9,<0.2" }, { name = "sqlite-vec", marker = "extra == 'server'", specifier = ">=0.1.9,<0.2" }, { name = "typer", marker = "extra == 'cli'", specifier = ">=0.16,<1" }, diff --git a/pyproject.toml b/pyproject.toml index 2fe4c600c2..7a72d73901 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,7 @@ cli = [ "inquirerpy>=0.3,<1", "packaging>=24,<27", "platformdirs>=4,<5", + "powercontext[builtin]", "powercontext[client]", "pyyaml>=6,<7", "typer>=0.16,<1", @@ -96,6 +97,7 @@ doctor = "powercontext.cli.system:doctor_app" service = "powercontext.service.cli:app" server = "powercontext.server.cli:app" setup = "powercontext.cli.system:setup_app" +archive = "powercontext.cli.archive:archive_app" [project.urls] Homepage = "https://github.com/oceanbase/powercontext" diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index 02e25abd0c..d130cb33ef 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -409,6 +409,27 @@ def _entry_text_type(): ), ) +PORTABLE_RESTORE_RECEIPTS_TABLE = Table( + "pc_portable_restore_receipts", + SHARED_METADATA, + Column("bundle_id", identity_string(36), primary_key=True), + Column("format_version", Integer, nullable=False), + Column("total_digest", identity_string(71), nullable=False), + Column("record_count", BigInteger, nullable=False), + Column("status", identity_string(32), nullable=False), + Column("inserted", BigInteger, nullable=False), + Column("already_present", BigInteger, nullable=False), + Column("projections_ready", Boolean, nullable=False), + Column("updated_at", DateTime(timezone=True), nullable=False), + CheckConstraint( + "status IN ('authoritative_restored', 'ready')", + name="ck_pc_portable_restore_receipts_status", + ), + CheckConstraint("record_count >= 0", name="ck_pc_portable_restore_receipts_record_count"), + CheckConstraint("inserted >= 0", name="ck_pc_portable_restore_receipts_inserted"), + CheckConstraint("already_present >= 0", name="ck_pc_portable_restore_receipts_existing"), +) + PROFILE_POLICIES_TABLE = Table( "pc_profile_policies", SHARED_METADATA, @@ -905,6 +926,7 @@ def _entry_text_type(): ARTIFACT_PUBLICATIONS_TABLE, ARTIFACT_CANDIDATE_VERSIONS_TABLE, ARTIFACT_CANDIDATE_HEADS_TABLE, + PORTABLE_RESTORE_RECEIPTS_TABLE, PROFILE_POLICIES_TABLE, SOURCE_CURSORS_TABLE, ARTIFACT_PROCESSING_LEASES_TABLE, diff --git a/src/powercontext/builtin/portability/__init__.py b/src/powercontext/builtin/portability/__init__.py new file mode 100644 index 0000000000..8ce6d3d273 --- /dev/null +++ b/src/powercontext/builtin/portability/__init__.py @@ -0,0 +1,39 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Versioned logical export and restore for built-in relational data.""" + +from powercontext.builtin.portability.bundle import ( + BundleConflictError, + BundleFormatError, + BundleInspection, + BundleProgress, + BundleReceipt, + BundleValidation, + ExportAuthorizer, + PortableBundleService, + ProgressObserver, +) + +__all__ = [ + "BundleConflictError", + "BundleFormatError", + "BundleInspection", + "BundleProgress", + "BundleReceipt", + "BundleValidation", + "ExportAuthorizer", + "PortableBundleService", + "ProgressObserver", +] diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py new file mode 100644 index 0000000000..61efd244c8 --- /dev/null +++ b/src/powercontext/builtin/portability/bundle.py @@ -0,0 +1,1569 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Portable, verified logical bundles for built-in relational storage. + +The archive deliberately contains domain rows only. Database files, search +indexes, trigger cursors, usage statistics, and host-local skill registrations +are deployment details and are not portable state. +""" + +# Error reason strings are the public inspection/validation contract. +# ruff: noqa: TRY003 + +from __future__ import annotations + +import base64 +import hashlib +import json +import re +import shutil +import sqlite3 +import tempfile +import zipfile +from collections import Counter +from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, date, datetime +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any, Literal, cast +from uuid import NAMESPACE_URL, UUID, uuid5 + +from sqlalchemy import insert, select, update +from sqlalchemy import inspect as sqlalchemy_inspect +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.tables import ( + ARTIFACT_CANDIDATE_HEADS_TABLE, + ARTIFACT_CANDIDATE_VERSIONS_TABLE, + ARTIFACT_HEADS_TABLE, + ARTIFACT_LINEAGE_ARTIFACTS_TABLE, + ARTIFACT_LINEAGE_SOURCES_TABLE, + ARTIFACT_PUBLICATIONS_TABLE, + ARTIFACTS_TABLE, + MEMORY_ENTRY_HEADS_TABLE, + MEMORY_ENTRY_VERSIONS_TABLE, + PORTABLE_RESTORE_RECEIPTS_TABLE, + SCOPE_CONTEXT_REFERENCES_TABLE, + SCOPE_CREATION_REQUESTS_TABLE, + SCOPE_EXTERNAL_REFERENCES_TABLE, + SCOPES_TABLE, + SOURCE_JOURNAL_HEADS_TABLE, + SOURCES_TABLE, +) + +FORMAT_VERSION = 1 +_RECORDS_NAME = "records.ndjson" +_MANIFEST_NAME = "manifest.json" +_SHA256 = "sha256:" +_MAX_ARCHIVE_BYTES = 512 * 1024 * 1024 +_MAX_RECORDS_BYTES = 2 * 1024 * 1024 * 1024 +_MAX_RECORDS = 2_000_000 +_MAX_LINE_BYTES = 4 * 1024 * 1024 +_MAX_MANIFEST_BYTES = 256 * 1024 +_HEX_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") +_ABSOLUTE_HOST_PATH = re.compile(r"^(?:/|file:/+|[A-Za-z]:[\\/]|\\\\)") +_SENSITIVE_KEYS = frozenset({ + "access_token", + "api_key", + "apikey", + "bearer_token", + "credential", + "credentials", + "password", + "provider_secret", + "refresh_token", + "secret", + "token", +}) +_EXCLUDED_RECORD_CLASSES = ( + "access_bindings", + "audit_records", + "credentials_and_provider_secrets", + "evaluation_receipts", + "external_skill_registrations", + "host_local_installation_state", + "restore_receipts", + "scope_default_selection", + "search_projections", + "source_cursors", + "usage_statistics", +) + +RecordType = Literal[ + "scope", + "scope_context_reference", + "scope_external_reference", + "scope_creation_request", + "source_journal_head", + "source", + "artifact_revision", + "artifact_lineage_source", + "artifact_lineage_artifact", + "artifact_publication", + "artifact_head", + "memory_entry_version", + "memory_entry_head", + "candidate_version", + "candidate_head", +] +ProjectionRebuilder = Callable[[tuple[str, ...]], Awaitable[None]] +ExportAuthorizer = Callable[[tuple[str, ...]], Awaitable[None]] +ProgressObserver = Callable[["BundleProgress"], None] + + +class BundleFormatError(ValueError): + """The supplied archive is malformed, corrupt, or unsupported.""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + + +class BundleConflictError(ValueError): + """A target immutable identity exists with different canonical content.""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + + +@dataclass(frozen=True, slots=True) +class BundleInspection: + """Content-free facts established by parsing and verifying a bundle.""" + + bundle_id: str + scopes: tuple[str, ...] + record_count: int + records_by_type: Mapping[str, int] + total_digest: str + format_version: int + producer_version: str + + +@dataclass(frozen=True, slots=True) +class BundleValidation(BundleInspection): + """Target compatibility established without performing writes.""" + + already_present: int + conflicts: int + projections_supported: bool + compatible: bool + required_source_types: tuple[str, ...] + unsupported_source_types: tuple[str, ...] + required_artifact_families: tuple[str, ...] + unsupported_artifact_families: tuple[str, ...] + reader_version: str + + +@dataclass(frozen=True, slots=True) +class BundleReceipt: + """Content-free result of an export or restore.""" + + bundle_id: str + record_count: int + total_digest: str + inserted: int = 0 + already_present: int = 0 + projections_ready: bool = False + status: Literal["exported", "authoritative_restored", "ready"] = "exported" + + +@dataclass(frozen=True, slots=True) +class BundleProgress: + """Content-free progress suitable for terminals and automation logs.""" + + phase: Literal["export", "inspect", "validate", "target", "restore", "projections"] + processed: int + total: int | None + + +@dataclass(frozen=True, slots=True) +class _Record: + record_type: RecordType + identity: Mapping[str, str | int] + payload: Mapping[str, Any] + digest: str + + +@dataclass(frozen=True, slots=True) +class _ParsedBundle: + inspection: BundleInspection + + +@dataclass(frozen=True, slots=True) +class _RecordSpec: + """Stable logical fields and their current relational persistence adapter.""" + + table: Any + identity: tuple[str, ...] + payload: tuple[str, ...] + scope_field: str = "scope_id" + + +_SPECS: dict[RecordType, _RecordSpec] = { + "scope": _RecordSpec(SCOPES_TABLE, ("scope_id",), ("title", "summary", "parent_scope_id", "version")), + "scope_context_reference": _RecordSpec(SCOPE_CONTEXT_REFERENCES_TABLE, ("scope_id", "referenced_scope_id"), ()), + "scope_external_reference": _RecordSpec( + SCOPE_EXTERNAL_REFERENCES_TABLE, + ("scope_id", "ordinal"), + ("kind", "value", "value_digest"), + ), + "scope_creation_request": _RecordSpec( + SCOPE_CREATION_REQUESTS_TABLE, + ("idempotency_key",), + ("request_digest", "scope_id"), + ), + "source_journal_head": _RecordSpec(SOURCE_JOURNAL_HEADS_TABLE, ("scope_id",), ("position",)), + "source": _RecordSpec( + SOURCES_TABLE, + ("scope_id", "source_type", "source_id"), + ("payload", "journal_position"), + ), + "artifact_revision": _RecordSpec( + ARTIFACTS_TABLE, + ("scope_id", "family", "artifact_id", "revision"), + ("content", "memory_citations"), + ), + "artifact_lineage_source": _RecordSpec( + ARTIFACT_LINEAGE_SOURCES_TABLE, + ("scope_id", "family", "artifact_id", "revision", "ordinal"), + ("source_type", "source_id"), + ), + "artifact_lineage_artifact": _RecordSpec( + ARTIFACT_LINEAGE_ARTIFACTS_TABLE, + ("scope_id", "family", "artifact_id", "revision", "ordinal"), + ("upstream_family", "upstream_artifact_id", "upstream_revision"), + ), + "artifact_publication": _RecordSpec( + ARTIFACT_PUBLICATIONS_TABLE, + ("target_scope_id", "target_family", "target_artifact_id", "target_revision"), + ( + "source_scope_id", + "source_family", + "source_artifact_id", + "source_revision", + "content_digest", + "idempotency_key", + ), + scope_field="target_scope_id", + ), + "artifact_head": _RecordSpec( + ARTIFACT_HEADS_TABLE, + ("scope_id", "family", "artifact_id"), + ("revision", "lifecycle_state", "replacement_artifact_id", "governance_generation"), + ), + "memory_entry_version": _RecordSpec( + MEMORY_ENTRY_VERSIONS_TABLE, + ("scope_id", "memory_artifact_id", "entry_version_id"), + ( + "family", + "entry_id", + "version", + "previous_version_id", + "kind", + "text", + "source_refs", + "artifact_refs", + "entry_content_hash", + "created_in_revision", + ), + ), + "memory_entry_head": _RecordSpec( + MEMORY_ENTRY_HEADS_TABLE, + ("scope_id", "memory_artifact_id", "entry_id"), + ("family", "head_revision", "entry_version_id", "entry_content_hash"), + ), + "candidate_version": _RecordSpec( + ARTIFACT_CANDIDATE_VERSIONS_TABLE, + ("scope_id", "candidate_id", "version"), + ( + "family", + "proposal", + "source_refs", + "artifact_refs", + "memory_citations", + "target_family", + "target_artifact_id", + "target_revision", + "reason", + ), + ), + "candidate_head": _RecordSpec( + ARTIFACT_CANDIDATE_HEADS_TABLE, + ("scope_id", "candidate_id"), + ( + "family", + "version", + "status", + "result_family", + "result_artifact_id", + "result_revision", + "decision_reason", + ), + ), +} + +# Parent rows always precede their dependent rows. This matters for SQLite as +# well as for MySQL/OceanBase foreign-key enforcement. +_EXPORT_ORDER: tuple[RecordType, ...] = tuple(_SPECS) + + +class PortableBundleService: + """Export and restore complete scopes without depending on a DB dialect. + + The service is intentionally application-facing rather than a database-file + utility. Callers must perform authorization before calling :meth:`export`; + records are never enumerated before the scope arguments are validated. + """ + + def __init__( + self, + database: AsyncDatabase, + /, + *, + projection_rebuilder: ProjectionRebuilder | None = None, + supported_source_types: Iterable[str] | None = None, + supported_artifact_families: Iterable[str] | None = None, + ) -> None: + self._database = database + self._projection_rebuilder = projection_rebuilder + self._supported_source_types = None if supported_source_types is None else frozenset(supported_source_types) + self._supported_artifact_families = ( + None if supported_artifact_families is None else frozenset(supported_artifact_families) + ) + + async def export( + self, + scopes: Iterable[str], + output: Path, + /, + *, + authorize: ExportAuthorizer, + progress: ProgressObserver | None = None, + compress: bool = True, + ) -> BundleReceipt: + """Write a verified archive containing all portable records for scopes. + + A single read transaction provides a consistent snapshot on supported + relational backends. The resulting ZIP is written only after the + snapshot has been completely read, so a failed export never presents a + partially finalized archive at ``output``. + """ + + selected = _validate_scopes(scopes) + # Authorization is deliberately outside the snapshot transaction and + # before the first database query or pagination operation. + await authorize(selected) + output.parent.mkdir(parents=True, exist_ok=True) + records_path = _temporary_path(output.parent, suffix=".records.ndjson") + archive_path = _temporary_path(output.parent, suffix=".pcb") + try: + with records_path.open("wb") as stream: + async with self._database.transaction() as connection: + await _establish_export_snapshot(connection) + await _validate_selected_scopes(connection, selected) + count, records_by_type, digest = await self._stream_export( + connection, selected, stream, progress=progress + ) + bundle_id = str(uuid5(NAMESPACE_URL, f"powercontext:portable:v{FORMAT_VERSION}:{digest}")) + manifest = { + "format_version": FORMAT_VERSION, + "bundle_id": bundle_id, + "producer": {"name": "powercontext", "version": _producer_version()}, + "scopes": list(selected), + "record_count": count, + "records_by_type": dict(sorted(records_by_type.items())), + "excluded_record_classes": list(_EXCLUDED_RECORD_CLASSES), + "total_digest": digest, + } + compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED + with zipfile.ZipFile( + archive_path, "w", compression=compression, compresslevel=9 if compress else None + ) as archive: + with ( + records_path.open("rb") as source, + archive.open(_zip_info(_RECORDS_NAME, compression), "w") as target, + ): + shutil.copyfileobj(source, target, length=1024 * 1024) + archive.writestr(_zip_info(_MANIFEST_NAME, compression), _canonical_json(manifest)) + archive_path.replace(output) + return BundleReceipt(bundle_id=bundle_id, record_count=count, total_digest=digest) + finally: + records_path.unlink(missing_ok=True) + archive_path.unlink(missing_ok=True) + + @staticmethod + async def inspect(source: Path, /, *, progress: ProgressObserver | None = None) -> BundleInspection: + """Verify structure and checksums without touching the target database.""" + + return _parse_bundle(source, progress=progress, phase="inspect").inspection + + async def validate( + self, + source: Path, + /, + *, + supported_source_types: Iterable[str] | None = None, + progress: ProgressObserver | None = None, + ) -> BundleValidation: + """Verify bundle integrity, dependencies, and optional adapter support.""" + + parsed = _parse_bundle(source, progress=progress, phase="validate") + _validate_dependencies(source) + required_sources = { + str(record.identity["source_type"]) for record in _iter_records(source) if record.record_type == "source" + } + configured = ( + self._supported_source_types if supported_source_types is None else frozenset(supported_source_types) + ) + missing_sources = () if configured is None else tuple(sorted(required_sources - configured)) + required_families = _required_artifact_families(_iter_records(source)) + missing_families = ( + () + if self._supported_artifact_families is None + else tuple(sorted(required_families - self._supported_artifact_families)) + ) + already_present, conflicts = await self._target_compatibility( + source, total=parsed.inspection.record_count, progress=progress + ) + return BundleValidation( + bundle_id=parsed.inspection.bundle_id, + scopes=parsed.inspection.scopes, + record_count=parsed.inspection.record_count, + records_by_type=parsed.inspection.records_by_type, + total_digest=parsed.inspection.total_digest, + format_version=parsed.inspection.format_version, + producer_version=parsed.inspection.producer_version, + already_present=already_present, + conflicts=conflicts, + projections_supported=self._projection_rebuilder is not None, + compatible=not missing_sources and not missing_families and conflicts == 0, + required_source_types=tuple(sorted(required_sources)), + unsupported_source_types=missing_sources, + required_artifact_families=tuple(sorted(required_families)), + unsupported_artifact_families=missing_families, + reader_version=_producer_version(), + ) + + async def restore( + self, + source: Path, + /, + *, + supported_source_types: Iterable[str] | None = None, + progress: ProgressObserver | None = None, + ) -> BundleReceipt: + """Restore a validated bundle atomically, skipping identical records. + + Existing rows are compared using the same portable record encoding. A + difference at the same primary identity aborts the entire transaction; + no existing revision or head is overwritten. + """ + + parsed = _parse_bundle(source) + validation = await self.validate(source, supported_source_types=supported_source_types, progress=progress) + if validation.conflicts: + raise BundleConflictError(f"target contains {validation.conflicts} conflicting record identities") + if not validation.compatible: + raise BundleFormatError("target is not compatible with bundle requirements") + try: + async with self._database.transaction() as connection: + inserted, existing = await self._write_records( + connection, source, total=parsed.inspection.record_count, progress=progress + ) + await self._write_restore_receipt( + connection, + parsed.inspection, + inserted=inserted, + already_present=existing, + projections_ready=False, + ) + except IntegrityError as error: + raise BundleConflictError("target constraint conflict during transactional restore") from error + projections_ready = False + if self._projection_rebuilder is not None: + _report_progress(progress, "projections", 0, 1) + await self._projection_rebuilder(parsed.inspection.scopes) + projections_ready = True + _report_progress(progress, "projections", 1, 1) + async with self._database.transaction() as connection: + await self._write_restore_receipt( + connection, + parsed.inspection, + inserted=inserted, + already_present=existing, + projections_ready=True, + ) + return BundleReceipt( + bundle_id=parsed.inspection.bundle_id, + record_count=parsed.inspection.record_count, + total_digest=parsed.inspection.total_digest, + inserted=inserted, + already_present=existing, + projections_ready=projections_ready, + status="ready" if projections_ready else "authoritative_restored", + ) + + async def _write_restore_receipt( + self, + connection: AsyncConnection, + inspection: BundleInspection, + /, + *, + inserted: int, + already_present: int, + projections_ready: bool, + ) -> None: + row = ( + ( + await connection.execute( + select(PORTABLE_RESTORE_RECEIPTS_TABLE).where( + PORTABLE_RESTORE_RECEIPTS_TABLE.c.bundle_id == inspection.bundle_id + ) + ) + ) + .mappings() + .one_or_none() + ) + if row is not None and ( + str(row["total_digest"]) != inspection.total_digest + or int(row["format_version"]) != inspection.format_version + or int(row["record_count"]) != inspection.record_count + ): + raise BundleConflictError("restore receipt bundle identity conflict") + values = { + "format_version": inspection.format_version, + "total_digest": inspection.total_digest, + "record_count": inspection.record_count, + "status": "ready" if projections_ready else "authoritative_restored", + "inserted": inserted, + "already_present": already_present, + "projections_ready": projections_ready, + "updated_at": datetime.now(UTC), + } + if row is None: + await connection.execute( + insert(PORTABLE_RESTORE_RECEIPTS_TABLE).values(bundle_id=inspection.bundle_id, **values) + ) + else: + await connection.execute( + update(PORTABLE_RESTORE_RECEIPTS_TABLE) + .where(PORTABLE_RESTORE_RECEIPTS_TABLE.c.bundle_id == inspection.bundle_id) + .values(**values) + ) + + async def _target_compatibility( + self, + source: Path, + /, + *, + total: int, + progress: ProgressObserver | None, + ) -> tuple[int, int]: + already_present = 0 + conflicts = 0 + async with self._database.transaction() as connection: + existing_tables = await connection.run_sync( + lambda sync_connection: { + spec.table.name + for spec in _SPECS.values() + if sqlalchemy_inspect(sync_connection).has_table(spec.table.name) + } + ) + for processed, record in enumerate(_iter_records(source), start=1): + table = _SPECS[record.record_type].table + if table.name not in existing_tables: + _report_progress(progress, "target", processed, total) + continue + where = [table.c[key] == _database_value(value) for key, value in record.identity.items()] + row = (await connection.execute(select(table).where(*where))).mappings().one_or_none() + if row is None: + _report_progress(progress, "target", processed, total) + continue + present = _record_from_row(record.record_type, cast(Mapping[str, Any], dict(row))) + if present.digest == record.digest: + already_present += 1 + else: + conflicts += 1 + _report_progress(progress, "target", processed, total) + return already_present, conflicts + + async def _write_records( + self, + connection: AsyncConnection, + source: Path, + /, + *, + total: int, + progress: ProgressObserver | None, + ) -> tuple[int, int]: + """Replay records in dependency order inside the caller transaction.""" + inserted = 0 + existing = 0 + processed = 0 + for record in _ordered_scope_records(source): + was_inserted = await self._write_record(connection, record) + inserted += int(was_inserted) + existing += int(not was_inserted) + processed += 1 + _report_progress(progress, "restore", processed, total) + # A portable bundle may be produced by another implementation, so do + # not trust its physical order. Re-scan once per dependency level + # instead of materializing its records merely to sort them. + for record_type in _EXPORT_ORDER: + if record_type == "scope": + continue + for record in _iter_records(source): + if record.record_type != record_type: + continue + was_inserted = await self._write_record(connection, record) + inserted += int(was_inserted) + existing += int(not was_inserted) + processed += 1 + _report_progress(progress, "restore", processed, total) + return inserted, existing + + async def _write_record(self, connection: AsyncConnection, record: _Record, /) -> bool: + table = _SPECS[record.record_type].table + where = [table.c[key] == _database_value(value) for key, value in record.identity.items()] + row = (await connection.execute(select(table).where(*where))).mappings().one_or_none() + if row is not None: + present = _record_from_row(record.record_type, cast(Mapping[str, Any], dict(row))) + if present.digest != record.digest: + raise BundleConflictError(f"immutable identity conflict: {record.record_type}") + return False + await connection.execute(insert(table).values(**_row_values(record))) + return True + + async def _stream_export( + self, + connection: AsyncConnection, + scopes: tuple[str, ...], + stream: Any, + *, + progress: ProgressObserver | None, + ) -> tuple[int, Counter[str], str]: + count = 0 + records_by_type: Counter[str] = Counter() + digest = _DigestAccumulator() + for record_type in _EXPORT_ORDER: + spec = _SPECS[record_type] + table = spec.table + statement = ( + select(table) + .where(table.c[spec.scope_field].in_(scopes)) + .order_by(*(table.c[field] for field in spec.identity)) + ) + rows = await connection.stream(statement) + async for row in rows.mappings(): + record = _record_from_row(record_type, cast(Mapping[str, Any], dict(row))) + _validate_record_safe(record) + stream.write(_canonical_json(_record_document(record)) + b"\n") + digest.add(record.digest) + count += 1 + records_by_type[record.record_type] += 1 + _report_progress(progress, "export", count, None) + _report_progress(progress, "export", count, count) + return count, records_by_type, digest.value() + + +def _record_from_row(record_type: RecordType, row: Mapping[str, Any]) -> _Record: + spec = _SPECS[record_type] + identity = {field: _json_value(row[field]) for field in spec.identity} + payload = {field: _json_value(row[field]) for field in spec.payload} + canonical = {"record_type": record_type, "schema_version": 1, "identity": identity, "payload": payload} + return _Record(record_type=record_type, identity=identity, payload=payload, digest=_digest(canonical)) + + +async def _establish_export_snapshot(connection: AsyncConnection, /) -> None: + """Pin all export reads to one database snapshot before enumeration.""" + + if connection.dialect.name == "mysql": + # AsyncDatabase has already issued START TRANSACTION for MySQL-mode + # engines. Replacing that empty transaction with a consistent snapshot + # is safe because authorization has completed and no query has run yet. + await connection.exec_driver_sql("START TRANSACTION WITH CONSISTENT SNAPSHOT") + + +def _record_document(record: _Record) -> dict[str, Any]: + return { + "record_type": record.record_type, + "schema_version": 1, + "identity": dict(record.identity), + "payload": dict(record.payload), + "digest": record.digest, + } + + +def _row_values(record: _Record) -> dict[str, Any]: + values = {key: _database_value(value) for key, value in record.identity.items()} + values.update({key: _database_value(value) for key, value in record.payload.items()}) + # Projections are intentionally excluded from logical bundles. Heads may + # use NULL searchable text; Memory projections are rebuilt by the runtime. + if record.record_type == "artifact_head": + values["searchable_text"] = None + elif record.record_type == "memory_entry_head": + values["searchable_text"] = str(record.payload["entry_content_hash"]) + elif record.record_type == "scope": + values["scope_id_search"] = str(record.identity["scope_id"]) + values["title_search"] = str(record.payload["title"]) + values["summary_search"] = str(record.payload["summary"]) + elif record.record_type == "scope_external_reference": + values["value_search"] = str(record.payload["value"]) + return values + + +def _parse_bundle( + source: Path, + *, + progress: ProgressObserver | None = None, + phase: Literal["inspect", "validate"] = "inspect", +) -> _ParsedBundle: + try: + with zipfile.ZipFile(source) as archive: + _validate_container(archive) + with archive.open(_MANIFEST_NAME) as stream: + manifest = json.loads(stream.read(_MAX_MANIFEST_BYTES + 1), parse_constant=_reject_json_constant) + _validate_manifest(manifest) + except (OSError, json.JSONDecodeError, zipfile.BadZipFile) as error: + raise BundleFormatError("cannot read portable bundle") from error + count = 0 + counts: Counter[str] = Counter() + digest = _DigestAccumulator() + for record in _iter_records(source): + count += 1 + counts[record.record_type] += 1 + digest.add(record.digest) + _report_progress(progress, phase, count, manifest["record_count"]) + expected_count = manifest["record_count"] + if count != expected_count: + raise BundleFormatError("manifest record count does not match records") + if digest.value() != manifest["total_digest"]: + raise BundleFormatError("bundle total digest does not match records") + sorted_counts = dict(sorted(counts.items())) + if sorted_counts != manifest["records_by_type"]: + raise BundleFormatError("manifest record classes do not match records") + inspection = BundleInspection( + bundle_id=manifest["bundle_id"], + scopes=tuple(manifest["scopes"]), + record_count=count, + records_by_type=sorted_counts, + total_digest=digest.value(), + format_version=manifest["format_version"], + producer_version=manifest["producer"]["version"], + ) + scope_ids = tuple( + str(record.identity["scope_id"]) for record in _iter_records(source) if record.record_type == "scope" + ) + if tuple(sorted(scope_ids)) != inspection.scopes: + raise BundleFormatError("manifest scopes do not match scope records") + return _ParsedBundle(inspection=inspection) + + +def _ordered_scope_records(source: Path, /) -> tuple[_Record, ...]: + pending = { + str(record.identity["scope_id"]): record for record in _iter_records(source) if record.record_type == "scope" + } + ordered: list[_Record] = [] + while pending: + ready = sorted( + scope_id + for scope_id, record in pending.items() + if record.payload["parent_scope_id"] is None or record.payload["parent_scope_id"] not in pending + ) + if not ready: + raise BundleFormatError("scope hierarchy contains a cycle") + for scope_id in ready: + ordered.append(pending.pop(scope_id)) + return tuple(ordered) + + +def _iter_records(source: Path, /) -> Iterator[_Record]: + """Yield verified records without retaining the archive body in memory.""" + + try: + with zipfile.ZipFile(source) as archive, archive.open(_RECORDS_NAME) as stream: + count = 0 + while line := stream.readline(_MAX_LINE_BYTES + 1): + if len(line) > _MAX_LINE_BYTES: + raise BundleFormatError("record line exceeds size limit") + if not line.strip(): + continue + count += 1 + if count > _MAX_RECORDS: + raise BundleFormatError("record count exceeds limit") + yield _parse_record(json.loads(line, parse_constant=_reject_json_constant)) + except (OSError, json.JSONDecodeError, zipfile.BadZipFile) as error: + raise BundleFormatError("cannot read portable bundle") from error + + +def _validate_container(archive: zipfile.ZipFile) -> None: + items = archive.infolist() + names = [item.filename for item in items] + if len(items) != 2 or set(names) != {_MANIFEST_NAME, _RECORDS_NAME}: + raise BundleFormatError("bundle must contain only manifest.json and records.ndjson") + if any( + item.is_dir() + or item.compress_type not in {zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED} + or item.compress_size > _MAX_ARCHIVE_BYTES + for item in items + ): + raise BundleFormatError("bundle exceeds resource limits") + if archive.getinfo(_MANIFEST_NAME).file_size > _MAX_MANIFEST_BYTES: + raise BundleFormatError("manifest exceeds resource limit") + if archive.getinfo(_RECORDS_NAME).file_size > _MAX_RECORDS_BYTES: + raise BundleFormatError("records exceed resource limit") + + +def _validate_manifest(manifest: object) -> None: # noqa: C901 - manifest fields have independent constraints. + if not isinstance(manifest, dict): + raise BundleFormatError("manifest must be an object") + manifest = cast(dict[str, object], manifest) + required = { + "format_version", + "bundle_id", + "producer", + "scopes", + "record_count", + "records_by_type", + "excluded_record_classes", + "total_digest", + } + if set(manifest) != required: + raise BundleFormatError("manifest has unknown or missing fields") + if manifest["format_version"] != FORMAT_VERSION: + raise BundleFormatError("unsupported bundle format version") + if ( + not isinstance(manifest["bundle_id"], str) + or isinstance(manifest["record_count"], bool) + or not isinstance(manifest["record_count"], int) + or manifest["record_count"] < 0 + or manifest["record_count"] > _MAX_RECORDS + ): + raise BundleFormatError("manifest has invalid identity or record count") + try: + UUID(manifest["bundle_id"]) + except ValueError as error: + raise BundleFormatError("manifest has invalid bundle identity") from error + producer = manifest["producer"] + if not isinstance(producer, dict): + raise BundleFormatError("manifest has invalid producer") + typed_producer = cast(dict[str, object], producer) + producer_version = typed_producer.get("version") + if ( + set(typed_producer) != {"name", "version"} + or typed_producer.get("name") != "powercontext" + or not isinstance(producer_version, str) + or not producer_version + ): + raise BundleFormatError("manifest has invalid producer") + scopes = manifest["scopes"] + if not isinstance(scopes, list) or not all(isinstance(scope, str) for scope in scopes): + raise BundleFormatError("manifest has invalid scopes") + typed_scopes = cast(list[str], scopes) + if _validate_scopes(typed_scopes) != tuple(typed_scopes): + raise BundleFormatError("manifest has invalid scopes") + counts = manifest["records_by_type"] + if ( + not isinstance(counts, dict) + or any( + key not in _SPECS or isinstance(value, bool) or not isinstance(value, int) or value < 0 + for key, value in counts.items() + ) + or sum(cast(dict[str, int], counts).values()) != manifest["record_count"] + or manifest["excluded_record_classes"] != list(_EXCLUDED_RECORD_CLASSES) + or not _valid_digest(manifest["total_digest"]) + ): + raise BundleFormatError("manifest has invalid checksums") + + +def _parse_record(value: object) -> _Record: + if not isinstance(value, dict): + raise BundleFormatError("record must be an object") + value = cast(dict[str, object], value) + if set(value) != {"record_type", "schema_version", "identity", "payload", "digest"}: + raise BundleFormatError("record has unknown or missing fields") + record_type = value.get("record_type") + if record_type not in _SPECS or value.get("schema_version") != 1: + raise BundleFormatError("unsupported record type or schema") + identity = value.get("identity") + payload = value.get("payload") + digest = value.get("digest") + if not isinstance(identity, dict) or not isinstance(payload, dict) or not isinstance(digest, str): + raise BundleFormatError("record has invalid fields") + identity = cast(dict[str, str | int], identity) + payload = cast(dict[str, Any], payload) + typed = cast(RecordType, record_type) + spec = _SPECS[typed] + if ( + set(identity) != set(spec.identity) + or set(payload) != set(spec.payload) + or any(isinstance(item, bool) or not isinstance(item, str | int) for item in identity.values()) + or not _valid_digest(digest) + ): + raise BundleFormatError("record has invalid identity or digest") + canonical = {"record_type": typed, "schema_version": 1, "identity": identity, "payload": payload} + if _digest(canonical) != digest: + raise BundleFormatError("record digest does not match content") + record = _Record(record_type=typed, identity=identity, payload=payload, digest=digest) + _validate_record_values(record) + _validate_record_safe(record) + return record + + +def _validate_record_values(record: _Record, /) -> None: # noqa: C901 - one versioned schema contract. + identity = record.identity + payload = record.payload + if any(isinstance(value, str) and (not value or value != value.strip()) for value in identity.values()): + raise BundleFormatError("record identity contains an invalid string") + positive_identity_fields = {"revision", "version"} + for field, value in identity.items(): + if isinstance(value, int) and (value < 0 or (field in positive_identity_fields and value == 0)): + raise BundleFormatError("record identity contains an invalid integer") + + def strings(*fields: str, nullable: tuple[str, ...] = ()) -> None: + for field in fields: + value = payload[field] + if field in nullable and value is None: + continue + if not isinstance(value, str): + raise BundleFormatError("record payload has an invalid string field") + + def integers(*fields: str, positive: bool = False) -> None: + for field in fields: + value = payload[field] + if isinstance(value, bool) or not isinstance(value, int) or value < int(positive): + raise BundleFormatError("record payload has an invalid integer field") + + def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: + for field in fields: + value = payload[field] + if field in nullable and value is None: + continue + encoded = cast(dict[str, object], value).get("base64") if isinstance(value, dict) else None + if not isinstance(value, dict) or set(value) != {"base64"} or not isinstance(encoded, str): + raise BundleFormatError("record payload has an invalid binary field") + try: + base64.b64decode(encoded, validate=True) + except ValueError as error: + raise BundleFormatError("record payload has an invalid binary field") from error + + kind = record.record_type + if kind == "scope": + strings("title", "summary", "parent_scope_id", nullable=("parent_scope_id",)) + integers("version", positive=True) + elif kind == "scope_context_reference": + if identity["scope_id"] == identity["referenced_scope_id"]: + raise BundleFormatError("scope cannot reference itself") + elif kind == "scope_external_reference": + strings("kind", "value", "value_digest") + elif kind == "scope_creation_request": + strings("request_digest", "scope_id") + elif kind == "source_journal_head": + integers("position") + elif kind == "source": + bytes_values("payload") + integers("journal_position", positive=True) + elif kind == "artifact_revision": + bytes_values("content", "memory_citations", nullable=("memory_citations",)) + elif kind == "artifact_lineage_source": + strings("source_type", "source_id") + elif kind == "artifact_lineage_artifact": + strings("upstream_family", "upstream_artifact_id") + integers("upstream_revision", positive=True) + elif kind == "artifact_publication": + strings("source_scope_id", "source_family", "source_artifact_id", "content_digest", "idempotency_key") + integers("source_revision", positive=True) + elif kind == "artifact_head": + strings("lifecycle_state", "replacement_artifact_id", nullable=("replacement_artifact_id",)) + integers("revision", positive=True) + integers("governance_generation") + if payload["lifecycle_state"] not in {"active", "deprecated", "retired"} or ( + payload["replacement_artifact_id"] is not None and payload["lifecycle_state"] != "deprecated" + ): + raise BundleFormatError("artifact head has invalid governance state") + elif kind == "memory_entry_version": + strings( + "family", + "entry_id", + "previous_version_id", + "kind", + "text", + "entry_content_hash", + nullable=("previous_version_id",), + ) + integers("version", "created_in_revision", positive=True) + bytes_values("source_refs", "artifact_refs") + elif kind == "memory_entry_head": + strings("family", "entry_version_id", "entry_content_hash") + integers("head_revision", positive=True) + elif kind == "candidate_version": + strings( + "family", + "target_family", + "target_artifact_id", + "reason", + nullable=("target_family", "target_artifact_id", "reason"), + ) + bytes_values("proposal", "source_refs", "artifact_refs", "memory_citations", nullable=("memory_citations",)) + target = (payload["target_family"], payload["target_artifact_id"], payload["target_revision"]) + if target != (None, None, None) and ( + target[0] is None + or target[1] is None + or isinstance(target[2], bool) + or not isinstance(target[2], int) + or target[2] < 1 + ): + raise BundleFormatError("candidate version has an incomplete target") + elif kind == "candidate_head": + strings( + "family", + "status", + "result_family", + "result_artifact_id", + "decision_reason", + nullable=("result_family", "result_artifact_id", "decision_reason"), + ) + integers("version", positive=True) + result = (payload["result_family"], payload["result_artifact_id"], payload["result_revision"]) + status = payload["status"] + valid = ( + (status == "pending" and result == (None, None, None) and payload["decision_reason"] is None) + or (status == "rejected" and result == (None, None, None) and payload["decision_reason"] is not None) + or ( + status == "approved" + and all(value is not None for value in result) + and isinstance(result[2], int) + and not isinstance(result[2], bool) + and result[2] > 0 + and payload["decision_reason"] is None + ) + ) + if not valid: + raise BundleFormatError("candidate head has invalid decision state") + + +async def _validate_selected_scopes(connection: AsyncConnection, scopes: tuple[str, ...], /) -> None: + selected = set(scopes) + rows = tuple( + ( + await connection.execute( + select(SCOPES_TABLE.c.scope_id, SCOPES_TABLE.c.parent_scope_id).where( + SCOPES_TABLE.c.scope_id.in_(scopes) + ) + ) + ).mappings() + ) + if {str(row["scope_id"]) for row in rows} != selected: + raise BundleFormatError("one or more selected scopes do not exist") + if any(row["parent_scope_id"] is not None and str(row["parent_scope_id"]) not in selected for row in rows): + raise BundleFormatError("selected scope set omits a required parent scope") + referenced = ( + await connection.execute( + select(SCOPE_CONTEXT_REFERENCES_TABLE.c.referenced_scope_id).where( + SCOPE_CONTEXT_REFERENCES_TABLE.c.scope_id.in_(scopes) + ) + ) + ).scalars() + if any(str(scope_id) not in selected for scope_id in referenced): + raise BundleFormatError("selected scope set omits a referenced context scope") + publication_sources = ( + await connection.execute( + select(ARTIFACT_PUBLICATIONS_TABLE.c.source_scope_id).where( + ARTIFACT_PUBLICATIONS_TABLE.c.target_scope_id.in_(scopes) + ) + ) + ).scalars() + if any(str(scope_id) not in selected for scope_id in publication_sources): + raise BundleFormatError("selected scope set omits an Artifact publication source scope") + + +def _validate_record_safe(record: _Record, /) -> None: + """Reject secret-shaped configuration and host paths without echoing content.""" + + for field, value in record.payload.items(): + encoded = cast(dict[str, object], value).get("base64") if isinstance(value, dict) else None + if isinstance(encoded, str): + try: + decoded = json.loads(base64.b64decode(encoded, validate=True)) + except (ValueError, UnicodeDecodeError, json.JSONDecodeError): + continue + _validate_safe_value(decoded, field) + else: + _validate_safe_value(value, field) + + +def _validate_safe_value(value: object, key: str | None = None) -> None: + normalized = "" if key is None else key.casefold().replace("-", "_") + sensitive_suffixes = ("_api_key", "_credential", "_credentials", "_password", "_secret", "_token") + if (normalized in _SENSITIVE_KEYS or normalized.endswith(sensitive_suffixes)) and value is not None and value != "": + raise BundleFormatError("bundle contains a credential-shaped field") + if normalized == "authorization" and isinstance(value, str) and value.casefold().startswith(("bearer ", "basic ")): + raise BundleFormatError("bundle contains a credential-shaped field") + if ( + isinstance(value, str) + and ( + normalized.endswith(("_path", "_directory", "_home", "_root")) + or normalized in {"cwd", "directory", "home", "path", "root", "working_directory", "workspace"} + ) + and _ABSOLUTE_HOST_PATH.match(value) + ): + raise BundleFormatError("bundle contains an absolute host path") + if isinstance(value, dict): + for child_key, child in value.items(): + _validate_safe_value(child, str(child_key)) + elif isinstance(value, list): + for child in value: + _validate_safe_value(child, key) + + +def _validate_dependencies(source: Path, /) -> None: + """Check references with a temporary on-disk identity index. + + The index deliberately stores only canonical identities, never record + payloads. This permits dependency validation for large bundles without + retaining their content in the importing process. + """ + + with _identity_index(source) as contains: + for record in _iter_records(source): + _validate_record_dependencies(record, contains) + + +def _validate_record_dependencies( # noqa: C901 - logical record kinds have distinct dependency edges. + record: _Record, + contains: Callable[[RecordType, Mapping[str, object]], bool], + /, +) -> None: + payload = record.payload + identity = record.identity + scope_field = _SPECS[record.record_type].scope_field + scope = identity.get(scope_field, payload.get(scope_field)) + if not isinstance(scope, str): + raise BundleFormatError("record has invalid scope identity") + if record.record_type != "scope" and not contains("scope", {"scope_id": scope}): + raise BundleFormatError("record references missing scope") + if record.record_type == "scope": + parent = payload["parent_scope_id"] + if parent is not None and not contains("scope", {"scope_id": parent}): + raise BundleFormatError("scope references missing parent") + elif record.record_type == "scope_context_reference": + if not contains("scope", {"scope_id": identity["referenced_scope_id"]}): + raise BundleFormatError("scope context references missing scope") + elif record.record_type == "source": + if not contains("source_journal_head", {"scope_id": scope}): + raise BundleFormatError("source has no journal head") + elif record.record_type == "artifact_lineage_source": + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": identity["family"], + "artifact_id": identity["artifact_id"], + "revision": identity["revision"], + }, + ): + raise BundleFormatError("artifact lineage references missing revision") + if not contains( + "source", + {"scope_id": scope, "source_type": payload["source_type"], "source_id": payload["source_id"]}, + ): + raise BundleFormatError("artifact lineage references missing source") + elif record.record_type == "artifact_lineage_artifact": + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": identity["family"], + "artifact_id": identity["artifact_id"], + "revision": identity["revision"], + }, + ): + raise BundleFormatError("artifact lineage references missing revision") + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": payload["upstream_family"], + "artifact_id": payload["upstream_artifact_id"], + "revision": payload["upstream_revision"], + }, + ): + raise BundleFormatError("record references missing artifact revision") + elif record.record_type == "artifact_publication": + if not contains("scope", {"scope_id": payload["source_scope_id"]}): + raise BundleFormatError("Artifact publication references missing source scope") + target_exists = contains( + "artifact_revision", + { + "scope_id": identity["target_scope_id"], + "family": identity["target_family"], + "artifact_id": identity["target_artifact_id"], + "revision": identity["target_revision"], + }, + ) + source_exists = contains( + "artifact_revision", + { + "scope_id": payload["source_scope_id"], + "family": payload["source_family"], + "artifact_id": payload["source_artifact_id"], + "revision": payload["source_revision"], + }, + ) + if not target_exists or not source_exists: + raise BundleFormatError("Artifact publication references missing revision") + elif record.record_type == "memory_entry_version": + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": payload["family"], + "artifact_id": identity["memory_artifact_id"], + "revision": payload["created_in_revision"], + }, + ): + raise BundleFormatError("record references missing artifact revision") + previous = payload["previous_version_id"] + if previous is not None and not contains( + "memory_entry_version", + { + "scope_id": scope, + "memory_artifact_id": identity["memory_artifact_id"], + "entry_version_id": previous, + }, + ): + raise BundleFormatError("memory version references missing previous version") + elif record.record_type == "artifact_head": + if not contains("artifact_revision", {**identity, "revision": payload["revision"]}): + raise BundleFormatError("artifact head references missing revision") + elif record.record_type == "memory_entry_head": + if not contains( + "memory_entry_version", + { + "scope_id": scope, + "memory_artifact_id": identity["memory_artifact_id"], + "entry_version_id": payload["entry_version_id"], + }, + ): + raise BundleFormatError("memory head references missing entry version") + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": payload["family"], + "artifact_id": identity["memory_artifact_id"], + "revision": payload["head_revision"], + }, + ): + raise BundleFormatError("memory head references missing artifact revision") + elif record.record_type == "candidate_version": + target_family = payload["target_family"] + if target_family is not None and not contains( + "artifact_revision", + { + "scope_id": scope, + "family": target_family, + "artifact_id": payload["target_artifact_id"], + "revision": payload["target_revision"], + }, + ): + raise BundleFormatError("candidate references missing target revision") + elif record.record_type == "candidate_head": + if not contains( + "candidate_version", + {"scope_id": scope, "candidate_id": identity["candidate_id"], "version": payload["version"]}, + ): + raise BundleFormatError("candidate head references missing version") + result_family = payload["result_family"] + if result_family is not None and not contains( + "artifact_revision", + { + "scope_id": scope, + "family": result_family, + "artifact_id": payload["result_artifact_id"], + "revision": payload["result_revision"], + }, + ): + raise BundleFormatError("candidate head references missing result revision") + _validate_embedded_references((record,), contains) + _validate_memory_citations(record, contains) + + +@contextmanager +def _identity_index(source: Path, /) -> Iterator[Callable[[RecordType, Mapping[str, object]], bool]]: + index_path = _temporary_path(Path(tempfile.gettempdir()), suffix=".identities.sqlite3") + connection: sqlite3.Connection | None = None + try: + connection = sqlite3.connect(index_path) + connection.execute( + "CREATE TABLE identities (record_type TEXT NOT NULL, identity TEXT NOT NULL, " + "PRIMARY KEY (record_type, identity))" + ) + for record in _iter_records(source): + try: + connection.execute( + "INSERT INTO identities (record_type, identity) VALUES (?, ?)", + (record.record_type, _identity_key(record.identity)), + ) + except sqlite3.IntegrityError as error: + raise BundleFormatError("bundle contains duplicate immutable identity") from error + connection.commit() + + def contains(record_type: RecordType, identity: Mapping[str, object]) -> bool: + return ( + connection.execute( + "SELECT 1 FROM identities WHERE record_type = ? AND identity = ?", + (record_type, _identity_key(identity)), + ).fetchone() + is not None + ) + + yield contains + finally: + if connection is not None: + connection.close() + index_path.unlink(missing_ok=True) + + +def _validate_embedded_references( + records: Iterable[_Record], + contains: Callable[[RecordType, Mapping[str, object]], bool], +) -> None: + for record in records: + if record.record_type not in {"memory_entry_version", "candidate_version"}: + continue + scope_id = record.identity["scope_id"] + for reference in _reference_items(record.payload["source_refs"], "source"): + if not contains( + "source", + { + "scope_id": scope_id, + "source_type": reference["source_type"], + "source_id": reference["source_id"], + }, + ): + raise BundleFormatError("record references missing source") + for reference in _reference_items(record.payload["artifact_refs"], "artifact"): + if not contains( + "artifact_revision", + { + "scope_id": scope_id, + "family": reference["family"], + "artifact_id": reference["artifact_id"], + "revision": reference["revision"], + }, + ): + raise BundleFormatError("record references missing artifact revision") + + +def _reference_items(value: object, kind: Literal["source", "artifact"], /) -> tuple[Mapping[str, object], ...]: + encoded = cast(dict[str, object], value).get("base64") if isinstance(value, dict) else None + if not isinstance(value, dict) or set(value) != {"base64"} or not isinstance(encoded, str): + raise BundleFormatError(f"{kind} references are not encoded bytes") + try: + decoded = json.loads(base64.b64decode(encoded, validate=True), parse_constant=_reject_json_constant) + except (ValueError, json.JSONDecodeError) as error: + raise BundleFormatError(f"{kind} references are not valid JSON") from error + if not isinstance(decoded, list): + raise BundleFormatError(f"{kind} references must be an array") + fields = {"source_type", "source_id"} if kind == "source" else {"family", "artifact_id", "revision"} + references: list[Mapping[str, object]] = [] + for item in decoded: + if ( + not isinstance(item, dict) + or set(item) != fields + or any(isinstance(field, bool) or not isinstance(field, str | int) for field in item.values()) + ): + raise BundleFormatError(f"{kind} reference has invalid fields") + references.append(cast(Mapping[str, object], item)) + return tuple(references) + + +def _validate_memory_citations( + record: _Record, + contains: Callable[[RecordType, Mapping[str, object]], bool], + /, +) -> None: + if record.record_type not in {"artifact_revision", "candidate_version"}: + return + encoded_value = record.payload["memory_citations"] + if encoded_value is None: + return + encoded = cast(dict[str, object], encoded_value).get("base64") if isinstance(encoded_value, dict) else None + if not isinstance(encoded, str): + raise BundleFormatError("memory citations are not encoded bytes") + try: + citations = json.loads(base64.b64decode(encoded, validate=True), parse_constant=_reject_json_constant) + except (ValueError, json.JSONDecodeError) as error: + raise BundleFormatError("memory citations are not valid JSON") from error + if not isinstance(citations, list): + raise BundleFormatError("memory citations must be an array") + scope_id = record.identity["scope_id"] + for citation in citations: + if not isinstance(citation, dict) or set(citation) != {"memory_ref", "entry_id", "entry_version_id"}: + raise BundleFormatError("memory citation has invalid fields") + memory_ref = citation["memory_ref"] + if ( + not isinstance(memory_ref, dict) + or set(memory_ref) != {"family", "artifact_id", "revision"} + or not isinstance(citation["entry_id"], str) + or not isinstance(citation["entry_version_id"], str) + ): + raise BundleFormatError("memory citation has invalid fields") + artifact_exists = contains( + "artifact_revision", + { + "scope_id": scope_id, + "family": memory_ref["family"], + "artifact_id": memory_ref["artifact_id"], + "revision": memory_ref["revision"], + }, + ) + version_exists = contains( + "memory_entry_version", + { + "scope_id": scope_id, + "memory_artifact_id": memory_ref["artifact_id"], + "entry_version_id": citation["entry_version_id"], + }, + ) + if not artifact_exists or not version_exists: + raise BundleFormatError("memory citation references missing version") + + +def _required_artifact_families(records: Iterable[_Record]) -> set[str]: + families: set[str] = set() + for record in records: + identity = record.identity + payload = record.payload + if record.record_type in { + "artifact_revision", + "artifact_lineage_source", + "artifact_lineage_artifact", + "artifact_head", + }: + families.add(str(identity["family"])) + if record.record_type == "artifact_publication": + families.update((str(identity["target_family"]), str(payload["source_family"]))) + if record.record_type in {"artifact_lineage_artifact", "candidate_version", "candidate_head"}: + for field in ("upstream_family", "target_family", "result_family"): + value = payload.get(field) + if value is not None: + families.add(str(value)) + if record.record_type in {"memory_entry_version", "memory_entry_head", "candidate_version", "candidate_head"}: + families.add(str(payload["family"])) + return families + + +def _validate_scopes(scopes: Iterable[str]) -> tuple[str, ...]: + if isinstance(scopes, str): + raise BundleFormatError("scopes must be an iterable of scope IDs") + try: + values = tuple(scopes) + except TypeError as error: + raise BundleFormatError("scopes must be an iterable of scope IDs") from error + if not values or any(not isinstance(scope, str) or not scope.strip() or scope != scope.strip() for scope in values): + raise BundleFormatError("scopes must contain at least one non-empty trimmed scope ID") + selected = tuple(sorted(set(values))) + return selected + + +def _identity_key(identity: Mapping[str, object], /) -> str: + return _canonical_json(identity).decode("utf-8") + + +class _DigestAccumulator: + """Incrementally hash a deterministic ordered record-digest sequence.""" + + def __init__(self) -> None: + self._hash = hashlib.sha256() + self._first = True + + def add(self, digest: str, /) -> None: + if not self._first: + self._hash.update(b"\n") + self._hash.update(digest.encode("ascii")) + self._first = False + + def value(self) -> str: + return _SHA256 + self._hash.hexdigest() + + +def _temporary_path(directory: Path, /, *, suffix: str) -> Path: + with tempfile.NamedTemporaryFile( + prefix=".powercontext-archive-", suffix=suffix, dir=directory, delete=False + ) as handle: + return Path(handle.name) + + +def _zip_info(name: str, compression: int, /) -> zipfile.ZipInfo: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = compression + info.external_attr = 0o600 << 16 + return info + + +def _digest(value: object) -> str: + return _SHA256 + hashlib.sha256(_canonical_json(value)).hexdigest() + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8") + + +def _json_value(value: object) -> Any: + if isinstance(value, bytes | bytearray | memoryview): + return {"base64": base64.b64encode(bytes(value)).decode("ascii")} + if isinstance(value, datetime | date): + return value.isoformat() + return value + + +def _database_value(value: object) -> object: + encoded = cast(dict[str, object], value).get("base64") if isinstance(value, dict) else None + if isinstance(value, dict) and set(value) == {"base64"} and isinstance(encoded, str): + try: + return base64.b64decode(encoded, validate=True) + except ValueError as error: + raise BundleFormatError("invalid base64 column value") from error + return value + + +def _valid_digest(value: object) -> bool: + return isinstance(value, str) and _HEX_DIGEST.fullmatch(value) is not None + + +def _reject_json_constant(_value: str, /) -> None: + raise BundleFormatError("bundle contains a non-standard JSON value") + + +def _report_progress( + observer: ProgressObserver | None, + phase: Literal["export", "inspect", "validate", "target", "restore", "projections"], + processed: int, + total: int | None, +) -> None: + if observer is not None and (processed in {0, 1, total} or processed % 1_000 == 0): + observer(BundleProgress(phase=phase, processed=processed, total=total)) + + +def _producer_version() -> str: + try: + return version("powercontext") + except PackageNotFoundError: + return "unknown" diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index 0f3b22b0a5..e9fd3042ec 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -267,6 +267,7 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler from powercontext.builtin.handoff_report.application import HandoffReportApplication + from powercontext.builtin.portability import PortableBundleService from powercontext.builtin.runtime.artifact_processing import ArtifactProcessingSupervisors TopicMemorySearch = Callable[..., Awaitable[TopicMemorySearchResult]] @@ -2236,6 +2237,7 @@ def __init__( remote_ingestion: RemoteIngestion | None = None, dream_service: DreamService | None = None, generation_concurrency: int = 4, + archive_service: PortableBundleService | None = None, ) -> None: if source_window_limit < 1: raise _RuntimeConfigurationError("source_window_limit") @@ -2319,6 +2321,7 @@ def __init__( self.skill = SkillApplication(self) self.remote_skills = RemoteSkillApplication(self) self.statistics = StatisticsApplication(self) + self.archive = archive_service self.handoff_report: HandoffReportApplication | None = None self.artifact_processing_supervisor: ArtifactProcessingSupervisors | None = None self.processor = None if scope_ids is None else ScheduledSourceProcessor(self, scope_ids) diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index 6994c880af..063d06f68b 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -516,6 +516,7 @@ async def run_profile(scope_id, high): scheduled_source_runner=scheduled_source_runner, scheduled_experience_runner=scheduled_experience_runner, remote_ingestion=contexts, + archive_service=contexts.portability, ) ) runtime.artifact_processing_supervisor = await resources.enter_async_context( diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index f5e423c916..47f6b9f8ef 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -150,6 +150,7 @@ from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, SOURCE_JOURNAL_HEADS_TABLE from powercontext.builtin.persistence.topic_memory import TopicMemoryRepository from powercontext.builtin.persistence.topic_memory_index import NoTopicMemoryIndex, TopicMemoryIndex +from powercontext.builtin.portability import PortableBundleService from powercontext.builtin.publication import ArtifactPublicationApplication from powercontext.builtin.review.generation import ( GeneratedCandidateResult, @@ -487,6 +488,12 @@ def __init__( self.database = database self.scopes = ScopeApplication(database, cursor_secret=cursor_secret) self.source_registry = source_registry or BUILTIN_SOURCE_REGISTRY + self.portability = PortableBundleService( + database, + projection_rebuilder=self.rebuild_portable_projections, + supported_source_types=tuple(definition.name for definition in self.source_registry.definitions), + supported_artifact_families=(Handoff.family, Memory.family, Experience.family, Skill.family), + ) self.index = NoMemoryIndex() if index is None else index self.topic_memory_index = NoTopicMemoryIndex() if topic_memory_index is None else topic_memory_index self.experience_index = NoExperienceIndex() if experience_index is None else experience_index @@ -1195,6 +1202,16 @@ async def handoff_scope_ids(self) -> tuple[str, ...]: ).scalars() return tuple(str(value) for value in values) + async def rebuild_portable_projections(self, scope_ids: tuple[str, ...], /) -> None: + """Rebuild target-local search projections after a logical restore.""" + + for scope_id in scope_ids: + services = self._services_for(scope_id) + _, catalog = services.sources() + await services.memory(catalog).rebuild_projections() + async with self.database.transaction() as connection: + await self.experience_index.initialize(connection) + async def process_memory( self, scope_id: str, diff --git a/src/powercontext/cli/archive.py b/src/powercontext/cli/archive.py new file mode 100644 index 0000000000..95dd2f2e46 --- /dev/null +++ b/src/powercontext/cli/archive.py @@ -0,0 +1,212 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Operator-facing portable bundle commands using a controlled built-in Runtime.""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import Awaitable, Callable +from dataclasses import asdict +from pathlib import Path +from typing import Annotated, TypeVar, assert_never + +import typer +from sqlalchemy.engine import make_url +from sqlalchemy.ext.asyncio import create_async_engine + +from powercontext.builtin.artifacts.experience import Experience +from powercontext.builtin.artifacts.handoff import Handoff +from powercontext.builtin.artifacts.memory import Memory +from powercontext.builtin.artifacts.skill import Skill +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile +from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.portability import ( + BundleConflictError, + BundleFormatError, + BundleInspection, + BundleProgress, + BundleReceipt, + BundleValidation, + PortableBundleService, +) +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_runtime +from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY +from powercontext.paths import default_scheduler_path +from powercontext.server.configuration import server_settings_context + +HELP_OPTION_NAMES = ("-h", "--help") +_ResultT = TypeVar("_ResultT", BundleInspection, BundleValidation, BundleReceipt) + +archive_app = typer.Typer( + name="archive", + context_settings={"help_option_names": HELP_OPTION_NAMES}, + help="Create, inspect, validate, and restore portable logical bundles.", + no_args_is_help=True, +) + + +@archive_app.command("export") +def export_bundle( + scope_id: Annotated[ + list[str], typer.Option("--scope-id", min=1, help="Complete scope to export; repeat as needed.") + ], + output: Annotated[Path, typer.Option("--output", help="New portable bundle file to create.")], + compress: Annotated[bool, typer.Option("--compress/--no-compress", help="Compress the portable bundle.")] = True, + env_file: Annotated[ + Path | None, typer.Option("--env-file", help="Deployment environment for SQLite, SeekDB, or OceanBase.") + ] = None, +) -> None: + """Create a verified logical bundle from complete scopes.""" + + _emit_archive_result( + lambda archive: archive.export( + scope_id, output, authorize=_authorize_local_export, progress=_emit_progress, compress=compress + ), + env_file=env_file, + ) + + +@archive_app.command("inspect") +def inspect_bundle( + source: Annotated[Path, typer.Argument(help="Portable bundle file to inspect.")], +) -> None: + """Verify archive structure and checksums without writing domain data.""" + + try: + _emit(asyncio.run(PortableBundleService.inspect(source, progress=_emit_progress))) + except BundleFormatError as error: + typer.echo(f"error: archive validation failed: {error}", err=True) + raise typer.Exit(code=2) from error + + +@archive_app.command("restore") +def restore_bundle( + source: Annotated[Path, typer.Argument(help="Portable bundle file to validate or restore.")], + dry_run: Annotated[bool, typer.Option("--dry-run", help="Validate only; never write domain data.")] = False, + yes: Annotated[bool, typer.Option("--yes", help="Confirm a write restore.")] = False, + env_file: Annotated[ + Path | None, typer.Option("--env-file", help="Target deployment environment for validation or restore.") + ] = None, +) -> None: + """Validate or restore a portable bundle into the configured deployment.""" + + if dry_run: + _emit_validation_result(source, env_file=env_file) + return + if not yes: + raise typer.BadParameter( # noqa: TRY003 + "restore writes immutable records; re-run with --yes to confirm", + param_hint="--yes", + ) + _emit_archive_result(lambda archive: archive.restore(source, progress=_emit_progress), env_file=env_file) + + +async def _run_archive( + operation: Callable[[PortableBundleService], Awaitable[_ResultT]], + /, + *, + env_file: Path | None, +) -> _ResultT: + with server_settings_context(env_file=env_file) as settings: + config = BuiltinConfig( + runtime=settings.runtime, + database=settings.database, + inference=settings.inference, + handoff_report=settings.handoff_report, + external_skills=settings.external_skills, + ) + async with open_builtin_runtime(config, scheduler_path=default_scheduler_path()) as runtime: + if runtime.archive is None: + raise RuntimeError("portable archive service is unavailable") # noqa: TRY003 + return await operation(runtime.archive) + + +async def _authorize_local_export(_scopes: tuple[str, ...], /) -> None: + """A local operator's authority is the OS permission to read the database.""" + + +async def _validate_only(source: Path, /, *, env_file: Path | None) -> BundleValidation: + """Open only the configured database profile; do not initialize Runtime state.""" + + with server_settings_context(env_file=env_file) as settings: + database = settings.database + if isinstance(database, SQLiteConfig): + configured_path = make_url(database.url).database + existing = configured_path is not None and await asyncio.to_thread(_database_file_exists, configured_path) + url = database.url if existing else "sqlite+aiosqlite:///:memory:" + engine = create_async_engine(url, echo=database.echo, hide_parameters=True) + validation_database = AsyncDatabase.own(engine, shared_connection=not existing) + try: + return await _validate_with_database(validation_database, source) + finally: + await validation_database.close() + if isinstance(database, OceanBaseConfig): + async with OceanBaseProfile.open(database, tables=()) as profile: + return await _validate_with_database(profile.database, source) + if isinstance(database, SeekDBConfig): + async with SeekDBProfile.open(database, tables=()) as profile: + return await _validate_with_database(profile.database, source) + assert_never(database) + + +async def _validate_with_database(database: AsyncDatabase, source: Path, /) -> BundleValidation: + async def projection_capability(_scopes: tuple[str, ...], /) -> None: + return None + + archive = PortableBundleService( + database, + projection_rebuilder=projection_capability, + supported_source_types=tuple(definition.name for definition in BUILTIN_SOURCE_REGISTRY.definitions), + supported_artifact_families=(Handoff.family, Memory.family, Experience.family, Skill.family), + ) + return await archive.validate(source, progress=_emit_progress) + + +def _database_file_exists(value: str, /) -> bool: + return Path(value).expanduser().is_file() + + +def _emit_progress(value: BundleProgress, /) -> None: + typer.echo(json.dumps({"progress": asdict(value)}, sort_keys=True), err=True) + + +def _emit_archive_result( + operation: Callable[[PortableBundleService], Awaitable[_ResultT]], /, *, env_file: Path | None +) -> None: + """Render expected archive failures without leaking stack traces or records.""" + + try: + _emit(asyncio.run(_run_archive(operation, env_file=env_file))) + except BundleFormatError as error: + typer.echo(f"error: archive validation failed: {error}", err=True) + raise typer.Exit(code=2) from error + except BundleConflictError as error: + typer.echo(f"error: archive restore conflict: {error}", err=True) + raise typer.Exit(code=3) from error + + +def _emit_validation_result(source: Path, /, *, env_file: Path | None) -> None: + try: + _emit(asyncio.run(_validate_only(source, env_file=env_file))) + except BundleFormatError as error: + typer.echo(f"error: archive validation failed: {error}", err=True) + raise typer.Exit(code=2) from error + + +def _emit(value: BundleInspection | BundleValidation | BundleReceipt, /) -> None: + typer.echo(json.dumps(asdict(value), default=list, sort_keys=True)) diff --git a/tests/builtin/portability/test_bundle.py b/tests/builtin/portability/test_bundle.py new file mode 100644 index 0000000000..3926193b4f --- /dev/null +++ b/tests/builtin/portability/test_bundle.py @@ -0,0 +1,611 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Observable contracts for portable logical bundles.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import tracemalloc +import zipfile +from pathlib import Path +from typing import cast + +import pytest +from sqlalchemy import delete, insert, select + +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import ( + ARTIFACT_CANDIDATE_HEADS_TABLE, + ARTIFACT_CANDIDATE_VERSIONS_TABLE, + ARTIFACT_HEADS_TABLE, + ARTIFACT_PUBLICATIONS_TABLE, + ARTIFACTS_TABLE, + BUILTIN_TABLES, + PORTABLE_RESTORE_RECEIPTS_TABLE, + SCOPE_CONTEXT_REFERENCES_TABLE, + SCOPE_CREATION_REQUESTS_TABLE, + SCOPES_TABLE, + SOURCE_JOURNAL_HEADS_TABLE, + SOURCES_TABLE, +) +from powercontext.builtin.portability import BundleConflictError, BundleFormatError, PortableBundleService + + +async def _authorize_export(_scopes: tuple[str, ...], /) -> None: + pass + + +async def _insert_scope(connection, scope_id: str = "project:one") -> None: + await connection.execute( + insert(SCOPES_TABLE).values( + scope_id=scope_id, + title="Portable fixture", + summary="Portable fixture", + scope_id_search=scope_id, + title_search="Portable fixture", + summary_search="Portable fixture", + parent_scope_id=None, + version=1, + ) + ) + + +def test_bundle_round_trips_authoritative_scope_data_and_excludes_projections(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=b'{"name":"source-1"}', + journal_position=1, + ) + ) + await connection.execute( + insert(ARTIFACTS_TABLE).values( + scope_id="project:one", + family="handoff", + artifact_id="handoff", + revision=1, + content=b'{"content":{"summary":"hello"}}', + ) + ) + await connection.execute( + insert(ARTIFACT_HEADS_TABLE).values( + scope_id="project:one", + family="handoff", + artifact_id="handoff", + revision=1, + searchable_text="not portable", + ) + ) + receipt = await PortableBundleService(source.database).export( + ["project:one"], archive, authorize=_authorize_export + ) + assert receipt.record_count == 5 + + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + service = PortableBundleService(target.database) + inspection = await service.validate(archive, supported_source_types=("content",)) + assert inspection.scopes == ("project:one",) + restored = await service.restore(archive, supported_source_types=("content",)) + assert restored.inserted == 5 + repeated = await service.restore(archive, supported_source_types=("content",)) + assert repeated.already_present == 5 + async with target.database.transaction() as connection: + assert await connection.scalar(select(ARTIFACT_HEADS_TABLE.c.searchable_text)) is None + + asyncio.run(scenario()) + + +def test_export_is_byte_deterministic_and_preserves_scope_relationships(tmp_path: Path) -> None: + async def scenario() -> None: + first = tmp_path / "first.pcb" + second = tmp_path / "second.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection, "project:parent") + await connection.execute( + insert(SCOPES_TABLE).values( + scope_id="workstream:child", + title="Child", + summary="Child scope", + scope_id_search="workstream:child", + title_search="Child", + summary_search="Child scope", + parent_scope_id="project:parent", + version=1, + ) + ) + await connection.execute( + insert(SCOPE_CONTEXT_REFERENCES_TABLE).values( + scope_id="workstream:child", referenced_scope_id="project:parent" + ) + ) + await connection.execute( + insert(SCOPE_CREATION_REQUESTS_TABLE).values( + idempotency_key="child-request", + request_digest="digest", + scope_id="workstream:child", + ) + ) + await connection.execute( + insert(ARTIFACTS_TABLE), + [ + { + "scope_id": "project:parent", + "family": "experience", + "artifact_id": "source-experience", + "revision": 1, + "content": b"source", + }, + { + "scope_id": "workstream:child", + "family": "experience", + "artifact_id": "published-experience", + "revision": 1, + "content": b"published", + }, + ], + ) + await connection.execute( + insert(ARTIFACT_PUBLICATIONS_TABLE).values( + target_scope_id="workstream:child", + target_family="experience", + target_artifact_id="published-experience", + target_revision=1, + source_scope_id="project:parent", + source_family="experience", + source_artifact_id="source-experience", + source_revision=1, + content_digest="d" * 64, + idempotency_key="publish-experience", + ) + ) + service = PortableBundleService(source.database) + one = await service.export(["workstream:child", "project:parent"], first, authorize=_authorize_export) + two = await service.export(["project:parent", "workstream:child"], second, authorize=_authorize_export) + assert one.bundle_id == two.bundle_id + assert first.read_bytes() == second.read_bytes() + + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + restored = await PortableBundleService(target.database).restore(first) + async with target.database.transaction() as connection: + parent = await connection.scalar( + select(SCOPES_TABLE.c.parent_scope_id).where(SCOPES_TABLE.c.scope_id == "workstream:child") + ) + context_count = await connection.scalar(select(SCOPE_CONTEXT_REFERENCES_TABLE.c.scope_id)) + creation_scope = await connection.scalar(select(SCOPE_CREATION_REQUESTS_TABLE.c.scope_id)) + publication_source = await connection.scalar(select(ARTIFACT_PUBLICATIONS_TABLE.c.source_scope_id)) + assert restored.record_count == 7 + assert parent == "project:parent" + assert context_count == "workstream:child" + assert creation_scope == "workstream:child" + assert publication_source == "project:parent" + + asyncio.run(scenario()) + + +def test_validation_accepts_a_clean_target_without_initializing_schema(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + async with SQLiteProfile.open(SQLiteConfig(), tables=()) as clean_target: + validation = await PortableBundleService( + clean_target.database, + projection_rebuilder=lambda _scopes: asyncio.sleep(0), + ).validate(archive) + assert validation.compatible is True + assert validation.already_present == 0 + assert validation.conflicts == 0 + + asyncio.run(scenario()) + + +def test_restore_rejects_divergent_immutable_identity_without_overwrite(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=b"original", + journal_position=1, + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + async with target.database.transaction() as connection: + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=b"different", + journal_position=1, + ) + ) + service = PortableBundleService(target.database) + validation = await service.validate(archive) + assert validation.compatible is False + assert validation.conflicts == 1 + with pytest.raises(BundleConflictError): + await service.restore(archive) + async with target.database.transaction() as connection: + assert await connection.scalar(select(SOURCES_TABLE.c.payload)) == b"different" + + asyncio.run(scenario()) + + +def test_failed_restore_rolls_back_earlier_records_and_can_be_retried(tmp_path: Path) -> None: + """A write-time constraint conflict must not strand earlier bundle rows.""" + + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=b"new source", + journal_position=1, + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + async with target.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="target-only", + payload=b"existing", + journal_position=1, + ) + ) + service = PortableBundleService(target.database) + validation = await service.validate(archive) + assert validation.conflicts == 0 + with pytest.raises(BundleConflictError, match="target constraint conflict"): + await service.restore(archive) + + async with target.database.transaction() as connection: + source_ids = tuple((await connection.execute(select(SOURCES_TABLE.c.source_id))).scalars()) + assert source_ids == ("target-only",) + await connection.execute(delete(SOURCES_TABLE)) + + retried = await service.restore(archive) + assert retried.inserted == 1 + assert retried.already_present == 2 + + asyncio.run(scenario()) + + +def test_restore_reports_projection_readiness_from_runtime_rebuilder(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + rebuilt: list[tuple[str, ...]] = [] + progress = [] + + async def rebuild(scopes: tuple[str, ...]) -> None: + rebuilt.append(scopes) + + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + receipt = await PortableBundleService(target.database, projection_rebuilder=rebuild).restore( + archive, progress=progress.append + ) + assert rebuilt == [("project:one",)] + assert receipt.projections_ready is True + assert receipt.status == "ready" + assert {event.phase for event in progress} >= {"validate", "target", "restore", "projections"} + + asyncio.run(scenario()) + + +def test_export_authorization_precedes_database_access(tmp_path: Path) -> None: + class _ForbiddenDatabase: + def transaction(self): + raise AssertionError("database was accessed before authorization") # noqa: TRY003 + + async def scenario() -> None: + async def deny(_scopes: tuple[str, ...]) -> None: + raise PermissionError("denied") + + service = PortableBundleService(cast(AsyncDatabase, _ForbiddenDatabase())) + with pytest.raises(PermissionError, match="denied"): + await service.export(["project:one"], tmp_path / "denied.pcb", authorize=deny) + assert not (tmp_path / "denied.pcb").exists() + + asyncio.run(scenario()) + + +def test_projection_failure_leaves_a_durable_not_ready_receipt(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + + async def fail_rebuild(_scopes: tuple[str, ...]) -> None: + raise RuntimeError("projection failed") # noqa: TRY003 + + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + with pytest.raises(RuntimeError, match="projection failed"): + await PortableBundleService(target.database, projection_rebuilder=fail_rebuild).restore(archive) + async with target.database.transaction() as connection: + row = (await connection.execute(select(PORTABLE_RESTORE_RECEIPTS_TABLE))).mappings().one() + assert row["status"] == "authoritative_restored" + assert row["projections_ready"] is False + + asyncio.run(scenario()) + + +def test_export_rejects_secret_fields_without_echoing_values(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "secret.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=b'{"metadata":{"api_key":"do-not-echo"}}', + journal_position=1, + ) + ) + with pytest.raises(BundleFormatError) as caught: + await PortableBundleService(source.database).export( + ["project:one"], archive, authorize=_authorize_export + ) + assert "credential-shaped" in str(caught.value) + assert "do-not-echo" not in str(caught.value) + assert not archive.exists() + + asyncio.run(scenario()) + + +def test_restore_accepts_a_valid_bundle_with_records_in_a_different_physical_order(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "ordered.pcb" + reordered = tmp_path / "reordered.pcb" + database = tmp_path / "source.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{database}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=b"source", + journal_position=1, + ) + ) + await connection.execute( + insert(ARTIFACTS_TABLE).values( + scope_id="project:one", + family="handoff", + artifact_id="handoff", + revision=1, + content=b"artifact", + ) + ) + await connection.execute( + insert(ARTIFACT_HEADS_TABLE).values( + scope_id="project:one", family="handoff", artifact_id="handoff", revision=1 + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + + with zipfile.ZipFile(archive) as input_archive: + manifest = json.loads(input_archive.read("manifest.json")) + records = list(reversed(input_archive.read("records.ndjson").splitlines())) + digests = [json.loads(record)["digest"] for record in records] + manifest["total_digest"] = "sha256:" + hashlib.sha256("\n".join(digests).encode()).hexdigest() + with zipfile.ZipFile(reordered, "w") as output_archive: + output_archive.writestr("manifest.json", json.dumps(manifest)) + output_archive.writestr("records.ndjson", b"\n".join(records) + b"\n") + + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + receipt = await PortableBundleService(target.database).restore(reordered) + assert receipt.inserted == 5 + + asyncio.run(scenario()) + + +def test_restore_keeps_large_bundle_payloads_out_of_process_memory(tmp_path: Path) -> None: + """The import budget is independent of aggregate NDJSON payload size.""" + + async def scenario() -> None: + archive = tmp_path / "large.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + payload = b"x" * 8_192 + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute( + insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1_500) + ) + await connection.execute( + insert(SOURCES_TABLE), + [ + { + "scope_id": "project:one", + "source_type": "content", + "source_id": f"source-{number}", + "payload": payload, + "journal_position": number, + } + for number in range(1, 1_501) + ], + ) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + + tracemalloc.start() + try: + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + receipt = await PortableBundleService(target.database).restore(archive) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert receipt.inserted == 1_502 + assert peak < 10 * 1024 * 1024 + + asyncio.run(scenario()) + + +def test_validate_rejects_an_artifact_family_the_target_runtime_cannot_restore(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute( + insert(ARTIFACTS_TABLE).values( + scope_id="project:one", + family="unavailable-family", + artifact_id="record-1", + revision=1, + content=b"payload", + ) + ) + await connection.execute( + insert(ARTIFACT_HEADS_TABLE).values( + scope_id="project:one", + family="unavailable-family", + artifact_id="record-1", + revision=1, + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + validation = await PortableBundleService( + target.database, + supported_artifact_families=("handoff",), + ).validate(archive) + assert validation.compatible is False + assert validation.unsupported_artifact_families == ("unavailable-family",) + + asyncio.run(scenario()) + + +def test_validate_rejects_candidate_evidence_that_is_not_in_the_bundle(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + database = tmp_path / "source.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{database}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute( + insert(ARTIFACT_CANDIDATE_VERSIONS_TABLE).values( + scope_id="project:one", + candidate_id="candidate-1", + version=1, + family="handoff", + proposal=b"{}", + source_refs=b'[{"source_type":"content","source_id":"missing"}]', + artifact_refs=b"[]", + target_family=None, + target_artifact_id=None, + target_revision=None, + reason=None, + ) + ) + await connection.execute( + insert(ARTIFACT_CANDIDATE_HEADS_TABLE).values( + scope_id="project:one", + candidate_id="candidate-1", + family="handoff", + version=1, + status="pending", + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + with pytest.raises(BundleFormatError, match="missing source"): + await PortableBundleService(target.database).validate(archive) + + asyncio.run(scenario()) diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py new file mode 100644 index 0000000000..ee3e6d7577 --- /dev/null +++ b/tests/e2e/test_portable_archive.py @@ -0,0 +1,118 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Portable archive acceptance through the public built-in Runtime.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +from powercontext.builtin.artifacts.memory import MemoryEntryInput +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime import ( + BuiltinConfig, + CaptureSource, + HandoffDraft, + HandoffSourceCitation, + HandoffStatement, + RememberMemoryRequest, + open_builtin_runtime, +) +from powercontext.builtin.scope import ScopeDraft +from powercontext.builtin.work import AcknowledgeHandoff, ReceiverChecks + + +def test_sqlite_portable_archive_restores_revisions_lineage_and_handoff_receipts(tmp_path: Path) -> None: + async def scenario() -> None: + archive_path = tmp_path / "portable.pcb" + source_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'source.db'}")) + target_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + + async with open_builtin_runtime(source_config) as source_runtime: + assert source_runtime.scopes is not None + scope = await source_runtime.scopes.create( + ScopeDraft( + title="Portable archive", summary="Archive recovery fixture", idempotency_key="portable-archive" + ) + ) + scope_id = scope.scope_id + source = await source_runtime.sources.for_scope(scope_id).capture( + CaptureSource( + source_id="turn-1", + content="The archive must preserve the handoff citation.", + metadata={"origin": "portable-archive-e2e"}, + ) + ) + memory = await source_runtime.memory.for_scope(scope_id).remember( + RememberMemoryRequest( + entries=(MemoryEntryInput(kind="decision", text="Use the portable archive for migration."),) + ) + ) + assert memory.entry is not None + handoffs = source_runtime.handoff.for_scope(scope_id) + prepared = await handoffs.finalize( + HandoffDraft( + objective="Restore an exact logical handoff.", + state=( + HandoffStatement( + text="The source citation is durable.", + citations=(HandoffSourceCitation(source_ref=source.source_ref),), + ), + ), + disposition="continuable", + next_action=HandoffStatement( + text="Open the restored handoff.", + citations=(HandoffSourceCitation(source_ref=source.source_ref),), + ), + ) + ) + committed = await handoffs.commit(prepared) + acknowledgement = await source_runtime.work.for_scope(scope_id).acknowledge( + AcknowledgeHandoff( + source_id="handoff-receipt-1", + receiver="portable-target", + status="accepted", + selection="exact", + revision=committed.as_ref(), + receiver_checks=ReceiverChecks( + live_state="confirmed", + capability="confirmed", + authorization="confirmed", + ), + ) + ) + assert source_runtime.archive is not None + + async def authorize(scopes: tuple[str, ...]) -> None: + assert scopes == (scope_id,) + + exported = await source_runtime.archive.export([scope_id], archive_path, authorize=authorize) + + async with open_builtin_runtime(target_config) as target_runtime: + assert target_runtime.archive is not None + restored = await target_runtime.archive.restore(archive_path) + latest = await target_runtime.handoff.for_scope(scope_id).latest() + continuity = await target_runtime.work.for_scope(scope_id).continuity() + + assert restored.record_count == exported.record_count + assert restored.projections_ready is True + assert latest == committed + assert latest is not None + assert latest.revision == 1 + assert latest.lineage.sources == (source.source_ref,) + assert continuity.coverage.acknowledgement_records == 1 + assert continuity.coverage.active_receipt_ref == acknowledgement.receipt.source_ref + + asyncio.run(scenario()) diff --git a/tests/e2e/test_portable_archive_oceanbase.py b/tests/e2e/test_portable_archive_oceanbase.py new file mode 100644 index 0000000000..61c16c5c29 --- /dev/null +++ b/tests/e2e/test_portable_archive_oceanbase.py @@ -0,0 +1,123 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Live SQLite-to-OceanBase portable archive acceptance.""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path +from uuid import uuid4 + +import pytest +from pydantic import SecretStr +from sqlalchemy import insert, select +from sqlalchemy.engine import make_url + +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import ( + ARTIFACT_HEADS_TABLE, + ARTIFACTS_TABLE, + BUILTIN_TABLES, + SCOPES_TABLE, +) +from powercontext.builtin.portability import PortableBundleService + +LIVE_URL = os.environ.get("POWERCONTEXT_TEST_OCEANBASE_URL") + + +async def _authorize(_scopes: tuple[str, ...], /) -> None: + pass + + +@pytest.mark.skipif( + not LIVE_URL, + reason="set POWERCONTEXT_TEST_OCEANBASE_URL with test database creation and deletion privileges", +) +def test_sqlite_bundle_restores_exact_revision_into_oceanbase(tmp_path: Path) -> None: + async def scenario() -> None: + assert LIVE_URL is not None + scope_id = "project:portable-oceanbase" + archive = tmp_path / "sqlite-to-oceanbase.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as sqlite: + async with sqlite.database.transaction() as connection: + await connection.execute( + insert(SCOPES_TABLE).values( + scope_id=scope_id, + title="Portable migration", + summary="SQLite to OceanBase", + scope_id_search=scope_id, + title_search="Portable migration", + summary_search="SQLite to OceanBase", + parent_scope_id=None, + version=1, + ) + ) + await connection.execute( + insert(ARTIFACTS_TABLE).values( + scope_id=scope_id, + family="handoff", + artifact_id="handoff", + revision=1, + content=b'{"content":{"objective":"portable"}}', + ) + ) + await connection.execute( + insert(ARTIFACT_HEADS_TABLE).values( + scope_id=scope_id, + family="handoff", + artifact_id="handoff", + revision=1, + ) + ) + exported = await PortableBundleService(sqlite.database).export([scope_id], archive, authorize=_authorize) + + database_name = f"pc_portable_{uuid4().hex}" + base_config = OceanBaseConfig(url=SecretStr(LIVE_URL)) + created = False + async with OceanBaseProfile.open(base_config, tables=()) as server: + try: + async with server.database.transaction() as connection: + await connection.exec_driver_sql(f"CREATE DATABASE `{database_name}`") + created = True + target_url = make_url(LIVE_URL).set(database=database_name).render_as_string(hide_password=False) + async with OceanBaseProfile.open( + OceanBaseConfig(url=SecretStr(target_url)), tables=BUILTIN_TABLES + ) as oceanbase: + restored = await PortableBundleService(oceanbase.database).restore(archive) + async with oceanbase.database.transaction() as connection: + row = ( + ( + await connection.execute( + select(ARTIFACTS_TABLE).where( + ARTIFACTS_TABLE.c.scope_id == scope_id, + ARTIFACTS_TABLE.c.family == "handoff", + ARTIFACTS_TABLE.c.artifact_id == "handoff", + ARTIFACTS_TABLE.c.revision == 1, + ) + ) + ) + .mappings() + .one() + ) + assert restored.record_count == exported.record_count + assert bytes(row["content"]) == b'{"content":{"objective":"portable"}}' + finally: + if created: + async with server.database.transaction() as connection: + await connection.exec_driver_sql(f"DROP DATABASE `{database_name}`") + + asyncio.run(scenario()) diff --git a/tests/test_archive_cli.py b/tests/test_archive_cli.py new file mode 100644 index 0000000000..2710e2248d --- /dev/null +++ b/tests/test_archive_cli.py @@ -0,0 +1,71 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio + +from typer.testing import CliRunner + +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_runtime +from powercontext.cli.app import create_cli +from powercontext.cli.archive import archive_app +from powercontext.paths import default_database_path, sqlite_url + + +def test_archive_cli_exposes_a_safe_restore_confirmation_and_dry_run() -> None: + cli = create_cli([archive_app]) + runner = CliRunner() + + help_result = runner.invoke(cli, ["archive", "--help"], terminal_width=160) + restore_help = runner.invoke(cli, ["archive", "restore", "--help"], terminal_width=160) + + assert help_result.exit_code == 0 + assert all(command in help_result.output for command in ("export", "inspect", "restore")) + assert restore_help.exit_code == 0 + assert "--dry-run" in restore_help.output + assert "--yes" in restore_help.output + + +def test_archive_cli_can_export_inspect_and_dry_run_a_scope(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "data")) + output = tmp_path / "scope.pcb" + cli = create_cli([archive_app]) + runner = CliRunner() + + async def default_scope_id() -> str: + config = BuiltinConfig(database=SQLiteConfig(url=sqlite_url(default_database_path()))) + async with open_builtin_runtime(config) as runtime: + assert runtime.scopes is not None + scope = await runtime.scopes.default_scope() + assert scope is not None + return scope.scope_id + + scope_id = asyncio.run(default_scope_id()) + exported = runner.invoke(cli, ["archive", "export", "--scope-id", scope_id, "--output", str(output)]) + inspected = runner.invoke(cli, ["archive", "inspect", str(output)]) + validated = runner.invoke(cli, ["archive", "restore", str(output), "--dry-run"]) + + assert exported.exit_code == 0, exported.output + assert output.is_file() + assert inspected.exit_code == 0, inspected.output + assert validated.exit_code == 0, validated.output + + +def test_archive_cli_reports_corrupt_or_missing_bundles_without_a_traceback() -> None: + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "inspect", "missing.pcb"]) + + assert result.exit_code == 2 + assert isinstance(result.exception, SystemExit) + assert "error: archive validation failed: cannot read portable bundle" in result.output + assert "Traceback" not in result.output diff --git a/uv.lock b/uv.lock index 1e0b9b439d..d6132f9e3f 100644 --- a/uv.lock +++ b/uv.lock @@ -1202,7 +1202,7 @@ name = "importlib-metadata" version = "9.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp" }, + { name = "zipp", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } wheels = [ @@ -2369,8 +2369,8 @@ name = "opendalfs" version = "0.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "fsspec" }, - { name = "opendal" }, + { name = "fsspec", marker = "python_full_version >= '3.12'" }, + { name = "opendal", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0d/23/fdb4d34f08ac68a27b2d553256e13b15ce139916d36bab33f9a0e71e8770/opendalfs-0.1.0.tar.gz", hash = "sha256:5ceaeccc0852ef10c8ef2ce20e5927dbc18083909c06550ad36f81bcacb5774d", size = 75112, upload-time = "2026-08-26T05:30:33.497Z" } wheels = [ @@ -2704,13 +2704,20 @@ builtin = [ { name = "sqlite-vec" }, ] cli = [ + { name = "aiosqlite" }, + { name = "apscheduler" }, { name = "httpx", extra = ["socks"] }, { name = "inquirerpy" }, + { name = "jsonschema" }, { name = "opentelemetry-api" }, { name = "packaging" }, { name = "platformdirs" }, + { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, { name = "pydantic-settings" }, + { name = "pyobvector" }, { name = "pyyaml" }, + { name = "sqlalchemy", extra = ["asyncio"] }, + { name = "sqlite-vec" }, { name = "typer" }, ] client = [ @@ -2796,9 +2803,11 @@ notebooks = [ [package.metadata] requires-dist = [ { name = "aiosqlite", marker = "extra == 'builtin'", specifier = ">=0.22,<1" }, + { name = "aiosqlite", marker = "extra == 'cli'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'seekdb'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'server'", specifier = ">=0.22,<1" }, { name = "apscheduler", marker = "extra == 'builtin'", specifier = ">=3.11,<4" }, + { name = "apscheduler", marker = "extra == 'cli'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'seekdb'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'server'", specifier = ">=3.11,<4" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115,<1" }, @@ -2809,6 +2818,7 @@ requires-dist = [ { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3,<1" }, { name = "jinja2", marker = "extra == 'server'", specifier = ">=3.1.6,<4" }, { name = "jsonschema", marker = "extra == 'builtin'", specifier = ">=4.23,<5" }, + { name = "jsonschema", marker = "extra == 'cli'", specifier = ">=4.23,<5" }, { name = "jsonschema", marker = "extra == 'seekdb'", specifier = ">=4.23,<5" }, { name = "jsonschema", marker = "extra == 'server'", specifier = ">=4.23,<5" }, { name = "opentelemetry-api", marker = "extra == 'cli'", specifier = ">=1.30,<2" }, @@ -2827,6 +2837,7 @@ requires-dist = [ { name = "pycasbin", marker = "extra == 'server'", specifier = ">=2.8,<3" }, { name = "pydantic", specifier = ">=2.10,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'builtin'", specifier = ">=2.27.1,<3" }, + { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'cli'", specifier = ">=2.27.1,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'seekdb'", specifier = ">=2.27.1,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'server'", specifier = ">=2.27.1,<3" }, { name = "pydantic-settings", marker = "extra == 'builtin'", specifier = ">=2.7,<3" }, @@ -2836,6 +2847,7 @@ requires-dist = [ { name = "pydantic-settings", marker = "extra == 'server'", specifier = ">=2.7,<3" }, { name = "pylibseekdb", marker = "(sys_platform == 'darwin' and extra == 'seekdb') or (sys_platform == 'linux' and extra == 'seekdb')", specifier = ">=1.3.0.post4,<2" }, { name = "pyobvector", marker = "extra == 'builtin'", specifier = ">=0.2.28,<0.3" }, + { name = "pyobvector", marker = "extra == 'cli'", specifier = ">=0.2.28,<0.3" }, { name = "pyobvector", marker = "extra == 'seekdb'", specifier = ">=0.2.28,<0.3" }, { name = "pyobvector", marker = "extra == 'server'", specifier = ">=0.2.28,<0.3" }, { name = "pyyaml", marker = "extra == 'builtin'", specifier = ">=6,<7" }, @@ -2846,9 +2858,11 @@ requires-dist = [ { name = "rfc8785", specifier = ">=0.1.4,<1" }, { name = "scalar-fastapi", marker = "extra == 'server'", specifier = ">=1.8.2,<2" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'builtin'", specifier = ">=2,<3" }, + { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'cli'", specifier = ">=2,<3" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'seekdb'", specifier = ">=2,<3" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'server'", specifier = ">=2,<3" }, { name = "sqlite-vec", marker = "extra == 'builtin'", specifier = ">=0.1.9,<0.2" }, + { name = "sqlite-vec", marker = "extra == 'cli'", specifier = ">=0.1.9,<0.2" }, { name = "sqlite-vec", marker = "extra == 'seekdb'", specifier = ">=0.1.9,<0.2" }, { name = "sqlite-vec", marker = "extra == 'server'", specifier = ">=0.1.9,<0.2" }, { name = "typer", marker = "extra == 'cli'", specifier = ">=0.16,<1" }, @@ -2895,8 +2909,8 @@ name = "powercontext-connector-opendal" version = "0.0.1" source = { editable = "integrations/opendal" } dependencies = [ - { name = "opendalfs" }, - { name = "powercontext", extra = ["client"] }, + { name = "opendalfs", marker = "python_full_version >= '3.12'" }, + { name = "powercontext", extra = ["client"], marker = "python_full_version >= '3.12'" }, ] [package.metadata] @@ -4297,18 +4311,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/5b/f2aa703a4fc5d2dff73460a7d46cc2f3f44aa0f3dd8eeb20d2a0ecf68862/tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07", size = 918110, upload-time = "2026-05-15T04:51:17.237Z" }, ] -[[package]] -name = "tinycss2" -version = "1.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "webencodings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a3/ae/2ca4913e5c0f09781d75482874c3a95db9105462a92ddd303c7d285d3df2/tinycss2-1.5.1.tar.gz", hash = "sha256:d339d2b616ba90ccce58da8495a78f46e55d4d25f9fd71dfd526f07e7d53f957", size = 88195, upload-time = "2025-11-23T10:29:10.082Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/60/45/c7b5c3168458db837e8ceab06dc77824e18202679d0463f0e8f002143a97/tinycss2-1.5.1-py3-none-any.whl", hash = "sha256:3415ba0f5839c062696996998176c4a3751d18b7edaaeeb658c9ce21ec150661", size = 28404, upload-time = "2025-11-23T10:29:08.676Z" }, -] - [[package]] name = "time-machine" version = "3.5.0" @@ -4373,6 +4375,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/e1/0fda7534a0b3af4f19fc70bcd4b421924427bc0645b7c1255617ae98de69/time_machine-3.5.0-cp315-cp315t-win_arm64.whl", hash = "sha256:a93195067e2ea6d64e17a0ce0b1eed1fa9bd056ace20c6338fe9be532917ca75", size = 27531, upload-time = "2026-08-25T00:18:31.616Z" }, ] +[[package]] +name = "tinycss2" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "webencodings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/ae/2ca4913e5c0f09781d75482874c3a95db9105462a92ddd303c7d285d3df2/tinycss2-1.5.1.tar.gz", hash = "sha256:d339d2b616ba90ccce58da8495a78f46e55d4d25f9fd71dfd526f07e7d53f957", size = 88195, upload-time = "2025-11-23T10:29:10.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/45/c7b5c3168458db837e8ceab06dc77824e18202679d0463f0e8f002143a97/tinycss2-1.5.1-py3-none-any.whl", hash = "sha256:3415ba0f5839c062696996998176c4a3751d18b7edaaeeb658c9ce21ec150661", size = 28404, upload-time = "2025-11-23T10:29:08.676Z" }, +] + [[package]] name = "tomli" version = "2.4.1" From bffe534268aa137478e157622341d43f10a461f1 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Sat, 12 Sep 2026 11:08:56 +0800 Subject: [PATCH 19/32] test: normalize archive CLI help output --- tests/test_archive_cli.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_archive_cli.py b/tests/test_archive_cli.py index 2710e2248d..ffb86ae56b 100644 --- a/tests/test_archive_cli.py +++ b/tests/test_archive_cli.py @@ -14,6 +14,7 @@ import asyncio +from click.utils import strip_ansi from typer.testing import CliRunner from powercontext.builtin.persistence.sqlite import SQLiteConfig @@ -30,11 +31,14 @@ def test_archive_cli_exposes_a_safe_restore_confirmation_and_dry_run() -> None: help_result = runner.invoke(cli, ["archive", "--help"], terminal_width=160) restore_help = runner.invoke(cli, ["archive", "restore", "--help"], terminal_width=160) + help_output = strip_ansi(help_result.output) + restore_output = strip_ansi(restore_help.output) + assert help_result.exit_code == 0 - assert all(command in help_result.output for command in ("export", "inspect", "restore")) + assert all(command in help_output for command in ("export", "inspect", "restore")) assert restore_help.exit_code == 0 - assert "--dry-run" in restore_help.output - assert "--yes" in restore_help.output + assert "--dry-run" in restore_output + assert "--yes" in restore_output def test_archive_cli_can_export_inspect_and_dry_run_a_scope(tmp_path, monkeypatch) -> None: From 62a73bbd8a1d8b301b589659fafbbc4f4391f126 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Sun, 20 Sep 2026 04:25:56 +0800 Subject: [PATCH 20/32] fix(archive): validate wire records and rebuild complete projections --- .../builtin/portability/bundle.py | 156 +++++++++++++----- .../builtin/portability/schema.py | 81 +++++++++ .../builtin/runtime/relational.py | 38 ++++- src/powercontext/cli/archive.py | 21 ++- tests/builtin/portability/test_bundle.py | 41 ++++- tests/e2e/test_portable_archive.py | 39 ++++- tests/test_archive_cli.py | 17 ++ 7 files changed, 339 insertions(+), 54 deletions(-) create mode 100644 src/powercontext/builtin/portability/schema.py diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index b9759d1e3c..7ca7f7aa45 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -14,6 +14,7 @@ import hashlib import json import re +import shutil import sqlite3 import tempfile import zipfile @@ -27,7 +28,7 @@ from typing import Any, Literal, cast from uuid import uuid4 -from sqlalchemy import insert, select +from sqlalchemy import insert, select, tuple_ from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.builtin.persistence.database import AsyncDatabase @@ -43,6 +44,7 @@ SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE, ) +from powercontext.builtin.portability.schema import BINARY_FIELDS, INTEGER_FIELDS, NULLABLE_FIELDS, RECORD_FIELDS FORMAT_VERSION = 1 _RECORDS_NAME = "records.ndjson" @@ -204,7 +206,8 @@ async def export(self, scopes: Iterable[str], output: Path, /) -> BundleReceipt: records_path.unlink(missing_ok=True) archive_path.unlink(missing_ok=True) - async def inspect(self, source: Path, /) -> BundleInspection: + @staticmethod + async def inspect(source: Path, /) -> BundleInspection: """Verify structure and checksums without touching the target database.""" return _parse_bundle(source).inspection @@ -214,9 +217,26 @@ async def validate( ) -> BundleInspection: """Verify bundle integrity, dependencies, and optional adapter support.""" + return await self.validate_archive( + source, + supported_source_types=( + self._supported_source_types if supported_source_types is None else supported_source_types + ), + supported_artifact_families=self._supported_artifact_families, + ) + + @staticmethod + async def validate_archive( + source: Path, + /, + *, + supported_source_types: Iterable[str] | None = None, + supported_artifact_families: Iterable[str] | None = None, + ) -> BundleInspection: + """Validate a wire bundle without opening or initializing any database.""" parsed = _parse_bundle(source) _validate_dependencies(source) - configured = self._supported_source_types if supported_source_types is None else set(supported_source_types) + configured = supported_source_types if configured is not None: supported = set(configured) missing = sorted( @@ -229,9 +249,9 @@ async def validate( ) if missing: raise BundleFormatError(f"target does not support source types: {', '.join(missing)}") - if self._supported_artifact_families is not None: + if supported_artifact_families is not None: required_families = _required_artifact_families(_iter_records(source)) - missing_families = sorted(required_families - self._supported_artifact_families) + missing_families = sorted(required_families - set(supported_artifact_families)) if missing_families: raise BundleFormatError(f"target does not support artifact families: {', '.join(missing_families)}") return parsed.inspection @@ -250,47 +270,64 @@ async def restore( no existing revision or head is overwritten. """ - parsed = _parse_bundle(source) - await self.validate(source, supported_source_types=supported_source_types) - async with self._database.transaction() as connection: - inserted, existing = await self._write_records(connection, source) + # Own a stable input across validation and replay, even if the caller + # replaces the original archive while a restore is running. + with tempfile.TemporaryDirectory(prefix="powercontext-restore-") as directory: + snapshot = Path(directory) / "bundle.pcb" + try: + shutil.copyfile(source, snapshot) + except OSError as error: + raise BundleFormatError("cannot read portable bundle") from error + inspection = await self.validate(snapshot, supported_source_types=supported_source_types) + with _staged_records(snapshot) as staged: + async with self._database.transaction() as connection: + inserted, existing = await self._write_records(connection, staged) projections_ready = False if self._projection_rebuilder is not None: - await self._projection_rebuilder(parsed.inspection.scopes) + await self._projection_rebuilder(inspection.scopes) projections_ready = True return BundleReceipt( - bundle_id=parsed.inspection.bundle_id, - record_count=parsed.inspection.record_count, - total_digest=parsed.inspection.total_digest, + bundle_id=inspection.bundle_id, + record_count=inspection.record_count, + total_digest=inspection.total_digest, inserted=inserted, already_present=existing, projections_ready=projections_ready, ) - async def _write_records(self, connection: AsyncConnection, source: Path, /) -> tuple[int, int]: + async def _write_records(self, connection: AsyncConnection, staged: sqlite3.Connection, /) -> tuple[int, int]: """Replay records in dependency order inside the caller transaction.""" inserted = 0 existing = 0 - # A portable bundle may be produced by another implementation, so do - # not trust its physical order. Re-scan once per dependency level - # instead of materializing its records merely to sort them. for record_type in _EXPORT_ORDER: - for record in _iter_records(source): - if record.record_type != record_type: - continue - table = _TABLES[record.record_type] - where = [table.c[key] == value for key, value in record.identity.items()] - row = (await connection.execute(select(table).where(*where))).mappings().one_or_none() - if row is not None: - present = _record_from_row(record.record_type, cast(Mapping[str, Any], dict(row))) - if present.digest != record.digest: - raise BundleConflictError( - f"immutable identity conflict: {record.record_type} {dict(record.identity)!r}" - ) - existing += 1 - continue - await connection.execute(insert(table).values(**_row_values(record))) - inserted += 1 + cursor = staged.execute("SELECT document FROM records WHERE kind = ? ORDER BY sequence", (record_type,)) + while documents := cursor.fetchmany(100): + records = tuple(_parse_record(json.loads(row[0])) for row in documents) + table = _TABLES[record_type] + keys = RECORD_FIELDS[record_type][0] + identities = [tuple(record.identity[key] for key in keys) for record in records] + rows = ( + await connection.execute( + select(table).where(tuple_(*(table.c[key] for key in keys)).in_(identities)) + ) + ).mappings() + present = { + _identity_key(record.identity): record.digest + for row in rows + for record in (_record_from_row(record_type, cast(Mapping[str, Any], dict(row))),) + } + values = [] + for record in records: + digest = present.get(_identity_key(record.identity)) + if digest is not None: + if digest != record.digest: + raise BundleConflictError(f"immutable identity conflict: {record_type}") + existing += 1 + else: + values.append(_row_values(record)) + if values: + await connection.execute(insert(table), values) + inserted += len(values) return inserted, existing async def _stream_export( @@ -315,14 +352,28 @@ async def _stream_export( return count, records_by_type, digest.value() +@contextmanager +def _staged_records(source: Path) -> Iterator[sqlite3.Connection]: + """Index validated records on disk before acquiring the target write transaction.""" + with tempfile.TemporaryDirectory(prefix="powercontext-records-") as directory: + connection = sqlite3.connect(Path(directory) / "records.db") + try: + connection.execute("CREATE TABLE records (sequence INTEGER PRIMARY KEY, kind TEXT, document BLOB)") + connection.executemany( + "INSERT INTO records(kind, document) VALUES (?, ?)", + ((record.record_type, _canonical_json(_record_document(record))) for record in _iter_records(source)), + ) + connection.execute("CREATE INDEX records_kind ON records(kind, sequence)") + connection.commit() + yield connection + finally: + connection.close() + + def _record_from_row(record_type: RecordType, row: Mapping[str, Any]) -> _Record: - table = _TABLES[record_type] - identity = {column.name: _json_value(row[column.name]) for column in table.primary_key.columns} - payload = { - column.name: _json_value(row[column.name]) - for column in table.columns - if column.name not in identity and column.name not in {"searchable_text"} - } + identity_fields, payload_fields = RECORD_FIELDS[record_type] + identity = {name: _json_value(row[name]) for name in identity_fields} + payload = {name: _json_value(row[name]) for name in payload_fields} canonical = {"record_type": record_type, "schema_version": 1, "identity": identity, "payload": payload} return _Record(record_type=record_type, identity=identity, payload=payload, digest=_digest(canonical)) @@ -361,6 +412,8 @@ def _parse_bundle(source: Path) -> _ParsedBundle: counts: Counter[str] = Counter() digest = _DigestAccumulator() for record in _iter_records(source): + if record.identity["scope_id"] not in manifest["scopes"]: + raise BundleFormatError("record scope is not declared in manifest") count += 1 counts[record.record_type] += 1 digest.add(record.digest) @@ -389,7 +442,7 @@ def _iter_records(source: Path, /) -> Iterator[_Record]: try: with zipfile.ZipFile(source) as archive, archive.open(_RECORDS_NAME) as stream: count = 0 - for line in stream: + while line := stream.readline(_MAX_LINE_BYTES + 1): if len(line) > _MAX_LINE_BYTES: raise BundleFormatError("record line exceeds size limit") if not line.strip(): @@ -446,15 +499,32 @@ def _parse_record(value: object) -> _Record: identity = cast(dict[str, str | int], identity) payload = cast(dict[str, Any], payload) typed = cast(RecordType, record_type) - expected = {column.name for column in _TABLES[typed].primary_key.columns} - if set(identity) != expected or not _valid_digest(digest): + identity_fields, payload_fields = RECORD_FIELDS[typed] + if set(identity) != set(identity_fields) or not _valid_digest(digest): raise BundleFormatError("record has invalid identity or digest") + if set(payload) != set(payload_fields): + raise BundleFormatError("record has invalid payload fields") + _validate_field_values(identity, payload) canonical = {"record_type": typed, "schema_version": 1, "identity": identity, "payload": payload} if _digest(canonical) != digest: raise BundleFormatError("record digest does not match content") return _Record(record_type=typed, identity=identity, payload=payload, digest=digest) +def _validate_field_values(identity: Mapping[str, object], payload: Mapping[str, object]) -> None: + for name, field in {**identity, **payload}.items(): + if field is None and name in NULLABLE_FIELDS and name not in identity: + continue + if name in BINARY_FIELDS: + if not isinstance(_database_value(field), bytes): + raise BundleFormatError("record has invalid binary field") + elif name in INTEGER_FIELDS: + if not isinstance(field, int) or isinstance(field, bool) or field < 0: + raise BundleFormatError("record has invalid integer field") + elif not isinstance(field, str): + raise BundleFormatError("record has invalid text field") + + def _validate_dependencies(source: Path, /) -> None: """Check references with a temporary on-disk identity index. diff --git a/src/powercontext/builtin/portability/schema.py b/src/powercontext/builtin/portability/schema.py new file mode 100644 index 0000000000..dda561276c --- /dev/null +++ b/src/powercontext/builtin/portability/schema.py @@ -0,0 +1,81 @@ +"""Version-one wire fields, independent of relational table metadata. + +Changing persistence columns must not silently change an existing archive format. +Each entry lists the identity fields followed by the authoritative payload fields. +""" + +RECORD_FIELDS: dict[str, tuple[tuple[str, ...], tuple[str, ...]]] = { + "source_journal_head": (("scope_id",), ("position",)), + "source": (("scope_id", "source_type", "source_id"), ("payload", "journal_position")), + "artifact_revision": (("scope_id", "family", "artifact_id", "revision"), ("content",)), + "artifact_lineage_source": ( + ("scope_id", "family", "artifact_id", "revision", "ordinal"), + ("source_type", "source_id"), + ), + "artifact_lineage_artifact": ( + ("scope_id", "family", "artifact_id", "revision", "ordinal"), + ("upstream_family", "upstream_artifact_id", "upstream_revision"), + ), + "artifact_head": (("scope_id", "family", "artifact_id"), ("revision",)), + "memory_entry_version": ( + ("scope_id", "memory_artifact_id", "entry_version_id"), + ( + "family", + "entry_id", + "version", + "previous_version_id", + "kind", + "text", + "source_refs", + "artifact_refs", + "entry_content_hash", + "created_in_revision", + ), + ), + "memory_entry_head": ( + ("scope_id", "memory_artifact_id", "entry_id"), + ("family", "head_revision", "entry_version_id", "entry_content_hash"), + ), + "candidate_version": ( + ("scope_id", "candidate_id", "version"), + ( + "family", + "proposal", + "source_refs", + "artifact_refs", + "target_family", + "target_artifact_id", + "target_revision", + "reason", + ), + ), + "candidate_head": ( + ("scope_id", "candidate_id"), + ("family", "version", "status", "result_family", "result_artifact_id", "result_revision", "decision_reason"), + ), +} + +BINARY_FIELDS = frozenset({"payload", "content", "proposal", "source_refs", "artifact_refs"}) +INTEGER_FIELDS = frozenset({ + "position", + "journal_position", + "revision", + "ordinal", + "upstream_revision", + "version", + "created_in_revision", + "head_revision", + "target_revision", + "result_revision", +}) +NULLABLE_FIELDS = frozenset({ + "previous_version_id", + "target_family", + "target_artifact_id", + "target_revision", + "result_family", + "result_artifact_id", + "result_revision", + "decision_reason", + "reason", +}) diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 9288a22855..b47a606954 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -19,13 +19,14 @@ import asyncio from collections.abc import Callable from dataclasses import dataclass +from pathlib import Path from typing import Any, cast from uuid import uuid4 from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncConnection -from powercontext.artifacts import Artifact +from powercontext.artifacts import Artifact, ArtifactRef from powercontext.builtin.artifacts.experience import ( EXPERIENCE_INCUBATION_CURSOR_NAME, Experience, @@ -77,7 +78,7 @@ from powercontext.builtin.persistence.sources import SourceRepository, StoredSource from powercontext.builtin.persistence.statistics import StatisticsRepository from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, SOURCE_JOURNAL_HEADS_TABLE -from powercontext.builtin.portability import PortableBundleService +from powercontext.builtin.portability import BundleInspection, PortableBundleService from powercontext.builtin.review.generation import ( GeneratedCandidateResult, GenerationCapabilityUnavailableError, @@ -123,6 +124,16 @@ CONTENT_SOURCE_ADAPTER, EXTERNAL_SKILL_SNAPSHOT_SOURCE_ADAPTER, ) +_PORTABLE_FAMILIES = (Handoff.family, Memory.family, Experience.family, Skill.family) + + +async def validate_builtin_archive(source: Path) -> BundleInspection: + """Check a bundle against built-in adapters without initializing Runtime state.""" + return await PortableBundleService.validate_archive( + source, + supported_source_types=tuple(adapter.name for adapter in _SOURCE_ADAPTERS), + supported_artifact_families=_PORTABLE_FAMILIES, + ) @dataclass(frozen=True, slots=True) @@ -310,7 +321,7 @@ def __init__( database, projection_rebuilder=self.rebuild_portable_projections, supported_source_types=tuple(adapter.name for adapter in _SOURCE_ADAPTERS), - supported_artifact_families=(Handoff.family, Memory.family, Experience.family, Skill.family), + supported_artifact_families=_PORTABLE_FAMILIES, ) self.index = NoMemoryIndex() if index is None else index self.experience_index = NoExperienceIndex() if experience_index is None else experience_index @@ -471,7 +482,26 @@ async def rebuild_portable_projections(self, scope_ids: tuple[str, ...], /) -> N for scope_id in scope_ids: services = self._services_for(scope_id) _, catalog = services.sources() - await services.memory(catalog).rebuild_projections() + await services.memory(catalog).rebuild_projections(self._embedding_model) + if self.index.capabilities.vector: + async with self.database.transaction() as connection: + heads = ( + await connection.execute( + select(ARTIFACT_HEADS_TABLE.c.artifact_id, ARTIFACT_HEADS_TABLE.c.revision).where( + ARTIFACT_HEADS_TABLE.c.scope_id == scope_id, + ARTIFACT_HEADS_TABLE.c.family == Memory.family, + ) + ) + ).all() + refs = tuple( + ArtifactRef(family=Memory.family, artifact_id=str(row[0]), revision=int(row[1])) + for row in heads + ) + model = self._embedding_model + if refs and ( + model is None or not await self.index.vector_complete(connection, scope_id, refs, model.profile) + ): + raise RuntimeError("restored Memory vector projection is incomplete") # noqa: TRY003 async with self.database.transaction() as connection: await self.experience_index.initialize(connection) diff --git a/src/powercontext/cli/archive.py b/src/powercontext/cli/archive.py index 5da586d16a..7f0bbc1279 100644 --- a/src/powercontext/cli/archive.py +++ b/src/powercontext/cli/archive.py @@ -4,9 +4,9 @@ import asyncio import json -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Coroutine from pathlib import Path -from typing import Annotated, TypeVar +from typing import Annotated, Any, TypeVar import typer @@ -19,6 +19,7 @@ PortableBundleService, ) from powercontext.builtin.runtime import BuiltinConfig, open_builtin_runtime +from powercontext.builtin.runtime.relational import validate_builtin_archive from powercontext.paths import default_database_path, default_scheduler_path, sqlite_url HELP_OPTION_NAMES = ("-h", "--help") @@ -50,7 +51,7 @@ def inspect_bundle( ) -> None: """Verify archive structure and checksums without writing domain data.""" - _emit_archive_result(lambda archive: archive.inspect(source)) + _emit_result(lambda: PortableBundleService.inspect(source)) @archive_app.command("restore") @@ -62,7 +63,7 @@ def restore_bundle( """Validate or restore a portable bundle into the configured deployment.""" if dry_run: - _emit_archive_result(lambda archive: archive.validate(source)) + _emit_result(lambda: _validate_only(source)) return if not yes: raise typer.BadParameter( # noqa: TRY003 @@ -83,8 +84,18 @@ async def _run_archive(operation: Callable[[PortableBundleService], Awaitable[_R def _emit_archive_result(operation: Callable[[PortableBundleService], Awaitable[_ResultT]], /) -> None: """Render expected archive failures without leaking stack traces or records.""" + _emit_result(lambda: _run_archive(operation)) + + +async def _validate_only(source: Path) -> BundleInspection: + # Validation checks the wire format and built-in adapters, not live Runtime + # projections. No database needs to be created or initialized for this. + return await validate_builtin_archive(source) + + +def _emit_result(operation: Callable[[], Coroutine[Any, Any, _ResultT]], /) -> None: try: - _emit(asyncio.run(_run_archive(operation))) + _emit(asyncio.run(operation())) except BundleFormatError as error: typer.echo(f"error: archive validation failed: {error}", err=True) raise typer.Exit(code=2) from error diff --git a/tests/builtin/portability/test_bundle.py b/tests/builtin/portability/test_bundle.py index 6d9e10f15b..af455bda67 100644 --- a/tests/builtin/portability/test_bundle.py +++ b/tests/builtin/portability/test_bundle.py @@ -30,6 +30,43 @@ from powercontext.builtin.portability import BundleConflictError, BundleFormatError, PortableBundleService +@pytest.mark.parametrize("mutation", ["identity_override", "unknown_field", "missing_field", "wrong_type", "scope"]) +def test_restore_rejects_invalid_wire_fields_before_writing(tmp_path: Path, mutation: str) -> None: + async def scenario() -> None: + archive = tmp_path / "invalid.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await PortableBundleService(source.database).export(["project:one"], archive) + with zipfile.ZipFile(archive) as bundle: + manifest = json.loads(bundle.read("manifest.json")) + record = json.loads(bundle.read("records.ndjson")) + if mutation == "identity_override": + record["payload"]["scope_id"] = "project:other" + elif mutation == "unknown_field": + record["payload"]["unexpected"] = "value" + elif mutation == "missing_field": + del record["payload"]["position"] + elif mutation == "wrong_type": + record["payload"]["position"] = True + else: + manifest["scopes"] = ["project:other"] + del record["digest"] + canonical = json.dumps(record, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() + record["digest"] = "sha256:" + hashlib.sha256(canonical).hexdigest() + manifest["total_digest"] = "sha256:" + hashlib.sha256(record["digest"].encode()).hexdigest() + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("manifest.json", json.dumps(manifest)) + bundle.writestr("records.ndjson", json.dumps(record) + "\n") + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + with pytest.raises(BundleFormatError): + await PortableBundleService(target.database).restore(archive) + async with target.database.transaction() as connection: + assert (await connection.execute(select(SOURCE_JOURNAL_HEADS_TABLE))).all() == [] + + asyncio.run(scenario()) + + def test_bundle_round_trips_authoritative_scope_data_and_excludes_projections(tmp_path: Path) -> None: async def scenario() -> None: archive = tmp_path / "scope.pcb" @@ -284,7 +321,9 @@ async def scenario() -> None: SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES ) as source: async with source.database.transaction() as connection: - await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1_500)) + await connection.execute( + insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1_500) + ) await connection.execute( insert(SOURCES_TABLE), [ diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index 6a948efbc9..a912ebd7bd 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -10,7 +10,8 @@ import asyncio from pathlib import Path -from powercontext.builtin.artifacts.memory import MemoryEntryInput +from powercontext.builtin.artifacts.memory import EmbeddingProfile, MemoryEntryInput +from powercontext.builtin.inference import EmbeddingResult from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.runtime import ( BuiltinConfig, @@ -19,6 +20,7 @@ HandoffSourceCitation, HandoffStatement, RememberMemoryRequest, + open_builtin_contexts, open_builtin_runtime, ) @@ -78,3 +80,38 @@ async def scenario() -> None: assert latest.lineage.sources == (source.source_ref,) asyncio.run(scenario()) + + +class _ArchiveEmbeddingModel: + profile = EmbeddingProfile( + profile_id="archive-test", model="test", dimension=3, distance="l2", normalization="unit" + ) + + async def embed(self, texts: tuple[str, ...], /) -> EmbeddingResult: + return EmbeddingResult(vectors=tuple((1.0, 0.0, 0.0) for _ in texts)) + + +def test_restored_memory_vector_search_survives_restart(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "vector.pcb" + target = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'vector.db'}")) + model = _ArchiveEmbeddingModel() + async with open_builtin_contexts(BuiltinConfig(), embedding_model=model) as source: + memory = await (await source.get("project:vector")).artifacts.memory.remember( + memory=None, + entries=(MemoryEntryInput(kind="fact", text="Archive preserves semantic search."),), + mode="append", + ) + assert memory is not None + await source.portability.export(["project:vector"], archive) + async with open_builtin_contexts(target, embedding_model=model) as restored: + assert (await restored.portability.restore(archive)).projections_ready + assert (await restored.portability.restore(archive)).inserted == 0 + async with open_builtin_contexts(target, embedding_model=model) as reopened: + service = (await reopened.get("project:vector")).artifacts.memory + for mode in ("vector", "hybrid"): + result = await service.search("semantic", memories=(memory,), mode=mode) + assert result.hits[0].text == "Archive preserves semantic search." + assert "vector" in result.hits[0].matched_by + + asyncio.run(scenario()) diff --git a/tests/test_archive_cli.py b/tests/test_archive_cli.py index be1fec332e..74fc723fad 100644 --- a/tests/test_archive_cli.py +++ b/tests/test_archive_cli.py @@ -39,6 +39,23 @@ def test_archive_cli_can_export_inspect_and_dry_run_an_empty_scope(tmp_path, mon assert validated.exit_code == 0, validated.output +def test_inspect_and_dry_run_do_not_create_target_database(tmp_path, monkeypatch) -> None: + source_home = tmp_path / "source" + monkeypatch.setenv("POWERCONTEXT_HOME", str(source_home)) + archive = tmp_path / "scope.pcb" + cli = create_cli([archive_app]) + runner = CliRunner() + assert ( + runner.invoke(cli, ["archive", "export", "--scope-id", "project:one", "--output", str(archive)]).exit_code == 0 + ) + target_home = tmp_path / "unused-target" + monkeypatch.setenv("POWERCONTEXT_HOME", str(target_home)) + for arguments in (["inspect", str(archive)], ["restore", str(archive), "--dry-run"]): + result = runner.invoke(cli, ["archive", *arguments]) + assert result.exit_code == 0, result.output + assert not target_home.exists() + + def test_archive_cli_reports_corrupt_or_missing_bundles_without_a_traceback() -> None: result = CliRunner().invoke(create_cli([archive_app]), ["archive", "inspect", "missing.pcb"]) From 9f1029922ac6b636fb51f7f9ac7b0fe371c92cfb Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Sun, 20 Sep 2026 16:39:00 +0800 Subject: [PATCH 21/32] refactor(archive): remove redundant code and unrelated changes --- scripts/integration_guidance_native.py | 14 +- scripts/scan_e2e_evidence.sh | 3 +- .../builtin/portability/bundle.py | 149 +++++++----------- tests/client/test_receiver_service.py | 6 - tests/e2e/test_portable_archive_oceanbase.py | 123 --------------- tests/e2e/test_tag_schema_upgrade.py | 4 +- uv.lock | 10 +- 7 files changed, 70 insertions(+), 239 deletions(-) delete mode 100644 tests/e2e/test_portable_archive_oceanbase.py diff --git a/scripts/integration_guidance_native.py b/scripts/integration_guidance_native.py index a7dcb5d948..decd7c3601 100644 --- a/scripts/integration_guidance_native.py +++ b/scripts/integration_guidance_native.py @@ -58,21 +58,9 @@ async def call(self, name: str, arguments: dict[str, Any], fixture: HandoffFixtu if node is None: message = "Native guidance evaluation requires Node.js 22.19+ and installed host dependencies" raise RuntimeError(message) - probe = await asyncio.create_subprocess_exec( - node, - "-p", - 'process.allowedNodeEnvironmentFlags.has("--experimental-transform-types")', - stdout=asyncio.subprocess.PIPE, - ) - flags, _ = await probe.communicate() - if probe.returncode: - message = "Could not inspect native guidance Node.js capabilities" - raise RuntimeError(message) - # Newer Node releases remove the flag after enabling TypeScript by default. - transform_flags = ("--experimental-transform-types",) if flags.strip() == b"true" else () self.process = await asyncio.create_subprocess_exec( node, - *transform_flags, + "--experimental-transform-types", str(Path(__file__).with_suffix(".mjs")), self.host, stdin=asyncio.subprocess.PIPE, diff --git a/scripts/scan_e2e_evidence.sh b/scripts/scan_e2e_evidence.sh index 0b0bee5ed4..84529e05c0 100644 --- a/scripts/scan_e2e_evidence.sh +++ b/scripts/scan_e2e_evidence.sh @@ -104,8 +104,7 @@ run_scan() { sanitize_detector() { local value="$1" - local lower_value - lower_value="$(printf '%s' "${value}" | LC_ALL=C tr '[:upper:]' '[:lower:]')" + local lower_value="${value,,}" if test "${#value}" -le 120; then case "${value}" in ""|[!A-Za-z0-9]*) ;; diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index e5fb99af95..e5591115d0 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -139,16 +139,10 @@ class BundleFormatError(ValueError): """The supplied archive is malformed, corrupt, or unsupported.""" - def __init__(self, reason: str) -> None: - super().__init__(reason) - class BundleConflictError(ValueError): """A target immutable identity exists with different canonical content.""" - def __init__(self, reason: str) -> None: - super().__init__(reason) - @dataclass(frozen=True, slots=True) class BundleInspection: @@ -208,11 +202,6 @@ class _Record: digest: str -@dataclass(frozen=True, slots=True) -class _ParsedBundle: - inspection: BundleInspection - - @dataclass(frozen=True, slots=True) class _RecordSpec: """Stable logical fields and their current relational persistence adapter.""" @@ -447,7 +436,7 @@ async def export( async def inspect(source: Path, /, *, progress: ProgressObserver | None = None) -> BundleInspection: """Verify structure and checksums without touching the target database.""" - return _parse_bundle(source, progress=progress, phase="inspect").inspection + return _parse_bundle(source, progress=progress, phase="inspect") async def validate( self, @@ -459,7 +448,7 @@ async def validate( ) -> BundleValidation: """Verify bundle integrity, dependencies, and optional adapter support.""" - parsed = _parse_bundle(source, progress=progress, phase="validate") + inspection = _parse_bundle(source, progress=progress, phase="validate") _validate_dependencies(source) required_sources = { str(record.identity["source_type"]) for record in _iter_records(source) if record.record_type == "source" @@ -475,16 +464,16 @@ async def validate( else tuple(sorted(required_families - self._supported_artifact_families)) ) already_present, conflicts = await self._target_compatibility( - source, total=parsed.inspection.record_count, progress=progress + source, total=inspection.record_count, progress=progress ) return BundleValidation( - bundle_id=parsed.inspection.bundle_id, - scopes=parsed.inspection.scopes, - record_count=parsed.inspection.record_count, - records_by_type=parsed.inspection.records_by_type, - total_digest=parsed.inspection.total_digest, - format_version=parsed.inspection.format_version, - producer_version=parsed.inspection.producer_version, + bundle_id=inspection.bundle_id, + scopes=inspection.scopes, + record_count=inspection.record_count, + records_by_type=inspection.records_by_type, + total_digest=inspection.total_digest, + format_version=inspection.format_version, + producer_version=inspection.producer_version, already_present=already_present, conflicts=conflicts, projections_supported=self._projection_rebuilder is not None, @@ -524,7 +513,6 @@ async def restore( async def _restore_snapshot( self, source: Path, /, *, supported_source_types: Iterable[str] | None, progress: ProgressObserver | None ) -> BundleReceipt: - parsed = _parse_bundle(source) validation = await self.validate(source, supported_source_types=supported_source_types, progress=progress) if validation.conflicts: raise BundleConflictError(f"target contains {validation.conflicts} conflicting record identities") @@ -534,11 +522,11 @@ async def _restore_snapshot( with _staged_records(source) as staged: async with self._database.transaction() as connection: inserted, existing = await self._write_records( - connection, staged, total=parsed.inspection.record_count, progress=progress + connection, staged, total=validation.record_count, progress=progress ) await self._write_restore_receipt( connection, - parsed.inspection, + validation, inserted=inserted, already_present=existing, projections_ready=False, @@ -548,21 +536,21 @@ async def _restore_snapshot( projections_ready = False if self._projection_rebuilder is not None: _report_progress(progress, "projections", 0, 1) - await self._projection_rebuilder(parsed.inspection.scopes) + await self._projection_rebuilder(validation.scopes) projections_ready = True _report_progress(progress, "projections", 1, 1) async with self._database.transaction() as connection: await self._write_restore_receipt( connection, - parsed.inspection, + validation, inserted=inserted, already_present=existing, projections_ready=True, ) return BundleReceipt( - bundle_id=parsed.inspection.bundle_id, - record_count=parsed.inspection.record_count, - total_digest=parsed.inspection.total_digest, + bundle_id=validation.bundle_id, + record_count=validation.record_count, + total_digest=validation.total_digest, inserted=inserted, already_present=existing, projections_ready=projections_ready, @@ -667,9 +655,9 @@ async def _write_records( existing = 0 processed = 0 for record in _ordered_scope_records(_staged_type_records(staged, "scope")): - was_inserted = await self._write_record(connection, record) - inserted += int(was_inserted) - existing += int(not was_inserted) + added, skipped = await self._write_batch(connection, "scope", (record,)) + inserted += added + existing += skipped processed += 1 _report_progress(progress, "restore", processed, total) # The disk index was built before acquiring the target write transaction. @@ -714,18 +702,6 @@ async def _write_batch( await connection.execute(insert(spec.table), values) return len(values), len(records) - len(values) - async def _write_record(self, connection: AsyncConnection, record: _Record, /) -> bool: - table = _SPECS[record.record_type].table - where = [table.c[key] == _database_value(value, table.c[key]) for key, value in record.identity.items()] - row = (await connection.execute(select(table).where(*where))).mappings().one_or_none() - if row is not None: - present = _record_from_row(record.record_type, cast(Mapping[str, Any], dict(row))) - if present.digest != record.digest: - raise BundleConflictError(f"immutable identity conflict: {record.record_type}") - return False - await connection.execute(insert(table).values(**_row_values(record))) - return True - async def _stream_export( self, connection: AsyncConnection, @@ -840,7 +816,7 @@ def _parse_bundle( *, progress: ProgressObserver | None = None, phase: Literal["inspect", "validate"] = "inspect", -) -> _ParsedBundle: +) -> BundleInspection: try: with zipfile.ZipFile(source) as archive: _validate_container(archive) @@ -852,7 +828,10 @@ def _parse_bundle( count = 0 counts: Counter[str] = Counter() digest = _DigestAccumulator() + scope_ids: list[str] = [] for record in _iter_records(source): + if record.record_type == "scope": + scope_ids.append(str(record.identity["scope_id"])) count += 1 counts[record.record_type] += 1 digest.add(record.digest) @@ -874,12 +853,9 @@ def _parse_bundle( format_version=manifest["format_version"], producer_version=manifest["producer"]["version"], ) - scope_ids = tuple( - str(record.identity["scope_id"]) for record in _iter_records(source) if record.record_type == "scope" - ) if tuple(sorted(scope_ids)) != inspection.scopes: raise BundleFormatError("manifest scopes do not match scope records") - return _ParsedBundle(inspection=inspection) + return inspection @contextmanager @@ -1506,7 +1482,7 @@ def _validate_record_dependencies( # noqa: C901 - logical record kinds have dis }, ): raise BundleFormatError("candidate head references missing result revision") - _validate_embedded_references((record,), contains) + _validate_embedded_references(record, contains) _validate_memory_citations(record, contains) @@ -1538,7 +1514,8 @@ def _validate_source_journal_records(source: Path, /) -> None: raise BundleFormatError("source journal snapshot is inconsistent") -def _validate_skill_package_contents(source: Path, /) -> None: +def _validated_skill_packages(source: Path, /) -> dict[tuple[str, str], tuple[str, int, int, int]]: + packages: dict[tuple[str, str], tuple[str, int, int, int]] = {} for record in _iter_records(source): if record.record_type != "skill_package": continue @@ -1557,20 +1534,17 @@ def _validate_skill_package_contents(source: Path, /) -> None: record.payload["manifest"] ): raise BundleFormatError("Skill package bytes do not match metadata") + packages[(str(record.identity["scope_id"]), str(record.identity["tree_digest"]))] = ( + snapshot.reference.archive_digest, + snapshot.reference.file_count, + snapshot.reference.uncompressed_size, + snapshot.reference.archive_size, + ) + return packages def _validate_skill_package_dependencies(source: Path, /) -> None: - _validate_skill_package_contents(source) - packages = { - (str(record.identity["scope_id"]), str(record.identity["tree_digest"])): ( - str(record.payload["archive_digest"]), - int(record.payload["file_count"]), - int(record.payload["uncompressed_size"]), - int(record.payload["archive_size"]), - ) - for record in _iter_records(source) - if record.record_type == "skill_package" - } + packages = _validated_skill_packages(source) for record in _iter_records(source): if record.record_type == "artifact_revision" and record.identity["family"] == "skill": content = _decoded_bytes_json(record.payload["content"], "Skill content") @@ -1664,34 +1638,33 @@ def contains(record_type: str, identity: Mapping[str, object]) -> bool: def _validate_embedded_references( - records: Iterable[_Record], + record: _Record, contains: Callable[[str, Mapping[str, object]], bool], ) -> None: - for record in records: - if record.record_type not in {"memory_entry_version", "candidate_version"}: - continue - scope_id = record.identity["scope_id"] - for reference in _reference_items(record.payload["source_refs"], "source"): - if not contains( - "source", - { - "scope_id": scope_id, - "source_type": reference["source_type"], - "source_id": reference["source_id"], - }, - ): - raise BundleFormatError("record references missing source") - for reference in _reference_items(record.payload["artifact_refs"], "artifact"): - if not contains( - "artifact_revision", - { - "scope_id": scope_id, - "family": reference["family"], - "artifact_id": reference["artifact_id"], - "revision": reference["revision"], - }, - ): - raise BundleFormatError("record references missing artifact revision") + if record.record_type not in {"memory_entry_version", "candidate_version"}: + return + scope_id = record.identity["scope_id"] + for reference in _reference_items(record.payload["source_refs"], "source"): + if not contains( + "source", + { + "scope_id": scope_id, + "source_type": reference["source_type"], + "source_id": reference["source_id"], + }, + ): + raise BundleFormatError("record references missing source") + for reference in _reference_items(record.payload["artifact_refs"], "artifact"): + if not contains( + "artifact_revision", + { + "scope_id": scope_id, + "family": reference["family"], + "artifact_id": reference["artifact_id"], + "revision": reference["revision"], + }, + ): + raise BundleFormatError("record references missing artifact revision") def _reference_items(value: object, kind: Literal["source", "artifact"], /) -> tuple[Mapping[str, object], ...]: diff --git a/tests/client/test_receiver_service.py b/tests/client/test_receiver_service.py index 59da92f87d..5886a97e9a 100644 --- a/tests/client/test_receiver_service.py +++ b/tests/client/test_receiver_service.py @@ -13,7 +13,6 @@ # limitations under the License. from pathlib import Path -from types import SimpleNamespace import pytest from pydantic import SecretStr @@ -27,11 +26,6 @@ from powercontext.client.skill_receiver import RemoteSkillReceiverConfig -@pytest.fixture(autouse=True) -def linux_service_platform(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(service_module, "sys", SimpleNamespace(platform="linux", argv=service_module.sys.argv)) - - def _config(tmp_path: Path) -> RemoteSkillReceiverConfig: return RemoteSkillReceiverConfig( server_url="https://powercontext.example.com", diff --git a/tests/e2e/test_portable_archive_oceanbase.py b/tests/e2e/test_portable_archive_oceanbase.py deleted file mode 100644 index 61c16c5c29..0000000000 --- a/tests/e2e/test_portable_archive_oceanbase.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (c) 2026 OceanBase. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Live SQLite-to-OceanBase portable archive acceptance.""" - -from __future__ import annotations - -import asyncio -import os -from pathlib import Path -from uuid import uuid4 - -import pytest -from pydantic import SecretStr -from sqlalchemy import insert, select -from sqlalchemy.engine import make_url - -from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile -from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile -from powercontext.builtin.persistence.tables import ( - ARTIFACT_HEADS_TABLE, - ARTIFACTS_TABLE, - BUILTIN_TABLES, - SCOPES_TABLE, -) -from powercontext.builtin.portability import PortableBundleService - -LIVE_URL = os.environ.get("POWERCONTEXT_TEST_OCEANBASE_URL") - - -async def _authorize(_scopes: tuple[str, ...], /) -> None: - pass - - -@pytest.mark.skipif( - not LIVE_URL, - reason="set POWERCONTEXT_TEST_OCEANBASE_URL with test database creation and deletion privileges", -) -def test_sqlite_bundle_restores_exact_revision_into_oceanbase(tmp_path: Path) -> None: - async def scenario() -> None: - assert LIVE_URL is not None - scope_id = "project:portable-oceanbase" - archive = tmp_path / "sqlite-to-oceanbase.pcb" - async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as sqlite: - async with sqlite.database.transaction() as connection: - await connection.execute( - insert(SCOPES_TABLE).values( - scope_id=scope_id, - title="Portable migration", - summary="SQLite to OceanBase", - scope_id_search=scope_id, - title_search="Portable migration", - summary_search="SQLite to OceanBase", - parent_scope_id=None, - version=1, - ) - ) - await connection.execute( - insert(ARTIFACTS_TABLE).values( - scope_id=scope_id, - family="handoff", - artifact_id="handoff", - revision=1, - content=b'{"content":{"objective":"portable"}}', - ) - ) - await connection.execute( - insert(ARTIFACT_HEADS_TABLE).values( - scope_id=scope_id, - family="handoff", - artifact_id="handoff", - revision=1, - ) - ) - exported = await PortableBundleService(sqlite.database).export([scope_id], archive, authorize=_authorize) - - database_name = f"pc_portable_{uuid4().hex}" - base_config = OceanBaseConfig(url=SecretStr(LIVE_URL)) - created = False - async with OceanBaseProfile.open(base_config, tables=()) as server: - try: - async with server.database.transaction() as connection: - await connection.exec_driver_sql(f"CREATE DATABASE `{database_name}`") - created = True - target_url = make_url(LIVE_URL).set(database=database_name).render_as_string(hide_password=False) - async with OceanBaseProfile.open( - OceanBaseConfig(url=SecretStr(target_url)), tables=BUILTIN_TABLES - ) as oceanbase: - restored = await PortableBundleService(oceanbase.database).restore(archive) - async with oceanbase.database.transaction() as connection: - row = ( - ( - await connection.execute( - select(ARTIFACTS_TABLE).where( - ARTIFACTS_TABLE.c.scope_id == scope_id, - ARTIFACTS_TABLE.c.family == "handoff", - ARTIFACTS_TABLE.c.artifact_id == "handoff", - ARTIFACTS_TABLE.c.revision == 1, - ) - ) - ) - .mappings() - .one() - ) - assert restored.record_count == exported.record_count - assert bytes(row["content"]) == b'{"content":{"objective":"portable"}}' - finally: - if created: - async with server.database.transaction() as connection: - await connection.exec_driver_sql(f"DROP DATABASE `{database_name}`") - - asyncio.run(scenario()) diff --git a/tests/e2e/test_tag_schema_upgrade.py b/tests/e2e/test_tag_schema_upgrade.py index 818533b547..9922c9eba7 100644 --- a/tests/e2e/test_tag_schema_upgrade.py +++ b/tests/e2e/test_tag_schema_upgrade.py @@ -61,9 +61,9 @@ async def _isolated_database(backend, tmp_path): @pytest.mark.parametrize("backend", ["seekdb", "oceanbase"]) @pytest.mark.parametrize("topic_family", [False, True]) -def test_real_legacy_tags_survive_repeated_server_startup(backend, topic_family, tmp_path, short_tmp_path): +def test_real_legacy_tags_survive_repeated_server_startup(backend, topic_family, tmp_path): async def scenario(): - async with _isolated_database(backend, short_tmp_path) as (database, open_profile): + async with _isolated_database(backend, tmp_path) as (database, open_profile): settings = ServerSettings( database=database, runtime=RuntimeConfig(artifact_processing_families=()), diff --git a/uv.lock b/uv.lock index 99bf11c0ef..4b3badbaf2 100644 --- a/uv.lock +++ b/uv.lock @@ -1202,7 +1202,7 @@ name = "importlib-metadata" version = "9.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp", marker = "python_full_version < '3.12'" }, + { name = "zipp" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } wheels = [ @@ -2369,8 +2369,8 @@ name = "opendalfs" version = "0.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "fsspec", marker = "python_full_version >= '3.12'" }, - { name = "opendal", marker = "python_full_version >= '3.12'" }, + { name = "fsspec" }, + { name = "opendal" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0d/23/fdb4d34f08ac68a27b2d553256e13b15ce139916d36bab33f9a0e71e8770/opendalfs-0.1.0.tar.gz", hash = "sha256:5ceaeccc0852ef10c8ef2ce20e5927dbc18083909c06550ad36f81bcacb5774d", size = 75112, upload-time = "2026-08-26T05:30:33.497Z" } wheels = [ @@ -2911,8 +2911,8 @@ name = "powercontext-connector-opendal" version = "0.0.1" source = { editable = "integrations/opendal" } dependencies = [ - { name = "opendalfs", marker = "python_full_version >= '3.12'" }, - { name = "powercontext", extra = ["client"], marker = "python_full_version >= '3.12'" }, + { name = "opendalfs" }, + { name = "powercontext", extra = ["client"] }, ] [package.metadata] From 7658ee30c6034b4b92037ba3631e659b949dcc00 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Mon, 21 Sep 2026 20:23:07 +0800 Subject: [PATCH 22/32] fix(archive): pin export snapshots and preserve remote sources --- docs/en/docs/operate/portable-archive.md | 17 +- docs/zh/docs/operate/portable-archive.md | 17 +- .../builtin/portability/bundle.py | 142 ++++++++-- tests/builtin/portability/test_bundle.py | 20 +- tests/e2e/test_portable_archive.py | 248 +++++++++++++++++- 5 files changed, 411 insertions(+), 33 deletions(-) diff --git a/docs/en/docs/operate/portable-archive.md b/docs/en/docs/operate/portable-archive.md index c404cfa55f..516e434358 100644 --- a/docs/en/docs/operate/portable-archive.md +++ b/docs/en/docs/operate/portable-archive.md @@ -12,6 +12,8 @@ backup, an offline transfer, or a future backend migration. This is an offline operator command. It does not call a remote Server API. Without `--env-file` it opens the default SQLite database; with `--env-file` it uses the same SQLite, SeekDB, or OceanBase settings as that deployment. Stop PowerContext writers before restore. Export uses one database transaction as its consistent logical snapshot. +MySQL-mode backends use `REPEATABLE READ` for the export transaction even when the deployment defaults to +`READ COMMITTED`; later transactions retain the deployment default. ## Prerequisites @@ -48,6 +50,8 @@ JSON on stderr while the final receipt remains JSON on stdout. `export` writes JSON containing the bundle ID, record count, and checksum. `inspect` verifies the ZIP structure, per-record digests, total digest, record counts, and selected scopes without opening the target database. Keep the reported checksum with the backup inventory if an external backup system needs an independent verification record. +Before publishing the file, export checks its format, resource limits, and dependencies. If validation fails, an +existing backup at the output path remains unchanged. ## Validate before a restore @@ -64,6 +68,12 @@ reason; record bodies are not printed. A successful report includes `already_present`, `conflicts`, required and unsupported Source types and Artifact families, and whether the target has a projection rebuilder. Run against a different deployment with `--env-file ./target.env`. +Native Sources require their Python adapter on the target. Worker-materialized `SourceObservation` records carry +their captured payload and projections, so they can be read without loading that adapter. The bundle includes only +the definition manifests referenced by the selected scopes and verifies observation identities and definition +fingerprints before restore. Missing definitions and conflicting target definitions prevent restoration; unreferenced +global definitions are not exported. + To perform the write, repeat the command with explicit confirmation: ```bash @@ -81,7 +91,7 @@ Format version 1 carries the portable relational representation of these support | Preserved | Not portable | | --- | --- | | Scope identity, hierarchy, context/external references, and creation identity | Scope access bindings and host-local default selection | -| Source journal heads and Source records | Search projections and indexes | +| Source journal heads, Source records, and referenced remote Source Definition manifests | Search projections and indexes | | Artifact Revisions, lineage, cross-Scope publication provenance, and heads | Source cursors and scheduler state | | Memory entry versions and heads | External Skill registrations and host-local installation state | | Candidate versions, decision heads, and their evidence references | Audit events, usage facts, evaluation receipts, and restore receipts | @@ -132,8 +142,9 @@ the Server configuration guide. Dry-run must report no unsupported families and ## Format compatibility -Format version `1` readers accept version `1` bundles from any PowerContext producer version. The producer version is -reported for diagnostics but does not replace the archive schema version. Writers produce only version `1`; there is no +Format version `1` readers require support for every record type present in the bundle. Older readers may reject +new record types even when the bundle uses format version `1`. The producer version is reported for diagnostics but +does not replace the archive schema version or record-type compatibility checks. Writers produce only version `1`; there is no format `0` downgrade. A reader rejects unknown archive or record schema versions before target writes. Keep the old binary available until a restore drill succeeds when upgrading across a PowerContext major release. diff --git a/docs/zh/docs/operate/portable-archive.md b/docs/zh/docs/operate/portable-archive.md index b9af9b1953..93de1292b0 100644 --- a/docs/zh/docs/operate/portable-archive.md +++ b/docs/zh/docs/operate/portable-archive.md @@ -10,7 +10,8 @@ description: 创建、验证并恢复经过校验的 PowerContext 逻辑归档 这是一个离线运维命令，不调用远程 Server API。不传 `--env-file` 时打开默认 SQLite 数据库；传入后使用该部署的 SQLite、SeekDB 或 OceanBase 配置。恢复前应停止 PowerContext 写入进程；导出使用一个数据库 transaction 获取一致的 -逻辑 snapshot。 +逻辑 snapshot。MySQL 模式的后端会为导出事务使用 `REPEATABLE READ`，即使部署默认采用 +`READ COMMITTED`；后续事务仍使用部署的默认隔离级别。 ## 前提条件 @@ -44,6 +45,7 @@ powercontext archive inspect ./backups/payments.pcb `export` 会输出 JSON，其中包括 bundle ID、record 数和 checksum。`inspect` 不打开目标数据库，只校验 ZIP 结构、 单 record digest、总 digest、record 数和所选 scope。若外部备份系统需要独立校验记录，请把输出的 checksum 一并保存。 +导出在发布文件前还会检查格式、资源限制和依赖完整性。验证失败时，输出路径上的旧备份保持不变。 ## 在恢复前验证 @@ -54,10 +56,14 @@ powercontext archive restore ./backups/payments.pcb --dry-run ``` dry-run 不会写入领域数据。它会校验 checksum、必需的 Source 和 Artifact 依赖，以及已配置 Runtime 是否支持归档中的 -source type 和 Artifact family。验证失败会以退出码 `2` 结束，只输出不含内容的原因，不会打印 record body。成功报告 +source type 和 Artifact family。归档无效时返回退出码 `2`，不兼容或目标冲突时返回 `3`；错误信息不打印 record body。成功报告 包含 `already_present`、`conflicts`、所需和不支持的 Source type/Artifact family，以及目标是否支持 projection rebuild。 可用 `--env-file ./target.env` 检查另一个部署。 +原生 Source 需要目标环境安装对应的 Python adapter。远程 worker 已物化的 `SourceObservation` 自带捕获内容和投影， +读取时不需要加载该 adapter。归档只携带所选 scope 实际引用的 Source Definition manifest，并在恢复前校验 observation +的身份和定义指纹。缺失定义或与目标定义冲突会阻止恢复；未引用的全局定义不会随包导出。 + 要执行写恢复，必须显式确认： ```bash @@ -75,7 +81,7 @@ powercontext archive restore ./backups/payments.pcb --yes | 会保留 | 不可移植 | | --- | --- | | Scope identity、层级、context/external reference 和创建 identity | Scope access binding 和 host-local default selection | -| Source journal head 和 Source record | 搜索 projection 和 index | +| Source journal head、Source record 和引用的远程 Source Definition manifest | 搜索 projection 和 index | | Artifact Revision、lineage、跨 Scope 发布来源和 head | Source cursor 和 scheduler state | | Memory entry version 和 head | External Skill registration 和 host-local installation state | | Candidate version、decision head 及其 evidence reference | Audit event、usage fact、evaluation receipt 和 restore receipt | @@ -122,8 +128,9 @@ powercontext archive restore ./payments.pcb --env-file ./oceanbase.env --yes ## 格式兼容窗口 -格式版本 `1` reader 接受任意 PowerContext producer version 生成的格式版本 `1` bundle。producer version 仅用于诊断， -不能替代 archive schema version。writer 只生成版本 `1`，不支持降级到格式 `0`。遇到未知 archive 或 record schema +格式版本 `1` reader 还必须支持归档中出现的全部 record type。即使归档使用格式版本 `1`，旧 reader 也可能拒绝新增的 +record type。producer version 仅用于诊断，不能替代 archive schema version 或记录类型兼容性检查。 +writer 只生成版本 `1`，不支持降级到格式 `0`。遇到未知 archive 或 record schema version 时，reader 会在目标写入前拒绝。跨 PowerContext 大版本升级时，应保留旧二进制，直到恢复演练通过。 ## 部署原生备份 diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index e5591115d0..0d9cfceb9d 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -33,7 +33,7 @@ import tempfile import zipfile from collections import Counter -from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping +from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Iterator, Mapping from contextlib import contextmanager from dataclasses import dataclass from datetime import UTC, date, datetime @@ -42,6 +42,7 @@ from typing import Any, Literal, cast from uuid import NAMESPACE_URL, UUID, uuid5 +from pydantic import ValidationError from sqlalchemy import DateTime, insert, or_, select, tuple_, update from sqlalchemy import inspect as sqlalchemy_inspect from sqlalchemy.exc import IntegrityError @@ -67,10 +68,12 @@ SCOPE_EXTERNAL_REFERENCES_TABLE, SCOPES_TABLE, SKILL_PACKAGES_TABLE, + SOURCE_DEFINITION_MANIFESTS_TABLE, SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE, TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE, ) +from powercontext.sources import SourceDefinitionManifest, SourceObservation FORMAT_VERSION = 1 _RECORDS_NAME = "records.ndjson" @@ -115,6 +118,7 @@ "scope_context_reference", "scope_external_reference", "scope_creation_request", + "source_definition", "source_journal_head", "source", "skill_package", @@ -209,7 +213,7 @@ class _RecordSpec: table: Any identity: tuple[str, ...] payload: tuple[str, ...] - scope_field: str = "scope_id" + scope_field: str | None = "scope_id" _SPECS: dict[RecordType, _RecordSpec] = { @@ -225,6 +229,12 @@ class _RecordSpec: ("idempotency_key",), ("request_digest", "scope_id"), ), + "source_definition": _RecordSpec( + SOURCE_DEFINITION_MANIFESTS_TABLE, + ("definition_name", "definition_version"), + ("fingerprint", "manifest"), + scope_field=None, + ), "source_journal_head": _RecordSpec(SOURCE_JOURNAL_HEADS_TABLE, ("scope_id",), ("position",)), "source": _RecordSpec( SOURCES_TABLE, @@ -426,6 +436,11 @@ async def export( ): shutil.copyfileobj(source, target, length=1024 * 1024) archive.writestr(_zip_info(_MANIFEST_NAME, compression), _canonical_json(manifest)) + # Publish only an archive with valid structure and complete dependencies. + # Reject missing dependencies and size-limit breaches + # without replacing a previous usable backup at the destination. + _parse_bundle(archive_path) + _validate_dependencies(archive_path) archive_path.replace(output) return BundleReceipt(bundle_id=bundle_id, record_count=count, total_digest=digest) finally: @@ -451,7 +466,9 @@ async def validate( inspection = _parse_bundle(source, progress=progress, phase="validate") _validate_dependencies(source) required_sources = { - str(record.identity["source_type"]) for record in _iter_records(source) if record.record_type == "source" + str(record.identity["source_type"]) + for record in _iter_records(source) + if record.record_type == "source" and _source_observation(record) is None } configured = ( self._supported_source_types if supported_source_types is None else frozenset(supported_source_types) @@ -714,15 +731,7 @@ async def _stream_export( records_by_type: Counter[str] = Counter() digest = _DigestAccumulator() for record_type in _EXPORT_ORDER: - spec = _SPECS[record_type] - table = spec.table - statement = ( - select(table) - .where(table.c[spec.scope_field].in_(scopes)) - .order_by(*(table.c[field] for field in spec.identity)) - ) - rows = await connection.stream(statement) - async for row in rows.mappings(): + async for row in _export_rows(connection, record_type, scopes): record = _record_from_row(record_type, cast(Mapping[str, Any], dict(row))) _validate_record_safe(record) stream.write(_canonical_json(_record_document(record)) + b"\n") @@ -734,6 +743,83 @@ async def _stream_export( return count, records_by_type, digest.value() +async def _export_rows( + connection: AsyncConnection, kind: RecordType, scopes: tuple[str, ...] +) -> AsyncIterator[Mapping[str, Any]]: + spec = _SPECS[kind] + if spec.scope_field is not None: + rows = await connection.stream( + select(spec.table) + .where(spec.table.c[spec.scope_field].in_(scopes)) + .order_by(*(spec.table.c[field] for field in spec.identity)) + ) + async for row in rows.mappings(): + yield cast(Mapping[str, Any], row) + return + + # Definition manifests are global, but export authority is scope-bound. + # Include only immutable declarations referenced by selected observations. + definitions: set[tuple[str, str]] = set() + async for row in _export_rows(connection, "source", scopes): + observation = _source_observation(_record_from_row("source", row)) + if observation is not None: + definitions.add((observation.source_type, observation.definition_version)) + table = SOURCE_DEFINITION_MANIFESTS_TABLE + for name, definition_version in sorted(definitions): + row = ( + ( + await connection.execute( + select(table).where( + table.c.definition_name == name, table.c.definition_version == definition_version + ) + ) + ) + .mappings() + .one_or_none() + ) + if row is None: + raise BundleFormatError("observation references missing Source definition") + yield cast(Mapping[str, Any], row) + + +def _source_observation(record: _Record) -> SourceObservation | None: + try: + value = json.loads(cast(bytes, _database_value(record.payload["payload"]))) + except (ValueError, UnicodeDecodeError): + return None # Native Source payloads are interpreted by their adapter. + if not isinstance(value, dict): + return None + if value.get("encoding") == "powercontext-source-v1": + if value.get("representation") != "observation": + return None + value = value.get("value") + elif "definition_fingerprint" not in value or "source_type" not in value: + return None # Legacy native payload. + try: + observation = SourceObservation.model_validate_json(_canonical_json(value)) + except ValidationError as error: + raise BundleFormatError("invalid Source observation") from error + if (observation.source_type, observation.name) != (record.identity["source_type"], record.identity["source_id"]): + raise BundleFormatError("Source observation identity mismatch") + return observation + + +def _source_definition(record: _Record) -> SourceDefinitionManifest: + try: + manifest = SourceDefinitionManifest.model_validate_json( + cast(bytes, _database_value(record.payload["manifest"])) + ) + except ValidationError as error: + raise BundleFormatError("invalid Source definition manifest") from error + if (manifest.name, manifest.version, manifest.fingerprint) != ( + record.identity["definition_name"], + record.identity["definition_version"], + record.payload["fingerprint"], + ): + raise BundleFormatError("Source definition identity mismatch") + return manifest + + def _record_from_row(record_type: RecordType, row: Mapping[str, Any]) -> _Record: spec = _SPECS[record_type] identity = {field: _json_value(row[field]) for field in spec.identity} @@ -746,9 +832,11 @@ async def _establish_export_snapshot(connection: AsyncConnection, /) -> None: """Pin all export reads to one database snapshot before enumeration.""" if connection.dialect.name == "mysql": - # AsyncDatabase has already issued START TRANSACTION for MySQL-mode - # engines. Replacing that empty transaction with a consistent snapshot - # is safe because authorization has completed and no query has run yet. + # Replace AsyncDatabase's empty transaction before any reads. SET + # TRANSACTION applies only to the next transaction, so pooled sessions + # retain their configured isolation after export. + await connection.exec_driver_sql("ROLLBACK") + await connection.exec_driver_sql("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") await connection.exec_driver_sql("START TRANSACTION WITH CONSISTENT SNAPSHOT") elif connection.dialect.name == "sqlite": # SQLAlchemy's SQLite transaction begins lazily. Pin the snapshot @@ -1094,6 +1182,10 @@ def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: strings("request_digest", "scope_id") elif kind == "source_journal_head": integers("position") + elif kind == "source_definition": + strings("fingerprint") + bytes_values("manifest") + _source_definition(record) elif kind == "source": bytes_values("payload") integers("journal_position", positive=True) @@ -1314,6 +1406,8 @@ def _validate_record_dependencies( # noqa: C901 - logical record kinds have dis payload = record.payload identity = record.identity scope_field = _SPECS[record.record_type].scope_field + if scope_field is None: + return scope = identity.get(scope_field, payload.get(scope_field)) if not isinstance(scope, str): raise BundleFormatError("record has invalid scope identity") @@ -1329,6 +1423,16 @@ def _validate_record_dependencies( # noqa: C901 - logical record kinds have dis elif record.record_type == "source": if not contains("source_journal_head", {"scope_id": scope}): raise BundleFormatError("source has no journal head") + observation = _source_observation(record) + if observation is not None and not contains( + "source_definition_fingerprint", + { + "definition_name": observation.source_type, + "definition_version": observation.definition_version, + "fingerprint": observation.definition_fingerprint, + }, + ): + raise BundleFormatError("observation references missing or mismatched Source definition") elif record.record_type == "artifact_revision" and identity["family"] == "topic-memory": if not contains("topic_memory_publication", identity): raise BundleFormatError("Topic Memory revision has no publication metadata") @@ -1605,6 +1709,14 @@ def _identity_index(source: Path, /) -> Iterator[Callable[[str, Mapping[str, obj ) except sqlite3.IntegrityError as error: raise BundleFormatError("bundle contains duplicate immutable identity") from error + if record.record_type == "source_definition": + connection.execute( + "INSERT INTO identities (record_type, identity) VALUES (?, ?)", + ( + "source_definition_fingerprint", + _identity_key({**record.identity, "fingerprint": record.payload["fingerprint"]}), + ), + ) if record.record_type == "memory_entry_version": # Tags survive retirement; inactive entries no longer have a # search projection but retain their authoritative versions. diff --git a/tests/builtin/portability/test_bundle.py b/tests/builtin/portability/test_bundle.py index c306949c7d..0c5bcf2761 100644 --- a/tests/builtin/portability/test_bundle.py +++ b/tests/builtin/portability/test_bundle.py @@ -614,7 +614,7 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_validate_rejects_candidate_evidence_that_is_not_in_the_bundle(tmp_path: Path) -> None: +def test_export_rejects_candidate_evidence_that_is_not_in_the_bundle(tmp_path: Path) -> None: async def scenario() -> None: archive = tmp_path / "scope.pcb" database = tmp_path / "source.db" @@ -647,10 +647,12 @@ async def scenario() -> None: status="pending", ) ) - await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) - async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + archive.write_bytes(b"previous backup") with pytest.raises(BundleFormatError, match="missing source"): - await PortableBundleService(target.database).validate(archive) + await PortableBundleService(source.database).export( + ["project:one"], archive, authorize=_authorize_export + ) + assert archive.read_bytes() == b"previous backup" asyncio.run(scenario()) @@ -767,7 +769,7 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_validation_rejects_package_backed_skill_without_its_package(tmp_path: Path) -> None: +def test_export_rejects_package_backed_skill_without_its_package(tmp_path: Path) -> None: async def scenario() -> None: archive = tmp_path / "missing-package.pcb" content = json.dumps({ @@ -800,10 +802,12 @@ async def scenario() -> None: scope_id="project:one", family="skill", artifact_id="broken", revision=1 ) ) - await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) - async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + archive.write_bytes(b"previous backup") with pytest.raises(BundleFormatError, match="missing or mismatched package"): - await PortableBundleService(target.database).validate(archive) + await PortableBundleService(source.database).export( + ["project:one"], archive, authorize=_authorize_export + ) + assert archive.read_bytes() == b"previous backup" asyncio.run(scenario()) diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index 321b08b33e..7cc543c81d 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -17,15 +17,20 @@ from __future__ import annotations import asyncio +import base64 +import hashlib import importlib.util +import json import os +import zipfile +from collections import Counter from collections.abc import Iterator from pathlib import Path from uuid import uuid4 import pytest from pydantic import SecretStr -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event, select, update from sqlalchemy.engine import make_url from powercontext.artifacts import ArtifactRef @@ -35,8 +40,9 @@ from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.seekdb import SeekDBConfig from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.tables import SCOPES_TABLE, SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE from powercontext.builtin.persistence.topic_memory import TopicMemoryStorageInvariantError -from powercontext.builtin.portability import PortableBundleService +from powercontext.builtin.portability import BundleConflictError, BundleFormatError, PortableBundleService from powercontext.builtin.records import ArtifactWrite from powercontext.builtin.runtime import ( BuiltinConfig, @@ -45,12 +51,22 @@ HandoffSourceCitation, HandoffStatement, RememberMemoryRequest, + SubmitSourceObservation, open_builtin_contexts, open_builtin_runtime, ) from powercontext.builtin.scope import ScopeDraft +from powercontext.builtin.sources import ContentCapture from powercontext.builtin.tags import MemoryEntryTagTarget, TagFilter from powercontext.builtin.work import AcknowledgeHandoff, ReceiverChecks +from powercontext.sources import ( + AdapterSourceDefinition, + SourceDefinitionRegistry, + SourceRef, + manifest_for_definition, + project_source_for_transport, +) +from tests.builtin.persistence.contract import NoteAdapter, NoteInput, NoteSource def test_topic_restore_can_retry_after_projection_failure_and_restart(tmp_path: Path) -> None: @@ -339,3 +355,231 @@ def target_config(tmp_path: Path, short_tmp_path: Path, request: pytest.FixtureR engine.dispose() else: yield BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + + +@pytest.mark.parametrize("legacy", [False, True]) +@pytest.mark.parametrize("target_definition", ["absent", "identical", "conflicting"]) +def test_archive_preserves_remote_observations_without_worker_adapter( + tmp_path: Path, legacy: bool, target_definition: str +) -> None: + + async def scenario() -> None: + definition = AdapterSourceDefinition(NoteAdapter()) + registry = SourceDefinitionRegistry((definition,)) + manifest = manifest_for_definition(definition) + observation = project_source_for_transport( + registry, await registry.resolve(NoteInput(note_id="one", body="remote")) + ) + archive = tmp_path / "remote.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Remote", summary="Remote sources", idempotency_key="remote") + ) + await source.register_source_definition(manifest) + + class UnrelatedAdapter(NoteAdapter): + name = "unrelated-note" + + await source.register_source_definition( + manifest_for_definition(AdapterSourceDefinition(UnrelatedAdapter())) + ) + await source.submit_source_observation( + SubmitSourceObservation(scope_id=scope.scope_id, observation=observation) + ) + if legacy: + async with source.database.transaction() as connection: + await connection.execute( + update(SOURCES_TABLE).values(payload=observation.model_dump_json().encode()) + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + assert (await source.portability.inspect(archive)).records_by_type["source_definition"] == 1 + target_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'remote.db'}")) + async with open_builtin_contexts(target_config) as target: + if target_definition == "identical": + await target.register_source_definition(manifest) + elif target_definition == "conflicting": + + class DifferentNoteSource(NoteSource): + category: str = "changed" + + class DifferentNoteAdapter(NoteAdapter): + source_class = DifferentNoteSource + + await target.register_source_definition( + manifest_for_definition(AdapterSourceDefinition(DifferentNoteAdapter())) + ) + validation = await target.portability.validate(archive) + if target_definition == "conflicting": + assert not validation.compatible + assert validation.conflicts == 1 + with pytest.raises(BundleConflictError): + await target.portability.restore(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar( + select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == scope.scope_id) + ) + is None + ) + return + assert validation.compatible + assert validation.unsupported_source_types == () + assert (await target.portability.restore(archive)).projections_ready + async with open_builtin_contexts(target_config) as target: + async with target.database.transaction() as connection: + stored = await target.repositories.sources.get( + connection, + scope.scope_id, + SourceRef(source_type=observation.source_type, source_id=observation.name), + ) + assert stored.value == observation + assert ( + await target.repositories.source_definitions.get(connection, manifest.name, manifest.version) + == manifest + ) + # Continuing ingestion must also work without registering the manifest again. + assert ( + await target.submit_source_observation( + SubmitSourceObservation(scope_id=scope.scope_id, observation=observation) + ) + ).sequence == 1 + + asyncio.run(scenario()) + + +def test_archive_snapshot_survives_concurrent_mysql_capture(tmp_path: Path, target_config: BuiltinConfig) -> None: + + if target_config.database.kind == "sqlite": + pytest.skip("SQLite snapshot has a separate concurrent-writer regression") + + async def scenario() -> None: + async with open_builtin_contexts(target_config) as source: + scope = await source.scopes.create( + ScopeDraft(title="Snapshot", summary="Concurrent writes", idempotency_key="snapshot") + ) + context = await source.get(scope.scope_id) + + await context.sources.capture(ContentCapture(source_id="before", content="before")) + async with source.database.transaction() as connection: + await connection.exec_driver_sql("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED") + captured = False + + async def write_concurrently(_driver): + # A separate task owns a separate transaction/connection. + await asyncio.create_task(context.sources.capture(ContentCapture(source_id="after", content="after"))) + + def after_read(connection, _cursor, statement, _parameters, _context, _many): + nonlocal captured + if not captured and statement.startswith("SELECT pc_source_journal_heads."): + captured = True + connection.connection.dbapi_connection.run_async(write_concurrently) + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + event.listen(source.database.engine.sync_engine, "after_cursor_execute", after_read) + try: + archive = tmp_path / "snapshot.pcb" + await source.portability.export([scope.scope_id], archive, authorize=authorize) + finally: + event.remove(source.database.engine.sync_engine, "after_cursor_execute", after_read) + assert captured + async with open_builtin_contexts(BuiltinConfig()) as target: + assert (await target.portability.validate(archive)).compatible + await target.portability.restore(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar( + select(SOURCE_JOURNAL_HEADS_TABLE.c.position).where( + SOURCE_JOURNAL_HEADS_TABLE.c.scope_id == scope.scope_id + ) + ) + == 1 + ) + assert ( + await connection.execute( + select(SOURCES_TABLE.c.source_id).where(SOURCES_TABLE.c.scope_id == scope.scope_id) + ) + ).scalars().all() == ["before"] + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("damage", ["missing_definition", "fingerprint", "identity", "native"]) +def test_restore_rejects_invalid_remote_source_dependencies_before_writing(tmp_path: Path, damage: str) -> None: + + async def scenario() -> None: + definition = AdapterSourceDefinition(NoteAdapter()) + registry = SourceDefinitionRegistry((definition,)) + observation = project_source_for_transport( + registry, await registry.resolve(NoteInput(note_id="one", body="remote")) + ) + archive = tmp_path / "invalid.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Remote", summary="Dependencies", idempotency_key="remote") + ) + await source.register_source_definition(manifest_for_definition(definition)) + await source.submit_source_observation( + SubmitSourceObservation(scope_id=scope.scope_id, observation=observation) + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + with zipfile.ZipFile(archive) as bundle: + manifest = json.loads(bundle.read("manifest.json")) + records = [json.loads(line) for line in bundle.read("records.ndjson").splitlines()] + if damage == "missing_definition": + records = [record for record in records if record["record_type"] != "source_definition"] + else: + record = next(record for record in records if record["record_type"] == "source") + envelope = json.loads(base64.b64decode(record["payload"]["payload"]["base64"])) + if damage == "fingerprint": + envelope["value"]["definition_fingerprint"] = "sha256:" + "0" * 64 + elif damage == "identity": + record["identity"]["source_id"] = "another" + else: + envelope["representation"] = "native" + envelope["value"] = envelope["value"]["payload"] + record["payload"]["payload"]["base64"] = base64.b64encode(json.dumps(envelope).encode()).decode() + for record in records: + record.pop("digest") + record["digest"] = ( + "sha256:" + + hashlib.sha256( + json.dumps(record, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + ) + manifest["record_count"] = len(records) + manifest["records_by_type"] = dict(Counter(record["record_type"] for record in records)) + manifest["total_digest"] = ( + "sha256:" + hashlib.sha256("\n".join(record["digest"] for record in records).encode()).hexdigest() + ) + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("manifest.json", json.dumps(manifest)) + bundle.writestr("records.ndjson", "\n".join(json.dumps(record) for record in records) + "\n") + async with open_builtin_contexts(BuiltinConfig()) as target: + if damage == "native": + validation = await target.portability.validate(archive) + assert not validation.compatible + assert validation.unsupported_source_types == ("note",) + else: + with pytest.raises(BundleFormatError): + await target.portability.validate(archive) + with pytest.raises(BundleFormatError): + await target.portability.restore(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar( + select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == scope.scope_id) + ) + is None + ) + + asyncio.run(scenario()) From e38614321d85623ddb86368eae0e93575f7bfe33 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Mon, 21 Sep 2026 20:42:12 +0800 Subject: [PATCH 23/32] test(archive): cover remote source roundtrips across backends --- tests/e2e/test_portable_archive.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index 7cc543c81d..452c0e800a 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -360,7 +360,7 @@ def target_config(tmp_path: Path, short_tmp_path: Path, request: pytest.FixtureR @pytest.mark.parametrize("legacy", [False, True]) @pytest.mark.parametrize("target_definition", ["absent", "identical", "conflicting"]) def test_archive_preserves_remote_observations_without_worker_adapter( - tmp_path: Path, legacy: bool, target_definition: str + tmp_path: Path, legacy: bool, target_definition: str, target_config: BuiltinConfig ) -> None: async def scenario() -> None: @@ -397,7 +397,6 @@ class UnrelatedAdapter(NoteAdapter): ) await source.portability.export([scope.scope_id], archive, authorize=authorize) assert (await source.portability.inspect(archive)).records_by_type["source_definition"] == 1 - target_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'remote.db'}")) async with open_builtin_contexts(target_config) as target: if target_definition == "identical": await target.register_source_definition(manifest) @@ -447,6 +446,17 @@ class DifferentNoteAdapter(NoteAdapter): SubmitSourceObservation(scope_id=scope.scope_id, observation=observation) ) ).sequence == 1 + roundtrip = tmp_path / "remote-roundtrip.pcb" + await target.portability.export([scope.scope_id], roundtrip, authorize=authorize) + async with open_builtin_contexts(BuiltinConfig()) as reverse: + assert (await reverse.portability.restore(roundtrip)).projections_ready + async with reverse.database.transaction() as connection: + restored = await reverse.repositories.sources.get( + connection, + scope.scope_id, + SourceRef(source_type=observation.source_type, source_id=observation.name), + ) + assert restored.value == observation asyncio.run(scenario()) From d269b7585c213204cd3d22b9184225a325cd6dbc Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Tue, 22 Sep 2026 02:31:28 +0800 Subject: [PATCH 24/32] refactor(archive): consolidate validation passes and family support --- .../builtin/portability/bundle.py | 140 +++++++----------- .../builtin/runtime/relational.py | 25 ++-- src/powercontext/cli/archive.py | 18 +-- 3 files changed, 64 insertions(+), 119 deletions(-) diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index 0d9cfceb9d..65e275afa0 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -43,7 +43,7 @@ from uuid import NAMESPACE_URL, UUID, uuid5 from pydantic import ValidationError -from sqlalchemy import DateTime, insert, or_, select, tuple_, update +from sqlalchemy import DateTime, insert, select, tuple_, update from sqlalchemy import inspect as sqlalchemy_inspect from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection @@ -414,7 +414,6 @@ async def export( count, records_by_type, digest = await self._stream_export( connection, selected, stream, progress=progress ) - await _validate_export_journal_snapshot(connection, selected) bundle_id = str(uuid5(NAMESPACE_URL, f"powercontext:portable:v{FORMAT_VERSION}:{digest}")) manifest = { "format_version": FORMAT_VERSION, @@ -464,17 +463,11 @@ async def validate( """Verify bundle integrity, dependencies, and optional adapter support.""" inspection = _parse_bundle(source, progress=progress, phase="validate") - _validate_dependencies(source) - required_sources = { - str(record.identity["source_type"]) - for record in _iter_records(source) - if record.record_type == "source" and _source_observation(record) is None - } + required_sources, required_families = _validate_dependencies(source) configured = ( self._supported_source_types if supported_source_types is None else frozenset(supported_source_types) ) missing_sources = () if configured is None else tuple(sorted(required_sources - configured)) - required_families = _required_artifact_families(_iter_records(source)) missing_families = ( () if self._supported_artifact_families is None @@ -845,29 +838,6 @@ async def _establish_export_snapshot(connection: AsyncConnection, /) -> None: await connection.exec_driver_sql("BEGIN") -async def _validate_export_journal_snapshot(connection: AsyncConnection, scopes: tuple[str, ...], /) -> None: - """Reject a snapshot whose Sources advance beyond its journal head.""" - - invalid = await connection.scalar( - select(SOURCES_TABLE.c.source_id) - .join( - SOURCE_JOURNAL_HEADS_TABLE, - SOURCE_JOURNAL_HEADS_TABLE.c.scope_id == SOURCES_TABLE.c.scope_id, - isouter=True, - ) - .where( - SOURCES_TABLE.c.scope_id.in_(scopes), - or_( - SOURCE_JOURNAL_HEADS_TABLE.c.position.is_(None), - SOURCES_TABLE.c.journal_position > SOURCE_JOURNAL_HEADS_TABLE.c.position, - ), - ) - .limit(1) - ) - if invalid is not None: - raise BundleFormatError("source journal snapshot is inconsistent") - - def _record_document(record: _Record) -> dict[str, Any]: return { "record_type": record.record_type, @@ -1383,19 +1353,23 @@ def _validate_safe_value(value: object, key: str | None = None) -> None: _validate_safe_value(child, key) -def _validate_dependencies(source: Path, /) -> None: - """Check references with a temporary on-disk identity index. - - The index deliberately stores only canonical identities, never record - payloads. This permits dependency validation for large bundles without - retaining their content in the importing process. - """ +def _validate_dependencies(source: Path, /) -> tuple[set[str], set[str]]: + """Validate dependencies with disk-backed identities and bounded metadata.""" + heads, packages = _dependency_metadata(source) + required_sources: set[str] = set() + required_families: set[str] = set() with _identity_index(source) as contains: for record in _iter_records(source): _validate_record_dependencies(record, contains) - _validate_skill_package_dependencies(source) - _validate_source_journal_records(source) + _validate_skill_package_dependency(record, packages) + if record.record_type == "source": + if int(record.payload["journal_position"]) > heads.get(str(record.identity["scope_id"]), -1): + raise BundleFormatError("source journal snapshot is inconsistent") + if _source_observation(record) is None: + required_sources.add(str(record.identity["source_type"])) + required_families.update(_required_artifact_families((record,))) + return required_sources, required_families def _validate_record_dependencies( # noqa: C901 - logical record kinds have distinct dependency edges. @@ -1605,22 +1579,12 @@ def _contains_memory_entry_target( ) -def _validate_source_journal_records(source: Path, /) -> None: - heads = { - str(record.identity["scope_id"]): int(record.payload["position"]) - for record in _iter_records(source) - if record.record_type == "source_journal_head" - } - for record in _iter_records(source): - if record.record_type == "source" and int(record.payload["journal_position"]) > heads.get( - str(record.identity["scope_id"]), -1 - ): - raise BundleFormatError("source journal snapshot is inconsistent") - - -def _validated_skill_packages(source: Path, /) -> dict[tuple[str, str], tuple[str, int, int, int]]: +def _dependency_metadata(source: Path, /) -> tuple[dict[str, int], dict[tuple[str, str], tuple[str, int, int, int]]]: + heads: dict[str, int] = {} packages: dict[tuple[str, str], tuple[str, int, int, int]] = {} for record in _iter_records(source): + if record.record_type == "source_journal_head": + heads[str(record.identity["scope_id"])] = int(record.payload["position"]) if record.record_type != "skill_package": continue try: @@ -1644,41 +1608,41 @@ def _validated_skill_packages(source: Path, /) -> dict[tuple[str, str], tuple[st snapshot.reference.uncompressed_size, snapshot.reference.archive_size, ) - return packages + return heads, packages -def _validate_skill_package_dependencies(source: Path, /) -> None: - packages = _validated_skill_packages(source) - for record in _iter_records(source): - if record.record_type == "artifact_revision" and record.identity["family"] == "skill": - content = _decoded_bytes_json(record.payload["content"], "Skill content") - elif record.record_type == "candidate_version" and record.payload["family"] == "skill": - content = _decoded_bytes_json(record.payload["proposal"], "Skill proposal") - else: - continue - package = content.get("package") if isinstance(content, dict) else None - if package is None: - continue - if not isinstance(package, dict): - raise BundleFormatError("Skill package reference is invalid") - typed_package = cast(dict[str, object], package) - tree_digest = typed_package.get("tree_digest") - archive_digest = typed_package.get("archive_digest") - sizes = ( - typed_package.get("file_count"), - typed_package.get("uncompressed_size"), - typed_package.get("archive_size"), - ) - if ( - not isinstance(tree_digest, str) - or not isinstance(archive_digest, str) - or any(isinstance(value, bool) or not isinstance(value, int) for value in sizes) - ): - raise BundleFormatError("Skill package reference is invalid") - expected = (archive_digest, cast(int, sizes[0]), cast(int, sizes[1]), cast(int, sizes[2])) - key = (str(record.identity["scope_id"]), tree_digest) - if packages.get(key) != expected: - raise BundleFormatError("Skill artifact references missing or mismatched package") +def _validate_skill_package_dependency( + record: _Record, packages: Mapping[tuple[str, str], tuple[str, int, int, int]], / +) -> None: + if record.record_type == "artifact_revision" and record.identity["family"] == "skill": + content = _decoded_bytes_json(record.payload["content"], "Skill content") + elif record.record_type == "candidate_version" and record.payload["family"] == "skill": + content = _decoded_bytes_json(record.payload["proposal"], "Skill proposal") + else: + return + package = content.get("package") if isinstance(content, dict) else None + if package is None: + return + if not isinstance(package, dict): + raise BundleFormatError("Skill package reference is invalid") + typed_package = cast(dict[str, object], package) + tree_digest = typed_package.get("tree_digest") + archive_digest = typed_package.get("archive_digest") + sizes = ( + typed_package.get("file_count"), + typed_package.get("uncompressed_size"), + typed_package.get("archive_size"), + ) + if ( + not isinstance(tree_digest, str) + or not isinstance(archive_digest, str) + or any(isinstance(value, bool) or not isinstance(value, int) for value in sizes) + ): + raise BundleFormatError("Skill package reference is invalid") + expected = (archive_digest, cast(int, sizes[0]), cast(int, sizes[1]), cast(int, sizes[2])) + key = (str(record.identity["scope_id"]), tree_digest) + if packages.get(key) != expected: + raise BundleFormatError("Skill artifact references missing or mismatched package") def _decoded_bytes_json(value: object, name: str, /) -> object: diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index be68270d46..c071af4144 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -258,6 +258,9 @@ def _artifact_identity(ref: ArtifactRef) -> tuple[str, str, int]: _MEMORY_COMMIT_ENTRY_VERSION_COUNT = "powercontext.memory.commit.entry_version_count" +BUILTIN_ARTIFACT_TYPES = (Handoff, Memory, Experience, Skill, Profile, Prompt, TopicMemory) + + @dataclass(frozen=True, slots=True) class _Repositories: """Repositories shared by every scoped context.""" @@ -527,28 +530,20 @@ def __init__( self.database = database self.scopes = ScopeApplication(database, cursor_secret=cursor_secret) self.source_registry = source_registry or BUILTIN_SOURCE_REGISTRY - self.portability = PortableBundleService( - database, - projection_rebuilder=self.rebuild_portable_projections, - supported_source_types=tuple(definition.name for definition in self.source_registry.definitions), - supported_artifact_families=( - Handoff.family, - Memory.family, - Experience.family, - Skill.family, - Profile.family, - Prompt.family, - TopicMemory.family, - ), - ) self.index = NoMemoryIndex() if index is None else index self.topic_memory_index = NoTopicMemoryIndex() if topic_memory_index is None else topic_memory_index self.experience_index = NoExperienceIndex() if experience_index is None else experience_index source_repository = SourceRepository(self.source_registry) artifact_repository = ArtifactRepository( - (Handoff, Memory, Experience, Skill, Profile, Prompt, TopicMemory), + BUILTIN_ARTIFACT_TYPES, sources=source_repository, ) + self.portability = PortableBundleService( + database, + projection_rebuilder=self.rebuild_portable_projections, + supported_source_types=tuple(definition.name for definition in self.source_registry.definitions), + supported_artifact_families=artifact_repository.families, + ) topic_memory_repository = TopicMemoryRepository(artifacts=artifact_repository, index=self.topic_memory_index) self.repositories = _Repositories( sources=source_repository, diff --git a/src/powercontext/cli/archive.py b/src/powercontext/cli/archive.py index 6b324119d8..54c60cbd7a 100644 --- a/src/powercontext/cli/archive.py +++ b/src/powercontext/cli/archive.py @@ -27,13 +27,6 @@ from sqlalchemy.engine import make_url from sqlalchemy.ext.asyncio import create_async_engine -from powercontext.builtin.artifacts.experience import Experience -from powercontext.builtin.artifacts.handoff import Handoff -from powercontext.builtin.artifacts.memory import Memory -from powercontext.builtin.artifacts.profile import Profile -from powercontext.builtin.artifacts.prompt import Prompt -from powercontext.builtin.artifacts.skill import Skill -from powercontext.builtin.artifacts.topic_memory import TopicMemory from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile @@ -48,6 +41,7 @@ PortableBundleService, ) from powercontext.builtin.runtime import BuiltinConfig, open_builtin_runtime +from powercontext.builtin.runtime.relational import BUILTIN_ARTIFACT_TYPES from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY from powercontext.paths import default_scheduler_path from powercontext.server.configuration import server_settings_context @@ -180,15 +174,7 @@ async def projection_capability(_scopes: tuple[str, ...], /) -> None: database, projection_rebuilder=projection_capability, supported_source_types=tuple(definition.name for definition in BUILTIN_SOURCE_REGISTRY.definitions), - supported_artifact_families=( - Handoff.family, - Memory.family, - Experience.family, - Skill.family, - Profile.family, - Prompt.family, - TopicMemory.family, - ), + supported_artifact_families=tuple(artifact.family for artifact in BUILTIN_ARTIFACT_TYPES), ) return await archive.validate(source, progress=_emit_progress) From fbdd194ab5f519cc6ac8c1f453343eede3208fec Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Tue, 22 Sep 2026 22:10:28 +0800 Subject: [PATCH 25/32] fix(archive): preserve frozen recurrence decisions and events --- docs/en/docs/operate/portable-archive.md | 4 + docs/zh/docs/operate/portable-archive.md | 4 + .../builtin/portability/bundle.py | 150 ++++++++++ tests/e2e/test_portable_archive.py | 267 +++++++++++++++++- 4 files changed, 424 insertions(+), 1 deletion(-) diff --git a/docs/en/docs/operate/portable-archive.md b/docs/en/docs/operate/portable-archive.md index 516e434358..04a6a9cc79 100644 --- a/docs/en/docs/operate/portable-archive.md +++ b/docs/en/docs/operate/portable-archive.md @@ -103,6 +103,10 @@ The archive has a versioned manifest (`format_version`, producer version, scopes checksum) plus NDJSON records. It is the authoritative round-trip format. CSV may be produced separately for bounded analysis, but it cannot preserve immutable revisions, lineage, or evidence references and must not be used for restore. +Frozen recurrence matches and events are preserved with their original idempotency keys, Source positions, and exact +Artifact revisions. Restore validates candidate revisions, Handoff/receipt references, and event-to-match dependencies. +Advancing an Experience head does not change this history; replay after restore reuses the saved decisions. + ## Recovery and conflicts Restore is idempotent. Replaying the same archive recognizes identical records as `already_present`. A different diff --git a/docs/zh/docs/operate/portable-archive.md b/docs/zh/docs/operate/portable-archive.md index 93de1292b0..88337f918e 100644 --- a/docs/zh/docs/operate/portable-archive.md +++ b/docs/zh/docs/operate/portable-archive.md @@ -93,6 +93,10 @@ powercontext archive restore ./backups/payments.pcb --yes 它是权威的 round-trip 格式。CSV 可以单独用于有边界的分析，但无法保留不可变 revision、lineage 或 evidence reference， 不得用于恢复。 +归档保留已冻结的 recurrence 匹配决策与事件，包括原始幂等键、Source 位置和精确 Artifact revision。 +恢复前会校验候选 revision、Handoff/receipt 引用及事件关联的匹配记录。Experience 当前版本推进后， +历史决策仍保持不变；恢复后的重放复用已保存的决策。 + ## 恢复和冲突处理 恢复是幂等的。重复回放同一归档时，内容相同的 record 会计入 `already_present`。已有不可变 identity 对应不同 payload diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index 65e275afa0..1b77e84cfe 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -48,6 +48,15 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection +from powercontext.builtin.artifacts.experience.recurrence import ( + RecurrenceMatch, + RecurrenceObservation, + canonical_digest, + match_key, + observation_identity, + selection_key, + verdict_key, +) from powercontext.builtin.artifacts.skill.package import SkillPackageError, capture_skill_archive from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.tables import ( @@ -63,6 +72,8 @@ MEMORY_ENTRY_VERSIONS_TABLE, PORTABLE_RESTORE_RECEIPTS_TABLE, PROFILE_POLICIES_TABLE, + RECURRENCE_MATCH_TABLE, + RECURRENCE_OBSERVATION_TABLE, SCOPE_CONTEXT_REFERENCES_TABLE, SCOPE_CREATION_REQUESTS_TABLE, SCOPE_EXTERNAL_REFERENCES_TABLE, @@ -134,6 +145,8 @@ "topic_memory_publication", "profile_policy", "artifact_tag", + "recurrence_match", + "recurrence_observation", ] ProjectionRebuilder = Callable[[tuple[str, ...]], Awaitable[None]] ExportAuthorizer = Callable[[tuple[str, ...]], Awaitable[None]] @@ -350,6 +363,44 @@ class _RecordSpec: ("scope_id", "family", "artifact_id", "target_type", "target_id", "tag_key"), ("tag_key_hash", "tag", "assigned_at"), ), + # Use ledger uniqueness identities so a changed decision is a conflict, + # even when its content-derived match key or observation ID also changes. + "recurrence_match": _RecordSpec( + RECURRENCE_MATCH_TABLE, + ("scope_id", "task_outcome_source_type", "task_outcome_source_id", "failure_item_kind", "failure_item_index"), + ( + "match_key", + "task_outcome_position", + "failure_item_digest", + "candidate_set_mode", + "candidate_set_digest", + "result", + "target_family", + "target_artifact_id", + "target_revision", + "signature_key", + "payload", + ), + ), + "recurrence_observation": _RecordSpec( + RECURRENCE_OBSERVATION_TABLE, + ("scope_id", "verdict_key"), + ( + "observation_id", + "selection_key", + "event", + "match_basis", + "family", + "artifact_id", + "revision", + "signature_key", + "signature_key_hash", + "task_outcome_source_type", + "task_outcome_source_id", + "task_outcome_position", + "payload", + ), + ), } # Parent rows always precede their dependent rows. This matters for SQLite as @@ -1159,6 +1210,9 @@ def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: elif kind == "source": bytes_values("payload") integers("journal_position", positive=True) + elif kind in {"recurrence_match", "recurrence_observation"}: + bytes_values("payload") + _recurrence_value(record) elif kind == "skill_package": strings("archive_digest") bytes_values("archive_bytes", "manifest") @@ -1272,6 +1326,54 @@ def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: raise BundleFormatError("artifact tag has invalid key hash") +def _recurrence_value(record: _Record, /) -> RecurrenceMatch | RecurrenceObservation: + """Check frozen payloads against their indexed fields and idempotency keys.""" + model = RecurrenceMatch if record.record_type == "recurrence_match" else RecurrenceObservation + try: + value = model.model_validate_json(cast(bytes, _database_value(record.payload["payload"])), strict=True) + except (ValidationError, ValueError, TypeError) as error: + raise BundleFormatError("recurrence payload is invalid") from error + expected = { + "scope_id": value.scope_id, + "task_outcome_source_type": value.task_outcome_ref.source_type, + "task_outcome_source_id": value.task_outcome_ref.source_id, + "task_outcome_position": value.task_outcome_position, + "signature_key": value.signature_key, + } + if isinstance(value, RecurrenceMatch): + ref = value.artifact_ref + expected.update({ + "match_key": match_key(value), + "failure_item_kind": value.failure_ref.item_kind, + "failure_item_index": value.failure_ref.item_index, + "failure_item_digest": value.failure_ref.item_digest, + "candidate_set_mode": value.candidate_set_mode, + "candidate_set_digest": value.candidate_set_digest, + "result": value.result, + "target_family": None if ref is None else ref.family, + "target_artifact_id": None if ref is None else ref.artifact_id, + "target_revision": None if ref is None else ref.revision, + }) + else: + if value.observation_id != canonical_digest(observation_identity(value)): + raise BundleFormatError("recurrence observation has an invalid identity") + expected.update({ + "observation_id": value.observation_id, + "selection_key": selection_key(value), + "verdict_key": verdict_key(value), + "event": value.event, + "match_basis": value.match_basis, + "family": value.artifact_ref.family, + "artifact_id": value.artifact_ref.artifact_id, + "revision": value.artifact_ref.revision, + "signature_key_hash": _json_value(hashlib.sha256(value.signature_key.encode()).digest()), + }) + actual = {**record.identity, **record.payload} + if any(type(actual[key]) is not type(field) or actual[key] != field for key, field in expected.items()): + raise BundleFormatError("recurrence payload does not match its indexed fields") + return value + + def _validate_datetime_text(value: object, /) -> None: try: datetime.fromisoformat(cast(str, value)) @@ -1407,6 +1509,32 @@ def _validate_record_dependencies( # noqa: C901 - logical record kinds have dis }, ): raise BundleFormatError("observation references missing or mismatched Source definition") + elif record.record_type in {"recurrence_match", "recurrence_observation"}: + value = _recurrence_value(record) + if not contains( + "source_position", + { + "scope_id": scope, + **value.task_outcome_ref.model_dump(), + "position": value.task_outcome_position, + }, + ): + raise BundleFormatError("recurrence references missing or mismatched Source") + if isinstance(value, RecurrenceMatch): + refs = value.candidate_refs + else: + refs = (value.artifact_ref,) if value.handoff_ref is None else (value.artifact_ref, value.handoff_ref) + if value.handoff_receipt_ref is not None and not contains( + "source", {"scope_id": scope, **value.handoff_receipt_ref.model_dump()} + ): + raise BundleFormatError("recurrence references missing receipt Source") + if value.recurrence_match_digest is not None and not contains( + "recurrence_match_digest", {"scope_id": scope, "match_key": value.recurrence_match_digest} + ): + raise BundleFormatError("recurrence references missing frozen match") + for ref in refs: + if not contains("artifact_revision", {"scope_id": scope, **ref.model_dump()}): + raise BundleFormatError("recurrence references missing exact Artifact revision") elif record.record_type == "artifact_revision" and identity["family"] == "topic-memory": if not contains("topic_memory_publication", identity): raise BundleFormatError("Topic Memory revision has no publication metadata") @@ -1673,6 +1801,28 @@ def _identity_index(source: Path, /) -> Iterator[Callable[[str, Mapping[str, obj ) except sqlite3.IntegrityError as error: raise BundleFormatError("bundle contains duplicate immutable identity") from error + if record.record_type == "source": + connection.execute( + "INSERT INTO identities (record_type, identity) VALUES (?, ?)", + ( + "source_position", + _identity_key({ + **record.identity, + "position": record.payload["journal_position"], + }), + ), + ) + if record.record_type == "recurrence_match": + connection.execute( + "INSERT INTO identities (record_type, identity) VALUES (?, ?)", + ( + "recurrence_match_digest", + _identity_key({ + "scope_id": record.identity["scope_id"], + "match_key": record.payload["match_key"], + }), + ), + ) if record.record_type == "source_definition": connection.execute( "INSERT INTO identities (record_type, identity) VALUES (?, ?)", diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index 452c0e800a..f531656b88 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -29,7 +29,7 @@ from uuid import uuid4 import pytest -from pydantic import SecretStr +from pydantic import JsonValue, SecretStr from sqlalchemy import create_engine, event, select, update from sqlalchemy.engine import make_url @@ -593,3 +593,268 @@ async def authorize(_scopes: tuple[str, ...]) -> None: ) asyncio.run(scenario()) + + +async def _replay_recurrence(contexts, scope_id): + from powercontext.builtin.evidence.resolver import EvidenceResolver + from powercontext.builtin.runtime.recurrence import RelationalRecurrenceLedger + + async def no_memory(*_args): + raise AssertionError("fixture has no Memory citations") # noqa: TRY003 + + repositories = contexts.repositories + ledger = RelationalRecurrenceLedger( + database=contexts.database, + scope_id=scope_id, + sources=repositories.sources, + artifacts=repositories.artifacts, + recurrence=repositories.recurrence, + evidence=EvidenceResolver( + scope_id=scope_id, + sources=repositories.sources, + artifacts=repositories.artifacts, + memory_reader=no_memory, + ), + ) + async with contexts.database.transaction() as connection: + position = await repositories.sources.journal_position(connection, scope_id) + rows = await repositories.sources.list_window(connection, scope_id, after=0, through=position) + await ledger.record_window(connection, rows) + return ( + await repositories.recurrence.matches(connection, scope_id), + await repositories.recurrence.observations(connection, scope_id), + ) + + +async def _recurrence_archive(tmp_path): + from powercontext.builtin.artifacts.handoff.models import HandoffArtifactCitation + from powercontext.builtin.work.models import HandoffReceipt, TaskCheck, TaskOutcome, WorkClaim + + archive = tmp_path / "recurrence.pcb" + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Recurrence", summary="Frozen decisions", idempotency_key="recurrence") + ) + sid = scope.scope_id + content: dict[str, JsonValue] = { + "situation": "Contract changed", + "action": "Regenerate client", + "outcome": "Client agrees", + "lesson": "Regenerate after changing the contract", + "failure": { + "signature": {"recall_cue": "client is stale", "symptom": "client diverges"}, + "repair_surface": "experience_content", + "verification": {"condition": "contract changed", "check_subject": "client agrees"}, + }, + } + experience = await source.records.create_artifact(sid, "experience", ArtifactWrite(content=content)) + ref = ArtifactRef(family="experience", artifact_id=experience.artifact_id, revision=1) + citation = HandoffArtifactCitation(artifact_ref=ref) + handoff = await source.records.create_artifact( + sid, + "handoff", + ArtifactWrite( + content={ + "objective": "Regenerate client", + "disposition": "continuable", + "state": [{"text": "Use this experience", "citations": [citation.model_dump(mode="json")]}], + } + ), + ) + context = await source.get(sid) + receipt = HandoffReceipt( + receiver="archive-test", + status="accepted", + selection="exact", + evidence_status="available", + selected_revision=ArtifactRef(family="handoff", artifact_id=handoff.artifact_id, revision=1), + ) + await context.sources.capture( + ContentCapture( + source_id="receipt", + content=receipt.model_dump_json(), + metadata={"kind": "handoff-receipt"}, + ) + ) + receipt_ref = SourceRef(source_type="content", source_id="receipt") + for name, linked in (("linked-failure", True), ("unlinked-failure", False)): + outcome = TaskOutcome( + objective="Regenerate client", + summary="Failed", + status="failed", + observations=(WorkClaim(text="Client regeneration failed", basis="declared"),), + handoff_receipt_ref=receipt_ref if linked else None, + checks=(TaskCheck(name="client is stale", status="failed", basis="verified", evidence=(citation,)),), + ) + await context.sources.capture( + ContentCapture( + source_id=name, + content=outcome.model_dump_json(), + metadata={"kind": "task-outcome"}, + ) + ) + success = TaskOutcome( + objective="Regenerate client", + summary="Passed", + status="succeeded", + handoff_receipt_ref=receipt_ref, + observations=(WorkClaim(text="contract changed", basis="verified", evidence=(citation,)),), + checks=(TaskCheck(name="client agrees", status="passed", basis="verified", evidence=(citation,)),), + ) + await context.sources.capture( + ContentCapture( + source_id="success", + content=success.model_dump_json(), + metadata={"kind": "task-outcome"}, + ) + ) + expected = await _replay_recurrence(source, sid) + assert len(expected[0]) == 2 + assert len(expected[1]) == 5 + assert {event.event for event in expected[1]} == {"selected", "recurred", "avoided"} + assert all(match.artifact_ref == ref for match in expected[0]) + # Remove the signature from the current head. Recomputing an unlinked + # failure against this head would lose the original revision-1 match. + content["failure"] = None + await source.records.replace_artifact( + sid, + "experience", + experience.artifact_id, + '"revision:1"', + ArtifactWrite(content=content), + ) + assert await _replay_recurrence(source, sid) == expected + + async def authorize(_scopes): + pass + + await source.portability.export([sid], archive, authorize=authorize) + return archive, sid, experience.artifact_id, expected + + +def test_archive_preserves_frozen_recurrence_after_restart(tmp_path: Path, target_config: BuiltinConfig) -> None: + async def scenario(): + archive, sid, artifact_id, expected = await _recurrence_archive(tmp_path) + async with open_builtin_contexts(target_config) as target: + assert (await target.portability.validate(archive)).compatible + assert (await target.portability.restore(archive)).projections_ready + assert (await target.portability.restore(archive)).inserted == 0 + async with open_builtin_contexts(target_config) as target: + assert (await target.records.get_artifact(sid, "experience", artifact_id)).revision == 2 + async with target.database.transaction() as connection: + assert await target.repositories.recurrence.matches(connection, sid) == expected[0] + assert await target.repositories.recurrence.observations(connection, sid) == expected[1] + assert await _replay_recurrence(target, sid) == expected + assert await _replay_recurrence(target, sid) == expected + + async def authorize(_scopes): + pass + + roundtrip = tmp_path / "recurrence-roundtrip.pcb" + await target.portability.export([sid], roundtrip, authorize=authorize) + async with open_builtin_contexts(BuiltinConfig()) as reverse: + await reverse.portability.restore(roundtrip) + assert await _replay_recurrence(reverse, sid) == expected + + asyncio.run(scenario()) + + +def _rewrite_archive_records(archive, transform): + with zipfile.ZipFile(archive) as bundle: + manifest = json.loads(bundle.read("manifest.json")) + records = [json.loads(line) for line in bundle.read("records.ndjson").splitlines()] + records = transform(records) + for record in records: + record.pop("digest", None) + record["digest"] = ( + "sha256:" + + hashlib.sha256( + json.dumps(record, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + ) + manifest["record_count"] = len(records) + manifest["records_by_type"] = dict(Counter(record["record_type"] for record in records)) + manifest["total_digest"] = ( + "sha256:" + hashlib.sha256("\n".join(record["digest"] for record in records).encode()).hexdigest() + ) + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("manifest.json", json.dumps(manifest)) + bundle.writestr("records.ndjson", "\n".join(json.dumps(record) for record in records) + "\n") + + +@pytest.mark.parametrize("damage", ["outcome", "receipt", "revision", "handoff", "match", "payload", "key", "position"]) +def test_archive_rejects_broken_recurrence_before_writing(tmp_path: Path, damage: str) -> None: + async def scenario(): + archive, sid, _, _ = await _recurrence_archive(tmp_path) + + def damage_records(records): + if damage in {"outcome", "receipt"}: + source_id = "unlinked-failure" if damage == "outcome" else "receipt" + return [ + r for r in records if not (r["record_type"] == "source" and r["identity"]["source_id"] == source_id) + ] + if damage in {"revision", "handoff"}: + family = "experience" if damage == "revision" else "handoff" + return [ + r + for r in records + if not ( + r["record_type"] == "artifact_revision" + and r["identity"]["family"] == family + and r["identity"]["revision"] == 1 + ) + ] + if damage == "match": + return [r for r in records if r["record_type"] != "recurrence_match"] + if damage == "position": + record = next( + r + for r in records + if r["record_type"] == "source" and r["identity"]["source_id"] == "unlinked-failure" + ) + record["payload"]["journal_position"] = 1 + return records + record = next(r for r in records if r["record_type"] == "recurrence_observation") + if damage == "payload": + record["payload"]["payload"] = {"base64": base64.b64encode(b"{}").decode()} + else: + record["payload"]["selection_key"] = "sha256:" + "0" * 64 + return records + + _rewrite_archive_records(archive, damage_records) + async with open_builtin_contexts(BuiltinConfig()) as target: + with pytest.raises(BundleFormatError): + await target.portability.validate(archive) + with pytest.raises(BundleFormatError): + await target.portability.restore(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar(select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == sid)) + is None + ) + + asyncio.run(scenario()) + + +def test_archive_rejects_recomputed_recurrence_without_overwriting_history(tmp_path: Path) -> None: + async def scenario(): + archive, sid, _, expected = await _recurrence_archive(tmp_path) + without_history = tmp_path / "without-history.pcb" + without_history.write_bytes(archive.read_bytes()) + _rewrite_archive_records( + without_history, + lambda records: [r for r in records if not r["record_type"].startswith("recurrence_")], + ) + async with open_builtin_contexts(BuiltinConfig()) as target: + await target.portability.restore(without_history) + recomputed = await _replay_recurrence(target, sid) + assert recomputed != expected + assert any(match.result == "unmatched" for match in recomputed[0]) + validation = await target.portability.validate(archive) + assert not validation.compatible + assert validation.conflicts > 0 + with pytest.raises(BundleConflictError): + await target.portability.restore(archive) + assert await _replay_recurrence(target, sid) == recomputed + + asyncio.run(scenario()) From f94213193e51f2202403896904d0f646e9f52881 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Tue, 29 Sep 2026 19:32:53 +0800 Subject: [PATCH 26/32] fix(archive): validate content and identities before restore --- .../builtin/persistence/artifacts.py | 12 +- .../builtin/portability/bundle.py | 73 +++++++-- .../builtin/runtime/relational.py | 1 + src/powercontext/cli/archive.py | 5 +- tests/builtin/portability/test_bundle.py | 27 ++-- tests/e2e/test_portable_archive.py | 149 ++++++++++++++++++ 6 files changed, 232 insertions(+), 35 deletions(-) diff --git a/src/powercontext/builtin/persistence/artifacts.py b/src/powercontext/builtin/persistence/artifacts.py index 5c018d765a..14f1e6995d 100644 --- a/src/powercontext/builtin/persistence/artifacts.py +++ b/src/powercontext/builtin/persistence/artifacts.py @@ -93,6 +93,12 @@ def families(self) -> frozenset[str]: return frozenset(self._by_family) + def decode_content(self, family: str, payload: bytes, /) -> BaseModel: + """Decode canonical content using the registered family's storage contract.""" + + self._artifact_type(family) + return load_model(self._content_types[family], payload, kind="artifact", name=family) + async def create( self, connection: AsyncConnection, @@ -455,11 +461,9 @@ def _decode_artifact( family = str(row["family"]) artifact_type = self._artifact_type(family) lineage = lineage.model_copy(update={"memory_citations": load_memory_citations(row.get("memory_citations"))}) - content = load_model( - self._content_types[family], + content = self.decode_content( + family, stored_bytes(row["content"], column="payload"), - kind="artifact", - name=family, ) ref = ArtifactRef( family=family, diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index 1b77e84cfe..a86590f665 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -43,7 +43,7 @@ from uuid import NAMESPACE_URL, UUID, uuid5 from pydantic import ValidationError -from sqlalchemy import DateTime, insert, select, tuple_, update +from sqlalchemy import DateTime, PrimaryKeyConstraint, UniqueConstraint, and_, insert, or_, select, tuple_, update from sqlalchemy import inspect as sqlalchemy_inspect from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection @@ -59,6 +59,7 @@ ) from powercontext.builtin.artifacts.skill.package import SkillPackageError, capture_skill_archive from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.errors import InvalidStoredPayloadError from powercontext.builtin.persistence.tables import ( ARTIFACT_CANDIDATE_HEADS_TABLE, ARTIFACT_CANDIDATE_VERSIONS_TABLE, @@ -424,9 +425,11 @@ def __init__( projection_rebuilder: ProjectionRebuilder | None = None, supported_source_types: Iterable[str] | None = None, supported_artifact_families: Iterable[str] | None = None, + artifact_content_decoder: Callable[[str, bytes], object] | None = None, ) -> None: self._database = database self._projection_rebuilder = projection_rebuilder + self._artifact_content_decoder = artifact_content_decoder self._supported_source_types = None if supported_source_types is None else frozenset(supported_source_types) self._supported_artifact_families = ( None if supported_artifact_families is None else frozenset(supported_artifact_families) @@ -685,23 +688,41 @@ async def _target_compatibility( } ) for processed, record in enumerate(_iter_records(source), start=1): + self._validate_artifact_content(record) table = _SPECS[record.record_type].table if table.name not in existing_tables: _report_progress(progress, "target", processed, total) continue - where = [table.c[key] == _database_value(value, table.c[key]) for key, value in record.identity.items()] - row = (await connection.execute(select(table).where(*where))).mappings().one_or_none() - if row is None: - _report_progress(progress, "target", processed, total) - continue - present = _record_from_row(record.record_type, cast(Mapping[str, Any], dict(row))) - if present.digest == record.digest: - already_present += 1 - else: - conflicts += 1 + keys = (record.identity, *_unique_identities(record)) + where = or_( + *( + and_(*(table.c[key] == _database_value(value, table.c[key]) for key, value in identity.items())) + for identity in keys + ) + ) + rows = (await connection.execute(select(table).where(where))).mappings().all() + if rows: + if all( + _record_from_row(record.record_type, cast(Mapping[str, Any], row)).digest == record.digest + for row in rows + ): + already_present += 1 + else: + conflicts += 1 _report_progress(progress, "target", processed, total) return already_present, conflicts + def _validate_artifact_content(self, record: _Record, /) -> None: + if record.record_type != "artifact_revision" or self._artifact_content_decoder is None: + return + family = str(record.identity["family"]) + if self._supported_artifact_families is not None and family not in self._supported_artifact_families: + return + try: + self._artifact_content_decoder(family, cast(bytes, _database_value(record.payload["content"]))) + except InvalidStoredPayloadError as error: + raise BundleFormatError("Artifact content does not match the target family decoder") from error + async def _write_records( self, connection: AsyncConnection, @@ -1140,7 +1161,7 @@ def _parse_record(value: object) -> _Record: if ( set(identity) != set(spec.identity) or set(payload) != set(spec.payload) - or any(isinstance(item, bool) or not isinstance(item, str | int) for item in identity.values()) + or any(type(item) is not spec.table.c[field].type.python_type for field, item in identity.items()) or not _valid_digest(digest) ): raise BundleFormatError("record has invalid identity or digest") @@ -1783,6 +1804,27 @@ def _decoded_bytes_json(value: object, name: str, /) -> object: raise BundleFormatError(f"{name} is not valid JSON") from error +def _unique_identities(record: _Record, /) -> Iterator[dict[str, Any]]: + """Yield physical uniqueness keys not already covered by the logical identity.""" + + spec = _SPECS[record.record_type] + values = {**record.identity, **record.payload} + constraints = ( + constraint.columns + for constraint in spec.table.constraints + if isinstance(constraint, PrimaryKeyConstraint | UniqueConstraint) + ) + indexes = (index.columns for index in spec.table.indexes if index.unique) + for columns in (*constraints, *indexes): + fields = tuple(column.name for column in columns) + if set(spec.identity).issubset(fields): + continue + identity = {field: values[field] for field in fields} + # SQL unique keys permit multiple rows when a participating value is NULL. + if all(value is not None for value in identity.values()): + yield identity + + @contextmanager def _identity_index(source: Path, /) -> Iterator[Callable[[str, Mapping[str, object]], bool]]: index_path = _temporary_path(Path(tempfile.gettempdir()), suffix=".identities.sqlite3") @@ -1799,8 +1841,13 @@ def _identity_index(source: Path, /) -> Iterator[Callable[[str, Mapping[str, obj "INSERT INTO identities (record_type, identity) VALUES (?, ?)", (record.record_type, _identity_key(record.identity)), ) + for identity in _unique_identities(record): + connection.execute( + "INSERT INTO identities (record_type, identity) VALUES (?, ?)", + (record.record_type + ":unique", _identity_key(identity)), + ) except sqlite3.IntegrityError as error: - raise BundleFormatError("bundle contains duplicate immutable identity") from error + raise BundleFormatError("bundle contains duplicate immutable or unique identity") from error if record.record_type == "source": connection.execute( "INSERT INTO identities (record_type, identity) VALUES (?, ?)", diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 8b10618378..654ca70827 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -568,6 +568,7 @@ def __init__( projection_rebuilder=self.rebuild_portable_projections, supported_source_types=tuple(definition.name for definition in self.source_registry.definitions), supported_artifact_families=artifact_repository.families, + artifact_content_decoder=artifact_repository.decode_content, ) topic_memory_repository = TopicMemoryRepository(artifacts=artifact_repository, index=self.topic_memory_index) self.repositories = _Repositories( diff --git a/src/powercontext/cli/archive.py b/src/powercontext/cli/archive.py index 54c60cbd7a..3faf14129e 100644 --- a/src/powercontext/cli/archive.py +++ b/src/powercontext/cli/archive.py @@ -27,6 +27,7 @@ from sqlalchemy.engine import make_url from sqlalchemy.ext.asyncio import create_async_engine +from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile @@ -170,11 +171,13 @@ async def _validate_with_database(database: AsyncDatabase, source: Path, /) -> B async def projection_capability(_scopes: tuple[str, ...], /) -> None: return None + artifacts = ArtifactRepository(BUILTIN_ARTIFACT_TYPES) archive = PortableBundleService( database, projection_rebuilder=projection_capability, supported_source_types=tuple(definition.name for definition in BUILTIN_SOURCE_REGISTRY.definitions), - supported_artifact_families=tuple(artifact.family for artifact in BUILTIN_ARTIFACT_TYPES), + supported_artifact_families=artifacts.families, + artifact_content_decoder=artifacts.decode_content, ) return await archive.validate(source, progress=_emit_progress) diff --git a/tests/builtin/portability/test_bundle.py b/tests/builtin/portability/test_bundle.py index 0c5bcf2761..1b0c82fa3c 100644 --- a/tests/builtin/portability/test_bundle.py +++ b/tests/builtin/portability/test_bundle.py @@ -27,7 +27,7 @@ from typing import cast import pytest -from sqlalchemy import delete, insert, select +from sqlalchemy import insert, select from powercontext.builtin.artifacts.skill import capture_skill_directory from powercontext.builtin.persistence.database import AsyncDatabase @@ -350,31 +350,24 @@ async def scenario() -> None: SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES ) as target: async with target.database.transaction() as connection: - await _insert_scope(connection) - await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) - await connection.execute( - insert(SOURCES_TABLE).values( - scope_id="project:one", - source_type="content", - source_id="target-only", - payload=b"existing", - journal_position=1, - ) + await connection.exec_driver_sql( + "CREATE TRIGGER reject_restore BEFORE INSERT ON pc_sources " + "BEGIN SELECT RAISE(ABORT, 'write rejected after preflight'); END" ) service = PortableBundleService(target.database) validation = await service.validate(archive) - assert validation.conflicts == 0 + assert validation.compatible with pytest.raises(BundleConflictError, match="target constraint conflict"): await service.restore(archive) async with target.database.transaction() as connection: - source_ids = tuple((await connection.execute(select(SOURCES_TABLE.c.source_id))).scalars()) - assert source_ids == ("target-only",) - await connection.execute(delete(SOURCES_TABLE)) + for table in (SCOPES_TABLE, SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE): + assert (await connection.execute(select(table))).all() == [] + await connection.exec_driver_sql("DROP TRIGGER reject_restore") retried = await service.restore(archive) - assert retried.inserted == 1 - assert retried.already_present == 2 + assert retried.inserted == 3 + assert retried.already_present == 0 asyncio.run(scenario()) diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index f531656b88..5d7aa60a9a 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -858,3 +858,152 @@ async def scenario(): assert await _replay_recurrence(target, sid) == recomputed asyncio.run(scenario()) + + +async def _validation_archive(tmp_path: Path) -> tuple[Path, str]: + archive = tmp_path / "validation.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Validation", summary="Recovery", idempotency_key="validation") + ) + context = await source.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="source-a", content="Portable source.")) + await context.sources.capture(ContentCapture(source_id="source-b", content="Another portable source.")) + await source.records.create_artifact( + scope.scope_id, + "prompt", + ArtifactWrite( + prompt_key="memory.extract", + content={ + "schema_version": "powercontext.prompt.v1", + "mode": "auto", + "instructions": "", + "demonstrations": [], + }, + ), + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + return archive, scope.scope_id + + +def test_archive_rejects_invalid_artifact_content_before_writing(tmp_path: Path, monkeypatch) -> None: + from typer.testing import CliRunner + + from powercontext.cli.app import create_cli + from powercontext.cli.archive import archive_app + + async def scenario() -> Path: + archive, sid = await _validation_archive(tmp_path) + + def damage(records): + record = next(r for r in records if r["record_type"] == "artifact_revision") + record["payload"]["content"] = {"base64": base64.b64encode(b"{}").decode()} + return records + + _rewrite_archive_records(archive, damage) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="Artifact content"): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar(select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == sid)) + is None + ) + return archive + + archive = asyncio.run(scenario()) + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "unused-target")) + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "restore", str(archive), "--dry-run"]) + assert result.exit_code == 2, result.output + assert "Artifact content" in result.output + assert not (tmp_path / "unused-target" / "powercontext.db").exists() + + +@pytest.mark.parametrize( + ("kind", "field", "value"), + [("source", "source_id", 123), ("source", "source_id", True), ("artifact_revision", "revision", "1")], +) +def test_archive_rejects_incorrect_identity_types(tmp_path: Path, kind: str, field: str, value: object) -> None: + async def scenario() -> None: + archive, sid = await _validation_archive(tmp_path) + + def damage(records): + record = next(r for r in records if r["record_type"] == kind) + record["identity"][field] = value + return records + + _rewrite_archive_records(archive, damage) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="identity"): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar(select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == sid)) + is None + ) + + asyncio.run(scenario()) + + +def test_archive_dry_run_detects_source_journal_collision(tmp_path: Path, target_config: BuiltinConfig) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + empty = tmp_path / "empty.pcb" + incoming = tmp_path / "incoming.pcb" + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Collision", summary="Journal", idempotency_key="collision") + ) + await source.portability.export([scope.scope_id], empty, authorize=authorize) + context = await source.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="source-a", content="Incoming source.")) + await source.portability.export([scope.scope_id], incoming, authorize=authorize) + async with open_builtin_contexts(target_config) as target: + await target.portability.restore(empty) + context = await target.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="source-b", content="Existing source.")) + validation = await target.portability.validate(incoming) + assert not validation.compatible + assert validation.conflicts == 1 + with pytest.raises(BundleConflictError): + await target.portability.restore(incoming) + async with target.database.transaction() as connection: + assert ( + await connection.scalars( + select(SOURCES_TABLE.c.source_id).where(SOURCES_TABLE.c.scope_id == scope.scope_id) + ) + ).all() == ["source-b"] + + asyncio.run(scenario()) + + +def test_archive_rejects_duplicate_journal_positions_before_writing(tmp_path: Path) -> None: + async def scenario() -> None: + archive, sid = await _validation_archive(tmp_path) + + def damage(records): + for record in records: + if record["record_type"] == "source": + record["payload"]["journal_position"] = 1 + return records + + _rewrite_archive_records(archive, damage) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="unique identity"): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar(select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == sid)) + is None + ) + + asyncio.run(scenario()) From d3c7486c784ee076bf7ef0e03ccfb978e5dbd55d Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Sun, 11 Oct 2026 20:52:45 +0800 Subject: [PATCH 27/32] fix(archive): align portable restore with atomic memory --- e2e/bub/uv.lock | 4 ++ .../builtin/persistence/artifacts.py | 1 - .../builtin/portability/bundle.py | 60 +------------------ .../builtin/runtime/relational.py | 52 +++++++++------- tests/e2e/test_portable_archive.py | 42 ++++++------- uv.lock | 18 ++++-- 6 files changed, 66 insertions(+), 111 deletions(-) diff --git a/e2e/bub/uv.lock b/e2e/bub/uv.lock index 98aa844f41..549201876e 100644 --- a/e2e/bub/uv.lock +++ b/e2e/bub/uv.lock @@ -1622,6 +1622,7 @@ requires-dist = [ { name = "aiosqlite", marker = "extra == 'seekdb'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'server'", specifier = ">=0.22,<1" }, { name = "alembic", marker = "extra == 'builtin'", specifier = ">=1.16,<2" }, + { name = "alembic", marker = "extra == 'cli'", specifier = ">=1.16,<2" }, { name = "alembic", marker = "extra == 'seekdb'", specifier = ">=1.16,<2" }, { name = "alembic", marker = "extra == 'server'", specifier = ">=1.16,<2" }, { name = "apscheduler", marker = "extra == 'builtin'", specifier = ">=3.11,<4" }, @@ -1631,6 +1632,7 @@ requires-dist = [ { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115,<1" }, { name = "fastmcp", marker = "extra == 'server'", specifier = ">=3.4,<4" }, { name = "filelock", marker = "extra == 'builtin'", specifier = ">=3.16,<4" }, + { name = "filelock", marker = "extra == 'cli'", specifier = ">=3.16,<4" }, { name = "filelock", marker = "extra == 'seekdb'", specifier = ">=3.16,<4" }, { name = "filelock", marker = "extra == 'server'", specifier = ">=3.16,<4" }, { name = "httpx", marker = "extra == 'server'", specifier = ">=0.28,<1" }, @@ -1670,6 +1672,7 @@ requires-dist = [ { name = "pydantic-settings", marker = "extra == 'server'", specifier = ">=2.7,<3" }, { name = "pylibseekdb", marker = "(sys_platform == 'darwin' and extra == 'seekdb') or (sys_platform == 'linux' and extra == 'seekdb')", specifier = ">=1.4.0.post1,<2" }, { name = "pymysql", marker = "extra == 'builtin'", specifier = ">=1.2,<1.2.1" }, + { name = "pymysql", marker = "extra == 'cli'", specifier = ">=1.2,<1.2.1" }, { name = "pymysql", marker = "extra == 'seekdb'", specifier = ">=1.2,<1.2.1" }, { name = "pymysql", marker = "extra == 'server'", specifier = ">=1.2,<1.2.1" }, { name = "pyobvector", marker = "extra == 'builtin'", specifier = ">=0.2.28,<0.3" }, @@ -1699,6 +1702,7 @@ requires-dist = [ { name = "typer", marker = "extra == 'cli'", specifier = ">=0.16,<1" }, { name = "typing-extensions", specifier = ">=4.12,<5" }, { name = "tzdata", marker = "extra == 'builtin'", specifier = ">=2026.3" }, + { name = "tzdata", marker = "extra == 'cli'", specifier = ">=2026.3" }, { name = "tzdata", marker = "extra == 'seekdb'", specifier = ">=2026.3" }, { name = "tzdata", marker = "extra == 'server'", specifier = ">=2026.3" }, { name = "uvicorn", marker = "extra == 'server'", specifier = ">=0.34,<1" }, diff --git a/src/powercontext/builtin/persistence/artifacts.py b/src/powercontext/builtin/persistence/artifacts.py index 8b1c02a1bf..0d05bc7864 100644 --- a/src/powercontext/builtin/persistence/artifacts.py +++ b/src/powercontext/builtin/persistence/artifacts.py @@ -653,7 +653,6 @@ def _decode_artifact( ) -> Artifact[Any]: family = str(row["family"]) artifact_type = self._artifact_type(family) - lineage = lineage.model_copy(update={"memory_citations": load_memory_citations(row.get("memory_citations"))}) content = self.decode_content( family, stored_bytes(row["content"], column="payload"), diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index a86590f665..4b33f88c2c 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -270,7 +270,7 @@ class _RecordSpec: "artifact_revision": _RecordSpec( ARTIFACTS_TABLE, ("scope_id", "family", "artifact_id", "revision"), - ("content", "memory_citations"), + ("content",), ), "artifact_lineage_source": _RecordSpec( ARTIFACT_LINEAGE_SOURCES_TABLE, @@ -329,7 +329,6 @@ class _RecordSpec: "proposal", "source_refs", "artifact_refs", - "memory_citations", "target_family", "target_artifact_id", "target_revision", @@ -1244,7 +1243,7 @@ def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: ): raise BundleFormatError("Skill package has invalid digest") elif kind == "artifact_revision": - bytes_values("content", "memory_citations", nullable=("memory_citations",)) + bytes_values("content") elif kind == "artifact_lineage_source": strings("source_type", "source_id") elif kind == "artifact_lineage_artifact": @@ -1284,7 +1283,7 @@ def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: "reason", nullable=("target_family", "target_artifact_id", "reason"), ) - bytes_values("proposal", "source_refs", "artifact_refs", "memory_citations", nullable=("memory_citations",)) + bytes_values("proposal", "source_refs", "artifact_refs") target = (payload["target_family"], payload["target_artifact_id"], payload["target_revision"]) if target != (None, None, None) and ( target[0] is None @@ -1710,7 +1709,6 @@ def _validate_record_dependencies( # noqa: C901 - logical record kinds have dis ): raise BundleFormatError("candidate head references missing result revision") _validate_embedded_references(record, contains) - _validate_memory_citations(record, contains) def _contains_memory_entry_target( @@ -1963,58 +1961,6 @@ def _reference_items(value: object, kind: Literal["source", "artifact"], /) -> t return tuple(references) -def _validate_memory_citations( - record: _Record, - contains: Callable[[str, Mapping[str, object]], bool], - /, -) -> None: - if record.record_type not in {"artifact_revision", "candidate_version"}: - return - encoded_value = record.payload["memory_citations"] - if encoded_value is None: - return - encoded = cast(dict[str, object], encoded_value).get("base64") if isinstance(encoded_value, dict) else None - if not isinstance(encoded, str): - raise BundleFormatError("memory citations are not encoded bytes") - try: - citations = json.loads(base64.b64decode(encoded, validate=True), parse_constant=_reject_json_constant) - except (ValueError, json.JSONDecodeError) as error: - raise BundleFormatError("memory citations are not valid JSON") from error - if not isinstance(citations, list): - raise BundleFormatError("memory citations must be an array") - scope_id = record.identity["scope_id"] - for citation in citations: - if not isinstance(citation, dict) or set(citation) != {"memory_ref", "entry_id", "entry_version_id"}: - raise BundleFormatError("memory citation has invalid fields") - memory_ref = citation["memory_ref"] - if ( - not isinstance(memory_ref, dict) - or set(memory_ref) != {"family", "artifact_id", "revision"} - or not isinstance(citation["entry_id"], str) - or not isinstance(citation["entry_version_id"], str) - ): - raise BundleFormatError("memory citation has invalid fields") - artifact_exists = contains( - "artifact_revision", - { - "scope_id": scope_id, - "family": memory_ref["family"], - "artifact_id": memory_ref["artifact_id"], - "revision": memory_ref["revision"], - }, - ) - version_exists = contains( - "memory_entry_version", - { - "scope_id": scope_id, - "memory_artifact_id": memory_ref["artifact_id"], - "entry_version_id": citation["entry_version_id"], - }, - ) - if not artifact_exists or not version_exists: - raise BundleFormatError("memory citation references missing version") - - def _required_artifact_families(records: Iterable[_Record]) -> set[str]: families: set[str] = set() for record in records: diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 95416f7715..c186f203ea 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -33,7 +33,7 @@ from jsonschema.protocols import Validator from referencing import Registry from referencing.exceptions import Unresolvable -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.artifacts import ArtifactRef @@ -267,7 +267,7 @@ def _artifact_identity(ref: ArtifactRef) -> tuple[str, str, int]: _MEMORY_COMMIT_ENTRY_VERSION_COUNT = "powercontext.memory.commit.entry_version_count" -BUILTIN_ARTIFACT_TYPES = (Handoff, Memory, AtomicMemory, Experience, Skill, Profile, Prompt, TopicMemory) +BUILTIN_ARTIFACT_TYPES = (Handoff, AtomicMemory, Experience, Skill, Profile, Prompt, TopicMemory) @dataclass(frozen=True, slots=True) @@ -1378,29 +1378,35 @@ async def rebuild_portable_projections(self, scope_ids: tuple[str, ...], /) -> N """Rebuild target-local search projections after a logical restore.""" for scope_id in scope_ids: - services = self._services_for(scope_id) - _, catalog = services.sources() - await services.memory(catalog).rebuild_projections(self._embedding_model) async with self.database.transaction() as connection: - memory_refs = tuple( - ArtifactRef(family=Memory.family, artifact_id=str(artifact_id), revision=int(revision)) - for artifact_id, revision in ( - await connection.execute( - select(ARTIFACT_HEADS_TABLE.c.artifact_id, ARTIFACT_HEADS_TABLE.c.revision) - .where( - ARTIFACT_HEADS_TABLE.c.scope_id == scope_id, - ARTIFACT_HEADS_TABLE.c.family == Memory.family, - ) - .order_by(ARTIFACT_HEADS_TABLE.c.artifact_id) - ) - ).all() + await self.atomic_memory_index.initialize(connection) + await connection.execute( + self.atomic_memory_index.table.delete().where(self.atomic_memory_index.table.c.scope_id == scope_id) + ) + atomic_memories = self.atomic_memory.for_scope(scope_id) + cursor = None + rebuilt = 0 + while True: + page = await atomic_memories.list(limit=100, cursor=cursor) + for record in page.items: + prepared = await self.atomic_memory.publisher.prepare(record.artifact.content) + async with self.database.transaction() as connection: + await self.atomic_memory.publisher.publish(connection, scope_id, record, prepared) + rebuilt += 1 + cursor = page.next_cursor + if cursor is None: + break + async with self.database.transaction() as connection: + projected = int( + await connection.scalar( + select(func.count()) + .select_from(self.atomic_memory_index.table) + .where(self.atomic_memory_index.table.c.scope_id == scope_id) + ) + or 0 ) - if self.index.capabilities.vector and memory_refs: - model = self._embedding_model - if model is None or not await self.index.vector_complete( - connection, scope_id, memory_refs, model.profile - ): - raise RuntimeError("restored Memory vector projection is incomplete") # noqa: TRY003 + if projected != rebuilt: + raise RuntimeError("restored Atomic Memory projection is incomplete") # noqa: TRY003 topic_heads = tuple( ( await connection.execute( diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index 5d7aa60a9a..00a60cc874 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -34,7 +34,8 @@ from sqlalchemy.engine import make_url from powercontext.artifacts import ArtifactRef -from powercontext.builtin.artifacts.memory import EmbeddingProfile, MemoryEntryInput +from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent +from powercontext.builtin.artifacts.memory import EmbeddingProfile from powercontext.builtin.artifacts.skill import capture_skill_directory, package_file from powercontext.builtin.inference import EmbeddingResult from powercontext.builtin.persistence.oceanbase import OceanBaseConfig @@ -50,14 +51,13 @@ HandoffDraft, HandoffSourceCitation, HandoffStatement, - RememberMemoryRequest, SubmitSourceObservation, open_builtin_contexts, open_builtin_runtime, ) from powercontext.builtin.scope import ScopeDraft from powercontext.builtin.sources import ContentCapture -from powercontext.builtin.tags import MemoryEntryTagTarget, TagFilter +from powercontext.builtin.tags import ArtifactTagTarget, TagFilter from powercontext.builtin.work import AcknowledgeHandoff, ReceiverChecks from powercontext.sources import ( AdapterSourceDefinition, @@ -123,12 +123,11 @@ async def scenario() -> None: metadata={"origin": "portable-archive-e2e"}, ) ) - memory = await source_runtime.memory.for_scope(scope_id).remember( - RememberMemoryRequest( - entries=(MemoryEntryInput(kind="decision", text="Use the portable archive for migration."),) - ) - ) - assert memory.entry is not None + assert source_runtime.atomic_memory is not None + memory = await source_runtime.atomic_memory.for_scope(scope_id).create(( + AtomicMemoryContent(kind="decision", text="Use the portable archive for migration."), + )) + assert memory.records handoffs = source_runtime.handoff.for_scope(scope_id) prepared = await handoffs.finalize( HandoffDraft( @@ -202,13 +201,12 @@ async def scenario() -> None: scope = await source.scopes.create( ScopeDraft(title="Archive", summary="Vector recovery", idempotency_key="archive-vector") ) - service = (await source.get(scope.scope_id)).artifacts.memory - memory = await service.remember( - memory=None, entries=(MemoryEntryInput(kind="fact", text="Portable vector recovery."),), mode="append" + memory = await source.records.create_artifact( + scope.scope_id, + "atomic-memory", + ArtifactWrite(content={"kind": "fact", "text": "Portable vector recovery."}), ) - assert memory is not None - entry = memory.content.manifest.entries[0] - tag_target = MemoryEntryTagTarget(artifact_id=memory.artifact_id, entry_id=entry.entry_id) + tag_target = ArtifactTagTarget(family="atomic-memory", artifact_id=memory.artifact_id) empty = await source.records.get_tags(scope.scope_id, tag_target) tagged = await source.records.replace_tags( scope.scope_id, tag_target, ("archive",), expected_etag=empty.etag @@ -223,12 +221,10 @@ async def authorize(_scopes: tuple[str, ...]) -> None: assert receipt.projections_ready assert await target.records.get_tags(scope.scope_id, tag_target) == tagged async with open_builtin_contexts(target_config, embedding_model=Embedding()) as target: - service = (await target.get(scope.scope_id)).artifacts.memory - for mode in ("fts", "vector", "hybrid"): - result = await service.search( - "portable", memories=(memory,), mode=mode, tag_filter=TagFilter(tags=("archive",)) - ) - assert [hit.entry_id for hit in result.hits] == [entry.entry_id] + service = target.atomic_memory.for_scope(scope.scope_id) + for mode in ("text", "vector", "hybrid"): + result = await service.search("portable", mode=mode, tag_filter=TagFilter(tags=("archive",))) + assert [hit.hit.artifact_ref.artifact_id for hit in result.hits] == [memory.artifact_id] asyncio.run(scenario()) @@ -599,9 +595,6 @@ async def _replay_recurrence(contexts, scope_id): from powercontext.builtin.evidence.resolver import EvidenceResolver from powercontext.builtin.runtime.recurrence import RelationalRecurrenceLedger - async def no_memory(*_args): - raise AssertionError("fixture has no Memory citations") # noqa: TRY003 - repositories = contexts.repositories ledger = RelationalRecurrenceLedger( database=contexts.database, @@ -613,7 +606,6 @@ async def no_memory(*_args): scope_id=scope_id, sources=repositories.sources, artifacts=repositories.artifacts, - memory_reader=no_memory, ), ) async with contexts.database.transaction() as connection: diff --git a/uv.lock b/uv.lock index 8c02188594..86c0610625 100644 --- a/uv.lock +++ b/uv.lock @@ -1216,7 +1216,7 @@ name = "importlib-metadata" version = "9.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp" }, + { name = "zipp", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } wheels = [ @@ -2395,8 +2395,8 @@ name = "opendalfs" version = "0.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "fsspec" }, - { name = "opendal" }, + { name = "fsspec", marker = "python_full_version >= '3.12'" }, + { name = "opendal", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0d/23/fdb4d34f08ac68a27b2d553256e13b15ce139916d36bab33f9a0e71e8770/opendalfs-0.1.0.tar.gz", hash = "sha256:5ceaeccc0852ef10c8ef2ce20e5927dbc18083909c06550ad36f81bcacb5774d", size = 75112, upload-time = "2026-08-26T05:30:33.497Z" } wheels = [ @@ -2735,7 +2735,9 @@ builtin = [ ] cli = [ { name = "aiosqlite" }, + { name = "alembic" }, { name = "apscheduler" }, + { name = "filelock" }, { name = "httpx", extra = ["socks"] }, { name = "inquirerpy" }, { name = "jsonschema" }, @@ -2744,11 +2746,13 @@ cli = [ { name = "platformdirs" }, { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, { name = "pydantic-settings" }, + { name = "pymysql" }, { name = "pyobvector" }, { name = "pyyaml" }, { name = "sqlalchemy", extra = ["asyncio"] }, { name = "sqlite-vec" }, { name = "typer" }, + { name = "tzdata" }, ] client = [ { name = "httpx", extra = ["socks"] }, @@ -2854,6 +2858,7 @@ requires-dist = [ { name = "aiosqlite", marker = "extra == 'seekdb'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'server'", specifier = ">=0.22,<1" }, { name = "alembic", marker = "extra == 'builtin'", specifier = ">=1.16,<2" }, + { name = "alembic", marker = "extra == 'cli'", specifier = ">=1.16,<2" }, { name = "alembic", marker = "extra == 'seekdb'", specifier = ">=1.16,<2" }, { name = "alembic", marker = "extra == 'server'", specifier = ">=1.16,<2" }, { name = "apscheduler", marker = "extra == 'builtin'", specifier = ">=3.11,<4" }, @@ -2863,6 +2868,7 @@ requires-dist = [ { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115,<1" }, { name = "fastmcp", marker = "extra == 'server'", specifier = ">=3.4,<4" }, { name = "filelock", marker = "extra == 'builtin'", specifier = ">=3.16,<4" }, + { name = "filelock", marker = "extra == 'cli'", specifier = ">=3.16,<4" }, { name = "filelock", marker = "extra == 'seekdb'", specifier = ">=3.16,<4" }, { name = "filelock", marker = "extra == 'server'", specifier = ">=3.16,<4" }, { name = "httpx", marker = "extra == 'server'", specifier = ">=0.28,<1" }, @@ -2902,6 +2908,7 @@ requires-dist = [ { name = "pydantic-settings", marker = "extra == 'server'", specifier = ">=2.7,<3" }, { name = "pylibseekdb", marker = "(sys_platform == 'darwin' and extra == 'seekdb') or (sys_platform == 'linux' and extra == 'seekdb')", specifier = ">=1.4.0.post1,<2" }, { name = "pymysql", marker = "extra == 'builtin'", specifier = ">=1.2,<1.2.1" }, + { name = "pymysql", marker = "extra == 'cli'", specifier = ">=1.2,<1.2.1" }, { name = "pymysql", marker = "extra == 'seekdb'", specifier = ">=1.2,<1.2.1" }, { name = "pymysql", marker = "extra == 'server'", specifier = ">=1.2,<1.2.1" }, { name = "pyobvector", marker = "extra == 'builtin'", specifier = ">=0.2.28,<0.3" }, @@ -2931,6 +2938,7 @@ requires-dist = [ { name = "typer", marker = "extra == 'cli'", specifier = ">=0.16,<1" }, { name = "typing-extensions", specifier = ">=4.12,<5" }, { name = "tzdata", marker = "extra == 'builtin'", specifier = ">=2026.3" }, + { name = "tzdata", marker = "extra == 'cli'", specifier = ">=2026.3" }, { name = "tzdata", marker = "extra == 'seekdb'", specifier = ">=2026.3" }, { name = "tzdata", marker = "extra == 'server'", specifier = ">=2026.3" }, { name = "uvicorn", marker = "extra == 'server'", specifier = ">=0.34,<1" }, @@ -2976,8 +2984,8 @@ name = "powercontext-connector-opendal" version = "0.0.1" source = { editable = "integrations/opendal" } dependencies = [ - { name = "opendalfs" }, - { name = "powercontext", extra = ["client"] }, + { name = "opendalfs", marker = "python_full_version >= '3.12'" }, + { name = "powercontext", extra = ["client"], marker = "python_full_version >= '3.12'" }, ] [package.metadata] From fd56c05695a1159a949b35bba6efd44a82d1daa0 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Tue, 13 Oct 2026 15:49:39 +0800 Subject: [PATCH 28/32] fix(ci): expand OceanBase acceptance datafile capacity --- .github/workflows/e2e-harness.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 8e36436a0e..ff3bee2ec4 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -325,6 +325,32 @@ jobs: compose=(docker compose -f e2e/bub/compose.yaml -f e2e/bub/compose.oceanbase.yaml) "${compose[@]}" up --detach --wait --wait-timeout 660 oceanbase container_id=$("${compose[@]}" ps -q oceanbase) + # Match the clean-migration job's capacity: repeated database/index + # creation can exhaust the slim image's default 1 GB datafile limit. + "${compose[@]}" exec -T oceanbase obclient -h127.0.0.1 -P2881 -uroot@sys \ + -e "ALTER SYSTEM SET datafile_maxsize='8G'; ALTER SYSTEM SET datafile_next='512M'; ALTER SYSTEM SET datafile_size='4G';" + datafile_ready=false + for config_attempt in $(seq 1 30); do + "${compose[@]}" exec -T oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@sys --batch --skip-column-names \ + -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next');" \ + > "$RUNNER_TEMP/acceptance-datafile.tsv" + if awk ' + $5 == "datafile_size" && $7 == "4G" { size = 1 } + $5 == "datafile_maxsize" && $7 == "8G" { maximum = 1 } + $5 == "datafile_next" && $7 == "512M" { increment = 1 } + END { exit !(size && maximum && increment) } + ' "$RUNNER_TEMP/acceptance-datafile.tsv"; then + datafile_ready=true + break + fi + sleep 2 + done + cat "$RUNNER_TEMP/acceptance-datafile.tsv" + if [ "$datafile_ready" != true ]; then + echo 'OceanBase datafile configuration did not become effective within one minute.' >&2 + exit 1 + fi oceanbase_ip=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container_id") test -n "$oceanbase_ip" export POWERCONTEXT_TEST_OCEANBASE_URL="mysql+aoceanbase://root%40test:powercontext-e2e@${oceanbase_ip}:2881/powercontext?charset=utf8mb4" From 6e7235905f5abdfff81639e74b7eff52ad209a20 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Tue, 20 Oct 2026 18:31:50 +0800 Subject: [PATCH 29/32] fix(archive): harden portable archive recovery --- .github/workflows/master.yml | 7 + docs/en/docs/operate/portable-archive.md | 61 ++- docs/zh/docs/operate/portable-archive.md | 41 +- .../builtin/persistence/sources.py | 16 +- .../builtin/portability/__init__.py | 4 + .../builtin/portability/bundle.py | 208 ++++++++-- .../builtin/runtime/composition.py | 31 +- .../builtin/runtime/relational.py | 90 +---- .../builtin/sources/observation_validation.py | 86 ++++ src/powercontext/cli/archive.py | 49 ++- tests/builtin/persistence/test_records.py | 4 +- tests/builtin/portability/test_bundle.py | 84 +++- tests/e2e/test_portable_archive.py | 376 +++++++++++++++++- tests/test_archive_cli.py | 196 +++++++++ tests/test_source_observations.py | 4 +- 15 files changed, 1079 insertions(+), 178 deletions(-) create mode 100644 src/powercontext/builtin/sources/observation_validation.py diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 1d760e52f3..4dc05868e7 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -322,6 +322,13 @@ jobs: run: | Set-ItemProperty -LiteralPath 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name LongPathsEnabled -Value 0 + - name: Verify portable archive restoration and SQLite URI targets on Windows + run: >- + uv run --locked --no-sync pytest -q + tests/builtin/portability/test_bundle.py + tests/test_archive_cli.py + tests/e2e/test_portable_archive.py + # Keep pytest's default temporary root: a short --basetemp would hide # regressions in the nested checkout fixtures covered by issue #1501. - name: Run Windows portability regressions diff --git a/docs/en/docs/operate/portable-archive.md b/docs/en/docs/operate/portable-archive.md index 771f56053e..a914f9fc4b 100644 --- a/docs/en/docs/operate/portable-archive.md +++ b/docs/en/docs/operate/portable-archive.md @@ -15,6 +15,28 @@ PowerContext writers before restore. Export uses one database transaction as its MySQL-mode backends use `REPEATABLE READ` for the export transaction even when the deployment defaults to `READ COMMITTED`; later transactions retain the deployment default. +## Choosing the recovery format + +Use a deployment-native backup when you need the complete database, deployment-local state, or point-in-time recovery. +A `.pcb` bundle addresses a narrower requirement: transfer selected complete scopes between supported backends while +preserving their exact identities, revisions, evidence and retry behavior, without copying access bindings, host-local +selection, scheduler state or derived indexes. A SQLite database copy is not an OceanBase database, and replaying a +backend's SQL dump does not perform these application-level selections and projection rebuilds. Schema migrations +upgrade a deployment's schema; they do not replace this scope-transfer operation. + +The logical record schema is therefore an explicit portability contract, not a second copy of every storage column. +It retains authoritative domain fields and maps them to each backend while excluding deployment-specific data. +Changing a domain model requires checking this mapping and its round-trip behavior; a checksum cannot discover a +field that a producer failed to export. Unknown record schema versions are rejected instead of silently supplying +defaults for new authoritative state. + +Import policy is independent of the checksum algorithm: an absent identity is inserted, identical canonical content is +skipped, and different content at an existing identity causes a conflict without overwriting it. Per-record SHA-256 +digests are the chosen fingerprints for this comparison and for stream verification; comparing canonical record bytes +would implement the same policy. The total digest covers the record stream, while ZIP checks detect damaged compressed +data. These checks do not authenticate the producer, establish completeness against the source database, or replace +Source/Artifact decoding and dependency validation. + ## Prerequisites Install the CLI from the same PowerContext revision that created the local data. The archive command uses the persistent @@ -52,6 +74,9 @@ per-record digests, total digest, record counts, and selected scopes without ope reported checksum with the backup inventory if an external backup system needs an independent verification record. Before publishing the file, export checks its format, resource limits, and dependencies. If validation fails, an existing backup at the output path remains unchanged. +The destination must not be the source SQLite database or one of its `-wal`, `-shm`, or `-journal` files; export rejects +these paths rather than replacing live storage with an archive. These sibling filenames are reserved regardless of +letter case, including on case-sensitive filesystems. ## Validate before a restore @@ -63,7 +88,8 @@ powercontext archive restore ./backups/payments.pcb --dry-run Dry-run performs no domain writes. It verifies checksums, required source and Artifact dependencies, and whether the configured Runtime can restore the archive's source types and Artifact families. A validation failure exits with code -`2` for an invalid bundle, or `3` for unsupported requirements or target conflicts, and reports only a content-free +`2` for an invalid bundle or an uninspectable SQLite filename, or `3` for unsupported requirements or target conflicts, +and reports only a content-free reason; record bodies are not printed. A successful report includes `already_present`, `conflicts`, required and unsupported Source types and Artifact families, and whether the target has a projection rebuilder. Candidate versions and decision heads use the same complete decoder as normal reads. Validation @@ -71,12 +97,19 @@ checks their family consistency, evidence, reasons, lifecycle state and family-o audit envelopes; malformed or unsupported Candidates are rejected before any domain write. Run against a different deployment with `--env-file ./target.env`. Existing SQLite targets, including `file:` URI filenames with `uri=true`, are inspected read-only. A missing SQLite target is validated without creating its database file or parent directory. - -Native Sources require their Python adapter on the target. Worker-materialized `SourceObservation` records carry -their captured payload and projections, so they can be read without loading that adapter. The bundle includes only -the definition manifests referenced by the selected scopes and verifies observation identities and definition +This includes absolute `file:///C:/...` paths on Windows and percent-encoded filenames. Unsupported URI authorities, +unreadable targets, directory targets and non-directory parent paths fail validation rather than being treated as an +empty database. + +Native Sources require their Python adapter on the target. Before writing records, validation uses the same registered +Source decoder as normal reads to check both payload shape and indexed identity, including supported legacy payloads. +Worker-materialized `SourceObservation` records carry their captured payload and projections. The same decoder +validates their storage envelope without loading the worker adapter. The bundle includes only the definition +manifests referenced by the selected scopes and verifies observation identities and definition fingerprints before restore. Missing definitions and conflicting target definitions prevent restoration; unreferenced global definitions are not exported. +Observation payloads and supplied projections must also satisfy the same manifest schemas checked during normal +ingestion. Restore does not fetch remote schema references. To perform the write, repeat the command with explicit confirmation: @@ -126,6 +159,13 @@ payload for an existing immutable identity never overwrites it: restore exits wi transaction rolled back. Resolve the conflicting target data or choose a clean destination, then run the same restore command again. +The restore command opens its maintenance Runtime without bootstrapping a new default scope. This lets an archive +containing the canonical default scope restore its original identity and creation key into a fresh target. On the next +normal startup, default-scope bootstrap reuses that creation identity when present. Default selection bindings remain +deployment-local; restoring an archive does not overwrite an existing deployment's selection or conflicting scope. +The maintenance Runtime does not start background Artifact processing supervisors, even when processing is configured +for the target deployment. Restart the normal deployment after recovery to resume background work. + If the command is interrupted before completion, do not claim recovery succeeded. Authoritative records are transactionally applied, so a failed write does not leave a successful-looking partial restore. Re-run the same archive against the same destination after the previous command has stopped. @@ -133,9 +173,11 @@ against the same destination after the previous command has stopped. The target database stores a durable receipt keyed by `bundle_id`. `authoritative_restored` means the logical rows committed but search rebuild has not completed; `ready` means projection rebuild completed. If projection rebuild fails, re-run the same restore: identical rows are skipped and the receipt advances to `ready` only after verification. -The restore command can reopen unfinished Topic Memory projections when a pending restore receipt exists. Normal -Server startup still rejects incomplete projections; finish the restore before serving requests. Use the same target -embedding configuration when retrying. +While any receipt has `projections_ready=false`, normal Runtime/Server startup and worker startup refuse to serve, +including when the archive contains only Atomic Memory or Experience. The explicit restore path may reopen the target +and unfinished Topic Memory projections to complete recovery. Re-run the same restore before serving requests, using +the same target embedding configuration. Export from an unfinished recovery exits with code `3` and requires recovery +to finish first. Restore stages records in a temporary disk index before starting the target write transaction. Reserve temporary disk space for both the archive copy and its expanded records. Existing MySQL-mode deployments may require an @@ -167,6 +209,9 @@ binary available until a restore drill succeeds when upgrading across a PowerCon `artifact_head`, `artifact_lineage_artifact` and `candidate_head` require record schema version `2`. Incomplete schema version `1` representations of these records are rejected rather than assigning default merge state or Candidate kind. Create a fresh bundle from the source database with the current binary when a bundle fails these compatibility checks. +Binary fields use canonical Base64, and timestamp fields use the canonical timezone-naive ISO text emitted by the +supported database profiles. Alternative encodings, offset-bearing timestamps and out-of-range storage values are +rejected before writes rather than accepted with digests that would change on readback or conflict on retry. ## Deployment-native backups diff --git a/docs/zh/docs/operate/portable-archive.md b/docs/zh/docs/operate/portable-archive.md index 66e380f568..de10db488f 100644 --- a/docs/zh/docs/operate/portable-archive.md +++ b/docs/zh/docs/operate/portable-archive.md @@ -13,6 +13,22 @@ SQLite、SeekDB 或 OceanBase 配置。恢复前应停止 PowerContext 写入进 逻辑 snapshot。MySQL 模式的后端会为导出事务使用 `REPEATABLE READ`，即使部署默认采用 `READ COMMITTED`；后续事务仍使用部署的默认隔离级别。 +## 选择恢复格式 + +需要完整数据库、部署本地状态或时间点恢复时，应使用部署原生备份。`.pcb` 解决的是更窄的需求：在受支持的后端之间迁移 +选定的完整 scope，保留原始 identity、revision、evidence 和重试语义，但不复制访问绑定、宿主本地选择、调度状态和派生索引。 +SQLite 数据库副本不能直接作为 OceanBase 数据库使用；回放后端的 SQL dump 也不会执行这些应用层的范围选择和投影重建。 +schema migration 负责升级现有部署的表结构，并不替代这种 scope 迁移操作。 + +逻辑 record schema 因此是显式的可移植协议，而非每个存储列的另一份副本。它保留权威领域字段，映射到各后端，并排除部署 +专有数据。领域模型变化时，仍需检查映射及其往返行为；checksum 无法发现生产端遗漏导出的字段。读取端拒绝未知的 record +schema version，不会为新增的权威状态静默补默认值。 + +导入策略独立于 checksum 算法：不存在的 identity 会插入，canonical 内容相同则跳过，已有 identity 的内容不同则报告冲突， +不覆盖原数据。逐条 SHA-256 digest 是当前用于比较和流校验的指纹表示；直接比较 canonical record 字节也能实现同一策略。 +总 digest 覆盖 record 流，ZIP 校验可发现压缩数据损坏。这些检查不提供生产者身份认证，不证明导出内容相对源数据库完整， +也不替代 Source/Artifact 解码与依赖校验。 + ## 前提条件 从创建本地数据的同一个 PowerContext revision 安装 CLI。归档命令使用 @@ -46,6 +62,7 @@ powercontext archive inspect ./backups/payments.pcb `export` 会输出 JSON，其中包括 bundle ID、record 数和 checksum。`inspect` 不打开目标数据库，只校验 ZIP 结构、 单 record digest、总 digest、record 数和所选 scope。若外部备份系统需要独立校验记录，请把输出的 checksum 一并保存。 导出在发布文件前还会检查格式、资源限制和依赖完整性。验证失败时，输出路径上的旧备份保持不变。 +输出路径不能是源 SQLite 数据库或对应的 `-wal`、`-shm`、`-journal` 文件；导出会拒绝这些路径，不会用归档替换正在使用的存储。同目录中的这些保留文件名不区分大小写，在大小写敏感的文件系统上也一样。 ## 在恢复前验证 @@ -56,7 +73,8 @@ powercontext archive restore ./backups/payments.pcb --dry-run ``` dry-run 不会写入领域数据。它会校验 checksum、必需的 Source 和 Artifact 依赖，以及已配置 Runtime 是否支持归档中的 -source type 和 Artifact family。归档无效时返回退出码 `2`，不兼容或目标冲突时返回 `3`；错误信息不打印 record body。成功报告 +source type 和 Artifact family。归档无效或无法检查本地 SQLite 文件名时返回退出码 `2`，不兼容或目标冲突时返回 `3`； +错误信息不打印 record body。成功报告 包含 `already_present`、`conflicts`、所需和不支持的 Source type/Artifact family，以及目标是否支持 projection rebuild。 可用 `--env-file ./target.env` 检查另一个部署。 @@ -64,10 +82,15 @@ Candidate version 和 decision head 使用正常读取路径的完整解码器 以及 family 自有的 proposal 协议和可选的 Dream audit envelope。格式错误或不支持的 Candidate 会在写入领域数据前被拒绝。 已有 SQLite 目标以只读方式检查，支持带 `uri=true` 的 `file:` URI 文件名。 目标不存在时，dry-run 不会创建数据库文件或父目录。 +这包括 Windows 的绝对 `file:///C:/...` 路径及经过百分号编码的文件名。不支持的 URI authority、无法读取的目标、目录目标 +或非目录的父路径会使验证失败，不会被当成空数据库。 -原生 Source 需要目标环境安装对应的 Python adapter。远程 worker 已物化的 `SourceObservation` 自带捕获内容和投影， -读取时不需要加载该 adapter。归档只携带所选 scope 实际引用的 Source Definition manifest，并在恢复前校验 observation +原生 Source 需要目标环境安装对应的 Python adapter。验证会在写入前使用正常读取路径的已注册 Source 解码器，检查 payload +结构与索引 identity 是否一致，也覆盖受支持的 legacy payload。远程 worker 已物化的 `SourceObservation` 自带捕获内容和投影， +其存储 envelope 也使用同一解码器验证，但不需要加载 worker adapter。归档只携带所选 scope 实际引用的 Source Definition +manifest，并在恢复前校验 observation 的身份和定义指纹。缺失定义或与目标定义冲突会阻止恢复；未引用的全局定义不会随包导出。 +Observation 的 payload 和每个 projection 还必须通过正常写入路径所使用的 manifest schema 校验。恢复不会请求远程 schema。 要执行写恢复，必须显式确认： @@ -116,14 +139,20 @@ Atomic Memory 迁移后，归档保留迁移后的 Atomic Memory identity、不 时，恢复绝不会覆盖它：命令以退出码 `3` 结束，并回滚本次写 transaction。解决目标数据冲突，或选择一个干净的目标后， 再次运行同一个恢复命令。 +restore 命令会以维护模式打开 Runtime，不预先创建新的默认 scope。因此，包含规范默认 scope 的归档可以在干净的目标中恢复 +原始 identity 和创建幂等键；下次正常启动时，默认 scope 初始化会复用该创建 identity。默认选择绑定仍属于部署本地状态， +恢复不会覆盖现有部署的选择，也不会覆盖冲突的 scope。 +维护 Runtime 不会启动后台 Artifact 处理 supervisor，即使目标部署配置了后台处理。完成恢复后，再启动正常部署以继续后台工作。 + 如果命令在完成前中断，不能宣称恢复已成功。权威 record 会在 transaction 中写入；失败的写入不会留下看似成功的部分 恢复。先确认上一次命令已经停止，再对同一目标重新运行同一个归档。 目标数据库会按 `bundle_id` 保存 durable receipt。`authoritative_restored` 表示逻辑数据已经提交但搜索尚未重建完成； `ready` 表示 projection rebuild 已完成。若重建失败，重新恢复同一 bundle 会跳过相同记录，只有验证完成后才把 receipt 推进为 `ready`。 -有未完成的恢复回执时，restore 命令允许重新打开投影尚未完成的 Topic Memory；正常 Server 启动仍会拒绝不完整投影。 -请使用相同的目标 embedding 配置重试，完成恢复后再提供服务。 +只要存在 `projections_ready=false` 的回执，正常 Runtime/Server 启动与 worker 启动都会被拒绝，即使归档仅包含 Atomic Memory +或 Experience。显式 restore 路径可以重新打开目标及未完成的 Topic Memory 投影，以继续恢复。请使用相同的目标 embedding +配置重跑同一归档，完成后再提供服务。恢复未完成时，export 返回退出码 `3`，要求先完成恢复。 恢复会在目标写事务开始前将记录存入临时磁盘索引，请为归档副本及解压后的记录预留临时磁盘空间。 已有 MySQL 模式数据库可能需要在 Runtime 启动时将可移植时间戳从 `DATETIME` 升级为 `DATETIME(6)`； @@ -153,6 +182,8 @@ version 时，reader 会在目标写入前拒绝。跨 PowerContext 大版本升 `artifact_head`、`artifact_lineage_artifact` 和 `candidate_head` 要求 record schema version `2`。 reader 会拒绝这些记录不完整的 schema version `1` 表示，而不是补上默认合并状态或 Candidate 类型。 归档不满足兼容检查时，请用当前二进制从源数据库重新导出。 +二进制字段使用 canonical Base64；时间戳使用受支持数据库 profile 输出的 canonical、无时区 ISO 文本。恢复会在写入前拒绝 +替代编码、携带时区偏移的时间戳及超出存储范围的值，不会接受读回后 digest 改变、再次恢复却产生冲突的表示。 ## 部署原生备份 diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index ba3cd56c56..260ffdd2b7 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -119,7 +119,7 @@ async def add_with_status( await _lock_journal_head(connection, scope_id) existing = await self._find_row(connection, scope_id, ref) if existing is not None: - stored = self._decode_row(existing) + stored = self.decode_source(existing) # A trusted acknowledgement may replay a pre-provenance receipt. # Upgrade only its server-owned attestation, never its content or position. if ( @@ -161,7 +161,7 @@ async def add_with_status( existing = await self._find_row(connection, scope_id, ref) if existing is None: raise - stored = self._decode_row(existing) + stored = self.decode_source(existing) if stored.value != source: raise StoredPayloadConflictError("source", (scope_id, ref)) from None return stored, False @@ -186,7 +186,7 @@ async def get( row = await self._find_row(connection, scope_id, ref) if row is None: raise RepositoryNotFoundError("source", (scope_id, ref)) - return self._decode_row(row) + return self.decode_source(row) async def get_many( self, @@ -223,7 +223,7 @@ async def get_many( decoded: dict[tuple[str, str], StoredSource] = {} for row in rows: try: - stored = self._decode_row(row) + stored = self.decode_source(row) except RepositoryNotFoundError: ref = SourceRef(source_type=str(row["source_type"]), source_id=str(row["source_id"])) raise RepositoryNotFoundError("source", (scope_id, ref)) from None @@ -291,7 +291,7 @@ async def iter_list( statement = statement.limit(limit) async with connection.stream(statement.execution_options(yield_per=1)) as result: async for row in result.mappings(): - yield self._decode_row(row) + yield self.decode_source(row) async def list_window( self, @@ -324,7 +324,7 @@ async def list_window( .mappings() .all() ) - return tuple(self._decode_row(row) for row in rows) + return tuple(self.decode_source(row) for row in rows) async def journal_position(self, connection: AsyncConnection, scope_id: str, /) -> int: """Return the current high watermark for one scope.""" @@ -370,7 +370,9 @@ async def _find_row( .one_or_none() ) - def _decode_row(self, row: Mapping[Any, Any]) -> StoredSource: + def decode_source(self, row: Mapping[Any, Any], /) -> StoredSource: + """Decode and validate one stored Source using the registered adapter contract.""" + source_type = str(row["source_type"]) source_id = str(row["source_id"]) payload = stored_bytes(row["payload"], column="payload") diff --git a/src/powercontext/builtin/portability/__init__.py b/src/powercontext/builtin/portability/__init__.py index 8ce6d3d273..b0928f7d84 100644 --- a/src/powercontext/builtin/portability/__init__.py +++ b/src/powercontext/builtin/portability/__init__.py @@ -15,11 +15,13 @@ """Versioned logical export and restore for built-in relational data.""" from powercontext.builtin.portability.bundle import ( + BundleCompatibilityError, BundleConflictError, BundleFormatError, BundleInspection, BundleProgress, BundleReceipt, + BundleRecoveryRequiredError, BundleValidation, ExportAuthorizer, PortableBundleService, @@ -27,11 +29,13 @@ ) __all__ = [ + "BundleCompatibilityError", "BundleConflictError", "BundleFormatError", "BundleInspection", "BundleProgress", "BundleReceipt", + "BundleRecoveryRequiredError", "BundleValidation", "ExportAuthorizer", "PortableBundleService", diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index 4b20b3f032..62a4d3365a 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -24,6 +24,7 @@ from __future__ import annotations +import asyncio import base64 import hashlib import json @@ -32,6 +33,7 @@ import sqlite3 import tempfile import zipfile +import zlib from collections import Counter from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Iterator, Mapping from contextlib import contextmanager @@ -43,7 +45,20 @@ from uuid import NAMESPACE_URL, UUID, uuid5 from pydantic import ValidationError -from sqlalchemy import DateTime, PrimaryKeyConstraint, UniqueConstraint, and_, insert, or_, select, tuple_, update +from sqlalchemy import ( + BigInteger, + DateTime, + Integer, + PrimaryKeyConstraint, + String, + UniqueConstraint, + and_, + insert, + or_, + select, + tuple_, + update, +) from sqlalchemy import inspect as sqlalchemy_inspect from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncConnection @@ -61,7 +76,11 @@ from powercontext.builtin.artifacts.skill.package import SkillPackageError, capture_skill_archive from powercontext.builtin.persistence.candidates import decode_candidate_proposal_envelope from powercontext.builtin.persistence.database import AsyncDatabase -from powercontext.builtin.persistence.errors import InvalidStoredPayloadError +from powercontext.builtin.persistence.errors import ( + IdentityMismatchError, + InvalidStoredPayloadError, + RepositoryNotFoundError, +) from powercontext.builtin.persistence.tables import ( ARTIFACT_HANDLES_TABLE, ARTIFACT_HEADS_TABLE, @@ -88,6 +107,11 @@ SOURCES_TABLE, TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE, ) +from powercontext.builtin.sources.observation_validation import ( + validate_source_definition_manifest, + validate_source_observation, +) +from powercontext.errors import SourceError from powercontext.sources import SourceDefinitionManifest, SourceObservation FORMAT_VERSION = 1 @@ -167,6 +191,17 @@ class BundleConflictError(ValueError): """A target immutable identity exists with different canonical content.""" +class BundleCompatibilityError(BundleFormatError): + """The target does not support the bundle's otherwise valid requirements.""" + + +class BundleRecoveryRequiredError(RuntimeError): + """A committed restore needs its projections rebuilt before normal startup.""" + + def __init__(self) -> None: + super().__init__("portable restore projections are incomplete; finish archive restore before serving requests") + + @dataclass(frozen=True, slots=True) class BundleInspection: """Content-free facts established by parsing and verifying a bundle.""" @@ -442,11 +477,13 @@ def __init__( supported_artifact_families: Iterable[str] | None = None, artifact_content_decoder: Callable[[str, bytes], object] | None = None, candidate_decoder: Callable[[Mapping[str, Any]], object] | None = None, + source_decoder: Callable[[Mapping[str, Any]], object] | None = None, ) -> None: self._database = database self._projection_rebuilder = projection_rebuilder self._artifact_content_decoder = artifact_content_decoder self._candidate_decoder = candidate_decoder + self._source_decoder = source_decoder self._supported_source_types = None if supported_source_types is None else frozenset(supported_source_types) self._supported_artifact_families = ( None if supported_artifact_families is None else frozenset(supported_artifact_families) @@ -479,8 +516,8 @@ async def export( archive_path = _temporary_path(output.parent, suffix=".pcb") try: with records_path.open("wb") as stream: - async with self._database.transaction() as connection: - await _establish_export_snapshot(connection) + async with self._database.transaction(consistent_snapshot=True) as connection: + await _validate_export_destination(connection, output) await _validate_selected_scopes(connection, selected) count, records_by_type, digest = await self._stream_export( connection, selected, stream, progress=progress @@ -545,7 +582,7 @@ async def validate( else tuple(sorted(required_families - self._supported_artifact_families)) ) already_present, conflicts = await self._target_compatibility( - source, total=inspection.record_count, progress=progress + source, supported_source_types=configured, total=inspection.record_count, progress=progress ) return BundleValidation( bundle_id=inspection.bundle_id, @@ -598,7 +635,7 @@ async def _restore_snapshot( if validation.conflicts: raise BundleConflictError(f"target contains {validation.conflicts} conflicting record identities") if not validation.compatible: - raise BundleFormatError("target is not compatible with bundle requirements") + raise BundleCompatibilityError("target is not compatible with bundle requirements") try: with _staged_records(source) as staged: async with self._database.transaction() as connection: @@ -691,6 +728,7 @@ async def _target_compatibility( source: Path, /, *, + supported_source_types: frozenset[str] | None, total: int, progress: ProgressObserver | None, ) -> tuple[int, int]: @@ -705,7 +743,7 @@ async def _target_compatibility( } ) for processed, record in enumerate(_iter_records(source), start=1): - self._validate_target_content(record) + self._validate_target_content(record, supported_source_types=supported_source_types) table = _SPECS[record.record_type].table if table.name not in existing_tables: _report_progress(progress, "target", processed, total) @@ -729,7 +767,10 @@ async def _target_compatibility( _report_progress(progress, "target", processed, total) return already_present, conflicts - def _validate_target_content(self, record: _Record, /) -> None: + def _validate_target_content(self, record: _Record, /, *, supported_source_types: frozenset[str] | None) -> None: + if record.record_type == "source": + self._validate_source(record, supported_source_types=supported_source_types) + return if record.record_type == "candidate_version": if self._candidate_decoder is None: raise BundleFormatError("target cannot validate Candidate proposals") @@ -744,6 +785,26 @@ def _validate_target_content(self, record: _Record, /) -> None: except InvalidStoredPayloadError as error: raise BundleFormatError("Artifact content does not match the target family decoder") from error + def _validate_source(self, record: _Record, /, *, supported_source_types: frozenset[str] | None) -> None: + if ( + _source_observation(record) is None + and supported_source_types is not None + and record.identity["source_type"] not in supported_source_types + ): + return + if self._source_decoder is None: + raise BundleFormatError("target cannot validate Source payloads") + try: + self._source_decoder(_row_values(record)) + except ( + InvalidStoredPayloadError, + IdentityMismatchError, + RepositoryNotFoundError, + SourceError, + ValueError, + ) as error: + raise BundleFormatError("Source payload does not match the target Source decoder") from error + async def _write_records( self, connection: AsyncConnection, @@ -892,7 +953,7 @@ def _source_observation(record: _Record) -> SourceObservation | None: return None # Legacy native payload. try: observation = SourceObservation.model_validate_json(_canonical_json(value)) - except ValidationError as error: + except ValueError as error: raise BundleFormatError("invalid Source observation") from error if (observation.source_type, observation.name) != (record.identity["source_type"], record.identity["source_id"]): raise BundleFormatError("Source observation identity mismatch") @@ -904,7 +965,8 @@ def _source_definition(record: _Record) -> SourceDefinitionManifest: manifest = SourceDefinitionManifest.model_validate_json( cast(bytes, _database_value(record.payload["manifest"])) ) - except ValidationError as error: + validate_source_definition_manifest(manifest) + except (SourceError, ValueError) as error: raise BundleFormatError("invalid Source definition manifest") from error if (manifest.name, manifest.version, manifest.fingerprint) != ( record.identity["definition_name"], @@ -928,21 +990,19 @@ def _record_from_row(record_type: RecordType, row: Mapping[str, Any]) -> _Record return _Record(record_type=record_type, identity=identity, payload=payload, digest=_digest(canonical)) -async def _establish_export_snapshot(connection: AsyncConnection, /) -> None: - """Pin all export reads to one database snapshot before enumeration.""" - - if connection.dialect.name == "mysql": - # Replace AsyncDatabase's empty transaction before any reads. SET - # TRANSACTION applies only to the next transaction, so pooled sessions - # retain their configured isolation after export. - await connection.exec_driver_sql("ROLLBACK") - await connection.exec_driver_sql("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") - await connection.exec_driver_sql("START TRANSACTION WITH CONSISTENT SNAPSHOT") - elif connection.dialect.name == "sqlite": - # SQLAlchemy's SQLite transaction begins lazily. Pin the snapshot - # before the first table is enumerated so later SELECTs cannot observe - # a writer that committed midway through the export. - await connection.exec_driver_sql("BEGIN") +async def _validate_export_destination(connection: AsyncConnection, output: Path, /) -> None: + if connection.dialect.name != "sqlite": + return + destination = await asyncio.to_thread(output.resolve) + for row in await connection.exec_driver_sql("PRAGMA database_list"): + if not row[2]: + continue + database = await asyncio.to_thread(Path(row[2]).resolve) + reserved_names = {database.name.casefold() + suffix for suffix in ("", "-wal", "-shm", "-journal")} + if destination.name.casefold() in reserved_names and await asyncio.to_thread( + destination.parent.samefile, database.parent + ): + raise BundleFormatError("export destination overlaps the source database or its journal") def _record_document(record: _Record) -> dict[str, Any]: @@ -988,7 +1048,17 @@ def _parse_bundle( with archive.open(_MANIFEST_NAME) as stream: manifest = json.loads(stream.read(_MAX_MANIFEST_BYTES + 1), parse_constant=_reject_json_constant) _validate_manifest(manifest) - except (OSError, json.JSONDecodeError, zipfile.BadZipFile) as error: + except ( + OSError, + EOFError, + ValueError, + RecursionError, + NotImplementedError, + zipfile.BadZipFile, + zlib.error, + ) as error: + if isinstance(error, BundleFormatError): + raise raise BundleFormatError("cannot read portable bundle") from error count = 0 counts: Counter[str] = Counter() @@ -1091,7 +1161,17 @@ def _iter_records(source: Path, /) -> Iterator[_Record]: if count > _MAX_RECORDS: raise BundleFormatError("record count exceeds limit") yield _parse_record(json.loads(line, parse_constant=_reject_json_constant)) - except (OSError, json.JSONDecodeError, zipfile.BadZipFile) as error: + except ( + OSError, + EOFError, + ValueError, + RecursionError, + NotImplementedError, + zipfile.BadZipFile, + zlib.error, + ) as error: + if isinstance(error, BundleFormatError): + raise raise BundleFormatError("cannot read portable bundle") from error @@ -1102,6 +1182,7 @@ def _validate_container(archive: zipfile.ZipFile) -> None: raise BundleFormatError("bundle must contain only manifest.json and records.ndjson") if any( item.is_dir() + or item.flag_bits & 1 or item.compress_type not in {zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED} or item.compress_size > _MAX_ARCHIVE_BYTES for item in items @@ -1129,7 +1210,7 @@ def _validate_manifest(manifest: object) -> None: # noqa: C901 - manifest field } if set(manifest) != required: raise BundleFormatError("manifest has unknown or missing fields") - if manifest["format_version"] != FORMAT_VERSION: + if type(manifest["format_version"]) is not int or manifest["format_version"] != FORMAT_VERSION: raise BundleFormatError("unsupported bundle format version") if ( not isinstance(manifest["bundle_id"], str) @@ -1182,7 +1263,12 @@ def _parse_record(value: object) -> _Record: if set(value) != {"record_type", "schema_version", "identity", "payload", "digest"}: raise BundleFormatError("record has unknown or missing fields") record_type = value.get("record_type") - if record_type not in _SPECS or value.get("schema_version") != _SPECS[cast(RecordType, record_type)].schema_version: + if ( + not isinstance(record_type, str) + or record_type not in _SPECS + or type(value.get("schema_version")) is not int + or value["schema_version"] != _SPECS[cast(RecordType, record_type)].schema_version + ): raise BundleFormatError("unsupported record type or schema") identity = value.get("identity") payload = value.get("payload") @@ -1217,6 +1303,19 @@ def _parse_record(value: object) -> _Record: def _validate_record_values(record: _Record, /) -> None: # noqa: C901 - one versioned schema contract. identity = record.identity payload = record.payload + for field, value in {**identity, **payload}.items(): + column_type = _SPECS[record.record_type].table.c[field].type + if ( + isinstance(value, str) + and isinstance(column_type, String) + and column_type.length is not None + and len(value) > column_type.length + ): + raise BundleFormatError("record string exceeds its portable storage limit") + if isinstance(value, int) and not isinstance(value, bool) and isinstance(column_type, Integer): + bits = 63 if isinstance(column_type, BigInteger) else 31 + if not -(1 << bits) <= value < (1 << bits): + raise BundleFormatError("record integer exceeds its portable storage limit") if any(isinstance(value, str) and (not value or value != value.strip()) for value in identity.values()): raise BundleFormatError("record identity contains an invalid string") positive_identity_fields = {"revision", "version"} @@ -1247,9 +1346,11 @@ def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: if not isinstance(value, dict) or set(value) != {"base64"} or not isinstance(encoded, str): raise BundleFormatError("record payload has an invalid binary field") try: - base64.b64decode(encoded, validate=True) + decoded = base64.b64decode(encoded, validate=True) except ValueError as error: raise BundleFormatError("record payload has an invalid binary field") from error + if base64.b64encode(decoded).decode("ascii") != encoded: + raise BundleFormatError("record payload has a non-canonical binary field") kind = record.record_type if kind == "scope": @@ -1260,6 +1361,8 @@ def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: raise BundleFormatError("scope cannot reference itself") elif kind == "scope_external_reference": strings("kind", "value", "value_digest") + if payload["value_digest"] != hashlib.sha256(payload["value"].encode()).hexdigest(): + raise BundleFormatError("scope external reference has invalid value digest") elif kind == "scope_creation_request": strings("request_digest", "scope_id") elif kind == "source_journal_head": @@ -1312,7 +1415,11 @@ def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: integers("revision", positive=True) integers("governance_generation") if payload["lifecycle_state"] not in {"active", "deprecated", "retired"} or ( - payload["replacement_artifact_id"] is not None and payload["lifecycle_state"] != "deprecated" + payload["replacement_artifact_id"] is not None + and ( + payload["lifecycle_state"] != "deprecated" + or payload["replacement_artifact_id"] == identity["artifact_id"] + ) ): raise BundleFormatError("artifact head has invalid governance state") if payload["merged_into_id"] is not None and ( @@ -1460,9 +1567,11 @@ def _recurrence_value(record: _Record, /) -> RecurrenceMatch | RecurrenceObserva def _validate_datetime_text(value: object, /) -> None: try: - datetime.fromisoformat(cast(str, value)) + parsed = datetime.fromisoformat(cast(str, value)) except ValueError as error: raise BundleFormatError("record payload has an invalid datetime field") from error + if parsed.tzinfo is not None or parsed.isoformat() != value: + raise BundleFormatError("record payload has a non-canonical datetime field") async def _validate_selected_scopes(connection: AsyncConnection, scopes: tuple[str, ...], /) -> None: @@ -1507,7 +1616,9 @@ def _validate_record_safe(record: _Record, /) -> None: encoded = cast(dict[str, object], value).get("base64") if isinstance(value, dict) else None if isinstance(encoded, str): try: - decoded = json.loads(base64.b64decode(encoded, validate=True)) + decoded = json.loads(base64.b64decode(encoded, validate=True), parse_constant=_reject_json_constant) + except BundleFormatError: + raise except (ValueError, UnicodeDecodeError, json.JSONDecodeError): continue _validate_safe_value(decoded, field) @@ -1561,6 +1672,7 @@ def contains(record_type: str, identity: Mapping[str, object]) -> bool: for record in _iter_records(source): _validate_record_dependencies(record, contains) _validate_candidate_record(record, index, candidate_decoder) + _validate_observation_record(record, index) _validate_skill_package_dependency(record, packages) if record.record_type == "source": if int(record.payload["journal_position"]) > heads.get(str(record.identity["scope_id"]), -1): @@ -1571,6 +1683,26 @@ def contains(record_type: str, identity: Mapping[str, object]) -> bool: return required_sources, required_families +def _validate_observation_record(record: _Record, index: sqlite3.Connection, /) -> None: + if record.record_type != "source": + return + observation = _source_observation(record) + if observation is None: + return + identity = {"definition_name": observation.source_type, "definition_version": observation.definition_version} + document = index.execute( + "SELECT document FROM identities WHERE record_type = 'source_definition' AND identity = ?", + (_identity_key(identity),), + ).fetchone() + if document is None: + raise BundleFormatError("observation references missing Source definition") + manifest = _source_definition(_parse_record(json.loads(document[0]))) + try: + validate_source_observation(observation, manifest) + except (SourceError, ValueError) as error: + raise BundleFormatError("Source observation does not match its definition manifest") from error + + def _validate_candidate_record( record: _Record, index: sqlite3.Connection, @@ -1780,6 +1912,10 @@ def _validate_record_dependencies( # noqa: C901 - logical record kinds have dis elif record.record_type == "artifact_head": if not contains("artifact_revision", {**identity, "revision": payload["revision"]}): raise BundleFormatError("artifact head references missing revision") + if payload["replacement_artifact_id"] is not None and not contains( + "artifact_head", {**identity, "artifact_id": payload["replacement_artifact_id"]} + ): + raise BundleFormatError("artifact head references missing replacement") if payload["merged_into_id"] is not None and not contains( "artifact_head", {**identity, "artifact_id": payload["merged_into_id"]} ): @@ -1859,7 +1995,10 @@ def _contains_memory_entry_target( def _dependency_metadata(source: Path, /) -> tuple[dict[str, int], dict[tuple[str, str], tuple[str, int, int, int]]]: heads: dict[str, int] = {} packages: dict[tuple[str, str], tuple[str, int, int, int]] = {} + scopes: list[_Record] = [] for record in _iter_records(source): + if record.record_type == "scope": + scopes.append(record) if record.record_type == "source_journal_head": heads[str(record.identity["scope_id"])] = int(record.payload["position"]) if record.record_type != "skill_package": @@ -1885,6 +2024,7 @@ def _dependency_metadata(source: Path, /) -> tuple[dict[str, int], dict[tuple[st snapshot.reference.uncompressed_size, snapshot.reference.archive_size, ) + _ordered_scope_records(scopes) return heads, packages @@ -1977,7 +2117,7 @@ def _identity_index(source: Path, /) -> Iterator[sqlite3.Connection]: record.record_type, _identity_key(record.identity), _canonical_json(_record_document(record)) - if record.record_type == "candidate_version" + if record.record_type in {"candidate_version", "source_definition"} else None, ), ) diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index fcadfbebf8..b9cd3f7910 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -135,6 +135,7 @@ CompositeTopicMemoryIndex, TopicMemoryIndex, ) +from powercontext.builtin.portability import BundleRecoveryRequiredError from powercontext.builtin.runtime._scope_cache import ScopeCacheObserver from powercontext.builtin.runtime.application import ( BuiltinRuntime, @@ -792,7 +793,9 @@ async def run_profile(scope_id, high): ) ) runtime.artifact_processing_supervisor = await resources.enter_async_context( - _open_artifact_processing_supervisor(config, contexts, processing_bindings, tracing=tracing) + _open_artifact_processing_supervisor( + config, contexts, processing_bindings, tracing=tracing, recovery=_archive_recovery + ) ) contexts.profiles.operation_context = runtime._operation if config.handoff_report.enabled: @@ -970,8 +973,9 @@ async def _open_artifact_processing_supervisor( bindings: Sequence[ArtifactProcessingBinding], *, tracing: RuntimeTracing | None = None, + recovery: bool = False, ) -> AsyncIterator[ArtifactProcessingSupervisors | None]: - if config.runtime.artifact_processing_role == "api" or not bindings: + if recovery or config.runtime.artifact_processing_role == "api" or not bindings: yield None return groups = ( @@ -1103,7 +1107,8 @@ async def open_builtin_contexts( tracing=tracing, ) try: - await contexts.scopes.bootstrap_default() + if not _archive_recovery: + await contexts.scopes.bootstrap_default() yield contexts finally: # Producers are stopped by now, and the profile below still owns @@ -1181,7 +1186,8 @@ async def open_builtin_contexts( tracing=tracing, ) try: - await contexts.scopes.bootstrap_default() + if not _archive_recovery: + await contexts.scopes.bootstrap_default() yield contexts finally: # Producers are stopped by now, and the profile below still owns the @@ -1192,18 +1198,13 @@ async def open_builtin_contexts( async def _initialize_archive_aware_topics( connection: AsyncConnection, index: TopicMemoryIndex, *, worker: bool, recovery: bool ) -> None: - # Only an explicit restore may open incomplete projections, and only when - # a committed restore receipt explains their absence. Normal startup and - # workers retain the strict storage invariant checks. - pending = ( - recovery - and not worker - and await connection.scalar( - select(PORTABLE_RESTORE_RECEIPTS_TABLE.c.bundle_id) - .where(PORTABLE_RESTORE_RECEIPTS_TABLE.c.projections_ready.is_(False)) - .limit(1) - ) + pending = await connection.scalar( + select(PORTABLE_RESTORE_RECEIPTS_TABLE.c.bundle_id) + .where(PORTABLE_RESTORE_RECEIPTS_TABLE.c.projections_ready.is_(False)) + .limit(1) ) + if pending and (not recovery or worker): + raise BundleRecoveryRequiredError() if pending: await index.initialize(connection) else: diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index cd5172ee59..be88964c7c 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -24,15 +24,9 @@ from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass, field, replace from datetime import UTC, datetime -from typing import TYPE_CHECKING, Any, Literal, cast +from typing import TYPE_CHECKING, Literal, cast from uuid import uuid4 -from jsonschema import Draft202012Validator -from jsonschema.exceptions import SchemaError -from jsonschema.exceptions import ValidationError as JsonSchemaValidationError -from jsonschema.protocols import Validator -from referencing import Registry -from referencing.exceptions import Unresolvable from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncConnection @@ -226,6 +220,10 @@ SourceJournalEntry, validate_scope_id, ) +from powercontext.builtin.sources.observation_validation import ( + validate_source_definition_manifest, + validate_source_observation, +) from powercontext.builtin.statistics import ModelUsageOperation, ModelUsagePurpose, RecallTokenMeasurement from powercontext.builtin.triggers import ( HANDOFF_BOUNDARY_TRIGGER_NAME, @@ -240,12 +238,10 @@ from powercontext.errors import ( ArtifactNotFoundError, InvalidSourceDefinitionError, - InvalidSourceObservationError, SourceConflictError, SourceDefinitionNotFoundError, SourceNotFoundError, ) -from powercontext.limits import MAX_SOURCE_OBSERVATION_BYTES from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, ConnectorBinding, @@ -255,7 +251,6 @@ SourceDefinitionRegistry, SourceObservation, SourceRef, - TextEvidence, ) IdFactory = Callable[[str], str] @@ -610,6 +605,7 @@ def __init__( supported_artifact_families=artifact_repository.families, artifact_content_decoder=artifact_repository.decode_content, candidate_decoder=self.repositories.candidates.decode_candidate, + source_decoder=source_repository.decode_source, ) self._model_usage = _ModelUsageRecorder( database, @@ -873,7 +869,13 @@ async def register_source_definition( ) -> SourceDefinitionManifest: """Register one immutable declarative Definition supplied by a worker.""" - _validate_source_definition_manifest(manifest, self.source_registry) + validate_source_definition_manifest(manifest) + try: + self.source_registry.definition_for_name(manifest.name) + except SourceDefinitionNotFoundError: + pass + else: + raise InvalidSourceDefinitionError(type(manifest), "name", "must not replace an active Source Definition") try: async with self.database.transaction() as connection: stored = await self.repositories.source_definitions.register(connection, manifest) @@ -905,7 +907,7 @@ async def submit_source_observation( ) except RepositoryNotFoundError: raise SourceDefinitionNotFoundError(observation.source_type, observation.definition_version) from None - _validate_source_observation(observation, manifest) + validate_source_observation(observation, manifest) services = self._services_for(request.scope_id) source_store, source_catalog = services.sources() stored = await source_store.add(observation) @@ -1961,70 +1963,6 @@ def _validate_experience_plans( ) -def _validate_source_definition_manifest( - manifest: SourceDefinitionManifest, - source_registry: SourceDefinitionRegistry, -) -> None: - if len(manifest.model_dump_json(by_alias=True).encode()) > 64 * 1024: - raise InvalidSourceDefinitionError(type(manifest), "manifest", "must not exceed 64 KiB") - try: - source_registry.definition_for_name(manifest.name) - except SourceDefinitionNotFoundError: - pass - else: - raise InvalidSourceDefinitionError(type(manifest), "name", "must not replace an active Source Definition") - _json_schema_validator(manifest.name, manifest.source_schema) - standard_text_schema = TextEvidence.model_json_schema() - for projection in manifest.projections: - _json_schema_validator(projection.key.name, projection.schema_) - if projection.key == TEXT_EVIDENCE_PROJECTION_KEY and projection.schema_ != standard_text_schema: - raise InvalidSourceDefinitionError( - type(manifest), - "projection", - f"{projection.key.name!r} must use the standard schema", - ) - - -def _validate_source_observation(source: SourceObservation, manifest: SourceDefinitionManifest) -> None: - if source.source_type != manifest.name or source.definition_version != manifest.version: - raise InvalidSourceObservationError("definition", "does not match the registered manifest identity") - if source.definition_fingerprint != manifest.fingerprint: - raise InvalidSourceObservationError("fingerprint", "does not match the registered manifest") - if len(source.model_dump_json().encode()) > MAX_SOURCE_OBSERVATION_BYTES: - raise InvalidSourceObservationError("size", "must not exceed 4 MiB") - _validate_schema_value(manifest.name, manifest.source_schema, source.payload) - - declarations = {projection.key: projection for projection in manifest.projections} - supplied = {projection.key: projection.value for projection in source.projections} - if declarations.keys() != supplied.keys(): - raise InvalidSourceObservationError("projections", "must exactly match the registered manifest") - for key, declaration in declarations.items(): - value = supplied[key] - _validate_schema_value(key.name, declaration.schema_, value) - if key == TEXT_EVIDENCE_PROJECTION_KEY: - evidence = TextEvidence.model_validate(value) - if evidence.source_type != source.source_type or evidence.source_id != source.name: - raise InvalidSourceObservationError( - "text-evidence", - "source identity does not match the observation envelope", - ) - - -def _json_schema_validator(name: str, schema: Mapping[str, Any]) -> Validator: - try: - Draft202012Validator.check_schema(schema) - return Draft202012Validator(schema, registry=Registry()) - except SchemaError as error: - raise InvalidSourceDefinitionError(type(schema), "schema", f"{name!r} is not valid JSON Schema") from error - - -def _validate_schema_value(name: str, schema: Mapping[str, Any], value: object) -> None: - try: - _json_schema_validator(name, schema).validate(value) - except (JsonSchemaValidationError, Unresolvable) as error: - raise InvalidSourceObservationError("schema", f"value does not match {name!r}") from error - - def _scoped_id_factory(delegate: IdFactory | None) -> IdFactory: def new_id(kind: str) -> str: if delegate is not None: diff --git a/src/powercontext/builtin/sources/observation_validation.py b/src/powercontext/builtin/sources/observation_validation.py new file mode 100644 index 0000000000..2145b55b60 --- /dev/null +++ b/src/powercontext/builtin/sources/observation_validation.py @@ -0,0 +1,86 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Validate transported Sources against their immutable definition manifests.""" + +from collections.abc import Mapping +from typing import Any + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError +from jsonschema.exceptions import ValidationError as JsonSchemaValidationError +from jsonschema.protocols import Validator +from referencing import Registry +from referencing.exceptions import Unresolvable + +from powercontext.errors import InvalidSourceDefinitionError, InvalidSourceObservationError +from powercontext.limits import MAX_SOURCE_OBSERVATION_BYTES +from powercontext.sources import TEXT_EVIDENCE_PROJECTION_KEY, SourceDefinitionManifest, SourceObservation, TextEvidence + + +def validate_source_definition_manifest(manifest: SourceDefinitionManifest, /) -> None: + """Validate the bounded, network-independent schemas declared by a worker.""" + + if len(manifest.model_dump_json(by_alias=True).encode()) > 64 * 1024: + raise InvalidSourceDefinitionError(type(manifest), "manifest", "must not exceed 64 KiB") + json_schema_validator(manifest.name, manifest.source_schema) + standard_text_schema = TextEvidence.model_json_schema() + for projection in manifest.projections: + json_schema_validator(projection.key.name, projection.schema_) + if projection.key == TEXT_EVIDENCE_PROJECTION_KEY and projection.schema_ != standard_text_schema: + raise InvalidSourceDefinitionError( + type(manifest), "projection", f"{projection.key.name!r} must use the standard schema" + ) + + +def validate_source_observation(source: SourceObservation, manifest: SourceDefinitionManifest, /) -> None: + """Check captured payloads and every projection using the owning declaration.""" + + if source.source_type != manifest.name or source.definition_version != manifest.version: + raise InvalidSourceObservationError("definition", "does not match the registered manifest identity") + if source.definition_fingerprint != manifest.fingerprint: + raise InvalidSourceObservationError("fingerprint", "does not match the registered manifest") + if len(source.model_dump_json().encode()) > MAX_SOURCE_OBSERVATION_BYTES: + raise InvalidSourceObservationError("size", "must not exceed 4 MiB") + _validate_schema_value(manifest.name, manifest.source_schema, source.payload) + declarations = {projection.key: projection for projection in manifest.projections} + supplied = {projection.key: projection.value for projection in source.projections} + if declarations.keys() != supplied.keys(): + raise InvalidSourceObservationError("projections", "must exactly match the registered manifest") + for key, declaration in declarations.items(): + value = supplied[key] + _validate_schema_value(key.name, declaration.schema_, value) + if key == TEXT_EVIDENCE_PROJECTION_KEY: + evidence = TextEvidence.model_validate(value) + if evidence.source_type != source.source_type or evidence.source_id != source.name: + raise InvalidSourceObservationError( + "text-evidence", "source identity does not match the observation envelope" + ) + + +def json_schema_validator(name: str, schema: Mapping[str, Any], /) -> Validator: + """Build a JSON Schema validator with no network-backed reference registry.""" + + try: + Draft202012Validator.check_schema(schema) + return Draft202012Validator(schema, registry=Registry()) + except SchemaError as error: + raise InvalidSourceDefinitionError(type(schema), "schema", f"{name!r} is not valid JSON Schema") from error + + +def _validate_schema_value(name: str, schema: Mapping[str, Any], value: object) -> None: + try: + json_schema_validator(name, schema).validate(value) + except (JsonSchemaValidationError, Unresolvable) as error: + raise InvalidSourceObservationError("schema", f"value does not match {name!r}") from error diff --git a/src/powercontext/cli/archive.py b/src/powercontext/cli/archive.py index cea2593be8..e575781169 100644 --- a/src/powercontext/cli/archive.py +++ b/src/powercontext/cli/archive.py @@ -18,11 +18,13 @@ import asyncio import json +import stat from collections.abc import Awaitable, Callable from dataclasses import asdict from pathlib import Path from typing import Annotated, TypeVar, assert_never -from urllib.parse import unquote, urlsplit +from urllib.parse import urlsplit +from urllib.request import url2pathname import typer from sqlalchemy.engine import make_url @@ -33,13 +35,16 @@ from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile +from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.portability import ( + BundleCompatibilityError, BundleConflictError, BundleFormatError, BundleInspection, BundleProgress, BundleReceipt, + BundleRecoveryRequiredError, BundleValidation, PortableBundleService, ) @@ -50,6 +55,7 @@ from powercontext.server.configuration import server_settings_context HELP_OPTION_NAMES = ("-h", "--help") +_SQLITE_TARGET_INSPECTION_ERROR = "cannot inspect configured SQLite target" _ResultT = TypeVar("_ResultT", BundleInspection, BundleValidation, BundleReceipt) archive_app = typer.Typer( @@ -182,18 +188,39 @@ async def projection_capability(_scopes: tuple[str, ...], /) -> None: supported_artifact_families=artifacts.families, artifact_content_decoder=artifacts.decode_content, candidate_decoder=CandidateRepository(BUILTIN_CANDIDATE_PROPOSAL_TYPES).decode_candidate, + source_decoder=SourceRepository(BUILTIN_SOURCE_REGISTRY).decode_source, ) return await archive.validate(source, progress=_emit_progress) def _database_file_exists(value: str, /) -> bool: - url = make_url(value) - arguments, options = url.get_dialect()().create_connect_args(url) - filename = str(arguments[0]) - if options.get("uri") and filename.startswith("file:"): - uri = urlsplit(filename.replace("\t", "%09").replace("\n", "%0A").replace("\r", "%0D")) - filename = unquote(uri.path).partition("\0")[0] - return Path(filename).is_file() + authority = "" + try: + url = make_url(value) + arguments, options = url.get_dialect()().create_connect_args(url) + filename = str(arguments[0]) + if options.get("uri") and filename.startswith("file:"): + uri = urlsplit(filename.replace("\t", "%09").replace("\n", "%0A").replace("\r", "%0D")) + authority = uri.netloc + filename = url2pathname(uri.path).partition("\0")[0] + except (OSError, ValueError) as error: + raise BundleFormatError(_SQLITE_TARGET_INSPECTION_ERROR) from error + if authority not in {"", "localhost"} or not filename: + raise BundleFormatError(_SQLITE_TARGET_INSPECTION_ERROR) + target = Path(filename) + for candidate in (target, *target.parents): + try: + mode = candidate.stat().st_mode + except FileNotFoundError: + continue + except (OSError, ValueError) as error: + raise BundleFormatError(_SQLITE_TARGET_INSPECTION_ERROR) from error + if candidate == target and stat.S_ISREG(mode): + return True + if candidate != target and stat.S_ISDIR(mode): + return False + raise BundleFormatError(_SQLITE_TARGET_INSPECTION_ERROR) + raise BundleFormatError(_SQLITE_TARGET_INSPECTION_ERROR) def _emit_progress(value: BundleProgress, /) -> None: @@ -211,12 +238,18 @@ def _emit_archive_result( try: _emit(asyncio.run(_run_archive(operation, env_file=env_file, recovery=recovery))) + except BundleCompatibilityError as error: + typer.echo(f"error: archive target is incompatible: {error}", err=True) + raise typer.Exit(code=3) from error except BundleFormatError as error: typer.echo(f"error: archive validation failed: {error}", err=True) raise typer.Exit(code=2) from error except BundleConflictError as error: typer.echo(f"error: archive restore conflict: {error}", err=True) raise typer.Exit(code=3) from error + except BundleRecoveryRequiredError as error: + typer.echo(f"error: archive recovery required: {error}", err=True) + raise typer.Exit(code=3) from error def _emit_validation_result(source: Path, /, *, env_file: Path | None) -> None: diff --git a/tests/builtin/persistence/test_records.py b/tests/builtin/persistence/test_records.py index 5d1166cc83..cdefb0f556 100644 --- a/tests/builtin/persistence/test_records.py +++ b/tests/builtin/persistence/test_records.py @@ -463,14 +463,14 @@ async def scenario() -> None: await records.create_source("scope-a", "content", f"{index}:" + "x" * 1_100_000) decoded = 0 - original_decode = sources._decode_row + original_decode = sources.decode_source def counting_decode(row): nonlocal decoded decoded += 1 return original_decode(row) - monkeypatch.setattr(sources, "_decode_row", counting_decode) + monkeypatch.setattr(sources, "decode_source", counting_decode) first = await records.list_sources("scope-a", limit=8, cursor=None, caller="user:one") assert len(first.items) == 3 diff --git a/tests/builtin/portability/test_bundle.py b/tests/builtin/portability/test_bundle.py index ab2b2b75f7..ebdb93d891 100644 --- a/tests/builtin/portability/test_bundle.py +++ b/tests/builtin/portability/test_bundle.py @@ -31,6 +31,7 @@ from powercontext.builtin.artifacts.skill import capture_skill_directory from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.sources import SourceRepository from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import ( ARTIFACT_HEADS_TABLE, @@ -51,6 +52,18 @@ TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE, ) from powercontext.builtin.portability import BundleConflictError, BundleFormatError, PortableBundleService +from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY, ContentSource +from powercontext.sources import SourceMaterialization + +_SOURCES = SourceRepository(BUILTIN_SOURCE_REGISTRY) + + +def _content_payload(source_id: str, content: str) -> bytes: + return ( + ContentSource(name=source_id, materialization=SourceMaterialization.CAPTURED, content=content) + .model_dump_json() + .encode() + ) async def _authorize_export(_scopes: tuple[str, ...], /) -> None: @@ -72,6 +85,45 @@ async def _insert_scope(connection, scope_id: str = "project:one") -> None: ) +@pytest.mark.parametrize("suffix", ["", "-wal", "-shm", "-journal"]) +@pytest.mark.parametrize("output_name", ["source.db", "SOURCE.DB"]) +def test_export_cannot_replace_its_sqlite_database_or_journals(tmp_path: Path, suffix: str, output_name: str) -> None: + async def scenario() -> None: + database = tmp_path / "source.db" + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{database}") + async with SQLiteProfile.open(config, tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + with pytest.raises(BundleFormatError, match="source database"): + await PortableBundleService(source.database).export( + ["project:one"], tmp_path / (output_name + suffix), authorize=_authorize_export + ) + assert database.read_bytes().startswith(b"SQLite format 3\0") + async with ( + SQLiteProfile.open(config, tables=BUILTIN_TABLES) as reopened, + reopened.database.transaction() as connection, + ): + assert await connection.scalar(select(SCOPES_TABLE.c.scope_id)) == "project:one" + + asyncio.run(scenario()) + + +def test_export_preserves_its_callers_sqlite_transaction(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "snapshot.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await PortableBundleService(source.database).export( + ["project:one"], archive, authorize=_authorize_export + ) + assert await connection.scalar(select(SCOPES_TABLE.c.scope_id)) == "project:one" + async with source.database.transaction() as connection: + assert await connection.scalar(select(SCOPES_TABLE.c.scope_id)) == "project:one" + + asyncio.run(scenario()) + + @pytest.mark.parametrize("mutation", ["identity_override", "unknown_field", "missing_field", "wrong_type", "scope"]) def test_restore_rejects_invalid_wire_fields_before_writing(tmp_path: Path, mutation: str) -> None: async def scenario() -> None: @@ -125,7 +177,7 @@ async def scenario() -> None: scope_id="project:one", source_type="content", source_id="source-1", - payload=b'{"name":"source-1"}', + payload=_content_payload("source-1", "Original captured content."), journal_position=1, ) ) @@ -155,7 +207,7 @@ async def scenario() -> None: async with SQLiteProfile.open( SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES ) as target: - service = PortableBundleService(target.database) + service = PortableBundleService(target.database, source_decoder=_SOURCES.decode_source) inspection = await service.validate(archive, supported_source_types=("content",)) assert inspection.scopes == ("project:one",) restored = await service.restore(archive, supported_source_types=("content",)) @@ -291,7 +343,7 @@ async def scenario() -> None: scope_id="project:one", source_type="content", source_id="source-1", - payload=b"original", + payload=_content_payload("source-1", "original"), journal_position=1, ) ) @@ -306,18 +358,20 @@ async def scenario() -> None: scope_id="project:one", source_type="content", source_id="source-1", - payload=b"different", + payload=_content_payload("source-1", "different"), journal_position=1, ) ) - service = PortableBundleService(target.database) + service = PortableBundleService(target.database, source_decoder=_SOURCES.decode_source) validation = await service.validate(archive) assert validation.compatible is False assert validation.conflicts == 1 with pytest.raises(BundleConflictError): await service.restore(archive) async with target.database.transaction() as connection: - assert await connection.scalar(select(SOURCES_TABLE.c.payload)) == b"different" + assert await connection.scalar(select(SOURCES_TABLE.c.payload)) == _content_payload( + "source-1", "different" + ) asyncio.run(scenario()) @@ -340,7 +394,7 @@ async def scenario() -> None: scope_id="project:one", source_type="content", source_id="source-1", - payload=b"new source", + payload=_content_payload("source-1", "new source"), journal_position=1, ) ) @@ -354,7 +408,7 @@ async def scenario() -> None: "CREATE TRIGGER reject_restore BEFORE INSERT ON pc_sources " "BEGIN SELECT RAISE(ABORT, 'write rejected after preflight'); END" ) - service = PortableBundleService(target.database) + service = PortableBundleService(target.database, source_decoder=_SOURCES.decode_source) validation = await service.validate(archive) assert validation.compatible with pytest.raises(BundleConflictError, match="target constraint conflict"): @@ -486,7 +540,7 @@ async def scenario() -> None: scope_id="project:one", source_type="content", source_id="source-1", - payload=b"source", + payload=_content_payload("source-1", "source"), journal_position=1, ) ) @@ -516,7 +570,9 @@ async def scenario() -> None: output_archive.writestr("records.ndjson", b"\n".join(records) + b"\n") async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: - receipt = await PortableBundleService(target.database).restore(reordered) + receipt = await PortableBundleService(target.database, source_decoder=_SOURCES.decode_source).restore( + reordered + ) assert receipt.inserted == 5 asyncio.run(scenario()) @@ -529,7 +585,7 @@ async def scenario() -> None: archive = tmp_path / "large.pcb" source_path = tmp_path / "source.db" target_path = tmp_path / "target.db" - payload = b"x" * 8_192 + payload = "x" * 8_192 async with SQLiteProfile.open( SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES ) as source: @@ -545,7 +601,7 @@ async def scenario() -> None: "scope_id": "project:one", "source_type": "content", "source_id": f"source-{number}", - "payload": payload, + "payload": _content_payload(f"source-{number}", payload), "journal_position": number, } for number in range(1, 1_501) @@ -558,7 +614,9 @@ async def scenario() -> None: async with SQLiteProfile.open( SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES ) as target: - receipt = await PortableBundleService(target.database).restore(archive) + receipt = await PortableBundleService(target.database, source_decoder=_SOURCES.decode_source).restore( + archive + ) _, peak = tracemalloc.get_traced_memory() finally: tracemalloc.stop() diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index e580fd1c2a..c21eae955c 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -53,8 +53,12 @@ SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE, ) -from powercontext.builtin.persistence.topic_memory import TopicMemoryStorageInvariantError -from powercontext.builtin.portability import BundleConflictError, BundleFormatError, PortableBundleService +from powercontext.builtin.portability import ( + BundleConflictError, + BundleFormatError, + BundleRecoveryRequiredError, + PortableBundleService, +) from powercontext.builtin.records import ArtifactWrite from powercontext.builtin.review.models import CandidateAudit, CandidateStatus from powercontext.builtin.runtime import ( @@ -69,10 +73,12 @@ open_builtin_contexts, open_builtin_runtime, ) -from powercontext.builtin.scope import ScopeDraft +from powercontext.builtin.runtime.artifact_processing import ArtifactProcessingBinding +from powercontext.builtin.scope import ScopeDraft, ScopeExternalReference from powercontext.builtin.sources import ContentCapture from powercontext.builtin.tags import ArtifactTagTarget, TagFilter from powercontext.builtin.work import AcknowledgeHandoff, ReceiverChecks +from powercontext.limits import MAX_ARTIFACT_ID_LENGTH from powercontext.sources import ( AdapterSourceDefinition, SourceDefinitionRegistry, @@ -105,8 +111,12 @@ async def scenario() -> None: await source.portability.export([scope.scope_id], archive, authorize=authorize) async with open_builtin_contexts(target_config) as target: with pytest.raises(RuntimeError, match="interrupted"): - await PortableBundleService(target.database, projection_rebuilder=fail_projection).restore(archive) - with pytest.raises(TopicMemoryStorageInvariantError): + await PortableBundleService( + target.database, + projection_rebuilder=fail_projection, + source_decoder=target.repositories.sources.decode_source, + ).restore(archive) + with pytest.raises(BundleRecoveryRequiredError): async with open_builtin_contexts(target_config): pass async with open_builtin_contexts(target_config, _archive_recovery=True) as target: @@ -201,6 +211,181 @@ async def authorize(scopes: tuple[str, ...]) -> None: asyncio.run(scenario()) +def test_archive_recovers_canonical_default_scope_into_a_fresh_deployment( + tmp_path: Path, target_config: BuiltinConfig +) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "default.pcb" + async with open_builtin_runtime(BuiltinConfig()) as source: + assert source.scopes is not None + default = await source.scopes.default_scope() + assert default is not None + capture = CaptureSource( + source_id="default-evidence", content="Restore the original default scope.", metadata={} + ) + await source.sources.for_scope(default.scope_id).capture(capture) + expected = await source.records.for_scope(default.scope_id).get_source("content", capture.source_id) + assert source.archive is not None + exported = await source.archive.export([default.scope_id], archive, authorize=authorize) + async with open_builtin_runtime(target_config, _archive_recovery=True) as target: + assert target.scopes is not None + assert await target.scopes.default_scope() is None + assert target.archive is not None + assert (await target.archive.restore(archive)).projections_ready + assert (await target.archive.restore(archive)).inserted == 0 + async with open_builtin_runtime(target_config) as target: + assert target.scopes is not None + restored = await target.scopes.default_scope() + assert restored is not None + assert restored.scope_id == default.scope_id + assert ( + await target.records.for_scope(restored.scope_id).get_source("content", capture.source_id) == expected + ) + assert target.archive is not None + reexported = await target.archive.export([restored.scope_id], tmp_path / "again.pcb", authorize=authorize) + assert reexported.total_digest == exported.total_digest + + asyncio.run(scenario()) + + +def test_archive_maintenance_runtime_does_not_start_background_processing() -> None: + class NeverLaunch: + async def start(self, assignment): + message = "Maintenance must not launch workers" + raise AssertionError(message) + + async def scenario() -> None: + binding = ArtifactProcessingBinding( + binding_name="archive-maintenance", artifact_family="skill", launcher=NeverLaunch() + ) + async with open_builtin_runtime( + BuiltinConfig(), _archive_recovery=True, artifact_processing_bindings=(binding,) + ) as runtime: + assert runtime.artifact_processing_supervisor is None + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("family", ["atomic-memory", "experience"]) +def test_interrupted_archive_restore_blocks_contexts_runtime_and_workers( + tmp_path: Path, family: str, target_config: BuiltinConfig +) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def interrupted(_scopes: tuple[str, ...]) -> None: + message = "interrupted projection rebuild" + raise RuntimeError(message) + + async def scenario() -> None: + archive = tmp_path / "interrupted.pcb" + content: dict[str, JsonValue] = ( + {"kind": "rule", "text": "Retain recovery evidence."} + if family == "atomic-memory" + else {"situation": "Recovery", "action": "Retry", "outcome": "Ready", "lesson": "Verify projections."} + ) + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Incomplete recovery", summary="Do not serve", idempotency_key="incomplete") + ) + artifact = await source.records.create_artifact(scope.scope_id, family, ArtifactWrite(content=content)) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + async with open_builtin_contexts(target_config) as target: + with pytest.raises(RuntimeError, match="interrupted projection rebuild"): + await PortableBundleService( + target.database, + projection_rebuilder=interrupted, + source_decoder=target.repositories.sources.decode_source, + ).restore(archive) + for opener in (open_builtin_contexts, open_builtin_runtime): + with pytest.raises(RuntimeError, match="portable restore projections are incomplete"): + async with opener(target_config): + pass + for recovery in (False, True): + with pytest.raises(BundleRecoveryRequiredError): + async with open_builtin_contexts(target_config, _topic_memory_worker=True, _archive_recovery=recovery): + pass + async with open_builtin_contexts(target_config, _archive_recovery=True) as target: + assert (await target.portability.restore(archive)).projections_ready + async with open_builtin_runtime(target_config) as target: + restored = await target.records.for_scope(scope.scope_id).get_artifact(family, artifact.artifact_id) + assert restored.artifact_id == artifact.artifact_id + if family == "atomic-memory": + assert target.atomic_memory is not None + found = await target.atomic_memory.for_scope(scope.scope_id).search("recovery evidence") + assert found.hits + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("legacy", [False, True]) +@pytest.mark.parametrize("damage", ["payload", "identity", "definition", "capability_override"]) +def test_archive_rejects_unreadable_native_sources_before_writing(tmp_path: Path, monkeypatch, legacy, damage) -> None: + from typer.testing import CliRunner + + from powercontext.cli.app import create_cli + from powercontext.cli.archive import archive_app + + async def scenario() -> Path: + archive = tmp_path / "native.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Native source", summary="Read contract", idempotency_key="native-source") + ) + context = await source.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="evidence", content="Original captured evidence.")) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + + def rewrite(records): + record = next(item for item in records if item["record_type"] == "source") + envelope = json.loads(base64.b64decode(record["payload"]["payload"]["base64"])) + value = dict(envelope["value"]) + if damage in {"payload", "capability_override"}: + value = {} + elif damage == "identity": + value["name"] = "different-source-id" + else: + value["definition_version"] = "unsupported" + payload = value if legacy else {**envelope, "value": value} + record["payload"]["payload"] = {"base64": base64.b64encode(json.dumps(payload).encode()).decode()} + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + service = target.portability + supported_source_types = None + if damage == "capability_override": + service = PortableBundleService( + target.database, + supported_source_types=(), + source_decoder=target.repositories.sources.decode_source, + ) + supported_source_types = ("content",) + async with target.database.transaction() as connection: + before = (await connection.execute(select(SCOPES_TABLE))).all() + for operation in (service.validate, service.restore): + with pytest.raises(BundleFormatError, match="Source"): + await operation(archive, supported_source_types=supported_source_types) + async with target.database.transaction() as connection: + assert (await connection.execute(select(SCOPES_TABLE))).all() == before + assert (await connection.execute(select(SOURCES_TABLE))).all() == [] + return archive + + archive = asyncio.run(scenario()) + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "unused-target")) + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "restore", str(archive), "--dry-run"]) + assert result.exit_code == 2, result.output + assert "Source" in result.output + assert not (tmp_path / "unused-target" / "powercontext.db").exists() + + def test_archive_restores_tagged_memory_vector_search_and_survives_restart( tmp_path: Path, target_config: BuiltinConfig ) -> None: @@ -251,8 +436,10 @@ def test_archive_restores_package_files_and_topic_search_after_restart( async def scenario() -> None: package_dir = tmp_path / "portable" package_dir.mkdir() - (package_dir / "SKILL.md").write_text("---\nname: portable\ndescription: Portable guide\n---\nRead guide.md.\n") - (package_dir / "guide.md").write_text("Exact portable instructions.\n") + (package_dir / "SKILL.md").write_text( + "---\nname: portable\ndescription: Portable guide\n---\nRead guide.md.\n", encoding="utf-8", newline="\n" + ) + (package_dir / "guide.md").write_text("Exact portable instructions.\n", encoding="utf-8", newline="\n") package = capture_skill_directory(package_dir) archive = tmp_path / "families.pcb" source_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'source.db'}")) @@ -531,7 +718,19 @@ async def authorize(_scopes: tuple[str, ...]) -> None: asyncio.run(scenario()) -@pytest.mark.parametrize("damage", ["missing_definition", "fingerprint", "identity", "native"]) +@pytest.mark.parametrize( + "damage", + [ + "missing_definition", + "fingerprint", + "identity", + "native", + "envelope", + "observation_payload", + "undeclared_projection", + "nonstandard_json", + ], +) def test_restore_rejects_invalid_remote_source_dependencies_before_writing(tmp_path: Path, damage: str) -> None: async def scenario() -> None: @@ -566,6 +765,19 @@ async def authorize(_scopes: tuple[str, ...]) -> None: envelope["value"]["definition_fingerprint"] = "sha256:" + "0" * 64 elif damage == "identity": record["identity"]["source_id"] = "another" + elif damage == "envelope": + envelope["unexpected"] = "not part of the stored Source envelope" + elif damage in {"observation_payload", "undeclared_projection", "nonstandard_json"}: + missing_body = dict(envelope["value"]["payload"]) + missing_body.pop("body") + changes = { + "observation_payload": {"payload": missing_body}, + "undeclared_projection": { + "projections": [{"key": {"name": "undeclared", "version": "1"}, "value": {}}] + }, + "nonstandard_json": {"payload": {**envelope["value"]["payload"], "nonstandard": float("nan")}}, + } + envelope["value"].update(changes[damage]) else: envelope["representation"] = "native" envelope["value"] = envelope["value"]["payload"] @@ -1141,6 +1353,154 @@ def rewrite(records): asyncio.run(scenario()) +@pytest.mark.parametrize("damage", ["long_identity", "large_integer", "missing_replacement", "self_replacement"]) +def test_archive_rejects_unusable_artifact_and_scope_rows_before_writing(tmp_path: Path, damage: str) -> None: + async def scenario() -> None: + archive = tmp_path / "unusable.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Storage bounds", summary="Portable values", idempotency_key="storage-bounds") + ) + await source.records.create_artifact( + scope.scope_id, "atomic-memory", ArtifactWrite(content={"kind": "fact", "text": "Portable evidence."}) + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + + def rewrite(records): + if damage == "long_identity": + for record in records: + if record["record_type"] in {"artifact_revision", "artifact_head", "artifact_lineage_source"}: + record["identity"]["artifact_id"] = "a" * (MAX_ARTIFACT_ID_LENGTH + 1) + elif damage == "large_integer": + next(record for record in records if record["record_type"] == "scope")["payload"]["version"] = 1 << 31 + else: + head = next(record for record in records if record["record_type"] == "artifact_head") + replacement = "missing" if damage == "missing_replacement" else head["identity"]["artifact_id"] + head["payload"].update(lifecycle_state="deprecated", replacement_artifact_id=replacement) + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar( + select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == scope.scope_id) + ) + is None + ) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("encoding", ["base64", "timestamp", "offset_timestamp"]) +def test_archive_rejects_wire_values_that_cannot_roundtrip(tmp_path: Path, encoding: str) -> None: + async def scenario() -> None: + archive = tmp_path / "canonical.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Canonical", summary="Stable retries", idempotency_key="canonical") + ) + context = await source.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="one", content="Canonical evidence.")) + await source.profiles.put_policy(scope.scope_id, generation_enabled=True, expected_version=0) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + + def rewrite(records): + if encoding != "base64": + policy = next(item for item in records if item["record_type"] == "profile_policy") + timestamp = policy["payload"]["updated_at"] + policy["payload"]["updated_at"] = ( + timestamp.replace("T", " ") if encoding == "timestamp" else timestamp + "Z" + ) + else: + source = next(item for item in records if item["record_type"] == "source") + payload = base64.b64decode(source["payload"]["payload"]["base64"]) + while len(payload) % 3 == 0: + payload += b" " + encoded = base64.b64encode(payload).decode() + alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + offset = len(encoded.rstrip("=")) - 1 + changed = encoded[:offset] + alphabet[alphabet.index(encoded[offset]) | 1] + encoded[offset + 1 :] + assert base64.b64decode(changed, validate=True) == payload + source["payload"]["payload"] = {"base64": changed} + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="canonical"): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar( + select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == scope.scope_id) + ) + is None + ) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("damage", ["cycle", "external_digest"]) +def test_archive_validates_scope_relationships_before_writing(tmp_path: Path, damage: str) -> None: + async def scenario() -> None: + archive = tmp_path / "scopes.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + parent = await source.scopes.create( + ScopeDraft( + title="Parent", + summary="Scope relationships", + idempotency_key="parent", + external_references=(ScopeExternalReference(kind="repository", value="example"),), + ) + ) + child = await source.scopes.create( + ScopeDraft( + title="Child", + summary="Scope relationships", + parent_scope_id=parent.scope_id, + idempotency_key="child", + ) + ) + await source.portability.export([parent.scope_id, child.scope_id], archive, authorize=authorize) + + def rewrite(records): + if damage == "cycle": + parent_record = next( + item + for item in records + if item["record_type"] == "scope" and item["identity"]["scope_id"] == parent.scope_id + ) + parent_record["payload"]["parent_scope_id"] = child.scope_id + else: + reference = next(item for item in records if item["record_type"] == "scope_external_reference") + reference["payload"]["value_digest"] = "0" * 64 + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError): + await operation(archive) + + asyncio.run(scenario()) + + def test_archive_exports_migrated_scopes_without_deleting_offline_history(tmp_path: Path) -> None: async def authorize(_scopes: tuple[str, ...]) -> None: pass diff --git a/tests/test_archive_cli.py b/tests/test_archive_cli.py index 889ef4a73a..01f724fd7e 100644 --- a/tests/test_archive_cli.py +++ b/tests/test_archive_cli.py @@ -14,6 +14,9 @@ import asyncio import json +import struct +import zipfile +from pathlib import Path import pytest from click.utils import strip_ansi @@ -28,6 +31,34 @@ from powercontext.cli.app import create_cli from powercontext.cli.archive import archive_app from powercontext.paths import default_database_path, sqlite_url +from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry +from tests.builtin.persistence.contract import NoteAdapter, NoteInput + + +def test_archive_cli_restore_reports_unsupported_native_sources_consistently(tmp_path, monkeypatch) -> None: + archive = tmp_path / "native.pcb" + + async def seed() -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter()),)) + async with open_builtin_contexts(BuiltinConfig(), source_registry=registry) as source: + scope = await source.scopes.create( + ScopeDraft(title="Native", summary="Missing target adapter", idempotency_key="native") + ) + native = await registry.resolve(NoteInput(note_id="one", body="Native evidence")) + async with source.database.transaction() as connection: + await source.repositories.sources.add(connection, scope.scope_id, native) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + + asyncio.run(seed()) + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "target")) + runner = CliRunner() + cli = create_cli([archive_app]) + for confirmation in ("--dry-run", "--yes"): + result = runner.invoke(cli, ["archive", "restore", str(archive), confirmation]) + assert result.exit_code == 3, result.output def test_archive_cli_exposes_a_safe_restore_confirmation_and_dry_run() -> None: @@ -47,6 +78,52 @@ def test_archive_cli_exposes_a_safe_restore_confirmation_and_dry_run() -> None: assert "--yes" in restore_output +@pytest.mark.parametrize( + ("field", "value"), + [("record_type", []), ("record_type", {}), ("schema_version", True), ("format_version", True)], +) +def test_archive_cli_normalizes_invalid_wire_types(tmp_path, field, value) -> None: + archive = tmp_path / "invalid.pcb" + + async def seed() -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Wire types", summary="Reject malformed input", idempotency_key="wire-types") + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + + asyncio.run(seed()) + with zipfile.ZipFile(archive) as bundle: + manifest = json.loads(bundle.read("manifest.json")) + records = [json.loads(line) for line in bundle.read("records.ndjson").splitlines()] + if field == "format_version": + manifest[field] = value + else: + records[0][field] = value + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("manifest.json", json.dumps(manifest)) + bundle.writestr("records.ndjson", "\n".join(json.dumps(record) for record in records)) + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "inspect", str(archive)]) + assert result.exit_code == 2, (result.output, result.exception) + assert "archive validation failed" in result.output + + +@pytest.mark.parametrize("operation", [("inspect",), ("restore", "--dry-run"), ("restore", "--yes")]) +def test_archive_cli_rejects_unsupported_zip_versions(tmp_path, monkeypatch, operation) -> None: + archive = tmp_path / "unsupported.pcb" + entry = zipfile.ZipInfo("manifest.json") + entry.extract_version = 64 + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr(entry, b"{}") + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "target")) + result = CliRunner().invoke(create_cli([archive_app]), ["archive", *operation, str(archive)]) + assert result.exit_code == 2, (result.output, result.exception) + assert "archive validation failed" in result.output + + def test_archive_cli_can_export_inspect_and_dry_run_a_scope(tmp_path, monkeypatch) -> None: monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "data")) output = tmp_path / "scope.pcb" @@ -73,6 +150,125 @@ async def default_scope_id() -> str: assert not (target_home / "powercontext.db").exists() assert inspected.exit_code == 0, inspected.output assert validated.exit_code == 0, validated.output + restored = runner.invoke(cli, ["archive", "restore", str(output), "--yes"]) + assert restored.exit_code == 0, restored.output + repeated = runner.invoke(cli, ["archive", "restore", str(output), "--yes"]) + assert repeated.exit_code == 0, repeated.output + assert json.loads(repeated.stdout)["inserted"] == 0 + assert asyncio.run(default_scope_id()) == scope_id + + +@pytest.mark.parametrize("entry", ["manifest.json", "records.ndjson"]) +@pytest.mark.parametrize("operation", ["inspect", "dry-run", "restore"]) +def test_archive_cli_rejects_invalid_deflate_streams(tmp_path, monkeypatch, entry, operation) -> None: + archive = tmp_path / "damaged.pcb" + + async def seed() -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as contexts: + scope = await contexts.scopes.create( + ScopeDraft(title="Compression", summary="Corrupt stream", idempotency_key="compression") + ) + await contexts.portability.export([scope.scope_id], archive, authorize=authorize) + + asyncio.run(seed()) + with zipfile.ZipFile(archive) as bundle: + header_offset = bundle.getinfo(entry).header_offset + data = bytearray(archive.read_bytes()) + filename_size, extra_size = struct.unpack_from(" None: + archive = tmp_path / "scope.pcb" + target = tmp_path / "target" + target.mkdir() + + async def seed() -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as contexts: + scope = await contexts.scopes.create( + ScopeDraft(title="Inspection", summary="Fail closed", idempotency_key="inspection") + ) + await contexts.portability.export([scope.scope_id], archive, authorize=authorize) + + asyncio.run(seed()) + if failure.startswith("parent-file"): + target.rmdir() + target.write_text("not a directory", encoding="utf-8") + target = target / "target.db" + if failure in {"permission", "parent-file-as-missing"}: + original_stat = Path.stat + + def inaccessible(path, *arguments, **keywords): + if path == target: + if failure == "permission": + raise PermissionError + raise FileNotFoundError + return original_stat(path, *arguments, **keywords) + + monkeypatch.setattr(Path, "stat", inaccessible) + url = f"sqlite+aiosqlite:///{target}" + if failure == "authority": + url = f"sqlite+aiosqlite:///file://other-host/{target.as_posix().lstrip('/')}?uri=true" + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_URL", url) + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "restore", str(archive), "--dry-run"]) + assert result.exit_code == 2, result.output + assert "cannot inspect configured SQLite target" in result.output + + +def test_archive_cli_requires_projection_recovery_before_export(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "target")) + archive = tmp_path / "recovery.pcb" + output = tmp_path / "exported.pcb" + + async def seed() -> str: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def interrupted(_scopes: tuple[str, ...]) -> None: + raise RuntimeError("interrupted") + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Recovery", summary="Incomplete projections", idempotency_key="incomplete") + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + config = BuiltinConfig(database=SQLiteConfig(url=sqlite_url(default_database_path()))) + async with open_builtin_contexts(config) as target: + with pytest.raises(RuntimeError, match="interrupted"): + await archive_module.PortableBundleService(target.database, projection_rebuilder=interrupted).restore( + archive + ) + return scope.scope_id + + scope_id = asyncio.run(seed()) + cli = create_cli([archive_app]) + runner = CliRunner() + arguments = ["archive", "export", "--scope-id", scope_id, "--output", str(output)] + blocked = runner.invoke(cli, arguments) + assert blocked.exit_code == 3, blocked.output + assert "archive recovery required" in blocked.output + assert not output.exists() + recovered = runner.invoke(cli, ["archive", "restore", str(archive), "--yes"]) + assert recovered.exit_code == 0, recovered.output + exported = runner.invoke(cli, arguments) + assert exported.exit_code == 0, exported.output def test_archive_cli_reports_corrupt_or_missing_bundles_without_a_traceback() -> None: diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index c313ae7962..b4933a7934 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -21,8 +21,8 @@ from pydantic import ValidationError from referencing.exceptions import Unresolvable -from powercontext.builtin.runtime.relational import _json_schema_validator from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION, ContentCapture +from powercontext.builtin.sources.observation_validation import json_schema_validator from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, Source, @@ -96,7 +96,7 @@ def reject_request(*_args: object, **_kwargs: object) -> None: raise OSError("network access is forbidden") # noqa: TRY003 monkeypatch.setattr(urllib.request, "urlopen", reject_request) - validator = _json_schema_validator("remote", {"$ref": "http://127.0.0.1:9/schema"}) + validator = json_schema_validator("remote", {"$ref": "http://127.0.0.1:9/schema"}) with pytest.raises(Unresolvable): validator.validate({}) From 5cec720fda55f72de75d79cd8db85aa5954b4955 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Wed, 21 Oct 2026 21:48:11 +0800 Subject: [PATCH 30/32] fix(archive): validate tag labels against normalized keys --- .../builtin/portability/bundle.py | 9 ++- tests/builtin/portability/test_bundle.py | 4 +- tests/e2e/test_portable_archive.py | 74 ++++++++++++++++++- 3 files changed, 82 insertions(+), 5 deletions(-) diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index 31caa036ce..fdff002d63 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -107,12 +107,13 @@ SOURCES_TABLE, TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE, ) -from powercontext.builtin.records import ArtifactRevisionActor +from powercontext.builtin.records import ArtifactRevisionActor, InvalidBaseAccessRequestError from powercontext.builtin.scope.models import ScopeExternalReference from powercontext.builtin.sources.observation_validation import ( validate_source_definition_manifest, validate_source_observation, ) +from powercontext.builtin.tags import normalize_tags from powercontext.errors import SourceError from powercontext.sources import SourceDefinitionManifest, SourceObservation @@ -1545,6 +1546,12 @@ def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: raise BundleFormatError("artifact tag has invalid target") if identity["target_type"] == "memory_entry" and identity["family"] != "memory": raise BundleFormatError("artifact tag has invalid target") + try: + normalized = normalize_tags((payload["tag"],)) + except InvalidBaseAccessRequestError as error: + raise BundleFormatError("artifact tag has invalid label") from error + if identity["tag_key"] not in normalized: + raise BundleFormatError("artifact tag label does not match its normalized key") if _database_value(payload["tag_key_hash"]) != hashlib.sha256(str(identity["tag_key"]).encode()).digest(): raise BundleFormatError("artifact tag has invalid key hash") diff --git a/tests/builtin/portability/test_bundle.py b/tests/builtin/portability/test_bundle.py index ebdb93d891..8968cfaa80 100644 --- a/tests/builtin/portability/test_bundle.py +++ b/tests/builtin/portability/test_bundle.py @@ -792,8 +792,8 @@ async def scenario() -> None: artifact_id="portable", target_type="artifact", target_id="portable", - tag_key_hash=hashlib.sha256(b"team").digest(), - tag_key="team", + tag_key_hash=hashlib.sha256(b"platform").digest(), + tag_key="platform", tag="platform", assigned_at=now, ) diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index 658a3729aa..213211792e 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -524,7 +524,7 @@ async def scenario() -> None: tag_target = ArtifactTagTarget(family="atomic-memory", artifact_id=memory.artifact_id) empty = await source.records.get_tags(scope.scope_id, tag_target) tagged = await source.records.replace_tags( - scope.scope_id, tag_target, ("archive",), expected_etag=empty.etag + scope.scope_id, tag_target, ("ARCHIVE", "Straße", "Café", "中文"), expected_etag=empty.etag ) async def authorize(_scopes: tuple[str, ...]) -> None: @@ -538,12 +538,82 @@ async def authorize(_scopes: tuple[str, ...]) -> None: async with open_builtin_contexts(target_config, embedding_model=Embedding()) as target: service = target.atomic_memory.for_scope(scope.scope_id) for mode in ("text", "vector", "hybrid"): - result = await service.search("portable", mode=mode, tag_filter=TagFilter(tags=("archive",))) + result = await service.search( + "portable", mode=mode, tag_filter=TagFilter(tags=("archive", "STRASSE", "Café", "中文")) + ) assert [hit.hit.artifact_ref.artifact_id for hit in result.hits] == [memory.artifact_id] asyncio.run(scenario()) +@pytest.mark.parametrize( + ("field", "invalid"), + [ + ("tag", "Different"), + ("tag", ""), + ("tag", " archive"), + ("tag", "archive "), + ("tag", "a\x00b"), + ("tag", "\u0378"), + ("tag_key", "ARCHIVE"), + ("tag_key", "different"), + ], +) +def test_archive_rejects_inconsistent_tag_labels_before_writing(tmp_path: Path, field: str, invalid: str) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "tags.pcb" + config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Tags", summary="Preserve searchable labels", idempotency_key="tag-labels") + ) + memory = await source.records.create_artifact( + scope.scope_id, "atomic-memory", ArtifactWrite(content={"kind": "fact", "text": "Portable tags."}) + ) + tag_target = ArtifactTagTarget(family="atomic-memory", artifact_id=memory.artifact_id) + empty = await source.records.get_tags(scope.scope_id, tag_target) + tagged = await source.records.replace_tags( + scope.scope_id, tag_target, ("archive",), expected_etag=empty.etag + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + original = archive.read_bytes() + + def rewrite(records): + record = next(record for record in records if record["record_type"] == "artifact_tag") + if field == "tag": + record["payload"]["tag"] = invalid + else: + record["identity"]["tag_key"] = invalid + record["payload"]["tag_key_hash"] = { + "base64": base64.b64encode(hashlib.sha256(invalid.encode()).digest()).decode() + } + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(config) as target: + before = await target.scopes.list() + for operation in (target.portability.inspect, target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="artifact tag"): + await operation(archive) + assert await target.scopes.list() == before + archive.write_bytes(original) + assert (await target.portability.validate(archive)).compatible + assert (await target.portability.restore(archive)).projections_ready + assert await target.records.get_tags(scope.scope_id, tag_target) == tagged + async with open_builtin_contexts(config) as target: + assert (await target.portability.restore(archive)).inserted == 0 + assert await target.records.get_tags(scope.scope_id, tag_target) == tagged + hits = await target.atomic_memory.for_scope(scope.scope_id).search( + "portable", tag_filter=TagFilter(tags=tagged.tags) + ) + assert [hit.hit.artifact_ref.artifact_id for hit in hits.hits] == [memory.artifact_id] + + asyncio.run(scenario()) + + def test_archive_restores_package_files_and_topic_search_after_restart( tmp_path: Path, target_config: BuiltinConfig ) -> None: From 5b27e71419b626cde6d52ec14ec7e5c74050d16f Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Thu, 22 Oct 2026 15:53:38 +0800 Subject: [PATCH 31/32] fix(archive): validate publication provenance before restore --- docs/en/docs/operate/portable-archive.md | 8 + docs/zh/docs/operate/portable-archive.md | 5 + .../builtin/portability/bundle.py | 123 +++++++++-- tests/e2e/test_portable_archive.py | 194 +++++++++++++++++- 4 files changed, 312 insertions(+), 18 deletions(-) diff --git a/docs/en/docs/operate/portable-archive.md b/docs/en/docs/operate/portable-archive.md index 4df032c81b..c2979bcb52 100644 --- a/docs/en/docs/operate/portable-archive.md +++ b/docs/en/docs/operate/portable-archive.md @@ -101,6 +101,14 @@ This includes absolute `file:///C:/...` paths on Windows and percent-encoded fil unreadable targets, directory targets and non-directory parent paths fail validation rather than being treated as an empty database. +Publication validation follows exact Artifact revision addresses through the complete chain. Cycles and inconsistent +content digests within a chain are rejected before domain writes, even if all endpoints and checksums are valid. +Compatibility checks also compare existing target publication links for restored revisions. A target link absent from +the archive is a conflict, even if its chain is valid: accepting it could change the restored revision's original +provenance. The graph is indexed in temporary local storage, and target links are read in batches without scanning +unrelated publication histories. Conflicting target histories are reported without overwriting them; recovery from an +already invalid history requires a clean target and a valid backup. + Native Sources require their Python adapter on the target. Before writing records, validation uses the same registered Source decoder as normal reads to check both payload shape and indexed identity, including supported legacy payloads. Worker-materialized `SourceObservation` records carry their captured payload and projections. The same decoder diff --git a/docs/zh/docs/operate/portable-archive.md b/docs/zh/docs/operate/portable-archive.md index df096ab9f4..a3a5716cd6 100644 --- a/docs/zh/docs/operate/portable-archive.md +++ b/docs/zh/docs/operate/portable-archive.md @@ -85,6 +85,11 @@ Candidate version 和 decision head 使用正常读取路径的完整解码器 这包括 Windows 的绝对 `file:///C:/...` 路径及经过百分号编码的文件名。不支持的 URI authority、无法读取的目标、目录目标 或非目录的父路径会使验证失败，不会被当成空数据库。 +发布关系验证会沿精确的 Artifact Revision 地址检查完整来源链。即使所有端点和 checksum 都有效，链中的环和不一致的 +content digest 仍会在领域写入前被拒绝。兼容性检查也会比对待恢复 Revision 在目标中已有的发布关系。归档未声明的目标发布 +关系即使自身合法，也会作为冲突拒绝，防止恢复后的原始来源被改变。关系图使用本地临时存储建立索引，目标关系按批读取， +不扫描无关的发布历史。存在冲突时不会覆盖目标历史；已经损坏的历史需要使用有效备份恢复到干净目标。 + 原生 Source 需要目标环境安装对应的 Python adapter。验证会在写入前使用正常读取路径的已注册 Source 解码器，检查 payload 结构与索引 identity 是否一致，也覆盖受支持的 legacy payload。远程 worker 已物化的 `SourceObservation` 自带捕获内容和投影， 其存储 envelope 也使用同一解码器验证，但不需要加载 worker adapter。归档只携带所选 scope 实际引用的 Source Definition diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index fdff002d63..7ae7b4ee76 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -551,8 +551,8 @@ async def export( # Reject missing dependencies and size-limit breaches # without replacing a previous usable backup at the destination. _parse_bundle(archive_path) - _validate_dependencies(archive_path, candidate_decoder=self._candidate_decoder) - archive_path.replace(output) + with _validate_dependencies(archive_path, candidate_decoder=self._candidate_decoder): + archive_path.replace(output) return BundleReceipt(bundle_id=bundle_id, record_count=count, total_digest=digest) finally: records_path.unlink(missing_ok=True) @@ -575,19 +575,27 @@ async def validate( """Verify bundle integrity, dependencies, and optional adapter support.""" inspection = _parse_bundle(source, progress=progress, phase="validate") - required_sources, required_families = _validate_dependencies(source, candidate_decoder=self._candidate_decoder) - configured = ( - self._supported_source_types if supported_source_types is None else frozenset(supported_source_types) - ) - missing_sources = () if configured is None else tuple(sorted(required_sources - configured)) - missing_families = ( - () - if self._supported_artifact_families is None - else tuple(sorted(required_families - self._supported_artifact_families)) - ) - already_present, conflicts = await self._target_compatibility( - source, supported_source_types=configured, total=inspection.record_count, progress=progress - ) + with _validate_dependencies(source, candidate_decoder=self._candidate_decoder) as ( + required_sources, + required_families, + index, + ): + configured = ( + self._supported_source_types if supported_source_types is None else frozenset(supported_source_types) + ) + missing_sources = () if configured is None else tuple(sorted(required_sources - configured)) + missing_families = ( + () + if self._supported_artifact_families is None + else tuple(sorted(required_families - self._supported_artifact_families)) + ) + already_present, conflicts = await self._target_compatibility( + source, + supported_source_types=configured, + total=inspection.record_count, + progress=progress, + publication_index=index, + ) return BundleValidation( bundle_id=inspection.bundle_id, scopes=inspection.scopes, @@ -735,6 +743,7 @@ async def _target_compatibility( supported_source_types: frozenset[str] | None, total: int, progress: ProgressObserver | None, + publication_index: sqlite3.Connection, ) -> tuple[int, int]: already_present = 0 conflicts = 0 @@ -783,6 +792,12 @@ async def _target_compatibility( else: conflicts += 1 _report_progress(progress, "target", processed, total) + if ( + conflicts == 0 + and ARTIFACT_PUBLICATIONS_TABLE.name in existing_tables + and not await _validate_target_publications(connection, publication_index) + ): + conflicts += 1 return already_present, conflicts def _validate_target_content(self, record: _Record, /, *, supported_source_types: frozenset[str] | None) -> None: @@ -1689,9 +1704,10 @@ def _validate_safe_value(value: object, key: str | None = None) -> None: _validate_safe_value(child, key) +@contextmanager def _validate_dependencies( source: Path, /, *, candidate_decoder: Callable[[Mapping[str, Any]], object] | None = None -) -> tuple[set[str], set[str]]: +) -> Iterator[tuple[set[str], set[str], sqlite3.Connection]]: """Validate dependencies with disk-backed identities and bounded metadata.""" heads, packages = _dependency_metadata(source) @@ -1719,7 +1735,73 @@ def contains(record_type: str, identity: Mapping[str, object]) -> bool: if _source_observation(record) is None: required_sources.add(str(record.identity["source_type"])) required_families.update(_required_artifact_families((record,))) - return required_sources, required_families + _validate_publication_chains(index) + yield required_sources, required_families, index + + +def _index_publication_dependencies(index: sqlite3.Connection, record: _Record, /) -> None: + if record.record_type == "artifact_revision": + index.execute( + "INSERT OR IGNORE INTO publication_nodes (identity) VALUES (?)", + (_identity_key({"target_" + field: value for field, value in record.identity.items()}),), + ) + return + if record.record_type != "artifact_publication": + return + target = _identity_key(record.identity) + source = _identity_key({ + "target_" + field: record.payload["source_" + field] + for field in ("scope_id", "family", "artifact_id", "revision") + }) + index.execute( + "INSERT OR IGNORE INTO publication_edges (target, source, digest) VALUES (?, ?, ?)", + (target, source, record.payload["content_digest"]), + ) + + +def _validate_publication_chains(index: sqlite3.Connection, /) -> None: + if ( + index.execute( + "SELECT 1 FROM publication_edges AS edge JOIN publication_edges AS parent ON edge.source = parent.target " + "WHERE edge.digest != parent.digest LIMIT 1" + ).fetchone() + is not None + ): + raise BundleFormatError("publication chain has inconsistent content digests") + while pending := index.execute("SELECT target FROM publication_edges WHERE state = 0 LIMIT 1").fetchone(): + current = pending[0] + while edge := index.execute( + "SELECT source, state FROM publication_edges WHERE target = ?", (current,) + ).fetchone(): + if edge[1] == 1: + raise BundleFormatError("publication chain contains a cycle") + if edge[1] == 2: + break + index.execute("UPDATE publication_edges SET state = 1 WHERE target = ?", (current,)) + current = edge[0] + index.execute("UPDATE publication_edges SET state = 2 WHERE state = 1") + + +async def _validate_target_publications(connection: AsyncConnection, index: sqlite3.Connection, /) -> bool: + table = ARTIFACT_PUBLICATIONS_TABLE + fields = _SPECS["artifact_publication"].identity + pending = index.execute("SELECT identity FROM publication_nodes ORDER BY identity") + while batch := pending.fetchmany(100): + identities = [json.loads(row[0]) for row in batch] + rows = ( + await connection.execute( + select(table).where( + tuple_(*(table.c[field] for field in fields)).in_([ + tuple(identity[field] for field in fields) for identity in identities + ]) + ) + ) + ).mappings() + for row in rows: + target = _identity_key({field: row[field] for field in fields}) + if index.execute("SELECT 1 FROM publication_edges WHERE target = ?", (target,)).fetchone() is None: + return False + return True def _validate_observation_record(record: _Record, index: sqlite3.Connection, /) -> None: @@ -2151,6 +2233,12 @@ def _identity_index(source: Path, /) -> Iterator[sqlite3.Connection]: "CREATE TABLE identities (record_type TEXT NOT NULL, identity TEXT NOT NULL, document BLOB, " "PRIMARY KEY (record_type, identity))" ) + connection.execute( + "CREATE TABLE publication_edges (target TEXT PRIMARY KEY, source TEXT NOT NULL, digest TEXT NOT NULL, " + "state INTEGER NOT NULL DEFAULT 0)" + ) + connection.execute("CREATE INDEX publication_edges_state ON publication_edges(state, target)") + connection.execute("CREATE TABLE publication_nodes (identity TEXT PRIMARY KEY)") for record in _iter_records(source): try: connection.execute( @@ -2170,6 +2258,7 @@ def _identity_index(source: Path, /) -> Iterator[sqlite3.Connection]: ) except sqlite3.IntegrityError as error: raise BundleFormatError("bundle contains duplicate immutable or unique identity") from error + _index_publication_dependencies(connection, record) if record.record_type == "candidate_version": connection.execute( "INSERT INTO identities (record_type, identity) VALUES (?, ?)", diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index 213211792e..a09493dcc4 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -34,7 +34,7 @@ from sqlalchemy import create_engine, event, insert, select, update from sqlalchemy.engine import make_url -from powercontext.artifacts import ArtifactRef, StableHandle +from powercontext.artifacts import ArtifactAddress, ArtifactRef, StableHandle from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent from powercontext.builtin.artifacts.atomic_memory.errors import InvalidAtomicMemoryStateError from powercontext.builtin.artifacts.experience import ExperienceContent @@ -60,6 +60,7 @@ BundleRecoveryRequiredError, PortableBundleService, ) +from powercontext.builtin.publication import ArtifactPublicationRequest from powercontext.builtin.records import ArtifactWrite from powercontext.builtin.review.models import CandidateAudit, CandidateStatus from powercontext.builtin.runtime import ( @@ -739,6 +740,197 @@ async def authorize(_scopes: tuple[str, ...]) -> None: asyncio.run(scenario()) +async def _publication_archive(tmp_path: Path): + archive = tmp_path / "publications.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scopes = tuple([ + await source.scopes.create(ScopeDraft(title=title, summary="Publication recovery", idempotency_key=title)) + for title in ("Origin", "Relay", "Delivery") + ]) + created = await source.records.create_artifact( + scopes[0].scope_id, + "experience", + ArtifactWrite( + content={"situation": "Recovery", "action": "Publish", "outcome": "Ready", "lesson": "Keep lineage"} + ), + ) + addresses = [ + ArtifactAddress( + scope_id=created.scope_id, + artifact=ArtifactRef(family=created.family, artifact_id=created.artifact_id, revision=created.revision), + ) + ] + for scope in (scopes[1], scopes[2], scopes[0]): + published = await source.publications.publish( + ArtifactPublicationRequest( + source=addresses[-1], target_scope_id=scope.scope_id, idempotency_key="portable-publication" + ) + ) + addresses.append(published.target) + independent = await source.records.create_artifact( + scopes[1].scope_id, + "experience", + ArtifactWrite( + content={"situation": "Separate", "action": "Keep", "outcome": "Root", "lesson": "No publication"} + ), + ) + addresses.append( + ArtifactAddress( + scope_id=independent.scope_id, + artifact=ArtifactRef( + family=independent.family, artifact_id=independent.artifact_id, revision=independent.revision + ), + ) + ) + expected = tuple([ + await source.records.get_artifact(address.scope_id, address.artifact.family, address.artifact.artifact_id) + for address in addresses + ]) + exported = await source.portability.export([scope.scope_id for scope in scopes], archive, authorize=authorize) + return archive, tuple(addresses), expected, exported + + +def test_archive_preserves_publication_chains_after_restart(tmp_path: Path, target_config: BuiltinConfig) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive, addresses, expected, exported = await _publication_archive(tmp_path) + async with open_builtin_contexts(target_config) as target: + assert (await target.portability.validate(archive)).compatible + assert (await target.portability.restore(archive)).projections_ready + for address, record in zip(addresses, expected, strict=True): + assert ( + await target.records.get_artifact( + address.scope_id, address.artifact.family, address.artifact.artifact_id + ) + == record + ) + async with open_builtin_contexts(target_config) as target: + assert (await target.portability.restore(archive)).inserted == 0 + for address, record in zip(addresses, expected, strict=True): + assert ( + await target.records.get_artifact( + address.scope_id, address.artifact.family, address.artifact.artifact_id + ) + == record + ) + roundtrip = await target.portability.export( + sorted({address.scope_id for address in addresses}), + tmp_path / "publications-roundtrip.pcb", + authorize=authorize, + ) + assert roundtrip.total_digest == exported.total_digest + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("damage", ["cycle", "self_cycle", "digest"]) +def test_archive_rejects_invalid_publication_chains_before_writing(tmp_path: Path, damage: str) -> None: + async def scenario() -> None: + archive, addresses, expected, _exported = await _publication_archive(tmp_path) + original = archive.read_bytes() + + def rewrite(records): + publication = next( + record + for record in records + if record["record_type"] == "artifact_publication" + and record["identity"]["target_scope_id"] == addresses[1].scope_id + ) + if damage == "digest": + publication["payload"]["content_digest"] = "0" * 64 + else: + source = addresses[1] if damage == "self_cycle" else addresses[2] + publication["payload"].update( + source_scope_id=source.scope_id, + source_family=source.artifact.family, + source_artifact_id=source.artifact.artifact_id, + source_revision=source.artifact.revision, + ) + return records + + _rewrite_archive_records(archive, rewrite) + config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + async with open_builtin_contexts(config) as target: + before = await target.scopes.list() + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="publication"): + await operation(archive) + assert await target.scopes.list() == before + archive.write_bytes(original) + assert (await target.portability.restore(archive)).projections_ready + async with open_builtin_contexts(config) as target: + assert (await target.portability.restore(archive)).inserted == 0 + for address, record in zip(addresses, expected, strict=True): + assert ( + await target.records.get_artifact( + address.scope_id, address.artifact.family, address.artifact.artifact_id + ) + == record + ) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("target_history", ["cycle", "different_origin"]) +def test_archive_detects_publication_conflicts_with_existing_target(tmp_path: Path, target_history: str) -> None: + async def scenario() -> None: + archive, addresses, _expected, _exported = await _publication_archive(tmp_path) + existing = tmp_path / "existing.pcb" + existing.write_bytes(archive.read_bytes()) + + def rewrite(records): + publication = next( + record + for record in records + if record["record_type"] == "artifact_publication" + and record["identity"]["target_scope_id"] == addresses[1].scope_id + ) + publication["identity"].update( + target_scope_id=addresses[0].scope_id, + target_family=addresses[0].artifact.family, + target_artifact_id=addresses[0].artifact.artifact_id, + target_revision=addresses[0].artifact.revision, + ) + source = addresses[1] if target_history == "cycle" else addresses[-1] + publication["payload"].update( + source_scope_id=source.scope_id, + source_family=source.artifact.family, + source_artifact_id=source.artifact.artifact_id, + source_revision=source.artifact.revision, + idempotency_key="target-existing-publication", + ) + return records + + _rewrite_archive_records(existing, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + assert (await target.portability.restore(existing)).projections_ready + before = tuple([ + await target.records.get_artifact( + address.scope_id, address.artifact.family, address.artifact.artifact_id + ) + for address in addresses + ]) + validation = await target.portability.validate(archive) + assert not validation.compatible + with pytest.raises(BundleConflictError): + await target.portability.restore(archive) + for address, record in zip(addresses, before, strict=True): + assert ( + await target.records.get_artifact( + address.scope_id, address.artifact.family, address.artifact.artifact_id + ) + == record + ) + + asyncio.run(scenario()) + + @pytest.fixture(params=["sqlite", "seekdb", "server"]) def target_config(tmp_path: Path, short_tmp_path: Path, request: pytest.FixtureRequest) -> Iterator[BuiltinConfig]: if request.param == "seekdb": From 552a32265f59887d7ab36560e2a8b31689290477 Mon Sep 17 00:00:00 2001 From: YuZhangLarry Date: Thu, 22 Oct 2026 18:29:26 +0800 Subject: [PATCH 32/32] fix(archive): enforce restore limits and identity invariants --- docs/en/docs/operate/portable-archive.md | 26 ++-- docs/zh/docs/operate/portable-archive.md | 17 +-- .../builtin/portability/bundle.py | 119 ++++++++++++++---- .../builtin/runtime/relational.py | 5 - tests/builtin/portability/test_bundle.py | 81 ++++++++++++ tests/e2e/test_portable_archive.py | 101 +++++++++++++++ 6 files changed, 300 insertions(+), 49 deletions(-) diff --git a/docs/en/docs/operate/portable-archive.md b/docs/en/docs/operate/portable-archive.md index c2979bcb52..d53274f328 100644 --- a/docs/en/docs/operate/portable-archive.md +++ b/docs/en/docs/operate/portable-archive.md @@ -24,18 +24,8 @@ selection, scheduler state or derived indexes. A SQLite database copy is not an backend's SQL dump does not perform these application-level selections and projection rebuilds. Schema migrations upgrade a deployment's schema; they do not replace this scope-transfer operation. -The logical record schema is therefore an explicit portability contract, not a second copy of every storage column. -It retains authoritative domain fields and maps them to each backend while excluding deployment-specific data. -Changing a domain model requires checking this mapping and its round-trip behavior; a checksum cannot discover a -field that a producer failed to export. Unknown record schema versions are rejected instead of silently supplying -defaults for new authoritative state. - -Import policy is independent of the checksum algorithm: an absent identity is inserted, identical canonical content is -skipped, and different content at an existing identity causes a conflict without overwriting it. Per-record SHA-256 -digests are the chosen fingerprints for this comparison and for stream verification; comparing canonical record bytes -would implement the same policy. The total digest covers the record stream, while ZIP checks detect damaged compressed -data. These checks do not authenticate the producer, establish completeness against the source database, or replace -Source/Artifact decoding and dependency validation. +Checksums detect corruption, not producer authenticity or completeness against the source database. Domain decoding +and dependency validation still run before restore. ## Prerequisites @@ -109,6 +99,10 @@ provenance. The graph is indexed in temporary local storage, and target links ar unrelated publication histories. Conflicting target histories are reported without overwriting them; recovery from an already invalid history requires a clean target and a valid backup. +Tag labels and complete tag sets use the normal domain validation, including the limit of 32 labels per target. New +tag identities are added without removing existing labels; the combined set must also fit that limit. An oversized +bundle tag set is invalid input; an oversized combined target set is a conflict, rejected before writes. + Native Sources require their Python adapter on the target. Before writing records, validation uses the same registered Source decoder as normal reads to check both payload shape and indexed identity, including supported legacy payloads. Worker-materialized `SourceObservation` records carry their captured payload and projections. The same decoder @@ -145,7 +139,8 @@ Format version 1 carries the portable relational representation of these support | Work contracts, task outcomes, Handoff boundaries and receipts stored as Sources | Credentials, bearer tokens, provider secrets, and host-local Skill installation state | The archive has a versioned manifest (`format_version`, producer version, scopes, counts, exclusions, and total -checksum) plus NDJSON records. It is the authoritative round-trip format. CSV may be produced separately for bounded +checksum) plus NDJSON records. Per-record digests compare canonical content for conflicts; the total digest verifies +the record stream. It is the authoritative round-trip format. CSV may be produced separately for bounded analysis, but it cannot preserve immutable revisions, lineage, or evidence references and must not be used for restore. Frozen recurrence matches and events are preserved with their original idempotency keys, Source positions, and exact @@ -208,6 +203,11 @@ the Server configuration guide. Dry-run must report no unsupported families and ## Format compatibility +Bundle IDs must use the canonical lowercase, hyphenated UUID representation. The entire archive file, including ZIP +prefixes or trailing bytes, is limited to 512 MiB. The manifest limit is 256 KiB; uncompressed records are limited to +2 GiB, with at most 2,000,000 records and 8 MiB per NDJSON line. Oversized files are rejected before restore copying, +and snapshot copying enforces the same limit if the source changes while being read. + Format version `1` readers require support for every record type present in the bundle. Older readers may reject new record types even when the bundle uses format version `1`. The producer version is reported for diagnostics but does not replace the archive schema version or record-type compatibility checks. Writers produce only version `1`; there is no diff --git a/docs/zh/docs/operate/portable-archive.md b/docs/zh/docs/operate/portable-archive.md index a3a5716cd6..77eeecf320 100644 --- a/docs/zh/docs/operate/portable-archive.md +++ b/docs/zh/docs/operate/portable-archive.md @@ -20,14 +20,7 @@ SQLite、SeekDB 或 OceanBase 配置。恢复前应停止 PowerContext 写入进 SQLite 数据库副本不能直接作为 OceanBase 数据库使用；回放后端的 SQL dump 也不会执行这些应用层的范围选择和投影重建。 schema migration 负责升级现有部署的表结构，并不替代这种 scope 迁移操作。 -逻辑 record schema 因此是显式的可移植协议，而非每个存储列的另一份副本。它保留权威领域字段，映射到各后端，并排除部署 -专有数据。领域模型变化时，仍需检查映射及其往返行为；checksum 无法发现生产端遗漏导出的字段。读取端拒绝未知的 record -schema version，不会为新增的权威状态静默补默认值。 - -导入策略独立于 checksum 算法：不存在的 identity 会插入，canonical 内容相同则跳过，已有 identity 的内容不同则报告冲突， -不覆盖原数据。逐条 SHA-256 digest 是当前用于比较和流校验的指纹表示；直接比较 canonical record 字节也能实现同一策略。 -总 digest 覆盖 record 流，ZIP 校验可发现压缩数据损坏。这些检查不提供生产者身份认证，不证明导出内容相对源数据库完整， -也不替代 Source/Artifact 解码与依赖校验。 +Checksum 用于发现数据损坏，不提供生产者身份认证，也不证明导出内容相对源数据库完整。恢复前仍会执行领域解码和依赖校验。 ## 前提条件 @@ -90,6 +83,9 @@ content digest 仍会在领域写入前被拒绝。兼容性检查也会比对 关系即使自身合法，也会作为冲突拒绝，防止恢复后的原始来源被改变。关系图使用本地临时存储建立索引，目标关系按批读取， 不扫描无关的发布历史。存在冲突时不会覆盖目标历史；已经损坏的历史需要使用有效备份恢复到干净目标。 +标签及完整标签集合使用正常领域校验，每个标签目标最多包含 32 个标签。新标签 identity 会追加到目标，不删除已有标签， +合并后的集合也必须满足同一上限。归档内的标签集合超限会作为无效输入拒绝；与目标合并后超限则作为冲突在写入前拒绝。 + 原生 Source 需要目标环境安装对应的 Python adapter。验证会在写入前使用正常读取路径的已注册 Source 解码器，检查 payload 结构与索引 identity 是否一致，也覆盖受支持的 legacy payload。远程 worker 已物化的 `SourceObservation` 自带捕获内容和投影， 其存储 envelope 也使用同一解码器验证，但不需要加载 worker adapter。归档只携带所选 scope 实际引用的 Source Definition @@ -123,6 +119,7 @@ powercontext archive restore ./backups/payments.pcb --yes | 作为 Source 保存的 Work contract、task outcome、Handoff boundary 和 receipt | Credential、bearer token、provider secret 和 host-local Skill 安装状态 | 归档包含带版本的 manifest（`format_version`、producer version、scope、数量、排除项和总 checksum）以及 NDJSON record。 +逐条 digest 用于比较 canonical 内容是否冲突，总 digest 用于校验 record 流。 它是权威的 round-trip 格式。CSV 可以单独用于有边界的分析，但无法保留不可变 revision、lineage 或 evidence reference， 不得用于恢复。 @@ -179,6 +176,10 @@ powercontext archive restore ./payments.pcb --env-file ./oceanbase.env --yes ## 格式兼容窗口 +Bundle ID 必须使用带连字符、小写的标准 UUID 表示。完整归档文件连同 ZIP 前缀或尾随字节不得超过 512 MiB；manifest +上限为 256 KiB，解压后的 record 流上限为 2 GiB，最多包含 2,000,000 条 record，每行 NDJSON 不超过 8 MiB。 +超限文件会在恢复复制前被拒绝；复制 snapshot 时也会执行相同上限，防止读取期间源文件变化绕过限制。 + 格式版本 `1` reader 还必须支持归档中出现的全部 record type。即使归档使用格式版本 `1`，旧 reader 也可能拒绝新增的 record type。producer version 仅用于诊断，不能替代 archive schema version 或记录类型兼容性检查。 writer 只生成版本 `1`，不支持降级到格式 `0`。遇到未知 archive 或 record schema diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py index 7ae7b4ee76..54bb08b822 100644 --- a/src/powercontext/builtin/portability/bundle.py +++ b/src/powercontext/builtin/portability/bundle.py @@ -594,7 +594,7 @@ async def validate( supported_source_types=configured, total=inspection.record_count, progress=progress, - publication_index=index, + dependency_index=index, ) return BundleValidation( bundle_id=inspection.bundle_id, @@ -633,7 +633,15 @@ async def restore( with tempfile.TemporaryDirectory(prefix="powercontext-restore-") as directory: snapshot = Path(directory) / "bundle.pcb" try: - shutil.copyfile(source, snapshot) + if (await asyncio.to_thread(source.stat)).st_size > _MAX_ARCHIVE_BYTES: + raise BundleFormatError("bundle exceeds resource limit") + with source.open("rb") as original, snapshot.open("wb") as target: + copied = 0 + while chunk := original.read(min(1024 * 1024, _MAX_ARCHIVE_BYTES - copied + 1)): + copied += len(chunk) + if copied > _MAX_ARCHIVE_BYTES: + raise BundleFormatError("bundle exceeds resource limit") + target.write(chunk) except OSError as error: raise BundleFormatError("cannot read portable bundle") from error return await self._restore_snapshot( @@ -743,7 +751,7 @@ async def _target_compatibility( supported_source_types: frozenset[str] | None, total: int, progress: ProgressObserver | None, - publication_index: sqlite3.Connection, + dependency_index: sqlite3.Connection, ) -> tuple[int, int]: already_present = 0 conflicts = 0 @@ -795,7 +803,13 @@ async def _target_compatibility( if ( conflicts == 0 and ARTIFACT_PUBLICATIONS_TABLE.name in existing_tables - and not await _validate_target_publications(connection, publication_index) + and not await _validate_target_publications(connection, dependency_index) + ): + conflicts += 1 + if ( + conflicts == 0 + and ARTIFACT_TAGS_TABLE.name in existing_tables + and not await _validate_target_tag_sets(connection, dependency_index) ): conflicts += 1 return already_present, conflicts @@ -1032,8 +1046,11 @@ async def _validate_export_destination(connection: AsyncConnection, output: Path continue database = await asyncio.to_thread(Path(row[2]).resolve) reserved_names = {database.name.casefold() + suffix for suffix in ("", "-wal", "-shm", "-journal")} - if destination.name.casefold() in reserved_names and await asyncio.to_thread( - destination.parent.samefile, database.parent + if ( + await asyncio.to_thread(destination.exists) and await asyncio.to_thread(destination.samefile, database) + ) or ( + destination.name.casefold() in reserved_names + and await asyncio.to_thread(destination.parent.samefile, database.parent) ): raise BundleFormatError("export destination overlaps the source database or its journal") @@ -1076,6 +1093,8 @@ def _parse_bundle( phase: Literal["inspect", "validate"] = "inspect", ) -> BundleInspection: try: + if source.stat().st_size > _MAX_ARCHIVE_BYTES: + raise BundleFormatError("bundle exceeds resource limit") with zipfile.ZipFile(source) as archive: _validate_container(archive) with archive.open(_MANIFEST_NAME) as stream: @@ -1254,9 +1273,11 @@ def _validate_manifest(manifest: object) -> None: # noqa: C901 - manifest field ): raise BundleFormatError("manifest has invalid identity or record count") try: - UUID(manifest["bundle_id"]) + bundle_id = UUID(manifest["bundle_id"]) except ValueError as error: raise BundleFormatError("manifest has invalid bundle identity") from error + if str(bundle_id) != manifest["bundle_id"]: + raise BundleFormatError("manifest has invalid bundle identity") producer = manifest["producer"] if not isinstance(producer, dict): raise BundleFormatError("manifest has invalid producer") @@ -1736,6 +1757,7 @@ def contains(record_type: str, identity: Mapping[str, object]) -> bool: required_sources.add(str(record.identity["source_type"])) required_families.update(_required_artifact_families((record,))) _validate_publication_chains(index) + _validate_tag_sets(index) yield required_sources, required_families, index @@ -1804,6 +1826,64 @@ async def _validate_target_publications(connection: AsyncConnection, index: sqli return True +def _index_tag_set(index: sqlite3.Connection, record: _Record, /) -> None: + if record.record_type != "artifact_tag": + return + target = _identity_key({field: value for field, value in record.identity.items() if field != "tag_key"}) + index.execute( + "INSERT OR IGNORE INTO tag_labels (target, tag_key, label) VALUES (?, ?, ?)", + (target, record.identity["tag_key"], record.payload["tag"]), + ) + index.execute( + "INSERT OR IGNORE INTO identities (record_type, identity) VALUES ('artifact_tag_target', ?)", (target,) + ) + + +def _validate_tag_sets(index: sqlite3.Connection, /) -> None: + current: str | None = None + labels: list[str] = [] + for target, label in index.execute("SELECT target, label FROM tag_labels ORDER BY target, tag_key"): + if current != target: + current = target + labels.clear() + labels.append(label) + try: + normalize_tags(tuple(labels)) + except InvalidBaseAccessRequestError as error: + raise BundleFormatError("artifact tag set has invalid labels") from error + + +async def _validate_target_tag_sets(connection: AsyncConnection, index: sqlite3.Connection, /) -> bool: + table = ARTIFACT_TAGS_TABLE + fields = tuple(field for field in _SPECS["artifact_tag"].identity if field != "tag_key") + pending = index.execute( + "SELECT identity FROM identities WHERE record_type = 'artifact_tag_target' ORDER BY identity" + ) + while batch := pending.fetchmany(100): + identities = [json.loads(row[0]) for row in batch] + rows = ( + await connection.execute( + select(table).where( + tuple_(*(table.c[field] for field in fields)).in_([ + tuple(identity[field] for field in fields) for identity in identities + ]) + ) + ) + ).mappings() + for row in rows: + record = _record_from_row("artifact_tag", cast(Mapping[str, Any], row)) + try: + _validate_record_values(record) + except BundleFormatError: + return False + _index_tag_set(index, record) + try: + _validate_tag_sets(index) + except BundleFormatError: + return False + return True + + def _validate_observation_record(record: _Record, index: sqlite3.Connection, /) -> None: if record.record_type != "source": return @@ -1939,7 +2019,10 @@ def _validate_record_dependencies( # noqa: C901 - logical record kinds have dis {"scope_id": scope, "family": identity["family"], "artifact_id": identity["artifact_id"]}, ): raise BundleFormatError("artifact tag references missing artifact head") - if identity["target_type"] == "memory_entry" and not _contains_memory_entry_target(record, contains): + if identity["target_type"] == "memory_entry" and not contains( + "memory_entry_target", + {"scope_id": scope, "memory_artifact_id": identity["artifact_id"], "entry_id": identity["target_id"]}, + ): raise BundleFormatError("artifact tag references missing memory entry") elif record.record_type == "artifact_lineage_source": if not contains( @@ -2101,21 +2184,6 @@ def _validate_record_dependencies( # noqa: C901 - logical record kinds have dis _validate_embedded_references(record, contains) -def _contains_memory_entry_target( - record: _Record, - contains: Callable[[str, Mapping[str, object]], bool], - /, -) -> bool: - return contains( - "memory_entry_target", - { - "scope_id": record.identity["scope_id"], - "memory_artifact_id": record.identity["artifact_id"], - "entry_id": record.identity["target_id"], - }, - ) - - def _dependency_metadata(source: Path, /) -> tuple[dict[str, int], dict[tuple[str, str], tuple[str, int, int, int]]]: heads: dict[str, int] = {} packages: dict[tuple[str, str], tuple[str, int, int, int]] = {} @@ -2239,6 +2307,10 @@ def _identity_index(source: Path, /) -> Iterator[sqlite3.Connection]: ) connection.execute("CREATE INDEX publication_edges_state ON publication_edges(state, target)") connection.execute("CREATE TABLE publication_nodes (identity TEXT PRIMARY KEY)") + connection.execute( + "CREATE TABLE tag_labels (target TEXT NOT NULL, tag_key TEXT NOT NULL, label TEXT NOT NULL, " + "PRIMARY KEY (target, tag_key))" + ) for record in _iter_records(source): try: connection.execute( @@ -2259,6 +2331,7 @@ def _identity_index(source: Path, /) -> Iterator[sqlite3.Connection]: except sqlite3.IntegrityError as error: raise BundleFormatError("bundle contains duplicate immutable or unique identity") from error _index_publication_dependencies(connection, record) + _index_tag_set(connection, record) if record.record_type == "candidate_version": connection.execute( "INSERT INTO identities (record_type, identity) VALUES (?, ?)", diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 077194cc87..accc72c88a 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -274,11 +274,6 @@ def _artifact_identity(ref: ArtifactRef) -> tuple[str, str, int]: return ref.family, ref.artifact_id, ref.revision -_MEMORY_COMMIT_STAGE = "memory.commit" -_MEMORY_COMMIT_MEMORY_CHANGED = "powercontext.memory.commit.memory_changed" -_MEMORY_COMMIT_ENTRY_VERSION_COUNT = "powercontext.memory.commit.entry_version_count" - - BUILTIN_ARTIFACT_TYPES = (Handoff, AtomicMemory, Experience, Skill, Profile, Prompt, TopicMemory) BUILTIN_CANDIDATE_PROPOSAL_TYPES = { Experience.family: ExperienceContent, diff --git a/tests/builtin/portability/test_bundle.py b/tests/builtin/portability/test_bundle.py index 8968cfaa80..41f7f6de8b 100644 --- a/tests/builtin/portability/test_bundle.py +++ b/tests/builtin/portability/test_bundle.py @@ -19,6 +19,7 @@ import asyncio import hashlib import json +import os import sqlite3 import tracemalloc import zipfile @@ -52,6 +53,7 @@ TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE, ) from powercontext.builtin.portability import BundleConflictError, BundleFormatError, PortableBundleService +from powercontext.builtin.portability import bundle as bundle_module from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY, ContentSource from powercontext.sources import SourceMaterialization @@ -124,6 +126,85 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_export_rejects_hard_link_to_source_database(tmp_path: Path) -> None: + async def scenario() -> None: + database = tmp_path / "source.db" + alias = tmp_path / "different-name.db" + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{database}") + async with SQLiteProfile.open(config, tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + try: + os.link(database, alias) + except OSError as error: + pytest.skip(f"hard links unavailable: {error}") + with pytest.raises(BundleFormatError, match="source database"): + await PortableBundleService(source.database).export(["project:one"], alias, authorize=_authorize_export) + assert alias.samefile(database) + assert database.read_bytes().startswith(b"SQLite format 3\0") + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("notation", ["urn", "braces", "compact", "uppercase"]) +def test_bundle_rejects_noncanonical_receipt_identity(tmp_path: Path, notation: str) -> None: + async def scenario() -> None: + archive = tmp_path / "identity.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + original = archive.read_bytes() + with zipfile.ZipFile(archive) as bundle: + manifest = json.loads(bundle.read("manifest.json")) + records = bundle.read("records.ndjson") + identity = manifest["bundle_id"] + manifest["bundle_id"] = { + "urn": "urn:uuid:" + identity, + "braces": "{" + identity + "}", + "compact": identity.replace("-", ""), + "uppercase": identity.upper(), + }[notation] + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("manifest.json", json.dumps(manifest)) + bundle.writestr("records.ndjson", records) + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + service = PortableBundleService(target.database) + for operation in (service.inspect, service.validate, service.restore): + with pytest.raises(BundleFormatError, match="bundle identity"): + await operation(archive) + async with target.database.transaction() as connection: + assert await connection.scalar(select(SCOPES_TABLE.c.scope_id)) is None + archive.write_bytes(original) + assert (await service.validate(archive)).compatible + assert (await service.restore(archive)).inserted == 1 + + asyncio.run(scenario()) + + +def test_bundle_size_limit_includes_zip_prefix(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + async def scenario() -> None: + archive = tmp_path / "padded.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + original = archive.read_bytes() + monkeypatch.setattr(bundle_module, "_MAX_ARCHIVE_BYTES", 32 * 1024) + archive.write_bytes(b"\0" * (32 * 1024 + 1) + original) + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + service = PortableBundleService(target.database) + for operation in (service.inspect, service.validate, service.restore): + with pytest.raises(BundleFormatError, match="resource limit"): + await operation(archive) + async with target.database.transaction() as connection: + assert await connection.scalar(select(SCOPES_TABLE.c.scope_id)) is None + archive.write_bytes(original) + assert (await service.restore(archive)).inserted == 1 + + asyncio.run(scenario()) + + @pytest.mark.parametrize("mutation", ["identity_override", "unknown_field", "missing_field", "wrong_type", "scope"]) def test_restore_rejects_invalid_wire_fields_before_writing(tmp_path: Path, mutation: str) -> None: async def scenario() -> None: diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py index a09493dcc4..a4101aace3 100644 --- a/tests/e2e/test_portable_archive.py +++ b/tests/e2e/test_portable_archive.py @@ -615,6 +615,107 @@ def rewrite(records): asyncio.run(scenario()) +def test_archive_rejects_unreadable_tag_sets_before_writing(tmp_path: Path) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "tags.pcb" + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Tags", summary="Complete tag set", idempotency_key="tags") + ) + memory = await source.records.create_artifact( + scope.scope_id, "atomic-memory", ArtifactWrite(content={"kind": "fact", "text": "Portable tag set."}) + ) + tag_target = ArtifactTagTarget(family="atomic-memory", artifact_id=memory.artifact_id) + empty = await source.records.get_tags(scope.scope_id, tag_target) + tagged = await source.records.replace_tags( + scope.scope_id, tag_target, tuple(f"tag-{number}" for number in range(32)), expected_etag=empty.etag + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + original = archive.read_bytes() + + def rewrite(records): + tag = next(record for record in records if record["record_type"] == "artifact_tag") + records.append({ + **tag, + "identity": {**tag["identity"], "tag_key": "tag-32"}, + "payload": { + **tag["payload"], + "tag": "tag-32", + "tag_key_hash": {"base64": base64.b64encode(hashlib.sha256(b"tag-32").digest()).decode()}, + }, + }) + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + before = await target.scopes.list() + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="tag set"): + await operation(archive) + assert await target.scopes.list() == before + archive.write_bytes(original) + assert (await target.portability.restore(archive)).projections_ready + assert await target.records.get_tags(scope.scope_id, tag_target) == tagged + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("incoming_count", [16, 17]) +def test_archive_validates_combined_tag_sets_before_writing( + tmp_path: Path, incoming_count: int, target_config: BuiltinConfig +) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + first, second = tmp_path / "first.pcb", tmp_path / "second.pcb" + previous_labels = tuple(f"left-{number}" for number in range(16)) + incoming_labels = tuple(f"right-{number}" for number in range(incoming_count)) + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Tags", summary="Combined tag set", idempotency_key="tags") + ) + memory = await source.records.create_artifact( + scope.scope_id, + "atomic-memory", + ArtifactWrite(content={"kind": "fact", "text": "Portable combined tags."}), + ) + tag_target = ArtifactTagTarget(family="atomic-memory", artifact_id=memory.artifact_id) + empty = await source.records.get_tags(scope.scope_id, tag_target) + previous = await source.records.replace_tags( + scope.scope_id, tag_target, previous_labels, expected_etag=empty.etag + ) + await source.portability.export([scope.scope_id], first, authorize=authorize) + await source.records.replace_tags(scope.scope_id, tag_target, incoming_labels, expected_etag=previous.etag) + await source.portability.export([scope.scope_id], second, authorize=authorize) + async with open_builtin_contexts(target_config) as target: + await target.portability.restore(first) + if incoming_count == 17: + assert not (await target.portability.validate(second)).compatible + with pytest.raises(BundleConflictError): + await target.portability.restore(second) + expected = set(previous_labels) + else: + assert (await target.portability.validate(second)).compatible + assert (await target.portability.restore(second)).projections_ready + assert (await target.portability.restore(second)).inserted == 0 + expected = set(previous_labels + incoming_labels) + assert set((await target.records.get_tags(scope.scope_id, tag_target)).tags) == expected + async with open_builtin_contexts(target_config) as target: + assert set((await target.records.get_tags(scope.scope_id, tag_target)).tags) == expected + hits = await target.atomic_memory.for_scope(scope.scope_id).search( + "portable", tag_filter=TagFilter(tags=("right-0",)) + ) + assert [hit.hit.artifact_ref.artifact_id for hit in hits.hits] == ( + [memory.artifact_id] if incoming_count == 16 else [] + ) + + asyncio.run(scenario()) + + def test_archive_restores_package_files_and_topic_search_after_restart( tmp_path: Path, target_config: BuiltinConfig ) -> None: