diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index a849ce938a..df3003dc8a 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -385,6 +385,9 @@ jobs: # Requested historical refs may predate the role smoke harness. build/cli-smoke-venv/bin/powercontext skill remote-sync --help fi + if [ -f src/powercontext/cli/archive.py ]; then + build/cli-smoke-venv/bin/powercontext archive --help + fi plugin_root="$BUNDLE_ROOT/integrations/plugins/powercontext" ( diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 01ff65b202..553ef5a1f6 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -435,6 +435,13 @@ jobs: run: | Set-ItemProperty -LiteralPath 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name LongPathsEnabled -Value 0 + - name: Verify portable archive restoration and SQLite URI targets on Windows + run: >- + uv run --locked --no-sync pytest -q + tests/builtin/portability/test_bundle.py + tests/test_archive_cli.py + tests/e2e/test_portable_archive.py + # Keep pytest's default temporary root: a short --basetemp would hide # regressions in the nested checkout fixtures covered by issue #1501. - name: Run Windows portability regressions diff --git a/docs/en/docs/operate/index.md b/docs/en/docs/operate/index.md index e28b03098f..fb1ef9a0c6 100644 --- a/docs/en/docs/operate/index.md +++ b/docs/en/docs/operate/index.md @@ -12,6 +12,7 @@ Start with the operational task you need: | Keep a personal Server running, deploy a container, or enable authentication | [Deploy the Server](deploy-server.md) | | Understand unified database upgrades, backups, and service lifecycle | [Database migration design](database-migrations.md) | | Upgrade or switch background supervision mode | [Migrate Artifact processing state](artifact-processing-migration.md) | +| Export a local SQLite archive or restore one | [Portable archive](portable-archive.md) | | Inspect logs, metrics, and traces | [Observability](observability.md) | | Send traces to an analysis service | [Phoenix](trace-with-phoenix.md), [Langfuse](trace-with-langfuse.md) | | Diagnose failures and restore service or data | [Troubleshoot](troubleshoot.md) | diff --git a/docs/en/docs/operate/meta.json b/docs/en/docs/operate/meta.json index 935976d355..d52400229f 100644 --- a/docs/en/docs/operate/meta.json +++ b/docs/en/docs/operate/meta.json @@ -6,6 +6,7 @@ "connect-remote-server", "database-migrations", "artifact-processing-migration", + "portable-archive", "atomic-memory-migration", "observability", "trace-with-phoenix", diff --git a/docs/en/docs/operate/portable-archive.md b/docs/en/docs/operate/portable-archive.md new file mode 100644 index 0000000000..d53274f328 --- /dev/null +++ b/docs/en/docs/operate/portable-archive.md @@ -0,0 +1,247 @@ +--- +title: Export and restore a portable archive +description: Create, validate, and restore a verified PowerContext logical bundle. +--- + +# Export and restore a portable archive + +A portable archive is a `.pcb` file for moving or recovering complete PowerContext scopes. It is a logical archive: +it preserves domain identities and immutable history rather than copying a SQLite file. Use it for a controlled local +backup, an offline transfer, or a future backend migration. + +This is an offline operator command. It does not call a remote Server API. Without `--env-file` it opens the default +SQLite database; with `--env-file` it uses the same SQLite, SeekDB, or OceanBase settings as that deployment. Stop +PowerContext writers before restore. Export uses one database transaction as its consistent logical snapshot. +MySQL-mode backends use `REPEATABLE READ` for the export transaction even when the deployment defaults to +`READ COMMITTED`; later transactions retain the deployment default. + +## Choosing the recovery format + +Use a deployment-native backup when you need the complete database, deployment-local state, or point-in-time recovery. +A `.pcb` bundle addresses a narrower requirement: transfer selected complete scopes between supported backends while +preserving their exact identities, revisions, evidence and retry behavior, without copying access bindings, host-local +selection, scheduler state or derived indexes. A SQLite database copy is not an OceanBase database, and replaying a +backend's SQL dump does not perform these application-level selections and projection rebuilds. Schema migrations +upgrade a deployment's schema; they do not replace this scope-transfer operation. + +Checksums detect corruption, not producer authenticity or completeness against the source database. Domain decoding +and dependency validation still run before restore. + +## Prerequisites + +Install the CLI from the same PowerContext revision that created the local data. The archive command uses the persistent +SQLite database at `POWERCONTEXT_HOME/powercontext.db`; without `POWERCONTEXT_HOME`, PowerContext uses its operating +system user-data directory. + +```bash +uv tool install "powercontext[cli] @ git+https://github.com/oceanbase/powercontext.git@master" +export POWERCONTEXT_HOME="$HOME/.local/share/powercontext" +``` + +The archive contains content and metadata needed for recovery. Store it in a protected location with access controls +and retention appropriate for the underlying project data. The format deliberately omits credentials and configured +provider secrets, but it does not encrypt the project content. + +## Create and inspect an archive + +Pass every complete scope to include. A scope cannot be selected by prefix or partially by record type. + +```bash +powercontext archive export \ + --scope-id project:payments \ + --output ./backups/payments.pcb + +powercontext archive inspect ./backups/payments.pcb +``` + +Add `--no-compress` when the surrounding storage system already compresses objects. Export authorization occurs before +the first database query. For this offline command, authority is the operating-system and database permission needed to +read the configured deployment; it never reuses or exports Server bearer tokens. Progress is emitted as content-free +JSON on stderr while the final receipt remains JSON on stdout. + +`export` writes JSON containing the bundle ID, record count, and checksum. `inspect` verifies the ZIP structure, +per-record digests, total digest, record counts, and selected scopes without opening the target database. Keep the +reported checksum with the backup inventory if an external backup system needs an independent verification record. +Before publishing the file, export checks its format, resource limits, and dependencies. If validation fails, an +existing backup at the output path remains unchanged. +The destination must not be the source SQLite database or one of its `-wal`, `-shm`, or `-journal` files; export rejects +these paths rather than replacing live storage with an archive. These sibling filenames are reserved regardless of +letter case, including on case-sensitive filesystems. + +## Validate before a restore + +Always validate the archive against the destination first: + +```bash +powercontext archive restore ./backups/payments.pcb --dry-run +``` + +Dry-run performs no domain writes. It verifies checksums, required source and Artifact dependencies, and whether the +configured Runtime can restore the archive's source types and Artifact families. A validation failure exits with code +`2` for an invalid bundle or an uninspectable SQLite filename, or `3` for unsupported requirements or target conflicts, +and reports only a content-free +reason; record bodies are not printed. A successful report includes +`already_present`, `conflicts`, required and unsupported Source types and Artifact families, and whether the target has +a projection rebuilder. Candidate versions and decision heads use the same complete decoder as normal reads. Validation +checks their family consistency, evidence, reasons, lifecycle state and family-owned proposals, including optional Dream +audit envelopes; malformed or unsupported Candidates are rejected before any domain write. Run against a different +deployment with `--env-file ./target.env`. Existing SQLite targets, including `file:` URI filenames with `uri=true`, +are inspected read-only. A missing SQLite target is validated without creating its database file or parent directory. +This includes absolute `file:///C:/...` paths on Windows and percent-encoded filenames. Unsupported URI authorities, +unreadable targets, directory targets and non-directory parent paths fail validation rather than being treated as an +empty database. + +Publication validation follows exact Artifact revision addresses through the complete chain. Cycles and inconsistent +content digests within a chain are rejected before domain writes, even if all endpoints and checksums are valid. +Compatibility checks also compare existing target publication links for restored revisions. A target link absent from +the archive is a conflict, even if its chain is valid: accepting it could change the restored revision's original +provenance. The graph is indexed in temporary local storage, and target links are read in batches without scanning +unrelated publication histories. Conflicting target histories are reported without overwriting them; recovery from an +already invalid history requires a clean target and a valid backup. + +Tag labels and complete tag sets use the normal domain validation, including the limit of 32 labels per target. New +tag identities are added without removing existing labels; the combined set must also fit that limit. An oversized +bundle tag set is invalid input; an oversized combined target set is a conflict, rejected before writes. + +Native Sources require their Python adapter on the target. Before writing records, validation uses the same registered +Source decoder as normal reads to check both payload shape and indexed identity, including supported legacy payloads. +Worker-materialized `SourceObservation` records carry their captured payload and projections. The same decoder +validates their storage envelope without loading the worker adapter. The bundle includes only the definition +manifests referenced by the selected scopes and verifies observation identities and definition +fingerprints before restore. Missing definitions and conflicting target definitions prevent restoration; unreferenced +global definitions are not exported. +Observation payloads and supplied projections must also satisfy the same manifest schemas checked during normal +ingestion. Restore does not fetch remote schema references. + +To perform the write, repeat the command with explicit confirmation: + +```bash +powercontext archive restore ./backups/payments.pcb --yes +``` + +The result is JSON with `inserted`, `already_present`, and `projections_ready`. Treat the restore as ready for search +only when `projections_ready` is `true`. The Runtime rebuilds Memory, Topic Memory, and Experience search projections +after the authoritative rows have been restored, including configured Memory and Topic Memory vectors. + +## What the bundle preserves + +Format version 1 carries the portable relational representation of these supported records: + +| Preserved | Not portable | +| --- | --- | +| Scope identity, hierarchy, context/external references, and creation identity | Scope access bindings and host-local default selection | +| Source journal heads, Source records, and referenced remote Source Definition manifests | Search projections and indexes | +| Artifact Revisions, their timestamps/actors/rollback metadata, stable handles, lineage, cross-Scope publication provenance, and heads | Source cursors and scheduler state | +| Atomic Memory merge relationships and marked merge-input lineage | External Skill registrations and host-local installation state | +| Candidate versions, decision heads, and their evidence references | Audit events, usage facts, evaluation receipts, and restore receipts | +| Managed Skill package bytes and manifests; Artifact tags | Download locations and host-local Skill paths | +| Profile policies, Prompt revisions, and Topic Memory publication metadata | Target-specific embedding configuration | +| Work contracts, task outcomes, Handoff boundaries and receipts stored as Sources | Credentials, bearer tokens, provider secrets, and host-local Skill installation state | + +The archive has a versioned manifest (`format_version`, producer version, scopes, counts, exclusions, and total +checksum) plus NDJSON records. Per-record digests compare canonical content for conflicts; the total digest verifies +the record stream. It is the authoritative round-trip format. CSV may be produced separately for bounded +analysis, but it cannot preserve immutable revisions, lineage, or evidence references and must not be used for restore. + +Frozen recurrence matches and events are preserved with their original idempotency keys, Source positions, and exact +Artifact revisions. Restore validates candidate revisions, Handoff/receipt references, and event-to-match dependencies. +Advancing an Experience head does not change this history; replay after restore reuses the saved decisions. + +Merged inputs remain frozen after restore, and marked merge-input lineage remains available for undo-merge. These fields +also participate in conflict checks: the same revision with a different merge relationship is not an identical record. + +After Atomic Memory migration, bundles preserve the migrated Atomic Memory identities, immutable revisions and lineage. +The unused legacy entry rows whose collection is no longer public and the offline Memory collection archive are not +portable Runtime state. Export leaves this retained history untouched in the source database. Keep a deployment-native +backup if you need the original pre-migration collection snapshots, migration records or access-control history. + +## Recovery and conflicts + +Restore is idempotent. Replaying the same archive recognizes identical records as `already_present`. A different +payload for an existing immutable identity never overwrites it: restore exits with code `3` and leaves the write +transaction rolled back. Resolve the conflicting target data or choose a clean destination, then run the same restore +command again. + +The restore command opens its maintenance Runtime without bootstrapping a new default scope. This lets an archive +containing the canonical default scope restore its original identity and creation key into a fresh target. On the next +normal startup, default-scope bootstrap reuses that creation identity when present. Default selection bindings remain +deployment-local; restoring an archive does not overwrite an existing deployment's selection or conflicting scope. +The maintenance Runtime does not start background Artifact processing supervisors, even when processing is configured +for the target deployment. Restart the normal deployment after recovery to resume background work. + +If the command is interrupted before completion, do not claim recovery succeeded. Authoritative records are +transactionally applied, so a failed write does not leave a successful-looking partial restore. Re-run the same archive +against the same destination after the previous command has stopped. + +The target database stores a durable receipt keyed by `bundle_id`. `authoritative_restored` means the logical rows +committed but search rebuild has not completed; `ready` means projection rebuild completed. If projection rebuild +fails, re-run the same restore: identical rows are skipped and the receipt advances to `ready` only after verification. +While any receipt has `projections_ready=false`, normal Runtime/Server startup and worker startup refuse to serve, +including when the archive contains only Atomic Memory or Experience. The explicit restore path may reopen the target +and unfinished Topic Memory projections to complete recovery. Re-run the same restore before serving requests, using +the same target embedding configuration. Export from an unfinished recovery exits with code `3` and requires recovery +to finish first. + +Restore stages records in a temporary disk index before starting the target write transaction. Reserve temporary +disk space for both the archive copy and its expanded records. Existing MySQL-mode deployments may require an +explicit Runtime startup schema upgrade from `DATETIME` to `DATETIME(6)` for portable timestamps; back up the database +and provide schema-alter permissions before that upgrade. + +## SQLite to OceanBase migration + +Create separate environment files without placing credentials in the bundle: + +```bash +powercontext archive export --env-file ./sqlite.env \ + --scope-id project:payments --output ./payments.pcb +powercontext archive restore ./payments.pcb --env-file ./oceanbase.env --dry-run +powercontext archive restore ./payments.pcb --env-file ./oceanbase.env --yes +``` + +The environment files use `POWERCONTEXT_SERVER_DATABASE_KIND` and `POWERCONTEXT_SERVER_DATABASE_URL` as documented in +the Server configuration guide. Dry-run must report no unsupported families and zero conflicts before the write. + +## Format compatibility + +Bundle IDs must use the canonical lowercase, hyphenated UUID representation. The entire archive file, including ZIP +prefixes or trailing bytes, is limited to 512 MiB. The manifest limit is 256 KiB; uncompressed records are limited to +2 GiB, with at most 2,000,000 records and 8 MiB per NDJSON line. Oversized files are rejected before restore copying, +and snapshot copying enforces the same limit if the source changes while being read. + +Format version `1` readers require support for every record type present in the bundle. Older readers may reject +new record types even when the bundle uses format version `1`. The producer version is reported for diagnostics but +does not replace the archive schema version or record-type compatibility checks. Writers produce only version `1`; there is no +format `0` downgrade. A reader rejects unknown archive or record schema versions before target writes. Keep the old +binary available until a restore drill succeeds when upgrading across a PowerContext major release. + +`artifact_revision`, `artifact_head`, `artifact_lineage_artifact` and `candidate_head` require record schema version `2`. +Incomplete schema version `1` representations of these records are rejected rather than discarding revision metadata +or assigning default merge state or Candidate kind. Legacy database revisions retain explicit null metadata; restore +does not invent an actor, timestamp or rollback history. A target missing required archive columns must first be +upgraded by the current Runtime; dry-run reports this incompatibility without changing the target. +Create a fresh bundle from the source database with the current binary when a bundle fails these compatibility checks. +Binary fields use canonical Base64, and timestamp fields use the canonical timezone-naive ISO text emitted by the +supported database profiles. Alternative encodings, offset-bearing timestamps and out-of-range storage values are +rejected before writes rather than accepted with digests that would change on readback or conflict on retry. + +## Deployment-native backups + +Logical export is not a substitute for a crash-consistent deployment backup: + +- SQLite: stop all PowerContext writers before copying the database, or use the SQLite online backup API, for example + `sqlite3 powercontext.db ".backup './backups/powercontext.db'"`. Verify the copy with + `sqlite3 ./backups/powercontext.db "PRAGMA integrity_check"`, protect it like the source database, and test reopening it. +- OceanBase: enable tenant data backup and log archiving according to the deployed OceanBase version, retain the backup + destination and encryption material independently, and perform a restore drill into an isolated tenant. Use native + backup for point-in-time recovery and the portable bundle for logical cross-backend migration. + +## Current boundaries + +Export streams database rows to a temporary NDJSON file. Validation stores record identities and Candidate versions in a +temporary on-disk index to verify each head against its exact version. Restore replays dependency levels as streams, so +aggregate archive payloads are not retained in +process memory. Scope metadata is bounded by the manifest scope list. The command is not a remote Server API and never +puts deployment configuration or database credentials into the archive. + +Artifact Candidates are supported; tag-change Candidates are rejected before export publishes an archive or restore +writes records. An archive must not silently drop a tag-change decision or reinterpret it as an Artifact Candidate. diff --git a/docs/en/docs/operate/troubleshoot.md b/docs/en/docs/operate/troubleshoot.md index bfbba715b0..06d9b97b9f 100644 --- a/docs/en/docs/operate/troubleshoot.md +++ b/docs/en/docs/operate/troubleshoot.md @@ -269,7 +269,7 @@ so the previous database remains available for recovery: ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_portable_restore_receipts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions' \ -f ``` diff --git a/docs/zh/docs/operate/index.md b/docs/zh/docs/operate/index.md index ae59d768e5..9a9f3f34e2 100644 --- a/docs/zh/docs/operate/index.md +++ b/docs/zh/docs/operate/index.md @@ -12,6 +12,7 @@ description: 持续运行 Server，查找部署、监控与恢复步骤。 | 持续运行个人 Server、部署容器或启用认证 | [部署 Server](deploy-server.md) | | 了解统一数据库升级、备份与服务启停设计 | [数据库迁移设计](database-migrations.md) | | 升级或切换后台监督模式 | [迁移 Artifact 处理状态](artifact-processing-migration.md) | +| 导出本地 SQLite 归档或恢复归档 | [可移植归档](portable-archive.md) | | 查看日志、指标和 Trace | [可观测性](observability.md) | | 将 Trace 发送到分析服务 | [Phoenix](trace-with-phoenix.md)、[Langfuse](trace-with-langfuse.md) | | 诊断失败并恢复服务或数据 | [诊断与恢复](troubleshoot.md) | diff --git a/docs/zh/docs/operate/meta.json b/docs/zh/docs/operate/meta.json index 6ada284c16..7abd7b3930 100644 --- a/docs/zh/docs/operate/meta.json +++ b/docs/zh/docs/operate/meta.json @@ -6,6 +6,7 @@ "connect-remote-server", "database-migrations", "artifact-processing-migration", + "portable-archive", "atomic-memory-migration", "observability", "trace-with-phoenix", diff --git a/docs/zh/docs/operate/portable-archive.md b/docs/zh/docs/operate/portable-archive.md new file mode 100644 index 0000000000..77eeecf320 --- /dev/null +++ b/docs/zh/docs/operate/portable-archive.md @@ -0,0 +1,214 @@ +--- +title: 导出和恢复可移植归档 +description: 创建、验证并恢复经过校验的 PowerContext 逻辑归档。 +--- + +# 导出和恢复可移植归档 + +可移植归档是用于迁移或恢复完整 PowerContext scope 的 `.pcb` 文件。它是逻辑归档：保存领域 identity 和不可变历史， +而不是复制 SQLite 文件。它适用于受控的本地备份、离线传输，以及未来的后端迁移。 + +这是一个离线运维命令，不调用远程 Server API。不传 `--env-file` 时打开默认 SQLite 数据库；传入后使用该部署的 +SQLite、SeekDB 或 OceanBase 配置。恢复前应停止 PowerContext 写入进程；导出使用一个数据库 transaction 获取一致的 +逻辑 snapshot。MySQL 模式的后端会为导出事务使用 `REPEATABLE READ`，即使部署默认采用 +`READ COMMITTED`；后续事务仍使用部署的默认隔离级别。 + +## 选择恢复格式 + +需要完整数据库、部署本地状态或时间点恢复时，应使用部署原生备份。`.pcb` 解决的是更窄的需求：在受支持的后端之间迁移 +选定的完整 scope，保留原始 identity、revision、evidence 和重试语义，但不复制访问绑定、宿主本地选择、调度状态和派生索引。 +SQLite 数据库副本不能直接作为 OceanBase 数据库使用；回放后端的 SQL dump 也不会执行这些应用层的范围选择和投影重建。 +schema migration 负责升级现有部署的表结构，并不替代这种 scope 迁移操作。 + +Checksum 用于发现数据损坏，不提供生产者身份认证，也不证明导出内容相对源数据库完整。恢复前仍会执行领域解码和依赖校验。 + +## 前提条件 + +从创建本地数据的同一个 PowerContext revision 安装 CLI。归档命令使用 +`POWERCONTEXT_HOME/powercontext.db` 中持久化的 SQLite 数据库；未设置 `POWERCONTEXT_HOME` 时，PowerContext +会使用操作系统用户数据目录。 + +```bash +uv tool install "powercontext[cli] @ git+https://github.com/oceanbase/powercontext.git@master" +export POWERCONTEXT_HOME="$HOME/.local/share/powercontext" +``` + +归档包含恢复所需的内容和元数据。应按项目数据的访问控制和保留要求，将它放在受保护的位置。格式会排除凭据和已配置的 +provider secret，但不会加密项目内容。 + +## 创建并检查归档 + +传入每一个需要包含的完整 scope。不能用前缀选择 scope，也不能只导出某一种 record。 + +```bash +powercontext archive export \ + --scope-id project:payments \ + --output ./backups/payments.pcb + +powercontext archive inspect ./backups/payments.pcb +``` + +外层存储已经压缩时可增加 `--no-compress`。导出授权一定发生在第一次数据库查询之前；离线命令以操作系统文件权限和数据库 +读取权限作为 operator authority，不复用或导出 Server bearer token。无内容的 JSON 进度写到 stderr，最终 receipt +仍以 JSON 写到 stdout。 + +`export` 会输出 JSON，其中包括 bundle ID、record 数和 checksum。`inspect` 不打开目标数据库，只校验 ZIP 结构、 +单 record digest、总 digest、record 数和所选 scope。若外部备份系统需要独立校验记录，请把输出的 checksum 一并保存。 +导出在发布文件前还会检查格式、资源限制和依赖完整性。验证失败时，输出路径上的旧备份保持不变。 +输出路径不能是源 SQLite 数据库或对应的 `-wal`、`-shm`、`-journal` 文件；导出会拒绝这些路径，不会用归档替换正在使用的存储。同目录中的这些保留文件名不区分大小写，在大小写敏感的文件系统上也一样。 + +## 在恢复前验证 + +恢复前始终先针对目标环境验证： + +```bash +powercontext archive restore ./backups/payments.pcb --dry-run +``` + +dry-run 不会写入领域数据。它会校验 checksum、必需的 Source 和 Artifact 依赖，以及已配置 Runtime 是否支持归档中的 +source type 和 Artifact family。归档无效或无法检查本地 SQLite 文件名时返回退出码 `2`，不兼容或目标冲突时返回 `3`； +错误信息不打印 record body。成功报告 +包含 `already_present`、`conflicts`、所需和不支持的 Source type/Artifact family，以及目标是否支持 projection rebuild。 +可用 `--env-file ./target.env` 检查另一个部署。 + +Candidate version 和 decision head 使用正常读取路径的完整解码器，校验 family 一致性、evidence、原因、生命周期状态， +以及 family 自有的 proposal 协议和可选的 Dream audit envelope。格式错误或不支持的 Candidate 会在写入领域数据前被拒绝。 +已有 SQLite 目标以只读方式检查，支持带 `uri=true` 的 `file:` URI 文件名。 +目标不存在时，dry-run 不会创建数据库文件或父目录。 +这包括 Windows 的绝对 `file:///C:/...` 路径及经过百分号编码的文件名。不支持的 URI authority、无法读取的目标、目录目标 +或非目录的父路径会使验证失败，不会被当成空数据库。 + +发布关系验证会沿精确的 Artifact Revision 地址检查完整来源链。即使所有端点和 checksum 都有效，链中的环和不一致的 +content digest 仍会在领域写入前被拒绝。兼容性检查也会比对待恢复 Revision 在目标中已有的发布关系。归档未声明的目标发布 +关系即使自身合法，也会作为冲突拒绝，防止恢复后的原始来源被改变。关系图使用本地临时存储建立索引，目标关系按批读取， +不扫描无关的发布历史。存在冲突时不会覆盖目标历史；已经损坏的历史需要使用有效备份恢复到干净目标。 + +标签及完整标签集合使用正常领域校验，每个标签目标最多包含 32 个标签。新标签 identity 会追加到目标，不删除已有标签， +合并后的集合也必须满足同一上限。归档内的标签集合超限会作为无效输入拒绝；与目标合并后超限则作为冲突在写入前拒绝。 + +原生 Source 需要目标环境安装对应的 Python adapter。验证会在写入前使用正常读取路径的已注册 Source 解码器，检查 payload +结构与索引 identity 是否一致，也覆盖受支持的 legacy payload。远程 worker 已物化的 `SourceObservation` 自带捕获内容和投影， +其存储 envelope 也使用同一解码器验证，但不需要加载 worker adapter。归档只携带所选 scope 实际引用的 Source Definition +manifest，并在恢复前校验 observation +的身份和定义指纹。缺失定义或与目标定义冲突会阻止恢复；未引用的全局定义不会随包导出。 +Observation 的 payload 和每个 projection 还必须通过正常写入路径所使用的 manifest schema 校验。恢复不会请求远程 schema。 + +要执行写恢复，必须显式确认： + +```bash +powercontext archive restore ./backups/payments.pcb --yes +``` + +结果为 JSON，包含 `inserted`、`already_present` 和 `projections_ready`。只有 `projections_ready` 为 `true` +时才能宣称搜索已就绪。Runtime 会在恢复权威数据后重建 Memory、Topic Memory 和 Experience 搜索投影， +包括已配置的 Memory 和 Topic Memory 向量投影。 + +## 归档保留的内容 + +格式版本 1 支持以下可移植关系记录： + +| 会保留 | 不可移植 | +| --- | --- | +| Scope identity、层级、context/external reference 和创建 identity | Scope access binding 和 host-local default selection | +| Source journal head、Source record 和引用的远程 Source Definition manifest | 搜索 projection 和 index | +| Artifact Revision 及其时间戳、actor、回滚元数据；稳定句柄、lineage、跨 Scope 发布来源和 head | Source cursor 和 scheduler state | +| Atomic Memory 合并关系和带合并输入标记的 lineage | External Skill registration 和 host-local installation state | +| Candidate version、decision head 及其 evidence reference | Audit event、usage fact、evaluation receipt 和 restore receipt | +| 托管 Skill 包的文件内容与 manifest；Artifact 标签 | 下载地址和主机本地 Skill 路径 | +| Profile policy、Prompt revision 和 Topic Memory 发布元数据 | 目标环境的 embedding 配置 | +| 作为 Source 保存的 Work contract、task outcome、Handoff boundary 和 receipt | Credential、bearer token、provider secret 和 host-local Skill 安装状态 | + +归档包含带版本的 manifest（`format_version`、producer version、scope、数量、排除项和总 checksum）以及 NDJSON record。 +逐条 digest 用于比较 canonical 内容是否冲突，总 digest 用于校验 record 流。 +它是权威的 round-trip 格式。CSV 可以单独用于有边界的分析，但无法保留不可变 revision、lineage 或 evidence reference， +不得用于恢复。 + +归档保留已冻结的 recurrence 匹配决策与事件，包括原始幂等键、Source 位置和精确 Artifact revision。 +恢复前会校验候选 revision、Handoff/receipt 引用及事件关联的匹配记录。Experience 当前版本推进后， +历史决策仍保持不变；恢复后的重放复用已保存的决策。 + +恢复后，已合并输入仍保持冻结，带标记的合并输入 lineage 仍可用于撤销合并。这些字段也参与冲突检查： +相同 revision 但合并关系不同的记录，不会被误判为相同内容。 + +Atomic Memory 迁移后，归档保留迁移后的 Atomic Memory identity、不可变 revision 和 lineage。 +原 collection 已退出公共读路径的旧 entry 表记录，以及离线 Memory collection archive，不属于可移植 Runtime 状态。 +导出不会修改或删除源数据库中保留的这些历史。若需要原始迁移前 collection 快照、迁移记录或访问控制历史， +请同时保留部署原生备份。 + +## 恢复和冲突处理 + +恢复是幂等的。重复回放同一归档时，内容相同的 record 会计入 `already_present`。已有不可变 identity 对应不同 payload +时，恢复绝不会覆盖它：命令以退出码 `3` 结束，并回滚本次写 transaction。解决目标数据冲突，或选择一个干净的目标后， +再次运行同一个恢复命令。 + +restore 命令会以维护模式打开 Runtime，不预先创建新的默认 scope。因此，包含规范默认 scope 的归档可以在干净的目标中恢复 +原始 identity 和创建幂等键；下次正常启动时，默认 scope 初始化会复用该创建 identity。默认选择绑定仍属于部署本地状态， +恢复不会覆盖现有部署的选择，也不会覆盖冲突的 scope。 +维护 Runtime 不会启动后台 Artifact 处理 supervisor，即使目标部署配置了后台处理。完成恢复后，再启动正常部署以继续后台工作。 + +如果命令在完成前中断，不能宣称恢复已成功。权威 record 会在 transaction 中写入；失败的写入不会留下看似成功的部分 +恢复。先确认上一次命令已经停止，再对同一目标重新运行同一个归档。 + +目标数据库会按 `bundle_id` 保存 durable receipt。`authoritative_restored` 表示逻辑数据已经提交但搜索尚未重建完成； +`ready` 表示 projection rebuild 已完成。若重建失败，重新恢复同一 bundle 会跳过相同记录，只有验证完成后才把 receipt +推进为 `ready`。 +只要存在 `projections_ready=false` 的回执，正常 Runtime/Server 启动与 worker 启动都会被拒绝，即使归档仅包含 Atomic Memory +或 Experience。显式 restore 路径可以重新打开目标及未完成的 Topic Memory 投影，以继续恢复。请使用相同的目标 embedding +配置重跑同一归档，完成后再提供服务。恢复未完成时，export 返回退出码 `3`，要求先完成恢复。 + +恢复会在目标写事务开始前将记录存入临时磁盘索引，请为归档副本及解压后的记录预留临时磁盘空间。 +已有 MySQL 模式数据库可能需要在 Runtime 启动时将可移植时间戳从 `DATETIME` 升级为 `DATETIME(6)`； +升级前备份数据库，并为升级提供修改表结构的权限。 + +## 从 SQLite 迁移到 OceanBase + +分别创建环境文件，数据库凭据不会写入 bundle： + +```bash +powercontext archive export --env-file ./sqlite.env \ + --scope-id project:payments --output ./payments.pcb +powercontext archive restore ./payments.pcb --env-file ./oceanbase.env --dry-run +powercontext archive restore ./payments.pcb --env-file ./oceanbase.env --yes +``` + +环境文件使用 Server 配置文档中的 `POWERCONTEXT_SERVER_DATABASE_KIND` 和 +`POWERCONTEXT_SERVER_DATABASE_URL`。写入前，dry-run 应报告不存在不支持的 family，且冲突数为零。 + +## 格式兼容窗口 + +Bundle ID 必须使用带连字符、小写的标准 UUID 表示。完整归档文件连同 ZIP 前缀或尾随字节不得超过 512 MiB；manifest +上限为 256 KiB，解压后的 record 流上限为 2 GiB，最多包含 2,000,000 条 record，每行 NDJSON 不超过 8 MiB。 +超限文件会在恢复复制前被拒绝；复制 snapshot 时也会执行相同上限，防止读取期间源文件变化绕过限制。 + +格式版本 `1` reader 还必须支持归档中出现的全部 record type。即使归档使用格式版本 `1`，旧 reader 也可能拒绝新增的 +record type。producer version 仅用于诊断，不能替代 archive schema version 或记录类型兼容性检查。 +writer 只生成版本 `1`，不支持降级到格式 `0`。遇到未知 archive 或 record schema +version 时，reader 会在目标写入前拒绝。跨 PowerContext 大版本升级时，应保留旧二进制，直到恢复演练通过。 + +`artifact_revision`、`artifact_head`、`artifact_lineage_artifact` 和 `candidate_head` 要求 record schema version `2`。 +reader 会拒绝这些记录不完整的 schema version `1` 表示，而不是丢弃 revision 元数据或补上默认合并状态、Candidate 类型。 +旧数据库中 revision 元数据为空时，归档保留显式 null，不会编造 actor、时间戳或回滚历史。目标缺少归档所需列时， +应先用当前 Runtime 升级；dry-run 会报告不兼容，不会修改目标数据库。 +归档不满足兼容检查时，请用当前二进制从源数据库重新导出。 +二进制字段使用 canonical Base64；时间戳使用受支持数据库 profile 输出的 canonical、无时区 ISO 文本。恢复会在写入前拒绝 +替代编码、携带时区偏移的时间戳及超出存储范围的值，不会接受读回后 digest 改变、再次恢复却产生冲突的表示。 + +## 部署原生备份 + +逻辑导出不能替代 crash-consistent 部署备份： + +- SQLite：复制数据库前停止所有 PowerContext writer，或使用 SQLite online backup API，例如 + `sqlite3 powercontext.db ".backup './backups/powercontext.db'"`。随后执行 + `sqlite3 ./backups/powercontext.db "PRAGMA integrity_check"`，按源数据库等级保护备份，并测试重新打开。 +- OceanBase：根据实际部署版本启用 tenant data backup 和 log archiving，独立保管备份目标与加密材料，并在隔离 tenant + 进行恢复演练。point-in-time recovery 使用原生备份；逻辑跨后端迁移使用 portable bundle。 + +## 当前边界 + +导出会把数据库 row 流式写到临时 NDJSON 文件。验证在临时磁盘 index 中保存 record identity 和 Candidate version， +用于校验每个 head 对应的精确版本。恢复则按依赖层流式 +回放，因此归档的聚合 payload 不会常驻进程内存；Scope metadata 由 manifest scope list 约束。命令不是远程 Server +API，也不会把部署配置或数据库凭据写入归档。 + +归档支持 Artifact Candidate；tag-change Candidate 会在导出发布文件或恢复写入记录前被明确拒绝， +不会被静默丢弃或错误解释成 Artifact Candidate。 diff --git a/docs/zh/docs/operate/troubleshoot.md b/docs/zh/docs/operate/troubleshoot.md index a8cd9bd9e7..a619c3b45a 100644 --- a/docs/zh/docs/operate/troubleshoot.md +++ b/docs/zh/docs/operate/troubleshoot.md @@ -260,7 +260,7 @@ collation，但不会包含数据库 URL 或凭据。 ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_portable_restore_receipts,pc_source_cursors,pc_memory_source_windows,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_work_blocks,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_recall_effort_daily,pc_receipt_migration_review,pc_decision_observations,pc_lexical_projection_versions' \ -f ``` diff --git a/e2e/bub/uv.lock b/e2e/bub/uv.lock index 612602e7ea..7755457c3f 100644 --- a/e2e/bub/uv.lock +++ b/e2e/bub/uv.lock @@ -1660,31 +1660,38 @@ client = [ [package.metadata] requires-dist = [ { name = "aiosqlite", marker = "extra == 'builtin'", specifier = ">=0.22,<1" }, + { name = "aiosqlite", marker = "extra == 'cli'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'seekdb'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'server'", specifier = ">=0.22,<1" }, { name = "alembic", marker = "extra == 'builtin'", specifier = ">=1.16,<2" }, + { name = "alembic", marker = "extra == 'cli'", specifier = ">=1.16,<2" }, { name = "alembic", marker = "extra == 'seekdb'", specifier = ">=1.16,<2" }, { name = "alembic", marker = "extra == 'server'", specifier = ">=1.16,<2" }, { name = "apscheduler", marker = "extra == 'builtin'", specifier = ">=3.11,<4" }, + { name = "apscheduler", marker = "extra == 'cli'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'seekdb'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'server'", specifier = ">=3.11,<4" }, { name = "click", marker = "extra == 'cli'", specifier = ">=8.1,<9" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115,<1" }, { name = "fastmcp", marker = "extra == 'server'", specifier = ">=3.4,<4" }, { name = "filelock", marker = "extra == 'builtin'", specifier = ">=3.16,<4" }, + { name = "filelock", marker = "extra == 'cli'", specifier = ">=3.16,<4" }, { name = "filelock", marker = "extra == 'seekdb'", specifier = ">=3.16,<4" }, { name = "filelock", marker = "extra == 'server'", specifier = ">=3.16,<4" }, { name = "httpx", marker = "extra == 'builtin'", specifier = ">=0.28,<1" }, + { name = "httpx", marker = "extra == 'cli'", specifier = ">=0.28,<1" }, { name = "httpx", marker = "extra == 'seekdb'", specifier = ">=0.28,<1" }, { name = "httpx", marker = "extra == 'server'", specifier = ">=0.28,<1" }, { name = "httpx", extras = ["socks"], marker = "extra == 'cli'", specifier = ">=0.28,<1" }, { name = "httpx", extras = ["socks"], marker = "extra == 'client'", specifier = ">=0.28,<1" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3,<1" }, { name = "jieba", marker = "extra == 'builtin'", specifier = "==0.42.1" }, + { name = "jieba", marker = "extra == 'cli'", specifier = "==0.42.1" }, { name = "jieba", marker = "extra == 'seekdb'", specifier = "==0.42.1" }, { name = "jieba", marker = "extra == 'server'", specifier = "==0.42.1" }, { name = "jinja2", marker = "extra == 'server'", specifier = ">=3.1.6,<4" }, { name = "jsonschema", marker = "extra == 'builtin'", specifier = ">=4.23,<5" }, + { name = "jsonschema", marker = "extra == 'cli'", specifier = ">=4.23,<5" }, { name = "jsonschema", marker = "extra == 'seekdb'", specifier = ">=4.23,<5" }, { name = "jsonschema", marker = "extra == 'server'", specifier = ">=4.23,<5" }, { name = "markdown-it-py", marker = "extra == 'server'", specifier = ">=4,<5" }, @@ -1702,9 +1709,13 @@ requires-dist = [ { name = "platformdirs", marker = "extra == 'server'", specifier = ">=4,<5" }, { name = "prometheus-client", marker = "extra == 'server'", specifier = ">=0.21,<1" }, { name = "psutil", marker = "sys_platform == 'darwin' and extra == 'code'", specifier = ">=7.2,<8" }, + { name = "pycasbin", marker = "extra == 'builtin'", specifier = ">=2.8,<3" }, + { name = "pycasbin", marker = "extra == 'cli'", specifier = ">=2.8,<3" }, + { name = "pycasbin", marker = "extra == 'seekdb'", specifier = ">=2.8,<3" }, { name = "pycasbin", marker = "extra == 'server'", specifier = ">=2.8,<3" }, { name = "pydantic", specifier = ">=2.10,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'builtin'", specifier = ">=2.27.1,<3" }, + { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'cli'", specifier = ">=2.27.1,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'seekdb'", specifier = ">=2.27.1,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'server'", specifier = ">=2.27.1,<3" }, { name = "pydantic-settings", marker = "extra == 'builtin'", specifier = ">=2.7,<3" }, @@ -1714,9 +1725,11 @@ requires-dist = [ { name = "pydantic-settings", marker = "extra == 'server'", specifier = ">=2.7,<3" }, { name = "pylibseekdb", marker = "(sys_platform == 'darwin' and extra == 'seekdb') or (sys_platform == 'linux' and extra == 'seekdb')", specifier = ">=1.4.0.post1,<2" }, { name = "pymysql", marker = "extra == 'builtin'", specifier = ">=1.2,<1.2.1" }, + { name = "pymysql", marker = "extra == 'cli'", specifier = ">=1.2,<1.2.1" }, { name = "pymysql", marker = "extra == 'seekdb'", specifier = ">=1.2,<1.2.1" }, { name = "pymysql", marker = "extra == 'server'", specifier = ">=1.2,<1.2.1" }, { name = "pyobvector", marker = "extra == 'builtin'", specifier = ">=0.2.28,<0.3" }, + { name = "pyobvector", marker = "extra == 'cli'", specifier = ">=0.2.28,<0.3" }, { name = "pyobvector", marker = "extra == 'seekdb'", specifier = ">=0.2.28,<0.3" }, { name = "pyobvector", marker = "extra == 'server'", specifier = ">=0.2.28,<0.3" }, { name = "pyyaml", marker = "extra == 'builtin'", specifier = ">=6,<7" }, @@ -1727,9 +1740,11 @@ requires-dist = [ { name = "rfc8785", specifier = ">=0.1.4,<1" }, { name = "scalar-fastapi", marker = "extra == 'server'", specifier = ">=1.8.2,<2" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'builtin'", specifier = ">=2,<3" }, + { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'cli'", specifier = ">=2,<3" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'seekdb'", specifier = ">=2,<3" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'server'", specifier = ">=2,<3" }, { name = "sqlite-vec", marker = "extra == 'builtin'", specifier = ">=0.1.9,<0.2" }, + { name = "sqlite-vec", marker = "extra == 'cli'", specifier = ">=0.1.9,<0.2" }, { name = "sqlite-vec", marker = "extra == 'seekdb'", specifier = ">=0.1.9,<0.2" }, { name = "sqlite-vec", marker = "extra == 'server'", specifier = ">=0.1.9,<0.2" }, { name = "tree-sitter", marker = "extra == 'code'", specifier = "==0.26.0" }, @@ -1740,6 +1755,7 @@ requires-dist = [ { name = "typer", marker = "extra == 'cli'", specifier = ">=0.16,<1" }, { name = "typing-extensions", specifier = ">=4.12,<5" }, { name = "tzdata", marker = "extra == 'builtin'", specifier = ">=2026.3" }, + { name = "tzdata", marker = "extra == 'cli'", specifier = ">=2026.3" }, { name = "tzdata", marker = "extra == 'seekdb'", specifier = ">=2026.3" }, { name = "tzdata", marker = "extra == 'server'", specifier = ">=2026.3" }, { name = "uvicorn", marker = "extra == 'server'", specifier = ">=0.34,<1" }, diff --git a/pyproject.toml b/pyproject.toml index cd42eaf4c9..7cf06e7214 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ builtin = [ "pydantic-ai-slim[anthropic,openai]>=2.27.1,<3", "pydantic-settings>=2.7,<3", "pyyaml>=6,<7", + "pycasbin>=2.8,<3", "pyobvector>=0.2.28,<0.3", # aiomysql 0.3.2 cannot import/encode bytes with PyMySQL 1.2.1-1.2.3. # This temporary bound covers both OceanBase and embedded seekdb. @@ -88,7 +89,6 @@ server = [ "platformdirs>=4,<5", "powercontext[builtin]", "prometheus-client>=0.21,<1", - "pycasbin>=2.8,<3", "pydantic-settings>=2.7,<3", "scalar-fastapi>=1.8.2,<2", "uvicorn>=0.34,<1", @@ -101,6 +101,7 @@ cli = [ "inquirerpy>=0.3,<1", "packaging>=24,<27", "platformdirs>=4,<5", + "powercontext[builtin]", "powercontext[client]", "pyyaml>=6,<7", "typer>=0.16,<1", @@ -116,6 +117,7 @@ doctor = "powercontext.cli.system:doctor_app" service = "powercontext.service.cli:app" server = "powercontext.server.cli:app" setup = "powercontext.cli.system:setup_app" +archive = "powercontext.cli.archive:archive_app" [project.urls] Homepage = "https://github.com/oceanbase/powercontext" diff --git a/scripts/ci_role_smoke.py b/scripts/ci_role_smoke.py index 633e483482..c18980606c 100644 --- a/scripts/ci_role_smoke.py +++ b/scripts/ci_role_smoke.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Check installed role behavior; run with the isolated wheel environment's Python -I.""" +"""Check installed roles, including the CLI's builtin dependency for offline archives, using Python -I.""" from __future__ import annotations @@ -63,7 +63,7 @@ def _check_cli(root: Path, extras: set[str]) -> None: environment: dict[str, str] = {**os.environ, "POWERCONTEXT_HOME": str(root / "home"), "NO_COLOR": "1"} environment.pop("PYTHONPATH", None) environment.pop("PYTHONHOME", None) - commands = [(), ("config",), ("setup",), ("doctor",), ("skill", "remote-sync")] + commands = [(), ("config",), ("setup",), ("doctor",), ("skill", "remote-sync"), ("archive",)] if extras & {"builtin", "server"}: commands.append(("code",)) if "server" in extras: @@ -101,6 +101,8 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("extras", choices=("builtin", "cli", "builtin,cli", "server,cli")) extras = set(parser.parse_args().extras.split(",")) + if "cli" in extras: + extras.add("builtin") with tempfile.TemporaryDirectory(prefix="powercontext-role-") as temporary: root = Path(temporary) if extras & {"builtin", "server"}: diff --git a/src/powercontext/builtin/persistence/candidates.py b/src/powercontext/builtin/persistence/candidates.py index a0f3792318..9431da86b8 100644 --- a/src/powercontext/builtin/persistence/candidates.py +++ b/src/powercontext/builtin/persistence/candidates.py @@ -27,7 +27,7 @@ from powercontext.artifacts import ArtifactRef from powercontext.builtin.evidence.models import content_digest from powercontext.builtin.persistence.codec import dump_model, load_model, stored_bytes -from powercontext.builtin.persistence.errors import InvalidRepositoryArgumentError +from powercontext.builtin.persistence.errors import InvalidRepositoryArgumentError, InvalidStoredPayloadError from powercontext.builtin.persistence.tables import ( CANDIDATE_HEADS_TABLE, CANDIDATE_VERSIONS_TABLE, @@ -125,7 +125,7 @@ async def get( row = await self._find_current(connection, scope_id, candidate_id, current=current) if row is None: raise CandidateNotFoundError(candidate_id) - return self._decode_row(row) + return self.decode_candidate(row) async def page(self, connection, scope_id, *, status, family, candidate_kind, cursor, limit, metadata_only=False): """Read a stable bounded window; Tag proposals retain their required target basis.""" @@ -170,7 +170,7 @@ async def page(self, connection, scope_id, *, status, family, candidate_kind, cu if row["candidate_kind"] == "tag" else CandidateCatalogItem.model_validate(row) if metadata_only - else self._decode_row(row) + else self.decode_candidate(row) for row in rows[:limit] ) return items, rows[limit - 1]["candidate_id"] if len(rows) > limit else None @@ -393,25 +393,23 @@ async def _find_current( statement = statement.with_for_update() return (await connection.execute(statement)).mappings().one_or_none() - def _decode_row(self, row: Mapping[Any, Any]) -> Candidate[Any]: + def decode_candidate(self, row: Mapping[Any, Any], /) -> Candidate[Any]: + """Decode and validate the complete stored Candidate using its public read contract.""" + family = str(row["family"]) proposal_type = self._proposal_types.get(family) if proposal_type is None: raise InvalidCandidateError("family", family) target_family = row["target_family"] result_family = row["result_family"] - proposal_payload, audit = _decode_proposal(row["proposal"]) + proposal_payload, audit = decode_candidate_proposal_envelope(stored_bytes(row["proposal"], column="proposal")) + proposal = load_model(proposal_type, proposal_payload, kind="candidate-proposal", name=family) return Candidate( candidate_id=str(row["candidate_id"]), version=int(row["version"]), family=family, status=CandidateStatus(str(row["status"])), - proposal=load_model( - proposal_type, - proposal_payload, - kind="candidate-proposal", - name=family, - ), + proposal=proposal, sources=load_model( _SourceRefs, stored_bytes(row["source_refs"], column="source_refs"), @@ -478,13 +476,17 @@ def _encode_proposal(candidate: Candidate[Any]) -> bytes: ).encode() -def _decode_proposal(value: object) -> tuple[bytes, CandidateAudit | None]: - payload = stored_bytes(value, column="proposal") - decoded = json.loads(payload) - if isinstance(decoded, dict) and decoded.get("candidate_payload_version") == 1: - return json.dumps(decoded["proposal"], ensure_ascii=False).encode(), CandidateAudit.model_validate( - decoded["audit"] - ) +def decode_candidate_proposal_envelope(payload: bytes, /) -> tuple[bytes, CandidateAudit | None]: + """Unwrap persisted proposal provenance without changing the family-owned payload.""" + + try: + decoded = json.loads(payload) + if isinstance(decoded, dict) and decoded.get("candidate_payload_version") == 1: + return json.dumps(decoded["proposal"], ensure_ascii=False).encode(), CandidateAudit.model_validate( + decoded["audit"] + ) + except (KeyError, ValueError, TypeError) as error: + raise InvalidStoredPayloadError("candidate-proposal", "envelope", "invalid proposal envelope") from error return payload, None @@ -515,4 +517,4 @@ def _require_scope(scope_id: object) -> None: ) -__all__ = ["CandidateRepository"] +__all__ = ["CandidateRepository", "decode_candidate_proposal_envelope"] diff --git a/src/powercontext/builtin/persistence/oceanbase/profile.py b/src/powercontext/builtin/persistence/oceanbase/profile.py index 68e8a72f5c..c8c0a88879 100644 --- a/src/powercontext/builtin/persistence/oceanbase/profile.py +++ b/src/powercontext/builtin/persistence/oceanbase/profile.py @@ -180,6 +180,12 @@ async def _require_mysql_tenant(connection: AsyncConnection) -> None: result = await connection.exec_driver_sql("SHOW VARIABLES LIKE 'ob_compatibility_mode'") row = result.first() mode = None if row is None or len(row) < 2 else str(row[1]).upper() + if row is None: + # Standalone SeekDB 1.4 is MySQL-only and no longer exposes the + # multi-tenant compatibility variable. Keep unknown servers rejected. + version_row = (await connection.exec_driver_sql("SELECT VERSION()")).first() + if version_row and "-OceanBase seekdb-" in str(version_row[0]): + return if mode != "MYSQL": raise UnsupportedOceanBaseTenantError(mode) diff --git a/src/powercontext/builtin/persistence/oceanbase/topic_memory_index.py b/src/powercontext/builtin/persistence/oceanbase/topic_memory_index.py index b344c7ccf3..efffa89d6d 100644 --- a/src/powercontext/builtin/persistence/oceanbase/topic_memory_index.py +++ b/src/powercontext/builtin/persistence/oceanbase/topic_memory_index.py @@ -70,7 +70,7 @@ _TOPIC_FTS_INDEX = "ix_pc_topic_memory_active_topics_fts" _CHUNK_FTS_INDEX = "ix_pc_topic_memory_active_chunks_fts" -_FTS_DDL_RETRY_DELAYS = (0.1, 0.2, 0.4, 0.8, 1.6) +_INDEX_DDL_RETRY_DELAYS = (0.1, 0.2, 0.4, 0.8, 1.6) _TOPIC_VECTOR_TABLE = "pc_topic_memory_vector_topics" _CHUNK_VECTOR_TABLE = "pc_topic_memory_vector_chunks" _TOPIC_VECTOR_INDEX = "ix_pc_topic_memory_vector_topics_embedding" @@ -327,7 +327,7 @@ async def search( @staticmethod async def _ensure_index(connection: AsyncConnection, table_name: str, index_name: str) -> None: - for attempt in range(len(_FTS_DDL_RETRY_DELAYS) + 1): + for attempt in range(len(_INDEX_DDL_RETRY_DELAYS) + 1): count = await connection.scalar( _INDEX_EXISTS_SQL, {"table_name": table_name, "index_name": index_name}, @@ -347,10 +347,10 @@ async def _ensure_index(connection: AsyncConnection, table_name: str, index_name error.orig is None or not error.orig.args or error.orig.args[0] != 1205 - or attempt == len(_FTS_DDL_RETRY_DELAYS) + or attempt == len(_INDEX_DDL_RETRY_DELAYS) ): raise - await asyncio.sleep(_FTS_DDL_RETRY_DELAYS[attempt]) + await asyncio.sleep(_INDEX_DDL_RETRY_DELAYS[attempt]) else: return @@ -417,7 +417,20 @@ async def initialize(self, connection: AsyncConnection, /) -> None: "vector", f"OceanBase projection {table.name} uses {actual!r}; expected {expected}", ) - await connection.run_sync(lambda sync: self._topic_index.create(sync, checkfirst=True)) + for attempt in range(len(_INDEX_DDL_RETRY_DELAYS) + 1): + try: + await connection.run_sync(lambda sync: self._topic_index.create(sync, checkfirst=True)) + except OperationalError as error: + if ( + error.orig is None + or not error.orig.args + or error.orig.args[0] != 1205 + or attempt == len(_INDEX_DDL_RETRY_DELAYS) + ): + raise + await asyncio.sleep(_INDEX_DDL_RETRY_DELAYS[attempt]) + else: + return async def validate_current(self, connection: AsyncConnection, /) -> None: await validate_current_topic_vectors(connection, self.topic_table, self.chunk_table, self._fingerprint) diff --git a/src/powercontext/builtin/persistence/receipt_migration.py b/src/powercontext/builtin/persistence/receipt_migration.py index e1d21d71cb..36b3b450eb 100644 --- a/src/powercontext/builtin/persistence/receipt_migration.py +++ b/src/powercontext/builtin/persistence/receipt_migration.py @@ -73,7 +73,7 @@ async def migrate_handoff_receipts( # noqa: C901 - Keep receipt classification async with connection.stream(statement.execution_options(yield_per=1)) as result: async for row in result.mappings(): seen.add(row["source_id"]) - source = sources._decode_row(row).value + source = sources.decode_source(row).value if isinstance(source, ContentSource): identity = (row["scope_id"], row["source_id"]) if source.handoff_receipt: diff --git a/src/powercontext/builtin/persistence/schema.py b/src/powercontext/builtin/persistence/schema.py index d1e4170bc3..57c0f97112 100644 --- a/src/powercontext/builtin/persistence/schema.py +++ b/src/powercontext/builtin/persistence/schema.py @@ -18,7 +18,7 @@ from collections.abc import Sequence -from sqlalchemy import MetaData, Table +from sqlalchemy import MetaData, Table, text from sqlalchemy.ext.asyncio import AsyncConnection @@ -37,3 +37,33 @@ async def create_tables(connection: AsyncConnection, tables: Sequence[Table], /) checkfirst=True, ) ) + + +async def ensure_portable_timestamp_precision(connection: AsyncConnection, tables: Sequence[Table]) -> None: + """Explicit runtime schema upgrade preserving portable microsecond timestamps.""" + + if connection.dialect.name != "mysql": + return + + portable_columns = { + "pc_artifacts": "created_at", + "pc_profile_policies": "updated_at", + "pc_topic_memory_revision_publications": "published_at", + "pc_artifact_tags": "assigned_at", + } + for table in tables: + column = portable_columns.get(table.name) + if column is None: + continue + precision = await connection.scalar( + text( + "SELECT DATETIME_PRECISION FROM information_schema.COLUMNS " + "WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = :table_name AND COLUMN_NAME = :column_name" + ), + {"table_name": table.name, "column_name": column}, + ) + if precision is not None and int(precision) < 6: + nullability = "NULL" if table.c[column].nullable else "NOT NULL" + await connection.exec_driver_sql( + f"ALTER TABLE `{table.name}` MODIFY COLUMN `{column}` DATETIME(6) {nullability}" + ) diff --git a/src/powercontext/builtin/persistence/sources.py b/src/powercontext/builtin/persistence/sources.py index ba3cd56c56..260ffdd2b7 100644 --- a/src/powercontext/builtin/persistence/sources.py +++ b/src/powercontext/builtin/persistence/sources.py @@ -119,7 +119,7 @@ async def add_with_status( await _lock_journal_head(connection, scope_id) existing = await self._find_row(connection, scope_id, ref) if existing is not None: - stored = self._decode_row(existing) + stored = self.decode_source(existing) # A trusted acknowledgement may replay a pre-provenance receipt. # Upgrade only its server-owned attestation, never its content or position. if ( @@ -161,7 +161,7 @@ async def add_with_status( existing = await self._find_row(connection, scope_id, ref) if existing is None: raise - stored = self._decode_row(existing) + stored = self.decode_source(existing) if stored.value != source: raise StoredPayloadConflictError("source", (scope_id, ref)) from None return stored, False @@ -186,7 +186,7 @@ async def get( row = await self._find_row(connection, scope_id, ref) if row is None: raise RepositoryNotFoundError("source", (scope_id, ref)) - return self._decode_row(row) + return self.decode_source(row) async def get_many( self, @@ -223,7 +223,7 @@ async def get_many( decoded: dict[tuple[str, str], StoredSource] = {} for row in rows: try: - stored = self._decode_row(row) + stored = self.decode_source(row) except RepositoryNotFoundError: ref = SourceRef(source_type=str(row["source_type"]), source_id=str(row["source_id"])) raise RepositoryNotFoundError("source", (scope_id, ref)) from None @@ -291,7 +291,7 @@ async def iter_list( statement = statement.limit(limit) async with connection.stream(statement.execution_options(yield_per=1)) as result: async for row in result.mappings(): - yield self._decode_row(row) + yield self.decode_source(row) async def list_window( self, @@ -324,7 +324,7 @@ async def list_window( .mappings() .all() ) - return tuple(self._decode_row(row) for row in rows) + return tuple(self.decode_source(row) for row in rows) async def journal_position(self, connection: AsyncConnection, scope_id: str, /) -> int: """Return the current high watermark for one scope.""" @@ -370,7 +370,9 @@ async def _find_row( .one_or_none() ) - def _decode_row(self, row: Mapping[Any, Any]) -> StoredSource: + def decode_source(self, row: Mapping[Any, Any], /) -> StoredSource: + """Decode and validate one stored Source using the registered adapter contract.""" + source_type = str(row["source_type"]) source_id = str(row["source_id"]) payload = stored_bytes(row["payload"], column="payload") diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index 2793f9d9d0..a548e5f4e6 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -32,7 +32,7 @@ Text, UniqueConstraint, ) -from sqlalchemy.dialects.mysql import BINARY, MEDIUMBLOB, MEDIUMTEXT, VARCHAR +from sqlalchemy.dialects.mysql import BINARY, DATETIME, MEDIUMBLOB, MEDIUMTEXT, VARCHAR from powercontext.limits import ( MAX_ARTIFACT_FAMILY_LENGTH, @@ -197,7 +197,7 @@ def _entry_text_type(): Column("artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), Column("revision", Integer, primary_key=True), Column("content", _canonical_payload_type(), nullable=False), - Column("created_at", DateTime(timezone=True), nullable=True), + Column("created_at", DateTime(timezone=True).with_variant(DATETIME(fsp=6), "mysql"), nullable=True), Column("created_by", String(MAX_ARTIFACT_REVISION_ACTOR_LENGTH), nullable=True), Column("restored_from_revision", Integer, nullable=True), Column("rollback_reason", String(1024), nullable=True), @@ -450,6 +450,27 @@ def _entry_text_type(): ), ) +PORTABLE_RESTORE_RECEIPTS_TABLE = Table( + "pc_portable_restore_receipts", + SHARED_METADATA, + Column("bundle_id", identity_string(36), primary_key=True), + Column("format_version", Integer, nullable=False), + Column("total_digest", identity_string(71), nullable=False), + Column("record_count", BigInteger, nullable=False), + Column("status", identity_string(32), nullable=False), + Column("inserted", BigInteger, nullable=False), + Column("already_present", BigInteger, nullable=False), + Column("projections_ready", Boolean, nullable=False), + Column("updated_at", DateTime(timezone=True), nullable=False), + CheckConstraint( + "status IN ('authoritative_restored', 'ready')", + name="ck_pc_portable_restore_receipts_status", + ), + CheckConstraint("record_count >= 0", name="ck_pc_portable_restore_receipts_record_count"), + CheckConstraint("inserted >= 0", name="ck_pc_portable_restore_receipts_inserted"), + CheckConstraint("already_present >= 0", name="ck_pc_portable_restore_receipts_existing"), +) + PROFILE_POLICIES_TABLE = Table( "pc_profile_policies", SHARED_METADATA, @@ -458,7 +479,7 @@ def _entry_text_type(): Column("activation_mode", identity_string(32), nullable=False), Column("pending_candidate_id", identity_string(MAX_ARTIFACT_ID_LENGTH)), Column("version", BigInteger, nullable=False), - Column("updated_at", DateTime(timezone=True), nullable=False), + Column("updated_at", DateTime(timezone=True).with_variant(DATETIME(fsp=6), "mysql"), nullable=False), ForeignKeyConstraint(("scope_id",), ("pc_scopes.scope_id",), ondelete="CASCADE"), ForeignKeyConstraint( ("scope_id", "pending_candidate_id"), @@ -701,7 +722,7 @@ def _entry_text_type(): Column("family", identity_string(MAX_ARTIFACT_FAMILY_LENGTH), primary_key=True), Column("artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), Column("revision", Integer, primary_key=True), - Column("published_at", DateTime(timezone=False), nullable=False), + Column("published_at", DateTime(timezone=False).with_variant(DATETIME(fsp=6), "mysql"), nullable=False), ForeignKeyConstraint( ("scope_id", "family", "artifact_id", "revision"), ( @@ -1047,6 +1068,7 @@ def _entry_text_type(): ARTIFACT_PUBLICATIONS_TABLE, CANDIDATE_VERSIONS_TABLE, CANDIDATE_HEADS_TABLE, + PORTABLE_RESTORE_RECEIPTS_TABLE, PROFILE_POLICIES_TABLE, SOURCE_CURSORS_TABLE, MEMORY_SOURCE_WINDOWS_TABLE, @@ -1162,7 +1184,7 @@ def _entry_text_type(): Column("tag_key_hash", LargeBinary(32).with_variant(BINARY(32), "mysql"), primary_key=True), Column("tag_key", identity_string(128), nullable=False), Column("tag", String(64), nullable=False), - Column("assigned_at", DateTime(timezone=True), nullable=False), + Column("assigned_at", DateTime(timezone=True).with_variant(DATETIME(fsp=6), "mysql"), nullable=False), ForeignKeyConstraint( ("scope_id", "family", "artifact_id"), ("pc_artifact_heads.scope_id", "pc_artifact_heads.family", "pc_artifact_heads.artifact_id"), diff --git a/src/powercontext/builtin/persistence/topic_memory.py b/src/powercontext/builtin/persistence/topic_memory.py index 9bf1f6a925..970c18c973 100644 --- a/src/powercontext/builtin/persistence/topic_memory.py +++ b/src/powercontext/builtin/persistence/topic_memory.py @@ -597,6 +597,38 @@ async def _activate( published_at=published_at, ) ) + await self.rebuild_current(connection, scope_id, topic, projection) + return published_at + + async def rebuild_current( + self, + connection: AsyncConnection, + scope_id: str, + topic: TopicMemory, + projection: TopicMemoryProjection, + /, + ) -> None: + """Replace one current Topic Memory's target-local search projection.""" + + projection = self._validate_projection(TopicMemoryDraft(content=topic.content), projection) + ref = topic.as_ref() + current_revision = await connection.scalar( + select(ARTIFACT_HEADS_TABLE.c.revision).where( + ARTIFACT_HEADS_TABLE.c.scope_id == scope_id, + ARTIFACT_HEADS_TABLE.c.family == TopicMemory.family, + ARTIFACT_HEADS_TABLE.c.artifact_id == ref.artifact_id, + ) + ) + publication = await connection.scalar( + select(TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE.c.revision).where( + TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE.c.scope_id == scope_id, + TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE.c.family == TopicMemory.family, + TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE.c.artifact_id == ref.artifact_id, + TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE.c.revision == ref.revision, + ) + ) + if current_revision != ref.revision or publication != ref.revision: + raise TopicMemoryStorageInvariantError("projection-source", (scope_id, ref)) await self.index.replace(connection, scope_id, ref, projection) await connection.execute( delete(TOPIC_MEMORY_ACTIVE_CHUNKS_TABLE).where( @@ -640,7 +672,6 @@ async def _activate( for chunk in projection.chunks ], ) - return published_at def _validate_projection( self, diff --git a/src/powercontext/builtin/portability/__init__.py b/src/powercontext/builtin/portability/__init__.py new file mode 100644 index 0000000000..b0928f7d84 --- /dev/null +++ b/src/powercontext/builtin/portability/__init__.py @@ -0,0 +1,43 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Versioned logical export and restore for built-in relational data.""" + +from powercontext.builtin.portability.bundle import ( + BundleCompatibilityError, + BundleConflictError, + BundleFormatError, + BundleInspection, + BundleProgress, + BundleReceipt, + BundleRecoveryRequiredError, + BundleValidation, + ExportAuthorizer, + PortableBundleService, + ProgressObserver, +) + +__all__ = [ + "BundleCompatibilityError", + "BundleConflictError", + "BundleFormatError", + "BundleInspection", + "BundleProgress", + "BundleReceipt", + "BundleRecoveryRequiredError", + "BundleValidation", + "ExportAuthorizer", + "PortableBundleService", + "ProgressObserver", +] diff --git a/src/powercontext/builtin/portability/bundle.py b/src/powercontext/builtin/portability/bundle.py new file mode 100644 index 0000000000..54bb08b822 --- /dev/null +++ b/src/powercontext/builtin/portability/bundle.py @@ -0,0 +1,2579 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Portable, verified logical bundles for built-in relational storage. + +The archive deliberately contains domain rows only. Database files, search +indexes, trigger cursors, usage statistics, and host-local skill registrations +are deployment details and are not portable state. +""" + +# Error reason strings are the public inspection/validation contract. +# ruff: noqa: TRY003 + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import json +import re +import shutil +import sqlite3 +import tempfile +import zipfile +import zlib +from collections import Counter +from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, date, datetime +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any, Literal, cast +from uuid import NAMESPACE_URL, UUID, uuid5 + +from pydantic import ValidationError +from sqlalchemy import ( + BigInteger, + DateTime, + Integer, + PrimaryKeyConstraint, + String, + UniqueConstraint, + and_, + insert, + or_, + select, + tuple_, + update, +) +from sqlalchemy import inspect as sqlalchemy_inspect +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.artifacts import StableHandle +from powercontext.builtin.artifacts.experience.recurrence import ( + RecurrenceMatch, + RecurrenceObservation, + canonical_digest, + match_key, + observation_identity, + selection_key, + verdict_key, +) +from powercontext.builtin.artifacts.skill.package import SkillPackageError, capture_skill_archive +from powercontext.builtin.persistence.candidates import decode_candidate_proposal_envelope +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.errors import ( + IdentityMismatchError, + InvalidStoredPayloadError, + RepositoryNotFoundError, +) +from powercontext.builtin.persistence.tables import ( + ARTIFACT_HANDLES_TABLE, + ARTIFACT_HEADS_TABLE, + ARTIFACT_LINEAGE_ARTIFACTS_TABLE, + ARTIFACT_LINEAGE_SOURCES_TABLE, + ARTIFACT_PUBLICATIONS_TABLE, + ARTIFACT_TAGS_TABLE, + ARTIFACTS_TABLE, + CANDIDATE_HEADS_TABLE, + CANDIDATE_VERSIONS_TABLE, + MEMORY_ENTRY_HEADS_TABLE, + MEMORY_ENTRY_VERSIONS_TABLE, + PORTABLE_RESTORE_RECEIPTS_TABLE, + PROFILE_POLICIES_TABLE, + RECURRENCE_MATCH_TABLE, + RECURRENCE_OBSERVATION_TABLE, + SCOPE_CONTEXT_REFERENCES_TABLE, + SCOPE_CREATION_REQUESTS_TABLE, + SCOPE_EXTERNAL_REFERENCES_TABLE, + SCOPES_TABLE, + SKILL_PACKAGES_TABLE, + SOURCE_DEFINITION_MANIFESTS_TABLE, + SOURCE_JOURNAL_HEADS_TABLE, + SOURCES_TABLE, + TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE, +) +from powercontext.builtin.records import ArtifactRevisionActor, InvalidBaseAccessRequestError +from powercontext.builtin.scope.models import ScopeExternalReference +from powercontext.builtin.sources.observation_validation import ( + validate_source_definition_manifest, + validate_source_observation, +) +from powercontext.builtin.tags import normalize_tags +from powercontext.errors import SourceError +from powercontext.sources import SourceDefinitionManifest, SourceObservation + +FORMAT_VERSION = 1 +_RECORDS_NAME = "records.ndjson" +_MANIFEST_NAME = "manifest.json" +_SHA256 = "sha256:" +_MAX_ARCHIVE_BYTES = 512 * 1024 * 1024 +_MAX_RECORDS_BYTES = 2 * 1024 * 1024 * 1024 +_MAX_RECORDS = 2_000_000 +_MAX_LINE_BYTES = 8 * 1024 * 1024 +_MAX_MANIFEST_BYTES = 256 * 1024 +_HEX_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") +_ABSOLUTE_HOST_PATH = re.compile(r"^(?:/|file:/+|[A-Za-z]:[\\/]|\\\\)") +_SENSITIVE_KEYS = frozenset({ + "access_token", + "api_key", + "apikey", + "bearer_token", + "credential", + "credentials", + "password", + "provider_secret", + "refresh_token", + "secret", + "token", +}) +_EXCLUDED_RECORD_CLASSES = ( + "access_bindings", + "audit_records", + "credentials_and_provider_secrets", + "evaluation_receipts", + "external_skill_registrations", + "host_local_installation_state", + "legacy_memory_migration_history", + "restore_receipts", + "scope_default_selection", + "search_projections", + "source_cursors", + "usage_statistics", +) + +RecordType = Literal[ + "scope", + "scope_context_reference", + "scope_external_reference", + "scope_creation_request", + "source_definition", + "source_journal_head", + "source", + "skill_package", + "artifact_revision", + "artifact_handle", + "artifact_lineage_source", + "artifact_lineage_artifact", + "artifact_publication", + "artifact_head", + "memory_entry_version", + "memory_entry_head", + "candidate_version", + "candidate_head", + "topic_memory_publication", + "profile_policy", + "artifact_tag", + "recurrence_match", + "recurrence_observation", +] +ProjectionRebuilder = Callable[[tuple[str, ...]], Awaitable[None]] +ExportAuthorizer = Callable[[tuple[str, ...]], Awaitable[None]] +ProgressObserver = Callable[["BundleProgress"], None] + + +class BundleFormatError(ValueError): + """The supplied archive is malformed, corrupt, or unsupported.""" + + +class BundleConflictError(ValueError): + """A target immutable identity exists with different canonical content.""" + + +class BundleCompatibilityError(BundleFormatError): + """The target does not support the bundle's otherwise valid requirements.""" + + +class BundleRecoveryRequiredError(RuntimeError): + """A committed restore needs its projections rebuilt before normal startup.""" + + def __init__(self) -> None: + super().__init__("portable restore projections are incomplete; finish archive restore before serving requests") + + +@dataclass(frozen=True, slots=True) +class BundleInspection: + """Content-free facts established by parsing and verifying a bundle.""" + + bundle_id: str + scopes: tuple[str, ...] + record_count: int + records_by_type: Mapping[str, int] + total_digest: str + format_version: int + producer_version: str + + +@dataclass(frozen=True, slots=True) +class BundleValidation(BundleInspection): + """Target compatibility established without performing writes.""" + + already_present: int + conflicts: int + projections_supported: bool + compatible: bool + required_source_types: tuple[str, ...] + unsupported_source_types: tuple[str, ...] + required_artifact_families: tuple[str, ...] + unsupported_artifact_families: tuple[str, ...] + reader_version: str + + +@dataclass(frozen=True, slots=True) +class BundleReceipt: + """Content-free result of an export or restore.""" + + bundle_id: str + record_count: int + total_digest: str + inserted: int = 0 + already_present: int = 0 + projections_ready: bool = False + status: Literal["exported", "authoritative_restored", "ready"] = "exported" + + +@dataclass(frozen=True, slots=True) +class BundleProgress: + """Content-free progress suitable for terminals and automation logs.""" + + phase: Literal["export", "inspect", "validate", "target", "restore", "projections"] + processed: int + total: int | None + + +@dataclass(frozen=True, slots=True) +class _Record: + record_type: RecordType + identity: Mapping[str, str | int] + payload: Mapping[str, Any] + digest: str + + +@dataclass(frozen=True, slots=True) +class _RecordSpec: + """Stable logical fields and their current relational persistence adapter.""" + + table: Any + identity: tuple[str, ...] + payload: tuple[str, ...] + scope_field: str | None = "scope_id" + schema_version: int = 1 + + +_SPECS: dict[RecordType, _RecordSpec] = { + "scope": _RecordSpec(SCOPES_TABLE, ("scope_id",), ("title", "summary", "parent_scope_id", "version")), + "scope_context_reference": _RecordSpec(SCOPE_CONTEXT_REFERENCES_TABLE, ("scope_id", "referenced_scope_id"), ()), + "scope_external_reference": _RecordSpec( + SCOPE_EXTERNAL_REFERENCES_TABLE, + ("scope_id", "ordinal"), + ("kind", "value", "value_digest"), + ), + "scope_creation_request": _RecordSpec( + SCOPE_CREATION_REQUESTS_TABLE, + ("idempotency_key",), + ("request_digest", "scope_id"), + ), + "source_definition": _RecordSpec( + SOURCE_DEFINITION_MANIFESTS_TABLE, + ("definition_name", "definition_version"), + ("fingerprint", "manifest"), + scope_field=None, + ), + "source_journal_head": _RecordSpec(SOURCE_JOURNAL_HEADS_TABLE, ("scope_id",), ("position",)), + "source": _RecordSpec( + SOURCES_TABLE, + ("scope_id", "source_type", "source_id"), + ("payload", "journal_position"), + ), + "skill_package": _RecordSpec( + SKILL_PACKAGES_TABLE, + ("scope_id", "tree_digest"), + ( + "archive_digest", + "archive_bytes", + "manifest", + "file_count", + "uncompressed_size", + "archive_size", + ), + ), + "artifact_revision": _RecordSpec( + ARTIFACTS_TABLE, + ("scope_id", "family", "artifact_id", "revision"), + ("content", "created_at", "created_by", "restored_from_revision", "rollback_reason"), + schema_version=2, + ), + "artifact_handle": _RecordSpec( + ARTIFACT_HANDLES_TABLE, + ("scope_id", "family", "handle_key"), + ("artifact_id", "revision"), + ), + "artifact_lineage_source": _RecordSpec( + ARTIFACT_LINEAGE_SOURCES_TABLE, + ("scope_id", "family", "artifact_id", "revision", "ordinal"), + ("source_type", "source_id"), + ), + "artifact_lineage_artifact": _RecordSpec( + ARTIFACT_LINEAGE_ARTIFACTS_TABLE, + ("scope_id", "family", "artifact_id", "revision", "ordinal"), + ("upstream_family", "upstream_artifact_id", "upstream_revision", "is_merge_input"), + schema_version=2, + ), + "artifact_publication": _RecordSpec( + ARTIFACT_PUBLICATIONS_TABLE, + ("target_scope_id", "target_family", "target_artifact_id", "target_revision"), + ( + "source_scope_id", + "source_family", + "source_artifact_id", + "source_revision", + "content_digest", + "idempotency_key", + ), + scope_field="target_scope_id", + ), + "artifact_head": _RecordSpec( + ARTIFACT_HEADS_TABLE, + ("scope_id", "family", "artifact_id"), + ("revision", "lifecycle_state", "replacement_artifact_id", "governance_generation", "merged_into_id"), + schema_version=2, + ), + "memory_entry_version": _RecordSpec( + MEMORY_ENTRY_VERSIONS_TABLE, + ("scope_id", "memory_artifact_id", "entry_version_id"), + ( + "family", + "entry_id", + "version", + "previous_version_id", + "kind", + "text", + "source_refs", + "artifact_refs", + "entry_content_hash", + "created_in_revision", + ), + ), + "memory_entry_head": _RecordSpec( + MEMORY_ENTRY_HEADS_TABLE, + ("scope_id", "memory_artifact_id", "entry_id"), + ("family", "head_revision", "entry_version_id", "entry_content_hash"), + ), + "candidate_version": _RecordSpec( + CANDIDATE_VERSIONS_TABLE, + ("scope_id", "candidate_id", "version"), + ( + "family", + "proposal", + "source_refs", + "artifact_refs", + "target_family", + "target_artifact_id", + "target_revision", + "reason", + ), + ), + "candidate_head": _RecordSpec( + CANDIDATE_HEADS_TABLE, + ("scope_id", "candidate_id"), + ( + "family", + "version", + "status", + "result_family", + "result_artifact_id", + "result_revision", + "decision_reason", + "candidate_kind", + "result_payload", + ), + schema_version=2, + ), + "topic_memory_publication": _RecordSpec( + TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE, + ("scope_id", "family", "artifact_id", "revision"), + ("published_at",), + ), + "profile_policy": _RecordSpec( + PROFILE_POLICIES_TABLE, + ("scope_id",), + ("generation_enabled", "activation_mode", "pending_candidate_id", "version", "updated_at"), + ), + "artifact_tag": _RecordSpec( + ARTIFACT_TAGS_TABLE, + ("scope_id", "family", "artifact_id", "target_type", "target_id", "tag_key"), + ("tag_key_hash", "tag", "assigned_at"), + ), + # Use ledger uniqueness identities so a changed decision is a conflict, + # even when its content-derived match key or observation ID also changes. + "recurrence_match": _RecordSpec( + RECURRENCE_MATCH_TABLE, + ("scope_id", "task_outcome_source_type", "task_outcome_source_id", "failure_item_kind", "failure_item_index"), + ( + "match_key", + "task_outcome_position", + "failure_item_digest", + "candidate_set_mode", + "candidate_set_digest", + "result", + "target_family", + "target_artifact_id", + "target_revision", + "signature_key", + "payload", + ), + ), + "recurrence_observation": _RecordSpec( + RECURRENCE_OBSERVATION_TABLE, + ("scope_id", "verdict_key"), + ( + "observation_id", + "selection_key", + "event", + "match_basis", + "family", + "artifact_id", + "revision", + "signature_key", + "signature_key_hash", + "task_outcome_source_type", + "task_outcome_source_id", + "task_outcome_position", + "payload", + ), + ), +} + +# Parent rows always precede their dependent rows. This matters for SQLite as +# well as for MySQL/OceanBase foreign-key enforcement. +_EXPORT_ORDER: tuple[RecordType, ...] = tuple(_SPECS) + + +class PortableBundleService: + """Export and restore complete scopes without depending on a DB dialect. + + The service is intentionally application-facing rather than a database-file + utility. Callers must perform authorization before calling :meth:`export`; + records are never enumerated before the scope arguments are validated. + """ + + def __init__( + self, + database: AsyncDatabase, + /, + *, + projection_rebuilder: ProjectionRebuilder | None = None, + supported_source_types: Iterable[str] | None = None, + supported_artifact_families: Iterable[str] | None = None, + artifact_content_decoder: Callable[[Mapping[str, Any]], object] | None = None, + candidate_decoder: Callable[[Mapping[str, Any]], object] | None = None, + source_decoder: Callable[[Mapping[str, Any]], object] | None = None, + ) -> None: + self._database = database + self._projection_rebuilder = projection_rebuilder + self._artifact_content_decoder = artifact_content_decoder + self._candidate_decoder = candidate_decoder + self._source_decoder = source_decoder + self._supported_source_types = None if supported_source_types is None else frozenset(supported_source_types) + self._supported_artifact_families = ( + None if supported_artifact_families is None else frozenset(supported_artifact_families) + ) + + async def export( + self, + scopes: Iterable[str], + output: Path, + /, + *, + authorize: ExportAuthorizer, + progress: ProgressObserver | None = None, + compress: bool = True, + ) -> BundleReceipt: + """Write a verified archive containing all portable records for scopes. + + A single read transaction provides a consistent snapshot on supported + relational backends. The resulting ZIP is written only after the + snapshot has been completely read, so a failed export never presents a + partially finalized archive at ``output``. + """ + + selected = _validate_scopes(scopes) + # Authorization is deliberately outside the snapshot transaction and + # before the first database query or pagination operation. + await authorize(selected) + output.parent.mkdir(parents=True, exist_ok=True) + records_path = _temporary_path(output.parent, suffix=".records.ndjson") + archive_path = _temporary_path(output.parent, suffix=".pcb") + try: + with records_path.open("wb") as stream: + async with self._database.transaction(consistent_snapshot=True) as connection: + await _validate_export_destination(connection, output) + await _validate_selected_scopes(connection, selected) + count, records_by_type, digest = await self._stream_export( + connection, selected, stream, progress=progress + ) + bundle_id = str(uuid5(NAMESPACE_URL, f"powercontext:portable:v{FORMAT_VERSION}:{digest}")) + manifest = { + "format_version": FORMAT_VERSION, + "bundle_id": bundle_id, + "producer": {"name": "powercontext", "version": _producer_version()}, + "scopes": list(selected), + "record_count": count, + "records_by_type": dict(sorted(records_by_type.items())), + "excluded_record_classes": list(_EXCLUDED_RECORD_CLASSES), + "total_digest": digest, + } + compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED + with zipfile.ZipFile( + archive_path, "w", compression=compression, compresslevel=9 if compress else None + ) as archive: + with ( + records_path.open("rb") as source, + archive.open(_zip_info(_RECORDS_NAME, compression), "w") as target, + ): + shutil.copyfileobj(source, target, length=1024 * 1024) + archive.writestr(_zip_info(_MANIFEST_NAME, compression), _canonical_json(manifest)) + # Publish only an archive with valid structure and complete dependencies. + # Reject missing dependencies and size-limit breaches + # without replacing a previous usable backup at the destination. + _parse_bundle(archive_path) + with _validate_dependencies(archive_path, candidate_decoder=self._candidate_decoder): + archive_path.replace(output) + return BundleReceipt(bundle_id=bundle_id, record_count=count, total_digest=digest) + finally: + records_path.unlink(missing_ok=True) + archive_path.unlink(missing_ok=True) + + @staticmethod + async def inspect(source: Path, /, *, progress: ProgressObserver | None = None) -> BundleInspection: + """Verify structure and checksums without touching the target database.""" + + return _parse_bundle(source, progress=progress, phase="inspect") + + async def validate( + self, + source: Path, + /, + *, + supported_source_types: Iterable[str] | None = None, + progress: ProgressObserver | None = None, + ) -> BundleValidation: + """Verify bundle integrity, dependencies, and optional adapter support.""" + + inspection = _parse_bundle(source, progress=progress, phase="validate") + with _validate_dependencies(source, candidate_decoder=self._candidate_decoder) as ( + required_sources, + required_families, + index, + ): + configured = ( + self._supported_source_types if supported_source_types is None else frozenset(supported_source_types) + ) + missing_sources = () if configured is None else tuple(sorted(required_sources - configured)) + missing_families = ( + () + if self._supported_artifact_families is None + else tuple(sorted(required_families - self._supported_artifact_families)) + ) + already_present, conflicts = await self._target_compatibility( + source, + supported_source_types=configured, + total=inspection.record_count, + progress=progress, + dependency_index=index, + ) + return BundleValidation( + bundle_id=inspection.bundle_id, + scopes=inspection.scopes, + record_count=inspection.record_count, + records_by_type=inspection.records_by_type, + total_digest=inspection.total_digest, + format_version=inspection.format_version, + producer_version=inspection.producer_version, + already_present=already_present, + conflicts=conflicts, + projections_supported=self._projection_rebuilder is not None, + compatible=not missing_sources and not missing_families and conflicts == 0, + required_source_types=tuple(sorted(required_sources)), + unsupported_source_types=missing_sources, + required_artifact_families=tuple(sorted(required_families)), + unsupported_artifact_families=missing_families, + reader_version=_producer_version(), + ) + + async def restore( + self, + source: Path, + /, + *, + supported_source_types: Iterable[str] | None = None, + progress: ProgressObserver | None = None, + ) -> BundleReceipt: + """Restore a validated bundle atomically, skipping identical records. + + Existing rows are compared using the same portable record encoding. A + difference at the same primary identity aborts the entire transaction; + no existing revision or head is overwritten. + """ + + with tempfile.TemporaryDirectory(prefix="powercontext-restore-") as directory: + snapshot = Path(directory) / "bundle.pcb" + try: + if (await asyncio.to_thread(source.stat)).st_size > _MAX_ARCHIVE_BYTES: + raise BundleFormatError("bundle exceeds resource limit") + with source.open("rb") as original, snapshot.open("wb") as target: + copied = 0 + while chunk := original.read(min(1024 * 1024, _MAX_ARCHIVE_BYTES - copied + 1)): + copied += len(chunk) + if copied > _MAX_ARCHIVE_BYTES: + raise BundleFormatError("bundle exceeds resource limit") + target.write(chunk) + except OSError as error: + raise BundleFormatError("cannot read portable bundle") from error + return await self._restore_snapshot( + snapshot, supported_source_types=supported_source_types, progress=progress + ) + + async def _restore_snapshot( + self, source: Path, /, *, supported_source_types: Iterable[str] | None, progress: ProgressObserver | None + ) -> BundleReceipt: + validation = await self.validate(source, supported_source_types=supported_source_types, progress=progress) + if validation.conflicts: + raise BundleConflictError(f"target contains {validation.conflicts} conflicting record identities") + if not validation.compatible: + raise BundleCompatibilityError("target is not compatible with bundle requirements") + try: + with _staged_records(source) as staged: + async with self._database.transaction() as connection: + inserted, existing = await self._write_records( + connection, staged, total=validation.record_count, progress=progress + ) + await self._write_restore_receipt( + connection, + validation, + inserted=inserted, + already_present=existing, + projections_ready=False, + ) + except IntegrityError as error: + raise BundleConflictError("target constraint conflict during transactional restore") from error + projections_ready = False + if self._projection_rebuilder is not None: + _report_progress(progress, "projections", 0, 1) + await self._projection_rebuilder(validation.scopes) + projections_ready = True + _report_progress(progress, "projections", 1, 1) + async with self._database.transaction() as connection: + await self._write_restore_receipt( + connection, + validation, + inserted=inserted, + already_present=existing, + projections_ready=True, + ) + return BundleReceipt( + bundle_id=validation.bundle_id, + record_count=validation.record_count, + total_digest=validation.total_digest, + inserted=inserted, + already_present=existing, + projections_ready=projections_ready, + status="ready" if projections_ready else "authoritative_restored", + ) + + async def _write_restore_receipt( + self, + connection: AsyncConnection, + inspection: BundleInspection, + /, + *, + inserted: int, + already_present: int, + projections_ready: bool, + ) -> None: + row = ( + ( + await connection.execute( + select(PORTABLE_RESTORE_RECEIPTS_TABLE).where( + PORTABLE_RESTORE_RECEIPTS_TABLE.c.bundle_id == inspection.bundle_id + ) + ) + ) + .mappings() + .one_or_none() + ) + if row is not None and ( + str(row["total_digest"]) != inspection.total_digest + or int(row["format_version"]) != inspection.format_version + or int(row["record_count"]) != inspection.record_count + ): + raise BundleConflictError("restore receipt bundle identity conflict") + values = { + "format_version": inspection.format_version, + "total_digest": inspection.total_digest, + "record_count": inspection.record_count, + "status": "ready" if projections_ready else "authoritative_restored", + "inserted": inserted, + "already_present": already_present, + "projections_ready": projections_ready, + "updated_at": datetime.now(UTC), + } + if row is None: + await connection.execute( + insert(PORTABLE_RESTORE_RECEIPTS_TABLE).values(bundle_id=inspection.bundle_id, **values) + ) + else: + await connection.execute( + update(PORTABLE_RESTORE_RECEIPTS_TABLE) + .where(PORTABLE_RESTORE_RECEIPTS_TABLE.c.bundle_id == inspection.bundle_id) + .values(**values) + ) + + async def _target_compatibility( + self, + source: Path, + /, + *, + supported_source_types: frozenset[str] | None, + total: int, + progress: ProgressObserver | None, + dependency_index: sqlite3.Connection, + ) -> tuple[int, int]: + already_present = 0 + conflicts = 0 + async with self._database.transaction() as connection: + existing_tables = await connection.run_sync( + lambda sync_connection: { + spec.table.name + for spec in _SPECS.values() + if sqlalchemy_inspect(sync_connection).has_table(spec.table.name) + } + ) + checked_types: set[RecordType] = set() + for processed, record in enumerate(_iter_records(source), start=1): + self._validate_target_content(record, supported_source_types=supported_source_types) + spec = _SPECS[record.record_type] + table = spec.table + if table.name not in existing_tables: + _report_progress(progress, "target", processed, total) + continue + if record.record_type not in checked_types: + columns = await connection.run_sync( + lambda sync_connection, table_name=table.name: { + column["name"] for column in sqlalchemy_inspect(sync_connection).get_columns(table_name) + } + ) + if not {*spec.identity, *spec.payload} <= columns: + raise BundleCompatibilityError( + "target schema lacks required archive fields; start the current Runtime to upgrade it" + ) + checked_types.add(record.record_type) + keys = (record.identity, *_unique_identities(record)) + where = or_( + *( + and_(*(table.c[key] == _database_value(value, table.c[key]) for key, value in identity.items())) + for identity in keys + ) + ) + statement = select(*(table.c[field] for field in (*spec.identity, *spec.payload))).where(where) + rows = (await connection.execute(statement)).mappings().all() + if rows: + if all( + _record_from_row(record.record_type, cast(Mapping[str, Any], row)).digest == record.digest + for row in rows + ): + already_present += 1 + else: + conflicts += 1 + _report_progress(progress, "target", processed, total) + if ( + conflicts == 0 + and ARTIFACT_PUBLICATIONS_TABLE.name in existing_tables + and not await _validate_target_publications(connection, dependency_index) + ): + conflicts += 1 + if ( + conflicts == 0 + and ARTIFACT_TAGS_TABLE.name in existing_tables + and not await _validate_target_tag_sets(connection, dependency_index) + ): + conflicts += 1 + return already_present, conflicts + + def _validate_target_content(self, record: _Record, /, *, supported_source_types: frozenset[str] | None) -> None: + if record.record_type == "source": + self._validate_source(record, supported_source_types=supported_source_types) + return + if record.record_type == "candidate_version": + if self._candidate_decoder is None: + raise BundleFormatError("target cannot validate Candidate proposals") + return + if record.record_type != "artifact_revision" or self._artifact_content_decoder is None: + return + family = str(record.identity["family"]) + if self._supported_artifact_families is not None and family not in self._supported_artifact_families: + return + try: + self._artifact_content_decoder(_row_values(record)) + except InvalidStoredPayloadError as error: + raise BundleFormatError("Artifact content does not match the target family decoder") from error + + def _validate_source(self, record: _Record, /, *, supported_source_types: frozenset[str] | None) -> None: + if ( + _source_observation(record) is None + and supported_source_types is not None + and record.identity["source_type"] not in supported_source_types + ): + return + if self._source_decoder is None: + raise BundleFormatError("target cannot validate Source payloads") + try: + self._source_decoder(_row_values(record)) + except ( + InvalidStoredPayloadError, + IdentityMismatchError, + RepositoryNotFoundError, + SourceError, + ValueError, + ) as error: + raise BundleFormatError("Source payload does not match the target Source decoder") from error + + async def _write_records( + self, + connection: AsyncConnection, + staged: sqlite3.Connection, + /, + *, + total: int, + progress: ProgressObserver | None, + ) -> tuple[int, int]: + """Replay records in dependency order inside the caller transaction.""" + inserted = 0 + existing = 0 + processed = 0 + for record in _ordered_scope_records(_staged_type_records(staged, "scope")): + added, skipped = await self._write_batch(connection, "scope", (record,)) + inserted += added + existing += skipped + processed += 1 + _report_progress(progress, "restore", processed, total) + # The disk index was built before acquiring the target write transaction. + for record_type in _EXPORT_ORDER: + if record_type == "scope": + continue + for records in _staged_batches(staged, record_type): + added, skipped = await self._write_batch(connection, record_type, records) + inserted += added + existing += skipped + processed += len(records) + _report_progress(progress, "restore", processed, total) + return inserted, existing + + async def _write_batch( + self, connection: AsyncConnection, kind: RecordType, records: tuple[_Record, ...] + ) -> tuple[int, int]: + spec = _SPECS[kind] + keys = spec.identity + identities = [ + tuple(_database_value(record.identity[key], spec.table.c[key]) for key in keys) for record in records + ] + rows = ( + await connection.execute( + select(spec.table).where(tuple_(*(spec.table.c[key] for key in keys)).in_(identities)) + ) + ).mappings() + present = { + _identity_key(record.identity): record.digest + for row in rows + for record in (_record_from_row(kind, cast(Mapping[str, Any], dict(row))),) + } + values = [] + for record in records: + digest = present.get(_identity_key(record.identity)) + if digest is not None: + if digest != record.digest: + raise BundleConflictError(f"immutable identity conflict: {kind}") + else: + values.append(_row_values(record)) + if values: + await connection.execute(insert(spec.table), values) + return len(values), len(records) - len(values) + + async def _stream_export( + self, + connection: AsyncConnection, + scopes: tuple[str, ...], + stream: Any, + *, + progress: ProgressObserver | None, + ) -> tuple[int, Counter[str], str]: + count = 0 + records_by_type: Counter[str] = Counter() + digest = _DigestAccumulator() + for record_type in _EXPORT_ORDER: + async for row in _export_rows(connection, record_type, scopes): + record = _record_from_row(record_type, cast(Mapping[str, Any], dict(row))) + _validate_record_safe(record) + stream.write(_canonical_json(_record_document(record)) + b"\n") + digest.add(record.digest) + count += 1 + records_by_type[record.record_type] += 1 + _report_progress(progress, "export", count, None) + _report_progress(progress, "export", count, count) + return count, records_by_type, digest.value() + + +async def _export_rows( + connection: AsyncConnection, kind: RecordType, scopes: tuple[str, ...] +) -> AsyncIterator[Mapping[str, Any]]: + spec = _SPECS[kind] + if spec.scope_field is not None: + predicate = spec.table.c[spec.scope_field].in_(scopes) + if kind in {"memory_entry_version", "memory_entry_head"}: + predicate &= ( + select(ARTIFACTS_TABLE.c.artifact_id) + .where( + ARTIFACTS_TABLE.c.scope_id == spec.table.c.scope_id, + ARTIFACTS_TABLE.c.family == spec.table.c.family, + ARTIFACTS_TABLE.c.artifact_id == spec.table.c.memory_artifact_id, + ) + .exists() + ) + rows = await connection.stream( + select(spec.table).where(predicate).order_by(*(spec.table.c[field] for field in spec.identity)) + ) + async for row in rows.mappings(): + yield cast(Mapping[str, Any], row) + return + + # Definition manifests are global, but export authority is scope-bound. + # Include only immutable declarations referenced by selected observations. + definitions: set[tuple[str, str]] = set() + async for row in _export_rows(connection, "source", scopes): + observation = _source_observation(_record_from_row("source", row)) + if observation is not None: + definitions.add((observation.source_type, observation.definition_version)) + table = SOURCE_DEFINITION_MANIFESTS_TABLE + for name, definition_version in sorted(definitions): + row = ( + ( + await connection.execute( + select(table).where( + table.c.definition_name == name, table.c.definition_version == definition_version + ) + ) + ) + .mappings() + .one_or_none() + ) + if row is None: + raise BundleFormatError("observation references missing Source definition") + yield cast(Mapping[str, Any], row) + + +def _source_observation(record: _Record) -> SourceObservation | None: + try: + value = json.loads(cast(bytes, _database_value(record.payload["payload"]))) + except (ValueError, UnicodeDecodeError): + return None # Native Source payloads are interpreted by their adapter. + if not isinstance(value, dict): + return None + if value.get("encoding") == "powercontext-source-v1": + if value.get("representation") != "observation": + return None + value = value.get("value") + elif "definition_fingerprint" not in value or "source_type" not in value: + return None # Legacy native payload. + try: + observation = SourceObservation.model_validate_json(_canonical_json(value)) + except ValueError as error: + raise BundleFormatError("invalid Source observation") from error + if (observation.source_type, observation.name) != (record.identity["source_type"], record.identity["source_id"]): + raise BundleFormatError("Source observation identity mismatch") + return observation + + +def _source_definition(record: _Record) -> SourceDefinitionManifest: + try: + manifest = SourceDefinitionManifest.model_validate_json( + cast(bytes, _database_value(record.payload["manifest"])) + ) + validate_source_definition_manifest(manifest) + except (SourceError, ValueError) as error: + raise BundleFormatError("invalid Source definition manifest") from error + if (manifest.name, manifest.version, manifest.fingerprint) != ( + record.identity["definition_name"], + record.identity["definition_version"], + record.payload["fingerprint"], + ): + raise BundleFormatError("Source definition identity mismatch") + return manifest + + +def _record_from_row(record_type: RecordType, row: Mapping[str, Any]) -> _Record: + spec = _SPECS[record_type] + identity = {field: _json_value(row[field]) for field in spec.identity} + payload = {field: _json_value(row[field]) for field in spec.payload} + canonical = { + "record_type": record_type, + "schema_version": spec.schema_version, + "identity": identity, + "payload": payload, + } + return _Record(record_type=record_type, identity=identity, payload=payload, digest=_digest(canonical)) + + +async def _validate_export_destination(connection: AsyncConnection, output: Path, /) -> None: + if connection.dialect.name != "sqlite": + return + destination = await asyncio.to_thread(output.resolve) + for row in await connection.exec_driver_sql("PRAGMA database_list"): + if not row[2]: + continue + database = await asyncio.to_thread(Path(row[2]).resolve) + reserved_names = {database.name.casefold() + suffix for suffix in ("", "-wal", "-shm", "-journal")} + if ( + await asyncio.to_thread(destination.exists) and await asyncio.to_thread(destination.samefile, database) + ) or ( + destination.name.casefold() in reserved_names + and await asyncio.to_thread(destination.parent.samefile, database.parent) + ): + raise BundleFormatError("export destination overlaps the source database or its journal") + + +def _record_document(record: _Record) -> dict[str, Any]: + return { + "record_type": record.record_type, + "schema_version": _SPECS[record.record_type].schema_version, + "identity": dict(record.identity), + "payload": dict(record.payload), + "digest": record.digest, + } + + +def _row_values(record: _Record) -> dict[str, Any]: + table = _SPECS[record.record_type].table + values = {key: _database_value(value, table.c[key]) for key, value in record.identity.items()} + values.update({key: _database_value(value, table.c[key]) for key, value in record.payload.items()}) + # Projections are intentionally excluded from logical bundles. Heads may + # use NULL searchable text; Memory projections are rebuilt by the runtime. + if record.record_type == "artifact_head": + values["searchable_text"] = None + elif record.record_type == "memory_entry_head": + values["searchable_text"] = str(record.payload["entry_content_hash"]) + elif record.record_type == "scope": + values["scope_id_search"] = str(record.identity["scope_id"]) + values["title_search"] = str(record.payload["title"]) + values["summary_search"] = str(record.payload["summary"]) + elif record.record_type == "scope_external_reference": + values["value_search"] = str(record.payload["value"]) + elif record.record_type == "skill_package": + values["created_at"] = datetime.now(UTC) + return values + + +def _parse_bundle( + source: Path, + *, + progress: ProgressObserver | None = None, + phase: Literal["inspect", "validate"] = "inspect", +) -> BundleInspection: + try: + if source.stat().st_size > _MAX_ARCHIVE_BYTES: + raise BundleFormatError("bundle exceeds resource limit") + with zipfile.ZipFile(source) as archive: + _validate_container(archive) + with archive.open(_MANIFEST_NAME) as stream: + manifest = json.loads(stream.read(_MAX_MANIFEST_BYTES + 1), parse_constant=_reject_json_constant) + _validate_manifest(manifest) + except ( + OSError, + EOFError, + ValueError, + RecursionError, + NotImplementedError, + zipfile.BadZipFile, + zlib.error, + ) as error: + if isinstance(error, BundleFormatError): + raise + raise BundleFormatError("cannot read portable bundle") from error + count = 0 + counts: Counter[str] = Counter() + digest = _DigestAccumulator() + scope_ids: list[str] = [] + for record in _iter_records(source): + if record.record_type == "scope": + scope_ids.append(str(record.identity["scope_id"])) + count += 1 + counts[record.record_type] += 1 + digest.add(record.digest) + _report_progress(progress, phase, count, manifest["record_count"]) + expected_count = manifest["record_count"] + if count != expected_count: + raise BundleFormatError("manifest record count does not match records") + if digest.value() != manifest["total_digest"]: + raise BundleFormatError("bundle total digest does not match records") + sorted_counts = dict(sorted(counts.items())) + if sorted_counts != manifest["records_by_type"]: + raise BundleFormatError("manifest record classes do not match records") + inspection = BundleInspection( + bundle_id=manifest["bundle_id"], + scopes=tuple(manifest["scopes"]), + record_count=count, + records_by_type=sorted_counts, + total_digest=digest.value(), + format_version=manifest["format_version"], + producer_version=manifest["producer"]["version"], + ) + if tuple(sorted(scope_ids)) != inspection.scopes: + raise BundleFormatError("manifest scopes do not match scope records") + return inspection + + +@contextmanager +def _staged_records(source: Path) -> Iterator[sqlite3.Connection]: + """Index verified records on disk before acquiring the target write transaction.""" + with tempfile.TemporaryDirectory(prefix="powercontext-records-") as directory: + connection = sqlite3.connect(Path(directory) / "records.db") + try: + connection.execute("CREATE TABLE records (sequence INTEGER PRIMARY KEY, kind TEXT, document BLOB)") + connection.executemany( + "INSERT INTO records(kind, document) VALUES (?, ?)", + ((record.record_type, _canonical_json(_record_document(record))) for record in _iter_records(source)), + ) + connection.execute("CREATE INDEX records_kind ON records(kind, sequence)") + connection.commit() + yield connection + finally: + connection.close() + + +def _staged_type_records(staged: sqlite3.Connection, kind: RecordType) -> Iterator[_Record]: + for row in staged.execute("SELECT document FROM records WHERE kind = ? ORDER BY sequence", (kind,)): + yield _parse_record(json.loads(row[0])) + + +def _staged_batches(staged: sqlite3.Connection, kind: RecordType) -> Iterator[tuple[_Record, ...]]: + batch: list[_Record] = [] + size = 0 + for row in staged.execute("SELECT document FROM records WHERE kind = ? ORDER BY sequence", (kind,)): + batch.append(_parse_record(json.loads(row[0]))) + size += len(row[0]) + if len(batch) >= 100 or size >= 1024 * 1024: + yield tuple(batch) + batch.clear() + size = 0 + if batch: + yield tuple(batch) + + +def _ordered_scope_records(records: Iterable[_Record], /) -> tuple[_Record, ...]: + pending = {str(record.identity["scope_id"]): record for record in records} + ordered: list[_Record] = [] + while pending: + ready = sorted( + scope_id + for scope_id, record in pending.items() + if record.payload["parent_scope_id"] is None or record.payload["parent_scope_id"] not in pending + ) + if not ready: + raise BundleFormatError("scope hierarchy contains a cycle") + for scope_id in ready: + ordered.append(pending.pop(scope_id)) + return tuple(ordered) + + +def _iter_records(source: Path, /) -> Iterator[_Record]: + """Yield verified records without retaining the archive body in memory.""" + + try: + with zipfile.ZipFile(source) as archive, archive.open(_RECORDS_NAME) as stream: + count = 0 + while line := stream.readline(_MAX_LINE_BYTES + 1): + if len(line) > _MAX_LINE_BYTES: + raise BundleFormatError("record line exceeds size limit") + if not line.strip(): + continue + count += 1 + if count > _MAX_RECORDS: + raise BundleFormatError("record count exceeds limit") + yield _parse_record(json.loads(line, parse_constant=_reject_json_constant)) + except ( + OSError, + EOFError, + ValueError, + RecursionError, + NotImplementedError, + zipfile.BadZipFile, + zlib.error, + ) as error: + if isinstance(error, BundleFormatError): + raise + raise BundleFormatError("cannot read portable bundle") from error + + +def _validate_container(archive: zipfile.ZipFile) -> None: + items = archive.infolist() + names = [item.filename for item in items] + if len(items) != 2 or set(names) != {_MANIFEST_NAME, _RECORDS_NAME}: + raise BundleFormatError("bundle must contain only manifest.json and records.ndjson") + if any( + item.is_dir() + or item.flag_bits & 1 + or item.compress_type not in {zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED} + or item.compress_size > _MAX_ARCHIVE_BYTES + for item in items + ): + raise BundleFormatError("bundle exceeds resource limits") + if archive.getinfo(_MANIFEST_NAME).file_size > _MAX_MANIFEST_BYTES: + raise BundleFormatError("manifest exceeds resource limit") + if archive.getinfo(_RECORDS_NAME).file_size > _MAX_RECORDS_BYTES: + raise BundleFormatError("records exceed resource limit") + + +def _validate_manifest(manifest: object) -> None: # noqa: C901 - manifest fields have independent constraints. + if not isinstance(manifest, dict): + raise BundleFormatError("manifest must be an object") + manifest = cast(dict[str, object], manifest) + required = { + "format_version", + "bundle_id", + "producer", + "scopes", + "record_count", + "records_by_type", + "excluded_record_classes", + "total_digest", + } + if set(manifest) != required: + raise BundleFormatError("manifest has unknown or missing fields") + if type(manifest["format_version"]) is not int or manifest["format_version"] != FORMAT_VERSION: + raise BundleFormatError("unsupported bundle format version") + if ( + not isinstance(manifest["bundle_id"], str) + or isinstance(manifest["record_count"], bool) + or not isinstance(manifest["record_count"], int) + or manifest["record_count"] < 0 + or manifest["record_count"] > _MAX_RECORDS + ): + raise BundleFormatError("manifest has invalid identity or record count") + try: + bundle_id = UUID(manifest["bundle_id"]) + except ValueError as error: + raise BundleFormatError("manifest has invalid bundle identity") from error + if str(bundle_id) != manifest["bundle_id"]: + raise BundleFormatError("manifest has invalid bundle identity") + producer = manifest["producer"] + if not isinstance(producer, dict): + raise BundleFormatError("manifest has invalid producer") + typed_producer = cast(dict[str, object], producer) + producer_version = typed_producer.get("version") + if ( + set(typed_producer) != {"name", "version"} + or typed_producer.get("name") != "powercontext" + or not isinstance(producer_version, str) + or not producer_version + ): + raise BundleFormatError("manifest has invalid producer") + scopes = manifest["scopes"] + if not isinstance(scopes, list) or not all(isinstance(scope, str) for scope in scopes): + raise BundleFormatError("manifest has invalid scopes") + typed_scopes = cast(list[str], scopes) + if _validate_scopes(typed_scopes) != tuple(typed_scopes): + raise BundleFormatError("manifest has invalid scopes") + counts = manifest["records_by_type"] + if ( + not isinstance(counts, dict) + or any( + key not in _SPECS or isinstance(value, bool) or not isinstance(value, int) or value < 0 + for key, value in counts.items() + ) + or sum(cast(dict[str, int], counts).values()) != manifest["record_count"] + or manifest["excluded_record_classes"] != list(_EXCLUDED_RECORD_CLASSES) + or not _valid_digest(manifest["total_digest"]) + ): + raise BundleFormatError("manifest has invalid checksums") + + +def _parse_record(value: object) -> _Record: + if not isinstance(value, dict): + raise BundleFormatError("record must be an object") + value = cast(dict[str, object], value) + if set(value) != {"record_type", "schema_version", "identity", "payload", "digest"}: + raise BundleFormatError("record has unknown or missing fields") + record_type = value.get("record_type") + if ( + not isinstance(record_type, str) + or record_type not in _SPECS + or type(value.get("schema_version")) is not int + or value["schema_version"] != _SPECS[cast(RecordType, record_type)].schema_version + ): + raise BundleFormatError("unsupported record type or schema") + identity = value.get("identity") + payload = value.get("payload") + digest = value.get("digest") + if not isinstance(identity, dict) or not isinstance(payload, dict) or not isinstance(digest, str): + raise BundleFormatError("record has invalid fields") + identity = cast(dict[str, str | int], identity) + payload = cast(dict[str, Any], payload) + typed = cast(RecordType, record_type) + spec = _SPECS[typed] + if ( + set(identity) != set(spec.identity) + or set(payload) != set(spec.payload) + or any(type(item) is not spec.table.c[field].type.python_type for field, item in identity.items()) + or not _valid_digest(digest) + ): + raise BundleFormatError("record has invalid identity or digest") + canonical = { + "record_type": typed, + "schema_version": spec.schema_version, + "identity": identity, + "payload": payload, + } + if _digest(canonical) != digest: + raise BundleFormatError("record digest does not match content") + record = _Record(record_type=typed, identity=identity, payload=payload, digest=digest) + _validate_record_values(record) + _validate_record_safe(record) + return record + + +def _validate_record_values(record: _Record, /) -> None: # noqa: C901 - one versioned schema contract. + identity = record.identity + payload = record.payload + for field, value in {**identity, **payload}.items(): + column_type = _SPECS[record.record_type].table.c[field].type + if ( + isinstance(value, str) + and isinstance(column_type, String) + and column_type.length is not None + and len(value) > column_type.length + ): + raise BundleFormatError("record string exceeds its portable storage limit") + if isinstance(value, int) and not isinstance(value, bool) and isinstance(column_type, Integer): + bits = 63 if isinstance(column_type, BigInteger) else 31 + if not -(1 << bits) <= value < (1 << bits): + raise BundleFormatError("record integer exceeds its portable storage limit") + if any(isinstance(value, str) and (not value or value != value.strip()) for value in identity.values()): + raise BundleFormatError("record identity contains an invalid string") + positive_identity_fields = {"revision", "version"} + for field, value in identity.items(): + if isinstance(value, int) and (value < 0 or (field in positive_identity_fields and value == 0)): + raise BundleFormatError("record identity contains an invalid integer") + + def strings(*fields: str, nullable: tuple[str, ...] = ()) -> None: + for field in fields: + value = payload[field] + if field in nullable and value is None: + continue + if not isinstance(value, str): + raise BundleFormatError("record payload has an invalid string field") + + def integers(*fields: str, positive: bool = False) -> None: + for field in fields: + value = payload[field] + if isinstance(value, bool) or not isinstance(value, int) or value < int(positive): + raise BundleFormatError("record payload has an invalid integer field") + + def bytes_values(*fields: str, nullable: tuple[str, ...] = ()) -> None: + for field in fields: + value = payload[field] + if field in nullable and value is None: + continue + encoded = cast(dict[str, object], value).get("base64") if isinstance(value, dict) else None + if not isinstance(value, dict) or set(value) != {"base64"} or not isinstance(encoded, str): + raise BundleFormatError("record payload has an invalid binary field") + try: + decoded = base64.b64decode(encoded, validate=True) + except ValueError as error: + raise BundleFormatError("record payload has an invalid binary field") from error + if base64.b64encode(decoded).decode("ascii") != encoded: + raise BundleFormatError("record payload has a non-canonical binary field") + + kind = record.record_type + if kind == "scope": + strings("title", "summary", "parent_scope_id", nullable=("parent_scope_id",)) + integers("version", positive=True) + elif kind == "scope_context_reference": + if identity["scope_id"] == identity["referenced_scope_id"]: + raise BundleFormatError("scope cannot reference itself") + elif kind == "scope_external_reference": + strings("kind", "value", "value_digest") + try: + ScopeExternalReference(kind=payload["kind"], value=payload["value"]) + except ValidationError as error: + raise BundleFormatError("scope external reference has invalid domain values") from error + if payload["value_digest"] != hashlib.sha256(payload["value"].encode()).hexdigest(): + raise BundleFormatError("scope external reference has invalid value digest") + elif kind == "scope_creation_request": + strings("request_digest", "scope_id") + elif kind == "source_journal_head": + integers("position") + elif kind == "source_definition": + strings("fingerprint") + bytes_values("manifest") + _source_definition(record) + elif kind == "source": + bytes_values("payload") + integers("journal_position", positive=True) + elif kind in {"recurrence_match", "recurrence_observation"}: + bytes_values("payload") + _recurrence_value(record) + elif kind == "skill_package": + strings("archive_digest") + bytes_values("archive_bytes", "manifest") + integers("file_count", "uncompressed_size", "archive_size", positive=True) + if not all( + isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) + for value in (identity["tree_digest"], payload["archive_digest"]) + ): + raise BundleFormatError("Skill package has invalid digest") + elif kind == "artifact_revision": + bytes_values("content") + strings("created_at", "created_by", "rollback_reason", nullable=("created_at", "created_by", "rollback_reason")) + if payload["created_at"] is not None: + _validate_datetime_text(payload["created_at"]) + if payload["created_by"] is not None: + try: + ArtifactRevisionActor.model_validate_json(payload["created_by"]) + except ValueError as error: + raise BundleFormatError("Artifact revision has invalid actor metadata") from error + restored = payload["restored_from_revision"] + if restored is not None and (type(restored) is not int or not 0 < restored < int(identity["revision"])): + raise BundleFormatError("Artifact revision has invalid rollback metadata") + elif kind == "artifact_handle": + strings("artifact_id") + integers("revision", positive=True) + try: + StableHandle(family=str(identity["family"]), key=str(identity["handle_key"])) + except ValueError as error: + raise BundleFormatError("artifact handle is invalid") from error + elif kind == "artifact_lineage_source": + strings("source_type", "source_id") + elif kind == "artifact_lineage_artifact": + strings("upstream_family", "upstream_artifact_id") + integers("upstream_revision", positive=True) + if type(payload["is_merge_input"]) is not bool: + raise BundleFormatError("artifact lineage has an invalid merge marker") + elif kind == "artifact_publication": + strings("source_scope_id", "source_family", "source_artifact_id", "content_digest", "idempotency_key") + integers("source_revision", positive=True) + elif kind == "artifact_head": + strings( + "lifecycle_state", + "replacement_artifact_id", + "merged_into_id", + nullable=("replacement_artifact_id", "merged_into_id"), + ) + integers("revision", positive=True) + integers("governance_generation") + if payload["lifecycle_state"] not in {"active", "deprecated", "retired"} or ( + payload["replacement_artifact_id"] is not None + and ( + payload["lifecycle_state"] != "deprecated" + or payload["replacement_artifact_id"] == identity["artifact_id"] + ) + ): + raise BundleFormatError("artifact head has invalid governance state") + if payload["merged_into_id"] is not None and ( + payload["lifecycle_state"] != "deprecated" + or not payload["merged_into_id"] + or payload["merged_into_id"] == identity["artifact_id"] + or payload["replacement_artifact_id"] is not None + ): + raise BundleFormatError("artifact head has invalid merge state") + elif kind == "memory_entry_version": + strings( + "family", + "entry_id", + "previous_version_id", + "kind", + "text", + "entry_content_hash", + nullable=("previous_version_id",), + ) + integers("version", "created_in_revision", positive=True) + bytes_values("source_refs", "artifact_refs") + elif kind == "memory_entry_head": + strings("family", "entry_version_id", "entry_content_hash") + integers("head_revision", positive=True) + elif kind == "candidate_version": + strings( + "family", + "target_family", + "target_artifact_id", + "reason", + nullable=("target_family", "target_artifact_id", "reason"), + ) + bytes_values("proposal", "source_refs", "artifact_refs") + target = (payload["target_family"], payload["target_artifact_id"], payload["target_revision"]) + if target != (None, None, None) and ( + target[0] is None + or target[1] is None + or isinstance(target[2], bool) + or not isinstance(target[2], int) + or target[2] < 1 + ): + raise BundleFormatError("candidate version has an incomplete target") + elif kind == "candidate_head": + if payload["candidate_kind"] != "artifact" or payload["result_payload"] is not None: + raise BundleFormatError("tag Candidates are not supported by portable archives") + strings( + "family", + "status", + "result_family", + "result_artifact_id", + "decision_reason", + nullable=("result_family", "result_artifact_id", "decision_reason"), + ) + integers("version", positive=True) + result = (payload["result_family"], payload["result_artifact_id"], payload["result_revision"]) + status = payload["status"] + valid = ( + (status == "pending" and result == (None, None, None) and payload["decision_reason"] is None) + or (status == "rejected" and result == (None, None, None) and payload["decision_reason"] is not None) + or ( + status == "approved" + and all(value is not None for value in result) + and isinstance(result[2], int) + and not isinstance(result[2], bool) + and result[2] > 0 + and payload["decision_reason"] is None + ) + ) + if not valid: + raise BundleFormatError("candidate head has invalid decision state") + elif kind == "topic_memory_publication": + strings("published_at") + _validate_datetime_text(payload["published_at"]) + if identity["family"] != "topic-memory": + raise BundleFormatError("topic memory publication has invalid family") + elif kind == "profile_policy": + if not isinstance(payload["generation_enabled"], bool): + raise BundleFormatError("profile policy has invalid generation state") + strings("activation_mode", "pending_candidate_id", "updated_at", nullable=("pending_candidate_id",)) + _validate_datetime_text(payload["updated_at"]) + integers("version", positive=True) + if payload["activation_mode"] not in {"automatic", "review_required"}: + raise BundleFormatError("profile policy has invalid activation mode") + elif kind == "artifact_tag": + bytes_values("tag_key_hash") + strings("tag", "assigned_at") + _validate_datetime_text(payload["assigned_at"]) + if identity["target_type"] not in {"artifact", "memory_entry"}: + raise BundleFormatError("artifact tag has invalid target type") + if identity["target_type"] == "artifact" and identity["target_id"] != identity["artifact_id"]: + raise BundleFormatError("artifact tag has invalid target") + if identity["target_type"] == "memory_entry" and identity["family"] != "memory": + raise BundleFormatError("artifact tag has invalid target") + try: + normalized = normalize_tags((payload["tag"],)) + except InvalidBaseAccessRequestError as error: + raise BundleFormatError("artifact tag has invalid label") from error + if identity["tag_key"] not in normalized: + raise BundleFormatError("artifact tag label does not match its normalized key") + if _database_value(payload["tag_key_hash"]) != hashlib.sha256(str(identity["tag_key"]).encode()).digest(): + raise BundleFormatError("artifact tag has invalid key hash") + + +def _recurrence_value(record: _Record, /) -> RecurrenceMatch | RecurrenceObservation: + """Check frozen payloads against their indexed fields and idempotency keys.""" + model = RecurrenceMatch if record.record_type == "recurrence_match" else RecurrenceObservation + try: + value = model.model_validate_json(cast(bytes, _database_value(record.payload["payload"])), strict=True) + except (ValidationError, ValueError, TypeError) as error: + raise BundleFormatError("recurrence payload is invalid") from error + expected = { + "scope_id": value.scope_id, + "task_outcome_source_type": value.task_outcome_ref.source_type, + "task_outcome_source_id": value.task_outcome_ref.source_id, + "task_outcome_position": value.task_outcome_position, + "signature_key": value.signature_key, + } + if isinstance(value, RecurrenceMatch): + ref = value.artifact_ref + expected.update({ + "match_key": match_key(value), + "failure_item_kind": value.failure_ref.item_kind, + "failure_item_index": value.failure_ref.item_index, + "failure_item_digest": value.failure_ref.item_digest, + "candidate_set_mode": value.candidate_set_mode, + "candidate_set_digest": value.candidate_set_digest, + "result": value.result, + "target_family": None if ref is None else ref.family, + "target_artifact_id": None if ref is None else ref.artifact_id, + "target_revision": None if ref is None else ref.revision, + }) + else: + if value.observation_id != canonical_digest(observation_identity(value)): + raise BundleFormatError("recurrence observation has an invalid identity") + expected.update({ + "observation_id": value.observation_id, + "selection_key": selection_key(value), + "verdict_key": verdict_key(value), + "event": value.event, + "match_basis": value.match_basis, + "family": value.artifact_ref.family, + "artifact_id": value.artifact_ref.artifact_id, + "revision": value.artifact_ref.revision, + "signature_key_hash": _json_value(hashlib.sha256(value.signature_key.encode()).digest()), + }) + actual = {**record.identity, **record.payload} + if any(type(actual[key]) is not type(field) or actual[key] != field for key, field in expected.items()): + raise BundleFormatError("recurrence payload does not match its indexed fields") + return value + + +def _validate_datetime_text(value: object, /) -> None: + try: + parsed = datetime.fromisoformat(cast(str, value)) + except ValueError as error: + raise BundleFormatError("record payload has an invalid datetime field") from error + if parsed.tzinfo is not None or parsed.isoformat() != value: + raise BundleFormatError("record payload has a non-canonical datetime field") + + +async def _validate_selected_scopes(connection: AsyncConnection, scopes: tuple[str, ...], /) -> None: + selected = set(scopes) + rows = tuple( + ( + await connection.execute( + select(SCOPES_TABLE.c.scope_id, SCOPES_TABLE.c.parent_scope_id).where( + SCOPES_TABLE.c.scope_id.in_(scopes) + ) + ) + ).mappings() + ) + if {str(row["scope_id"]) for row in rows} != selected: + raise BundleFormatError("one or more selected scopes do not exist") + if any(row["parent_scope_id"] is not None and str(row["parent_scope_id"]) not in selected for row in rows): + raise BundleFormatError("selected scope set omits a required parent scope") + referenced = ( + await connection.execute( + select(SCOPE_CONTEXT_REFERENCES_TABLE.c.referenced_scope_id).where( + SCOPE_CONTEXT_REFERENCES_TABLE.c.scope_id.in_(scopes) + ) + ) + ).scalars() + if any(str(scope_id) not in selected for scope_id in referenced): + raise BundleFormatError("selected scope set omits a referenced context scope") + publication_sources = ( + await connection.execute( + select(ARTIFACT_PUBLICATIONS_TABLE.c.source_scope_id).where( + ARTIFACT_PUBLICATIONS_TABLE.c.target_scope_id.in_(scopes) + ) + ) + ).scalars() + if any(str(scope_id) not in selected for scope_id in publication_sources): + raise BundleFormatError("selected scope set omits an Artifact publication source scope") + + +def _validate_record_safe(record: _Record, /) -> None: + """Reject secret-shaped configuration and host paths without echoing content.""" + + for field, value in record.payload.items(): + encoded = cast(dict[str, object], value).get("base64") if isinstance(value, dict) else None + if isinstance(encoded, str): + try: + decoded = json.loads(base64.b64decode(encoded, validate=True), parse_constant=_reject_json_constant) + except BundleFormatError: + raise + except (ValueError, UnicodeDecodeError, json.JSONDecodeError): + continue + _validate_safe_value(decoded, field) + else: + _validate_safe_value(value, field) + + +def _validate_safe_value(value: object, key: str | None = None) -> None: + normalized = "" if key is None else key.casefold().replace("-", "_") + sensitive_suffixes = ("_api_key", "_credential", "_credentials", "_password", "_secret", "_token") + if (normalized in _SENSITIVE_KEYS or normalized.endswith(sensitive_suffixes)) and value is not None and value != "": + raise BundleFormatError("bundle contains a credential-shaped field") + if normalized == "authorization" and isinstance(value, str) and value.casefold().startswith(("bearer ", "basic ")): + raise BundleFormatError("bundle contains a credential-shaped field") + if ( + isinstance(value, str) + and ( + normalized.endswith(("_path", "_directory", "_home", "_root")) + or normalized in {"cwd", "directory", "home", "path", "root", "working_directory", "workspace"} + ) + and _ABSOLUTE_HOST_PATH.match(value) + ): + raise BundleFormatError("bundle contains an absolute host path") + if isinstance(value, dict): + for child_key, child in value.items(): + _validate_safe_value(child, str(child_key)) + elif isinstance(value, list): + for child in value: + _validate_safe_value(child, key) + + +@contextmanager +def _validate_dependencies( + source: Path, /, *, candidate_decoder: Callable[[Mapping[str, Any]], object] | None = None +) -> Iterator[tuple[set[str], set[str], sqlite3.Connection]]: + """Validate dependencies with disk-backed identities and bounded metadata.""" + + heads, packages = _dependency_metadata(source) + required_sources: set[str] = set() + required_families: set[str] = set() + with _identity_index(source) as index: + + def contains(record_type: str, identity: Mapping[str, object]) -> bool: + return ( + index.execute( + "SELECT 1 FROM identities WHERE record_type = ? AND identity = ?", + (record_type, _identity_key(identity)), + ).fetchone() + is not None + ) + + for record in _iter_records(source): + _validate_record_dependencies(record, contains) + _validate_candidate_record(record, index, candidate_decoder) + _validate_observation_record(record, index) + _validate_skill_package_dependency(record, packages) + if record.record_type == "source": + if int(record.payload["journal_position"]) > heads.get(str(record.identity["scope_id"]), -1): + raise BundleFormatError("source journal snapshot is inconsistent") + if _source_observation(record) is None: + required_sources.add(str(record.identity["source_type"])) + required_families.update(_required_artifact_families((record,))) + _validate_publication_chains(index) + _validate_tag_sets(index) + yield required_sources, required_families, index + + +def _index_publication_dependencies(index: sqlite3.Connection, record: _Record, /) -> None: + if record.record_type == "artifact_revision": + index.execute( + "INSERT OR IGNORE INTO publication_nodes (identity) VALUES (?)", + (_identity_key({"target_" + field: value for field, value in record.identity.items()}),), + ) + return + if record.record_type != "artifact_publication": + return + target = _identity_key(record.identity) + source = _identity_key({ + "target_" + field: record.payload["source_" + field] + for field in ("scope_id", "family", "artifact_id", "revision") + }) + index.execute( + "INSERT OR IGNORE INTO publication_edges (target, source, digest) VALUES (?, ?, ?)", + (target, source, record.payload["content_digest"]), + ) + + +def _validate_publication_chains(index: sqlite3.Connection, /) -> None: + if ( + index.execute( + "SELECT 1 FROM publication_edges AS edge JOIN publication_edges AS parent ON edge.source = parent.target " + "WHERE edge.digest != parent.digest LIMIT 1" + ).fetchone() + is not None + ): + raise BundleFormatError("publication chain has inconsistent content digests") + while pending := index.execute("SELECT target FROM publication_edges WHERE state = 0 LIMIT 1").fetchone(): + current = pending[0] + while edge := index.execute( + "SELECT source, state FROM publication_edges WHERE target = ?", (current,) + ).fetchone(): + if edge[1] == 1: + raise BundleFormatError("publication chain contains a cycle") + if edge[1] == 2: + break + index.execute("UPDATE publication_edges SET state = 1 WHERE target = ?", (current,)) + current = edge[0] + index.execute("UPDATE publication_edges SET state = 2 WHERE state = 1") + + +async def _validate_target_publications(connection: AsyncConnection, index: sqlite3.Connection, /) -> bool: + table = ARTIFACT_PUBLICATIONS_TABLE + fields = _SPECS["artifact_publication"].identity + pending = index.execute("SELECT identity FROM publication_nodes ORDER BY identity") + while batch := pending.fetchmany(100): + identities = [json.loads(row[0]) for row in batch] + rows = ( + await connection.execute( + select(table).where( + tuple_(*(table.c[field] for field in fields)).in_([ + tuple(identity[field] for field in fields) for identity in identities + ]) + ) + ) + ).mappings() + for row in rows: + target = _identity_key({field: row[field] for field in fields}) + if index.execute("SELECT 1 FROM publication_edges WHERE target = ?", (target,)).fetchone() is None: + return False + return True + + +def _index_tag_set(index: sqlite3.Connection, record: _Record, /) -> None: + if record.record_type != "artifact_tag": + return + target = _identity_key({field: value for field, value in record.identity.items() if field != "tag_key"}) + index.execute( + "INSERT OR IGNORE INTO tag_labels (target, tag_key, label) VALUES (?, ?, ?)", + (target, record.identity["tag_key"], record.payload["tag"]), + ) + index.execute( + "INSERT OR IGNORE INTO identities (record_type, identity) VALUES ('artifact_tag_target', ?)", (target,) + ) + + +def _validate_tag_sets(index: sqlite3.Connection, /) -> None: + current: str | None = None + labels: list[str] = [] + for target, label in index.execute("SELECT target, label FROM tag_labels ORDER BY target, tag_key"): + if current != target: + current = target + labels.clear() + labels.append(label) + try: + normalize_tags(tuple(labels)) + except InvalidBaseAccessRequestError as error: + raise BundleFormatError("artifact tag set has invalid labels") from error + + +async def _validate_target_tag_sets(connection: AsyncConnection, index: sqlite3.Connection, /) -> bool: + table = ARTIFACT_TAGS_TABLE + fields = tuple(field for field in _SPECS["artifact_tag"].identity if field != "tag_key") + pending = index.execute( + "SELECT identity FROM identities WHERE record_type = 'artifact_tag_target' ORDER BY identity" + ) + while batch := pending.fetchmany(100): + identities = [json.loads(row[0]) for row in batch] + rows = ( + await connection.execute( + select(table).where( + tuple_(*(table.c[field] for field in fields)).in_([ + tuple(identity[field] for field in fields) for identity in identities + ]) + ) + ) + ).mappings() + for row in rows: + record = _record_from_row("artifact_tag", cast(Mapping[str, Any], row)) + try: + _validate_record_values(record) + except BundleFormatError: + return False + _index_tag_set(index, record) + try: + _validate_tag_sets(index) + except BundleFormatError: + return False + return True + + +def _validate_observation_record(record: _Record, index: sqlite3.Connection, /) -> None: + if record.record_type != "source": + return + observation = _source_observation(record) + if observation is None: + return + identity = {"definition_name": observation.source_type, "definition_version": observation.definition_version} + document = index.execute( + "SELECT document FROM identities WHERE record_type = 'source_definition' AND identity = ?", + (_identity_key(identity),), + ).fetchone() + if document is None: + raise BundleFormatError("observation references missing Source definition") + manifest = _source_definition(_parse_record(json.loads(document[0]))) + try: + validate_source_observation(observation, manifest) + except (SourceError, ValueError) as error: + raise BundleFormatError("Source observation does not match its definition manifest") from error + + +def _validate_candidate_record( + record: _Record, + index: sqlite3.Connection, + decoder: Callable[[Mapping[str, Any]], object] | None, +) -> None: + if decoder is None or record.record_type not in {"candidate_version", "candidate_head"}: + return + version = record + if record.record_type == "candidate_head": + identity = {**record.identity, "version": record.payload["version"]} + document = index.execute( + "SELECT document FROM identities WHERE record_type = 'candidate_version' AND identity = ?", + (_identity_key(identity),), + ).fetchone() + if document is None: + raise BundleFormatError("candidate head references missing version") + version = _parse_record(json.loads(document[0])) + values = { + **_row_values(version), + "status": "pending", + "result_family": None, + "result_artifact_id": None, + "result_revision": None, + "decision_reason": None, + } + if record.record_type == "candidate_head": + values.update(_row_values(record)) + try: + decoder(values) + except (InvalidStoredPayloadError, ValueError) as error: + raise BundleFormatError("Candidate proposal or envelope does not match the target decoder") from error + + +def _validate_record_dependencies( # noqa: C901 - logical record kinds have distinct dependency edges. + record: _Record, + contains: Callable[[str, Mapping[str, object]], bool], + /, +) -> None: + payload = record.payload + identity = record.identity + scope_field = _SPECS[record.record_type].scope_field + if scope_field is None: + return + scope = identity.get(scope_field, payload.get(scope_field)) + if not isinstance(scope, str): + raise BundleFormatError("record has invalid scope identity") + if record.record_type != "scope" and not contains("scope", {"scope_id": scope}): + raise BundleFormatError("record references missing scope") + if record.record_type == "scope": + parent = payload["parent_scope_id"] + if parent is not None and not contains("scope", {"scope_id": parent}): + raise BundleFormatError("scope references missing parent") + elif record.record_type == "scope_context_reference": + if not contains("scope", {"scope_id": identity["referenced_scope_id"]}): + raise BundleFormatError("scope context references missing scope") + elif record.record_type == "source": + if not contains("source_journal_head", {"scope_id": scope}): + raise BundleFormatError("source has no journal head") + observation = _source_observation(record) + if observation is not None and not contains( + "source_definition_fingerprint", + { + "definition_name": observation.source_type, + "definition_version": observation.definition_version, + "fingerprint": observation.definition_fingerprint, + }, + ): + raise BundleFormatError("observation references missing or mismatched Source definition") + elif record.record_type in {"recurrence_match", "recurrence_observation"}: + value = _recurrence_value(record) + if not contains( + "source_position", + { + "scope_id": scope, + **value.task_outcome_ref.model_dump(), + "position": value.task_outcome_position, + }, + ): + raise BundleFormatError("recurrence references missing or mismatched Source") + if isinstance(value, RecurrenceMatch): + refs = value.candidate_refs + else: + refs = (value.artifact_ref,) if value.handoff_ref is None else (value.artifact_ref, value.handoff_ref) + if value.handoff_receipt_ref is not None and not contains( + "source", {"scope_id": scope, **value.handoff_receipt_ref.model_dump()} + ): + raise BundleFormatError("recurrence references missing receipt Source") + if value.recurrence_match_digest is not None and not contains( + "recurrence_match_digest", {"scope_id": scope, "match_key": value.recurrence_match_digest} + ): + raise BundleFormatError("recurrence references missing frozen match") + for ref in refs: + if not contains("artifact_revision", {"scope_id": scope, **ref.model_dump()}): + raise BundleFormatError("recurrence references missing exact Artifact revision") + elif record.record_type == "artifact_revision": + restored = payload["restored_from_revision"] + if restored is not None and not contains("artifact_revision", {**identity, "revision": restored}): + raise BundleFormatError("Artifact rollback references missing revision") + if identity["family"] == "topic-memory" and not contains("topic_memory_publication", identity): + raise BundleFormatError("Topic Memory revision has no publication metadata") + elif record.record_type == "topic_memory_publication": + if not contains("artifact_revision", identity): + raise BundleFormatError("topic memory publication references missing revision") + elif record.record_type == "profile_policy": + candidate_id = payload["pending_candidate_id"] + if candidate_id is not None and not contains( + "candidate_head", {"scope_id": scope, "candidate_id": candidate_id} + ): + raise BundleFormatError("profile policy references missing candidate") + elif record.record_type == "artifact_tag": + if not contains( + "artifact_head", + {"scope_id": scope, "family": identity["family"], "artifact_id": identity["artifact_id"]}, + ): + raise BundleFormatError("artifact tag references missing artifact head") + if identity["target_type"] == "memory_entry" and not contains( + "memory_entry_target", + {"scope_id": scope, "memory_artifact_id": identity["artifact_id"], "entry_id": identity["target_id"]}, + ): + raise BundleFormatError("artifact tag references missing memory entry") + elif record.record_type == "artifact_lineage_source": + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": identity["family"], + "artifact_id": identity["artifact_id"], + "revision": identity["revision"], + }, + ): + raise BundleFormatError("artifact lineage references missing revision") + if not contains( + "source", + {"scope_id": scope, "source_type": payload["source_type"], "source_id": payload["source_id"]}, + ): + raise BundleFormatError("artifact lineage references missing source") + elif record.record_type == "artifact_lineage_artifact": + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": identity["family"], + "artifact_id": identity["artifact_id"], + "revision": identity["revision"], + }, + ): + raise BundleFormatError("artifact lineage references missing revision") + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": payload["upstream_family"], + "artifact_id": payload["upstream_artifact_id"], + "revision": payload["upstream_revision"], + }, + ): + raise BundleFormatError("record references missing artifact revision") + elif record.record_type == "artifact_publication": + if not contains("scope", {"scope_id": payload["source_scope_id"]}): + raise BundleFormatError("Artifact publication references missing source scope") + target_exists = contains( + "artifact_revision", + { + "scope_id": identity["target_scope_id"], + "family": identity["target_family"], + "artifact_id": identity["target_artifact_id"], + "revision": identity["target_revision"], + }, + ) + source_exists = contains( + "artifact_revision", + { + "scope_id": payload["source_scope_id"], + "family": payload["source_family"], + "artifact_id": payload["source_artifact_id"], + "revision": payload["source_revision"], + }, + ) + if not target_exists or not source_exists: + raise BundleFormatError("Artifact publication references missing revision") + elif record.record_type == "memory_entry_version": + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": payload["family"], + "artifact_id": identity["memory_artifact_id"], + "revision": payload["created_in_revision"], + }, + ): + raise BundleFormatError("record references missing artifact revision") + previous = payload["previous_version_id"] + if previous is not None and not contains( + "memory_entry_version", + { + "scope_id": scope, + "memory_artifact_id": identity["memory_artifact_id"], + "entry_version_id": previous, + }, + ): + raise BundleFormatError("memory version references missing previous version") + elif record.record_type == "artifact_handle": + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": identity["family"], + "artifact_id": payload["artifact_id"], + "revision": payload["revision"], + }, + ): + raise BundleFormatError("artifact handle references missing revision") + elif record.record_type == "artifact_head": + if not contains("artifact_revision", {**identity, "revision": payload["revision"]}): + raise BundleFormatError("artifact head references missing revision") + if payload["replacement_artifact_id"] is not None and not contains( + "artifact_head", {**identity, "artifact_id": payload["replacement_artifact_id"]} + ): + raise BundleFormatError("artifact head references missing replacement") + if payload["merged_into_id"] is not None and not contains( + "artifact_head", {**identity, "artifact_id": payload["merged_into_id"]} + ): + raise BundleFormatError("artifact head references missing merge result") + elif record.record_type == "memory_entry_head": + if not contains( + "memory_entry_version", + { + "scope_id": scope, + "memory_artifact_id": identity["memory_artifact_id"], + "entry_version_id": payload["entry_version_id"], + }, + ): + raise BundleFormatError("memory head references missing entry version") + if not contains( + "artifact_revision", + { + "scope_id": scope, + "family": payload["family"], + "artifact_id": identity["memory_artifact_id"], + "revision": payload["head_revision"], + }, + ): + raise BundleFormatError("memory head references missing artifact revision") + elif record.record_type == "candidate_version": + target_family = payload["target_family"] + if target_family is not None and not contains( + "artifact_revision", + { + "scope_id": scope, + "family": target_family, + "artifact_id": payload["target_artifact_id"], + "revision": payload["target_revision"], + }, + ): + raise BundleFormatError("candidate references missing target revision") + elif record.record_type == "candidate_head": + if not contains( + "candidate_version", + {"scope_id": scope, "candidate_id": identity["candidate_id"], "version": payload["version"]}, + ): + raise BundleFormatError("candidate head references missing version") + if not contains( + "candidate_version_family", + {**identity, "version": payload["version"], "family": payload["family"]}, + ): + raise BundleFormatError("candidate head family does not match version") + result_family = payload["result_family"] + if result_family is not None and not contains( + "artifact_revision", + { + "scope_id": scope, + "family": result_family, + "artifact_id": payload["result_artifact_id"], + "revision": payload["result_revision"], + }, + ): + raise BundleFormatError("candidate head references missing result revision") + _validate_embedded_references(record, contains) + + +def _dependency_metadata(source: Path, /) -> tuple[dict[str, int], dict[tuple[str, str], tuple[str, int, int, int]]]: + heads: dict[str, int] = {} + packages: dict[tuple[str, str], tuple[str, int, int, int]] = {} + scopes: list[_Record] = [] + for record in _iter_records(source): + if record.record_type == "scope": + scopes.append(record) + if record.record_type == "source_journal_head": + heads[str(record.identity["scope_id"])] = int(record.payload["position"]) + if record.record_type != "skill_package": + continue + try: + snapshot = capture_skill_archive(cast(bytes, _database_value(record.payload["archive_bytes"]))) + except SkillPackageError as error: + raise BundleFormatError("Skill package archive is invalid") from error + expected = { + "tree_digest": record.identity["tree_digest"], + **{ + field: record.payload[field] + for field in ("archive_digest", "file_count", "uncompressed_size", "archive_size") + }, + } + if snapshot.reference.model_dump() != expected or snapshot.manifest_bytes != _database_value( + record.payload["manifest"] + ): + raise BundleFormatError("Skill package bytes do not match metadata") + packages[(str(record.identity["scope_id"]), str(record.identity["tree_digest"]))] = ( + snapshot.reference.archive_digest, + snapshot.reference.file_count, + snapshot.reference.uncompressed_size, + snapshot.reference.archive_size, + ) + _ordered_scope_records(scopes) + return heads, packages + + +def _validate_skill_package_dependency( + record: _Record, packages: Mapping[tuple[str, str], tuple[str, int, int, int]], / +) -> None: + if record.record_type == "artifact_revision" and record.identity["family"] == "skill": + content = _decoded_bytes_json(record.payload["content"], "Skill content") + elif record.record_type == "candidate_version" and record.payload["family"] == "skill": + try: + proposal, _audit = decode_candidate_proposal_envelope( + cast(bytes, _database_value(record.payload["proposal"])) + ) + except InvalidStoredPayloadError as error: + raise BundleFormatError("Candidate proposal envelope is invalid") from error + content = json.loads(proposal) + else: + return + package = content.get("package") if isinstance(content, dict) else None + if package is None: + return + if not isinstance(package, dict): + raise BundleFormatError("Skill package reference is invalid") + typed_package = cast(dict[str, object], package) + tree_digest = typed_package.get("tree_digest") + archive_digest = typed_package.get("archive_digest") + sizes = ( + typed_package.get("file_count"), + typed_package.get("uncompressed_size"), + typed_package.get("archive_size"), + ) + if ( + not isinstance(tree_digest, str) + or not isinstance(archive_digest, str) + or any(isinstance(value, bool) or not isinstance(value, int) for value in sizes) + ): + raise BundleFormatError("Skill package reference is invalid") + expected = (archive_digest, cast(int, sizes[0]), cast(int, sizes[1]), cast(int, sizes[2])) + key = (str(record.identity["scope_id"]), tree_digest) + if packages.get(key) != expected: + raise BundleFormatError("Skill artifact references missing or mismatched package") + + +def _decoded_bytes_json(value: object, name: str, /) -> object: + encoded = cast(dict[str, object], value).get("base64") if isinstance(value, dict) else None + if not isinstance(encoded, str): + raise BundleFormatError(f"{name} is not encoded bytes") + try: + return json.loads(base64.b64decode(encoded, validate=True), parse_constant=_reject_json_constant) + except (ValueError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise BundleFormatError(f"{name} is not valid JSON") from error + + +def _unique_identities(record: _Record, /) -> Iterator[dict[str, Any]]: + """Yield physical uniqueness keys not already covered by the logical identity.""" + + spec = _SPECS[record.record_type] + values = {**record.identity, **record.payload} + constraints = ( + constraint.columns + for constraint in spec.table.constraints + if isinstance(constraint, PrimaryKeyConstraint | UniqueConstraint) + ) + indexes = (index.columns for index in spec.table.indexes if index.unique) + for columns in (*constraints, *indexes): + fields = tuple(column.name for column in columns) + if set(spec.identity).issubset(fields): + continue + identity = {field: values[field] for field in fields} + # SQL unique keys permit multiple rows when a participating value is NULL. + if all(value is not None for value in identity.values()): + yield identity + + +@contextmanager +def _identity_index(source: Path, /) -> Iterator[sqlite3.Connection]: + index_path = _temporary_path(Path(tempfile.gettempdir()), suffix=".identities.sqlite3") + connection: sqlite3.Connection | None = None + try: + connection = sqlite3.connect(index_path) + connection.execute( + "CREATE TABLE identities (record_type TEXT NOT NULL, identity TEXT NOT NULL, document BLOB, " + "PRIMARY KEY (record_type, identity))" + ) + connection.execute( + "CREATE TABLE publication_edges (target TEXT PRIMARY KEY, source TEXT NOT NULL, digest TEXT NOT NULL, " + "state INTEGER NOT NULL DEFAULT 0)" + ) + connection.execute("CREATE INDEX publication_edges_state ON publication_edges(state, target)") + connection.execute("CREATE TABLE publication_nodes (identity TEXT PRIMARY KEY)") + connection.execute( + "CREATE TABLE tag_labels (target TEXT NOT NULL, tag_key TEXT NOT NULL, label TEXT NOT NULL, " + "PRIMARY KEY (target, tag_key))" + ) + for record in _iter_records(source): + try: + connection.execute( + "INSERT INTO identities (record_type, identity, document) VALUES (?, ?, ?)", + ( + record.record_type, + _identity_key(record.identity), + _canonical_json(_record_document(record)) + if record.record_type in {"candidate_version", "source_definition"} + else None, + ), + ) + for identity in _unique_identities(record): + connection.execute( + "INSERT INTO identities (record_type, identity) VALUES (?, ?)", + (record.record_type + ":unique", _identity_key(identity)), + ) + except sqlite3.IntegrityError as error: + raise BundleFormatError("bundle contains duplicate immutable or unique identity") from error + _index_publication_dependencies(connection, record) + _index_tag_set(connection, record) + if record.record_type == "candidate_version": + connection.execute( + "INSERT INTO identities (record_type, identity) VALUES (?, ?)", + ( + "candidate_version_family", + _identity_key({**record.identity, "family": record.payload["family"]}), + ), + ) + if record.record_type == "source": + connection.execute( + "INSERT INTO identities (record_type, identity) VALUES (?, ?)", + ( + "source_position", + _identity_key({ + **record.identity, + "position": record.payload["journal_position"], + }), + ), + ) + if record.record_type == "recurrence_match": + connection.execute( + "INSERT INTO identities (record_type, identity) VALUES (?, ?)", + ( + "recurrence_match_digest", + _identity_key({ + "scope_id": record.identity["scope_id"], + "match_key": record.payload["match_key"], + }), + ), + ) + if record.record_type == "source_definition": + connection.execute( + "INSERT INTO identities (record_type, identity) VALUES (?, ?)", + ( + "source_definition_fingerprint", + _identity_key({**record.identity, "fingerprint": record.payload["fingerprint"]}), + ), + ) + if record.record_type == "memory_entry_version": + # Tags survive retirement; inactive entries no longer have a + # search projection but retain their authoritative versions. + connection.execute( + "INSERT OR IGNORE INTO identities (record_type, identity) VALUES (?, ?)", + ( + "memory_entry_target", + _identity_key({ + "scope_id": record.identity["scope_id"], + "memory_artifact_id": record.identity["memory_artifact_id"], + "entry_id": record.payload["entry_id"], + }), + ), + ) + connection.commit() + + yield connection + finally: + if connection is not None: + connection.close() + index_path.unlink(missing_ok=True) + + +def _validate_embedded_references( + record: _Record, + contains: Callable[[str, Mapping[str, object]], bool], +) -> None: + if record.record_type not in {"memory_entry_version", "candidate_version"}: + return + scope_id = record.identity["scope_id"] + for reference in _reference_items(record.payload["source_refs"], "source"): + if not contains( + "source", + { + "scope_id": scope_id, + "source_type": reference["source_type"], + "source_id": reference["source_id"], + }, + ): + raise BundleFormatError("record references missing source") + for reference in _reference_items(record.payload["artifact_refs"], "artifact"): + if not contains( + "artifact_revision", + { + "scope_id": scope_id, + "family": reference["family"], + "artifact_id": reference["artifact_id"], + "revision": reference["revision"], + }, + ): + raise BundleFormatError("record references missing artifact revision") + + +def _reference_items(value: object, kind: Literal["source", "artifact"], /) -> tuple[Mapping[str, object], ...]: + encoded = cast(dict[str, object], value).get("base64") if isinstance(value, dict) else None + if not isinstance(value, dict) or set(value) != {"base64"} or not isinstance(encoded, str): + raise BundleFormatError(f"{kind} references are not encoded bytes") + try: + decoded = json.loads(base64.b64decode(encoded, validate=True), parse_constant=_reject_json_constant) + except (ValueError, json.JSONDecodeError) as error: + raise BundleFormatError(f"{kind} references are not valid JSON") from error + if not isinstance(decoded, list): + raise BundleFormatError(f"{kind} references must be an array") + fields = {"source_type", "source_id"} if kind == "source" else {"family", "artifact_id", "revision"} + references: list[Mapping[str, object]] = [] + for item in decoded: + if ( + not isinstance(item, dict) + or set(item) != fields + or any(isinstance(field, bool) or not isinstance(field, str | int) for field in item.values()) + ): + raise BundleFormatError(f"{kind} reference has invalid fields") + references.append(cast(Mapping[str, object], item)) + return tuple(references) + + +def _required_artifact_families(records: Iterable[_Record]) -> set[str]: + families: set[str] = set() + for record in records: + identity = record.identity + payload = record.payload + if record.record_type in { + "artifact_revision", + "artifact_handle", + "artifact_lineage_source", + "artifact_lineage_artifact", + "artifact_head", + "topic_memory_publication", + "artifact_tag", + }: + families.add(str(identity["family"])) + if record.record_type == "profile_policy": + families.add("profile") + if record.record_type == "artifact_publication": + families.update((str(identity["target_family"]), str(payload["source_family"]))) + if record.record_type in {"artifact_lineage_artifact", "candidate_version", "candidate_head"}: + for field in ("upstream_family", "target_family", "result_family"): + value = payload.get(field) + if value is not None: + families.add(str(value)) + if record.record_type in {"memory_entry_version", "memory_entry_head", "candidate_version", "candidate_head"}: + families.add(str(payload["family"])) + return families + + +def _validate_scopes(scopes: Iterable[str]) -> tuple[str, ...]: + if isinstance(scopes, str): + raise BundleFormatError("scopes must be an iterable of scope IDs") + try: + values = tuple(scopes) + except TypeError as error: + raise BundleFormatError("scopes must be an iterable of scope IDs") from error + if not values or any(not isinstance(scope, str) or not scope.strip() or scope != scope.strip() for scope in values): + raise BundleFormatError("scopes must contain at least one non-empty trimmed scope ID") + selected = tuple(sorted(set(values))) + return selected + + +def _identity_key(identity: Mapping[str, object], /) -> str: + return _canonical_json(identity).decode("utf-8") + + +class _DigestAccumulator: + """Incrementally hash a deterministic ordered record-digest sequence.""" + + def __init__(self) -> None: + self._hash = hashlib.sha256() + self._first = True + + def add(self, digest: str, /) -> None: + if not self._first: + self._hash.update(b"\n") + self._hash.update(digest.encode("ascii")) + self._first = False + + def value(self) -> str: + return _SHA256 + self._hash.hexdigest() + + +def _temporary_path(directory: Path, /, *, suffix: str) -> Path: + with tempfile.NamedTemporaryFile( + prefix=".powercontext-archive-", suffix=suffix, dir=directory, delete=False + ) as handle: + return Path(handle.name) + + +def _zip_info(name: str, compression: int, /) -> zipfile.ZipInfo: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = compression + info.external_attr = 0o600 << 16 + return info + + +def _digest(value: object) -> str: + return _SHA256 + hashlib.sha256(_canonical_json(value)).hexdigest() + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8") + + +def _json_value(value: object) -> Any: + if isinstance(value, bytes | bytearray | memoryview): + return {"base64": base64.b64encode(bytes(value)).decode("ascii")} + if isinstance(value, datetime | date): + return value.isoformat() + return value + + +def _database_value(value: object, column: Any | None = None) -> object: + encoded = cast(dict[str, object], value).get("base64") if isinstance(value, dict) else None + if isinstance(value, dict) and set(value) == {"base64"} and isinstance(encoded, str): + try: + return base64.b64decode(encoded, validate=True) + except ValueError as error: + raise BundleFormatError("invalid base64 column value") from error + if isinstance(value, str) and column is not None and isinstance(column.type, DateTime): + try: + return datetime.fromisoformat(value) + except ValueError as error: + raise BundleFormatError("invalid datetime column value") from error + return value + + +def _valid_digest(value: object) -> bool: + return isinstance(value, str) and _HEX_DIGEST.fullmatch(value) is not None + + +def _reject_json_constant(_value: str, /) -> None: + raise BundleFormatError("bundle contains a non-standard JSON value") + + +def _report_progress( + observer: ProgressObserver | None, + phase: Literal["export", "inspect", "validate", "target", "restore", "projections"], + processed: int, + total: int | None, +) -> None: + if observer is not None and (processed in {0, 1, total} or processed % 1_000 == 0): + observer(BundleProgress(phase=phase, processed=processed, total=total)) + + +def _producer_version() -> str: + try: + return version("powercontext") + except PackageNotFoundError: + return "unknown" diff --git a/src/powercontext/builtin/review/service.py b/src/powercontext/builtin/review/service.py index 3951eeb49d..7ca39c4a4b 100644 --- a/src/powercontext/builtin/review/service.py +++ b/src/powercontext/builtin/review/service.py @@ -404,7 +404,7 @@ async def history(self, candidate_id): ) ).mappings() return tuple( - self._candidates._decode_row({ + self._candidates.decode_candidate({ **row, "status": "pending", "result_family": None, diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index c3409d05ad..3fdba4986b 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -341,6 +341,7 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler from powercontext.builtin.handoff_report.application import HandoffReportApplication + from powercontext.builtin.portability import PortableBundleService from powercontext.builtin.runtime.artifact_processing import ArtifactProcessingSupervisors from powercontext.builtin.runtime.scope_reads import ScopeReadSnapshot @@ -3802,6 +3803,7 @@ def __init__( remote_ingestion: RemoteIngestion | None = None, dream_service: DreamService | None = None, generation_concurrency: int = 4, + archive_service: PortableBundleService | None = None, ) -> None: if source_window_limit < 1: raise _RuntimeConfigurationError("source_window_limit") @@ -3927,6 +3929,7 @@ def __init__( self.skill = SkillApplication(self) self.remote_skills = RemoteSkillApplication(self) self.statistics = StatisticsApplication(self) + self.archive = archive_service self.handoff_report: HandoffReportApplication | None = None self.artifact_processing_supervisor: ArtifactProcessingSupervisors | None = None source_ids = scope_ids if source_scope_ids is None else source_scope_ids diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index f02631453f..c5722cb17c 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -29,6 +29,8 @@ from urllib.parse import urlsplit from pydantic import AnyHttpUrl, JsonValue, SecretStr +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncConnection from typing_extensions import override from powercontext._logging import log_safely @@ -118,6 +120,7 @@ bootstrap_processing_schema, ) from powercontext.builtin.persistence.revision_metadata_schema import ensure_artifact_revision_metadata_schema +from powercontext.builtin.persistence.schema import ensure_portable_timestamp_precision from powercontext.builtin.persistence.scope_search_schema import ensure_scope_search_schema from powercontext.builtin.persistence.seekdb.profile import SeekDBConfig, SeekDBProfile from powercontext.builtin.persistence.skill_distribution_schema import ensure_skill_distribution_schema @@ -128,7 +131,7 @@ SQLiteTopicMemoryFTSIndex, SQLiteTopicMemoryVectorIndex, ) -from powercontext.builtin.persistence.tables import BUILTIN_TABLES +from powercontext.builtin.persistence.tables import BUILTIN_TABLES, PORTABLE_RESTORE_RECEIPTS_TABLE from powercontext.builtin.persistence.tag_schema import ensure_topic_memory_tag_schema from powercontext.builtin.persistence.topic_memory import TopicMemoryRepository from powercontext.builtin.persistence.topic_memory_budget import record_topic_memory_work_failure @@ -136,6 +139,7 @@ CompositeTopicMemoryIndex, TopicMemoryIndex, ) +from powercontext.builtin.portability import BundleRecoveryRequiredError from powercontext.builtin.runtime._scope_cache import ScopeCacheObserver from powercontext.builtin.runtime.application import ( BuiltinRuntime, @@ -523,6 +527,7 @@ async def open_builtin_runtime( cursor_secret: bytes | None = None, handoff_verification_keys: tuple[bytes, ...] = (), recall_effort_sink: RecallEffortSink | None = None, + _archive_recovery: bool = False, ) -> AsyncIterator[BuiltinRuntime]: """Open the selected database, inference adapters, and built-in runtime. @@ -646,6 +651,7 @@ async def open_builtin_runtime( contexts = await resources.enter_async_context( open_builtin_contexts( config, + _archive_recovery=_archive_recovery, model_usage_observer=model_usage_observer, candidate_pipeline=configured_pipeline, experience_pipeline=configured_incubation, @@ -867,11 +873,17 @@ async def run_profile(scope_id, high): scheduled_source_runner=scheduled_source_runner, scheduled_experience_runner=scheduled_experience_runner, remote_ingestion=contexts, + archive_service=contexts.portability, ) ) runtime.artifact_processing_supervisor = await resources.enter_async_context( _open_artifact_processing_supervisor( - config, contexts, processing_bindings, tracing=tracing, processing_observer=processing_observer + config, + contexts, + processing_bindings, + tracing=tracing, + recovery=_archive_recovery, + processing_observer=processing_observer, ) ) contexts.profiles.operation_context = runtime._operation @@ -1059,9 +1071,10 @@ async def _open_artifact_processing_supervisor( bindings: Sequence[ArtifactProcessingBinding], *, tracing: RuntimeTracing | None = None, + recovery: bool = False, processing_observer: Callable[[str, str, float], None] | None = None, ) -> AsyncIterator[ArtifactProcessingSupervisors | None]: - if config.runtime.artifact_processing_role == "api" or not bindings: + if recovery or config.runtime.artifact_processing_role == "api" or not bindings: yield None return groups = ( @@ -1111,6 +1124,7 @@ async def open_builtin_contexts( handoff_verification_keys: tuple[bytes, ...] = (), model_usage_observer: Callable[[int, int, int, int], None] | None = None, _topic_memory_worker: bool = False, + _archive_recovery: bool = False, ) -> AsyncIterator[RelationalContexts]: """Open the selected database and expose scope-bound PowerContext providers.""" @@ -1143,6 +1157,7 @@ async def open_builtin_contexts( await ensure_topic_memory_tag_schema(connection) await ensure_scope_search_schema(connection) await ensure_artifact_revision_metadata_schema(connection) + await ensure_portable_timestamp_precision(connection, BUILTIN_TABLES) # A Topic child reuses its parent's schema. It never reads or # writes Memory/Experience projections; rebuilding their FTS # indexes here would take the shared SQLite write lock once @@ -1157,8 +1172,8 @@ async def open_builtin_contexts( await ensure_artifact_head_searchable_text(connection) await assert_atomic_memory_migration_ready(connection) await experience_index.initialize(connection) - await TopicMemoryRepository(index=topic_index).initialize( - connection, configure_retrieval_shape=not _topic_memory_worker + await _initialize_archive_aware_topics( + connection, topic_index, worker=_topic_memory_worker, recovery=_archive_recovery ) contexts = RelationalContexts( model_usage_observer=model_usage_observer, @@ -1199,13 +1214,8 @@ async def open_builtin_contexts( cursor_secret=cursor_secret, tracing=tracing, ) - try: - await contexts.scopes.bootstrap_default() + async with _contexts_lifecycle(contexts, recovery=_archive_recovery): yield contexts - finally: - # Producers are stopped by now, and the profile below still owns - # the database this recorder writes into. - await contexts.aclose_usage_recorder() return experience_index = OceanBaseExperienceFTSIndex() topic_indexes: list[TopicMemoryIndex] = [OceanBaseTopicMemoryFTSIndex()] @@ -1229,6 +1239,7 @@ async def open_builtin_contexts( await ensure_topic_memory_tag_schema(connection) await ensure_scope_search_schema(connection) await ensure_artifact_revision_metadata_schema(connection) + await ensure_portable_timestamp_precision(connection, BUILTIN_TABLES) if not _topic_memory_worker: await _initialize_atomic_memory_authority(connection) await atomic_index.initialize(connection) @@ -1239,8 +1250,8 @@ async def open_builtin_contexts( await ensure_artifact_head_searchable_text(connection) await assert_atomic_memory_migration_ready(connection) await experience_index.initialize(connection) - await TopicMemoryRepository(index=topic_index).initialize( - connection, configure_retrieval_shape=not _topic_memory_worker + await _initialize_archive_aware_topics( + connection, topic_index, worker=_topic_memory_worker, recovery=_archive_recovery ) contexts = RelationalContexts( model_usage_observer=model_usage_observer, @@ -1279,13 +1290,36 @@ async def open_builtin_contexts( cursor_secret=cursor_secret, tracing=tracing, ) - try: - await contexts.scopes.bootstrap_default() + async with _contexts_lifecycle(contexts, recovery=_archive_recovery): yield contexts - finally: - # Producers are stopped by now, and the profile below still owns the - # database this recorder writes into. - await contexts.aclose_usage_recorder() + + +@asynccontextmanager +async def _contexts_lifecycle(contexts: RelationalContexts, /, *, recovery: bool) -> AsyncIterator[None]: + """Expose initialized Contexts and settle their usage before the owning database closes.""" + + try: + if not recovery: + await contexts.scopes.bootstrap_default() + yield + finally: + await contexts.aclose_usage_recorder() + + +async def _initialize_archive_aware_topics( + connection: AsyncConnection, index: TopicMemoryIndex, *, worker: bool, recovery: bool +) -> None: + pending = await connection.scalar( + select(PORTABLE_RESTORE_RECEIPTS_TABLE.c.bundle_id) + .where(PORTABLE_RESTORE_RECEIPTS_TABLE.c.projections_ready.is_(False)) + .limit(1) + ) + if pending and (not recovery or worker): + raise BundleRecoveryRequiredError() + if pending: + await index.initialize(connection) + else: + await TopicMemoryRepository(index=index).initialize(connection, configure_retrieval_shape=not worker) def _register_prompt_demonstrators( diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 8b308073ac..accc72c88a 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -24,16 +24,10 @@ from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass, field, replace from datetime import UTC, datetime -from typing import TYPE_CHECKING, Any, Literal, cast +from typing import TYPE_CHECKING, Literal, cast from uuid import uuid4 -from jsonschema import Draft202012Validator -from jsonschema.exceptions import SchemaError -from jsonschema.exceptions import ValidationError as JsonSchemaValidationError -from jsonschema.protocols import Validator -from referencing import Registry -from referencing.exceptions import Unresolvable -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.artifacts import ArtifactRef @@ -179,6 +173,7 @@ from powercontext.builtin.persistence.topic_memory import TopicMemoryRepository from powercontext.builtin.persistence.topic_memory_index import NoTopicMemoryIndex, TopicMemoryIndex from powercontext.builtin.persistence.topic_memory_management import TopicMemoryManagementWriter +from powercontext.builtin.portability import PortableBundleService from powercontext.builtin.publication import ArtifactPublicationApplication from powercontext.builtin.review.generation import ( GeneratedCandidateResult, @@ -231,6 +226,10 @@ SourceJournalEntry, validate_scope_id, ) +from powercontext.builtin.sources.observation_validation import ( + validate_source_definition_manifest, + validate_source_observation, +) from powercontext.builtin.statistics import ModelUsageOperation, ModelUsagePurpose, RecallTokenMeasurement from powercontext.builtin.triggers import ( HANDOFF_BOUNDARY_TRIGGER_NAME, @@ -245,12 +244,10 @@ from powercontext.errors import ( ArtifactNotFoundError, InvalidSourceDefinitionError, - InvalidSourceObservationError, SourceConflictError, SourceDefinitionNotFoundError, SourceNotFoundError, ) -from powercontext.limits import MAX_SOURCE_OBSERVATION_BYTES from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, ConnectorBinding, @@ -260,7 +257,6 @@ SourceDefinitionRegistry, SourceObservation, SourceRef, - TextEvidence, ) IdFactory = Callable[[str], str] @@ -278,6 +274,18 @@ def _artifact_identity(ref: ArtifactRef) -> tuple[str, str, int]: return ref.family, ref.artifact_id, ref.revision +BUILTIN_ARTIFACT_TYPES = (Handoff, AtomicMemory, Experience, Skill, Profile, Prompt, TopicMemory) +BUILTIN_CANDIDATE_PROPOSAL_TYPES = { + Experience.family: ExperienceContent, + Skill.family: SkillContent, + Profile.family: ProfileCandidateProposal, + AtomicMemory.family: AtomicMemoryContent, + TopicMemory.family: TopicMemoryContent, + Handoff.family: HandoffContent, + Prompt.family: PromptContent, +} + + @dataclass(frozen=True, slots=True) class _Repositories: """Repositories shared by every scoped context.""" @@ -526,7 +534,7 @@ def __init__( self.experience_index = NoExperienceIndex() if experience_index is None else experience_index source_repository = SourceRepository(self.source_registry) artifact_repository = ArtifactRepository( - (Handoff, AtomicMemory, Experience, Skill, Profile, Prompt, TopicMemory), + BUILTIN_ARTIFACT_TYPES, sources=source_repository, ) if atomic_memory_index is None: @@ -577,15 +585,7 @@ def __init__( sources=source_repository, artifacts=artifact_repository, governance=ArtifactGovernanceRepository(), - candidates=CandidateRepository({ - Experience.family: ExperienceContent, - Skill.family: SkillContent, - Profile.family: ProfileCandidateProposal, - AtomicMemory.family: AtomicMemoryContent, - TopicMemory.family: TopicMemoryContent, - Handoff.family: HandoffContent, - Prompt.family: PromptContent, - }), + candidates=CandidateRepository(BUILTIN_CANDIDATE_PROPOSAL_TYPES), connector_checkpoints=ConnectorCheckpointRepository(), source_definitions=SourceDefinitionManifestRepository(), cursors=SourceCursorRepository(), @@ -601,6 +601,15 @@ def __init__( topic_memories=topic_memory_repository, decision_observations=DecisionObservationRepository(), ) + self.portability = PortableBundleService( + database, + projection_rebuilder=self.rebuild_portable_projections, + supported_source_types=tuple(definition.name for definition in self.source_registry.definitions), + supported_artifact_families=artifact_repository.families, + artifact_content_decoder=artifact_repository.decode_content, + candidate_decoder=self.repositories.candidates.decode_candidate, + source_decoder=source_repository.decode_source, + ) self.list_skills = SkillArtifactSearcher(database, self.repositories.artifacts, self.experience_index) self._model_usage = _ModelUsageRecorder( database, @@ -865,7 +874,13 @@ async def register_source_definition( ) -> SourceDefinitionManifest: """Register one immutable declarative Definition supplied by a worker.""" - _validate_source_definition_manifest(manifest, self.source_registry) + validate_source_definition_manifest(manifest) + try: + self.source_registry.definition_for_name(manifest.name) + except SourceDefinitionNotFoundError: + pass + else: + raise InvalidSourceDefinitionError(type(manifest), "name", "must not replace an active Source Definition") try: async with self.database.transaction() as connection: stored = await self.repositories.source_definitions.register(connection, manifest) @@ -897,7 +912,7 @@ async def submit_source_observation( ) except RepositoryNotFoundError: raise SourceDefinitionNotFoundError(observation.source_type, observation.definition_version) from None - _validate_source_observation(observation, manifest) + validate_source_observation(observation, manifest) services = self._services_for(request.scope_id) source_store, source_catalog = services.sources() stored = await source_store.add(observation) @@ -1400,6 +1415,97 @@ async def handoff_scope_ids(self) -> tuple[str, ...]: ).scalars() return tuple(str(value) for value in values) + async def rebuild_portable_projections(self, scope_ids: tuple[str, ...], /) -> None: + """Rebuild target-local search projections after a logical restore.""" + + for scope_id in scope_ids: + async with self.database.transaction() as connection: + await self.atomic_memory_index.initialize(connection) + await connection.execute( + self.atomic_memory_index.table.delete().where(self.atomic_memory_index.table.c.scope_id == scope_id) + ) + after = "" + rebuilt = 0 + while True: + async with self.database.transaction() as connection: + heads = ARTIFACT_HEADS_TABLE + statement = with_artifact_content( + select(heads) + .where( + heads.c.scope_id == scope_id, + heads.c.family == AtomicMemory.family, + heads.c.lifecycle_state == "active", + heads.c.merged_into_id.is_(None), + heads.c.artifact_id > after, + ) + .order_by(heads.c.artifact_id) + .limit(100), + heads, + ) + rows = (await connection.execute(statement)).mappings().all() + records = await self.atomic_memory.service.read_many(connection, scope_id, rows) + if not records: + break + for record in records: + prepared = await self.atomic_memory.publisher.prepare(record.artifact.content) + async with self.database.transaction() as connection: + await self.atomic_memory.publisher.publish(connection, scope_id, record, prepared) + rebuilt += 1 + after = records[-1].artifact.artifact_id + async with self.database.transaction() as connection: + projected = int( + await connection.scalar( + select(func.count()) + .select_from(self.atomic_memory_index.table) + .where(self.atomic_memory_index.table.c.scope_id == scope_id) + ) + or 0 + ) + if projected != rebuilt: + raise RuntimeError("restored Atomic Memory projection is incomplete") # noqa: TRY003 + topic_heads = tuple( + ( + await connection.execute( + select(ARTIFACT_HEADS_TABLE.c.artifact_id, ARTIFACT_HEADS_TABLE.c.revision) + .where( + ARTIFACT_HEADS_TABLE.c.scope_id == scope_id, + ARTIFACT_HEADS_TABLE.c.family == TopicMemory.family, + ) + .order_by(ARTIFACT_HEADS_TABLE.c.artifact_id) + ) + ).all() + ) + for artifact_id, revision in topic_heads: + ref = ArtifactRef( + family=TopicMemory.family, + artifact_id=str(artifact_id), + revision=int(revision), + ) + async with self.database.transaction() as connection: + topic = await self.repositories.artifacts.get(connection, scope_id, ref) + if not isinstance(topic, TopicMemory): + raise TypeError("restored Topic Memory decoded to the wrong Artifact type") # noqa: TRY003 + projection = await self._topic_memory_writer.prepare(topic.content, usage_scope_id=scope_id) + async with self.database.transaction() as connection: + await self.repositories.topic_memories.rebuild_current(connection, scope_id, topic, projection) + async with self.database.transaction() as connection: + await self.topic_memory_index.validate_current(connection) + async with self.database.transaction() as connection: + await self.experience_index.rebuild(connection) + if self.experience_index.families: + missing = await connection.scalar( + select(ARTIFACT_HEADS_TABLE.c.artifact_id) + .where( + ARTIFACT_HEADS_TABLE.c.scope_id.in_(scope_ids), + ARTIFACT_HEADS_TABLE.c.family.in_(tuple(self.experience_index.families)), + ARTIFACT_HEADS_TABLE.c.searchable_text.is_(None), + ) + .limit(1) + ) + if missing is not None: + message = "restored approved Artifact projection is incomplete" + raise RuntimeError(message) + async def commit_handoff(self, scope_id: str, prepared, *, on_commit) -> Handoff: return await self._services_for(scope_id).handoff(on_commit=on_commit).commit(prepared) @@ -1865,70 +1971,6 @@ def _validate_experience_plans( ) -def _validate_source_definition_manifest( - manifest: SourceDefinitionManifest, - source_registry: SourceDefinitionRegistry, -) -> None: - if len(manifest.model_dump_json(by_alias=True).encode()) > 64 * 1024: - raise InvalidSourceDefinitionError(type(manifest), "manifest", "must not exceed 64 KiB") - try: - source_registry.definition_for_name(manifest.name) - except SourceDefinitionNotFoundError: - pass - else: - raise InvalidSourceDefinitionError(type(manifest), "name", "must not replace an active Source Definition") - _json_schema_validator(manifest.name, manifest.source_schema) - standard_text_schema = TextEvidence.model_json_schema() - for projection in manifest.projections: - _json_schema_validator(projection.key.name, projection.schema_) - if projection.key == TEXT_EVIDENCE_PROJECTION_KEY and projection.schema_ != standard_text_schema: - raise InvalidSourceDefinitionError( - type(manifest), - "projection", - f"{projection.key.name!r} must use the standard schema", - ) - - -def _validate_source_observation(source: SourceObservation, manifest: SourceDefinitionManifest) -> None: - if source.source_type != manifest.name or source.definition_version != manifest.version: - raise InvalidSourceObservationError("definition", "does not match the registered manifest identity") - if source.definition_fingerprint != manifest.fingerprint: - raise InvalidSourceObservationError("fingerprint", "does not match the registered manifest") - if len(source.model_dump_json().encode()) > MAX_SOURCE_OBSERVATION_BYTES: - raise InvalidSourceObservationError("size", "must not exceed 4 MiB") - _validate_schema_value(manifest.name, manifest.source_schema, source.payload) - - declarations = {projection.key: projection for projection in manifest.projections} - supplied = {projection.key: projection.value for projection in source.projections} - if declarations.keys() != supplied.keys(): - raise InvalidSourceObservationError("projections", "must exactly match the registered manifest") - for key, declaration in declarations.items(): - value = supplied[key] - _validate_schema_value(key.name, declaration.schema_, value) - if key == TEXT_EVIDENCE_PROJECTION_KEY: - evidence = TextEvidence.model_validate(value) - if evidence.source_type != source.source_type or evidence.source_id != source.name: - raise InvalidSourceObservationError( - "text-evidence", - "source identity does not match the observation envelope", - ) - - -def _json_schema_validator(name: str, schema: Mapping[str, Any]) -> Validator: - try: - Draft202012Validator.check_schema(schema) - return Draft202012Validator(schema, registry=Registry()) - except SchemaError as error: - raise InvalidSourceDefinitionError(type(schema), "schema", f"{name!r} is not valid JSON Schema") from error - - -def _validate_schema_value(name: str, schema: Mapping[str, Any], value: object) -> None: - try: - _json_schema_validator(name, schema).validate(value) - except (JsonSchemaValidationError, Unresolvable) as error: - raise InvalidSourceObservationError("schema", f"value does not match {name!r}") from error - - def _scoped_id_factory(delegate: IdFactory | None) -> IdFactory: def new_id(kind: str) -> str: if delegate is not None: diff --git a/src/powercontext/builtin/sources/observation_validation.py b/src/powercontext/builtin/sources/observation_validation.py new file mode 100644 index 0000000000..2145b55b60 --- /dev/null +++ b/src/powercontext/builtin/sources/observation_validation.py @@ -0,0 +1,86 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Validate transported Sources against their immutable definition manifests.""" + +from collections.abc import Mapping +from typing import Any + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError +from jsonschema.exceptions import ValidationError as JsonSchemaValidationError +from jsonschema.protocols import Validator +from referencing import Registry +from referencing.exceptions import Unresolvable + +from powercontext.errors import InvalidSourceDefinitionError, InvalidSourceObservationError +from powercontext.limits import MAX_SOURCE_OBSERVATION_BYTES +from powercontext.sources import TEXT_EVIDENCE_PROJECTION_KEY, SourceDefinitionManifest, SourceObservation, TextEvidence + + +def validate_source_definition_manifest(manifest: SourceDefinitionManifest, /) -> None: + """Validate the bounded, network-independent schemas declared by a worker.""" + + if len(manifest.model_dump_json(by_alias=True).encode()) > 64 * 1024: + raise InvalidSourceDefinitionError(type(manifest), "manifest", "must not exceed 64 KiB") + json_schema_validator(manifest.name, manifest.source_schema) + standard_text_schema = TextEvidence.model_json_schema() + for projection in manifest.projections: + json_schema_validator(projection.key.name, projection.schema_) + if projection.key == TEXT_EVIDENCE_PROJECTION_KEY and projection.schema_ != standard_text_schema: + raise InvalidSourceDefinitionError( + type(manifest), "projection", f"{projection.key.name!r} must use the standard schema" + ) + + +def validate_source_observation(source: SourceObservation, manifest: SourceDefinitionManifest, /) -> None: + """Check captured payloads and every projection using the owning declaration.""" + + if source.source_type != manifest.name or source.definition_version != manifest.version: + raise InvalidSourceObservationError("definition", "does not match the registered manifest identity") + if source.definition_fingerprint != manifest.fingerprint: + raise InvalidSourceObservationError("fingerprint", "does not match the registered manifest") + if len(source.model_dump_json().encode()) > MAX_SOURCE_OBSERVATION_BYTES: + raise InvalidSourceObservationError("size", "must not exceed 4 MiB") + _validate_schema_value(manifest.name, manifest.source_schema, source.payload) + declarations = {projection.key: projection for projection in manifest.projections} + supplied = {projection.key: projection.value for projection in source.projections} + if declarations.keys() != supplied.keys(): + raise InvalidSourceObservationError("projections", "must exactly match the registered manifest") + for key, declaration in declarations.items(): + value = supplied[key] + _validate_schema_value(key.name, declaration.schema_, value) + if key == TEXT_EVIDENCE_PROJECTION_KEY: + evidence = TextEvidence.model_validate(value) + if evidence.source_type != source.source_type or evidence.source_id != source.name: + raise InvalidSourceObservationError( + "text-evidence", "source identity does not match the observation envelope" + ) + + +def json_schema_validator(name: str, schema: Mapping[str, Any], /) -> Validator: + """Build a JSON Schema validator with no network-backed reference registry.""" + + try: + Draft202012Validator.check_schema(schema) + return Draft202012Validator(schema, registry=Registry()) + except SchemaError as error: + raise InvalidSourceDefinitionError(type(schema), "schema", f"{name!r} is not valid JSON Schema") from error + + +def _validate_schema_value(name: str, schema: Mapping[str, Any], value: object) -> None: + try: + json_schema_validator(name, schema).validate(value) + except (JsonSchemaValidationError, Unresolvable) as error: + raise InvalidSourceObservationError("schema", f"value does not match {name!r}") from error diff --git a/src/powercontext/cli/archive.py b/src/powercontext/cli/archive.py new file mode 100644 index 0000000000..17757f7b03 --- /dev/null +++ b/src/powercontext/cli/archive.py @@ -0,0 +1,270 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Operator-facing portable bundle commands using a controlled built-in Runtime.""" + +from __future__ import annotations + +import asyncio +import json +import stat +from collections.abc import Awaitable, Callable +from dataclasses import asdict +from pathlib import Path +from typing import Annotated, TypeVar, assert_never +from urllib.parse import urlsplit +from urllib.request import url2pathname + +import typer +from sqlalchemy.engine import make_url +from sqlalchemy.ext.asyncio import create_async_engine + +from powercontext.builtin.persistence.artifacts import ArtifactRepository +from powercontext.builtin.persistence.candidates import CandidateRepository +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile +from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile +from powercontext.builtin.persistence.sources import SourceRepository +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.portability import ( + BundleCompatibilityError, + BundleConflictError, + BundleFormatError, + BundleInspection, + BundleProgress, + BundleReceipt, + BundleRecoveryRequiredError, + BundleValidation, + PortableBundleService, +) +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_runtime +from powercontext.builtin.runtime.relational import BUILTIN_ARTIFACT_TYPES, BUILTIN_CANDIDATE_PROPOSAL_TYPES +from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY +from powercontext.paths import default_scheduler_path +from powercontext.server.configuration import server_settings_context + +HELP_OPTION_NAMES = ("-h", "--help") +_SQLITE_TARGET_INSPECTION_ERROR = "cannot inspect configured SQLite target" +_ResultT = TypeVar("_ResultT", BundleInspection, BundleValidation, BundleReceipt) + +archive_app = typer.Typer( + name="archive", + context_settings={"help_option_names": HELP_OPTION_NAMES}, + help="Create, inspect, validate, and restore portable logical bundles.", + no_args_is_help=True, +) + + +@archive_app.command("export") +def export_bundle( + scope_id: Annotated[ + list[str], typer.Option("--scope-id", min=1, help="Complete scope to export; repeat as needed.") + ], + output: Annotated[Path, typer.Option("--output", help="New portable bundle file to create.")], + compress: Annotated[bool, typer.Option("--compress/--no-compress", help="Compress the portable bundle.")] = True, + env_file: Annotated[ + Path | None, typer.Option("--env-file", help="Deployment environment for SQLite, SeekDB, or OceanBase.") + ] = None, +) -> None: + """Create a verified logical bundle from complete scopes.""" + + _emit_archive_result( + lambda archive: archive.export( + scope_id, output, authorize=_authorize_local_export, progress=_emit_progress, compress=compress + ), + env_file=env_file, + ) + + +@archive_app.command("inspect") +def inspect_bundle( + source: Annotated[Path, typer.Argument(help="Portable bundle file to inspect.")], +) -> None: + """Verify archive structure and checksums without writing domain data.""" + + try: + _emit(asyncio.run(PortableBundleService.inspect(source, progress=_emit_progress))) + except BundleFormatError as error: + typer.echo(f"error: archive validation failed: {error}", err=True) + raise typer.Exit(code=2) from error + + +@archive_app.command("restore") +def restore_bundle( + source: Annotated[Path, typer.Argument(help="Portable bundle file to validate or restore.")], + dry_run: Annotated[bool, typer.Option("--dry-run", help="Validate only; never write domain data.")] = False, + yes: Annotated[bool, typer.Option("--yes", help="Confirm a write restore.")] = False, + env_file: Annotated[ + Path | None, typer.Option("--env-file", help="Target deployment environment for validation or restore.") + ] = None, +) -> None: + """Validate or restore a portable bundle into the configured deployment.""" + + if dry_run: + _emit_validation_result(source, env_file=env_file) + return + if not yes: + raise typer.BadParameter( # noqa: TRY003 + "restore writes immutable records; re-run with --yes to confirm", + param_hint="--yes", + ) + _emit_archive_result( + lambda archive: archive.restore(source, progress=_emit_progress), env_file=env_file, recovery=True + ) + + +async def _run_archive( + operation: Callable[[PortableBundleService], Awaitable[_ResultT]], + /, + *, + env_file: Path | None, + recovery: bool = False, +) -> _ResultT: + with server_settings_context(env_file=env_file) as settings: + config = BuiltinConfig( + runtime=settings.runtime, + database=settings.database, + inference=settings.inference, + handoff_report=settings.handoff_report, + external_skills=settings.external_skills, + ) + async with open_builtin_runtime( + config, scheduler_path=default_scheduler_path(), _archive_recovery=recovery + ) as runtime: + if runtime.archive is None: + raise RuntimeError("portable archive service is unavailable") # noqa: TRY003 + return await operation(runtime.archive) + + +async def _authorize_local_export(_scopes: tuple[str, ...], /) -> None: + """A local operator's authority is the OS permission to read the database.""" + + +async def _validate_only(source: Path, /, *, env_file: Path | None) -> BundleValidation: + """Open only the configured database profile; do not initialize Runtime state.""" + + with server_settings_context(env_file=env_file) as settings: + database = settings.database + if isinstance(database, SQLiteConfig): + existing = database.is_persistent and await asyncio.to_thread(_database_file_exists, database.url) + if existing: + async with SQLiteProfile.open_readonly(database) as profile: + return await _validate_with_database(profile.database, source) + engine = create_async_engine("sqlite+aiosqlite:///:memory:", echo=database.echo, hide_parameters=True) + validation_database = AsyncDatabase.own(engine, shared_connection=True) + try: + return await _validate_with_database(validation_database, source) + finally: + await validation_database.close() + if isinstance(database, OceanBaseConfig): + async with OceanBaseProfile.open(database, tables=()) as profile: + return await _validate_with_database(profile.database, source) + if isinstance(database, SeekDBConfig): + async with SeekDBProfile.open(database, tables=()) as profile: + return await _validate_with_database(profile.database, source) + assert_never(database) + + +async def _validate_with_database(database: AsyncDatabase, source: Path, /) -> BundleValidation: + async def projection_capability(_scopes: tuple[str, ...], /) -> None: + return None + + artifacts = ArtifactRepository(BUILTIN_ARTIFACT_TYPES) + archive = PortableBundleService( + database, + projection_rebuilder=projection_capability, + supported_source_types=tuple(definition.name for definition in BUILTIN_SOURCE_REGISTRY.definitions), + supported_artifact_families=artifacts.families, + artifact_content_decoder=artifacts.decode_content, + candidate_decoder=CandidateRepository(BUILTIN_CANDIDATE_PROPOSAL_TYPES).decode_candidate, + source_decoder=SourceRepository(BUILTIN_SOURCE_REGISTRY).decode_source, + ) + return await archive.validate(source, progress=_emit_progress) + + +def _database_file_exists(value: str, /) -> bool: + authority = "" + try: + url = make_url(value) + arguments, options = url.get_dialect()().create_connect_args(url) + filename = str(arguments[0]) + if options.get("uri") and filename.startswith("file:"): + uri = urlsplit(filename.replace("\t", "%09").replace("\n", "%0A").replace("\r", "%0D")) + authority = uri.netloc + filename = url2pathname(uri.path).partition("\0")[0] + except (OSError, ValueError) as error: + raise BundleFormatError(_SQLITE_TARGET_INSPECTION_ERROR) from error + if authority not in {"", "localhost"} or not filename: + raise BundleFormatError(_SQLITE_TARGET_INSPECTION_ERROR) + target = Path(filename) + for candidate in (target, *target.parents): + try: + mode = candidate.stat().st_mode + except FileNotFoundError: + continue + except (OSError, ValueError) as error: + raise BundleFormatError(_SQLITE_TARGET_INSPECTION_ERROR) from error + if candidate == target and stat.S_ISREG(mode): + return True + if candidate != target and stat.S_ISDIR(mode): + return False + raise BundleFormatError(_SQLITE_TARGET_INSPECTION_ERROR) + raise BundleFormatError(_SQLITE_TARGET_INSPECTION_ERROR) + + +def _emit_progress(value: BundleProgress, /) -> None: + typer.echo(json.dumps({"progress": asdict(value)}, sort_keys=True), err=True) + + +def _emit_archive_result( + operation: Callable[[PortableBundleService], Awaitable[_ResultT]], + /, + *, + env_file: Path | None, + recovery: bool = False, +) -> None: + """Render expected archive failures without leaking stack traces or records.""" + + try: + _emit(asyncio.run(_run_archive(operation, env_file=env_file, recovery=recovery))) + except BundleCompatibilityError as error: + typer.echo(f"error: archive target is incompatible: {error}", err=True) + raise typer.Exit(code=3) from error + except BundleFormatError as error: + typer.echo(f"error: archive validation failed: {error}", err=True) + raise typer.Exit(code=2) from error + except BundleConflictError as error: + typer.echo(f"error: archive restore conflict: {error}", err=True) + raise typer.Exit(code=3) from error + except BundleRecoveryRequiredError as error: + typer.echo(f"error: archive recovery required: {error}", err=True) + raise typer.Exit(code=3) from error + + +def _emit_validation_result(source: Path, /, *, env_file: Path | None) -> None: + try: + validation = asyncio.run(_validate_only(source, env_file=env_file)) + _emit(validation) + if not validation.compatible: + raise typer.Exit(code=3) + except BundleCompatibilityError as error: + typer.echo(f"error: archive target is incompatible: {error}", err=True) + raise typer.Exit(code=3) from error + except BundleFormatError as error: + typer.echo(f"error: archive validation failed: {error}", err=True) + raise typer.Exit(code=2) from error + + +def _emit(value: BundleInspection | BundleValidation | BundleReceipt, /) -> None: + typer.echo(json.dumps(asdict(value), default=list, sort_keys=True)) diff --git a/tests/builtin/persistence/test_oceanbase_profile.py b/tests/builtin/persistence/test_oceanbase_profile.py index 0b44c7a2aa..cba0b75f7d 100644 --- a/tests/builtin/persistence/test_oceanbase_profile.py +++ b/tests/builtin/persistence/test_oceanbase_profile.py @@ -211,6 +211,32 @@ async def create_no_tables(_connection: object, _tables: tuple[Table, ...]) -> N asyncio.run(scenario()) +@pytest.mark.parametrize("version", ["5.7.25-OceanBase seekdb-v1.4.0.0", "8.0.40", "unknown"]) +def test_profile_recognizes_mysql_only_seekdb_without_tenant_variable(version, monkeypatch) -> None: + class Connection(_Connection): + async def exec_driver_sql(self, statement, parameters=None): + if statement == "SELECT VERSION()": + return _Result(((version,),)) + return await super().exec_driver_sql(statement, parameters) + + async def create_no_tables(_connection, _tables): + pass + + async def scenario(): + engine = _Engine(connection=Connection(row=None)) + monkeypatch.setattr(oceanbase_profile_module, "create_tables", create_no_tables) + monkeypatch.setattr(oceanbase_profile_module, "assert_dream_schema_ready", _skip_dream_preflight) + if "seekdb" in version: + async with OceanBaseProfile.attach(cast(AsyncEngine, engine), tables=()): + pass + else: + with pytest.raises(UnsupportedOceanBaseTenantError): + async with OceanBaseProfile.attach(cast(AsyncEngine, engine), tables=()): + pass + + asyncio.run(scenario()) + + def test_profile_rejects_legacy_identity_column_collation(monkeypatch: pytest.MonkeyPatch) -> None: async def scenario() -> None: extension_table = Table( diff --git a/tests/builtin/persistence/test_records.py b/tests/builtin/persistence/test_records.py index 887f5cac12..1c6384dba9 100644 --- a/tests/builtin/persistence/test_records.py +++ b/tests/builtin/persistence/test_records.py @@ -468,14 +468,14 @@ async def scenario() -> None: await records.create_source("scope-a", "content", f"{index}:" + "x" * 1_100_000) decoded = 0 - original_decode = sources._decode_row + original_decode = sources.decode_source def counting_decode(row): nonlocal decoded decoded += 1 return original_decode(row) - monkeypatch.setattr(sources, "_decode_row", counting_decode) + monkeypatch.setattr(sources, "decode_source", counting_decode) first = await records.list_sources("scope-a", limit=8, cursor=None, caller="user:one") assert len(first.items) == 3 diff --git a/tests/builtin/persistence/test_topic_memory_oceanbase.py b/tests/builtin/persistence/test_topic_memory_oceanbase.py index c1abfaba02..037d8aa73a 100644 --- a/tests/builtin/persistence/test_topic_memory_oceanbase.py +++ b/tests/builtin/persistence/test_topic_memory_oceanbase.py @@ -76,7 +76,7 @@ def test_oceanbase_topic_schema_compiles_native_text_and_vector_storage() -> Non vector_topics = str(CreateTable(vector_index.topic_table).compile(dialect=dialect)) vector_chunks = str(CreateTable(vector_index.chunk_table).compile(dialect=dialect)) - assert "published_at DATETIME NOT NULL" in publication + assert "published_at DATETIME(6) NOT NULL" in publication assert "FOREIGN KEY(scope_id, family, artifact_id, revision)" in publication assert "shape VARCHAR(16)" in retrieval_shape assert "profile_fingerprint VARCHAR(64)" in retrieval_shape @@ -188,6 +188,53 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_oceanbase_vector_startup_recovers_from_ddl_lock_conflict() -> None: + async def scenario() -> None: + connection = AsyncMock(spec=AsyncConnection) + connection.dialect = mysql.dialect() + connection.scalar.return_value = "VECTOR(3)" + connection.run_sync.side_effect = [ + OperationalError("CREATE VECTOR INDEX", None, MySQLOperationalError(1205, "DDL lock conflict")), + None, + ] + profile = EmbeddingProfile( + profile_id="topic-test", model="test", dimension=3, distance="l2", normalization="unit" + ) + await OceanBaseTopicMemoryVectorIndex(profile).initialize(cast(AsyncConnection, connection)) + connection.commit.assert_not_awaited() + connection.rollback.assert_not_awaited() + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("failure", [1142, 1213, "cancelled", "persistent-lock"]) +def test_oceanbase_vector_startup_preserves_failures_and_bounds_lock_retries(failure) -> None: + async def scenario() -> None: + connection = AsyncMock(spec=AsyncConnection) + connection.dialect = mysql.dialect() + connection.scalar.return_value = "VECTOR(3)" + error = ( + asyncio.CancelledError() + if failure == "cancelled" + else OperationalError( + "CREATE VECTOR INDEX", + None, + MySQLOperationalError(1205 if failure == "persistent-lock" else failure, "DDL failed"), + ) + ) + connection.run_sync.side_effect = error if failure == "persistent-lock" else [error, None] + profile = EmbeddingProfile( + profile_id="topic-test", model="test", dimension=3, distance="l2", normalization="unit" + ) + with pytest.raises(type(error)) as raised: + await asyncio.wait_for( + OceanBaseTopicMemoryVectorIndex(profile).initialize(cast(AsyncConnection, connection)), timeout=5 + ) + assert raised.value is error + + asyncio.run(scenario()) + + def test_oceanbase_fts_initializes_and_queries_both_current_projection_channels() -> None: async def scenario() -> None: connection = AsyncMock(spec=AsyncConnection) diff --git a/tests/builtin/portability/test_bundle.py b/tests/builtin/portability/test_bundle.py new file mode 100644 index 0000000000..41f7f6de8b --- /dev/null +++ b/tests/builtin/portability/test_bundle.py @@ -0,0 +1,984 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Observable contracts for portable logical bundles.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import sqlite3 +import tracemalloc +import zipfile +from datetime import UTC, datetime +from pathlib import Path +from typing import cast + +import pytest +from sqlalchemy import insert, select + +from powercontext.builtin.artifacts.skill import capture_skill_directory +from powercontext.builtin.persistence.database import AsyncDatabase +from powercontext.builtin.persistence.sources import SourceRepository +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import ( + ARTIFACT_HEADS_TABLE, + ARTIFACT_PUBLICATIONS_TABLE, + ARTIFACT_TAGS_TABLE, + ARTIFACTS_TABLE, + BUILTIN_TABLES, + CANDIDATE_HEADS_TABLE, + CANDIDATE_VERSIONS_TABLE, + PORTABLE_RESTORE_RECEIPTS_TABLE, + PROFILE_POLICIES_TABLE, + SCOPE_CONTEXT_REFERENCES_TABLE, + SCOPE_CREATION_REQUESTS_TABLE, + SCOPES_TABLE, + SKILL_PACKAGES_TABLE, + SOURCE_JOURNAL_HEADS_TABLE, + SOURCES_TABLE, + TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE, +) +from powercontext.builtin.portability import BundleConflictError, BundleFormatError, PortableBundleService +from powercontext.builtin.portability import bundle as bundle_module +from powercontext.builtin.sources import BUILTIN_SOURCE_REGISTRY, ContentSource +from powercontext.sources import SourceMaterialization + +_SOURCES = SourceRepository(BUILTIN_SOURCE_REGISTRY) + + +def _content_payload(source_id: str, content: str) -> bytes: + return ( + ContentSource(name=source_id, materialization=SourceMaterialization.CAPTURED, content=content) + .model_dump_json() + .encode() + ) + + +async def _authorize_export(_scopes: tuple[str, ...], /) -> None: + pass + + +async def _insert_scope(connection, scope_id: str = "project:one") -> None: + await connection.execute( + insert(SCOPES_TABLE).values( + scope_id=scope_id, + title="Portable fixture", + summary="Portable fixture", + scope_id_search=scope_id, + title_search="Portable fixture", + summary_search="Portable fixture", + parent_scope_id=None, + version=1, + ) + ) + + +@pytest.mark.parametrize("suffix", ["", "-wal", "-shm", "-journal"]) +@pytest.mark.parametrize("output_name", ["source.db", "SOURCE.DB"]) +def test_export_cannot_replace_its_sqlite_database_or_journals(tmp_path: Path, suffix: str, output_name: str) -> None: + async def scenario() -> None: + database = tmp_path / "source.db" + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{database}") + async with SQLiteProfile.open(config, tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + with pytest.raises(BundleFormatError, match="source database"): + await PortableBundleService(source.database).export( + ["project:one"], tmp_path / (output_name + suffix), authorize=_authorize_export + ) + assert database.read_bytes().startswith(b"SQLite format 3\0") + async with ( + SQLiteProfile.open(config, tables=BUILTIN_TABLES) as reopened, + reopened.database.transaction() as connection, + ): + assert await connection.scalar(select(SCOPES_TABLE.c.scope_id)) == "project:one" + + asyncio.run(scenario()) + + +def test_export_preserves_its_callers_sqlite_transaction(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "snapshot.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await PortableBundleService(source.database).export( + ["project:one"], archive, authorize=_authorize_export + ) + assert await connection.scalar(select(SCOPES_TABLE.c.scope_id)) == "project:one" + async with source.database.transaction() as connection: + assert await connection.scalar(select(SCOPES_TABLE.c.scope_id)) == "project:one" + + asyncio.run(scenario()) + + +def test_export_rejects_hard_link_to_source_database(tmp_path: Path) -> None: + async def scenario() -> None: + database = tmp_path / "source.db" + alias = tmp_path / "different-name.db" + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{database}") + async with SQLiteProfile.open(config, tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + try: + os.link(database, alias) + except OSError as error: + pytest.skip(f"hard links unavailable: {error}") + with pytest.raises(BundleFormatError, match="source database"): + await PortableBundleService(source.database).export(["project:one"], alias, authorize=_authorize_export) + assert alias.samefile(database) + assert database.read_bytes().startswith(b"SQLite format 3\0") + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("notation", ["urn", "braces", "compact", "uppercase"]) +def test_bundle_rejects_noncanonical_receipt_identity(tmp_path: Path, notation: str) -> None: + async def scenario() -> None: + archive = tmp_path / "identity.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + original = archive.read_bytes() + with zipfile.ZipFile(archive) as bundle: + manifest = json.loads(bundle.read("manifest.json")) + records = bundle.read("records.ndjson") + identity = manifest["bundle_id"] + manifest["bundle_id"] = { + "urn": "urn:uuid:" + identity, + "braces": "{" + identity + "}", + "compact": identity.replace("-", ""), + "uppercase": identity.upper(), + }[notation] + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("manifest.json", json.dumps(manifest)) + bundle.writestr("records.ndjson", records) + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + service = PortableBundleService(target.database) + for operation in (service.inspect, service.validate, service.restore): + with pytest.raises(BundleFormatError, match="bundle identity"): + await operation(archive) + async with target.database.transaction() as connection: + assert await connection.scalar(select(SCOPES_TABLE.c.scope_id)) is None + archive.write_bytes(original) + assert (await service.validate(archive)).compatible + assert (await service.restore(archive)).inserted == 1 + + asyncio.run(scenario()) + + +def test_bundle_size_limit_includes_zip_prefix(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + async def scenario() -> None: + archive = tmp_path / "padded.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + original = archive.read_bytes() + monkeypatch.setattr(bundle_module, "_MAX_ARCHIVE_BYTES", 32 * 1024) + archive.write_bytes(b"\0" * (32 * 1024 + 1) + original) + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + service = PortableBundleService(target.database) + for operation in (service.inspect, service.validate, service.restore): + with pytest.raises(BundleFormatError, match="resource limit"): + await operation(archive) + async with target.database.transaction() as connection: + assert await connection.scalar(select(SCOPES_TABLE.c.scope_id)) is None + archive.write_bytes(original) + assert (await service.restore(archive)).inserted == 1 + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("mutation", ["identity_override", "unknown_field", "missing_field", "wrong_type", "scope"]) +def test_restore_rejects_invalid_wire_fields_before_writing(tmp_path: Path, mutation: str) -> None: + async def scenario() -> None: + archive = tmp_path / "invalid.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + with zipfile.ZipFile(archive) as bundle: + manifest = json.loads(bundle.read("manifest.json")) + record = json.loads(bundle.read("records.ndjson")) + if mutation == "identity_override": + record["payload"]["scope_id"] = "project:other" + elif mutation == "unknown_field": + record["payload"]["unexpected"] = "value" + elif mutation == "missing_field": + del record["payload"]["title"] + elif mutation == "wrong_type": + record["payload"]["version"] = True + else: + manifest["scopes"] = ["project:other"] + del record["digest"] + canonical = json.dumps(record, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() + record["digest"] = "sha256:" + hashlib.sha256(canonical).hexdigest() + manifest["total_digest"] = "sha256:" + hashlib.sha256(record["digest"].encode()).hexdigest() + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("manifest.json", json.dumps(manifest)) + bundle.writestr("records.ndjson", json.dumps(record) + "\n") + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + with pytest.raises(BundleFormatError): + await PortableBundleService(target.database).restore(archive) + async with target.database.transaction() as connection: + assert (await connection.execute(select(SCOPES_TABLE))).all() == [] + + asyncio.run(scenario()) + + +def test_bundle_round_trips_authoritative_scope_data_and_excludes_projections(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=_content_payload("source-1", "Original captured content."), + journal_position=1, + ) + ) + await connection.execute( + insert(ARTIFACTS_TABLE).values( + scope_id="project:one", + family="handoff", + artifact_id="handoff", + revision=1, + content=b'{"content":{"summary":"hello"}}', + ) + ) + await connection.execute( + insert(ARTIFACT_HEADS_TABLE).values( + scope_id="project:one", + family="handoff", + artifact_id="handoff", + revision=1, + searchable_text="not portable", + ) + ) + receipt = await PortableBundleService(source.database).export( + ["project:one"], archive, authorize=_authorize_export + ) + assert receipt.record_count == 5 + + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + service = PortableBundleService(target.database, source_decoder=_SOURCES.decode_source) + inspection = await service.validate(archive, supported_source_types=("content",)) + assert inspection.scopes == ("project:one",) + restored = await service.restore(archive, supported_source_types=("content",)) + assert restored.inserted == 5 + repeated = await service.restore(archive, supported_source_types=("content",)) + assert repeated.already_present == 5 + async with target.database.transaction() as connection: + assert await connection.scalar(select(ARTIFACT_HEADS_TABLE.c.searchable_text)) is None + + asyncio.run(scenario()) + + +def test_export_is_byte_deterministic_and_preserves_scope_relationships(tmp_path: Path) -> None: + async def scenario() -> None: + first = tmp_path / "first.pcb" + second = tmp_path / "second.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection, "project:parent") + await connection.execute( + insert(SCOPES_TABLE).values( + scope_id="workstream:child", + title="Child", + summary="Child scope", + scope_id_search="workstream:child", + title_search="Child", + summary_search="Child scope", + parent_scope_id="project:parent", + version=1, + ) + ) + await connection.execute( + insert(SCOPE_CONTEXT_REFERENCES_TABLE).values( + scope_id="workstream:child", referenced_scope_id="project:parent" + ) + ) + await connection.execute( + insert(SCOPE_CREATION_REQUESTS_TABLE).values( + idempotency_key="child-request", + request_digest="digest", + scope_id="workstream:child", + ) + ) + await connection.execute( + insert(ARTIFACTS_TABLE), + [ + { + "scope_id": "project:parent", + "family": "experience", + "artifact_id": "source-experience", + "revision": 1, + "content": b"source", + }, + { + "scope_id": "workstream:child", + "family": "experience", + "artifact_id": "published-experience", + "revision": 1, + "content": b"published", + }, + ], + ) + await connection.execute( + insert(ARTIFACT_PUBLICATIONS_TABLE).values( + target_scope_id="workstream:child", + target_family="experience", + target_artifact_id="published-experience", + target_revision=1, + source_scope_id="project:parent", + source_family="experience", + source_artifact_id="source-experience", + source_revision=1, + content_digest="d" * 64, + idempotency_key="publish-experience", + ) + ) + service = PortableBundleService(source.database) + one = await service.export(["workstream:child", "project:parent"], first, authorize=_authorize_export) + two = await service.export(["project:parent", "workstream:child"], second, authorize=_authorize_export) + assert one.bundle_id == two.bundle_id + assert first.read_bytes() == second.read_bytes() + + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + restored = await PortableBundleService(target.database).restore(first) + async with target.database.transaction() as connection: + parent = await connection.scalar( + select(SCOPES_TABLE.c.parent_scope_id).where(SCOPES_TABLE.c.scope_id == "workstream:child") + ) + context_count = await connection.scalar(select(SCOPE_CONTEXT_REFERENCES_TABLE.c.scope_id)) + creation_scope = await connection.scalar(select(SCOPE_CREATION_REQUESTS_TABLE.c.scope_id)) + publication_source = await connection.scalar(select(ARTIFACT_PUBLICATIONS_TABLE.c.source_scope_id)) + assert restored.record_count == 7 + assert parent == "project:parent" + assert context_count == "workstream:child" + assert creation_scope == "workstream:child" + assert publication_source == "project:parent" + + asyncio.run(scenario()) + + +def test_validation_accepts_a_clean_target_without_initializing_schema(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + async with SQLiteProfile.open(SQLiteConfig(), tables=()) as clean_target: + validation = await PortableBundleService( + clean_target.database, + projection_rebuilder=lambda _scopes: asyncio.sleep(0), + ).validate(archive) + assert validation.compatible is True + assert validation.already_present == 0 + assert validation.conflicts == 0 + + asyncio.run(scenario()) + + +def test_restore_rejects_divergent_immutable_identity_without_overwrite(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=_content_payload("source-1", "original"), + journal_position=1, + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + async with target.database.transaction() as connection: + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=_content_payload("source-1", "different"), + journal_position=1, + ) + ) + service = PortableBundleService(target.database, source_decoder=_SOURCES.decode_source) + validation = await service.validate(archive) + assert validation.compatible is False + assert validation.conflicts == 1 + with pytest.raises(BundleConflictError): + await service.restore(archive) + async with target.database.transaction() as connection: + assert await connection.scalar(select(SOURCES_TABLE.c.payload)) == _content_payload( + "source-1", "different" + ) + + asyncio.run(scenario()) + + +def test_failed_restore_rolls_back_earlier_records_and_can_be_retried(tmp_path: Path) -> None: + """A write-time constraint conflict must not strand earlier bundle rows.""" + + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=_content_payload("source-1", "new source"), + journal_position=1, + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + async with target.database.transaction() as connection: + await connection.exec_driver_sql( + "CREATE TRIGGER reject_restore BEFORE INSERT ON pc_sources " + "BEGIN SELECT RAISE(ABORT, 'write rejected after preflight'); END" + ) + service = PortableBundleService(target.database, source_decoder=_SOURCES.decode_source) + validation = await service.validate(archive) + assert validation.compatible + with pytest.raises(BundleConflictError, match="target constraint conflict"): + await service.restore(archive) + + async with target.database.transaction() as connection: + for table in (SCOPES_TABLE, SOURCE_JOURNAL_HEADS_TABLE, SOURCES_TABLE): + assert (await connection.execute(select(table))).all() == [] + await connection.exec_driver_sql("DROP TRIGGER reject_restore") + + retried = await service.restore(archive) + assert retried.inserted == 3 + assert retried.already_present == 0 + + asyncio.run(scenario()) + + +def test_restore_reports_projection_readiness_from_runtime_rebuilder(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + rebuilt: list[tuple[str, ...]] = [] + progress = [] + + async def rebuild(scopes: tuple[str, ...]) -> None: + rebuilt.append(scopes) + + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + receipt = await PortableBundleService(target.database, projection_rebuilder=rebuild).restore( + archive, progress=progress.append + ) + assert rebuilt == [("project:one",)] + assert receipt.projections_ready is True + assert receipt.status == "ready" + assert {event.phase for event in progress} >= {"validate", "target", "restore", "projections"} + + asyncio.run(scenario()) + + +def test_export_authorization_precedes_database_access(tmp_path: Path) -> None: + class _ForbiddenDatabase: + def transaction(self): + raise AssertionError("database was accessed before authorization") # noqa: TRY003 + + async def scenario() -> None: + async def deny(_scopes: tuple[str, ...]) -> None: + raise PermissionError("denied") + + service = PortableBundleService(cast(AsyncDatabase, _ForbiddenDatabase())) + with pytest.raises(PermissionError, match="denied"): + await service.export(["project:one"], tmp_path / "denied.pcb", authorize=deny) + assert not (tmp_path / "denied.pcb").exists() + + asyncio.run(scenario()) + + +def test_projection_failure_leaves_a_durable_not_ready_receipt(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + + async def fail_rebuild(_scopes: tuple[str, ...]) -> None: + raise RuntimeError("projection failed") # noqa: TRY003 + + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + with pytest.raises(RuntimeError, match="projection failed"): + await PortableBundleService(target.database, projection_rebuilder=fail_rebuild).restore(archive) + async with target.database.transaction() as connection: + row = (await connection.execute(select(PORTABLE_RESTORE_RECEIPTS_TABLE))).mappings().one() + assert row["status"] == "authoritative_restored" + assert row["projections_ready"] is False + + asyncio.run(scenario()) + + +def test_export_rejects_secret_fields_without_echoing_values(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "secret.pcb" + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=b'{"metadata":{"api_key":"do-not-echo"}}', + journal_position=1, + ) + ) + with pytest.raises(BundleFormatError) as caught: + await PortableBundleService(source.database).export( + ["project:one"], archive, authorize=_authorize_export + ) + assert "credential-shaped" in str(caught.value) + assert "do-not-echo" not in str(caught.value) + assert not archive.exists() + + asyncio.run(scenario()) + + +def test_restore_accepts_a_valid_bundle_with_records_in_a_different_physical_order(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "ordered.pcb" + reordered = tmp_path / "reordered.pcb" + database = tmp_path / "source.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{database}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1)) + await connection.execute( + insert(SOURCES_TABLE).values( + scope_id="project:one", + source_type="content", + source_id="source-1", + payload=_content_payload("source-1", "source"), + journal_position=1, + ) + ) + await connection.execute( + insert(ARTIFACTS_TABLE).values( + scope_id="project:one", + family="handoff", + artifact_id="handoff", + revision=1, + content=b"artifact", + ) + ) + await connection.execute( + insert(ARTIFACT_HEADS_TABLE).values( + scope_id="project:one", family="handoff", artifact_id="handoff", revision=1 + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + + with zipfile.ZipFile(archive) as input_archive: + manifest = json.loads(input_archive.read("manifest.json")) + records = list(reversed(input_archive.read("records.ndjson").splitlines())) + digests = [json.loads(record)["digest"] for record in records] + manifest["total_digest"] = "sha256:" + hashlib.sha256("\n".join(digests).encode()).hexdigest() + with zipfile.ZipFile(reordered, "w") as output_archive: + output_archive.writestr("manifest.json", json.dumps(manifest)) + output_archive.writestr("records.ndjson", b"\n".join(records) + b"\n") + + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + receipt = await PortableBundleService(target.database, source_decoder=_SOURCES.decode_source).restore( + reordered + ) + assert receipt.inserted == 5 + + asyncio.run(scenario()) + + +def test_restore_keeps_large_bundle_payloads_out_of_process_memory(tmp_path: Path) -> None: + """The import budget is independent of aggregate NDJSON payload size.""" + + async def scenario() -> None: + archive = tmp_path / "large.pcb" + source_path = tmp_path / "source.db" + target_path = tmp_path / "target.db" + payload = "x" * 8_192 + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute( + insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=1_500) + ) + await connection.execute( + insert(SOURCES_TABLE), + [ + { + "scope_id": "project:one", + "source_type": "content", + "source_id": f"source-{number}", + "payload": _content_payload(f"source-{number}", payload), + "journal_position": number, + } + for number in range(1, 1_501) + ], + ) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + + tracemalloc.start() + try: + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}"), tables=BUILTIN_TABLES + ) as target: + receipt = await PortableBundleService(target.database, source_decoder=_SOURCES.decode_source).restore( + archive + ) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert receipt.inserted == 1_502 + assert peak < 10 * 1024 * 1024 + + asyncio.run(scenario()) + + +def test_validate_rejects_an_artifact_family_the_target_runtime_cannot_restore(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + source_path = tmp_path / "source.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{source_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute( + insert(ARTIFACTS_TABLE).values( + scope_id="project:one", + family="unavailable-family", + artifact_id="record-1", + revision=1, + content=b"payload", + ) + ) + await connection.execute( + insert(ARTIFACT_HEADS_TABLE).values( + scope_id="project:one", + family="unavailable-family", + artifact_id="record-1", + revision=1, + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + validation = await PortableBundleService( + target.database, + supported_artifact_families=("handoff",), + ).validate(archive) + assert validation.compatible is False + assert validation.unsupported_artifact_families == ("unavailable-family",) + + asyncio.run(scenario()) + + +def test_export_rejects_candidate_evidence_that_is_not_in_the_bundle(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + database = tmp_path / "source.db" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{database}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute( + insert(CANDIDATE_VERSIONS_TABLE).values( + scope_id="project:one", + candidate_id="candidate-1", + version=1, + family="handoff", + proposal=b"{}", + source_refs=b'[{"source_type":"content","source_id":"missing"}]', + artifact_refs=b"[]", + target_family=None, + target_artifact_id=None, + target_revision=None, + reason=None, + ) + ) + await connection.execute( + insert(CANDIDATE_HEADS_TABLE).values( + scope_id="project:one", + candidate_id="candidate-1", + family="handoff", + version=1, + status="pending", + ) + ) + archive.write_bytes(b"previous backup") + with pytest.raises(BundleFormatError, match="missing source"): + await PortableBundleService(source.database).export( + ["project:one"], archive, authorize=_authorize_export + ) + assert archive.read_bytes() == b"previous backup" + + asyncio.run(scenario()) + + +def test_bundle_preserves_registered_family_dependencies_and_tags(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "families.pcb" + now = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC) + package_path = tmp_path / "portable" + package_path.mkdir() + (package_path / "SKILL.md").write_text( + "---\nname: portable\ndescription: portable skill\n---\nRead guide.md.\n" + ) + (package_path / "guide.md").write_text("Verify the restored package.\n") + package = capture_skill_directory(package_path) + skill_content = package.as_skill_content().model_dump_json().encode() + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute( + insert(SKILL_PACKAGES_TABLE).values( + scope_id="project:one", + **package.reference.model_dump(), + archive_bytes=package.archive_bytes, + manifest=package.manifest_bytes, + created_at=now, + ) + ) + await connection.execute( + insert(ARTIFACTS_TABLE), + [ + { + "scope_id": "project:one", + "family": "skill", + "artifact_id": "portable", + "revision": 1, + "content": skill_content, + }, + { + "scope_id": "project:one", + "family": "topic-memory", + "artifact_id": "topic", + "revision": 1, + "content": b'{"title":"Topic","summary":"Summary","detail":"Detail"}', + }, + { + "scope_id": "project:one", + "family": "prompt", + "artifact_id": "prompt", + "revision": 1, + "content": b"{}", + }, + ], + ) + await connection.execute( + insert(ARTIFACT_HEADS_TABLE), + [ + {"scope_id": "project:one", "family": "skill", "artifact_id": "portable", "revision": 1}, + {"scope_id": "project:one", "family": "topic-memory", "artifact_id": "topic", "revision": 1}, + {"scope_id": "project:one", "family": "prompt", "artifact_id": "prompt", "revision": 1}, + ], + ) + await connection.execute( + insert(TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE).values( + scope_id="project:one", + family="topic-memory", + artifact_id="topic", + revision=1, + published_at=now.replace(tzinfo=None), + ) + ) + await connection.execute( + insert(PROFILE_POLICIES_TABLE).values( + scope_id="project:one", + generation_enabled=True, + activation_mode="automatic", + pending_candidate_id=None, + version=1, + updated_at=now, + ) + ) + await connection.execute( + insert(ARTIFACT_TAGS_TABLE).values( + scope_id="project:one", + family="skill", + artifact_id="portable", + target_type="artifact", + target_id="portable", + tag_key_hash=hashlib.sha256(b"platform").digest(), + tag_key="platform", + tag="platform", + assigned_at=now, + ) + ) + await PortableBundleService(source.database).export(["project:one"], archive, authorize=_authorize_export) + + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + service = PortableBundleService( + target.database, + supported_artifact_families=("skill", "topic-memory", "prompt", "profile"), + ) + validation = await service.validate(archive) + assert validation.compatible is True + assert validation.required_artifact_families == ("profile", "prompt", "skill", "topic-memory") + await service.restore(archive) + async with target.database.transaction() as connection: + assert await connection.scalar(select(SKILL_PACKAGES_TABLE.c.archive_bytes)) == package.archive_bytes + assert await connection.scalar(select(ARTIFACT_TAGS_TABLE.c.tag)) == "platform" + assert await connection.scalar(select(PROFILE_POLICIES_TABLE.c.activation_mode)) == "automatic" + assert ( + await connection.scalar(select(TOPIC_MEMORY_REVISION_PUBLICATIONS_TABLE.c.artifact_id)) == "topic" + ) + + asyncio.run(scenario()) + + +def test_export_rejects_package_backed_skill_without_its_package(tmp_path: Path) -> None: + async def scenario() -> None: + archive = tmp_path / "missing-package.pcb" + content = json.dumps({ + "name": "broken", + "description": "broken", + "instructions": "", + "validation": [], + "package": { + "tree_digest": "a" * 64, + "archive_digest": "b" * 64, + "file_count": 1, + "uncompressed_size": 1, + "archive_size": 1, + }, + "license": None, + "compatibility": None, + "metadata": {}, + "allowed_tools": None, + }).encode() + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute( + insert(ARTIFACTS_TABLE).values( + scope_id="project:one", family="skill", artifact_id="broken", revision=1, content=content + ) + ) + await connection.execute( + insert(ARTIFACT_HEADS_TABLE).values( + scope_id="project:one", family="skill", artifact_id="broken", revision=1 + ) + ) + archive.write_bytes(b"previous backup") + with pytest.raises(BundleFormatError, match="missing or mismatched package"): + await PortableBundleService(source.database).export( + ["project:one"], archive, authorize=_authorize_export + ) + assert archive.read_bytes() == b"previous backup" + + asyncio.run(scenario()) + + +def test_sqlite_export_uses_one_snapshot_during_concurrent_capture(tmp_path: Path) -> None: + async def scenario() -> None: + database_path = tmp_path / "concurrent.db" + archive = tmp_path / "snapshot.pcb" + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{database_path}"), tables=BUILTIN_TABLES + ) as source: + async with source.database.transaction() as connection: + await _insert_scope(connection) + await connection.execute(insert(SOURCE_JOURNAL_HEADS_TABLE).values(scope_id="project:one", position=0)) + committed = False + + def capture_during_export(progress): + nonlocal committed + if progress.phase != "export" or committed: + return + with sqlite3.connect(database_path) as writer: + writer.execute( + "UPDATE pc_source_journal_heads SET position = 1 WHERE scope_id = ?", ("project:one",) + ) + writer.execute( + "INSERT INTO pc_sources (scope_id, source_type, source_id, payload, journal_position) " + "VALUES (?, ?, ?, ?, ?)", + ("project:one", "content", "concurrent", b"{}", 1), + ) + committed = True + + await PortableBundleService(source.database).export( + ["project:one"], archive, authorize=_authorize_export, progress=capture_during_export + ) + assert committed + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as target: + await PortableBundleService(target.database).restore(archive) + async with target.database.transaction() as connection: + assert await connection.scalar(select(SOURCE_JOURNAL_HEADS_TABLE.c.position)) == 0 + assert await connection.scalar(select(SOURCES_TABLE.c.source_id)) is None + + asyncio.run(scenario()) diff --git a/tests/e2e/test_portable_archive.py b/tests/e2e/test_portable_archive.py new file mode 100644 index 0000000000..a4101aace3 --- /dev/null +++ b/tests/e2e/test_portable_archive.py @@ -0,0 +1,2483 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Portable archive acceptance through the public built-in Runtime.""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import importlib.util +import json +import os +import zipfile +from collections import Counter +from collections.abc import Iterator +from datetime import UTC, datetime +from pathlib import Path +from uuid import uuid4 + +import pytest +from pydantic import JsonValue, SecretStr +from sqlalchemy import create_engine, event, insert, select, update +from sqlalchemy.engine import make_url + +from powercontext.artifacts import ArtifactAddress, ArtifactRef, StableHandle +from powercontext.builtin.artifacts.atomic_memory import AtomicMemoryContent +from powercontext.builtin.artifacts.atomic_memory.errors import InvalidAtomicMemoryStateError +from powercontext.builtin.artifacts.experience import ExperienceContent +from powercontext.builtin.artifacts.memory import EmbeddingProfile +from powercontext.builtin.artifacts.profile.models import ProfileCandidateProposal, SourceWindow +from powercontext.builtin.artifacts.skill import capture_skill_directory, package_file +from powercontext.builtin.inference import EmbeddingResult +from powercontext.builtin.persistence.candidates import attach_candidate_audit, proposal_digest +from powercontext.builtin.persistence.migrations.atomic_memory_archive import ARCHIVE_TABLE +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from powercontext.builtin.persistence.seekdb import SeekDBConfig +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.tables import ( + ARTIFACTS_TABLE, + MEMORY_ENTRY_VERSIONS_TABLE, + SCOPES_TABLE, + SOURCE_JOURNAL_HEADS_TABLE, + SOURCES_TABLE, +) +from powercontext.builtin.portability import ( + BundleConflictError, + BundleFormatError, + BundleRecoveryRequiredError, + PortableBundleService, +) +from powercontext.builtin.publication import ArtifactPublicationRequest +from powercontext.builtin.records import ArtifactWrite +from powercontext.builtin.review.models import CandidateAudit, CandidateStatus +from powercontext.builtin.runtime import ( + BuiltinConfig, + CaptureSource, + CaptureWorkingNote, + HandoffDraft, + HandoffSourceCitation, + HandoffStatement, + SubmitSourceObservation, + WorkingNoteKind, + open_builtin_contexts, + open_builtin_runtime, +) +from powercontext.builtin.runtime.artifact_processing import ArtifactProcessingBinding +from powercontext.builtin.scope import ScopeDraft, ScopeExternalReference +from powercontext.builtin.sources import ContentCapture +from powercontext.builtin.tags import ArtifactTagTarget, TagFilter +from powercontext.builtin.work import AcknowledgeHandoff, ReceiverChecks +from powercontext.limits import MAX_ARTIFACT_ID_LENGTH +from powercontext.sources import ( + AdapterSourceDefinition, + SourceDefinitionRegistry, + SourceRef, + manifest_for_definition, + project_source_for_transport, +) +from tests.builtin.persistence.contract import NoteAdapter, NoteInput, NoteSource +from tests.e2e.atomic_memory_migration_fixture import SCOPE as MIGRATED_SCOPE +from tests.e2e.atomic_memory_migration_fixture import _seed_and_migrate + + +def test_topic_restore_can_retry_after_projection_failure_and_restart(tmp_path: Path) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def fail_projection(_scopes: tuple[str, ...]) -> None: + raise RuntimeError("interrupted projection rebuild") # noqa: TRY003 + + async def scenario() -> None: + archive = tmp_path / "topic.pcb" + target_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create(ScopeDraft(title="Recovery", summary="Retry", idempotency_key="retry")) + topic = await source.records.create_artifact( + scope.scope_id, + "topic-memory", + ArtifactWrite(content={"title": "Recovery", "summary": "Retry", "detail": "Retry after interruption."}), + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + async with open_builtin_contexts(target_config) as target: + with pytest.raises(RuntimeError, match="interrupted"): + await PortableBundleService( + target.database, + projection_rebuilder=fail_projection, + source_decoder=target.repositories.sources.decode_source, + ).restore(archive) + with pytest.raises(BundleRecoveryRequiredError): + async with open_builtin_contexts(target_config): + pass + async with open_builtin_contexts(target_config, _archive_recovery=True) as target: + assert (await target.portability.restore(archive)).projections_ready + async with open_builtin_contexts(target_config) as target: + restored = await target.records.get_artifact(scope.scope_id, "topic-memory", topic.artifact_id) + assert restored.artifact_id == topic.artifact_id + + asyncio.run(scenario()) + + +def test_sqlite_portable_archive_restores_revisions_lineage_and_handoff_receipts(tmp_path: Path) -> None: + async def scenario() -> None: + archive_path = tmp_path / "portable.pcb" + source_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'source.db'}")) + target_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + + async with open_builtin_runtime(source_config) as source_runtime: + assert source_runtime.scopes is not None + scope = await source_runtime.scopes.create( + ScopeDraft( + title="Portable archive", summary="Archive recovery fixture", idempotency_key="portable-archive" + ) + ) + scope_id = scope.scope_id + source = await source_runtime.sources.for_scope(scope_id).capture( + CaptureSource( + source_id="turn-1", + content="The archive must preserve the handoff citation.", + metadata={"origin": "portable-archive-e2e"}, + ) + ) + assert source_runtime.atomic_memory is not None + memory = await source_runtime.atomic_memory.for_scope(scope_id).create(( + AtomicMemoryContent(kind="decision", text="Use the portable archive for migration."), + )) + assert memory.records + handoffs = source_runtime.handoff.for_scope(scope_id) + prepared = await handoffs.finalize( + HandoffDraft( + objective="Restore an exact logical handoff.", + state=( + HandoffStatement( + text="The source citation is durable.", + citations=(HandoffSourceCitation(source_ref=source.source_ref),), + ), + ), + disposition="continuable", + next_action=HandoffStatement( + text="Open the restored handoff.", + citations=(HandoffSourceCitation(source_ref=source.source_ref),), + ), + ) + ) + committed = await handoffs.commit(prepared) + acknowledgement = await source_runtime.work.for_scope(scope_id).acknowledge( + AcknowledgeHandoff( + source_id="handoff-receipt-1", + receiver="portable-target", + status="accepted", + selection="exact", + revision=committed.as_ref(), + receiver_checks=ReceiverChecks( + live_state="confirmed", + capability="confirmed", + authorization="confirmed", + ), + ) + ) + assert source_runtime.archive is not None + + async def authorize(scopes: tuple[str, ...]) -> None: + assert scopes == (scope_id,) + + exported = await source_runtime.archive.export([scope_id], archive_path, authorize=authorize) + + async with open_builtin_runtime(target_config) as target_runtime: + assert target_runtime.archive is not None + restored = await target_runtime.archive.restore(archive_path) + latest = await target_runtime.handoff.for_scope(scope_id).latest() + continuity = await target_runtime.work.for_scope(scope_id).continuity() + + assert restored.record_count == exported.record_count + assert restored.projections_ready is True + assert latest == committed + assert latest is not None + assert latest.revision == 1 + assert latest.lineage.sources == (source.source_ref,) + assert continuity.coverage.acknowledgement_records == 1 + assert continuity.coverage.active_receipt_ref == acknowledgement.receipt.source_ref + + asyncio.run(scenario()) + + +def test_archive_recovers_canonical_default_scope_into_a_fresh_deployment( + tmp_path: Path, target_config: BuiltinConfig +) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "default.pcb" + async with open_builtin_runtime(BuiltinConfig()) as source: + assert source.scopes is not None + default = await source.scopes.default_scope() + assert default is not None + capture = CaptureSource( + source_id="default-evidence", content="Restore the original default scope.", metadata={} + ) + await source.sources.for_scope(default.scope_id).capture(capture) + expected = await source.records.for_scope(default.scope_id).get_source("content", capture.source_id) + assert source.archive is not None + exported = await source.archive.export([default.scope_id], archive, authorize=authorize) + async with open_builtin_runtime(target_config, _archive_recovery=True) as target: + assert target.scopes is not None + assert await target.scopes.default_scope() is None + assert target.archive is not None + assert (await target.archive.restore(archive)).projections_ready + assert (await target.archive.restore(archive)).inserted == 0 + async with open_builtin_runtime(target_config) as target: + assert target.scopes is not None + restored = await target.scopes.default_scope() + assert restored is not None + assert restored.scope_id == default.scope_id + assert ( + await target.records.for_scope(restored.scope_id).get_source("content", capture.source_id) == expected + ) + assert target.archive is not None + reexported = await target.archive.export([restored.scope_id], tmp_path / "again.pcb", authorize=authorize) + assert reexported.total_digest == exported.total_digest + + asyncio.run(scenario()) + + +def test_archive_preserves_revision_metadata_after_restart(tmp_path: Path, target_config: BuiltinConfig) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "revisions.pcb" + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Revision history", summary="Preserve audit evidence", idempotency_key="revisions") + ) + created = await source.records.create_artifact( + scope.scope_id, "profile", ArtifactWrite(content={"content": "# Profile\n\nKeep the original choice."}) + ) + original = await source.records.get_artifact(scope.scope_id, "profile", created.artifact_id) + await source.records.replace_artifact( + scope.scope_id, + "profile", + created.artifact_id, + '"revision:1"', + ArtifactWrite(content={"content": "# Profile\n\nUse a different choice."}), + ) + rollback = await source.records.replace_artifact( + scope.scope_id, + "profile", + created.artifact_id, + '"revision:2"', + ArtifactWrite( + content={"content": original.content["content"]}, + restored_from_revision=1, + reason="Restore the confirmed choice", + ), + actor_type="user", + actor_id="archive-operator", + ) + assert rollback.created_at is not None + assert rollback.created_by is not None and rollback.created_by.id == "archive-operator" + expected = await source.records.list_artifact_revisions( + scope.scope_id, "profile", created.artifact_id, limit=10, cursor=None + ) + exported = await source.portability.export([scope.scope_id], archive, authorize=authorize) + async with open_builtin_contexts(target_config) as legacy: + legacy_scope = await legacy.scopes.create( + ScopeDraft(title="Legacy metadata", summary="Unknown timestamp", idempotency_key="legacy-metadata") + ) + legacy_profile = await legacy.records.create_artifact( + legacy_scope.scope_id, + "profile", + ArtifactWrite(content={"content": "# Profile\n\nKeep unknown history."}), + ) + async with legacy.database.transaction() as connection: + await connection.execute( + update(ARTIFACTS_TABLE) + .where(ARTIFACTS_TABLE.c.scope_id == legacy_scope.scope_id) + .values(created_at=None) + ) + if connection.dialect.name == "mysql": + await connection.exec_driver_sql("ALTER TABLE pc_artifacts MODIFY COLUMN created_at DATETIME NULL") + async with open_builtin_contexts(target_config) as target: + assert ( + await target.records.get_artifact(legacy_scope.scope_id, "profile", legacy_profile.artifact_id) + ).created_at is None + assert (await target.portability.restore(archive)).projections_ready + assert (await target.portability.restore(archive)).inserted == 0 + async with open_builtin_contexts(target_config) as target: + assert await target.records.get_artifact(scope.scope_id, "profile", created.artifact_id) == rollback + assert ( + await target.records.list_artifact_revisions( + scope.scope_id, "profile", created.artifact_id, limit=10, cursor=None + ) + == expected + ) + reexported = await target.portability.export( + [scope.scope_id], tmp_path / "reexport.pcb", authorize=authorize + ) + assert reexported.total_digest == exported.total_digest + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + ("field", "value"), + [("created_at", None), ("created_by", '{"type":"user","id":"another-operator"}')], +) +def test_archive_detects_revision_metadata_conflicts(tmp_path: Path, field: str, value: object) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "metadata-conflict.pcb" + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Metadata conflict", summary="Preserve immutable history", idempotency_key="metadata") + ) + created = await source.records.create_artifact( + scope.scope_id, "profile", ArtifactWrite(content={"content": "# Profile\n\nKeep the recorded choice."}) + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + async with open_builtin_contexts(BuiltinConfig()) as target: + await target.portability.restore(archive) + async with target.database.transaction() as connection: + await connection.execute( + update(ARTIFACTS_TABLE).where(ARTIFACTS_TABLE.c.scope_id == scope.scope_id).values({field: value}) + ) + before = await target.records.get_artifact(scope.scope_id, "profile", created.artifact_id) + validation = await target.portability.validate(archive) + assert not validation.compatible and validation.conflicts == 1 + with pytest.raises(BundleConflictError): + await target.portability.restore(archive) + assert await target.records.get_artifact(scope.scope_id, "profile", created.artifact_id) == before + + asyncio.run(scenario()) + + +def test_archive_maintenance_runtime_does_not_start_background_processing() -> None: + class NeverLaunch: + async def start(self, assignment): + message = "Maintenance must not launch workers" + raise AssertionError(message) + + async def scenario() -> None: + binding = ArtifactProcessingBinding( + binding_name="archive-maintenance", artifact_family="skill", launcher=NeverLaunch() + ) + async with open_builtin_runtime( + BuiltinConfig(), _archive_recovery=True, artifact_processing_bindings=(binding,) + ) as runtime: + assert runtime.artifact_processing_supervisor is None + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("family", ["atomic-memory", "experience"]) +def test_interrupted_archive_restore_blocks_contexts_runtime_and_workers( + tmp_path: Path, family: str, target_config: BuiltinConfig +) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def interrupted(_scopes: tuple[str, ...]) -> None: + message = "interrupted projection rebuild" + raise RuntimeError(message) + + async def scenario() -> None: + archive = tmp_path / "interrupted.pcb" + content: dict[str, JsonValue] = ( + {"kind": "rule", "text": "Retain recovery evidence."} + if family == "atomic-memory" + else {"situation": "Recovery", "action": "Retry", "outcome": "Ready", "lesson": "Verify projections."} + ) + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Incomplete recovery", summary="Do not serve", idempotency_key="incomplete") + ) + artifact = await source.records.create_artifact(scope.scope_id, family, ArtifactWrite(content=content)) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + async with open_builtin_contexts(target_config) as target: + with pytest.raises(RuntimeError, match="interrupted projection rebuild"): + await PortableBundleService( + target.database, + projection_rebuilder=interrupted, + source_decoder=target.repositories.sources.decode_source, + ).restore(archive) + for opener in (open_builtin_contexts, open_builtin_runtime): + with pytest.raises(RuntimeError, match="portable restore projections are incomplete"): + async with opener(target_config): + pass + for recovery in (False, True): + with pytest.raises(BundleRecoveryRequiredError): + async with open_builtin_contexts(target_config, _topic_memory_worker=True, _archive_recovery=recovery): + pass + async with open_builtin_contexts(target_config, _archive_recovery=True) as target: + assert (await target.portability.restore(archive)).projections_ready + async with open_builtin_runtime(target_config) as target: + restored = await target.records.for_scope(scope.scope_id).get_artifact(family, artifact.artifact_id) + assert restored.artifact_id == artifact.artifact_id + if family == "atomic-memory": + assert target.atomic_memory is not None + found = await target.atomic_memory.for_scope(scope.scope_id).search("recovery evidence") + assert found.hits + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("legacy", [False, True]) +@pytest.mark.parametrize("damage", ["payload", "identity", "definition", "capability_override"]) +def test_archive_rejects_unreadable_native_sources_before_writing(tmp_path: Path, monkeypatch, legacy, damage) -> None: + from typer.testing import CliRunner + + from powercontext.cli.app import create_cli + from powercontext.cli.archive import archive_app + + async def scenario() -> Path: + archive = tmp_path / "native.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Native source", summary="Read contract", idempotency_key="native-source") + ) + context = await source.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="evidence", content="Original captured evidence.")) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + + def rewrite(records): + record = next(item for item in records if item["record_type"] == "source") + envelope = json.loads(base64.b64decode(record["payload"]["payload"]["base64"])) + value = dict(envelope["value"]) + if damage in {"payload", "capability_override"}: + value = {} + elif damage == "identity": + value["name"] = "different-source-id" + else: + value["definition_version"] = "unsupported" + payload = value if legacy else {**envelope, "value": value} + record["payload"]["payload"] = {"base64": base64.b64encode(json.dumps(payload).encode()).decode()} + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + service = target.portability + supported_source_types = None + if damage == "capability_override": + service = PortableBundleService( + target.database, + supported_source_types=(), + source_decoder=target.repositories.sources.decode_source, + ) + supported_source_types = ("content",) + async with target.database.transaction() as connection: + before = (await connection.execute(select(SCOPES_TABLE))).all() + for operation in (service.validate, service.restore): + with pytest.raises(BundleFormatError, match="Source"): + await operation(archive, supported_source_types=supported_source_types) + async with target.database.transaction() as connection: + assert (await connection.execute(select(SCOPES_TABLE))).all() == before + assert (await connection.execute(select(SOURCES_TABLE))).all() == [] + return archive + + archive = asyncio.run(scenario()) + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "unused-target")) + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "restore", str(archive), "--dry-run"]) + assert result.exit_code == 2, result.output + assert "Source" in result.output + assert not (tmp_path / "unused-target" / "powercontext.db").exists() + + +def test_archive_restores_tagged_memory_vector_search_and_survives_restart( + tmp_path: Path, target_config: BuiltinConfig +) -> None: + class Embedding: + profile = EmbeddingProfile(profile_id="archive-test", model="archive-test", dimension=3) + + async def embed(self, texts: tuple[str, ...], /) -> EmbeddingResult: + return EmbeddingResult(vectors=tuple((1.0, 0.0, 0.0) for _ in texts)) + + async def scenario() -> None: + archive = tmp_path / "tagged.pcb" + source_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'source.db'}")) + async with open_builtin_contexts(source_config, embedding_model=Embedding()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Archive", summary="Vector recovery", idempotency_key="archive-vector") + ) + memory = await source.records.create_artifact( + scope.scope_id, + "atomic-memory", + ArtifactWrite(content={"kind": "fact", "text": "Portable vector recovery."}), + ) + tag_target = ArtifactTagTarget(family="atomic-memory", artifact_id=memory.artifact_id) + empty = await source.records.get_tags(scope.scope_id, tag_target) + tagged = await source.records.replace_tags( + scope.scope_id, tag_target, ("ARCHIVE", "Straße", "Café", "中文"), expected_etag=empty.etag + ) + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + await source.portability.export([scope.scope_id], archive, authorize=authorize) + async with open_builtin_contexts(target_config, embedding_model=Embedding()) as target: + receipt = await target.portability.restore(archive) + assert receipt.projections_ready + assert await target.records.get_tags(scope.scope_id, tag_target) == tagged + async with open_builtin_contexts(target_config, embedding_model=Embedding()) as target: + service = target.atomic_memory.for_scope(scope.scope_id) + for mode in ("text", "vector", "hybrid"): + result = await service.search( + "portable", mode=mode, tag_filter=TagFilter(tags=("archive", "STRASSE", "Café", "中文")) + ) + assert [hit.hit.artifact_ref.artifact_id for hit in result.hits] == [memory.artifact_id] + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + ("field", "invalid"), + [ + ("tag", "Different"), + ("tag", ""), + ("tag", " archive"), + ("tag", "archive "), + ("tag", "a\x00b"), + ("tag", "\u0378"), + ("tag_key", "ARCHIVE"), + ("tag_key", "different"), + ], +) +def test_archive_rejects_inconsistent_tag_labels_before_writing(tmp_path: Path, field: str, invalid: str) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "tags.pcb" + config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Tags", summary="Preserve searchable labels", idempotency_key="tag-labels") + ) + memory = await source.records.create_artifact( + scope.scope_id, "atomic-memory", ArtifactWrite(content={"kind": "fact", "text": "Portable tags."}) + ) + tag_target = ArtifactTagTarget(family="atomic-memory", artifact_id=memory.artifact_id) + empty = await source.records.get_tags(scope.scope_id, tag_target) + tagged = await source.records.replace_tags( + scope.scope_id, tag_target, ("archive",), expected_etag=empty.etag + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + original = archive.read_bytes() + + def rewrite(records): + record = next(record for record in records if record["record_type"] == "artifact_tag") + if field == "tag": + record["payload"]["tag"] = invalid + else: + record["identity"]["tag_key"] = invalid + record["payload"]["tag_key_hash"] = { + "base64": base64.b64encode(hashlib.sha256(invalid.encode()).digest()).decode() + } + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(config) as target: + before = await target.scopes.list() + for operation in (target.portability.inspect, target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="artifact tag"): + await operation(archive) + assert await target.scopes.list() == before + archive.write_bytes(original) + assert (await target.portability.validate(archive)).compatible + assert (await target.portability.restore(archive)).projections_ready + assert await target.records.get_tags(scope.scope_id, tag_target) == tagged + async with open_builtin_contexts(config) as target: + assert (await target.portability.restore(archive)).inserted == 0 + assert await target.records.get_tags(scope.scope_id, tag_target) == tagged + hits = await target.atomic_memory.for_scope(scope.scope_id).search( + "portable", tag_filter=TagFilter(tags=tagged.tags) + ) + assert [hit.hit.artifact_ref.artifact_id for hit in hits.hits] == [memory.artifact_id] + + asyncio.run(scenario()) + + +def test_archive_rejects_unreadable_tag_sets_before_writing(tmp_path: Path) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "tags.pcb" + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Tags", summary="Complete tag set", idempotency_key="tags") + ) + memory = await source.records.create_artifact( + scope.scope_id, "atomic-memory", ArtifactWrite(content={"kind": "fact", "text": "Portable tag set."}) + ) + tag_target = ArtifactTagTarget(family="atomic-memory", artifact_id=memory.artifact_id) + empty = await source.records.get_tags(scope.scope_id, tag_target) + tagged = await source.records.replace_tags( + scope.scope_id, tag_target, tuple(f"tag-{number}" for number in range(32)), expected_etag=empty.etag + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + original = archive.read_bytes() + + def rewrite(records): + tag = next(record for record in records if record["record_type"] == "artifact_tag") + records.append({ + **tag, + "identity": {**tag["identity"], "tag_key": "tag-32"}, + "payload": { + **tag["payload"], + "tag": "tag-32", + "tag_key_hash": {"base64": base64.b64encode(hashlib.sha256(b"tag-32").digest()).decode()}, + }, + }) + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + before = await target.scopes.list() + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="tag set"): + await operation(archive) + assert await target.scopes.list() == before + archive.write_bytes(original) + assert (await target.portability.restore(archive)).projections_ready + assert await target.records.get_tags(scope.scope_id, tag_target) == tagged + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("incoming_count", [16, 17]) +def test_archive_validates_combined_tag_sets_before_writing( + tmp_path: Path, incoming_count: int, target_config: BuiltinConfig +) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + first, second = tmp_path / "first.pcb", tmp_path / "second.pcb" + previous_labels = tuple(f"left-{number}" for number in range(16)) + incoming_labels = tuple(f"right-{number}" for number in range(incoming_count)) + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Tags", summary="Combined tag set", idempotency_key="tags") + ) + memory = await source.records.create_artifact( + scope.scope_id, + "atomic-memory", + ArtifactWrite(content={"kind": "fact", "text": "Portable combined tags."}), + ) + tag_target = ArtifactTagTarget(family="atomic-memory", artifact_id=memory.artifact_id) + empty = await source.records.get_tags(scope.scope_id, tag_target) + previous = await source.records.replace_tags( + scope.scope_id, tag_target, previous_labels, expected_etag=empty.etag + ) + await source.portability.export([scope.scope_id], first, authorize=authorize) + await source.records.replace_tags(scope.scope_id, tag_target, incoming_labels, expected_etag=previous.etag) + await source.portability.export([scope.scope_id], second, authorize=authorize) + async with open_builtin_contexts(target_config) as target: + await target.portability.restore(first) + if incoming_count == 17: + assert not (await target.portability.validate(second)).compatible + with pytest.raises(BundleConflictError): + await target.portability.restore(second) + expected = set(previous_labels) + else: + assert (await target.portability.validate(second)).compatible + assert (await target.portability.restore(second)).projections_ready + assert (await target.portability.restore(second)).inserted == 0 + expected = set(previous_labels + incoming_labels) + assert set((await target.records.get_tags(scope.scope_id, tag_target)).tags) == expected + async with open_builtin_contexts(target_config) as target: + assert set((await target.records.get_tags(scope.scope_id, tag_target)).tags) == expected + hits = await target.atomic_memory.for_scope(scope.scope_id).search( + "portable", tag_filter=TagFilter(tags=("right-0",)) + ) + assert [hit.hit.artifact_ref.artifact_id for hit in hits.hits] == ( + [memory.artifact_id] if incoming_count == 16 else [] + ) + + asyncio.run(scenario()) + + +def test_archive_restores_package_files_and_topic_search_after_restart( + tmp_path: Path, target_config: BuiltinConfig +) -> None: + async def scenario() -> None: + package_dir = tmp_path / "portable" + package_dir.mkdir() + (package_dir / "SKILL.md").write_text( + "---\nname: portable\ndescription: Portable guide\n---\nRead guide.md.\n", encoding="utf-8", newline="\n" + ) + (package_dir / "guide.md").write_text("Exact portable instructions.\n", encoding="utf-8", newline="\n") + package = capture_skill_directory(package_dir) + archive = tmp_path / "families.pcb" + source_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'source.db'}")) + async with open_builtin_contexts(source_config) as source: + scope = await source.scopes.create( + ScopeDraft(title="Families", summary="Portable families", idempotency_key="families") + ) + await source.upload_skill_package(scope.scope_id, package.archive_bytes, None, None) + skill = await source.records.create_artifact( + scope.scope_id, "skill", ArtifactWrite(content=package.as_skill_content().model_dump(mode="json")) + ) + experience = await source.records.create_artifact( + scope.scope_id, + "experience", + ArtifactWrite( + content={ + "situation": "Portable archive recovery", + "action": "Restore the archive", + "outcome": "Artifacts remain searchable", + "lesson": "Verify portable search after restart", + } + ), + ) + assert [ + hit.artifact_ref.artifact_id for hit in await source.search_skills(scope.scope_id, "portable", 5) + ] == [skill.artifact_id] + assert [ + hit.artifact_ref.artifact_id for hit in await source.search_experience(scope.scope_id, "portable", 5) + ] == [experience.artifact_id] + topic = await source.records.create_artifact( + scope.scope_id, + "topic-memory", + ArtifactWrite( + content={ + "title": "Portable archives", + "summary": "Recovery", + "detail": "Portable archive recovery details.", + } + ), + ) + profile = await source.records.create_artifact( + scope.scope_id, "profile", ArtifactWrite(content={"content": "# Profile\nUse portable archives.\n"}) + ) + prompt = await source.records.create_artifact( + scope.scope_id, + "prompt", + ArtifactWrite( + prompt_key="memory.extract", + content={ + "schema_version": "powercontext.prompt.v1", + "mode": "auto", + "instructions": "", + "demonstrations": [], + }, + ), + ) + expected_profile = await source.records.get_artifact(scope.scope_id, "profile", profile.artifact_id) + expected_prompt = await source.records.get_artifact(scope.scope_id, "prompt", prompt.artifact_id) + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + exported = await source.portability.export([scope.scope_id], archive, authorize=authorize) + if target_config.database.kind != "sqlite": + async with ( + open_builtin_contexts(target_config) as legacy, + legacy.database.transaction() as connection, + ): + await connection.exec_driver_sql( + "ALTER TABLE pc_topic_memory_revision_publications MODIFY COLUMN published_at DATETIME NOT NULL" + ) + await connection.exec_driver_sql( + "ALTER TABLE pc_profile_policies MODIFY COLUMN updated_at DATETIME NOT NULL" + ) + async with open_builtin_contexts(target_config) as target: + receipt = await target.portability.restore(archive) + assert receipt.projections_ready + assert [ + hit.artifact_ref.artifact_id for hit in await target.search_skills(scope.scope_id, "portable", 5) + ] == [skill.artifact_id] + assert [ + hit.artifact_ref.artifact_id for hit in await target.search_experience(scope.scope_id, "portable", 5) + ] == [experience.artifact_id] + repeated = await target.portability.restore(archive) + assert repeated.inserted == 0 + async with open_builtin_contexts(target_config) as target: + assert [ + hit.artifact_ref.artifact_id for hit in await target.search_skills(scope.scope_id, "portable", 5) + ] == [skill.artifact_id] + assert [ + hit.artifact_ref.artifact_id for hit in await target.search_experience(scope.scope_id, "portable", 5) + ] == [experience.artifact_id] + restored = await target.skill_package( + scope.scope_id, ArtifactRef(family="skill", artifact_id=skill.artifact_id, revision=skill.revision) + ) + assert restored.archive_bytes == package.archive_bytes + assert package_file(restored, "guide.md") == b"Exact portable instructions.\n" + assert await target.records.get_artifact(scope.scope_id, "profile", profile.artifact_id) == expected_profile + assert await target.records.get_artifact(scope.scope_id, "prompt", prompt.artifact_id) == expected_prompt + async with target.database.transaction() as connection: + found = await target.repositories.topic_memories.search(connection, scope.scope_id, "portable", limit=5) + assert [hit.artifact_ref.artifact_id for hit in found.hits] == [topic.artifact_id] + roundtrip = tmp_path / "roundtrip.pcb" + reexported = await target.portability.export([scope.scope_id], roundtrip, authorize=authorize) + assert reexported.total_digest == exported.total_digest + reverse_config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'reverse.db'}")) + async with open_builtin_contexts(reverse_config) as reverse: + assert (await reverse.portability.restore(roundtrip)).projections_ready + assert ( + await reverse.records.get_artifact(scope.scope_id, "profile", profile.artifact_id) == expected_profile + ) + + asyncio.run(scenario()) + + +async def _publication_archive(tmp_path: Path): + archive = tmp_path / "publications.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scopes = tuple([ + await source.scopes.create(ScopeDraft(title=title, summary="Publication recovery", idempotency_key=title)) + for title in ("Origin", "Relay", "Delivery") + ]) + created = await source.records.create_artifact( + scopes[0].scope_id, + "experience", + ArtifactWrite( + content={"situation": "Recovery", "action": "Publish", "outcome": "Ready", "lesson": "Keep lineage"} + ), + ) + addresses = [ + ArtifactAddress( + scope_id=created.scope_id, + artifact=ArtifactRef(family=created.family, artifact_id=created.artifact_id, revision=created.revision), + ) + ] + for scope in (scopes[1], scopes[2], scopes[0]): + published = await source.publications.publish( + ArtifactPublicationRequest( + source=addresses[-1], target_scope_id=scope.scope_id, idempotency_key="portable-publication" + ) + ) + addresses.append(published.target) + independent = await source.records.create_artifact( + scopes[1].scope_id, + "experience", + ArtifactWrite( + content={"situation": "Separate", "action": "Keep", "outcome": "Root", "lesson": "No publication"} + ), + ) + addresses.append( + ArtifactAddress( + scope_id=independent.scope_id, + artifact=ArtifactRef( + family=independent.family, artifact_id=independent.artifact_id, revision=independent.revision + ), + ) + ) + expected = tuple([ + await source.records.get_artifact(address.scope_id, address.artifact.family, address.artifact.artifact_id) + for address in addresses + ]) + exported = await source.portability.export([scope.scope_id for scope in scopes], archive, authorize=authorize) + return archive, tuple(addresses), expected, exported + + +def test_archive_preserves_publication_chains_after_restart(tmp_path: Path, target_config: BuiltinConfig) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive, addresses, expected, exported = await _publication_archive(tmp_path) + async with open_builtin_contexts(target_config) as target: + assert (await target.portability.validate(archive)).compatible + assert (await target.portability.restore(archive)).projections_ready + for address, record in zip(addresses, expected, strict=True): + assert ( + await target.records.get_artifact( + address.scope_id, address.artifact.family, address.artifact.artifact_id + ) + == record + ) + async with open_builtin_contexts(target_config) as target: + assert (await target.portability.restore(archive)).inserted == 0 + for address, record in zip(addresses, expected, strict=True): + assert ( + await target.records.get_artifact( + address.scope_id, address.artifact.family, address.artifact.artifact_id + ) + == record + ) + roundtrip = await target.portability.export( + sorted({address.scope_id for address in addresses}), + tmp_path / "publications-roundtrip.pcb", + authorize=authorize, + ) + assert roundtrip.total_digest == exported.total_digest + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("damage", ["cycle", "self_cycle", "digest"]) +def test_archive_rejects_invalid_publication_chains_before_writing(tmp_path: Path, damage: str) -> None: + async def scenario() -> None: + archive, addresses, expected, _exported = await _publication_archive(tmp_path) + original = archive.read_bytes() + + def rewrite(records): + publication = next( + record + for record in records + if record["record_type"] == "artifact_publication" + and record["identity"]["target_scope_id"] == addresses[1].scope_id + ) + if damage == "digest": + publication["payload"]["content_digest"] = "0" * 64 + else: + source = addresses[1] if damage == "self_cycle" else addresses[2] + publication["payload"].update( + source_scope_id=source.scope_id, + source_family=source.artifact.family, + source_artifact_id=source.artifact.artifact_id, + source_revision=source.artifact.revision, + ) + return records + + _rewrite_archive_records(archive, rewrite) + config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + async with open_builtin_contexts(config) as target: + before = await target.scopes.list() + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="publication"): + await operation(archive) + assert await target.scopes.list() == before + archive.write_bytes(original) + assert (await target.portability.restore(archive)).projections_ready + async with open_builtin_contexts(config) as target: + assert (await target.portability.restore(archive)).inserted == 0 + for address, record in zip(addresses, expected, strict=True): + assert ( + await target.records.get_artifact( + address.scope_id, address.artifact.family, address.artifact.artifact_id + ) + == record + ) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("target_history", ["cycle", "different_origin"]) +def test_archive_detects_publication_conflicts_with_existing_target(tmp_path: Path, target_history: str) -> None: + async def scenario() -> None: + archive, addresses, _expected, _exported = await _publication_archive(tmp_path) + existing = tmp_path / "existing.pcb" + existing.write_bytes(archive.read_bytes()) + + def rewrite(records): + publication = next( + record + for record in records + if record["record_type"] == "artifact_publication" + and record["identity"]["target_scope_id"] == addresses[1].scope_id + ) + publication["identity"].update( + target_scope_id=addresses[0].scope_id, + target_family=addresses[0].artifact.family, + target_artifact_id=addresses[0].artifact.artifact_id, + target_revision=addresses[0].artifact.revision, + ) + source = addresses[1] if target_history == "cycle" else addresses[-1] + publication["payload"].update( + source_scope_id=source.scope_id, + source_family=source.artifact.family, + source_artifact_id=source.artifact.artifact_id, + source_revision=source.artifact.revision, + idempotency_key="target-existing-publication", + ) + return records + + _rewrite_archive_records(existing, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + assert (await target.portability.restore(existing)).projections_ready + before = tuple([ + await target.records.get_artifact( + address.scope_id, address.artifact.family, address.artifact.artifact_id + ) + for address in addresses + ]) + validation = await target.portability.validate(archive) + assert not validation.compatible + with pytest.raises(BundleConflictError): + await target.portability.restore(archive) + for address, record in zip(addresses, before, strict=True): + assert ( + await target.records.get_artifact( + address.scope_id, address.artifact.family, address.artifact.artifact_id + ) + == record + ) + + asyncio.run(scenario()) + + +@pytest.fixture(params=["sqlite", "seekdb", "server"]) +def target_config(tmp_path: Path, short_tmp_path: Path, request: pytest.FixtureRequest) -> Iterator[BuiltinConfig]: + if request.param == "seekdb": + if importlib.util.find_spec("pylibseekdb") is None: + pytest.skip("install powercontext[seekdb] for the embedded database") + yield BuiltinConfig(database=SeekDBConfig(path=short_tmp_path / "seekdb")) + elif request.param == "server": + live_url = os.environ.get("POWERCONTEXT_TEST_OCEANBASE_URL") + if not live_url: + pytest.skip("set POWERCONTEXT_TEST_OCEANBASE_URL for an isolated server database") + url = make_url(live_url) + name = "pc_archive_" + uuid4().hex + engine = create_engine(url.set(drivername="mysql+pymysql"), hide_parameters=True) + try: + with engine.begin() as connection: + connection.exec_driver_sql(f"CREATE DATABASE `{name}`") + try: + yield BuiltinConfig( + database=OceanBaseConfig( + url=SecretStr(url.set(database=name).render_as_string(hide_password=False)) + ) + ) + finally: + with engine.begin() as connection: + connection.exec_driver_sql(f"DROP DATABASE `{name}`") + finally: + engine.dispose() + else: + yield BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + + +@pytest.mark.parametrize("legacy", [False, True]) +@pytest.mark.parametrize("target_definition", ["absent", "identical", "conflicting"]) +def test_archive_preserves_remote_observations_without_worker_adapter( + tmp_path: Path, legacy: bool, target_definition: str, target_config: BuiltinConfig +) -> None: + + async def scenario() -> None: + definition = AdapterSourceDefinition(NoteAdapter()) + registry = SourceDefinitionRegistry((definition,)) + manifest = manifest_for_definition(definition) + observation = project_source_for_transport( + registry, await registry.resolve(NoteInput(note_id="one", body="remote")) + ) + archive = tmp_path / "remote.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Remote", summary="Remote sources", idempotency_key="remote") + ) + await source.register_source_definition(manifest) + + class UnrelatedAdapter(NoteAdapter): + name = "unrelated-note" + + await source.register_source_definition( + manifest_for_definition(AdapterSourceDefinition(UnrelatedAdapter())) + ) + await source.submit_source_observation( + SubmitSourceObservation(scope_id=scope.scope_id, observation=observation) + ) + if legacy: + async with source.database.transaction() as connection: + await connection.execute( + update(SOURCES_TABLE).values(payload=observation.model_dump_json().encode()) + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + assert (await source.portability.inspect(archive)).records_by_type["source_definition"] == 1 + async with open_builtin_contexts(target_config) as target: + if target_definition == "identical": + await target.register_source_definition(manifest) + elif target_definition == "conflicting": + + class DifferentNoteSource(NoteSource): + category: str = "changed" + + class DifferentNoteAdapter(NoteAdapter): + source_class = DifferentNoteSource + + await target.register_source_definition( + manifest_for_definition(AdapterSourceDefinition(DifferentNoteAdapter())) + ) + validation = await target.portability.validate(archive) + if target_definition == "conflicting": + assert not validation.compatible + assert validation.conflicts == 1 + with pytest.raises(BundleConflictError): + await target.portability.restore(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar( + select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == scope.scope_id) + ) + is None + ) + return + assert validation.compatible + assert validation.unsupported_source_types == () + assert (await target.portability.restore(archive)).projections_ready + async with open_builtin_contexts(target_config) as target: + async with target.database.transaction() as connection: + stored = await target.repositories.sources.get( + connection, + scope.scope_id, + SourceRef(source_type=observation.source_type, source_id=observation.name), + ) + assert stored.value == observation + assert ( + await target.repositories.source_definitions.get(connection, manifest.name, manifest.version) + == manifest + ) + # Continuing ingestion must also work without registering the manifest again. + assert ( + await target.submit_source_observation( + SubmitSourceObservation(scope_id=scope.scope_id, observation=observation) + ) + ).sequence == 1 + roundtrip = tmp_path / "remote-roundtrip.pcb" + await target.portability.export([scope.scope_id], roundtrip, authorize=authorize) + async with open_builtin_contexts(BuiltinConfig()) as reverse: + assert (await reverse.portability.restore(roundtrip)).projections_ready + async with reverse.database.transaction() as connection: + restored = await reverse.repositories.sources.get( + connection, + scope.scope_id, + SourceRef(source_type=observation.source_type, source_id=observation.name), + ) + assert restored.value == observation + + asyncio.run(scenario()) + + +def test_archive_snapshot_survives_concurrent_mysql_capture(tmp_path: Path, target_config: BuiltinConfig) -> None: + + if target_config.database.kind == "sqlite": + pytest.skip("SQLite snapshot has a separate concurrent-writer regression") + + async def scenario() -> None: + async with open_builtin_contexts(target_config) as source: + scope = await source.scopes.create( + ScopeDraft(title="Snapshot", summary="Concurrent writes", idempotency_key="snapshot") + ) + context = await source.get(scope.scope_id) + + await context.sources.capture(ContentCapture(source_id="before", content="before")) + async with source.database.transaction() as connection: + await connection.exec_driver_sql("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED") + captured = False + + async def write_concurrently(_driver): + # A separate task owns a separate transaction/connection. + await asyncio.create_task(context.sources.capture(ContentCapture(source_id="after", content="after"))) + + def after_read(connection, _cursor, statement, _parameters, _context, _many): + nonlocal captured + if not captured and statement.startswith("SELECT pc_source_journal_heads."): + captured = True + connection.connection.dbapi_connection.run_async(write_concurrently) + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + event.listen(source.database.engine.sync_engine, "after_cursor_execute", after_read) + try: + archive = tmp_path / "snapshot.pcb" + await source.portability.export([scope.scope_id], archive, authorize=authorize) + finally: + event.remove(source.database.engine.sync_engine, "after_cursor_execute", after_read) + assert captured + async with open_builtin_contexts(BuiltinConfig()) as target: + assert (await target.portability.validate(archive)).compatible + await target.portability.restore(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar( + select(SOURCE_JOURNAL_HEADS_TABLE.c.position).where( + SOURCE_JOURNAL_HEADS_TABLE.c.scope_id == scope.scope_id + ) + ) + == 1 + ) + assert ( + await connection.execute( + select(SOURCES_TABLE.c.source_id).where(SOURCES_TABLE.c.scope_id == scope.scope_id) + ) + ).scalars().all() == ["before"] + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + "damage", + [ + "missing_definition", + "fingerprint", + "identity", + "native", + "envelope", + "observation_payload", + "undeclared_projection", + "nonstandard_json", + ], +) +def test_restore_rejects_invalid_remote_source_dependencies_before_writing(tmp_path: Path, damage: str) -> None: + + async def scenario() -> None: + definition = AdapterSourceDefinition(NoteAdapter()) + registry = SourceDefinitionRegistry((definition,)) + observation = project_source_for_transport( + registry, await registry.resolve(NoteInput(note_id="one", body="remote")) + ) + archive = tmp_path / "invalid.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Remote", summary="Dependencies", idempotency_key="remote") + ) + await source.register_source_definition(manifest_for_definition(definition)) + await source.submit_source_observation( + SubmitSourceObservation(scope_id=scope.scope_id, observation=observation) + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + with zipfile.ZipFile(archive) as bundle: + manifest = json.loads(bundle.read("manifest.json")) + records = [json.loads(line) for line in bundle.read("records.ndjson").splitlines()] + if damage == "missing_definition": + records = [record for record in records if record["record_type"] != "source_definition"] + else: + record = next(record for record in records if record["record_type"] == "source") + envelope = json.loads(base64.b64decode(record["payload"]["payload"]["base64"])) + if damage == "fingerprint": + envelope["value"]["definition_fingerprint"] = "sha256:" + "0" * 64 + elif damage == "identity": + record["identity"]["source_id"] = "another" + elif damage == "envelope": + envelope["unexpected"] = "not part of the stored Source envelope" + elif damage in {"observation_payload", "undeclared_projection", "nonstandard_json"}: + missing_body = dict(envelope["value"]["payload"]) + missing_body.pop("body") + changes = { + "observation_payload": {"payload": missing_body}, + "undeclared_projection": { + "projections": [{"key": {"name": "undeclared", "version": "1"}, "value": {}}] + }, + "nonstandard_json": {"payload": {**envelope["value"]["payload"], "nonstandard": float("nan")}}, + } + envelope["value"].update(changes[damage]) + else: + envelope["representation"] = "native" + envelope["value"] = envelope["value"]["payload"] + record["payload"]["payload"]["base64"] = base64.b64encode(json.dumps(envelope).encode()).decode() + for record in records: + record.pop("digest") + record["digest"] = ( + "sha256:" + + hashlib.sha256( + json.dumps(record, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + ) + manifest["record_count"] = len(records) + manifest["records_by_type"] = dict(Counter(record["record_type"] for record in records)) + manifest["total_digest"] = ( + "sha256:" + hashlib.sha256("\n".join(record["digest"] for record in records).encode()).hexdigest() + ) + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("manifest.json", json.dumps(manifest)) + bundle.writestr("records.ndjson", "\n".join(json.dumps(record) for record in records) + "\n") + async with open_builtin_contexts(BuiltinConfig()) as target: + if damage == "native": + validation = await target.portability.validate(archive) + assert not validation.compatible + assert validation.unsupported_source_types == ("note",) + else: + with pytest.raises(BundleFormatError): + await target.portability.validate(archive) + with pytest.raises(BundleFormatError): + await target.portability.restore(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar( + select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == scope.scope_id) + ) + is None + ) + + asyncio.run(scenario()) + + +async def _replay_recurrence(contexts, scope_id): + from powercontext.builtin.evidence.resolver import EvidenceResolver + from powercontext.builtin.runtime.recurrence import RelationalRecurrenceLedger + + repositories = contexts.repositories + ledger = RelationalRecurrenceLedger( + database=contexts.database, + scope_id=scope_id, + sources=repositories.sources, + artifacts=repositories.artifacts, + recurrence=repositories.recurrence, + evidence=EvidenceResolver( + scope_id=scope_id, + sources=repositories.sources, + artifacts=repositories.artifacts, + ), + ) + async with contexts.database.transaction() as connection: + position = await repositories.sources.journal_position(connection, scope_id) + rows = await repositories.sources.list_window(connection, scope_id, after=0, through=position) + await ledger.record_window(connection, rows) + return ( + await repositories.recurrence.matches(connection, scope_id), + await repositories.recurrence.observations(connection, scope_id), + ) + + +async def _recurrence_archive(tmp_path): + from powercontext.builtin.artifacts.handoff.models import HandoffArtifactCitation + from powercontext.builtin.work.models import HandoffReceipt, TaskCheck, TaskOutcome, WorkClaim + + archive = tmp_path / "recurrence.pcb" + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Recurrence", summary="Frozen decisions", idempotency_key="recurrence") + ) + sid = scope.scope_id + content: dict[str, JsonValue] = { + "situation": "Contract changed", + "action": "Regenerate client", + "outcome": "Client agrees", + "lesson": "Regenerate after changing the contract", + "failure": { + "signature": {"recall_cue": "client is stale", "symptom": "client diverges"}, + "repair_surface": "experience_content", + "verification": {"condition": "contract changed", "check_subject": "client agrees"}, + }, + } + experience = await source.records.create_artifact(sid, "experience", ArtifactWrite(content=content)) + ref = ArtifactRef(family="experience", artifact_id=experience.artifact_id, revision=1) + citation = HandoffArtifactCitation(artifact_ref=ref) + handoff = await source.records.create_artifact( + sid, + "handoff", + ArtifactWrite( + content={ + "objective": "Regenerate client", + "disposition": "continuable", + "state": [{"text": "Use this experience", "citations": [citation.model_dump(mode="json")]}], + } + ), + ) + context = await source.get(sid) + receipt = HandoffReceipt( + receiver="archive-test", + status="accepted", + selection="exact", + evidence_status="available", + selected_revision=ArtifactRef(family="handoff", artifact_id=handoff.artifact_id, revision=1), + ) + await context.sources.capture( + ContentCapture( + source_id="receipt", + content=receipt.model_dump_json(), + metadata={"kind": "handoff-receipt"}, + ) + ) + receipt_ref = SourceRef(source_type="content", source_id="receipt") + for name, linked in (("linked-failure", True), ("unlinked-failure", False)): + outcome = TaskOutcome( + objective="Regenerate client", + summary="Failed", + status="failed", + observations=(WorkClaim(text="Client regeneration failed", basis="declared"),), + handoff_receipt_ref=receipt_ref if linked else None, + checks=(TaskCheck(name="client is stale", status="failed", basis="verified", evidence=(citation,)),), + ) + await context.sources.capture( + ContentCapture( + source_id=name, + content=outcome.model_dump_json(), + metadata={"kind": "task-outcome"}, + ) + ) + success = TaskOutcome( + objective="Regenerate client", + summary="Passed", + status="succeeded", + handoff_receipt_ref=receipt_ref, + observations=(WorkClaim(text="contract changed", basis="verified", evidence=(citation,)),), + checks=(TaskCheck(name="client agrees", status="passed", basis="verified", evidence=(citation,)),), + ) + await context.sources.capture( + ContentCapture( + source_id="success", + content=success.model_dump_json(), + metadata={"kind": "task-outcome"}, + ) + ) + expected = await _replay_recurrence(source, sid) + assert len(expected[0]) == 2 + assert len(expected[1]) == 5 + assert {event.event for event in expected[1]} == {"selected", "recurred", "avoided"} + assert all(match.artifact_ref == ref for match in expected[0]) + # Remove the signature from the current head. Recomputing an unlinked + # failure against this head would lose the original revision-1 match. + content["failure"] = None + await source.records.replace_artifact( + sid, + "experience", + experience.artifact_id, + '"revision:1"', + ArtifactWrite(content=content), + ) + assert await _replay_recurrence(source, sid) == expected + + async def authorize(_scopes): + pass + + await source.portability.export([sid], archive, authorize=authorize) + return archive, sid, experience.artifact_id, expected + + +def test_archive_preserves_frozen_recurrence_after_restart(tmp_path: Path, target_config: BuiltinConfig) -> None: + async def scenario(): + archive, sid, artifact_id, expected = await _recurrence_archive(tmp_path) + async with open_builtin_contexts(target_config) as target: + assert (await target.portability.validate(archive)).compatible + assert (await target.portability.restore(archive)).projections_ready + assert (await target.portability.restore(archive)).inserted == 0 + async with open_builtin_contexts(target_config) as target: + assert (await target.records.get_artifact(sid, "experience", artifact_id)).revision == 2 + async with target.database.transaction() as connection: + assert await target.repositories.recurrence.matches(connection, sid) == expected[0] + assert await target.repositories.recurrence.observations(connection, sid) == expected[1] + assert await _replay_recurrence(target, sid) == expected + assert await _replay_recurrence(target, sid) == expected + + async def authorize(_scopes): + pass + + roundtrip = tmp_path / "recurrence-roundtrip.pcb" + await target.portability.export([sid], roundtrip, authorize=authorize) + async with open_builtin_contexts(BuiltinConfig()) as reverse: + await reverse.portability.restore(roundtrip) + assert await _replay_recurrence(reverse, sid) == expected + + asyncio.run(scenario()) + + +def _rewrite_archive_records(archive, transform): + with zipfile.ZipFile(archive) as bundle: + manifest = json.loads(bundle.read("manifest.json")) + records = [json.loads(line) for line in bundle.read("records.ndjson").splitlines()] + records = transform(records) + for record in records: + record.pop("digest", None) + record["digest"] = ( + "sha256:" + + hashlib.sha256( + json.dumps(record, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + ) + manifest["record_count"] = len(records) + manifest["records_by_type"] = dict(Counter(record["record_type"] for record in records)) + manifest["total_digest"] = ( + "sha256:" + hashlib.sha256("\n".join(record["digest"] for record in records).encode()).hexdigest() + ) + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("manifest.json", json.dumps(manifest)) + bundle.writestr("records.ndjson", "\n".join(json.dumps(record) for record in records) + "\n") + + +@pytest.mark.parametrize("damage", ["outcome", "receipt", "revision", "handoff", "match", "payload", "key", "position"]) +def test_archive_rejects_broken_recurrence_before_writing(tmp_path: Path, damage: str) -> None: + async def scenario(): + archive, sid, _, _ = await _recurrence_archive(tmp_path) + + def damage_records(records): + if damage in {"outcome", "receipt"}: + source_id = "unlinked-failure" if damage == "outcome" else "receipt" + return [ + r for r in records if not (r["record_type"] == "source" and r["identity"]["source_id"] == source_id) + ] + if damage in {"revision", "handoff"}: + family = "experience" if damage == "revision" else "handoff" + return [ + r + for r in records + if not ( + r["record_type"] == "artifact_revision" + and r["identity"]["family"] == family + and r["identity"]["revision"] == 1 + ) + ] + if damage == "match": + return [r for r in records if r["record_type"] != "recurrence_match"] + if damage == "position": + record = next( + r + for r in records + if r["record_type"] == "source" and r["identity"]["source_id"] == "unlinked-failure" + ) + record["payload"]["journal_position"] = 1 + return records + record = next(r for r in records if r["record_type"] == "recurrence_observation") + if damage == "payload": + record["payload"]["payload"] = {"base64": base64.b64encode(b"{}").decode()} + else: + record["payload"]["selection_key"] = "sha256:" + "0" * 64 + return records + + _rewrite_archive_records(archive, damage_records) + async with open_builtin_contexts(BuiltinConfig()) as target: + with pytest.raises(BundleFormatError): + await target.portability.validate(archive) + with pytest.raises(BundleFormatError): + await target.portability.restore(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar(select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == sid)) + is None + ) + + asyncio.run(scenario()) + + +def test_archive_rejects_recomputed_recurrence_without_overwriting_history(tmp_path: Path) -> None: + async def scenario(): + archive, sid, _, expected = await _recurrence_archive(tmp_path) + without_history = tmp_path / "without-history.pcb" + without_history.write_bytes(archive.read_bytes()) + _rewrite_archive_records( + without_history, + lambda records: [r for r in records if not r["record_type"].startswith("recurrence_")], + ) + async with open_builtin_contexts(BuiltinConfig()) as target: + await target.portability.restore(without_history) + recomputed = await _replay_recurrence(target, sid) + assert recomputed != expected + assert any(match.result == "unmatched" for match in recomputed[0]) + validation = await target.portability.validate(archive) + assert not validation.compatible + assert validation.conflicts > 0 + with pytest.raises(BundleConflictError): + await target.portability.restore(archive) + assert await _replay_recurrence(target, sid) == recomputed + + asyncio.run(scenario()) + + +async def _validation_archive(tmp_path: Path) -> tuple[Path, str]: + archive = tmp_path / "validation.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Validation", summary="Recovery", idempotency_key="validation") + ) + context = await source.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="source-a", content="Portable source.")) + await context.sources.capture(ContentCapture(source_id="source-b", content="Another portable source.")) + await source.records.create_artifact( + scope.scope_id, + "prompt", + ArtifactWrite( + prompt_key="memory.extract", + content={ + "schema_version": "powercontext.prompt.v1", + "mode": "auto", + "instructions": "", + "demonstrations": [], + }, + ), + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + return archive, scope.scope_id + + +def test_archive_rejects_invalid_artifact_content_before_writing(tmp_path: Path, monkeypatch) -> None: + from typer.testing import CliRunner + + from powercontext.cli.app import create_cli + from powercontext.cli.archive import archive_app + + async def scenario() -> Path: + archive, sid = await _validation_archive(tmp_path) + + def damage(records): + record = next(r for r in records if r["record_type"] == "artifact_revision") + record["payload"]["content"] = {"base64": base64.b64encode(b"{}").decode()} + return records + + _rewrite_archive_records(archive, damage) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="Artifact content"): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar(select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == sid)) + is None + ) + return archive + + archive = asyncio.run(scenario()) + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "unused-target")) + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "restore", str(archive), "--dry-run"]) + assert result.exit_code == 2, result.output + assert "Artifact content" in result.output + assert not (tmp_path / "unused-target" / "powercontext.db").exists() + + +@pytest.mark.parametrize( + ("kind", "field", "value"), + [("source", "source_id", 123), ("source", "source_id", True), ("artifact_revision", "revision", "1")], +) +def test_archive_rejects_incorrect_identity_types(tmp_path: Path, kind: str, field: str, value: object) -> None: + async def scenario() -> None: + archive, sid = await _validation_archive(tmp_path) + + def damage(records): + record = next(r for r in records if r["record_type"] == kind) + record["identity"][field] = value + return records + + _rewrite_archive_records(archive, damage) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="identity"): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar(select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == sid)) + is None + ) + + asyncio.run(scenario()) + + +def test_archive_dry_run_detects_source_journal_collision(tmp_path: Path, target_config: BuiltinConfig) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + empty = tmp_path / "empty.pcb" + incoming = tmp_path / "incoming.pcb" + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Collision", summary="Journal", idempotency_key="collision") + ) + await source.portability.export([scope.scope_id], empty, authorize=authorize) + context = await source.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="source-a", content="Incoming source.")) + await source.portability.export([scope.scope_id], incoming, authorize=authorize) + async with open_builtin_contexts(target_config) as target: + await target.portability.restore(empty) + context = await target.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="source-b", content="Existing source.")) + validation = await target.portability.validate(incoming) + assert not validation.compatible + assert validation.conflicts == 1 + with pytest.raises(BundleConflictError): + await target.portability.restore(incoming) + async with target.database.transaction() as connection: + assert ( + await connection.scalars( + select(SOURCES_TABLE.c.source_id).where(SOURCES_TABLE.c.scope_id == scope.scope_id) + ) + ).all() == ["source-b"] + + asyncio.run(scenario()) + + +def test_archive_rejects_duplicate_journal_positions_before_writing(tmp_path: Path) -> None: + async def scenario() -> None: + archive, sid = await _validation_archive(tmp_path) + + def damage(records): + for record in records: + if record["record_type"] == "source": + record["payload"]["journal_position"] = 1 + return records + + _rewrite_archive_records(archive, damage) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="unique identity"): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar(select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == sid)) + is None + ) + + asyncio.run(scenario()) + + +async def _merged_archive(tmp_path: Path): + archive = tmp_path / "merged.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create(ScopeDraft(title="Merge", summary="Recovery", idempotency_key="merge")) + memory = source.atomic_memory.for_scope(scope.scope_id) + created = await source.records.create_atomic_memories( + scope.scope_id, + ({"kind": "rule", "text": "Preserve input one."}, {"kind": "rule", "text": "Preserve input two."}), + ) + originals = tuple([await memory.get(item.artifact_id) for item in created]) + merged = await memory.merge( + tuple(item.as_read() for item in originals), + AtomicMemoryContent(kind="rule", text="Preserve both inputs."), + ) + expected = tuple([await memory.get(item.ref.artifact_id) for item in originals]) + exported = await source.portability.export([scope.scope_id], archive, authorize=authorize) + return archive, scope.scope_id, expected, merged.primary, exported + + +def test_archive_preserves_merge_state_and_undo_after_restart(tmp_path: Path, target_config: BuiltinConfig) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive, scope_id, originals, merged, exported = await _merged_archive(tmp_path) + async with open_builtin_contexts(target_config) as target: + assert (await target.portability.restore(archive)).status == "ready" + assert (await target.portability.restore(archive)).inserted == 0 + roundtrip = await target.portability.export([scope_id], tmp_path / "roundtrip.pcb", authorize=authorize) + assert roundtrip.total_digest == exported.total_digest + async with open_builtin_contexts(target_config) as target: + memory = target.atomic_memory.for_scope(scope_id) + for original in originals: + assert await memory.get(original.ref.artifact_id) == original + assert original.state.state == "merged" + assert original.state.merged_into_id == merged.ref.artifact_id + with pytest.raises(InvalidAtomicMemoryStateError, match="cannot be edited"): + await target.records.replace_artifact( + scope_id, + "atomic-memory", + original.ref.artifact_id, + '"revision:1"', + ArtifactWrite(content={"kind": "rule", "text": "Cannot revise merged input."}), + ) + result = await memory.restore(merged.ref.artifact_id, operation="undo_merge") + assert {item.artifact_id for item in result.restored} == {item.ref.artifact_id for item in originals} + for original in originals: + assert (await memory.get(original.ref.artifact_id)).state.state == "active" + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("field", ["merged_into_id", "is_merge_input"]) +def test_archive_detects_conflicting_merge_topology(tmp_path: Path, field: str) -> None: + async def scenario() -> None: + archive, scope_id, originals, merged, _exported = await _merged_archive(tmp_path) + async with open_builtin_contexts(BuiltinConfig()) as target: + await target.portability.restore(archive) + + def damage(records): + kind = "artifact_head" if field == "merged_into_id" else "artifact_lineage_artifact" + record = next( + item for item in records if item["record_type"] == kind and item["payload"][field] is not None + ) + record["payload"][field] = None if field == "merged_into_id" else False + return records + + _rewrite_archive_records(archive, damage) + validation = await target.portability.validate(archive) + assert not validation.compatible + assert validation.conflicts == 1 + with pytest.raises(BundleConflictError): + await target.portability.restore(archive) + memory = target.atomic_memory.for_scope(scope_id) + assert await memory.get(originals[0].ref.artifact_id) == originals[0] + assert await memory.get(merged.ref.artifact_id) == merged + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("damage", ["missing_result", "old_head_schema", "old_lineage_schema", "invalid_marker"]) +def test_archive_rejects_invalid_merge_records_before_writing(tmp_path: Path, damage: str) -> None: + async def scenario() -> None: + archive, scope_id, _originals, _merged, _exported = await _merged_archive(tmp_path) + + def rewrite(records): + if damage in {"missing_result", "old_head_schema"}: + record = next( + item + for item in records + if item["record_type"] == "artifact_head" and item["payload"]["merged_into_id"] is not None + ) + if damage == "missing_result": + record["payload"]["merged_into_id"] = "missing-merge-result" + else: + record["schema_version"] = 1 + del record["payload"]["merged_into_id"] + else: + record = next(item for item in records if item["record_type"] == "artifact_lineage_artifact") + if damage == "invalid_marker": + record["payload"]["is_merge_input"] = "true" + else: + record["schema_version"] = 1 + del record["payload"]["is_merge_input"] + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar(select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == scope_id)) + is None + ) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("damage", ["long_identity", "large_integer", "missing_replacement", "self_replacement"]) +def test_archive_rejects_unusable_artifact_and_scope_rows_before_writing(tmp_path: Path, damage: str) -> None: + async def scenario() -> None: + archive = tmp_path / "unusable.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Storage bounds", summary="Portable values", idempotency_key="storage-bounds") + ) + await source.records.create_artifact( + scope.scope_id, "atomic-memory", ArtifactWrite(content={"kind": "fact", "text": "Portable evidence."}) + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + + def rewrite(records): + if damage == "long_identity": + for record in records: + if record["record_type"] in {"artifact_revision", "artifact_head", "artifact_lineage_source"}: + record["identity"]["artifact_id"] = "a" * (MAX_ARTIFACT_ID_LENGTH + 1) + elif damage == "large_integer": + next(record for record in records if record["record_type"] == "scope")["payload"]["version"] = 1 << 31 + else: + head = next(record for record in records if record["record_type"] == "artifact_head") + replacement = "missing" if damage == "missing_replacement" else head["identity"]["artifact_id"] + head["payload"].update(lifecycle_state="deprecated", replacement_artifact_id=replacement) + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar( + select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == scope.scope_id) + ) + is None + ) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("encoding", ["base64", "timestamp", "offset_timestamp"]) +def test_archive_rejects_wire_values_that_cannot_roundtrip(tmp_path: Path, encoding: str) -> None: + async def scenario() -> None: + archive = tmp_path / "canonical.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Canonical", summary="Stable retries", idempotency_key="canonical") + ) + context = await source.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="one", content="Canonical evidence.")) + await source.profiles.put_policy(scope.scope_id, generation_enabled=True, expected_version=0) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + + def rewrite(records): + if encoding != "base64": + policy = next(item for item in records if item["record_type"] == "profile_policy") + timestamp = policy["payload"]["updated_at"] + policy["payload"]["updated_at"] = ( + timestamp.replace("T", " ") if encoding == "timestamp" else timestamp + "Z" + ) + else: + source = next(item for item in records if item["record_type"] == "source") + payload = base64.b64decode(source["payload"]["payload"]["base64"]) + while len(payload) % 3 == 0: + payload += b" " + encoded = base64.b64encode(payload).decode() + alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + offset = len(encoded.rstrip("=")) - 1 + changed = encoded[:offset] + alphabet[alphabet.index(encoded[offset]) | 1] + encoded[offset + 1 :] + assert base64.b64decode(changed, validate=True) == payload + source["payload"]["payload"] = {"base64": changed} + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="canonical"): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar( + select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == scope.scope_id) + ) + is None + ) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("field", ["kind", "value"]) +@pytest.mark.parametrize("invalid", ["", " \t", " padded", "padded "]) +def test_archive_rejects_unreadable_scope_external_references(tmp_path: Path, field: str, invalid: str) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "scope.pcb" + config = BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'target.db'}")) + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft( + title="Scope references", + summary="Preserve readable external references", + idempotency_key="scope-references", + external_references=(ScopeExternalReference(kind="代码库", value="team/项目 name"),), + ) + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + original = archive.read_bytes() + + def rewrite(records): + reference = next(record for record in records if record["record_type"] == "scope_external_reference") + reference["payload"][field] = invalid + reference["payload"]["value_digest"] = hashlib.sha256(reference["payload"]["value"].encode()).hexdigest() + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(config) as target: + before = await target.scopes.list() + for operation in (target.portability.inspect, target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="external reference"): + await operation(archive) + assert await target.scopes.list() == before + archive.write_bytes(original) + assert (await target.portability.validate(archive)).compatible + assert (await target.portability.restore(archive)).status == "ready" + assert await target.scopes.get(scope.scope_id) == scope + async with open_builtin_contexts(config) as target: + assert await target.scopes.get(scope.scope_id) == scope + assert (await target.portability.restore(archive)).inserted == 0 + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("damage", ["cycle", "external_digest"]) +def test_archive_validates_scope_relationships_before_writing(tmp_path: Path, damage: str) -> None: + async def scenario() -> None: + archive = tmp_path / "scopes.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + parent = await source.scopes.create( + ScopeDraft( + title="Parent", + summary="Scope relationships", + idempotency_key="parent", + external_references=(ScopeExternalReference(kind="repository", value="example"),), + ) + ) + child = await source.scopes.create( + ScopeDraft( + title="Child", + summary="Scope relationships", + parent_scope_id=parent.scope_id, + idempotency_key="child", + ) + ) + await source.portability.export([parent.scope_id, child.scope_id], archive, authorize=authorize) + + def rewrite(records): + if damage == "cycle": + parent_record = next( + item + for item in records + if item["record_type"] == "scope" and item["identity"]["scope_id"] == parent.scope_id + ) + parent_record["payload"]["parent_scope_id"] = child.scope_id + else: + reference = next(item for item in records if item["record_type"] == "scope_external_reference") + reference["payload"]["value_digest"] = "0" * 64 + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError): + await operation(archive) + + asyncio.run(scenario()) + + +def test_archive_exports_migrated_scopes_without_deleting_offline_history(tmp_path: Path) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'migrated.db'}") + async with ( + open_builtin_contexts(BuiltinConfig(database=config)) as source, + source.database.transaction() as connection, + ): + await connection.execute( + insert(SCOPES_TABLE).values( + scope_id=MIGRATED_SCOPE, + title="Migrated", + summary="Retained history", + scope_id_search=MIGRATED_SCOPE, + title_search="Migrated", + summary_search="Retained history", + version=1, + ) + ) + await _seed_and_migrate(config) + archive = tmp_path / "migrated.pcb" + async with open_builtin_contexts(BuiltinConfig(database=config)) as source: + async with source.database.transaction() as connection: + retained_versions = (await connection.execute(select(MEMORY_ENTRY_VERSIONS_TABLE))).all() + retained_archive = (await connection.execute(select(ARCHIVE_TABLE))).all() + assert retained_versions and retained_archive + exported = await source.portability.export([MIGRATED_SCOPE], archive, authorize=authorize) + expected = await source.atomic_memory.for_scope(MIGRATED_SCOPE).list() + async with source.database.transaction() as connection: + assert (await connection.execute(select(MEMORY_ENTRY_VERSIONS_TABLE))).all() == retained_versions + assert (await connection.execute(select(ARCHIVE_TABLE))).all() == retained_archive + inspected = await PortableBundleService.inspect(archive) + assert "memory_entry_version" not in inspected.records_by_type + async with open_builtin_contexts(BuiltinConfig()) as target: + assert (await target.portability.restore(archive)).status == "ready" + assert await target.atomic_memory.for_scope(MIGRATED_SCOPE).list() == expected + reexported = await target.portability.export([MIGRATED_SCOPE], tmp_path / "again.pcb", authorize=authorize) + assert reexported.total_digest == exported.total_digest + + asyncio.run(scenario()) + + +async def _candidate_archive(tmp_path: Path, family: str, *, audited: bool = False): + archive = tmp_path / "candidate.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + proposal = ( + ExperienceContent(situation="Archive", action="Validate", outcome="Readable", lesson="Check proposals.") + if family == "experience" + else ProfileCandidateProposal( + content="Keep exact profile instructions.", + generator_id="test", + generator_version="1", + created_at=datetime(2026, 10, 1, tzinfo=UTC), + source_window=SourceWindow(after=0, through=1), + ) + ) + package = None + if family == "skill": + package_dir = tmp_path / "portable-skill" + package_dir.mkdir() + (package_dir / "SKILL.md").write_text( + "---\nname: portable-skill\ndescription: Portable instructions\n---\nValidate archives.\n" + ) + package = capture_skill_directory(package_dir) + proposal = package.as_skill_content() + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Candidate", summary="Recovery", idempotency_key="candidate") + ) + context = await source.get(scope.scope_id) + await context.sources.capture(ContentCapture(source_id="evidence", content="Candidate evidence.")) + if package is not None: + await source.upload_skill_package(scope.scope_id, package.archive_bytes, None, None) + async with source.database.transaction() as connection: + candidate = await source.repositories.candidates.create( + connection, + scope.scope_id, + "candidate-1", + family, + proposal, + sources=(SourceRef(source_type="content", source_id="evidence"),), + artifacts=(), + target=None, + reason=None, + ) + if audited: + await attach_candidate_audit( + connection, + scope.scope_id, + candidate, + CandidateAudit( + operation="correct", + dream_run_id="dream-1", + spec_version="1", + proposal_digest=proposal_digest(proposal), + evidence_manifest_ref="source:evidence", + validation_policy_digest="sha256:" + "a" * 64, + ), + ) + candidate = await source.repositories.candidates.get(connection, scope.scope_id, candidate.candidate_id) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + return archive, scope.scope_id, candidate + + +@pytest.mark.parametrize("family", ["experience", "profile", "skill"]) +@pytest.mark.parametrize("audited", [False, True]) +def test_archive_restores_candidates_through_supported_read_paths( + tmp_path: Path, family: str, audited: bool, target_config: BuiltinConfig +) -> None: + async def scenario() -> None: + archive, scope_id, candidate = await _candidate_archive(tmp_path, family, audited=audited) + async with open_builtin_contexts(target_config) as target: + assert (await target.portability.restore(archive)).status == "ready" + review = target.review(scope_id) + restored = await review.get_candidate(candidate.candidate_id) + assert restored.proposal == candidate.proposal + assert restored.audit == candidate.audit + page = await review.list_candidates(status=CandidateStatus.PENDING, family=family, cursor=None, limit=10) + assert restored in page.candidates + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("family", ["experience", "profile", "skill"]) +@pytest.mark.parametrize("damage", ["empty", "invalid_envelope", "invalid_audit"]) +def test_archive_rejects_invalid_candidate_proposals_before_writing( + tmp_path: Path, monkeypatch, family: str, damage: str +) -> None: + from typer.testing import CliRunner + + from powercontext.cli.app import create_cli + from powercontext.cli.archive import archive_app + + async def scenario() -> Path: + archive, scope_id, _candidate = await _candidate_archive(tmp_path, family) + + def rewrite(records): + record = next(item for item in records if item["record_type"] == "candidate_version") + payload = b"{}" if damage == "empty" else b'{"candidate_payload_version":1}' + if damage == "invalid_audit": + payload = b'{"candidate_payload_version":1,"proposal":{},"audit":{}}' + record["payload"]["proposal"] = {"base64": base64.b64encode(payload).decode()} + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="Candidate proposal"): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar(select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == scope_id)) + is None + ) + return archive + + archive = asyncio.run(scenario()) + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "unused-target")) + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "restore", str(archive), "--dry-run"]) + assert result.exit_code == 2, result.output + assert "Candidate proposal" in result.output + assert not (tmp_path / "unused-target" / "powercontext.db").exists() + + +def test_archive_rejects_tag_candidates_before_writing(tmp_path: Path) -> None: + async def scenario() -> None: + archive, scope_id, _candidate = await _candidate_archive(tmp_path, "experience") + + def rewrite(records): + record = next(item for item in records if item["record_type"] == "candidate_head") + record["payload"]["candidate_kind"] = "tag" + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="tag Candidates"): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar(select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == scope_id)) + is None + ) + + asyncio.run(scenario()) + + +def test_archive_rejects_missing_audited_skill_candidate_package(tmp_path: Path) -> None: + async def scenario() -> None: + archive, scope_id, candidate = await _candidate_archive(tmp_path, "skill", audited=True) + + def rewrite(records): + return [ + item + for item in records + if item["record_type"] != "skill_package" + and ( + item["record_type"] not in {"candidate_version", "candidate_head"} + or item["identity"]["candidate_id"] == candidate.candidate_id + ) + ] + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match="missing or mismatched package"): + await operation(archive) + async with target.database.transaction() as connection: + assert ( + await connection.scalar(select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == scope_id)) + is None + ) + + asyncio.run(scenario()) + + +def test_archive_preserves_stable_handles_and_detects_binding_conflicts( + tmp_path: Path, target_config: BuiltinConfig +) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "handles.pcb" + handle = StableHandle(family="atomic-memory", key="portable-rule") + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Handles", summary="Recovery", idempotency_key="handles") + ) + memories = await source.records.create_atomic_memories( + scope.scope_id, + ({"kind": "rule", "text": "Keep original binding."}, {"kind": "rule", "text": "Other binding."}), + ) + original, alternate = tuple( + ArtifactRef(family="atomic-memory", artifact_id=item.artifact_id, revision=item.revision) + for item in memories + ) + await source.records.bind_artifact_handle(scope.scope_id, handle, original) + exported = await source.portability.export([scope.scope_id], archive, authorize=authorize) + async with open_builtin_contexts(target_config) as target: + await target.portability.restore(archive) + async with open_builtin_contexts(target_config) as target: + assert await target.records.resolve_artifact_handle(scope.scope_id, handle) == original + assert (await target.portability.restore(archive)).inserted == 0 + roundtrip = await target.portability.export([scope.scope_id], tmp_path / "again.pcb", authorize=authorize) + assert roundtrip.total_digest == exported.total_digest + + def rewrite(records): + record = next(item for item in records if item["record_type"] == "artifact_handle") + record["payload"]["artifact_id"] = alternate.artifact_id + return records + + _rewrite_archive_records(archive, rewrite) + assert (await target.portability.validate(archive)).conflicts == 1 + with pytest.raises(BundleConflictError): + await target.portability.restore(archive) + assert await target.records.resolve_artifact_handle(scope.scope_id, handle) == original + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + "damage", ["head_family", "empty_evidence", "blank_reason", "blank_decision_reason", "historical_reason"] +) +def test_archive_rejects_invalid_candidate_envelopes_before_writing(tmp_path: Path, monkeypatch, damage: str) -> None: + from typer.testing import CliRunner + + from powercontext.cli.app import create_cli + from powercontext.cli.archive import archive_app + + async def scenario() -> Path: + archive, scope_id, _candidate = await _candidate_archive(tmp_path, "experience") + + def rewrite(records): + version = next(item for item in records if item["record_type"] == "candidate_version") + head = next(item for item in records if item["record_type"] == "candidate_head") + if damage == "head_family": + head["payload"]["family"] = "profile" + elif damage == "empty_evidence": + version["payload"]["source_refs"] = {"base64": base64.b64encode(b"[]").decode()} + version["payload"]["artifact_refs"] = {"base64": base64.b64encode(b"[]").decode()} + elif damage == "blank_decision_reason": + head["payload"].update(status="rejected", decision_reason="") + else: + if damage == "historical_reason": + records.append({ + **version, + "identity": {**version["identity"], "version": 2}, + "payload": dict(version["payload"]), + }) + head["payload"]["version"] = 2 + version["payload"]["reason"] = "" + return records + + _rewrite_archive_records(archive, rewrite) + async with open_builtin_contexts(BuiltinConfig()) as target: + existing = await target.scopes.create( + ScopeDraft(title="Existing target", summary="Must remain unchanged", idempotency_key="existing") + ) + async with target.database.transaction() as connection: + before = (await connection.execute(select(SCOPES_TABLE))).all() + for operation in (target.portability.validate, target.portability.restore): + with pytest.raises(BundleFormatError, match=r"[Cc]andidate"): + await operation(archive) + async with target.database.transaction() as connection: + assert (await connection.execute(select(SCOPES_TABLE))).all() == before + assert ( + await connection.scalar(select(SCOPES_TABLE.c.scope_id).where(SCOPES_TABLE.c.scope_id == scope_id)) + is None + ) + assert existing.scope_id in {row.scope_id for row in before} + return archive + + archive = asyncio.run(scenario()) + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "unused-target")) + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "restore", str(archive), "--dry-run"]) + assert result.exit_code == 2, result.output + assert "candidate" in result.output.lower() + assert not (tmp_path / "unused-target" / "powercontext.db").exists() + + +@pytest.mark.parametrize("status", ["pending", "approved", "rejected"]) +def test_archive_preserves_candidate_versions_and_lifecycle_after_restart( + tmp_path: Path, status: str, target_config: BuiltinConfig +) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + original, scope_id, candidate = await _candidate_archive(tmp_path, "experience") + archive = tmp_path / "candidate-history.pcb" + async with open_builtin_contexts(BuiltinConfig()) as source: + await source.portability.restore(original) + review = source.review(scope_id) + revised = await review.revise( + candidate.candidate_id, + candidate.version, + candidate.proposal, + sources=candidate.sources, + artifacts=candidate.artifacts, + target=candidate.target, + reason="Refined from the same evidence.", + ) + if status == "approved": + await review.approve(revised.candidate_id, revised.version) + elif status == "rejected": + await review.reject(revised.candidate_id, revised.version, "Not needed.") + expected = await review.get_candidate(candidate.candidate_id) + expected_history = await review.history(candidate.candidate_id) + await source.portability.export([scope_id], archive, authorize=authorize) + _rewrite_archive_records(archive, lambda records: list(reversed(records))) + inspected = await PortableBundleService.inspect(archive) + assert inspected.records_by_type["candidate_version"] == 2 + async with open_builtin_contexts(target_config) as target: + assert (await target.portability.restore(archive)).status == "ready" + assert (await target.portability.restore(archive)).inserted == 0 + async with open_builtin_contexts(target_config) as target: + review = target.review(scope_id) + assert await review.get_candidate(candidate.candidate_id) == expected + assert await review.history(candidate.candidate_id) == expected_history + page = await review.list_candidates( + status=CandidateStatus(status), + family="experience", + cursor=None, + limit=10, + ) + assert page.candidates == (expected,) + + asyncio.run(scenario()) + + +def test_archive_preserves_working_note_identity_and_handoff_after_restart( + tmp_path: Path, target_config: BuiltinConfig +) -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def scenario() -> None: + archive = tmp_path / "working-note.pcb" + capture = CaptureWorkingNote( + source_id="validation-note", + content="The archive roundtrip passed without promoting session state to Memory.", + session_id="session-1", + note_kind=WorkingNoteKind.VALIDATION, + expires_at=datetime(2099, 1, 1, tzinfo=UTC), + ) + async with open_builtin_runtime(BuiltinConfig()) as source: + assert source.scopes is not None + scope = await source.scopes.create( + ScopeDraft(title="Working note", summary="Portable session state", idempotency_key="working-note") + ) + note = await source.sources.for_scope(scope.scope_id).capture_working_note(capture) + expected = await source.records.for_scope(scope.scope_id).get_source("content", capture.source_id) + draft = HandoffDraft( + objective="Continue archive verification.", + state=( + HandoffStatement( + text="The recorded validation remains available.", + citations=(HandoffSourceCitation(source_ref=note.source_ref),), + ), + ), + disposition="continuable", + ) + handoffs = source.handoff.for_scope(scope.scope_id) + committed = await handoffs.commit(await handoffs.finalize(draft)) + assert source.archive is not None + exported = await source.archive.export([scope.scope_id], archive, authorize=authorize) + async with open_builtin_runtime(target_config) as target: + assert target.archive is not None + assert (await target.archive.restore(archive)).status == "ready" + async with open_builtin_runtime(target_config) as target: + assert await target.records.for_scope(scope.scope_id).get_source("content", capture.source_id) == expected + assert await target.sources.for_scope(scope.scope_id).capture_working_note(capture) == note + handoffs = target.handoff.for_scope(scope.scope_id) + assert await handoffs.latest() == committed + assert (await handoffs.finalize(draft)).content == committed.content + assert target.atomic_memory is not None + assert (await target.atomic_memory.for_scope(scope.scope_id).list()).items == () + assert target.archive is not None + reexported = await target.archive.export([scope.scope_id], tmp_path / "again.pcb", authorize=authorize) + assert reexported.total_digest == exported.total_digest + + asyncio.run(scenario()) diff --git a/tests/test_archive_cli.py b/tests/test_archive_cli.py new file mode 100644 index 0000000000..b0e0ce51ca --- /dev/null +++ b/tests/test_archive_cli.py @@ -0,0 +1,418 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import json +import sqlite3 +import struct +import zipfile +from pathlib import Path + +import pytest +from click.utils import strip_ansi +from sqlalchemy import update +from typer.testing import CliRunner + +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.tables import SCOPES_TABLE +from powercontext.builtin.records import ArtifactWrite +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts, open_builtin_runtime +from powercontext.builtin.scope import ScopeDraft +from powercontext.cli import archive as archive_module +from powercontext.cli.app import create_cli +from powercontext.cli.archive import archive_app +from powercontext.paths import default_database_path, sqlite_url +from powercontext.sources import AdapterSourceDefinition, SourceDefinitionRegistry +from tests.builtin.persistence.contract import NoteAdapter, NoteInput + + +def test_archive_cli_dry_run_reports_legacy_target_without_migrating_it(tmp_path, monkeypatch) -> None: + archive = tmp_path / "revisions.pcb" + target_file = tmp_path / "target.db" + target_url = sqlite_url(target_file) + target_config = BuiltinConfig(database=SQLiteConfig(url=target_url)) + + async def seed() -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Revision metadata", summary="Upgrade before validation", idempotency_key="metadata") + ) + await source.records.create_artifact( + scope.scope_id, "profile", ArtifactWrite(content={"content": "# Profile\n\nRetain revision metadata."}) + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + async with open_builtin_contexts(target_config): + pass + + asyncio.run(seed()) + with sqlite3.connect(target_file) as connection: + for column in ("created_at", "created_by", "restored_from_revision", "rollback_reason"): + connection.execute(f"ALTER TABLE pc_artifacts DROP COLUMN {column}") + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "home")) + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_KIND", "sqlite") + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_URL", target_url) + runner = CliRunner() + cli = create_cli([archive_app]) + before = target_file.read_bytes() + dry_run = runner.invoke(cli, ["archive", "restore", str(archive), "--dry-run"]) + assert dry_run.exit_code == 3, dry_run.output + assert "target schema lacks required archive fields" in dry_run.output + assert target_file.read_bytes() == before + restored = runner.invoke(cli, ["archive", "restore", str(archive), "--yes"]) + assert restored.exit_code == 0, restored.output + compatible = runner.invoke(cli, ["archive", "restore", str(archive), "--dry-run"]) + assert compatible.exit_code == 0, compatible.output + + +def test_archive_cli_restore_reports_unsupported_native_sources_consistently(tmp_path, monkeypatch) -> None: + archive = tmp_path / "native.pcb" + + async def seed() -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + registry = SourceDefinitionRegistry((AdapterSourceDefinition(NoteAdapter()),)) + async with open_builtin_contexts(BuiltinConfig(), source_registry=registry) as source: + scope = await source.scopes.create( + ScopeDraft(title="Native", summary="Missing target adapter", idempotency_key="native") + ) + native = await registry.resolve(NoteInput(note_id="one", body="Native evidence")) + async with source.database.transaction() as connection: + await source.repositories.sources.add(connection, scope.scope_id, native) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + + asyncio.run(seed()) + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "target")) + runner = CliRunner() + cli = create_cli([archive_app]) + for confirmation in ("--dry-run", "--yes"): + result = runner.invoke(cli, ["archive", "restore", str(archive), confirmation]) + assert result.exit_code == 3, result.output + + +def test_archive_cli_exposes_a_safe_restore_confirmation_and_dry_run() -> None: + cli = create_cli([archive_app]) + runner = CliRunner() + + help_result = runner.invoke(cli, ["archive", "--help"], terminal_width=160) + restore_help = runner.invoke(cli, ["archive", "restore", "--help"], terminal_width=160) + + help_output = strip_ansi(help_result.output) + restore_output = strip_ansi(restore_help.output) + + assert help_result.exit_code == 0 + assert all(command in help_output for command in ("export", "inspect", "restore")) + assert restore_help.exit_code == 0 + assert "--dry-run" in restore_output + assert "--yes" in restore_output + + +@pytest.mark.parametrize( + ("field", "value"), + [("record_type", []), ("record_type", {}), ("schema_version", True), ("format_version", True)], +) +def test_archive_cli_normalizes_invalid_wire_types(tmp_path, field, value) -> None: + archive = tmp_path / "invalid.pcb" + + async def seed() -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Wire types", summary="Reject malformed input", idempotency_key="wire-types") + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + + asyncio.run(seed()) + with zipfile.ZipFile(archive) as bundle: + manifest = json.loads(bundle.read("manifest.json")) + records = [json.loads(line) for line in bundle.read("records.ndjson").splitlines()] + if field == "format_version": + manifest[field] = value + else: + records[0][field] = value + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("manifest.json", json.dumps(manifest)) + bundle.writestr("records.ndjson", "\n".join(json.dumps(record) for record in records)) + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "inspect", str(archive)]) + assert result.exit_code == 2, (result.output, result.exception) + assert "archive validation failed" in result.output + + +@pytest.mark.parametrize("operation", [("inspect",), ("restore", "--dry-run"), ("restore", "--yes")]) +def test_archive_cli_rejects_unsupported_zip_versions(tmp_path, monkeypatch, operation) -> None: + archive = tmp_path / "unsupported.pcb" + entry = zipfile.ZipInfo("manifest.json") + entry.extract_version = 64 + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr(entry, b"{}") + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "target")) + result = CliRunner().invoke(create_cli([archive_app]), ["archive", *operation, str(archive)]) + assert result.exit_code == 2, (result.output, result.exception) + assert "archive validation failed" in result.output + + +def test_archive_cli_can_export_inspect_and_dry_run_a_scope(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "data")) + output = tmp_path / "scope.pcb" + cli = create_cli([archive_app]) + runner = CliRunner() + + async def default_scope_id() -> str: + config = BuiltinConfig(database=SQLiteConfig(url=sqlite_url(default_database_path()))) + async with open_builtin_runtime(config) as runtime: + assert runtime.scopes is not None + scope = await runtime.scopes.default_scope() + assert scope is not None + return scope.scope_id + + scope_id = asyncio.run(default_scope_id()) + exported = runner.invoke(cli, ["archive", "export", "--scope-id", scope_id, "--output", str(output)]) + target_home = tmp_path / "unused-target" + monkeypatch.setenv("POWERCONTEXT_HOME", str(target_home)) + inspected = runner.invoke(cli, ["archive", "inspect", str(output)]) + validated = runner.invoke(cli, ["archive", "restore", str(output), "--dry-run"]) + + assert exported.exit_code == 0, exported.output + assert output.is_file() + assert not (target_home / "powercontext.db").exists() + assert inspected.exit_code == 0, inspected.output + assert validated.exit_code == 0, validated.output + restored = runner.invoke(cli, ["archive", "restore", str(output), "--yes"]) + assert restored.exit_code == 0, restored.output + repeated = runner.invoke(cli, ["archive", "restore", str(output), "--yes"]) + assert repeated.exit_code == 0, repeated.output + assert json.loads(repeated.stdout)["inserted"] == 0 + assert asyncio.run(default_scope_id()) == scope_id + + +@pytest.mark.parametrize("entry", ["manifest.json", "records.ndjson"]) +@pytest.mark.parametrize("operation", ["inspect", "dry-run", "restore"]) +def test_archive_cli_rejects_invalid_deflate_streams(tmp_path, monkeypatch, entry, operation) -> None: + archive = tmp_path / "damaged.pcb" + + async def seed() -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as contexts: + scope = await contexts.scopes.create( + ScopeDraft(title="Compression", summary="Corrupt stream", idempotency_key="compression") + ) + await contexts.portability.export([scope.scope_id], archive, authorize=authorize) + + asyncio.run(seed()) + with zipfile.ZipFile(archive) as bundle: + header_offset = bundle.getinfo(entry).header_offset + data = bytearray(archive.read_bytes()) + filename_size, extra_size = struct.unpack_from(" None: + archive = tmp_path / "scope.pcb" + target = tmp_path / "target" + target.mkdir() + + async def seed() -> None: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async with open_builtin_contexts(BuiltinConfig()) as contexts: + scope = await contexts.scopes.create( + ScopeDraft(title="Inspection", summary="Fail closed", idempotency_key="inspection") + ) + await contexts.portability.export([scope.scope_id], archive, authorize=authorize) + + asyncio.run(seed()) + if failure.startswith("parent-file"): + target.rmdir() + target.write_text("not a directory", encoding="utf-8") + target = target / "target.db" + if failure in {"permission", "parent-file-as-missing"}: + original_stat = Path.stat + + def inaccessible(path, *arguments, **keywords): + if path == target: + if failure == "permission": + raise PermissionError + raise FileNotFoundError + return original_stat(path, *arguments, **keywords) + + monkeypatch.setattr(Path, "stat", inaccessible) + url = f"sqlite+aiosqlite:///{target}" + if failure == "authority": + url = f"sqlite+aiosqlite:///file://other-host/{target.as_posix().lstrip('/')}?uri=true" + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_URL", url) + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "restore", str(archive), "--dry-run"]) + assert result.exit_code == 2, result.output + assert "cannot inspect configured SQLite target" in result.output + + +def test_archive_cli_requires_projection_recovery_before_export(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("POWERCONTEXT_HOME", str(tmp_path / "target")) + archive = tmp_path / "recovery.pcb" + output = tmp_path / "exported.pcb" + + async def seed() -> str: + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def interrupted(_scopes: tuple[str, ...]) -> None: + raise RuntimeError("interrupted") + + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="Recovery", summary="Incomplete projections", idempotency_key="incomplete") + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + config = BuiltinConfig(database=SQLiteConfig(url=sqlite_url(default_database_path()))) + async with open_builtin_contexts(config) as target: + with pytest.raises(RuntimeError, match="interrupted"): + await archive_module.PortableBundleService(target.database, projection_rebuilder=interrupted).restore( + archive + ) + return scope.scope_id + + scope_id = asyncio.run(seed()) + cli = create_cli([archive_app]) + runner = CliRunner() + arguments = ["archive", "export", "--scope-id", scope_id, "--output", str(output)] + blocked = runner.invoke(cli, arguments) + assert blocked.exit_code == 3, blocked.output + assert "archive recovery required" in blocked.output + assert not output.exists() + recovered = runner.invoke(cli, ["archive", "restore", str(archive), "--yes"]) + assert recovered.exit_code == 0, recovered.output + exported = runner.invoke(cli, arguments) + assert exported.exit_code == 0, exported.output + + +def test_archive_cli_reports_corrupt_or_missing_bundles_without_a_traceback() -> None: + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "inspect", "missing.pcb"]) + + assert result.exit_code == 2 + assert isinstance(result.exception, SystemExit) + assert "error: archive validation failed: cannot read portable bundle" in result.output + assert "Traceback" not in result.output + + +def test_archive_cli_dry_run_exits_nonzero_when_target_is_incompatible(tmp_path, monkeypatch) -> None: + async def incompatible(_source, /, *, env_file): + del env_file + return archive_module.BundleValidation( + bundle_id="00000000-0000-0000-0000-000000000000", + scopes=("project:one",), + record_count=1, + records_by_type={"scope": 1}, + total_digest="sha256:" + "0" * 64, + format_version=1, + producer_version="test", + already_present=0, + conflicts=0, + projections_supported=True, + compatible=False, + required_source_types=(), + unsupported_source_types=(), + required_artifact_families=("future-family",), + unsupported_artifact_families=("future-family",), + reader_version="test", + ) + + monkeypatch.setattr(archive_module, "_validate_only", incompatible) + result = CliRunner().invoke( + create_cli([archive_app]), + ["archive", "restore", str(tmp_path / "bundle.pcb"), "--dry-run"], + ) + + assert result.exit_code == 3 + assert '"compatible": false' in result.output + + +@pytest.mark.parametrize("uri_kind", ["absolute", "relative", "escaped"]) +def test_archive_cli_dry_run_checks_existing_sqlite_uri_target(tmp_path, monkeypatch, uri_kind) -> None: + monkeypatch.chdir(tmp_path) + target_path = tmp_path / ("target #1.db" if uri_kind == "escaped" else "target.db") + archive = tmp_path / "scope.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def prepare() -> None: + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create(ScopeDraft(title="Original", summary="Archive", idempotency_key="uri")) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + async with open_builtin_contexts( + BuiltinConfig(database=SQLiteConfig(url=f"sqlite+aiosqlite:///{target_path}")) + ) as target: + await target.portability.restore(archive) + async with target.database.transaction() as connection: + await connection.execute( + update(SCOPES_TABLE) + .where(SCOPES_TABLE.c.scope_id == scope.scope_id) + .values(title="Target differs", title_search="Target differs", version=2) + ) + + asyncio.run(prepare()) + filename = "file:target.db" if uri_kind == "relative" else target_path.as_uri() + uri_flag = "on" if uri_kind == "relative" else "true" + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_KIND", "sqlite") + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_URL", f"sqlite+aiosqlite:///{filename}?uri={uri_flag}") + before = target_path.read_bytes() + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "restore", str(archive), "--dry-run"]) + + assert result.exit_code == 3, result.output + report = json.loads(result.stdout) + assert not report["compatible"] + assert report["conflicts"] == 1 + assert target_path.read_bytes() == before + + +def test_archive_cli_dry_run_does_not_create_missing_sqlite_uri_target(tmp_path, monkeypatch) -> None: + archive = tmp_path / "scope.pcb" + + async def authorize(_scopes: tuple[str, ...]) -> None: + pass + + async def prepare() -> None: + async with open_builtin_contexts(BuiltinConfig()) as source: + scope = await source.scopes.create( + ScopeDraft(title="New", summary="Archive", idempotency_key="missing-uri") + ) + await source.portability.export([scope.scope_id], archive, authorize=authorize) + + asyncio.run(prepare()) + target_path = tmp_path / "missing" / "target.db" + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_KIND", "sqlite") + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_URL", f"sqlite+aiosqlite:///{target_path.as_uri()}?uri=true") + result = CliRunner().invoke(create_cli([archive_app]), ["archive", "restore", str(archive), "--dry-run"]) + + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["compatible"] + assert not target_path.parent.exists() diff --git a/tests/test_source_observations.py b/tests/test_source_observations.py index c313ae7962..b4933a7934 100644 --- a/tests/test_source_observations.py +++ b/tests/test_source_observations.py @@ -21,8 +21,8 @@ from pydantic import ValidationError from referencing.exceptions import Unresolvable -from powercontext.builtin.runtime.relational import _json_schema_validator from powercontext.builtin.sources import CONTENT_SOURCE_DEFINITION, ContentCapture +from powercontext.builtin.sources.observation_validation import json_schema_validator from powercontext.sources import ( TEXT_EVIDENCE_PROJECTION_KEY, Source, @@ -96,7 +96,7 @@ def reject_request(*_args: object, **_kwargs: object) -> None: raise OSError("network access is forbidden") # noqa: TRY003 monkeypatch.setattr(urllib.request, "urlopen", reject_request) - validator = _json_schema_validator("remote", {"$ref": "http://127.0.0.1:9/schema"}) + validator = json_schema_validator("remote", {"$ref": "http://127.0.0.1:9/schema"}) with pytest.raises(Unresolvable): validator.validate({}) diff --git a/uv.lock b/uv.lock index 9a77ef5541..de9cd7d4f2 100644 --- a/uv.lock +++ b/uv.lock @@ -1225,7 +1225,7 @@ name = "importlib-metadata" version = "9.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp" }, + { name = "zipp", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } wheels = [ @@ -2410,8 +2410,8 @@ name = "opendalfs" version = "0.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "fsspec" }, - { name = "opendal" }, + { name = "fsspec", marker = "python_full_version >= '3.12'" }, + { name = "opendal", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0d/23/fdb4d34f08ac68a27b2d553256e13b15ce139916d36bab33f9a0e71e8770/opendalfs-0.1.0.tar.gz", hash = "sha256:5ceaeccc0852ef10c8ef2ce20e5927dbc18083909c06550ad36f81bcacb5774d", size = 75112, upload-time = "2026-08-26T05:30:33.497Z" } wheels = [ @@ -2741,6 +2741,7 @@ builtin = [ { name = "jieba" }, { name = "jsonschema" }, { name = "packaging" }, + { name = "pycasbin" }, { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, { name = "pydantic-settings" }, { name = "pymysql" }, @@ -2751,15 +2752,28 @@ builtin = [ { name = "tzdata" }, ] cli = [ + { name = "aiosqlite" }, + { name = "alembic" }, + { name = "apscheduler" }, { name = "click" }, + { name = "filelock" }, { name = "httpx", extra = ["socks"] }, { name = "inquirerpy" }, + { name = "jieba" }, + { name = "jsonschema" }, { name = "opentelemetry-api" }, { name = "packaging" }, { name = "platformdirs" }, + { name = "pycasbin" }, + { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, { name = "pydantic-settings" }, + { name = "pymysql" }, + { name = "pyobvector" }, { name = "pyyaml" }, + { name = "sqlalchemy", extra = ["asyncio"] }, + { name = "sqlite-vec" }, { name = "typer" }, + { name = "tzdata" }, ] client = [ { name = "httpx", extra = ["socks"] }, @@ -2785,6 +2799,7 @@ seekdb = [ { name = "jieba" }, { name = "jsonschema" }, { name = "packaging" }, + { name = "pycasbin" }, { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, { name = "pydantic-settings" }, { name = "pylibseekdb", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -2865,31 +2880,38 @@ notebooks = [ [package.metadata] requires-dist = [ { name = "aiosqlite", marker = "extra == 'builtin'", specifier = ">=0.22,<1" }, + { name = "aiosqlite", marker = "extra == 'cli'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'seekdb'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'server'", specifier = ">=0.22,<1" }, { name = "alembic", marker = "extra == 'builtin'", specifier = ">=1.16,<2" }, + { name = "alembic", marker = "extra == 'cli'", specifier = ">=1.16,<2" }, { name = "alembic", marker = "extra == 'seekdb'", specifier = ">=1.16,<2" }, { name = "alembic", marker = "extra == 'server'", specifier = ">=1.16,<2" }, { name = "apscheduler", marker = "extra == 'builtin'", specifier = ">=3.11,<4" }, + { name = "apscheduler", marker = "extra == 'cli'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'seekdb'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'server'", specifier = ">=3.11,<4" }, { name = "click", marker = "extra == 'cli'", specifier = ">=8.1,<9" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115,<1" }, { name = "fastmcp", marker = "extra == 'server'", specifier = ">=3.4,<4" }, { name = "filelock", marker = "extra == 'builtin'", specifier = ">=3.16,<4" }, + { name = "filelock", marker = "extra == 'cli'", specifier = ">=3.16,<4" }, { name = "filelock", marker = "extra == 'seekdb'", specifier = ">=3.16,<4" }, { name = "filelock", marker = "extra == 'server'", specifier = ">=3.16,<4" }, { name = "httpx", marker = "extra == 'builtin'", specifier = ">=0.28,<1" }, + { name = "httpx", marker = "extra == 'cli'", specifier = ">=0.28,<1" }, { name = "httpx", marker = "extra == 'seekdb'", specifier = ">=0.28,<1" }, { name = "httpx", marker = "extra == 'server'", specifier = ">=0.28,<1" }, { name = "httpx", extras = ["socks"], marker = "extra == 'cli'", specifier = ">=0.28,<1" }, { name = "httpx", extras = ["socks"], marker = "extra == 'client'", specifier = ">=0.28,<1" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3,<1" }, { name = "jieba", marker = "extra == 'builtin'", specifier = "==0.42.1" }, + { name = "jieba", marker = "extra == 'cli'", specifier = "==0.42.1" }, { name = "jieba", marker = "extra == 'seekdb'", specifier = "==0.42.1" }, { name = "jieba", marker = "extra == 'server'", specifier = "==0.42.1" }, { name = "jinja2", marker = "extra == 'server'", specifier = ">=3.1.6,<4" }, { name = "jsonschema", marker = "extra == 'builtin'", specifier = ">=4.23,<5" }, + { name = "jsonschema", marker = "extra == 'cli'", specifier = ">=4.23,<5" }, { name = "jsonschema", marker = "extra == 'seekdb'", specifier = ">=4.23,<5" }, { name = "jsonschema", marker = "extra == 'server'", specifier = ">=4.23,<5" }, { name = "markdown-it-py", marker = "extra == 'server'", specifier = ">=4,<5" }, @@ -2907,9 +2929,13 @@ requires-dist = [ { name = "platformdirs", marker = "extra == 'server'", specifier = ">=4,<5" }, { name = "prometheus-client", marker = "extra == 'server'", specifier = ">=0.21,<1" }, { name = "psutil", marker = "sys_platform == 'darwin' and extra == 'code'", specifier = ">=7.2,<8" }, + { name = "pycasbin", marker = "extra == 'builtin'", specifier = ">=2.8,<3" }, + { name = "pycasbin", marker = "extra == 'cli'", specifier = ">=2.8,<3" }, + { name = "pycasbin", marker = "extra == 'seekdb'", specifier = ">=2.8,<3" }, { name = "pycasbin", marker = "extra == 'server'", specifier = ">=2.8,<3" }, { name = "pydantic", specifier = ">=2.10,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'builtin'", specifier = ">=2.27.1,<3" }, + { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'cli'", specifier = ">=2.27.1,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'seekdb'", specifier = ">=2.27.1,<3" }, { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], marker = "extra == 'server'", specifier = ">=2.27.1,<3" }, { name = "pydantic-settings", marker = "extra == 'builtin'", specifier = ">=2.7,<3" }, @@ -2919,9 +2945,11 @@ requires-dist = [ { name = "pydantic-settings", marker = "extra == 'server'", specifier = ">=2.7,<3" }, { name = "pylibseekdb", marker = "(sys_platform == 'darwin' and extra == 'seekdb') or (sys_platform == 'linux' and extra == 'seekdb')", specifier = ">=1.4.0.post1,<2" }, { name = "pymysql", marker = "extra == 'builtin'", specifier = ">=1.2,<1.2.1" }, + { name = "pymysql", marker = "extra == 'cli'", specifier = ">=1.2,<1.2.1" }, { name = "pymysql", marker = "extra == 'seekdb'", specifier = ">=1.2,<1.2.1" }, { name = "pymysql", marker = "extra == 'server'", specifier = ">=1.2,<1.2.1" }, { name = "pyobvector", marker = "extra == 'builtin'", specifier = ">=0.2.28,<0.3" }, + { name = "pyobvector", marker = "extra == 'cli'", specifier = ">=0.2.28,<0.3" }, { name = "pyobvector", marker = "extra == 'seekdb'", specifier = ">=0.2.28,<0.3" }, { name = "pyobvector", marker = "extra == 'server'", specifier = ">=0.2.28,<0.3" }, { name = "pyyaml", marker = "extra == 'builtin'", specifier = ">=6,<7" }, @@ -2932,9 +2960,11 @@ requires-dist = [ { name = "rfc8785", specifier = ">=0.1.4,<1" }, { name = "scalar-fastapi", marker = "extra == 'server'", specifier = ">=1.8.2,<2" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'builtin'", specifier = ">=2,<3" }, + { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'cli'", specifier = ">=2,<3" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'seekdb'", specifier = ">=2,<3" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'server'", specifier = ">=2,<3" }, { name = "sqlite-vec", marker = "extra == 'builtin'", specifier = ">=0.1.9,<0.2" }, + { name = "sqlite-vec", marker = "extra == 'cli'", specifier = ">=0.1.9,<0.2" }, { name = "sqlite-vec", marker = "extra == 'seekdb'", specifier = ">=0.1.9,<0.2" }, { name = "sqlite-vec", marker = "extra == 'server'", specifier = ">=0.1.9,<0.2" }, { name = "tree-sitter", marker = "extra == 'code'", specifier = "==0.26.0" }, @@ -2945,6 +2975,7 @@ requires-dist = [ { name = "typer", marker = "extra == 'cli'", specifier = ">=0.16,<1" }, { name = "typing-extensions", specifier = ">=4.12,<5" }, { name = "tzdata", marker = "extra == 'builtin'", specifier = ">=2026.3" }, + { name = "tzdata", marker = "extra == 'cli'", specifier = ">=2026.3" }, { name = "tzdata", marker = "extra == 'seekdb'", specifier = ">=2026.3" }, { name = "tzdata", marker = "extra == 'server'", specifier = ">=2026.3" }, { name = "uvicorn", marker = "extra == 'server'", specifier = ">=0.34,<1" }, @@ -2991,8 +3022,8 @@ name = "powercontext-connector-opendal" version = "0.0.1" source = { editable = "integrations/opendal" } dependencies = [ - { name = "opendalfs" }, - { name = "powercontext", extra = ["client"] }, + { name = "opendalfs", marker = "python_full_version >= '3.12'" }, + { name = "powercontext", extra = ["client"], marker = "python_full_version >= '3.12'" }, ] [package.metadata]