From 18a0b256a1b80d86e1d7b16522aae5186bff1a1a Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Sat, 12 Sep 2026 11:46:25 +0800 Subject: [PATCH 01/43] docs: record ECC runtime consolidation plan --- docs/index.md | 1 + .../adr-headless-engine-contract.md | 80 +++++ .../ecc-runtime-consolidation-plan.zh-CN.md | 221 +++++++++++++ .../specification/glossary-headless-engine.md | 89 ++++++ .../specification/headless-engine-contract.md | 290 ++++++++++++++++++ 5 files changed, 681 insertions(+) create mode 100644 docs/specification/adr-headless-engine-contract.md create mode 100644 docs/specification/ecc-runtime-consolidation-plan.zh-CN.md create mode 100644 docs/specification/glossary-headless-engine.md create mode 100644 docs/specification/headless-engine-contract.md diff --git a/docs/index.md b/docs/index.md index efe18b71..24afee4a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -43,6 +43,7 @@ ChipCompiler supports various EDA file formats. Technical specifications for par ### CLI Specifications +- **[ECC Runtime Consolidation Plan](specification/ecc-runtime-consolidation-plan.zh-CN.md)** - ECC-owned Runtime, shared CLI/Engine writes, Studio Snapshot boundary, phased migration and protocol cleanup - **[CLI Design](specification/cli-design.md)** - Progressive-disclosure CLI design and roadmap - Grep-friendly summary lines with disclosure commands - Project, run, step, metric, artifact, issue, and config object model diff --git a/docs/specification/adr-headless-engine-contract.md b/docs/specification/adr-headless-engine-contract.md new file mode 100644 index 00000000..687c814a --- /dev/null +++ b/docs/specification/adr-headless-engine-contract.md @@ -0,0 +1,80 @@ +# ADR: Headless Engine Contract Scope (PR 275) + +> **Superseded.** The accepted PR 275 scope was Studio-only. The current +> cross-repository decision is documented in +> [ECC Runtime Consolidation v1](ecc-runtime-consolidation-plan.zh-CN.md) and +> moves all CLI Workspace writes onto the same ECC Engine commit paths. + +Status: accepted during review of +[PR 275](https://github.com/openecos-projects/ecc/pull/275) +(`ekko/refactor-ecc-cli-integration`). Implementation spec: +[headless-engine-contract.md](headless-engine-contract.md). + +## Context + +This PR removes the in-process RPC runtime and adds headless workspace APIs +under `chipcompiler.engine`. Two interpretations were available: + +- Treat the new APIs as a shared CLI + Studio contract in this PR. +- Treat them as a Studio-owned headless contract, and leave CLI on its + existing `create_workspace` / `EngineFlow` path. + +`main` already had two products, not one transaction model: + +- CLI serialized runs with a sibling `{workspace}.lock` and rolled back + `ecc param --workspace` from in-memory file bytes. +- RPC/Studio serialized mutations with a process-local session lock and + had no file-backed configuration transaction. + +## Decision + +This PR delivers a **Studio-only** headless contract. CLI does not adopt +the new create / configure / execute APIs in this round. + +Consequences that follow from that scope: + +1. Studio create, structural update, and configuration update share one + sibling `{workspace}.lock`. Configuration backup remains a Studio + transaction implementation and does **not** own + `.{workspace}.configuration.lock` (that file is new in this PR and is + removed). Backup and crash recovery run only while the ownership lock + is already held. +2. CLI keeps `{workspace}.lock` and `_snapshot_transaction`. Those are + the pre-existing CLI design; this PR does not merge them into + `WorkspaceFileTransaction`. +3. Studio execution goes through `chipcompiler.engine.execute`, which + commits the Engineering Snapshot. CLI continues to call `EngineFlow` + directly and does not write snapshots. +4. Studio query APIs (`describe`, `assess`, `read_*`) load workspaces + `read_only=True`. They must not migrate, create home files, or append + logs. Mutating APIs may load writable workspaces. +5. Unused runtime extraction in `workspace_flow.py` is deleted. `execute` + calls `EngineFlow` directly. +6. `create_workspace_from_spec` may delete `target` only if this call + created that directory. A pre-existing Workspace is never removed on + create failure (`workspace_exists` instead). +7. `chipcompiler.engine` exports the Studio contract: lifecycle, + configuration, `execute`, Engineering Snapshot reads, and signoff + inspect/export. It does not export analysis builders or `score_qor`; + those stay internal to snapshot/execute. +8. Workspace Spec parameter applicability uses `normalize_flow_step()`. + The handwritten step-alias table in spec validation is deleted. + +## Non-goals for this PR + +- Wiring CLI `ecc run`, `ecc workspace refresh`, or `ecc param --workspace` + through `create_workspace_from_spec` / `update_workspace_configuration` + / `execute`. +- Making CLI and Studio lock files mutually exclusive. +- Replacing CLI in-memory parameter rollback with the Studio file + transaction. +- Changing CLI `sys.modules` manifest aliases. +- Making CLI `ecc signoff inspect` a read-only load. +- Splitting `metrics.py`. +- Treating full-suite / PyInstaller / Nix runs as a contract change + (they remain a merge checklist, not a scope item). + +## Follow-up + +A later PR may adopt the headless contract in CLI. That change is +explicitly out of scope here and would be a separate decision. diff --git a/docs/specification/ecc-runtime-consolidation-plan.zh-CN.md b/docs/specification/ecc-runtime-consolidation-plan.zh-CN.md new file mode 100644 index 00000000..ea5a863a --- /dev/null +++ b/docs/specification/ecc-runtime-consolidation-plan.zh-CN.md @@ -0,0 +1,221 @@ +# ECC Runtime Consolidation v1 + +状态：已确认，准备实施 + +本文是 ECC 与 ECOS Studio 的联合实施方案。目标是删除 +`ecos-studio-runtime`/`ecos-ecc-runtime-adapter` 的独立实现，同时保留 +ECOS Studio 的全部能力，包括 Workspace Snapshot、Project Comparison、QoR、 +Signoff、Layout/Floorplan 编辑、取消、恢复和后台执行。 + +## 1. 最终边界 + +```text +ECC Engine 领域事务、配置、Flow 执行、Revision、Engineering Snapshot +ECC Runtime ECC 内部 JSON-RPC、Session、Operation、Ledger、事件、取消、恢复 +ECC CLI CLI 解析、输出和只读查询；写操作调用 ECC Engine +Electron 产品策略、权限、路径授权、进程监督、Snapshot 读取和展示 +``` + +- 只发布一个 `ecc` 可执行文件。普通 CLI 与隐藏的 `ecc rpc serve` 使用同一 + 可执行文件、同一 Python 环境和同一 ECC 版本。 +- ECC Runtime 是 ECC 的私有 Studio 集成接口，不承诺第三方协议兼容性。 +- 每个有执行或编辑工作的 Workspace 使用独立 Runtime 进程和故障域。 +- Workspace 无活动 Operation、Layout Edit 或 finalization 工作后，Electron + 释放该 Workspace Runtime；Workspace 无关请求使用一个应用生命周期内的懒加载 + Control Runtime，但它仍是同一 `ecc` 进程模型中的路由细节。 +- Electron Agent、Quick Start 和后台产品命令必须走 + `Electron Product Command -> ECC Runtime -> ECC Engine`，不能直接导入 ECC + 或绕过 Runtime 调用 Engine。 + +## 2. Snapshot 决策 + +### Engineering Snapshot + +`home/engineering-snapshot.json` 是 Studio 的 committed engineering read +model，不是 CLI 查询 API。 + +- ECC 是唯一生成者和写入者。 +- Workspace 创建、结构更新、配置更新、rerun preparation、保存 Layout 和每个 + Flow Step Commit 都按统一 Revision 提交 Snapshot。 +- 文件保持现有 schema v2 和文件名；`engineering-snapshot.stale.json` 继续表示 + 单个 stale predecessor。 +- Flow 状态、Snapshot 和 commit metadata 必须以 Atomic Engineering Commit + 一起暂存、验证并原子替换；任何失败都保留前一组 committed 文件且不发 + `workspace.committed`。 +- Electron 直接读取并校验 Snapshot（路径、大小、JSON、schema、Workspace ID、 + Revision、section 和 Artifact 指纹），Runtime Event 只作为失效提示。 +- Snapshot 缺失、损坏或不支持时不回退到旧文件、不自动转换、不在查询时物化；旧 + Workspace 使用显式 `ecc workspace refresh`、重新创建或已有 `--overwrite` 流程。 + +### Execution Snapshot + +`workspace.snapshot` 是 Runtime 的运行态视图，由 live Session、Active +Operation 和有界 Operation Ledger 重建；它不替代 Engineering Snapshot。 + +- Ledger 始终保留 Active Operation 和最近 256 个 terminal Operation。 +- 更早记录及其 command-id 去重结果可以删除，不提供用户历史记录功能。 +- 进程重启后无法确认的非终态 Operation 统一恢复为 `interrupted`。 +- Event 可以重复、延迟或丢失；序列缺口或重启后必须重新查询 Execution Snapshot + 和 Engineering Snapshot。 + +## 3. 一致性、锁和 Revision + +- Workspace Revision 是唯一单调序列：创建为 1，配置/结构更新、rerun preparation、 + Layout 保存和每个 Step Commit 各推进一次。 +- Runtime 与 CLI 使用同一个 Workspace Ownership Lock（Workspace 同级的 + `.lock`）。Runtime fail-fast 返回 `workspace_busy`；同一 Runtime + 已有冲突 Operation 返回 `operation_conflict`。CLI 保持现有阻塞等待行为。 +- 同时触及 Project Manifest 和 Workspace 的 API 固定锁顺序： + `Project Manifest Lock -> Workspace Ownership Lock`。 +- Workspace-only 只取得 Workspace Lock；Manifest-only 只取得 Manifest Lock。 +- Runtime `workspace.open` 是短暂的 Open Transaction：校验 Snapshot、Bindings、 + Ledger 和恢复状态后释放锁。Operation 或 Layout Edit 才持有长生命周期锁。 +- Layout Edit 从 `layout.edit.begin` 持锁到 save/discard/close/process loss；save + 先提交 Revision 与 Snapshot，再释放锁；进程丢失不发布未提交编辑。 +- Bindings 只保存为机器本地绝对路径，不能写入 Descriptor/Manifest、推进 Revision + 或单独生成 Snapshot。 + +## 4. CLI 合同 + +### 写路径 + +下列 CLI 写操作全部改为直接调用 ECC Engine 的事务和提交路径： + +- Workspace 创建、Project 管理的 create/update 和 `ecc workspace refresh`； +- `ecc param set/unset --workspace`； +- rerun preparation、Layout 保存和 Flow 执行； +- `ecc run --overwrite`。 + +CLI 继续负责参数优先级、命令名称、阻塞锁、`--overwrite` 临时备份、输出记录和 +错误退出码。CLI 不调用 ECC Runtime，也不保留自己的 Snapshot/参数回滚实现。 +Project Manifest 与 Workspace 的联合更新使用现有 ECC API staging、Manifest 原子 +更新和失败回滚，不引入 two-phase commit。 + +CLI 解析 `--resume`、`--from`、`--to`、`--only`、`--force` 和 preset，解析为规范化 +有序 `step_ids` 后传给 Engine `ExecutionPlan`。Engine 负责依赖验证、执行、提交和 +Revision；Engine 不认识 CLI flag 名称。 + +Engine `ExecutionResult` 至少提供 `succeeded`、最终状态、可选 `step_id`、按执行顺序 +排列的 `executed_steps` 和可选 `failed_step`。CLI 只把它映射为现有 +`StepRunResult`/`CommandResult`，不重新扫描 `flow.json` 推断结果。 + +没有显式 rerun selector 且目标已经完成时，Engine 返回成功 no-op：不创建 Operation、 +不写 Workspace、不推进 Revision、不刷新 Engineering Snapshot。显式 `--force`、 +`--from`、`--only` 等仍然是 mutation。 + +### 读路径 + +第一阶段保留现有查询输出和实现边界： + +- `status` 直接读取 `home/flow.json`； +- `report step` 直接读取 Flow 和声明的 Step artifacts； +- `signoff inspect` 保留现有 Signoff 查询结果； +- `report qor/checklist/summary` 保留现有报告构建和写入默认或显式的 + `signoff/` 报告文件。 + +这些命令都不能创建或刷新 Engineering Snapshot。需要加载 Workspace 对象的报告和 +Signoff 查询必须使用 `load_workspace(read_only=True)`，不能触发迁移、事务恢复、 +配置文件名重写或 Workspace 日志写入；报告目标文件是唯一有意写入。 + +## 5. ECC 实施步骤 + +### Phase 0：基线与冻结 + +1. 固定 ECC git commit、Studio gitlink、当前磁盘文件 schema 和现有 CLI/RPC 测试。 +2. 建立跨组件 fixture：ECC 创建的 Workspace、Studio 创建的 Workspace、已有 + Snapshot、Snapshot-less Workspace、无效 Snapshot、revision 冲突和 lock 占用。 +3. 记录现有 `ecos/runtime-adapter` 的模块、测试、依赖、启动脚本和打包资源清单。 + +### Phase 1：ECC ownership migration（保持线协议兼容） + +1. 将 `ecos/runtime-adapter/ecos_runtime_adapter/*.py` 直接移动到 ECC 的 + `chipcompiler/runtime/`，按现有模块拆分重写 import；测试移动到 `test/runtime/`。 +2. 把 `jsonrpcserver>=5.0.9,<6` 移入 ECC `pyproject.toml`，删除 Adapter + requirements、wrapper、重复 package、重复测试和独立 Adapter binary。 +3. 保留 JSON-RPC method、字段、错误码、事件和内部旧 aliases，确保旧 Studio 与 + 新 ECC 的 parity；普通 CLI help 隐藏 `rpc`。 +4. 在 Engine 中实现统一 Workspace Ownership Lock、Manifest->Workspace 锁顺序、 + Atomic Engineering Commit、统一 Revision 和 Stable Domain Error 映射。 +5. 让 Engine 的 create/update/configuration/rerun/layout/execute 共享同一提交路径； + Flow Step Commit 必须先完成 Flow + Snapshot，再通知 Runtime。 +6. 将所有 CLI Workspace 写操作接入 Engine；保留只读命令输出和报告文件输出。 +7. 将 report/Signoff 的 Workspace hydration 改为 `read_only=True`，补充无副作用断言。 +8. 为 `ExecutionPlan`、`ExecutionResult`、no-op、`workspace_busy`、 + `operation_conflict`、revision conflict、overwrite rollback 和 Snapshot failure + 补齐 Engine/CLI/runtime 测试。 +9. 运行 ECC Ruff、全量 Python tests、PyInstaller 和 framed-stdio Runtime smoke。 + +### Phase 1.5：Studio 切换与跨仓库验证 + +1. 先合并并发布 ECC Phase 1，再更新 Studio ECC gitlink。 +2. Electron 启动命令切换为 pinned `ecc rpc serve --stdio --persistent-db`，删除 + Adapter binary staging、resource assertions 和独立 requirements 安装。 +3. 保留现有 Workspace Runtime/Control Runtime 路由、事件、取消、恢复、Layout 编辑 + 和 Project Snapshot reader；只替换后端进程来源。 +4. 验证 ECC 生成 Workspace 可被 Studio 打开，Studio 生成 Workspace 可继续被 CLI + 执行；验证 Project Comparison 直接读取持久化 Snapshot。 +5. 运行 Studio frozen install、typecheck、lint/format、Electron/Renderer tests、 + build、desktop smoke 和 parent packaging build。 + +### Phase 2：协议清理（必须完成） + +Phase 2 只有在 Phase 1 parity、打包和跨组件 smoke 全部通过后开始，不能无限期保留 +兼容分支： + +1. 强制 Control Runtime 和每个 Workspace Runtime 在任何业务命令前完成 `rpc.hello`。 +2. 使用 private protocol v2，返回 `protocolVersion`、ECC package version 和明确 + capabilities；缺失或不匹配在任何写操作前返回 `runtime_incompatible`。 +3. 删除过渡性的 `workspace.engineering_snapshot` RPC，统一为 Electron 直接读取 + `home/engineering-snapshot.json`。 +4. 删除 `adapterVersion`、`runtime.adapter.v1`、Adapter-named capability、旧 DTO、 + 旧字段 aliases 和兼容错误映射。 +5. 删除 `ecos-ecc-runtime-adapter` 名称、wrapper、资源断言和所有重复实现；CI 以 + grep/contract test 阻止旧名称回归。 +6. 保留一个 Snapshot validator、一个 Runtime capability vocabulary、一个 private + protocol version 和一个 canonical read path；ECC 与 Studio 作为一个 pinned update + 发布。 + +## 6. 失败与恢复语义 + +- Snapshot 提交失败：返回 `engineering_snapshot_commit_failed`，保留前一 Snapshot， + Flow Operation 失败，不发送 committed event。 +- Snapshot 缺失/损坏/版本不支持：查询返回稳定的 unavailable/invalid reason，不回退 + 为 `unstart`，不自动写文件；显式 refresh、run 或 overwrite 才能修复。 +- Runtime 无法确认进程被终止时：Ledger 恢复为 `interrupted`，不能伪装成 + `cancelled` 或 succeeded。 +- 普通 cancel 只在 Flow Step 边界生效；运行中的 EDA 工具继续完成当前 Step。 +- Force Quit 才会硬终止 Runtime，最多进行 3 秒清理，然后依靠 Ledger 恢复。 +- 已有 `--overwrite` 的临时备份只用于失败恢复；成功后删除，不形成用户历史归档。 + +## 7. 验收与合并门禁 + +必须同时满足： + +- ECC：Ruff、全量 Python suite、Engine/Runtime/CLI 聚焦测试、PyInstaller、framed + stdio smoke（hello、create/open、Execution Snapshot、Flow、cancel、recovery）。 +- Studio：frozen install、typecheck、lint/format、Electron/Renderer tests、build、 + desktop smoke、parent packaging build。 +- 跨组件：ECC->Studio Workspace、Studio->CLI Workspace、overwrite rollback、 + Snapshot missing/invalid、revision conflict、Workspace Busy、background run、 + Force Quit recovery、Layout Edit lock lifetime、报告输出兼容。 +- 打包产物：只包含一个 `ecc` executable，不包含 Adapter binary、wrapper 或第二套 + Python requirements。 +- 只读门禁：status/report/signoff 查询不迁移、不恢复、不追加 Workspace 日志；报告 + 命令只写其声明的报告目标。 +- 事件门禁：事件重复、延迟、丢失和序列缺口都能通过重新查询 Snapshot 恢复，正确性不 + 依赖 Renderer ACK。 + +## 8. 不在本方案内 + +- 不做旧 Workspace 或旧 Snapshot schema 的隐式迁移和 raw-file fallback。 +- 不保留第二个 Adapter forwarding package、symlink 或独立 runtime binary。 +- 不把 CLI 只读命令整体迁移为 Snapshot reader；后续若统一 read model，另立变更。 +- 不实现用户可见的 Operation 历史、无限 Ledger、Snapshot 历史时间旅行或通用 Artifact + 浏览器。 +- 不重做 Studio UI、QoR 算法、Signoff 判定、Project Manifest 领域归属或 ECC-FE。 + +## 9. 决策记录 + +本方案对应 Studio 架构 ADR-0033（ECC owns the private Studio runtime）以及 ECC +Headless Engine 既有 glossary。旧的 Studio-only Headless Engine Contract 文档只保留 +作历史记录，不能覆盖本文确认的 CLI 共用 Engine、Snapshot 边界和 Phase 2 清理要求。 diff --git a/docs/specification/glossary-headless-engine.md b/docs/specification/glossary-headless-engine.md new file mode 100644 index 00000000..e8c3af42 --- /dev/null +++ b/docs/specification/glossary-headless-engine.md @@ -0,0 +1,89 @@ +# Glossary: Headless Engine and Workspace Lifecycle + +> **Historical vocabulary.** This glossary was written for the earlier +> Studio-only Headless Engine Contract. The current CLI/Studio ownership, +> Snapshot boundary and protocol phases are defined in +> [ECC Runtime Consolidation v1](ecc-runtime-consolidation-plan.zh-CN.md). + +Terms used by the Studio-owned Headless Engine Contract introduced in +PR 275. Implementation spec: +[headless-engine-contract.md](headless-engine-contract.md). +CLI vocabulary (`ecc.toml`, `--workspace`, `ecc run`) is unchanged. + +## Workspace + +A directory that holds one flow execution: `origin/`, `home/`, `config/`, +step directories, and logs. Identified on disk by its path. Studio also +identifies it by `workspaceId` inside the Engineering Snapshot. + +## Workspace Spec + +The Studio input that describes what a Workspace should contain: design +inputs, PDK binding, flow range, and parameters. Validated by +`validate_workspace_spec` before create or structural update. Parameter +applicability uses `normalize_flow_step()`, the same canonical step names +as flow construction. + +## Bindings + +Resolved host paths for Spec roles (RTL, filelist, PDK files, and so on). +The Spec names roles; Bindings point at files on this machine. + +## Engineering Snapshot + +`home/engineering-snapshot.json`. Studio's committed view of a Workspace: +`workspaceId`, `workspaceRevision`, analysis, QoR assessment, signoff +assessment, and artifact index. CLI-created workspaces do not have this +file in this PR. + +## Workspace Revision + +Monotonic integer in the Engineering Snapshot. Studio configuration and +structural updates are revision-aware: a stale expected revision is a +conflict, not a silent overwrite. + +## Headless Engine Contract + +The Studio Python APIs exported by `chipcompiler.engine`: create, +structural update, configuration update, query, `execute`, Engineering +Snapshot reads, and signoff inspect/export. Analysis builders and QoR +scoring stay internal. This PR does not make the CLI a client of that +contract. + +## Workspace Ownership Lock + +Sibling file `{workspace-name}.lock` next to the Workspace directory. It +survives directory replacement. Studio create / update / configuration +and CLI run / CLI workspace-param all use this filename, but this PR does +not treat CLI and Studio as one locking domain. + +## Configuration Transaction + +Studio-only disk backup used by `update_workspace_configuration`. It +restores params, flow, snapshot, and related files if a configuration +update fails or the process dies mid-commit. It runs only while the +Workspace Ownership Lock is already held. It is not a second lock file, +and it is not CLI parameter rollback. + +## CLI Parameter Rollback + +In-memory bytes snapshot used by `ecc param --workspace`. Process-local +only. Pre-existing CLI behavior; not merged into the Configuration +Transaction in this PR. + +## `execute` + +Studio entry point that runs an `EngineFlow` with an `ExecutionPlan` and +commits the Engineering Snapshot after completed steps. CLI does not call +it in this PR. + +## `EngineFlow` + +The existing flow runner. CLI `ecc run` calls it directly. Studio +reaches it through `execute`. + +## Read-only Workspace Load + +`load_workspace(..., read_only=True)`. Studio query APIs use this so +inspection cannot migrate configs, create `home/` files, or write logs. +Mutating Studio APIs load a writable Workspace. diff --git a/docs/specification/headless-engine-contract.md b/docs/specification/headless-engine-contract.md new file mode 100644 index 00000000..9c9154f8 --- /dev/null +++ b/docs/specification/headless-engine-contract.md @@ -0,0 +1,290 @@ +# Headless Engine Contract (PR 275 follow-through) + +> **Historical document.** This earlier spec described a Studio-only Engine +> contract and explicitly left CLI writes on the old path. It is superseded by +> [ECC Runtime Consolidation v1](ecc-runtime-consolidation-plan.zh-CN.md), which +> makes ECC Engine the shared mutation path for CLI and Studio while keeping CLI +> read commands and report output compatible. Do not use the old CLI scope or +> Snapshot ownership statements below as the current contract. + +Status: ready for implementation. Scope is the Studio-owned Headless Engine +Contract already sketched in this branch; this spec closes the gaps decided +during review. Vocabulary follows +[glossary-headless-engine.md](glossary-headless-engine.md). Binding ADR: +[adr-headless-engine-contract.md](adr-headless-engine-contract.md). + +## Problem Statement + +Studio needs a process-local Python contract to create, inspect, configure, +and run a Workspace after the in-process RPC runtime is gone. The branch +already exposes lifecycle and configuration APIs, but they are not yet a +single contract: configuration uses a second lock file, query APIs can +migrate the Workspace, create can delete a directory it did not create, +execution is not the documented Studio entry, and the public engine surface +does not export snapshot or signoff reads. + +CLI already has its own ownership lock and in-memory parameter rollback. +Those must keep working. This spec does not ask CLI to adopt the Headless +Engine Contract. + +## Solution + +Finish the Studio-only Headless Engine Contract on this branch: + +- One Workspace Ownership Lock for every Studio mutation. +- Configuration Transaction as disk backup only, never a second lock. +- Query APIs load a Workspace read-only and never migrate. +- Studio execution goes through `execute` and commits the Engineering + Snapshot. +- Create failure deletes only a directory this call created. +- The engine package exports snapshot reads and signoff inspect/export, not + scoring internals. +- Workspace Spec parameter applicability uses canonical flow step names. + +CLI `ecc run` and `ecc param --workspace` stay on their existing paths. + +## User Stories + +1. As a Studio engineer, I want to create a Workspace from a Workspace Spec + and Bindings, so that the GUI does not speak RPC. +2. As a Studio engineer, I want a successful create to write an Engineering + Snapshot at revision 1, so that later updates have a committed identity. +3. As a Studio engineer, I want repeating the same create command id to + return the existing Workspace, so that retries are safe. +4. As a Studio engineer, I want create to fail with `workspace_exists` when + the directory already belongs to someone else, so that another Workspace + is never deleted. +5. As a Studio engineer, I want create to remove only the directory this + call created if setup fails after mkdir, so that half-written trees are + cleaned up without touching pre-existing data. +6. As a Studio engineer, I want structural update to replace design, PDK, + flow range, and parameters atomically, so that a failed update does not + leave a mixed Workspace. +7. As a Studio engineer, I want structural update to require the expected + Workspace Revision, so that two Studio sessions cannot silently overwrite + each other. +8. As a Studio engineer, I want a matching command id on structural update + to be idempotent, so that a retried request does not bump the revision + twice. +9. As a Studio engineer, I want a stale expected revision to fail with + `revision_conflict` without creating a missing snapshot, so that conflict + handling cannot invent identity. +10. As a Studio engineer, I want configuration update to change only + parameters (not design or PDK structure), so that geometry and PDK + edits stay on the structural-update path. +11. As a Studio engineer, I want configuration update to take the same + Workspace Ownership Lock as create and structural update, so that + Studio cannot configure a Workspace while another Studio mutation + replaces it. +12. As a Studio engineer, I want configuration update to keep a disk backup + and restore it if refresh or invalidate fails, so that params, flow, + and snapshot stay consistent. +13. As a Studio engineer, I want configuration crash recovery to run only + while the Workspace Ownership Lock is already held, so that restore + cannot race another Studio mutation. +14. As a Studio engineer, I want no `.configuration.lock` sibling file, so + that lock ownership is one filename and one story. +15. As a Studio engineer, I want `describe_workspace_binding_requirement` + to report family, version, and mode without writing files, so that + opening a Workspace in the GUI is inspection, not migration. +16. As a Studio engineer, I want `assess_execution_readiness` to say whether + Bindings are enough to run, without creating `home/` files or logs. +17. As a Studio engineer, I want `read_workspace_configuration` and + `read_step_configuration` from a directory to be read-only, so that + viewing settings cannot rewrite configs. +18. As a Studio engineer, I want a missing `home.json` or checklist to stay + missing after a query, so that inspection never materializes defaults. +19. As a Studio engineer, I want `execute` to run a full range or a single + step, so that the GUI has one run entry. +20. As a Studio engineer, I want `execute` to commit the Engineering + Snapshot after completed steps, so that QoR and artifacts in the GUI + match disk. +21. As a Studio engineer, I want a failed `execute` to keep the same + blocking semantics as `EngineFlow` (stop vs continue), so that run + behavior does not fork. +22. As a Studio engineer, I want to read the committed Engineering Snapshot + from the engine package, so that the GUI does not import snapshot + internals. +23. As a Studio engineer, I want to inspect the current Signoff Assessment + without persisting a checklist, so that signoff views are safe. +24. As a Studio engineer, I want to export a signoff archive through the + engine package, so that package creation stays on the same contract. +25. As a Studio engineer, I want analysis builders and QoR scoring to stay + internal, so that the GUI reads committed snapshot data instead of + recomputing scores. +26. As a Studio engineer, I want Workspace Spec validation to accept the + same step spellings as flow construction, so that `synth` and + `Synthesis` are not two applicability worlds. +27. As a Studio engineer, I want a parameter that does not apply to the + requested flow range to be a structured spec issue, so that the GUI can + show the field path. +28. As a Studio engineer, I want unused runtime flow helpers gone, so that + the only Studio run path is `execute` over `EngineFlow`. +29. As a CLI user, I want `ecc run` to keep calling `EngineFlow` directly, + so that CLI workspaces do not suddenly grow an Engineering Snapshot. +30. As a CLI user, I want `ecc param --workspace` to keep in-memory + rollback and the sibling ownership lock, so that parameter edits still + undo on refresh failure in the same process. +31. As a CLI user, I want CLI and Studio to share a lock filename without + this change promising cross-product exclusion, so that CLI behavior + stays what it was on main. +32. As a CLI user, I want `ecc signoff inspect` unchanged, so that CLI + inspection is not silently rewritten as a Studio query. +33. As a reviewer, I want the engine package export list to be the Studio + contract, so that deep imports are not required for snapshot or + signoff. +34. As a reviewer, I want tests to speak only through public engine APIs + and on-disk Workspace files, so that lock-file names inside the + transaction helper can change without rewriting the suite. +35. As a Studio engineer, I want Bindings applied at execute/update time, + so that a Workspace can be copied to another machine and rebound. +36. As a Studio engineer, I want a PDK root mismatch to be + `pdk_binding_mismatch`, so that the GUI can ask the user to rebind + instead of running the wrong PDK. +37. As a Studio engineer, I want step configuration update to invalidate + only from the target step, so that earlier successful steps are not + thrown away. +38. As a Studio engineer, I want configuration update to reject design or + PDK sections, so that the GUI cannot hide a structural change inside + a parameter save. +39. As a Studio engineer, I want `execute` to apply Bindings before + running, so that a default-PDK Workspace can resolve files on this + host. +40. As a maintainer, I want `execute` to call `EngineFlow` methods + directly, so that runtime signature probing is not part of the + contract. + +## Implementation Decisions + +- The Headless Engine Contract is Studio-only for this change. CLI does not + call create-from-spec, configuration update, or `execute`. +- Studio mutations (create, structural update, configuration update) all + acquire the Workspace Ownership Lock: the sibling `{workspace-name}.lock` + next to the Workspace directory. That filename already exists for CLI + runs; this change does not add a second Studio ownership lock. +- The Configuration Transaction keeps its disk backup under the Workspace + `home/` directory and still restores on failure or crash. It must not + open or flock a `.configuration.lock` file. Backup, commit, rollback, and + crash recovery run only while the caller already holds the Workspace + Ownership Lock. +- Writable `load_workspace` (and therefore transaction crash recovery + hooked off load) is allowed on mutating Studio paths. Query paths must + not go through that load. +- Query APIs are `describe_workspace_binding_requirement`, + `assess_execution_readiness`, `read_workspace_configuration_from_directory`, + `read_step_configuration_from_directory`, Engineering Snapshot reads from + a directory, and signoff inspect. Each loads with `read_only=True` and + must not migrate filenames, create `home/` files, rebuild a persisted + checklist, or append logs. +- In-memory Workspace objects passed into `read_workspace_configuration` / + `read_step_configuration` are already loaded; those helpers must not + persist side effects either. +- `create_workspace_from_spec` treats an existing target as + `workspace_exists`, except when `command_id` matches a completed create + (idempotent retry). Failure cleanup may `rmtree` only if this invocation + created the target directory. A pre-existing directory is never removed. +- Structural update may keep using a staging directory that this call + created; that staging directory is owned by the call and may be deleted + on failure. +- Studio execution uses `execute` with an `ExecutionPlan`. The default + observer commits the Engineering Snapshot after completed steps. CLI + continues to construct `EngineFlow` and call `run_steps` / `run_step`. +- `execute` calls `EngineFlow.run_steps` / `run_step` directly with `rerun` + and observer arguments. No `inspect.signature` probing. +- Delete unused runtime-extraction helpers that have no production callers + (`workspace_step_from_flow`, DB-init-for-step, success/state adapters). + Tests that only needed `build_flow_for_workspace` should construct + `EngineFlow` themselves. +- `chipcompiler.engine` exports, in addition to the current lifecycle / + configuration / `execute` surface: Engineering Snapshot read helpers and + signoff inspect/export. It does not export analysis builders or + `score_qor`. +- Snapshot *writes* stay internal to create, update, configuration, and + `execute`. Studio reads committed snapshots; it does not call create / + ensure / invalidate snapshot helpers as public API. +- Workspace Spec parameter applicability uses `normalize_flow_step()` for + both the requested flow steps and each schema's `applies` value. The + handwritten alias map (`synthesis→synth`, and the rest) is deleted. + Parameters with `applies == "all"` remain applicable to any range. +- Keep CLI in-memory parameter rollback and CLI use of the sibling lock. + Do not route CLI param through the Configuration Transaction. +- Keep CLI `sys.modules` manifest aliases and CLI signoff inspect as they + are. +- Do not split the metrics module in this change. +- Typed errors stay `WorkspaceLifecycleError` with stable `code` values + already used by create/update/configuration (`workspace_exists`, + `revision_conflict`, `workspace_spec_invalid`, `pdk_binding_missing`, + `pdk_binding_mismatch`, `workspace_structure_change_requires_update`, + and the existing snapshot/config invalid codes). + +## Testing Decisions + +Good tests assert observable contract behavior: return values, exception +codes, Engineering Snapshot identity/revision, and whether files on disk +appeared, changed, or stayed absent. They do not assert the Configuration +Transaction's internal lock filename, backup directory name, or helper +class. They do not import analysis/scoring modules to prove the public +contract. + +Highest seam: `chipcompiler.engine` public APIs plus the Workspace +directory they mutate. That is the existing `test/engine/` seam +(`test_workspace_spec.py`, `test_workspace_configuration.py`, +`test_execution.py`, `test_signoff_assessment.py`). Prefer extending those +modules over adding a new test package. + +Cover: + +- Create idempotent retry vs `workspace_exists`; create failure does not + delete a pre-existing directory (the call did not create it). +- Configuration update and create/structural update cannot interleave: a + holder of the Workspace Ownership Lock blocks the other Studio mutation + (same style as existing CLI lock-wait tests, but through engine APIs). +- After removing the independent configuration lock, crash-recovery tests + still restore params/snapshot when the process dies mid-commit; recovery + is observed via a later mutating API or a read of committed files, not + via the transaction lock path. +- Query APIs with missing `home.json` / checklist / log dir / missing PDK + root do not recreate those paths. Extend the existing read-only + configuration test to `describe`, `assess`, and step-configuration read. +- `execute` commits snapshot after a completed step; already present in + `test_execution.py` and remains required. +- Spec applicability: a parameter whose canonical `applies` step is + outside the requested range is a structured issue; a non-canonical + spelling that `normalize_flow_step()` maps into the range is allowed. +- Engine package export list includes snapshot reads and signoff + inspect/export, and does not include `score_qor` or analysis builders. +- Existing revision/idempotency/rollback tests in workspace spec and + configuration suites still pass. + +CLI suites are regression guards, not the place to add Studio contract +tests. Do not add tests that require CLI to write an Engineering Snapshot. + +## Out of Scope + +- Wiring CLI create, refresh, param, or run through the Headless Engine + Contract. +- Making CLI and Studio one locking domain (cross-product mutual + exclusion). +- Replacing CLI Parameter Rollback with the Configuration Transaction. +- Changing CLI manifest import aliases. +- Making CLI signoff inspect a read-only load. +- Splitting the metrics module. +- Exporting analysis builders or QoR scoring from the engine package. +- Teaching CLI `ecc run` to write an Engineering Snapshot. +- Full-suite, PyInstaller, and Nix runs as spec items (merge checklist + only). + +## Further Notes + +`main` already had two products. CLI serialized with the sibling ownership +lock and rolled back workspace params from in-memory bytes. RPC/Studio +serialized with a process-local session lock and had no file-backed +configuration transaction. This spec does not merge those histories; it +gives Studio a file-backed contract after RPC removal. + +The independent `.configuration.lock` file is new on this branch, not +inherited from `main`. Removing it is restoring a single ownership lock, +not deleting CLI behavior. + +A later change may adopt the Headless Engine Contract in CLI. That is a +separate decision. From d29589f8aaffaa36273b4751506d55c908e8b8fb Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Tue, 8 Sep 2026 23:38:43 +0800 Subject: [PATCH 02/43] fix(cli): align parameter filters with flow steps --- chipcompiler/cli/command_handlers/param.py | 30 +++++++++++++- .../cli/command_handlers/workspace_params.py | 15 ++++--- test/cli/params/test_commands.py | 39 +++++++++++++++++++ test/cli/params/test_workspace_commands.py | 31 +++++++++++++++ 4 files changed, 108 insertions(+), 7 deletions(-) diff --git a/chipcompiler/cli/command_handlers/param.py b/chipcompiler/cli/command_handlers/param.py index 8a56b08f..0fe62d6f 100644 --- a/chipcompiler/cli/command_handlers/param.py +++ b/chipcompiler/cli/command_handlers/param.py @@ -11,6 +11,7 @@ validate_pdk_target, validate_value, ) +from chipcompiler.rtl2gds import get_flow_builders, normalize_flow_step from chipcompiler.utility.file import write_text_atomic @@ -29,6 +30,20 @@ def _manifest_mode_error(ctx: CommandContext) -> CommandResult | None: return None +def _parameter_flow_error(schema, ctx: CommandContext) -> CommandResult | None: + preset = getattr(ctx.config, "flow_preset", "") + builder = get_flow_builders().get(preset) + if builder is None or schema.applies == "all" or schema.pdk_target is not None: + return None + flow_steps = {normalize_flow_step(step).casefold() for step, _tool, _state in builder()} + if normalize_flow_step(schema.applies).casefold() in flow_steps: + return None + return CommandResult.err( + [error_record("parameter_not_in_flow", param=schema.param, preset=preset)], + exit_code=1, + ) + + def param_list(args, ctx: CommandContext) -> CommandResult: if getattr(args, "workspace", None) is not None: from chipcompiler.cli.command_handlers import workspace_params @@ -45,12 +60,17 @@ def param_list(args, ctx: CommandContext) -> CommandResult: resolved, _ = resolve_parameters(toml_overrides=toml_overrides) project = ctx.project - selected_step = (getattr(args, "step", None) or "").casefold() + selected_step = normalize_flow_step(getattr(args, "step", None) or "").casefold() show_all = bool(getattr(args, "all", False)) records = [] for rp in resolved: s = rp.schema - if selected_step and selected_step not in {s.group.casefold(), s.applies.casefold()}: + schema_steps = { + normalize_flow_step(s.group).casefold(), + normalize_flow_step(s.applies).casefold(), + } + global_at_first_step = s.applies == "all" and selected_step == "synthesis" + if selected_step and selected_step not in schema_steps and not global_at_first_step: continue if not selected_step and not show_all and s.has_direct_target and not rp.is_explicit: continue @@ -160,6 +180,9 @@ def param_set(args, ctx: CommandContext) -> CommandResult: ], exit_code=1, ) + flow_error = _parameter_flow_error(schema, ctx) + if flow_error is not None: + return flow_error try: value = parse_value(raw_value, schema) @@ -248,6 +271,9 @@ def param_unset(args, ctx: CommandContext) -> CommandResult: ], exit_code=1, ) + flow_error = _parameter_flow_error(schema, ctx) + if flow_error is not None: + return flow_error config_path = _find_config_path(ctx.project_dir) if config_path is None: diff --git a/chipcompiler/cli/command_handlers/workspace_params.py b/chipcompiler/cli/command_handlers/workspace_params.py index 8c576c77..673de650 100644 --- a/chipcompiler/cli/command_handlers/workspace_params.py +++ b/chipcompiler/cli/command_handlers/workspace_params.py @@ -12,6 +12,7 @@ workspace_param_step, workspace_param_value, ) +from chipcompiler.rtl2gds import normalize_flow_step def param_set(args, ctx: CommandContext) -> CommandResult: @@ -62,15 +63,19 @@ def param_list(args, ctx: CommandContext) -> CommandResult: if workspace_error is not None: return workspace_error overrides = {record["key"] for record in workspace_param_diff(workspace)} - selected_step = (args.step or "").casefold() + selected_step = normalize_flow_step(args.step or "").casefold() + flow_steps = workspace.flow.data.get("steps", []) + first_step = normalize_flow_step(flow_steps[0]["name"]).casefold() if flow_steps else "" records = [] for schema in list_schemas(): if schema.pdk_target is not None or (not args.all and schema.param not in overrides): continue - if selected_step and selected_step not in { - schema.group.casefold(), - schema.applies.casefold(), - }: + schema_steps = { + normalize_flow_step(schema.group).casefold(), + normalize_flow_step(schema.applies).casefold(), + } + global_at_first_step = schema.applies == "all" and selected_step == first_step + if selected_step and selected_step not in schema_steps and not global_at_first_step: continue try: value = workspace_param_value(workspace, schema) diff --git a/test/cli/params/test_commands.py b/test/cli/params/test_commands.py index abeeac71..015e7836 100644 --- a/test/cli/params/test_commands.py +++ b/test/cli/params/test_commands.py @@ -26,6 +26,20 @@ def test_param_list_step_includes_config_schemas(self, tmp_path, capsys, create_ assert rc == 0 assert "cts.skew_bound" in capsys.readouterr().out + def test_param_list_synthesis_includes_global_parameters( + self, tmp_path, capsys, create_cli_project, plain_records + ): + project_dir = create_cli_project() + rc = cli_main.run( + ["param", "list", "--step", "synthesis", "--project", project_dir, "--plain"] + ) + + assert rc == 0 + assert [record["param"] for record in plain_records(capsys.readouterr().out)] == [ + "design.frequency_mhz", + "flow.run_analysis", + ] + class TestParamShow: def test_param_show_known_key(self, tmp_path, capsys, create_cli_project): @@ -90,6 +104,31 @@ def test_param_set_rejects_invalid_value(self, tmp_path, capsys, create_cli_proj rc = cli_main.run(["param", "set", "place.target_density", "1.5", "--project", project_dir]) assert rc == 1 + def test_param_set_rejects_parameter_outside_project_flow( + self, tmp_path, capsys, create_cli_project, plain_records + ): + project_dir = create_cli_project() + config_path = tmp_path / "gcd" / "ecc.toml" + config_path.write_text( + config_path.read_text().replace('preset = "rtl2gds"', 'preset = "syn_sta"') + ) + + rc = cli_main.run( + [ + "param", + "set", + "place.target_density", + "0.65", + "--project", + project_dir, + "--plain", + ] + ) + + assert rc == 1 + assert plain_records(capsys.readouterr().out)[0]["error"] == "parameter_not_in_flow" + assert "target_density" not in config_path.read_text() + def test_param_set_accepts_negative_value( self, tmp_path, capsys, create_cli_project, plain_records ): diff --git a/test/cli/params/test_workspace_commands.py b/test/cli/params/test_workspace_commands.py index 684988f2..cdfa0aa9 100644 --- a/test/cli/params/test_workspace_commands.py +++ b/test/cli/params/test_workspace_commands.py @@ -200,3 +200,34 @@ def test_workspace_param_list_honors_step_filter( assert plain_records(capsys.readouterr().out) == [ {"param": "list", "status": "clean", "workspace": "baseline"} ] + + +def test_workspace_param_list_matches_first_step_configuration( + capsys, create_cli_project, monkeypatch, plain_records +): + project_dir = create_cli_project() + workspace_dir = Path(project_dir) / "baseline" + _write_manifest(project_dir) + workspace = _workspace(workspace_dir) + monkeypatch.setattr("chipcompiler.data.load_workspace", lambda _path: workspace) + + rc = cli_main.run( + [ + "param", + "list", + "--workspace", + "baseline", + "--step", + "synthesis", + "--all", + "--project", + project_dir, + "--plain", + ] + ) + + assert rc == 0 + assert [record["param"] for record in plain_records(capsys.readouterr().out)] == [ + "design.frequency_mhz", + "flow.run_analysis", + ] From c658a906f208272db71bdc077dc7562eb2d228cc Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 00:16:36 +0800 Subject: [PATCH 03/43] feat(workspace): share project and workspace contracts --- chipcompiler/cli/project/manifest_write.py | 64 ++- chipcompiler/engine/__init__.py | 6 + chipcompiler/engine/pdk_binding.py | 64 +++ chipcompiler/engine/workspace_lifecycle.py | 144 +++++++ chipcompiler/engine/workspace_spec.py | 431 +++++++++++++++++++ chipcompiler/project/__init__.py | 15 + chipcompiler/project/api.py | 279 ++++++++++++ test/engine/test_workspace_spec.py | 151 +++++++ test/fixtures/workspace_spec/input/gcd.v | 3 + test/fixtures/workspace_spec/input/sources.f | 1 + test/fixtures/workspace_spec/invalid.json | 67 +++ test/fixtures/workspace_spec/pdk/.keep | 1 + test/fixtures/workspace_spec/valid.json | 68 +++ test/project/__init__.py | 0 test/project/test_manifest.py | 53 +++ 15 files changed, 1327 insertions(+), 20 deletions(-) create mode 100644 chipcompiler/engine/pdk_binding.py create mode 100644 chipcompiler/engine/workspace_lifecycle.py create mode 100644 chipcompiler/engine/workspace_spec.py create mode 100644 chipcompiler/project/__init__.py create mode 100644 chipcompiler/project/api.py create mode 100644 test/engine/test_workspace_spec.py create mode 100644 test/fixtures/workspace_spec/input/gcd.v create mode 100644 test/fixtures/workspace_spec/input/sources.f create mode 100644 test/fixtures/workspace_spec/invalid.json create mode 100644 test/fixtures/workspace_spec/pdk/.keep create mode 100644 test/fixtures/workspace_spec/valid.json create mode 100644 test/project/__init__.py create mode 100644 test/project/test_manifest.py diff --git a/chipcompiler/cli/project/manifest_write.py b/chipcompiler/cli/project/manifest_write.py index c1d422db..bc90e154 100644 --- a/chipcompiler/cli/project/manifest_write.py +++ b/chipcompiler/cli/project/manifest_write.py @@ -96,16 +96,51 @@ def build_manifest_document( ) -> dict: """Assemble a schema-v1 manifest for a virgin project's first run.""" now = _now_iso() - name = os.path.basename(os.path.normpath(project_dir)) or "project" - document: dict[str, Any] = { + document = build_project_document( + project_dir, + design_name=design_name, + base_design=base_design, + now=now, + ) + document["workspaces"] = [ + manifest_workspace_entry( + workspace_id, + name=design_name, + workspace_path=workspace_path, + start_step=start_step, + end_step=end_step, + status=status, + now=now, + ) + ] + document["qor_baseline"] = { + "workspace_id": workspace_id, + "reason": "Default project QoR baseline", + } + return document + + +def build_project_document( + project_dir: str, + *, + design_name: str, + base_design: dict, + name: str | None = None, + now: str | None = None, + mpc: dict | None = None, +) -> dict[str, Any]: + """Assemble a schema-v1 Project Manifest without a Workspace.""" + timestamp = now or _now_iso() + project_name = name or os.path.basename(os.path.normpath(project_dir)) or "project" + return { "schema_version": 1, - "project_id": f"proj_{_slugify(name)}", - "name": name, + "project_id": f"proj_{_slugify(project_name)}", + "name": project_name, "design_name": design_name, "description": "", "root_path": project_dir, - "created_at": now, - "updated_at": now, + "created_at": timestamp, + "updated_at": timestamp, "base_design": { **{key: value for key, value in base_design.items() if key != "parameters" and value}, "parameters": _record(base_design.get("parameters")), @@ -114,22 +149,11 @@ def build_manifest_document( ], }, "objectives": json.loads(json.dumps(DEFAULT_OBJECTIVES)), - "workspaces": [ - manifest_workspace_entry( - workspace_id, - name=design_name, - workspace_path=workspace_path, - start_step=start_step, - end_step=end_step, - status=status, - now=now, - ) - ], - "mpc": None, + "workspaces": [], + "mpc": deepcopy(mpc), "best_workspace": None, - "qor_baseline": {"workspace_id": workspace_id, "reason": "Default project QoR baseline"}, + "qor_baseline": None, } - return document def write_manifest_if_absent(project_dir: str, document: dict) -> bool: diff --git a/chipcompiler/engine/__init__.py b/chipcompiler/engine/__init__.py index f75ec0ef..459a008e 100644 --- a/chipcompiler/engine/__init__.py +++ b/chipcompiler/engine/__init__.py @@ -2,6 +2,8 @@ from .flow import EngineFlow from .rerun import StepRunResult from .signoff import SignoffPackageCollector, SignoffPackageOptions +from .workspace_lifecycle import WorkspaceLifecycleError, create_workspace_from_spec +from .workspace_spec import describe_workspace_spec, validate_workspace_spec __all__ = [ "EngineDB", @@ -9,4 +11,8 @@ "StepRunResult", "SignoffPackageCollector", "SignoffPackageOptions", + "WorkspaceLifecycleError", + "create_workspace_from_spec", + "describe_workspace_spec", + "validate_workspace_spec", ] diff --git a/chipcompiler/engine/pdk_binding.py b/chipcompiler/engine/pdk_binding.py new file mode 100644 index 00000000..f26134ac --- /dev/null +++ b/chipcompiler/engine/pdk_binding.py @@ -0,0 +1,64 @@ +import hashlib +import json +from pathlib import Path +from typing import Any + + +def pdk_binding_content_hash( + spec: dict[str, Any], + bindings: dict[str, Any], +) -> str: + digest = hashlib.sha256() + digest.update(str(spec.get("familyId", "")).encode()) + digest.update(str(spec.get("version") or bindings.get("version") or "").encode()) + digest.update(json.dumps(spec.get("overrides", {}), sort_keys=True).encode()) + for path in _selected_paths(spec, bindings): + digest.update(path.read_bytes()) + return digest.hexdigest() + + +def _selected_paths(spec: dict[str, Any], bindings: dict[str, Any]) -> list[Path]: + if spec.get("mode") == "manual": + files = bindings.get("files") + if not isinstance(files, dict): + return [] + return [ + Path(files[ref["fileId"]]) + for ref in spec.get("files", []) + if isinstance(ref, dict) + and isinstance(ref.get("fileId"), str) + and ref["fileId"] in files + and Path(files[ref["fileId"]]).is_file() + ] + + try: + from chipcompiler.data import get_pdk + + pdk = get_pdk( + spec.get("familyId", ""), + pdk_root=bindings.get("root", ""), + overrides=pdk_overrides_from_binding(spec, bindings), + ) + except (OSError, ValueError): + return [] + candidates = [pdk.tech, *pdk.lefs, *pdk.libs, pdk.mapping_file] + return [Path(path) for path in candidates if path is not None and Path(path).is_file()] + + +def pdk_overrides_from_binding(spec: dict[str, Any], bindings: dict[str, Any]) -> dict[str, Any]: + overrides = dict(spec.get("overrides", {})) + files = bindings.get("files", {}) + by_role: dict[str, list[str]] = {} + for ref in spec.get("files", []): + path = files.get(ref.get("fileId")) if isinstance(files, dict) else None + if isinstance(path, str): + by_role.setdefault(ref["role"], []).append(path) + if by_role.get("tech"): + overrides["tech"] = by_role["tech"][0] + if by_role.get("lef"): + overrides["lefs"] = by_role["lef"] + if by_role.get("liberty"): + overrides["libs"] = by_role["liberty"] + if by_role.get("mapping"): + overrides["mapping_file"] = by_role["mapping"][0] + return overrides diff --git a/chipcompiler/engine/workspace_lifecycle.py b/chipcompiler/engine/workspace_lifecycle.py new file mode 100644 index 00000000..ea561c5f --- /dev/null +++ b/chipcompiler/engine/workspace_lifecycle.py @@ -0,0 +1,144 @@ +import shutil +import tempfile +from pathlib import Path +from typing import Any + +from chipcompiler.cli.project.params import build_backend_overrides, resolve_parameters +from chipcompiler.data import PDK, create_workspace +from chipcompiler.engine.workspace_spec import validate_workspace_spec +from chipcompiler.rtl2gds import get_flow_builders + + +class WorkspaceLifecycleError(RuntimeError): + def __init__(self, code: str, message: str, details: dict[str, Any] | None = None): + super().__init__(message) + self.code = code + self.details = details or {} + + +def create_workspace_from_spec( + target_directory: str | Path, + spec: object, + bindings: object, + command_id: str = "", +): + del command_id + target = Path(target_directory).expanduser().resolve() + if target.exists(): + raise WorkspaceLifecycleError("workspace_exists", f"Workspace already exists: {target}") + validation = validate_workspace_spec(spec, bindings) + issues = validation["issues"] + if issues: + raise WorkspaceLifecycleError( + "workspace_spec_invalid", + "Workspace Spec validation failed", + {"issues": issues}, + ) + resolved = validation["resolvedWorkspaceSpec"] + binding_map = _string_keyed_dict(bindings) + input_bindings = _string_keyed_dict(binding_map.get("inputs")) + input_paths = { + item["role"]: str(input_bindings[item["inputId"]]) for item in resolved["inputs"] + } + rtl_paths = [ + str(input_bindings[item["inputId"]]) for item in resolved["inputs"] if item["role"] == "rtl" + ] + parameters, errors = resolve_parameters(toml_overrides=resolved["parameters"]) + if errors: + raise WorkspaceLifecycleError( + "workspace_spec_invalid", + "Workspace parameters are invalid", + {"issues": errors}, + ) + backend_parameters = build_backend_overrides(parameters) + backend_parameters.update( + { + "design": resolved["design"]["name"], + "top_module": resolved["design"]["topModule"], + "clock": resolved["design"].get("clockPort", ""), + } + ) + flow_steps = get_flow_builders()[resolved["flow"]["flowId"]]() + flow_config = { + "start_step": str(getattr(flow_steps[0][0], "value", flow_steps[0][0])), + "end_step": str(getattr(flow_steps[-1][0], "value", flow_steps[-1][0])), + } + pdk, pdk_root, pdk_overrides = _bound_pdk( + resolved["pdk"], _string_keyed_dict(binding_map["pdk"]) + ) + + generated_filelist: str | None = None + if len(rtl_paths) > 1: + with tempfile.NamedTemporaryFile( + "w", encoding="utf-8", suffix=".f", delete=False + ) as handle: + handle.write("\n".join(rtl_paths) + "\n") + generated_filelist = handle.name + + try: + workspace = create_workspace( + directory=target, + origin_def=input_paths.get("def", ""), + origin_verilog=( + input_paths.get("netlist", "") or (rtl_paths[0] if len(rtl_paths) == 1 else "") + ), + input_filelist=input_paths.get("filelist", "") or generated_filelist or "", + golden_verilog=input_paths.get("goldenNetlist", ""), + sdc=input_paths.get("sdc", ""), + spef=input_paths.get("spef", ""), + pdk=pdk, + pdk_root=pdk_root, + pdk_overrides=pdk_overrides, + parameters=backend_parameters, + flow_config=flow_config, + ) + if workspace is None: + raise WorkspaceLifecycleError( + "workspace_create_failed", f"Workspace creation failed: {target}" + ) + return workspace + except Exception: + shutil.rmtree(target, ignore_errors=True) + raise + finally: + if generated_filelist is not None: + Path(generated_filelist).unlink(missing_ok=True) + + +def _bound_pdk(spec: dict, binding: dict) -> tuple[PDK | str, str, dict | None]: + root = Path(str(binding["root"])) + overrides = spec.get("overrides", {}) + if spec["mode"] != "manual": + return ( + spec["familyId"], + str(root), + { + **({"dont_use": overrides["dont_use"]} if "dont_use" in overrides else {}), + **({"abc_load": overrides["abc_load"]} if "abc_load" in overrides else {}), + }, + ) + files = binding.get("files", {}) + by_role: dict[str, list[Path]] = {} + for item in spec.get("files", []): + by_role.setdefault(item["role"], []).append(Path(str(files[item["fileId"]]))) + return ( + PDK( + name=spec["familyId"], + version=spec.get("version", ""), + root=root, + tech=by_role["tech"][0], + lefs=by_role.get("lef", []), + libs=by_role.get("liberty", []), + mapping_file=(by_role.get("mapping") or [None])[0], + dont_use=overrides.get("dont_use", []), + abc_load=float(overrides.get("abc_load", 0.015)), + ), + "", + None, + ) + + +def _string_keyed_dict(value: object) -> dict[str, Any]: + if not isinstance(value, dict): + return {} + return {str(key): item for key, item in value.items()} diff --git a/chipcompiler/engine/workspace_spec.py b/chipcompiler/engine/workspace_spec.py new file mode 100644 index 00000000..9facf0fd --- /dev/null +++ b/chipcompiler/engine/workspace_spec.py @@ -0,0 +1,431 @@ +from copy import deepcopy +from pathlib import Path +from typing import Any + +from chipcompiler.cli.project.params import list_schemas, resolve_parameters +from chipcompiler.engine.pdk_binding import pdk_binding_content_hash +from chipcompiler.rtl2gds import get_flow_builders, normalize_flow_step +from chipcompiler.utility.filelist import validate_filelist + +SCHEMA_VERSION = 1 +_INPUT_ROLES = frozenset({"rtl", "filelist", "netlist", "goldenNetlist", "def", "sdc", "spef"}) +_PDK_FILE_ROLES = frozenset({"tech", "lef", "liberty", "mapping"}) +_SPEC_FIELDS = frozenset( + {"schemaVersion", "design", "inputMode", "inputs", "pdk", "flow", "mpc", "parameters"} +) + + +def describe_workspace_spec() -> dict[str, Any]: + flows = [] + for flow_id, builder in sorted(get_flow_builders().items()): + steps = [_enum_value(step) for step, _tool, _state in builder()] + flows.append({"flowId": flow_id, "stepIds": steps}) + return { + "schemaVersion": SCHEMA_VERSION, + "parameterCatalog": [ + { + "id": schema.param, + "type": schema.type, + "default": deepcopy(schema.default), + "appliesTo": schema.applies, + "backendMapping": deepcopy(schema.maps_to), + **({"range": list(schema.range)} if schema.range else {}), + **({"choices": list(schema.choices)} if schema.choices else {}), + **({"unit": schema.unit} if schema.unit else {}), + } + for schema in list_schemas() + ], + "flowDefinitions": flows, + "inputRoleRules": { + "roles": sorted(_INPUT_ROLES), + "rtl": {"rtl": "oneOrMore", "filelist": "exactlyOneAlternative"}, + "postSynthesis": {"netlist": "exactlyOne"}, + "manualPdk": { + "tech": "exactlyOne", + "lef": "oneOrMore", + "liberty": "oneOrMore", + "mapping": "zeroOrOne", + }, + }, + } + + +def validate_workspace_spec(spec: object, bindings: object) -> dict[str, Any]: + issues: list[dict[str, Any]] = [] + if not isinstance(spec, dict): + return {"issues": [_issue("invalid_type", "", expected="object")]} + spec = {str(key): value for key, value in spec.items()} + if not isinstance(bindings, dict): + bindings = {} + issues.append(_issue("invalid_type", "/bindings", expected="object")) + else: + bindings = {str(key): value for key, value in bindings.items()} + + _unknown_fields(spec, _SPEC_FIELDS, "", issues) + if spec.get("schemaVersion") != SCHEMA_VERSION or isinstance(spec.get("schemaVersion"), bool): + issues.append( + _issue( + "unsupported_schema_version", + "/schemaVersion", + supported=SCHEMA_VERSION, + ) + ) + + design = _mapping(spec.get("design"), "/design", issues) + _unknown_fields(design, {"name", "topModule", "clockPort"}, "/design", issues) + _required_string(design, "name", "/design/name", issues) + _required_string(design, "topModule", "/design/topModule", issues) + if "clockPort" in design and not _nonempty_string(design["clockPort"]): + issues.append(_issue("invalid_value", "/design/clockPort")) + + input_mode = spec.get("inputMode") + if input_mode not in {"rtl", "postSynthesis"}: + issues.append(_issue("invalid_input_mode", "/inputMode")) + inputs = _refs(spec.get("inputs"), "inputId", _INPUT_ROLES, "/inputs", issues) + input_bindings = _input_bindings(bindings.get("inputs"), issues) + _validate_binding_set(inputs, input_bindings, issues) + _validate_input_roles(input_mode, inputs, issues) + _validate_input_files(inputs, input_bindings, issues) + + pdk = _mapping(spec.get("pdk"), "/pdk", issues) + _unknown_fields(pdk, {"familyId", "version", "mode", "files", "overrides"}, "/pdk", issues) + _required_string(pdk, "familyId", "/pdk/familyId", issues) + mode = pdk.get("mode") + if mode not in {"default", "manual"}: + issues.append(_issue("invalid_pdk_mode", "/pdk/mode")) + overrides = pdk.get("overrides", {}) + if not isinstance(overrides, dict): + issues.append(_issue("invalid_type", "/pdk/overrides", expected="object")) + else: + _unknown_fields(overrides, {"dont_use", "abc_load"}, "/pdk/overrides", issues) + if "dont_use" in overrides and not ( + isinstance(overrides["dont_use"], list) + and all(isinstance(value, str) for value in overrides["dont_use"]) + ): + issues.append(_issue("invalid_type", "/pdk/overrides/dont_use", expected="array")) + if "abc_load" in overrides and ( + isinstance(overrides["abc_load"], bool) + or not isinstance(overrides["abc_load"], (int, float)) + ): + issues.append(_issue("invalid_type", "/pdk/overrides/abc_load", expected="number")) + pdk_bindings = _mapping(bindings.get("pdk"), "/bindings/pdk", issues) + root = pdk_bindings.get("root") + if not isinstance(root, str) or not root.strip() or not Path(root).is_dir(): + issues.append(_issue("pdk_binding_missing", "/bindings/pdk/root")) + _validate_pdk_files(pdk, pdk_bindings, issues) + requested_version = pdk.get("version") + bound_version = pdk_bindings.get("version") + if requested_version and bound_version and requested_version != bound_version: + issues.append( + _issue( + "pdk_binding_mismatch", + "/bindings/pdk/version", + expected=requested_version, + actual=bound_version, + ) + ) + + flow = _mapping(spec.get("flow"), "/flow", issues) + _unknown_fields(flow, {"flowId", "fromStepId", "throughStepId"}, "/flow", issues) + flow_steps = _validate_flow(flow, issues) + + parameters = spec.get("parameters", {}) + if not isinstance(parameters, dict): + issues.append(_issue("invalid_type", "/parameters", expected="object")) + parameters = {} + else: + parameters = {str(key): value for key, value in parameters.items()} + known_parameters = {schema.param for schema in list_schemas()} + for parameter_id in parameters: + if parameter_id not in known_parameters: + issues.append(_issue("unknown_parameter", f"/parameters/{_pointer(parameter_id)}")) + resolved_parameters, parameter_errors = resolve_parameters( + toml_overrides={key: value for key, value in parameters.items() if key in known_parameters} + ) + for error in parameter_errors: + parameter_id = next((key for key in parameters if key in error), "") + issues.append( + _issue( + "invalid_parameter", + f"/parameters/{_pointer(parameter_id)}" if parameter_id else "/parameters", + reason=error, + ) + ) + _validate_parameter_applicability(parameters, flow_steps, issues) + + mpc = spec.get("mpc") + if mpc is not None: + mpc = _mapping(mpc, "/mpc", issues) + _unknown_fields(mpc, {"resourceId", "version", "designId"}, "/mpc", issues) + for key in ("resourceId", "version", "designId"): + _required_string(mpc, key, f"/mpc/{key}", issues) + mpc_binding = _mapping(bindings.get("mpc"), "/bindings/mpc", issues) + if not isinstance(mpc_binding.get("template"), dict): + issues.append(_issue("mpc_binding_missing", "/bindings/mpc/template")) + + if any(issue["severity"] == "error" for issue in issues): + return {"issues": issues} + + resolved = deepcopy(spec) + resolved["parameters"] = _effective_parameter_values(resolved_parameters, flow_steps) + resolved["pdk"] = { + **deepcopy(pdk), + "version": requested_version or bound_version or "unversioned", + "contentHash": pdk_binding_content_hash(pdk, pdk_bindings), + } + return {"resolvedWorkspaceSpec": resolved, "issues": issues} + + +def _validate_input_roles( + input_mode: object, + inputs: list[dict[str, str]], + issues: list[dict[str, Any]], +) -> None: + roles = [item["role"] for item in inputs] + if input_mode == "rtl": + if ("rtl" in roles) == ("filelist" in roles): + issues.append(_issue("conflicting_input_roles", "/inputs")) + if roles.count("filelist") > 1: + issues.append(_issue("filelist_cardinality", "/inputs")) + if "netlist" in roles: + issues.append(_issue("input_role_not_allowed", "/inputs", mode="rtl")) + elif input_mode == "postSynthesis": + if roles.count("netlist") != 1: + issues.append(_issue("netlist_cardinality", "/inputs")) + if any(role in {"rtl", "filelist"} for role in roles): + issues.append(_issue("input_role_not_allowed", "/inputs", mode="postSynthesis")) + for role in ("netlist", "goldenNetlist", "def", "sdc", "spef"): + if roles.count(role) > 1: + issues.append(_issue(f"{role}_cardinality", "/inputs")) + + +def _validate_input_files( + refs: list[dict[str, str]], + bindings: dict[str, str], + issues: list[dict[str, Any]], +) -> None: + for ref in refs: + input_id = ref["inputId"] + path = bindings.get(input_id) + if not path: + continue + source = Path(path) + pointer = f"/bindings/inputs/{_pointer(input_id)}" + if not source.is_file(): + issues.append(_issue("input_unreadable", pointer)) + continue + if ref["role"] != "filelist": + continue + try: + _existing, missing = validate_filelist(str(source)) + except (OSError, ValueError) as exc: + issues.append(_issue("filelist_invalid", pointer, reason=str(exc))) + continue + if missing: + issues.append(_issue("filelist_source_missing", pointer, missing=missing)) + + +def _validate_binding_set( + refs: list[dict[str, str]], + bindings: dict[str, str], + issues: list[dict[str, Any]], +) -> None: + ref_ids = {item["inputId"] for item in refs} + for input_id in sorted(ref_ids - bindings.keys()): + issues.append(_issue("input_binding_missing", f"/bindings/inputs/{_pointer(input_id)}")) + for input_id in sorted(bindings.keys() - ref_ids): + issues.append(_issue("unknown_input_binding", f"/bindings/inputs/{_pointer(input_id)}")) + + +def _validate_pdk_files( + pdk: dict[str, Any], + bindings: dict[str, Any], + issues: list[dict[str, Any]], +) -> None: + mode = pdk.get("mode") + refs = _refs(pdk.get("files", []), "fileId", _PDK_FILE_ROLES, "/pdk/files", issues) + roles = [item["role"] for item in refs] + rules = ( + ( + ("tech", roles.count("tech") == 1), + ("lef", roles.count("lef") >= 1), + ("liberty", roles.count("liberty") >= 1), + ("mapping", roles.count("mapping") <= 1), + ) + if mode == "manual" + else ( + ("tech", roles.count("tech") <= 1), + ("mapping", roles.count("mapping") <= 1), + ) + ) + for role, valid in rules: + if not valid: + issues.append(_issue(f"pdk_{role}_cardinality", "/pdk/files")) + raw_file_bindings = bindings.get("files", {}) + file_bindings = raw_file_bindings if isinstance(raw_file_bindings, dict) else {} + ref_ids = {item["fileId"] for item in refs} + for file_id in sorted(ref_ids - file_bindings.keys()): + issues.append( + _issue("pdk_file_binding_missing", f"/bindings/pdk/files/{_pointer(file_id)}") + ) + for file_id in sorted(file_bindings.keys() - ref_ids): + issues.append( + _issue("unknown_pdk_file_binding", f"/bindings/pdk/files/{_pointer(file_id)}") + ) + for file_id, path in file_bindings.items(): + if file_id in ref_ids and (not isinstance(path, str) or not Path(path).is_file()): + issues.append(_issue("pdk_file_unreadable", f"/bindings/pdk/files/{_pointer(file_id)}")) + + +def _validate_flow(flow: dict[str, Any], issues: list[dict[str, Any]]) -> set[str]: + flow_id = flow.get("flowId") + builder = get_flow_builders().get(flow_id) if isinstance(flow_id, str) else None + if builder is None: + issues.append(_issue("unknown_flow", "/flow/flowId")) + return set() + steps = [_enum_value(step) for step, _tool, _state in builder()] + start = flow.get("fromStepId") + end = flow.get("throughStepId") + if (start is None) != (end is None): + issues.append(_issue("flow_boundaries_incomplete", "/flow")) + return set(steps) + if start is None: + return set(steps) + if start not in steps: + issues.append(_issue("unknown_flow_boundary", "/flow/fromStepId")) + if end not in steps: + issues.append(_issue("unknown_flow_boundary", "/flow/throughStepId")) + if start in steps and end in steps: + start_index, end_index = steps.index(start), steps.index(end) + if start_index > end_index: + issues.append(_issue("flow_boundary_reversed", "/flow")) + else: + return set(steps[start_index : end_index + 1]) + return set() + + +def _validate_parameter_applicability( + explicit: dict[str, Any], + flow_steps: set[str], + issues: list[dict[str, Any]], +) -> None: + normalized_steps = {step.lower() for step in flow_steps} + aliases = { + "synthesis": "synth", + "floorplan": "floor", + "fixfanout": "fanout", + "placement": "place", + "routing": "route", + } + normalized_steps |= {aliases.get(step, step) for step in normalized_steps} + for schema in list_schemas(): + if schema.param not in explicit: + continue + if schema.applies == "all": + continue + applies = aliases.get(schema.applies.lower(), schema.applies.lower()) + if flow_steps and applies not in normalized_steps: + issues.append( + _issue( + "inapplicable_parameter", + f"/parameters/{_pointer(schema.param)}", + appliesTo=schema.applies, + ) + ) + + +def _effective_parameter_values(resolved, steps: set[str]) -> dict[str, object]: + available = {normalize_flow_step(step).casefold() for step in steps} + return { + parameter.param: deepcopy(parameter.value) + for parameter in resolved + if parameter.schema.pdk_target is None + and ( + parameter.schema.applies == "all" + or normalize_flow_step(parameter.schema.applies).casefold() in available + ) + } + + +def _refs( + value: object, + id_key: str, + roles: frozenset[str], + path: str, + issues: list[dict[str, Any]], +) -> list[dict[str, str]]: + if not isinstance(value, list): + issues.append(_issue("invalid_type", path, expected="array")) + return [] + refs = [] + seen = set() + for index, value_item in enumerate(value): + item_path = f"{path}/{index}" + if not isinstance(value_item, dict): + issues.append(_issue("invalid_type", item_path, expected="object")) + continue + item = {str(key): field for key, field in value_item.items()} + _unknown_fields(item, {id_key, "role"}, item_path, issues) + item_id = item.get(id_key) + role = item.get("role") + if not _nonempty_string(item_id): + issues.append(_issue("invalid_value", f"{item_path}/{id_key}")) + continue + if item_id in seen: + issues.append(_issue("duplicate_ref", f"{item_path}/{id_key}")) + continue + seen.add(item_id) + if role not in roles: + issues.append(_issue("unknown_role", f"{item_path}/role")) + continue + refs.append({id_key: item_id, "role": role}) + return refs + + +def _input_bindings(value: object, issues: list[dict[str, Any]]) -> dict[str, str]: + if not isinstance(value, dict): + issues.append(_issue("invalid_type", "/bindings/inputs", expected="object")) + return {} + return { + key: path for key, path in value.items() if isinstance(key, str) and isinstance(path, str) + } + + +def _mapping(value: object, path: str, issues: list[dict[str, Any]]) -> dict[str, Any]: + if not isinstance(value, dict): + issues.append(_issue("invalid_type", path, expected="object")) + return {} + return {str(key): item for key, item in value.items()} + + +def _unknown_fields( + value: dict[str, Any], + allowed: set[str] | frozenset[str], + path: str, + issues: list[dict[str, Any]], +) -> None: + for key in value.keys() - allowed: + issues.append(_issue("unknown_field", f"{path}/{_pointer(key)}")) + + +def _required_string( + value: dict[str, Any], key: str, path: str, issues: list[dict[str, Any]] +) -> None: + if not _nonempty_string(value.get(key)): + issues.append(_issue("required", path)) + + +def _nonempty_string(value: object) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _enum_value(value: object) -> str: + return str(getattr(value, "value", value)) + + +def _pointer(value: str) -> str: + return value.replace("~", "~0").replace("/", "~1") + + +def _issue(code: str, path: str, **details: Any) -> dict[str, Any]: + return {"code": code, "path": path, "severity": "error", "details": details} diff --git a/chipcompiler/project/__init__.py b/chipcompiler/project/__init__.py new file mode 100644 index 00000000..98ff93c2 --- /dev/null +++ b/chipcompiler/project/__init__.py @@ -0,0 +1,15 @@ +from .api import ( + create_project_manifest, + create_project_workspace, + discover_project_manifest, + load_project_manifest, + mutate_project_manifest, +) + +__all__ = [ + "create_project_manifest", + "create_project_workspace", + "discover_project_manifest", + "load_project_manifest", + "mutate_project_manifest", +] diff --git a/chipcompiler/project/api.py b/chipcompiler/project/api.py new file mode 100644 index 00000000..4662f38e --- /dev/null +++ b/chipcompiler/project/api.py @@ -0,0 +1,279 @@ +import json +import shutil +from copy import deepcopy +from datetime import UTC, datetime +from pathlib import Path + +from chipcompiler.cli.project.manifest import ( + _CANONICAL_TO_MANIFEST_STEP, + ManifestError, + load_manifest, +) +from chipcompiler.cli.project.manifest_write import ( + build_project_document, + manifest_workspace_entry, + update_manifest, + write_manifest_if_absent, +) + + +def load_project_manifest(project_dir: str | Path) -> dict: + manifest = load_manifest(str(Path(project_dir).expanduser().resolve())) + document = deepcopy(manifest.raw) + document.update( + { + "schema_version": 1, + "project_id": manifest.project_id, + "name": manifest.name, + "design_name": manifest.design_name, + "root_path": manifest.project_dir, + "base_design": deepcopy(manifest.base_design), + "objectives": deepcopy(manifest.objectives), + "workspaces": [_workspace_document(item) for item in manifest.workspaces], + "qor_baseline": deepcopy(manifest.qor_baseline), + } + ) + return document + + +def discover_project_manifest(directory: str | Path) -> tuple[Path, dict] | None: + current = Path(directory).expanduser().resolve() + if not current.is_dir(): + current = current.parent + for project in (current, *current.parents): + if (project / "project.json").exists() or (project / "project.json").is_symlink(): + return project, load_project_manifest(project) + return None + + +def create_project_manifest( + project_dir: str | Path, + name: str, + design_name: str, + *, + now: str | None = None, + mpc: dict | None = None, +) -> dict: + project = Path(project_dir).expanduser().resolve() + project.mkdir(parents=True, exist_ok=True) + document = build_project_document( + str(project), + design_name=design_name, + base_design={"rtl_list": [], "parameters": {"design": design_name}}, + name=name, + now=now, + mpc=mpc, + ) + if not write_manifest_if_absent(str(project), document): + raise ManifestError(f"Project Manifest already exists: {project / 'project.json'}") + try: + return load_project_manifest(project) + except Exception: + (project / "project.json").unlink(missing_ok=True) + raise + + +def mutate_project_manifest(project_dir: str | Path, mutation: dict) -> dict: + project = Path(project_dir).expanduser().resolve() + load_project_manifest(project) + + def apply(document: dict) -> None: + kind = mutation.get("type") + if kind == "register_workspace": + _register_workspace(document, mutation, project) + elif kind in {"select_qor_baseline", "select_best_workspace"}: + _select_workspace(document, mutation) + elif kind == "archive_workspace": + _archive_workspace(document, mutation) + elif kind == "delete_workspace": + _delete_workspace(document, mutation) + else: + raise ManifestError(f"unsupported Project mutation: {kind}") + + if not update_manifest(str(project), apply): + raise ManifestError("Project Manifest update failed") + return load_project_manifest(project) + + +def create_project_workspace( + project_dir: str | Path, + target_directory: str | Path, + spec: object, + bindings: object, + *, + command_id: str, + workspace_id: str | None = None, + name: str | None = None, + source_workspace_id: str | None = None, + expected_project_id: str | None = None, + now: str | None = None, +): + from chipcompiler.engine import create_workspace_from_spec + + project = Path(project_dir).expanduser().resolve() + target = Path(target_directory).expanduser().resolve() + try: + target.relative_to(project) + except ValueError as exc: + raise ManifestError("Workspace must be inside the Project root") from exc + manifest = load_project_manifest(project) + if expected_project_id is not None and manifest["project_id"] != expected_project_id: + raise ManifestError("Project identity does not match") + existed = target.exists() + workspace = create_workspace_from_spec(str(target), spec, bindings, command_id) + if workspace is None: + raise ManifestError("Workspace creation returned no Workspace") + identity = workspace_id or target.name + try: + mutate_project_manifest( + project, + { + "type": "register_workspace", + "workspace_id": identity, + "name": name or identity, + "workspace_path": str(target), + "source_workspace_id": source_workspace_id, + "created_at": now, + "updated_at": now, + }, + ) + except Exception: + if not existed: + shutil.rmtree(target, ignore_errors=True) + raise + return workspace + + +def _workspace_document(workspace) -> dict: + document = deepcopy(workspace.raw) + document.update( + { + "workspace_id": workspace.workspace_id, + "workspace_path": workspace.workspace_path, + "start_step": workspace.start_step, + "end_step": workspace.end_step, + "status": workspace.status, + "parameter_patch": deepcopy(workspace.parameter_patch), + } + ) + return document + + +def _register_workspace(document: dict, mutation: dict, project: Path) -> None: + workspace_id = _required_string(mutation, "workspace_id") + workspace_path = _resolve_workspace(project, _required_string(mutation, "workspace_path")) + for existing in document.get("workspaces", []): + same_id = existing.get("workspace_id") == workspace_id + same_path = ( + _resolve_workspace(project, str(existing.get("workspace_path", ""))) == workspace_path + ) + if same_id and same_path: + return + if same_id or same_path: + raise ManifestError(f"Workspace registration conflicts with: {workspace_id}") + start_step, end_step = _workspace_range(workspace_path, mutation) + timestamp = str( + mutation.get("updated_at") or mutation.get("created_at") or datetime.now(UTC).isoformat() + ) + entry = manifest_workspace_entry( + workspace_id, + name=str(mutation.get("name") or workspace_id), + workspace_path=str(workspace_path), + start_step=start_step, + end_step=end_step, + status="archived" if mutation.get("lifecycle") == "archived" else "not_started", + now=timestamp, + ) + for key in ( + "source_workspace_id", + "branch_from", + "parameter_patch", + "metrics_summary", + "step_metrics", + ): + if key in mutation: + entry[key] = deepcopy(mutation[key]) + document.setdefault("workspaces", []).append(entry) + document["updated_at"] = timestamp + + +def _workspace_range(workspace: Path, mutation: dict) -> tuple[str, str]: + start = mutation.get("start_step") + end = mutation.get("end_step") + if isinstance(start, str) and isinstance(end, str) and start and end: + return start, end + try: + ledger = json.loads((workspace / "home" / "flow.json").read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return "Synth", "Harden" + names = [ + step.get("name") + for step in ledger.get("steps", []) + if isinstance(step, dict) and isinstance(step.get("name"), str) + ] + if not names: + return "Synth", "Harden" + return ( + _CANONICAL_TO_MANIFEST_STEP.get(names[0], names[0]), + _CANONICAL_TO_MANIFEST_STEP.get(names[-1], names[-1]), + ) + + +def _select_workspace(document: dict, mutation: dict) -> None: + workspace_id = _required_string(mutation, "workspace_id") + workspace = _find_workspace(document, workspace_id) + if workspace.get("status") == "archived": + raise ManifestError(f"Workspace is not active: {workspace_id}") + field = "qor_baseline" if mutation["type"] == "select_qor_baseline" else "best_workspace" + document[field] = {"workspace_id": workspace_id, "reason": str(mutation.get("reason") or "")} + document["updated_at"] = str(mutation.get("updated_at") or "") + + +def _archive_workspace(document: dict, mutation: dict) -> None: + workspace = _find_workspace(document, _required_string(mutation, "workspace_id")) + workspace["status"] = "archived" + workspace["updated_at"] = str(mutation.get("updated_at") or "") + document["updated_at"] = workspace["updated_at"] + + +def _delete_workspace(document: dict, mutation: dict) -> None: + workspace_id = _required_string(mutation, "workspace_id") + _find_workspace(document, workspace_id) + document["workspaces"] = [ + workspace + for workspace in document.get("workspaces", []) + if workspace.get("workspace_id") != workspace_id + ] + for workspace in document["workspaces"]: + if workspace.get("source_workspace_id") == workspace_id: + workspace["source_workspace_id"] = None + for field in ("qor_baseline", "best_workspace"): + selection = document.get(field) + if isinstance(selection, dict) and selection.get("workspace_id") == workspace_id: + document[field] = None + document["updated_at"] = str(mutation.get("updated_at") or "") + + +def _find_workspace(document: dict, workspace_id: str) -> dict: + for workspace in document.get("workspaces", []): + if isinstance(workspace, dict) and workspace.get("workspace_id") == workspace_id: + return workspace + raise ManifestError(f"Workspace is not registered: {workspace_id}") + + +def _required_string(value: dict, key: str) -> str: + result = value.get(key) + if not isinstance(result, str) or not result.strip(): + raise ManifestError(f"Project Manifest {key} is required") + return result.strip() + + +def _resolve_workspace(project: Path, declared: str) -> Path: + workspace = Path(declared) + workspace = workspace if workspace.is_absolute() else project / workspace + resolved = workspace.resolve() + try: + resolved.relative_to(project) + except ValueError as exc: + raise ManifestError("Workspace must be inside the Project root") from exc + return resolved diff --git a/test/engine/test_workspace_spec.py b/test/engine/test_workspace_spec.py new file mode 100644 index 00000000..fbe833d2 --- /dev/null +++ b/test/engine/test_workspace_spec.py @@ -0,0 +1,151 @@ +import json +from copy import deepcopy +from pathlib import Path + +from chipcompiler.engine.workspace_spec import ( + describe_workspace_spec, + validate_workspace_spec, +) + + +def _valid_spec(): + return { + "schemaVersion": 1, + "design": {"name": "gcd", "topModule": "gcd", "clockPort": "clk"}, + "inputMode": "rtl", + "inputs": [{"inputId": "rtl-main", "role": "rtl"}], + "pdk": {"familyId": "ics55", "mode": "default"}, + "flow": {"flowId": "rtl2gds"}, + "parameters": {"design.frequency_mhz": 200.0}, + } + + +def _shared_fixture(name: str): + root = Path(__file__).parents[1] / "fixtures" / "workspace_spec" + payload = json.loads((root / name).read_text(encoding="utf-8")) + bindings = deepcopy(payload["workspaceBindings"]) + bindings["inputs"] = {key: str(root / value) for key, value in bindings["inputs"].items()} + bindings["pdk"]["root"] = str(root / bindings["pdk"]["root"]) + return payload, bindings + + +def test_shared_workspace_spec_fixtures_cover_valid_and_invalid_contracts(): + valid, valid_bindings = _shared_fixture("valid.json") + assert validate_workspace_spec(valid["workspaceSpec"], valid_bindings)["issues"] == [] + + invalid, invalid_bindings = _shared_fixture("invalid.json") + result = validate_workspace_spec(invalid["workspaceSpec"], invalid_bindings) + assert {issue["code"] for issue in result["issues"]} >= set(invalid["expectedIssueCodes"]) + + +def test_workspace_spec_discovery_and_validation_are_canonical_and_side_effect_free(tmp_path): + rtl = tmp_path / "gcd.v" + rtl.write_text("module gcd(input clk); endmodule\n") + pdk = tmp_path / "pdk" + pdk.mkdir() + before = sorted(str(path.relative_to(tmp_path)) for path in tmp_path.rglob("*")) + + discovery = describe_workspace_spec() + result = validate_workspace_spec( + _valid_spec(), + {"inputs": {"rtl-main": str(rtl)}, "pdk": {"root": str(pdk)}}, + ) + + assert discovery["schemaVersion"] == 1 + assert {item["id"] for item in discovery["parameterCatalog"]} >= { + "design.frequency_mhz", + "floorplan.core_util", + } + assert result["issues"] == [] + assert result["resolvedWorkspaceSpec"]["parameters"]["design.frequency_mhz"] == 200.0 + assert result["resolvedWorkspaceSpec"]["parameters"]["floorplan.core_util"] == 0.4 + assert len(result["resolvedWorkspaceSpec"]["pdk"]["contentHash"]) == 64 + assert str(pdk) not in json.dumps(result["resolvedWorkspaceSpec"]) + assert sorted(str(path.relative_to(tmp_path)) for path in tmp_path.rglob("*")) == before + + +def test_workspace_spec_reports_structured_binding_and_input_cardinality_issues(tmp_path): + filelist = tmp_path / "sources.f" + filelist.write_text("missing.v\n") + spec = _valid_spec() + spec["inputs"] = [ + {"inputId": "rtl-main", "role": "rtl"}, + {"inputId": "sources", "role": "filelist"}, + ] + + result = validate_workspace_spec( + spec, + { + "inputs": {"sources": str(filelist), "unknown": str(filelist)}, + "pdk": {"root": str(tmp_path)}, + }, + ) + + assert {(issue["code"], issue["path"]) for issue in result["issues"]} >= { + ("conflicting_input_roles", "/inputs"), + ("input_binding_missing", "/bindings/inputs/rtl-main"), + ("unknown_input_binding", "/bindings/inputs/unknown"), + ("filelist_source_missing", "/bindings/inputs/sources"), + } + assert "resolvedWorkspaceSpec" not in result + + +def test_workspace_spec_rejects_multiple_manual_pdk_technology_files(tmp_path): + rtl = tmp_path / "gcd.v" + rtl.write_text("module gcd; endmodule\n") + for name in ("a.tech.lef", "b.tech.lef", "cells.lef", "typ.lib"): + (tmp_path / name).write_text(name) + spec = _valid_spec() + spec["pdk"] = { + "familyId": "ics55", + "mode": "manual", + "files": [ + {"fileId": "tech-a", "role": "tech"}, + {"fileId": "tech-b", "role": "tech"}, + {"fileId": "cells", "role": "lef"}, + {"fileId": "lib", "role": "liberty"}, + ], + } + + result = validate_workspace_spec( + json.loads(json.dumps(spec)), + { + "inputs": {"rtl-main": str(rtl)}, + "pdk": { + "root": str(tmp_path), + "files": { + "tech-a": str(tmp_path / "a.tech.lef"), + "tech-b": str(tmp_path / "b.tech.lef"), + "cells": str(tmp_path / "cells.lef"), + "lib": str(tmp_path / "typ.lib"), + }, + }, + }, + ) + + assert any(issue["code"] == "pdk_tech_cardinality" for issue in result["issues"]) + + +def test_validated_workspace_spec_creates_main_compatible_workspace( + tmp_path, minimal_ics55_pdk_factory +): + from chipcompiler.data import load_workspace + from chipcompiler.engine import create_workspace_from_spec + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + target = tmp_path / "workspace" + + created = create_workspace_from_spec( + target, + payload["workspaceSpec"], + bindings, + "create-1", + ) + reopened = load_workspace(target) + + assert created.directory == target + assert reopened.directory == target + assert reopened.design.name == "gcd" + assert reopened.parameters.data["frequency_max"] == 200.0 + assert reopened.flow.steps()[0]["name"] == "Synthesis" diff --git a/test/fixtures/workspace_spec/input/gcd.v b/test/fixtures/workspace_spec/input/gcd.v new file mode 100644 index 00000000..5705cb05 --- /dev/null +++ b/test/fixtures/workspace_spec/input/gcd.v @@ -0,0 +1,3 @@ +module gcd(input clk, output y); + assign y = clk; +endmodule diff --git a/test/fixtures/workspace_spec/input/sources.f b/test/fixtures/workspace_spec/input/sources.f new file mode 100644 index 00000000..369c2e56 --- /dev/null +++ b/test/fixtures/workspace_spec/input/sources.f @@ -0,0 +1 @@ +missing.v diff --git a/test/fixtures/workspace_spec/invalid.json b/test/fixtures/workspace_spec/invalid.json new file mode 100644 index 00000000..b179d679 --- /dev/null +++ b/test/fixtures/workspace_spec/invalid.json @@ -0,0 +1,67 @@ +{ + "cliConfig": { + "design_name": "gcd", + "design_rtl": ["input/gcd.v"], + "design_top": "gcd", + "flow_preset": "rtl2gds", + "pdk_name": "ics55", + "pdk_overrides": { + "mystery": "unsupported" + }, + "pdk_root": "pdk" + }, + "expectedCliError": "unsupported_legacy_field: pdk.overrides.mystery", + "expectedIssueCodes": [ + "conflicting_input_roles", + "filelist_source_missing" + ], + "expectedStudioError": "unsupported_legacy_field: parameters.mystery_knob", + "studioDraft": { + "commandId": "workspace-create-invalid-shared-fixture", + "designInputMode": "rtl", + "directory": "workspace", + "originVerilog": "input/gcd.v", + "parameters": { + "design": "gcd", + "mystery_knob": 3, + "top_module": "gcd" + }, + "pdk": "ics55", + "pdkRoot": "pdk" + }, + "workspaceBindings": { + "inputs": { + "filelist": "input/sources.f", + "rtl-main": "input/gcd.v" + }, + "pdk": { + "root": "pdk" + } + }, + "workspaceSpec": { + "design": { + "name": "gcd", + "topModule": "gcd" + }, + "flow": { + "flowId": "rtl2gds" + }, + "inputMode": "rtl", + "inputs": [ + { + "inputId": "rtl-main", + "role": "rtl" + }, + { + "inputId": "filelist", + "role": "filelist" + } + ], + "parameters": {}, + "pdk": { + "familyId": "ics55", + "mode": "default" + }, + "schemaVersion": 1 + } +} diff --git a/test/fixtures/workspace_spec/pdk/.keep b/test/fixtures/workspace_spec/pdk/.keep new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/test/fixtures/workspace_spec/pdk/.keep @@ -0,0 +1 @@ + diff --git a/test/fixtures/workspace_spec/valid.json b/test/fixtures/workspace_spec/valid.json new file mode 100644 index 00000000..94a43fe0 --- /dev/null +++ b/test/fixtures/workspace_spec/valid.json @@ -0,0 +1,68 @@ +{ + "cliConfig": { + "design_clock_port": "clk", + "design_frequency_mhz": 200, + "design_name": "gcd", + "design_rtl": ["input/gcd.v"], + "design_top": "gcd", + "flow_preset": "rtl2gds", + "params_overrides": { + "floorplan.core_util": 0.6, + "place.target_density": 0.7 + }, + "pdk_name": "ics55", + "pdk_root": "pdk" + }, + "studioDraft": { + "commandId": "workspace-create-shared-fixture", + "designInputMode": "rtl", + "directory": "workspace", + "originVerilog": "input/gcd.v", + "parameters": { + "clock": "clk", + "core_utilization": 0.6, + "design": "gcd", + "frequency_max": 200, + "target_density": 0.7, + "top_module": "gcd" + }, + "pdk": "ics55", + "pdkConfigMode": "default", + "pdkRoot": "pdk" + }, + "workspaceBindings": { + "inputs": { + "rtl-main": "input/gcd.v" + }, + "pdk": { + "root": "pdk" + } + }, + "workspaceSpec": { + "design": { + "clockPort": "clk", + "name": "gcd", + "topModule": "gcd" + }, + "flow": { + "flowId": "rtl2gds" + }, + "inputMode": "rtl", + "inputs": [ + { + "inputId": "rtl-main", + "role": "rtl" + } + ], + "parameters": { + "design.frequency_mhz": 200.0, + "floorplan.core_util": 0.6, + "place.target_density": 0.7 + }, + "pdk": { + "familyId": "ics55", + "mode": "default" + }, + "schemaVersion": 1 + } +} diff --git a/test/project/__init__.py b/test/project/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/project/test_manifest.py b/test/project/test_manifest.py new file mode 100644 index 00000000..c93ed17e --- /dev/null +++ b/test/project/test_manifest.py @@ -0,0 +1,53 @@ +import json + +from chipcompiler.cli.project.manifest import load_manifest +from chipcompiler.project import ( + create_project_manifest, + discover_project_manifest, + load_project_manifest, + mutate_project_manifest, +) + + +def test_domain_created_manifest_is_cli_readable(tmp_path): + created = create_project_manifest(tmp_path, "Demo", "gcd", now="2026-01-01T00:00:00Z") + + loaded = load_manifest(str(tmp_path)) + + assert created["root_path"] == str(tmp_path.resolve()) + assert loaded.project_id == created["project_id"] + assert loaded.design_name == "gcd" + assert loaded.workspaces == () + + +def test_project_manifest_is_discovered_from_nested_workspace(tmp_path): + workspace = tmp_path / "experiment" + workspace.mkdir() + created = create_project_manifest(tmp_path, "Demo", "gcd", now="2026-01-01T00:00:00Z") + + assert discover_project_manifest(workspace) == (tmp_path.resolve(), created) + + +def test_register_workspace_uses_main_manifest_shape(tmp_path): + workspace = tmp_path / "experiment" + (workspace / "home").mkdir(parents=True) + (workspace / "home" / "flow.json").write_text( + json.dumps({"steps": [{"name": "Synthesis", "state": "Unstart"}]}) + ) + create_project_manifest(tmp_path, "Demo", "gcd", now="2026-01-01T00:00:00Z") + + updated = mutate_project_manifest( + tmp_path, + { + "type": "register_workspace", + "workspace_id": "experiment", + "workspace_path": str(workspace), + "name": "Experiment", + "created_at": "2026-02-01T00:00:00Z", + "updated_at": "2026-02-01T00:00:00Z", + }, + ) + + assert updated == load_project_manifest(tmp_path) + assert updated["workspaces"][0]["start_step"] == "Synth" + assert updated["workspaces"][0]["end_step"] == "Synth" From 728793326d79678362d6da36975e01b943456e64 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 00:33:49 +0800 Subject: [PATCH 04/43] feat(workspace): add revision-aware configuration updates --- chipcompiler/cli/project/workspace_params.py | 8 +- chipcompiler/engine/__init__.py | 8 + chipcompiler/engine/snapshot.py | 172 ++++++++++++ .../engine/workspace_configuration.py | 253 ++++++++++++++++++ chipcompiler/engine/workspace_lifecycle.py | 88 +++++- test/engine/test_workspace_configuration.py | 94 +++++++ 6 files changed, 620 insertions(+), 3 deletions(-) create mode 100644 chipcompiler/engine/snapshot.py create mode 100644 chipcompiler/engine/workspace_configuration.py create mode 100644 test/engine/test_workspace_configuration.py diff --git a/chipcompiler/cli/project/workspace_params.py b/chipcompiler/cli/project/workspace_params.py index e03481b0..3c48b6f2 100644 --- a/chipcompiler/cli/project/workspace_params.py +++ b/chipcompiler/cli/project/workspace_params.py @@ -48,6 +48,12 @@ def workspace_param_value(workspace, schema) -> object: def set_workspace_param(workspace, schema, value: object) -> tuple[object, str]: + baseline = update_workspace_param_value(workspace, schema, value) + return baseline, workspace_param_step(schema) + + +def update_workspace_param_value(workspace, schema, value: object) -> object: + """Apply and track one public parameter without choosing invalidation scope.""" records = _override_records(workspace.parameters.data) record = next((item for item in records if item["key"] == schema.param), None) if record is None: @@ -59,7 +65,7 @@ def set_workspace_param(workspace, schema, value: object) -> tuple[object, str]: record["value"] = deepcopy(value) _set_override_records(workspace.parameters.data, records) _apply_workspace_value(workspace, schema, value) - return record["baseline"], workspace_param_step(schema) + return record["baseline"] def unset_workspace_param(workspace, schema) -> tuple[object, str] | None: diff --git a/chipcompiler/engine/__init__.py b/chipcompiler/engine/__init__.py index 459a008e..98c13608 100644 --- a/chipcompiler/engine/__init__.py +++ b/chipcompiler/engine/__init__.py @@ -2,6 +2,11 @@ from .flow import EngineFlow from .rerun import StepRunResult from .signoff import SignoffPackageCollector, SignoffPackageOptions +from .workspace_configuration import ( + read_workspace_configuration, + read_workspace_configuration_from_directory, + update_workspace_configuration, +) from .workspace_lifecycle import WorkspaceLifecycleError, create_workspace_from_spec from .workspace_spec import describe_workspace_spec, validate_workspace_spec @@ -14,5 +19,8 @@ "WorkspaceLifecycleError", "create_workspace_from_spec", "describe_workspace_spec", + "read_workspace_configuration", + "read_workspace_configuration_from_directory", + "update_workspace_configuration", "validate_workspace_spec", ] diff --git a/chipcompiler/engine/snapshot.py b/chipcompiler/engine/snapshot.py new file mode 100644 index 00000000..e2e36426 --- /dev/null +++ b/chipcompiler/engine/snapshot.py @@ -0,0 +1,172 @@ +from copy import deepcopy +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from chipcompiler.utility import JsonReadError, json_read, json_read_strict, json_write + +SNAPSHOT_SCHEMA_VERSION = 2 +SNAPSHOT_FILENAME = "engineering-snapshot.json" +STALE_SNAPSHOT_FILENAME = "engineering-snapshot.stale.json" + + +class EngineeringSnapshotError(RuntimeError): + pass + + +def create_engineering_snapshot( + workspace: Any, + *, + workspace_id: str | None = None, + workspace_revision: int = 1, + cause: str = "workspace.created", +) -> dict[str, Any]: + snapshot = _build_snapshot( + workspace, + workspace_id=workspace_id or f"workspace-{uuid4().hex}", + workspace_revision=workspace_revision, + cause=cause, + ) + _write_snapshot(_snapshot_path(workspace), snapshot) + return snapshot + + +def ensure_engineering_snapshot(workspace: Any) -> dict[str, Any]: + path = _snapshot_path(workspace) + if path.is_file(): + return _read_snapshot(path) + return create_engineering_snapshot(workspace, cause="workspace.migrated") + + +def read_engineering_snapshot(workspace: Any) -> dict[str, Any]: + return _read_snapshot(_snapshot_path(workspace)) + + +def read_engineering_snapshot_from_directory(directory: str | Path) -> dict[str, Any]: + return _read_snapshot(Path(directory).expanduser().resolve() / "home" / SNAPSHOT_FILENAME) + + +def read_stale_engineering_snapshot(workspace: Any) -> dict[str, Any] | None: + path = _stale_snapshot_path(workspace) + return _read_snapshot(path) if path.is_file() else None + + +def invalidate_engineering_snapshot( + workspace: Any, + *, + workspace_id: str, + cause: str, + first_invalidated_step: str | None = None, +) -> dict[str, Any]: + current = read_engineering_snapshot(workspace) + if current["workspaceId"] != workspace_id: + raise EngineeringSnapshotError("Workspace identity changed before invalidation") + flow = deepcopy(current.get("flow", {})) + steps = flow.get("steps", []) if isinstance(flow, dict) else [] + invalidated_index = 0 + if first_invalidated_step is not None: + invalidated_index = next( + ( + index + for index, step in enumerate(steps) + if isinstance(step, dict) + and str(step.get("name", "")).casefold() == first_invalidated_step.casefold() + ), + -1, + ) + if invalidated_index < 0: + raise EngineeringSnapshotError(f"Flow Step not found: {first_invalidated_step}") + for step in steps[invalidated_index:]: + if isinstance(step, dict): + step["state"] = "Unstart" + step.pop("runtime", None) + step.pop("peak memory (mb)", None) + stale_path = _stale_snapshot_path(workspace) + if not stale_path.is_file(): + _write_snapshot(stale_path, current) + invalidated = [ + str(step["name"]) + for step in steps[invalidated_index:] + if isinstance(step, dict) and step.get("name") + ] + snapshot = _build_snapshot( + workspace, + workspace_id=workspace_id, + workspace_revision=current["workspaceRevision"] + 1, + cause=cause, + ) + snapshot["flow"] = flow + snapshot["stalePredecessor"] = { + "workspaceRevision": current["workspaceRevision"], + "invalidatedStepIds": invalidated, + } + _write_snapshot(_snapshot_path(workspace), snapshot) + return snapshot + + +def _build_snapshot( + workspace: Any, + *, + workspace_id: str, + workspace_revision: int, + cause: str, +) -> dict[str, Any]: + flow_owner = getattr(workspace, "flow", None) + flow = _data_mapping(flow_owner) + if not flow and flow_owner is not None: + steps = flow_owner.steps() + flow = {"steps": deepcopy(steps)} if steps else {} + home = _data_mapping(getattr(workspace, "home", None)) + checklist_path = home.get("checklist") + checklist = json_read(checklist_path) if isinstance(checklist_path, (str, Path)) else {} + return { + "schemaVersion": SNAPSHOT_SCHEMA_VERSION, + "workspaceId": workspace_id, + "workspaceRevision": workspace_revision, + "cause": cause, + "flow": flow, + "parameters": _data_mapping(getattr(workspace, "parameters", None)), + "checklist": checklist if isinstance(checklist, dict) else {}, + "analysis": {"steps": []}, + "metrics": {}, + "qorAssessment": {"status": "unavailable", "metrics": {}, "dimensions": {}}, + "signoffAssessment": {"status": "blocked", "groups": [], "risks": []}, + "artifacts": [], + } + + +def _snapshot_path(workspace: Any) -> Path: + return Path(workspace.directory) / "home" / SNAPSHOT_FILENAME + + +def _stale_snapshot_path(workspace: Any) -> Path: + return Path(workspace.directory) / "home" / STALE_SNAPSHOT_FILENAME + + +def _data_mapping(owner: Any) -> dict[str, Any]: + data = getattr(owner, "data", {}) + return deepcopy(data) if isinstance(data, dict) else {} + + +def _write_snapshot(path: Path, snapshot: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not json_write(path, snapshot): + raise EngineeringSnapshotError(f"failed to persist Engineering Snapshot: {path}") + + +def _read_snapshot(path: Path) -> dict[str, Any]: + try: + snapshot = json_read_strict(path) + except (OSError, JsonReadError) as exc: + raise EngineeringSnapshotError(f"invalid Engineering Snapshot: {path}") from exc + if ( + not isinstance(snapshot, dict) + or snapshot.get("schemaVersion") != SNAPSHOT_SCHEMA_VERSION + or not isinstance(snapshot.get("workspaceId"), str) + or not snapshot["workspaceId"] + or isinstance(snapshot.get("workspaceRevision"), bool) + or not isinstance(snapshot.get("workspaceRevision"), int) + or snapshot["workspaceRevision"] < 1 + ): + raise EngineeringSnapshotError(f"invalid Engineering Snapshot: {path}") + return snapshot diff --git a/chipcompiler/engine/workspace_configuration.py b/chipcompiler/engine/workspace_configuration.py new file mode 100644 index 00000000..57357baa --- /dev/null +++ b/chipcompiler/engine/workspace_configuration.py @@ -0,0 +1,253 @@ +from pathlib import Path +from typing import Any + +from chipcompiler.cli.project.params import ( + _validate_schema_type, + list_schemas, + lookup_schema, + validate_value, +) +from chipcompiler.cli.project.workspace_params import ( + update_workspace_param_value, + workspace_param_value, +) +from chipcompiler.data import load_workspace, save_parameter +from chipcompiler.data.workspace import workspace_config_paths +from chipcompiler.engine.flow import EngineFlow +from chipcompiler.engine.rerun import invalidate_from +from chipcompiler.engine.snapshot import ( + SNAPSHOT_FILENAME, + STALE_SNAPSHOT_FILENAME, + ensure_engineering_snapshot, + invalidate_engineering_snapshot, +) +from chipcompiler.rtl2gds import get_flow_builders + +from .workspace_lifecycle import ( + WorkspaceLifecycleError, + _command_retry_matches, + _load_committed_workspace, + _workspace_command_fingerprint, + _write_workspace_command, +) + + +def update_workspace_configuration( + target_directory: str | Path, + expected_workspace_revision: int, + configuration: object, + workspace_bindings: object, + command_id: str = "", +): + target = Path(target_directory).expanduser().resolve() + if not target.is_dir(): + raise WorkspaceLifecycleError("workspace_missing", f"Workspace not found: {target}") + before = _snapshot_files(_workspace_file_paths(target)) + try: + return _update_workspace_configuration( + target, + expected_workspace_revision, + configuration, + workspace_bindings, + command_id, + ) + except BaseException: + _restore_files(before) + raise + + +def _update_workspace_configuration( + target: Path, + expected_workspace_revision: int, + configuration: object, + workspace_bindings: object, + command_id: str, +): + if not isinstance(configuration, dict) or not isinstance(workspace_bindings, dict): + raise WorkspaceLifecycleError( + "workspace_spec_invalid", "Workspace configuration and bindings must be objects" + ) + config = {str(key): value for key, value in configuration.items()} + unknown_sections = config.keys() - {"design", "pdk", "parameters"} + if unknown_sections: + raise WorkspaceLifecycleError( + "workspace_spec_invalid", f"Unknown configuration section: {min(unknown_sections)}" + ) + if config.get("design") or config.get("pdk"): + raise WorkspaceLifecycleError( + "workspace_structure_change_requires_update", + "Design and PDK changes require a structural Workspace Update", + ) + requested = config.get("parameters", {}) + if not isinstance(requested, dict): + raise WorkspaceLifecycleError("workspace_spec_invalid", "parameters must be an object") + parameters = {str(key): value for key, value in requested.items()} + fingerprint = _workspace_command_fingerprint( + "configuration", config, workspace_bindings, expected_workspace_revision + ) + if command_id and _command_retry_matches(target, command_id, fingerprint): + return _load_committed_workspace(target) + + workspace = _load_committed_workspace(target) + current = ensure_engineering_snapshot(workspace) + if current["workspaceRevision"] != expected_workspace_revision: + raise WorkspaceLifecycleError( + "revision_conflict", + "Workspace Revision does not match", + { + "expectedWorkspaceRevision": expected_workspace_revision, + "actualWorkspaceRevision": current["workspaceRevision"], + }, + ) + + changed = False + for parameter, value in parameters.items(): + schema = lookup_schema(parameter) + if schema is None or schema.pdk_target is not None: + raise WorkspaceLifecycleError("unknown_parameter", f"Unknown parameter: {parameter}") + normalized, type_error = _validate_schema_type(value, schema) + errors = [type_error] if type_error else validate_value(normalized, schema) + if errors: + raise WorkspaceLifecycleError("invalid_parameter", str(errors[0])) + if workspace_param_value(workspace, schema) != normalized: + update_workspace_param_value(workspace, schema, normalized) + changed = True + + if not changed: + _write_workspace_command( + target, + command_id, + fingerprint, + current["workspaceId"], + current["workspaceRevision"], + ) + return workspace + if not save_parameter(workspace.parameters): + raise OSError("Failed to save Workspace parameters") + from chipcompiler.data import refresh_workspace_config + + refresh_workspace_config(workspace) + flow = EngineFlow(workspace) + steps = workspace.flow.steps() + if steps: + invalidate_from(flow, str(steps[0]["name"])) + updated = invalidate_engineering_snapshot( + workspace, + workspace_id=current["workspaceId"], + cause="workspace.configuration_updated", + ) + _write_workspace_command( + target, + command_id, + fingerprint, + updated["workspaceId"], + updated["workspaceRevision"], + ) + return _load_committed_workspace(target) + + +def read_workspace_configuration(workspace: Any) -> dict[str, Any]: + parameters = {} + for schema in list_schemas(): + if schema.pdk_target is not None: + continue + try: + parameters[schema.param] = workspace_param_value(workspace, schema) + except (OSError, ValueError): + continue + steps = workspace.flow.steps() + flow_names = [str(step.get("name", "")) for step in steps if step.get("name")] + flow = {"flowId": _flow_id(flow_names)} + if flow_names: + flow.update({"fromStepId": flow_names[0], "throughStepId": flow_names[-1]}) + inputs, input_bindings = _workspace_inputs(workspace) + return { + "workspaceSpec": { + "schemaVersion": 1, + "design": { + "name": workspace.design.name, + "topModule": workspace.design.top_module, + "clockPort": str(workspace.parameters.data.get("clock", "")), + }, + "inputMode": "postSynthesis" if workspace.design.origin_def is not None else "rtl", + "inputs": inputs, + "pdk": { + "familyId": workspace.pdk.name, + "version": workspace.pdk.version, + "mode": "default", + }, + "flow": flow, + "parameters": parameters, + }, + "workspaceBindings": { + "inputs": input_bindings, + "pdk": { + "root": str(workspace.pdk.root or ""), + "version": workspace.pdk.version, + }, + }, + } + + +def read_workspace_configuration_from_directory( + target_directory: str | Path, +) -> dict[str, Any]: + target = Path(target_directory).expanduser().resolve() + workspace = load_workspace(target) + if workspace is None: + raise WorkspaceLifecycleError("workspace_missing", f"Workspace not found: {target}") + return read_workspace_configuration(workspace) + + +def _workspace_inputs(workspace: Any) -> tuple[list[dict[str, str]], dict[str, str]]: + values = [] + if workspace.design.input_filelist is not None: + values.append(("filelist", workspace.design.input_filelist)) + elif workspace.design.origin_verilog is not None: + role = "netlist" if workspace.design.origin_def is not None else "rtl" + values.append((role, workspace.design.origin_verilog)) + for role, path in ( + ("def", workspace.design.origin_def), + ("goldenNetlist", workspace.design.golden_verilog), + ("sdc", workspace.pdk.sdc), + ("spef", workspace.pdk.spef), + ): + if path is not None: + values.append((role, path)) + return ( + [{"inputId": role, "role": role} for role, _path in values], + {role: str(path) for role, path in values}, + ) + + +def _flow_id(names: list[str]) -> str: + for flow_id, builder in get_flow_builders().items(): + candidate = [str(getattr(step, "value", step)) for step, _tool, _state in builder()] + if candidate == names: + return flow_id + return "custom" + + +def _workspace_file_paths(target: Path) -> list[Path]: + paths = [ + target / "home" / "params.toml", + target / "home" / "flow.json", + target / "home" / SNAPSHOT_FILENAME, + target / "home" / STALE_SNAPSHOT_FILENAME, + target / "home" / "workspace-commands.json", + ] + paths.extend(path for key, path in workspace_config_paths(target).items() if key != "dir") + return paths + + +def _snapshot_files(paths: list[Path]) -> dict[Path, bytes | None]: + return {path: path.read_bytes() if path.is_file() else None for path in paths} + + +def _restore_files(snapshot: dict[Path, bytes | None]) -> None: + for path, content in snapshot.items(): + if content is None: + path.unlink(missing_ok=True) + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) diff --git a/chipcompiler/engine/workspace_lifecycle.py b/chipcompiler/engine/workspace_lifecycle.py index ea561c5f..aa41cc74 100644 --- a/chipcompiler/engine/workspace_lifecycle.py +++ b/chipcompiler/engine/workspace_lifecycle.py @@ -1,10 +1,13 @@ +import hashlib +import json import shutil import tempfile from pathlib import Path from typing import Any from chipcompiler.cli.project.params import build_backend_overrides, resolve_parameters -from chipcompiler.data import PDK, create_workspace +from chipcompiler.data import PDK, create_workspace, load_workspace +from chipcompiler.engine.snapshot import create_engineering_snapshot from chipcompiler.engine.workspace_spec import validate_workspace_spec from chipcompiler.rtl2gds import get_flow_builders @@ -22,9 +25,11 @@ def create_workspace_from_spec( bindings: object, command_id: str = "", ): - del command_id target = Path(target_directory).expanduser().resolve() + fingerprint = _workspace_command_fingerprint("create", spec, bindings) if target.exists(): + if command_id and _command_retry_matches(target, command_id, fingerprint): + return _load_committed_workspace(target) raise WorkspaceLifecycleError("workspace_exists", f"Workspace already exists: {target}") validation = validate_workspace_spec(spec, bindings) issues = validation["issues"] @@ -96,6 +101,14 @@ def create_workspace_from_spec( raise WorkspaceLifecycleError( "workspace_create_failed", f"Workspace creation failed: {target}" ) + snapshot = create_engineering_snapshot(workspace) + _write_workspace_command( + target, + command_id, + fingerprint, + snapshot["workspaceId"], + snapshot["workspaceRevision"], + ) return workspace except Exception: shutil.rmtree(target, ignore_errors=True) @@ -142,3 +155,74 @@ def _string_keyed_dict(value: object) -> dict[str, Any]: if not isinstance(value, dict): return {} return {str(key): item for key, item in value.items()} + + +def _load_committed_workspace(path: Path): + workspace = load_workspace(path) + if workspace is None: + raise WorkspaceLifecycleError("workspace_invalid", f"Workspace cannot be opened: {path}") + return workspace + + +def _workspace_command_fingerprint( + kind: str, + spec: object, + bindings: object, + expected_revision: int | None = None, +) -> str: + payload = json.dumps( + { + "kind": kind, + "spec": spec, + "bindings": bindings, + "expectedWorkspaceRevision": expected_revision, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + return hashlib.sha256(payload).hexdigest() + + +def _workspace_commands(path: Path) -> dict[str, Any]: + command_path = path / "home" / "workspace-commands.json" + try: + payload = json.loads(command_path.read_text(encoding="utf-8")) + except FileNotFoundError: + return {"schemaVersion": 1, "commands": {}} + if payload.get("schemaVersion") != 1 or not isinstance(payload.get("commands"), dict): + raise WorkspaceLifecycleError("workspace_invalid", "Invalid Workspace command ledger") + return payload + + +def _command_retry_matches(path: Path, command_id: str, fingerprint: str) -> bool: + record = _workspace_commands(path)["commands"].get(command_id) + if record is None: + return False + if not isinstance(record, dict) or record.get("fingerprint") != fingerprint: + raise WorkspaceLifecycleError( + "idempotency_conflict", f"command id reused with different input: {command_id}" + ) + return True + + +def _write_workspace_command( + path: Path, + command_id: str, + fingerprint: str, + workspace_id: str, + workspace_revision: int, +) -> None: + if not command_id: + return + from chipcompiler.utility import json_write + + payload = _workspace_commands(path) + payload["commands"][command_id] = { + "fingerprint": fingerprint, + "result": { + "workspaceId": workspace_id, + "workspaceRevision": workspace_revision, + }, + } + if not json_write(path / "home" / "workspace-commands.json", payload): + raise OSError("Failed to persist Workspace command ledger") diff --git a/test/engine/test_workspace_configuration.py b/test/engine/test_workspace_configuration.py new file mode 100644 index 00000000..dc9caca5 --- /dev/null +++ b/test/engine/test_workspace_configuration.py @@ -0,0 +1,94 @@ +import json +from copy import deepcopy +from pathlib import Path + +import pytest + +from chipcompiler.engine import ( + WorkspaceLifecycleError, + create_workspace_from_spec, + read_workspace_configuration_from_directory, + update_workspace_configuration, +) +from chipcompiler.engine.snapshot import read_engineering_snapshot + + +def _workspace_spec_fixture() -> tuple[dict, dict]: + root = Path(__file__).parents[1] / "fixtures" / "workspace_spec" + payload = json.loads((root / "valid.json").read_text(encoding="utf-8")) + bindings = deepcopy(payload["workspaceBindings"]) + bindings["inputs"] = {key: str(root / value) for key, value in bindings["inputs"].items()} + return payload["workspaceSpec"], bindings + + +def test_workspace_configuration_update_is_revision_aware_and_idempotent( + tmp_path, minimal_ics55_pdk_factory +): + spec, bindings = _workspace_spec_fixture() + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings, "create-1") + initial = read_engineering_snapshot(workspace) + + updated = update_workspace_configuration( + workspace.directory, + initial["workspaceRevision"], + {"parameters": {"design.frequency_mhz": 250.0}}, + bindings, + "configuration-1", + ) + current = read_engineering_snapshot(updated) + repeated = update_workspace_configuration( + workspace.directory, + initial["workspaceRevision"], + {"parameters": {"design.frequency_mhz": 250.0}}, + bindings, + "configuration-1", + ) + + assert initial["workspaceRevision"] == 1 + assert current["workspaceRevision"] == 2 + assert read_engineering_snapshot(repeated)["workspaceRevision"] == 2 + assert repeated.parameters.data["frequency_max"] == 250.0 + assert ( + read_workspace_configuration_from_directory(workspace.directory)["workspaceSpec"][ + "parameters" + ]["design.frequency_mhz"] + == 250.0 + ) + + with pytest.raises(WorkspaceLifecycleError) as conflict: + update_workspace_configuration( + workspace.directory, + 1, + {"parameters": {"design.frequency_mhz": 300.0}}, + bindings, + "configuration-2", + ) + assert conflict.value.code == "revision_conflict" + assert read_engineering_snapshot(updated)["workspaceRevision"] == 2 + + +def test_workspace_configuration_update_rolls_back_refresh_failure( + tmp_path, minimal_ics55_pdk_factory, monkeypatch +): + spec, bindings = _workspace_spec_fixture() + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings, "create-1") + before_config = (workspace.directory / "home" / "params.toml").read_bytes() + before_snapshot = read_engineering_snapshot(workspace) + + def fail_refresh(_workspace): + raise RuntimeError("config regeneration failed") + + monkeypatch.setattr("chipcompiler.data.refresh_workspace_config", fail_refresh) + with pytest.raises(RuntimeError, match="config regeneration failed"): + update_workspace_configuration( + workspace.directory, + before_snapshot["workspaceRevision"], + {"parameters": {"design.frequency_mhz": 250.0}}, + bindings, + "configuration-1", + ) + + assert (workspace.directory / "home" / "params.toml").read_bytes() == before_config + assert read_engineering_snapshot(workspace) == before_snapshot From 6c5e7a0f8b6031b6d5c55a6771e975e00f5f36db Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 00:46:47 +0800 Subject: [PATCH 05/43] feat(workspace): add scoped step configuration --- chipcompiler/engine/__init__.py | 6 + .../engine/workspace_configuration.py | 177 +++++++++++++++++- test/engine/test_workspace_configuration.py | 63 ++++++- 3 files changed, 244 insertions(+), 2 deletions(-) diff --git a/chipcompiler/engine/__init__.py b/chipcompiler/engine/__init__.py index 98c13608..a6449663 100644 --- a/chipcompiler/engine/__init__.py +++ b/chipcompiler/engine/__init__.py @@ -3,9 +3,12 @@ from .rerun import StepRunResult from .signoff import SignoffPackageCollector, SignoffPackageOptions from .workspace_configuration import ( + read_step_configuration, + read_step_configuration_from_directory, read_workspace_configuration, read_workspace_configuration_from_directory, update_workspace_configuration, + update_workspace_step_configuration, ) from .workspace_lifecycle import WorkspaceLifecycleError, create_workspace_from_spec from .workspace_spec import describe_workspace_spec, validate_workspace_spec @@ -19,8 +22,11 @@ "WorkspaceLifecycleError", "create_workspace_from_spec", "describe_workspace_spec", + "read_step_configuration", + "read_step_configuration_from_directory", "read_workspace_configuration", "read_workspace_configuration_from_directory", + "update_workspace_step_configuration", "update_workspace_configuration", "validate_workspace_spec", ] diff --git a/chipcompiler/engine/workspace_configuration.py b/chipcompiler/engine/workspace_configuration.py index 57357baa..196e62a6 100644 --- a/chipcompiler/engine/workspace_configuration.py +++ b/chipcompiler/engine/workspace_configuration.py @@ -1,3 +1,4 @@ +from copy import deepcopy from pathlib import Path from typing import Any @@ -21,7 +22,7 @@ ensure_engineering_snapshot, invalidate_engineering_snapshot, ) -from chipcompiler.rtl2gds import get_flow_builders +from chipcompiler.rtl2gds import get_flow_builders, normalize_flow_step from .workspace_lifecycle import ( WorkspaceLifecycleError, @@ -189,6 +190,180 @@ def read_workspace_configuration(workspace: Any) -> dict[str, Any]: } +def read_step_configuration(workspace: Any, step_id: str) -> dict[str, Any]: + step, schemas = _step_catalog(workspace, step_id) + snapshot = ensure_engineering_snapshot(workspace) + if not schemas: + raise WorkspaceLifecycleError( + "step_configuration_unavailable", + f"Flow Step has no configurable parameters: {step}", + { + "workspaceId": snapshot["workspaceId"], + "workspaceRevision": snapshot["workspaceRevision"], + }, + ) + return { + "step": step, + "stepId": step, + "parameters": [_public_parameter_record(workspace, schema) for schema in schemas], + "workspaceId": snapshot["workspaceId"], + "workspaceRevision": snapshot["workspaceRevision"], + } + + +def read_step_configuration_from_directory( + target_directory: str | Path, + step_id: str, +) -> dict[str, Any]: + workspace = _load_committed_workspace(Path(target_directory).expanduser().resolve()) + return read_step_configuration(workspace, step_id) + + +def update_workspace_step_configuration( + target_directory: str | Path, + expected_workspace_revision: int, + step_id: str, + parameters: object, + command_id: str = "", +): + target = Path(target_directory).expanduser().resolve() + if not target.is_dir(): + raise WorkspaceLifecycleError("workspace_missing", f"Workspace not found: {target}") + before = _snapshot_files(_workspace_file_paths(target)) + try: + return _update_workspace_step_configuration( + target, + expected_workspace_revision, + step_id, + parameters, + command_id, + ) + except BaseException: + _restore_files(before) + raise + + +def _update_workspace_step_configuration( + target: Path, + expected_workspace_revision: int, + step_id: str, + parameters: object, + command_id: str, +): + if not isinstance(step_id, str) or not step_id.strip() or not isinstance(parameters, dict): + raise WorkspaceLifecycleError( + "workspace_spec_invalid", "Step identity and parameters object are required" + ) + patch = {str(key): value for key, value in parameters.items()} + fingerprint = _workspace_command_fingerprint( + "step_configuration", + {"stepId": step_id, "parameters": patch}, + {}, + expected_workspace_revision, + ) + if command_id and _command_retry_matches(target, command_id, fingerprint): + return _load_committed_workspace(target) + + workspace = _load_committed_workspace(target) + snapshot = ensure_engineering_snapshot(workspace) + if snapshot["workspaceRevision"] != expected_workspace_revision: + raise WorkspaceLifecycleError( + "revision_conflict", + "Workspace Revision does not match", + { + "expectedWorkspaceRevision": expected_workspace_revision, + "actualWorkspaceRevision": snapshot["workspaceRevision"], + }, + ) + step, schemas = _step_catalog(workspace, step_id) + allowed = {schema.param: schema for schema in schemas} + changed = False + for parameter, value in patch.items(): + schema = lookup_schema(parameter) + if schema is None or schema.pdk_target is not None: + raise WorkspaceLifecycleError("unknown_parameter", f"Unknown parameter: {parameter}") + if parameter not in allowed: + raise WorkspaceLifecycleError( + "parameter_not_applicable", + f"Parameter {parameter} is not configurable at {step}", + ) + normalized, type_error = _validate_schema_type(value, schema) + errors = [type_error] if type_error else validate_value(normalized, schema) + if errors: + raise WorkspaceLifecycleError("invalid_parameter", str(errors[0])) + if workspace_param_value(workspace, schema) != normalized: + update_workspace_param_value(workspace, schema, normalized) + changed = True + + if not changed: + _write_workspace_command( + target, + command_id, + fingerprint, + snapshot["workspaceId"], + snapshot["workspaceRevision"], + ) + return workspace + if not save_parameter(workspace.parameters): + raise OSError("Failed to save Workspace parameters") + from chipcompiler.data import refresh_workspace_config + + refresh_workspace_config(workspace) + invalidate_from(EngineFlow(workspace), step) + updated = invalidate_engineering_snapshot( + workspace, + workspace_id=snapshot["workspaceId"], + cause="workspace.step_configuration_updated", + first_invalidated_step=step, + ) + _write_workspace_command( + target, + command_id, + fingerprint, + updated["workspaceId"], + updated["workspaceRevision"], + ) + return _load_committed_workspace(target) + + +def _step_catalog(workspace: Any, step_id: str): + identity = normalize_flow_step(step_id).casefold() + steps = [str(step.get("name", "")) for step in workspace.flow.steps() if step.get("name")] + step = next( + (candidate for candidate in steps if normalize_flow_step(candidate).casefold() == identity), + None, + ) + if step is None: + raise WorkspaceLifecycleError("unknown_flow_step", f"Flow Step not found: {step_id}") + first = normalize_flow_step(steps[0]).casefold() + schemas = tuple( + schema + for schema in list_schemas() + if schema.pdk_target is None + and ( + normalize_flow_step(schema.applies).casefold() == identity + or (schema.applies == "all" and identity == first) + ) + ) + return step, schemas + + +def _public_parameter_record(workspace: Any, schema) -> dict[str, Any]: + record = { + "param": schema.param, + "type": schema.type, + "value": workspace_param_value(workspace, schema), + "default": deepcopy(schema.default), + "applies": schema.applies, + "description": schema.description, + } + for field in ("range", "choices", "unit"): + value = getattr(schema, field) + if value is not None: + record[field] = list(value) if isinstance(value, tuple) else value + return record + + def read_workspace_configuration_from_directory( target_directory: str | Path, ) -> dict[str, Any]: diff --git a/test/engine/test_workspace_configuration.py b/test/engine/test_workspace_configuration.py index dc9caca5..2b7702cd 100644 --- a/test/engine/test_workspace_configuration.py +++ b/test/engine/test_workspace_configuration.py @@ -5,12 +5,15 @@ import pytest from chipcompiler.engine import ( + EngineFlow, WorkspaceLifecycleError, create_workspace_from_spec, + read_step_configuration, read_workspace_configuration_from_directory, update_workspace_configuration, + update_workspace_step_configuration, ) -from chipcompiler.engine.snapshot import read_engineering_snapshot +from chipcompiler.engine.snapshot import create_engineering_snapshot, read_engineering_snapshot def _workspace_spec_fixture() -> tuple[dict, dict]: @@ -92,3 +95,61 @@ def fail_refresh(_workspace): assert (workspace.directory / "home" / "params.toml").read_bytes() == before_config assert read_engineering_snapshot(workspace) == before_snapshot + + +def test_step_configuration_update_invalidates_only_target_suffix( + tmp_path, minimal_ics55_pdk_factory +): + spec, bindings = _workspace_spec_fixture() + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings, "create-1") + flow = EngineFlow(workspace) + for step in flow.workspace.flow.data["steps"][:3]: + step["state"] = "Success" + assert flow.save() + initial = read_engineering_snapshot(workspace) + create_engineering_snapshot( + workspace, + workspace_id=initial["workspaceId"], + workspace_revision=initial["workspaceRevision"], + ) + + configuration = read_step_configuration(workspace, "floor-plan") + synthesis = read_step_configuration(workspace, "synthesis") + updated = update_workspace_step_configuration( + workspace.directory, + configuration["workspaceRevision"], + "Floorplan", + {"floorplan.core_util": 0.55}, + "step-configuration-1", + ) + + records = {record["param"]: record for record in configuration["parameters"]} + synthesis_params = {record["param"] for record in synthesis["parameters"]} + assert {"design.frequency_mhz", "flow.run_analysis"} <= synthesis_params + assert "floorplan.core_util" in records + assert set(records["floorplan.core_util"]) == { + "param", + "type", + "value", + "default", + "applies", + "description", + "range", + } + states = {step["name"]: step["state"] for step in updated.flow.steps()} + assert states["Synthesis"] == "Success" + assert states["lec"] == "Success" + assert states["Floorplan"] == "Unstart" + assert read_engineering_snapshot(updated)["workspaceRevision"] == 2 + + with pytest.raises(WorkspaceLifecycleError) as inapplicable: + update_workspace_step_configuration( + workspace.directory, + 2, + "Floorplan", + {"place.target_density": 0.6}, + "step-configuration-2", + ) + assert inapplicable.value.code == "parameter_not_applicable" + assert read_engineering_snapshot(updated)["workspaceRevision"] == 2 From 7b0e20d96bcc106417b4735c16c144887b5030ea Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 00:54:31 +0800 Subject: [PATCH 06/43] feat(engine): add headless execution contracts --- chipcompiler/engine/__init__.py | 15 +++- chipcompiler/engine/execution.py | 85 +++++++++++++++++++++ chipcompiler/engine/snapshot.py | 35 +++++++++ chipcompiler/engine/workspace_flow.py | 89 ++++++++++++++++++++++ chipcompiler/engine/workspace_lifecycle.py | 47 +++++++++++- test/engine/test_execution.py | 50 ++++++++++++ test/engine/test_workspace_flow.py | 14 ++++ 7 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 chipcompiler/engine/execution.py create mode 100644 chipcompiler/engine/workspace_flow.py create mode 100644 test/engine/test_execution.py create mode 100644 test/engine/test_workspace_flow.py diff --git a/chipcompiler/engine/__init__.py b/chipcompiler/engine/__init__.py index a6449663..95a7ad30 100644 --- a/chipcompiler/engine/__init__.py +++ b/chipcompiler/engine/__init__.py @@ -1,4 +1,5 @@ from .db import EngineDB +from .execution import ExecutionPlan, ExecutionResult, execute from .flow import EngineFlow from .rerun import StepRunResult from .signoff import SignoffPackageCollector, SignoffPackageOptions @@ -10,18 +11,30 @@ update_workspace_configuration, update_workspace_step_configuration, ) -from .workspace_lifecycle import WorkspaceLifecycleError, create_workspace_from_spec +from .workspace_lifecycle import ( + WorkspaceLifecycleError, + apply_workspace_bindings, + assess_execution_readiness, + create_workspace_from_spec, + describe_workspace_binding_requirement, +) from .workspace_spec import describe_workspace_spec, validate_workspace_spec __all__ = [ "EngineDB", "EngineFlow", + "ExecutionPlan", + "ExecutionResult", "StepRunResult", "SignoffPackageCollector", "SignoffPackageOptions", "WorkspaceLifecycleError", + "apply_workspace_bindings", + "assess_execution_readiness", "create_workspace_from_spec", "describe_workspace_spec", + "describe_workspace_binding_requirement", + "execute", "read_step_configuration", "read_step_configuration_from_directory", "read_workspace_configuration", diff --git a/chipcompiler/engine/execution.py b/chipcompiler/engine/execution.py new file mode 100644 index 00000000..e25a2cd0 --- /dev/null +++ b/chipcompiler/engine/execution.py @@ -0,0 +1,85 @@ +import inspect +from dataclasses import dataclass +from typing import Any, Literal + +from chipcompiler.data import StateEnum + + +@dataclass(frozen=True) +class ExecutionPlan: + intent: Literal["run", "rerun"] + step_id: str | None = None + + +@dataclass(frozen=True) +class ExecutionResult: + succeeded: bool + state: str + step_id: str | None = None + + +def execute(flow: Any, plan: ExecutionPlan, *, event_sink: Any = None) -> ExecutionResult: + if plan.intent not in {"run", "rerun"}: + raise ValueError(f"unsupported execution intent: {plan.intent}") + if event_sink is None and getattr(flow.workspace, "directory", None): + event_sink = _EngineeringCommitSink(flow.workspace) + rerun = plan.intent == "rerun" + if plan.step_id is None: + succeeded = bool(_invoke(flow.run_steps, rerun=rerun, observer=event_sink)) + return ExecutionResult( + succeeded=succeeded, + state=StateEnum.Success.value if succeeded else StateEnum.Imcomplete.value, + ) + + get_workspace_step = getattr(flow, "get_workspace_step", None) + step = ( + get_workspace_step(plan.step_id) + if callable(get_workspace_step) + else next( + ( + candidate + for candidate in getattr(flow, "workspace_steps", []) + if getattr(candidate, "name", None) == plan.step_id + ), + None, + ) + ) + if step is None: + raise ValueError(f"step not found: {plan.step_id}") + state = _invoke(flow.run_step, step, rerun=rerun, observer=event_sink) + return ExecutionResult( + succeeded=state is StateEnum.Success, + state=getattr(state, "value", str(state)), + step_id=plan.step_id, + ) + + +class _EngineeringCommitSink: + def __init__(self, workspace: Any): + from chipcompiler.engine.snapshot import ensure_engineering_snapshot + + self.workspace = workspace + self.snapshot = ensure_engineering_snapshot(workspace) + + def commit_step(self, _step: Any, state: Any, _error: str | None = None) -> None: + from chipcompiler.engine.snapshot import commit_engineering_snapshot + + self.snapshot = commit_engineering_snapshot( + self.workspace, + workspace_id=self.snapshot["workspaceId"], + cause=f"flow_step.{getattr(state, 'value', str(state)).lower()}", + ) + + +def _invoke(callback, *args, rerun: bool, observer: Any): + parameters = inspect.signature(callback).parameters.values() + accepts_kwargs = any( + parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters + ) + kwargs = {} + names = {parameter.name for parameter in parameters} + if accepts_kwargs or "rerun" in names: + kwargs["rerun"] = rerun + if observer is not None and (accepts_kwargs or "observer" in names): + kwargs["observer"] = observer + return callback(*args, **kwargs) diff --git a/chipcompiler/engine/snapshot.py b/chipcompiler/engine/snapshot.py index e2e36426..34a6fa6d 100644 --- a/chipcompiler/engine/snapshot.py +++ b/chipcompiler/engine/snapshot.py @@ -51,6 +51,41 @@ def read_stale_engineering_snapshot(workspace: Any) -> dict[str, Any] | None: return _read_snapshot(path) if path.is_file() else None +def commit_engineering_snapshot( + workspace: Any, + *, + workspace_id: str, + cause: str, +) -> dict[str, Any]: + current = read_engineering_snapshot(workspace) + if current["workspaceId"] != workspace_id: + raise EngineeringSnapshotError("Workspace identity changed before commit") + snapshot = _build_snapshot( + workspace, + workspace_id=workspace_id, + workspace_revision=current["workspaceRevision"] + 1, + cause=cause, + ) + stale = current.get("stalePredecessor") + if isinstance(stale, dict): + states = { + str(step.get("name")): step.get("state") + for step in snapshot.get("flow", {}).get("steps", []) + if isinstance(step, dict) and step.get("name") + } + remaining = [ + step_id + for step_id in stale.get("invalidatedStepIds", []) + if states.get(step_id) not in {"Success", "Skipped"} + ] + if remaining: + snapshot["stalePredecessor"] = {**deepcopy(stale), "invalidatedStepIds": remaining} + else: + _stale_snapshot_path(workspace).unlink(missing_ok=True) + _write_snapshot(_snapshot_path(workspace), snapshot) + return snapshot + + def invalidate_engineering_snapshot( workspace: Any, *, diff --git a/chipcompiler/engine/workspace_flow.py b/chipcompiler/engine/workspace_flow.py new file mode 100644 index 00000000..0d10bd13 --- /dev/null +++ b/chipcompiler/engine/workspace_flow.py @@ -0,0 +1,89 @@ +from typing import Any + + +def build_flow_for_workspace(workspace, *, create_step_workspaces: bool = True): + import chipcompiler.engine as engine_api + + engine_flow = engine_api.EngineFlow(workspace=workspace) + if not engine_flow.has_init(): + raise ValueError("Workspace has no committed Flow") + + if create_step_workspaces: + engine_flow.create_step_workspaces() + return engine_flow + + +def workspace_step_from_flow(workspace, name: str): + previous_step = None + loader = getattr(workspace.flow, "steps", None) + steps = loader() if callable(loader) else workspace.flow.data.get("steps", []) + for flow_step in steps: + workspace_step = _build_workspace_step_for_info(workspace, flow_step, previous_step) + if flow_step.get("name") == name: + return workspace_step + if workspace_step is not None: + previous_step = workspace_step + return None + + +def _build_workspace_step_for_info(workspace, flow_step: dict, previous_step): + step_name = flow_step.get("name") + tool = flow_step.get("tool") + if not step_name or not tool: + return None + + if previous_step is None: + input_def = workspace.design.origin_def + input_verilog = workspace.design.origin_verilog + input_db = None + else: + input_def = previous_step.output.def_ or "" + input_verilog = previous_step.output.verilog or "" + input_db = previous_step.output.db or "" + + builder = _load_tool_builder(tool) + if builder is None or not hasattr(builder, "build_step"): + return None + + return builder.build_step( + workspace=workspace, + step_name=step_name, + input_def=input_def, + input_verilog=input_verilog, + input_db=input_db, + ) + + +def _load_tool_builder(tool: str): + import importlib + + module_alias = { + "klayout": "klayout_tool", + "dreamplace": "ecc_dreamplace", + "sizer": "ecc_sizer", + } + module_name = module_alias.get(tool, tool) + return importlib.import_module(f"chipcompiler.tools.{module_name}.builder") + + +def init_db_engine_for_workspace_step(engine_flow, workspace_step): + engine_db = getattr(engine_flow, "engine_db", None) + if engine_db is None: + from chipcompiler.engine import EngineDB + + engine_db = EngineDB(workspace=engine_flow.workspace) + engine_flow.engine_db = engine_db + elif engine_db.has_init(): + return True + + return engine_db.create_db_engine(step=workspace_step) + + +def success_state(): + from chipcompiler.data import StateEnum + + return StateEnum.Success + + +def state_value(state: Any) -> str: + return getattr(state, "value", str(state)) diff --git a/chipcompiler/engine/workspace_lifecycle.py b/chipcompiler/engine/workspace_lifecycle.py index aa41cc74..679e4ca5 100644 --- a/chipcompiler/engine/workspace_lifecycle.py +++ b/chipcompiler/engine/workspace_lifecycle.py @@ -6,7 +6,7 @@ from typing import Any from chipcompiler.cli.project.params import build_backend_overrides, resolve_parameters -from chipcompiler.data import PDK, create_workspace, load_workspace +from chipcompiler.data import PDK, create_workspace, get_pdk, load_workspace from chipcompiler.engine.snapshot import create_engineering_snapshot from chipcompiler.engine.workspace_spec import validate_workspace_spec from chipcompiler.rtl2gds import get_flow_builders @@ -19,6 +19,51 @@ def __init__(self, code: str, message: str, details: dict[str, Any] | None = Non self.details = details or {} +def describe_workspace_binding_requirement( + workspace_directory: str | Path, +) -> dict[str, Any]: + workspace = _load_committed_workspace(Path(workspace_directory).expanduser().resolve()) + return { + "familyId": workspace.pdk.name, + "version": workspace.pdk.version or "unversioned", + "mode": "default", + } + + +def assess_execution_readiness( + workspace_directory: str | Path, + bindings: object | None, +) -> dict[str, Any]: + try: + workspace = _load_committed_workspace(Path(workspace_directory).expanduser().resolve()) + except (OSError, ValueError, WorkspaceLifecycleError): + return {"ready": False, "code": "workspace_invalid"} + binding_map = _string_keyed_dict(bindings) + pdk_binding = _string_keyed_dict(binding_map.get("pdk")) + root = pdk_binding.get("root") + if not isinstance(root, str) or not Path(root).is_dir(): + return {"ready": False, "code": "pdk_binding_missing"} + try: + get_pdk(workspace.pdk.name, pdk_root=root).validate() + except (OSError, ValueError): + return {"ready": False, "code": "pdk_binding_mismatch"} + return {"ready": True} + + +def apply_workspace_bindings(workspace, bindings: object) -> None: + binding_map = _string_keyed_dict(bindings) + pdk_binding = _string_keyed_dict(binding_map.get("pdk")) + root = pdk_binding.get("root") + if not isinstance(root, str) or not Path(root).is_dir(): + raise WorkspaceLifecycleError("pdk_binding_missing", "PDK binding root is required") + sdc, spef = workspace.pdk.sdc, workspace.pdk.spef + workspace.pdk = get_pdk(workspace.pdk.name, pdk_root=root) + workspace.pdk.sdc, workspace.pdk.spef = sdc, spef + from chipcompiler.data import refresh_workspace_config + + refresh_workspace_config(workspace) + + def create_workspace_from_spec( target_directory: str | Path, spec: object, diff --git a/test/engine/test_execution.py b/test/engine/test_execution.py new file mode 100644 index 00000000..ba0851f0 --- /dev/null +++ b/test/engine/test_execution.py @@ -0,0 +1,50 @@ +from types import SimpleNamespace + +from chipcompiler.data import StateEnum +from chipcompiler.engine.execution import ExecutionPlan, execute + + +def test_execution_plan_dispatches_full_flow_and_single_step(): + calls = [] + + class Flow: + def run_steps(self, *, rerun=False, observer=None): + calls.append(("flow", rerun, observer)) + return True + + def get_workspace_step(self, step_id): + return SimpleNamespace(name=step_id) + + def run_step(self, step, *, rerun=False, observer=None): + calls.append((step.name, rerun, observer)) + return StateEnum.Success + + observer = object() + flow = Flow() + + assert execute(flow, ExecutionPlan(intent="run"), event_sink=observer).succeeded + step_result = execute( + flow, + ExecutionPlan(intent="rerun", step_id="Floorplan"), + event_sink=observer, + ) + + assert step_result == step_result.__class__( + succeeded=True, + state=StateEnum.Success.value, + step_id="Floorplan", + ) + assert calls == [("flow", False, observer), ("Floorplan", True, observer)] + + +def test_execution_failure_keeps_main_blocking_semantics(): + class Flow: + workspace = SimpleNamespace(flow=SimpleNamespace(data={"steps": []})) + + def run_steps(self, *, rerun=False, observer=None): + return False + + result = execute(Flow(), ExecutionPlan(intent="run"), event_sink=object()) + + assert not result.succeeded + assert result.state == StateEnum.Imcomplete.value diff --git a/test/engine/test_workspace_flow.py b/test/engine/test_workspace_flow.py new file mode 100644 index 00000000..06d54bf7 --- /dev/null +++ b/test/engine/test_workspace_flow.py @@ -0,0 +1,14 @@ +from chipcompiler.engine.workspace_flow import build_flow_for_workspace + + +def test_build_flow_requires_committed_flow(tmp_path): + from chipcompiler.data import Workspace + + workspace = Workspace(directory=tmp_path) + + try: + build_flow_for_workspace(workspace, create_step_workspaces=False) + except ValueError as error: + assert str(error) == "Workspace has no committed Flow" + else: + raise AssertionError("missing committed Flow was accepted") From eeefd81c045a232a49f77e1667a4c824a3fb782f Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 01:00:39 +0800 Subject: [PATCH 07/43] feat(engine): expose analysis and signoff facts --- chipcompiler/engine/analysis.py | 431 ++++++++++++++++++++++ chipcompiler/engine/qor.py | 123 ++++++ chipcompiler/engine/qor_scoring.py | 163 ++++++++ chipcompiler/engine/signoff_assessment.py | 174 +++++++++ chipcompiler/engine/signoff_export.py | 72 ++++ chipcompiler/engine/snapshot.py | 16 +- test/engine/test_qor_scoring.py | 34 ++ 7 files changed, 1008 insertions(+), 5 deletions(-) create mode 100644 chipcompiler/engine/analysis.py create mode 100644 chipcompiler/engine/qor.py create mode 100644 chipcompiler/engine/qor_scoring.py create mode 100644 chipcompiler/engine/signoff_assessment.py create mode 100644 chipcompiler/engine/signoff_export.py create mode 100644 test/engine/test_qor_scoring.py diff --git a/chipcompiler/engine/analysis.py b/chipcompiler/engine/analysis.py new file mode 100644 index 00000000..2e1c58c2 --- /dev/null +++ b/chipcompiler/engine/analysis.py @@ -0,0 +1,431 @@ +import hashlib +import math +from pathlib import Path +from typing import Any, TypeGuard + +from chipcompiler.data.step_dirs import STEP_DIRECTORIES +from chipcompiler.tools.ecc.sta_qor import STA_POWER_REPORT_FILENAME, STA_REPORT_FILENAMES +from chipcompiler.utility import JsonReadError, file_digest, json_read_strict + +_ANALYSIS_FILES = ( + ("metrics", "qor_metrics", "qor_metrics.json", 3), + ("summary", "qor_summary", "qor_summary.json", 4), + ("hotspots", "qor_hotspots", "qor_hotspots.json", 3), +) +_TIMING_FILE = ("timingIssues", "sta_timing_issues", "sta_timing_issues.json", 1) +_SUBFLOW_MAX_BYTES = 1024 * 1024 +_SUBFLOW_MAX_STEPS = 256 +_ARTIFACT_HASH_MAX_BYTES = 16 * 1024 * 1024 +_CONGESTION_IMAGES = ( + ("egr_congestion_map", "{step}_egr_horizontal_overflow.png"), + ("egr_congestion_map", "{step}_egr_vertical_overflow.png"), + ("egr_congestion_map", "{step}_egr_union_overflow.png"), + ("RUDY_map", "{step}_rudy_horizontal.png"), + ("RUDY_map", "{step}_rudy_vertical.png"), + ("RUDY_map", "{step}_rudy_union.png"), + ("RUDY_map", "{step}_lut_rudy_horizontal.png"), + ("RUDY_map", "{step}_lut_rudy_vertical.png"), + ("RUDY_map", "{step}_lut_rudy_union.png"), + ("density_map", "{step}_allcell_density.png"), +) + + +def build_workspace_analysis( + workspace: Any, workspace_id: str +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + root = Path(workspace.directory).resolve() + flow = getattr(getattr(workspace, "flow", None), "data", {}) + raw_steps = flow.get("steps", []) if isinstance(flow, dict) else [] + design = str(getattr(getattr(workspace, "design", None), "name", "")).strip() + steps: list[dict[str, Any]] = [] + artifacts: list[dict[str, Any]] = [] + for order, raw_step in enumerate(raw_steps): + if not isinstance(raw_step, dict): + continue + step_id = raw_step.get("name") + tool_id = raw_step.get("tool") + if not _safe_segment(step_id) or not _safe_segment(tool_id): + continue + identity = "".join(character for character in step_id.casefold() if character.isalnum()) + if identity == "fixfanout": + continue + step_dir = root / STEP_DIRECTORIES.get(step_id, f"{step_id}_{tool_id}") + step: dict[str, Any] = { + "stepId": step_id, + "toolId": tool_id, + "order": order, + "flowState": str(raw_step.get("state", "")), + } + declared = list(_ANALYSIS_FILES) + if step_id.lower() == "sta": + declared.append(_TIMING_FILE) + for field, kind, filename, schema_version in declared: + path = step_dir / "analysis" / filename + reference = path.relative_to(root).as_posix() + artifact = _artifact_ref( + path, + workspace_id=workspace_id, + reference=reference, + step_id=step_id, + kind=kind, + root=root, + ) + artifacts.append(artifact) + step[field] = _analysis_file(path, artifact["artifactId"], schema_version, root) + if "timingIssues" not in step: + step["timingIssues"] = None + if tool_id.lower() == "yosys_lec" and design: + result = step_dir / "output" / f"{design}_{step_id}_result.json" + artifact = _artifact_ref( + result, + workspace_id=workspace_id, + reference=result.relative_to(root).as_posix(), + step_id=step_id, + kind="lec_result", + root=root, + ) + artifacts.append(artifact) + step["lecResult"] = _lec_result_file(result, artifact["artifactId"], root) + step["subflow"] = _subflow_summary(step_dir / "subflow.json", root) + if design: + layout = step_dir / "output" / f"{design}_{step_id}.png" + artifacts.append( + _artifact_ref( + layout, + workspace_id=workspace_id, + reference=layout.relative_to(root).as_posix(), + step_id=step_id, + kind="layout_image", + root=root, + ) + ) + if step_id.lower() == "harden": + for suffix in ("gds", "lef", "lib"): + output = step_dir / "output" / f"{design}_{step_id}.{suffix}" + artifacts.append( + _artifact_ref( + output, + workspace_id=workspace_id, + reference=output.relative_to(root).as_posix(), + step_id=step_id, + kind="harden_output", + root=root, + ) + ) + geometry = step_dir / "output" / "geometry" / "geometry.manifest" + artifacts.append( + _artifact_ref( + geometry, + workspace_id=workspace_id, + reference=geometry.relative_to(root).as_posix(), + step_id=step_id, + kind="layout_geometry", + root=root, + ) + ) + report_names = ( + (f"{step_id}_stat.json", f"{step_id}_check.rpt") + if tool_id.lower() == "yosys" + else (f"{step_id}.db.rpt", f"{step_id}.rpt") + ) + for report_name in report_names: + report = step_dir / "report" / report_name + artifacts.append( + _artifact_ref( + report, + workspace_id=workspace_id, + reference=report.relative_to(root).as_posix(), + step_id=step_id, + kind="report_text", + root=root, + ) + ) + timing_files: list[tuple[str, Path]] = [] + if step_id.lower() == "synthesis": + timing_root = step_dir / "feature" / "post_synthesis" + timing_files.extend( + ( + ("timing_summary", timing_root / "qor_summary.json"), + ("timing_paths", timing_root / "timing_paths.json"), + ) + ) + timing_data = step.get("timingIssues") + timing_payload = timing_data.get("data") if isinstance(timing_data, dict) else None + timing_paths = ( + timing_payload.get("artifact_paths") if isinstance(timing_payload, dict) else None + ) + if step_id.lower() == "sta": + for item in timing_paths[:32] if isinstance(timing_paths, list) else []: + if not isinstance(item, dict): + continue + report_dir = _safe_step_artifact_path(step_dir, item.get("report_dir"), root) + if report_dir is None: + continue + try: + relative_report_dir = report_dir.relative_to(step_dir / "report") + except ValueError: + continue + report_names = list(STA_REPORT_FILENAMES) + if (report_dir / STA_POWER_REPORT_FILENAME).is_file(): + report_names.append(STA_POWER_REPORT_FILENAME) + for report_name in report_names: + report = report_dir / report_name + artifact = _artifact_ref( + report, + workspace_id=workspace_id, + reference=report.relative_to(root).as_posix(), + step_id=step_id, + kind="report_text", + root=root, + ) + artifact["name"] = (relative_report_dir / report_name).as_posix() + artifacts.append(artifact) + for item in timing_paths[:32] if isinstance(timing_paths, list) else []: + if not isinstance(item, dict): + continue + for kind, field in ( + ("timing_summary", "qor_summary_file"), + ("timing_paths", "timing_paths_file"), + ): + timing_path = _safe_step_artifact_path(step_dir, item.get(field), root) + if timing_path is not None: + timing_files.append((kind, timing_path)) + for kind, path in timing_files: + artifacts.append( + _artifact_ref( + path, + workspace_id=workspace_id, + reference=path.relative_to(root).as_posix(), + step_id=step_id, + kind=kind, + root=root, + ) + ) + if step_id.lower() in {"place", "cts"}: + for directory, filename in _CONGESTION_IMAGES: + image = step_dir / "feature" / directory / filename.format(step=step_id) + artifacts.append( + _artifact_ref( + image, + workspace_id=workspace_id, + reference=image.relative_to(root).as_posix(), + step_id=step_id, + kind="congestion_image", + root=root, + ) + ) + steps.append(step) + return {"steps": steps}, artifacts + + +def _lec_result_file(path: Path, artifact_id: str, root: Path) -> dict[str, Any]: + if _has_symlink(path, root): + return { + "artifactId": artifact_id, + "status": "unsafe", + "reasonCode": "LEC_RESULT_UNSAFE", + "data": None, + } + if not path.is_file(): + return { + "artifactId": artifact_id, + "status": "missing", + "reasonCode": "LEC_RESULT_MISSING", + "data": None, + } + try: + data = json_read_strict(path) + except (OSError, JsonReadError): + return { + "artifactId": artifact_id, + "status": "invalid", + "reasonCode": "LEC_RESULT_INVALID", + "data": None, + } + if not isinstance(data, dict): + return { + "artifactId": artifact_id, + "status": "invalid", + "reasonCode": "LEC_RESULT_INVALID", + "data": None, + } + result = dict(data) + result["freshness_status"] = _lec_freshness_status(data, root) + return {"artifactId": artifact_id, "status": "available", "data": result} + + +def _lec_freshness_status(data: dict[str, Any], root: Path) -> str: + if data.get("status") != "proven": + return "incomplete" + for role in ("golden", "gate"): + path = data.get(f"{role}_verilog") + digest = data.get(f"{role}_sha256") + size = data.get(f"{role}_size_bytes") + if not isinstance(path, str) or not isinstance(digest, str) or type(size) is not int: + return "stale" + candidate = Path(path).resolve() + if not candidate.is_relative_to(root) or not candidate.is_file(): + return "stale" + actual = file_digest(candidate) + if actual is None or actual != (digest, size): + return "stale" + return "proven" + + +def _analysis_file(path: Path, artifact_id: str, schema_version: int, root: Path) -> dict[str, Any]: + if _has_symlink(path, root): + return { + "artifactId": artifact_id, + "status": "unsafe", + "reasonCode": "ANALYSIS_REFERENCE_UNSAFE", + "data": None, + } + if not path.is_file(): + return { + "artifactId": artifact_id, + "status": "missing", + "reasonCode": "ANALYSIS_FILE_MISSING", + "data": None, + } + try: + data = json_read_strict(path) + except (OSError, JsonReadError): + return { + "artifactId": artifact_id, + "status": "invalid", + "reasonCode": "ANALYSIS_FILE_INVALID", + "data": None, + } + if not isinstance(data, dict): + return { + "artifactId": artifact_id, + "status": "invalid", + "reasonCode": "ANALYSIS_FILE_INVALID", + "data": None, + } + if data.get("schema_version") != schema_version: + return { + "artifactId": artifact_id, + "status": "unsupported", + "reasonCode": "ANALYSIS_SCHEMA_UNSUPPORTED", + "data": None, + } + return {"artifactId": artifact_id, "status": "available", "data": data} + + +def _subflow_summary(path: Path, root: Path) -> dict[str, Any]: + if _has_symlink(path, root): + return {"status": "unsafe", "steps": []} + if not path.is_file(): + return {"status": "missing", "steps": []} + try: + if path.stat().st_size > _SUBFLOW_MAX_BYTES: + return {"status": "oversized", "steps": []} + data = json_read_strict(path) + except (OSError, JsonReadError): + return {"status": "invalid", "steps": []} + raw_steps = data.get("steps") if isinstance(data, dict) else None + if not isinstance(raw_steps, list) or len(raw_steps) > _SUBFLOW_MAX_STEPS: + return {"status": "invalid", "steps": []} + steps: list[dict[str, Any]] = [] + for item in raw_steps: + if not isinstance(item, dict): + return {"status": "invalid", "steps": []} + name = item.get("name") + state = item.get("state") + if not isinstance(name, str) or not name or not isinstance(state, str): + return {"status": "invalid", "steps": []} + runtime = item.get("runtime") + peak_memory = item.get("peak memory (mb)") + step: dict[str, Any] = {"name": name, "state": state} + if isinstance(runtime, str): + step["runtime"] = runtime + if ( + isinstance(peak_memory, (int, float)) + and not isinstance(peak_memory, bool) + and math.isfinite(peak_memory) + ): + step["peakMemoryMb"] = peak_memory + steps.append(step) + return {"status": "available", "steps": steps} + + +def _artifact_ref( + path: Path, + *, + workspace_id: str, + reference: str, + step_id: str, + kind: str, + root: Path, +) -> dict[str, Any]: + artifact: dict[str, Any] = { + "artifactId": _artifact_id(workspace_id, reference), + "kind": kind, + "name": path.name, + "stepId": step_id, + "availability": "missing", + "reference": reference, + } + try: + if path.is_file() and not _has_symlink(path, root): + size = path.stat().st_size + if size <= _ARTIFACT_HASH_MAX_BYTES: + artifact.update( + availability="available", + sizeBytes=size, + sha256=_sha256(path), + ) + else: + artifact.update(availability="stale", sizeBytes=size) + except OSError: + pass + return artifact + + +def _safe_step_artifact_path(step_dir: Path, value: object, root: Path) -> Path | None: + if not isinstance(value, str) or not value: + return None + candidate = Path(value) + if candidate.is_absolute() or ".." in candidate.parts: + return None + resolved = step_dir / candidate + try: + resolved.relative_to(root) + except ValueError: + return None + return resolved + + +def _safe_segment(value: object) -> TypeGuard[str]: + return ( + isinstance(value, str) + and bool(value) + and value not in {".", ".."} + and "/" not in value + and "\\" not in value + ) + + +def _has_symlink(path: Path, root: Path) -> bool: + current = path + while current != root: + if current.is_symlink(): + return True + if current.parent == current: + return True + current = current.parent + return False + + +def _artifact_id(workspace_id: str, reference: str) -> str: + digest = hashlib.sha256(f"{workspace_id}\0{reference}".encode()).hexdigest() + return f"artifact-{digest[:32]}" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as artifact: + for chunk in iter(lambda: artifact.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/chipcompiler/engine/qor.py b/chipcompiler/engine/qor.py new file mode 100644 index 00000000..6cef28c2 --- /dev/null +++ b/chipcompiler/engine/qor.py @@ -0,0 +1,123 @@ +import math +from typing import Any + +from .qor_scoring import ( + DIMENSION_WEIGHTS, + QOR_SCORE_THRESHOLD, + QorScoringMetric, + score_qor, +) + + +def build_workspace_qor_assessment(analysis: dict[str, Any]) -> dict[str, Any]: + metrics = [] + step_summaries = [] + metric_steps = [] + for step in analysis["steps"]: + if step["flowState"] != "Success": + continue + step_id = step["stepId"] + metrics_file = step["metrics"] + payload = metrics_file["data"] if metrics_file["status"] == "available" else {} + records = payload.get("metrics") + valid_records = [record for record in records or [] if _valid_metric(record)] + metrics.extend(valid_records) + metric_steps.extend((step_id, record) for record in valid_records) + summary_file = step["summary"] + summary = summary_file["data"] if summary_file["status"] == "available" else {} + summary_status = ( + str(summary.get("quality_status", "incomplete")) + if summary.get("schema_version") == 4 + else "unavailable" + ) + step_summaries.append( + { + "stepId": step_id, + "order": step["order"], + "name": step_id, + "status": summary_status, + "summaryMetricCount": len(valid_records), + } + ) + + if not metrics: + return { + "status": "unavailable", + "score": {"value": None, "threshold": QOR_SCORE_THRESHOLD, "gate": "unavailable"}, + "areaScoringStep": None, + "dimensionScores": {}, + "metrics": [], + "steps": step_summaries, + } + + gate = _gate_status(step_summaries) + scoring = score_qor( + [ + QorScoringMetric( + step=step_id, + metric_id=record["id"], + value=float(record["value"]), + dimension=record["category"], + direction=record["direction"], + scope=record["scope"], + corner=record.get("corner"), + project_role=record["project_role"], + rating_score=record["rating"]["score"], + ) + for step_id, record in metric_steps + ] + ) + return { + "status": "ready", + "score": { + "value": scoring.overall_score, + "threshold": QOR_SCORE_THRESHOLD, + "gate": gate, + }, + "areaScoringStep": scoring.area_scoring_step, + "dimensionScores": { + dimension: score for dimension, (score, _count) in scoring.dimensions.items() + }, + "metrics": metrics, + "steps": step_summaries, + } + + +def _valid_metric(record: Any) -> bool: + if not isinstance(record, dict): + return False + value = record.get("value") + rating = record.get("rating") + corner = record.get("corner") + corner_context = record.get("corner_context") + return ( + isinstance(record.get("id"), str) + and isinstance(record.get("display_name"), str) + and isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and record.get("category") in DIMENSION_WEIGHTS + and record.get("direction") + in {"higher_is_better", "lower_is_better", "target_range", "trend_only"} + and isinstance(record.get("scope"), str) + and (corner is None or isinstance(corner, str)) + and (corner_context is None or isinstance(corner_context, dict)) + and isinstance(record.get("analysis_group"), str) + and isinstance(rating, dict) + and isinstance(rating.get("gate"), bool) + and isinstance(rating.get("score"), bool) + and isinstance(rating.get("trend"), bool) + and record.get("project_role") in {"final", "trend", "gate", "none"} + and record.get("step_role") in {"primary", "secondary", "detail", "hidden"} + and record.get("confidence") in {"high", "medium", "low"} + and isinstance(record.get("source"), dict) + ) + + +def _gate_status(steps: list[dict[str, Any]]) -> str: + statuses = {step["status"] for step in steps} + if "blocked" in statuses: + return "blocked" + if statuses & {"incomplete", "unavailable"}: + return "incomplete" + return "pass" diff --git a/chipcompiler/engine/qor_scoring.py b/chipcompiler/engine/qor_scoring.py new file mode 100644 index 00000000..00fc6283 --- /dev/null +++ b/chipcompiler/engine/qor_scoring.py @@ -0,0 +1,163 @@ +from dataclasses import dataclass +from typing import Any + +QOR_SCORE_THRESHOLD = 60 + +DIMENSION_WEIGHTS = { + "timing": 0.35, + "power_integrity": 0.25, + "routability_physical": 0.2, + "area_cost": 0.1, + "clock_robustness_dfm": 0.1, + "runtime": 0.0, +} + +METRIC_FAIL_VALUES = { + "drc_count": 10, + "lvs_count": 10, + "route_wirelength": 6000, + "route_via_count": 2000, + "cts_buffer_count": 20, + "cts_buffer_area": 40, + "clock_wirelength": 400000, + "cts_clock_wirelength_max": 100000, + "cts_clock_tree_max_level": 20, + "die_area": 3000, + "core_area": 2500, + "core_utilization": 0.85, + "synthesis_cell_area": 3000, + "fanout_max": 100, + "place_hpwl": 10000, + "place_grwl": 12000, + "place_flute_wirelength": 10000, + "place_congestion_egr_overflow_total": 100, + "place_congestion_egr_overflow_max": 20, + "place_rudy_utilization_max": 1, + "place_lutrudy_utilization_max": 1, + "route_dr_total_violation_count": 50, + "route_dr_total_patch_count": 100, + "route_dr_total_wirelength": 6000, + "route_dr_total_via_count": 2000, + "route_la_total_overflow": 100, + "rcx_missing_corner_count": 9, + "sta_setup_wns": -0.2, + "sta_setup_tns": -1, + "sta_hold_wns": -0.2, + "sta_hold_tns": -1, + "sta_frequency_mhz": 100, + "sta_setup_violation_count": 1, + "sta_hold_violation_count": 1, + "sta_missing_corner_count": 1, + "harden_artifact_missing_count": 6, +} + +_SLACK_METRICS = {"sta_setup_wns", "sta_setup_tns", "sta_hold_wns", "sta_hold_tns"} +_ROLE_PRIORITY = {"final": 0, "gate": 1, "trend": 2, "none": 3} + + +@dataclass(frozen=True) +class QorScoringMetric: + step: str + metric_id: str + value: float + dimension: str + direction: str + scope: str + corner: str | None + project_role: str + rating_score: bool + payload: Any = None + + +@dataclass(frozen=True) +class ScoredQorMetric: + metric: QorScoringMetric + score: float | None + + +@dataclass(frozen=True) +class QorScoringResult: + area_scoring_step: str | None + metrics: tuple[ScoredQorMetric, ...] + dimensions: dict[str, tuple[float, int]] + overall_score: float | None + + +def score_qor(records: list[QorScoringMetric]) -> QorScoringResult: + area_step = next( + ( + record.step + for record in reversed(records) + if record.dimension == "area_cost" and record.rating_score + ), + None, + ) + selected: dict[tuple[str, str, str], tuple[int, int, QorScoringMetric]] = {} + for order, record in enumerate(records): + if record.project_role == "none": + continue + if record.dimension == "area_cost" and record.step != area_step: + continue + key = (record.metric_id, record.scope, record.corner or "") + candidate = (_ROLE_PRIORITY.get(record.project_role, 3), -order, record) + if key not in selected or candidate[:2] < selected[key][:2]: + selected[key] = candidate + + scored = tuple( + ScoredQorMetric(record, score_metric(record) if record.rating_score else None) + for _role, _order, record in sorted(selected.values(), key=lambda item: item[2].metric_id) + ) + by_dimension: dict[str, list[float]] = {} + for item in scored: + if item.score is not None: + by_dimension.setdefault(item.metric.dimension, []).append(item.score) + dimensions = { + dimension: ( + round(sum(by_dimension[dimension]) / len(by_dimension[dimension]), 1), + len(by_dimension[dimension]), + ) + for dimension in DIMENSION_WEIGHTS + if dimension in by_dimension + } + weighted = sum( + score * DIMENSION_WEIGHTS[dimension] + for dimension, (score, _count) in dimensions.items() + if DIMENSION_WEIGHTS[dimension] > 0 + ) + overall = ( + round(weighted, 1) + if any(DIMENSION_WEIGHTS[dimension] > 0 for dimension in dimensions) + else None + ) + return QorScoringResult(area_step, scored, dimensions, overall) + + +def score_metric(record: QorScoringMetric) -> float | None: + if record.direction == "trend_only": + return None + fail = METRIC_FAIL_VALUES.get(record.metric_id) + if fail is None: + return None + if record.metric_id in _SLACK_METRICS: + if fail >= 0: + return None + return 100.0 if record.value >= 0 else _clamp(100 * (record.value - fail) / -fail) + if record.direction == "target_range": + if record.metric_id != "core_utilization": + return None + if 0.45 <= record.value <= 0.7: + return 100.0 + if record.value < 0.45: + return _clamp(100 * record.value / 0.45) + return _clamp(100 * (fail - record.value) / (fail - 0.7)) + if fail <= 0: + return None + if record.direction == "lower_is_better": + return _clamp(100 * (fail - record.value) / fail) + if record.direction == "higher_is_better": + return _clamp(100 * record.value / fail) + return None + + +def _clamp(value: float) -> float: + return max(0.0, min(100.0, value)) diff --git a/chipcompiler/engine/signoff_assessment.py b/chipcompiler/engine/signoff_assessment.py new file mode 100644 index 00000000..9b378ce2 --- /dev/null +++ b/chipcompiler/engine/signoff_assessment.py @@ -0,0 +1,174 @@ +from pathlib import Path +from typing import Any, TypedDict + +from chipcompiler.utility import json_read + +_REVIEW_GROUPS: tuple[tuple[str, str], ...] = ( + ("initial", "Initial"), + ("config", "Config"), + ("harden", "Harden"), + ("final_design", "Final Design"), + ("sta", "STA"), + ("spef", "SPEF"), + ("reports", "Reports"), +) + + +class _ReviewGroup(TypedDict): + id: str + label: str + available: int + expected: int + blocked: list[dict] + attention: list[dict] + + +def build_signoff_assessment(workspace: Any) -> dict[str, Any]: + checklist = json_read(Path(workspace.directory) / "home" / "checklist.json") + if checklist.get("schema_version") != 3 or checklist.get("kind") != "signoff_checklist": + return _unavailable_assessment() + + groups: dict[str, _ReviewGroup] = {} + for group_id, label in _REVIEW_GROUPS: + groups[group_id] = { + "id": group_id, + "label": label, + "available": 0, + "expected": 0, + "blocked": [], + "attention": [], + } + for item in checklist.get("checklist", []): + if not isinstance(item, dict): + continue + group = groups[_group_for(item)] + group["expected"] += 1 + if item.get("state") == "pass": + group["available"] += 1 + elif item.get("blocked") is True: + group["blocked"].append(_detail(item)) + else: + group["attention"].append(_detail(item)) + + review_groups = [] + risks = [] + for group_id, _label in _REVIEW_GROUPS: + group = groups[group_id] + blocked = group["blocked"] + attention = group["attention"] + if blocked: + status = "blocked" + summary = f"{len(blocked)} blocking checklist requirements" + risks.append(_risk(group, "blocked", blocked, summary)) + if attention: + risks.append( + _risk( + group, + "warning", + attention, + f"{len(attention)} attention-only checklist requirements", + ) + ) + elif attention: + status = "attention" + summary = f"{len(attention)} attention-only checklist requirements" + risks.append(_risk(group, "warning", attention, summary)) + else: + status = "ready" + summary = ( + f"{group['available']} of {group['expected']} requirements ready" + if group["expected"] + else "No requirements" + ) + review_groups.append( + { + "id": group_id, + "label": group["label"], + "status": status, + "available": group["available"], + "expected": group["expected"], + "summary": summary, + } + ) + + status = checklist.get("status") + return { + "status": status if status in {"ready", "attention", "blocked"} else "blocked", + "groups": review_groups, + "risks": sorted(risks, key=lambda risk: risk["severity"] != "blocked"), + } + + +def _risk(group: _ReviewGroup, severity: str, details: list[dict], summary: str) -> dict: + result = "requirements block export" if severity == "blocked" else "attention" + return { + "severity": severity, + "title": f"{group['label']} signoff {result}", + "summary": summary, + "details": details, + } + + +def _detail(item: dict) -> dict: + source = item.get("source", {}) + source = source if isinstance(source, dict) else {} + evidence = item.get("evidence", []) + return { + "kind": item.get("category", "checklist"), + "label": item.get("title", "Checklist item"), + "location": source.get("path", "home/checklist.json"), + "reason": item.get("summary", ""), + "owner": item.get("owner", "checklist"), + "policy": item.get("policy", "warn"), + "state": item.get("state", "unavailable"), + "evidence": evidence if isinstance(evidence, list) else [], + } + + +def _group_for(item: dict) -> str: + step = str(item.get("step", "")) + category = str(item.get("category", "")) + source = item.get("source", {}) + path = source.get("path", "") if isinstance(source, dict) else "" + if category == "configuration" or path.startswith("config/"): + return "config" + if category == "provenance" or path.startswith(("origin/", "initial/")): + return "initial" + if step == "Harden" or path.startswith(("Harden_ecc/", "harden/")): + return "harden" + if step == "sta" or path.startswith(("sta_ecc/", "final/timing/sta/")): + return "sta" + if step == "RCX" or path.startswith(("RCX_ecc/", "final/timing/spef/")): + return "spef" + if step in {"Route", "drc", "lvs", "filler"} or path.startswith( + ("route_ecc/", "drc_ecc/", "lvs_ecc/", "filler_ecc/", "final/design/") + ): + return "final_design" + return "reports" + + +def _unavailable_assessment() -> dict[str, Any]: + return { + "status": "blocked", + "groups": [ + { + "id": group_id, + "label": label, + "status": "blocked" if group_id == "reports" else "ready", + "available": 0, + "expected": 0, + "summary": ( + "Checklist unavailable" if group_id == "reports" else "No requirements" + ), + } + for group_id, label in _REVIEW_GROUPS + ], + "risks": [ + { + "severity": "blocked", + "title": "Signoff checklist unavailable", + "summary": "Re-run signoff inspection after current-output analysis completes.", + "details": [], + } + ], + } diff --git a/chipcompiler/engine/signoff_export.py b/chipcompiler/engine/signoff_export.py new file mode 100644 index 00000000..1c7a19d9 --- /dev/null +++ b/chipcompiler/engine/signoff_export.py @@ -0,0 +1,72 @@ +import os +import shutil +import tempfile +from pathlib import Path + +from chipcompiler.engine import EngineFlow, SignoffPackageOptions +from chipcompiler.engine.signoff_assessment import build_signoff_assessment + + +class SignoffExportError(RuntimeError): + pass + + +def inspect_signoff_package(workspace) -> dict: + """Return the current Signoff Assessment without mutating the Workspace.""" + return build_signoff_assessment(workspace) + + +def export_signoff_package_archive( + workspace, + output_path: str, + additional_files: list[dict[str, str]] | None = None, + *, + include_debug: bool = False, +) -> str: + raw_destination = Path(output_path).expanduser() + destination = raw_destination.parent.resolve() / raw_destination.name + + with tempfile.TemporaryDirectory(prefix="ecc-signoff-") as temporary_root: + result = EngineFlow(workspace).collect_signoff_package( + SignoffPackageOptions( + output_dir=temporary_root, + archive=False, + include_debug=include_debug, + materialize=False, + refresh_analysis=False, + ) + ) + if not result.ok: + missing = ", ".join(result.missing_required) or "unknown required resources" + raise SignoffExportError(f"signoff package is incomplete: {missing}") + if not result.package_dir: + raise SignoffExportError("signoff package directory was not created") + + package_dir = Path(result.package_dir) + + if additional_files: + for file_info in additional_files: + p = package_dir / file_info["archivePath"] + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(file_info["content"], encoding="utf-8") + + archive = package_dir.with_suffix(".tar.gz") + import tarfile + + with tarfile.open(archive, "w:gz") as tar: + tar.add(package_dir, arcname=package_dir.name) + + destination.parent.mkdir(parents=True, exist_ok=True) + descriptor, staged_name = tempfile.mkstemp( + dir=destination.parent, + prefix=f".{destination.name}.", + ) + os.close(descriptor) + staged_path = Path(staged_name) + try: + shutil.copy2(archive, staged_path) + os.replace(staged_path, destination) + finally: + staged_path.unlink(missing_ok=True) + + return str(destination) diff --git a/chipcompiler/engine/snapshot.py b/chipcompiler/engine/snapshot.py index 34a6fa6d..86984912 100644 --- a/chipcompiler/engine/snapshot.py +++ b/chipcompiler/engine/snapshot.py @@ -154,6 +154,12 @@ def _build_snapshot( home = _data_mapping(getattr(workspace, "home", None)) checklist_path = home.get("checklist") checklist = json_read(checklist_path) if isinstance(checklist_path, (str, Path)) else {} + from chipcompiler.engine.analysis import build_workspace_analysis + from chipcompiler.engine.qor import build_workspace_qor_assessment + from chipcompiler.engine.signoff_assessment import build_signoff_assessment + + analysis, artifacts = build_workspace_analysis(workspace, workspace_id) + qor_assessment = build_workspace_qor_assessment(analysis) return { "schemaVersion": SNAPSHOT_SCHEMA_VERSION, "workspaceId": workspace_id, @@ -162,11 +168,11 @@ def _build_snapshot( "flow": flow, "parameters": _data_mapping(getattr(workspace, "parameters", None)), "checklist": checklist if isinstance(checklist, dict) else {}, - "analysis": {"steps": []}, - "metrics": {}, - "qorAssessment": {"status": "unavailable", "metrics": {}, "dimensions": {}}, - "signoffAssessment": {"status": "blocked", "groups": [], "risks": []}, - "artifacts": [], + "analysis": analysis, + "metrics": deepcopy(qor_assessment["metrics"]), + "qorAssessment": qor_assessment, + "signoffAssessment": build_signoff_assessment(workspace), + "artifacts": artifacts, } diff --git a/test/engine/test_qor_scoring.py b/test/engine/test_qor_scoring.py new file mode 100644 index 00000000..d61317db --- /dev/null +++ b/test/engine/test_qor_scoring.py @@ -0,0 +1,34 @@ +from chipcompiler.engine.qor_scoring import QorScoringMetric, score_qor + + +def _metric(step, metric_id, value, dimension, direction="lower_is_better", **kwargs): + return QorScoringMetric( + step=step, + metric_id=metric_id, + value=value, + dimension=dimension, + direction=direction, + scope="workspace", + corner=None, + project_role=kwargs.get("project_role", "final"), + rating_score=kwargs.get("rating_score", True), + ) + + +def test_qor_scoring_selects_latest_area_and_combines_dimension_weights(): + result = score_qor( + [ + _metric("Floor", "die_area", 300, "area_cost"), + _metric("STA", "sta_setup_wns", -0.1, "timing", "higher_is_better"), + _metric("Harden", "die_area", 1500, "area_cost"), + _metric("DRC", "drc_count", 0, "routability_physical"), + ] + ) + + assert result.area_scoring_step == "Harden" + assert result.dimensions == { + "timing": (50.0, 1), + "routability_physical": (100.0, 1), + "area_cost": (50.0, 1), + } + assert result.overall_score == 42.5 From c2bdb316de2bb7f1c190462f366df10ca15669e3 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 01:05:56 +0800 Subject: [PATCH 08/43] refactor(cli): use headless signoff interface --- chipcompiler/cli/command_handlers/signoff.py | 11 ++++++----- test/cli/commands/test_signoff.py | 8 ++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/chipcompiler/cli/command_handlers/signoff.py b/chipcompiler/cli/command_handlers/signoff.py index aa55c376..80f53e73 100644 --- a/chipcompiler/cli/command_handlers/signoff.py +++ b/chipcompiler/cli/command_handlers/signoff.py @@ -9,7 +9,7 @@ def inspect(command_input, ctx: CommandContext) -> CommandResult: if failure is not None: return failure - from chipcompiler.runtime.signoff_export import inspect_signoff_package + from chipcompiler.engine.signoff_export import inspect_signoff_package review = inspect_signoff_package(workspace) project = ctx.project @@ -56,17 +56,18 @@ def export(command_input, ctx: CommandContext) -> CommandResult: if failure is not None: return failure - from chipcompiler.runtime.workspace_api import RuntimeApiError - try: - from chipcompiler.runtime.signoff_export import export_signoff_package_archive + from chipcompiler.engine.signoff_export import ( + SignoffExportError, + export_signoff_package_archive, + ) output_path = export_signoff_package_archive( workspace, command_input.output_path, include_debug=command_input.include_debug, ) - except RuntimeApiError as exc: + except SignoffExportError as exc: return CommandResult.err( [ error_record( diff --git a/test/cli/commands/test_signoff.py b/test/cli/commands/test_signoff.py index 9df84510..07947cf9 100644 --- a/test/cli/commands/test_signoff.py +++ b/test/cli/commands/test_signoff.py @@ -63,7 +63,7 @@ def fake_load_workspace(path): def _patch_inspect(monkeypatch, review=REVIEW): monkeypatch.setattr( - "chipcompiler.runtime.signoff_export.inspect_signoff_package", + "chipcompiler.engine.signoff_export.inspect_signoff_package", lambda workspace: review, ) @@ -85,7 +85,7 @@ def fake_export(workspace, output_path, additional_files=None, *, include_debug= return destination monkeypatch.setattr( - "chipcompiler.runtime.signoff_export.export_signoff_package_archive", fake_export + "chipcompiler.engine.signoff_export.export_signoff_package_archive", fake_export ) return calls @@ -226,13 +226,13 @@ def test_export_forwards_include_debug( def test_export_incomplete_maps_to_error( self, tmp_path, capsys, monkeypatch, create_cli_project, workspace_stub, plain_records ): - from chipcompiler.runtime.workspace_api import RuntimeApiError + from chipcompiler.engine.signoff_export import SignoffExportError project_dir = create_cli_project() os.makedirs(os.path.join(project_dir, "default")) _patch_export( monkeypatch, - error=RuntimeApiError("command_failed", "signoff package is incomplete: x"), + error=SignoffExportError("signoff package is incomplete: x"), ) rc = cli_main.run( From 05459e58d83a553150f31ff11472ad526c29ba2c Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 01:48:40 +0800 Subject: [PATCH 09/43] fix(workspace): complete cross-creator update contract --- chipcompiler/cli/command_handlers/project.py | 14 ++ chipcompiler/engine/__init__.py | 2 + chipcompiler/engine/workspace_lifecycle.py | 143 ++++++++++++++++++- test/engine/test_workspace_spec.py | 100 +++++++++++++ 4 files changed, 255 insertions(+), 4 deletions(-) diff --git a/chipcompiler/cli/command_handlers/project.py b/chipcompiler/cli/command_handlers/project.py index 6ea8a597..23d30117 100644 --- a/chipcompiler/cli/command_handlers/project.py +++ b/chipcompiler/cli/command_handlers/project.py @@ -444,6 +444,20 @@ def error(kind: str, **fields) -> CommandResult: return error("flow_range_requires_pair") if fresh_target and (command_input.resume or command_input.only is not None): return error("selector_requires_workspace") + if not fresh_target and ctx.project_state == "manifest" and cfg.manifest_driven: + return run_dispatch.dispatch_project_run( + command_input, + ctx, + cfg, + run_dir, + run_name, + cli_overrides, + flow_config, + project_state, + warning_records, + workspace_registered=workspace_registered, + execute_flow=execute_flow, + ) errors = effective_config.validate_effective( ctx, cfg, diff --git a/chipcompiler/engine/__init__.py b/chipcompiler/engine/__init__.py index 95a7ad30..c523bb8e 100644 --- a/chipcompiler/engine/__init__.py +++ b/chipcompiler/engine/__init__.py @@ -17,6 +17,7 @@ assess_execution_readiness, create_workspace_from_spec, describe_workspace_binding_requirement, + update_workspace_from_spec, ) from .workspace_spec import describe_workspace_spec, validate_workspace_spec @@ -39,6 +40,7 @@ "read_step_configuration_from_directory", "read_workspace_configuration", "read_workspace_configuration_from_directory", + "update_workspace_from_spec", "update_workspace_step_configuration", "update_workspace_configuration", "validate_workspace_spec", diff --git a/chipcompiler/engine/workspace_lifecycle.py b/chipcompiler/engine/workspace_lifecycle.py index 679e4ca5..97c826e0 100644 --- a/chipcompiler/engine/workspace_lifecycle.py +++ b/chipcompiler/engine/workspace_lifecycle.py @@ -1,5 +1,8 @@ +import ctypes +import errno import hashlib import json +import os import shutil import tempfile from pathlib import Path @@ -26,7 +29,7 @@ def describe_workspace_binding_requirement( return { "familyId": workspace.pdk.name, "version": workspace.pdk.version or "unversioned", - "mode": "default", + "mode": "manual" if workspace.parameters.data.get("pdk_config") else "default", } @@ -44,7 +47,12 @@ def assess_execution_readiness( if not isinstance(root, str) or not Path(root).is_dir(): return {"ready": False, "code": "pdk_binding_missing"} try: - get_pdk(workspace.pdk.name, pdk_root=root).validate() + if workspace.parameters.data.get("pdk_config"): + if workspace.pdk.root is None or Path(root).resolve() != workspace.pdk.root.resolve(): + return {"ready": False, "code": "pdk_binding_mismatch"} + workspace.pdk.validate() + else: + get_pdk(workspace.pdk.name, pdk_root=root).validate() except (OSError, ValueError): return {"ready": False, "code": "pdk_binding_mismatch"} return {"ready": True} @@ -57,7 +65,12 @@ def apply_workspace_bindings(workspace, bindings: object) -> None: if not isinstance(root, str) or not Path(root).is_dir(): raise WorkspaceLifecycleError("pdk_binding_missing", "PDK binding root is required") sdc, spef = workspace.pdk.sdc, workspace.pdk.spef - workspace.pdk = get_pdk(workspace.pdk.name, pdk_root=root) + if workspace.parameters.data.get("pdk_config"): + if workspace.pdk.root is None or Path(root).resolve() != workspace.pdk.root.resolve(): + raise WorkspaceLifecycleError("pdk_binding_mismatch", "PDK binding root does not match") + workspace.pdk.validate() + else: + workspace.pdk = get_pdk(workspace.pdk.name, pdk_root=root) workspace.pdk.sdc, workspace.pdk.spef = sdc, spef from chipcompiler.data import refresh_workspace_config @@ -118,12 +131,19 @@ def create_workspace_from_spec( ) generated_filelist: str | None = None + generated_pdk_config: str | None = None if len(rtl_paths) > 1: with tempfile.NamedTemporaryFile( "w", encoding="utf-8", suffix=".f", delete=False ) as handle: handle.write("\n".join(rtl_paths) + "\n") generated_filelist = handle.name + if isinstance(pdk, PDK): + with tempfile.NamedTemporaryFile( + "w", encoding="utf-8", suffix=".json", delete=False + ) as handle: + json.dump(_pdk_config(pdk), handle) + generated_pdk_config = handle.name try: workspace = create_workspace( @@ -136,8 +156,9 @@ def create_workspace_from_spec( golden_verilog=input_paths.get("goldenNetlist", ""), sdc=input_paths.get("sdc", ""), spef=input_paths.get("spef", ""), - pdk=pdk, + pdk=pdk.name if isinstance(pdk, PDK) else pdk, pdk_root=pdk_root, + pdk_json=generated_pdk_config or "", pdk_overrides=pdk_overrides, parameters=backend_parameters, flow_config=flow_config, @@ -161,6 +182,78 @@ def create_workspace_from_spec( finally: if generated_filelist is not None: Path(generated_filelist).unlink(missing_ok=True) + if generated_pdk_config is not None: + Path(generated_pdk_config).unlink(missing_ok=True) + + +def update_workspace_from_spec( + target_directory: str | Path, + expected_workspace_revision: int, + spec: object, + bindings: object, + command_id: str = "", +): + from chipcompiler.engine.snapshot import ensure_engineering_snapshot + + target = Path(target_directory).expanduser().resolve() + fingerprint = _workspace_command_fingerprint( + "update", spec, bindings, expected_workspace_revision + ) + if command_id and _command_retry_matches(target, command_id, fingerprint): + return _load_committed_workspace(target) + current = _load_committed_workspace(target) + snapshot = ensure_engineering_snapshot(current) + if snapshot["workspaceRevision"] != expected_workspace_revision: + raise WorkspaceLifecycleError( + "revision_conflict", + "Workspace Revision does not match", + { + "expectedWorkspaceRevision": expected_workspace_revision, + "actualWorkspaceRevision": snapshot["workspaceRevision"], + }, + ) + + staging = Path(tempfile.mkdtemp(prefix=f".{target.name}.staging-", dir=target.parent)) + staging.rmdir() + try: + staged = create_workspace_from_spec(staging, spec, bindings) + create_engineering_snapshot( + staged, + workspace_id=snapshot["workspaceId"], + workspace_revision=snapshot["workspaceRevision"] + 1, + cause="workspace.updated", + ) + _copy_workspace_commands(target, staging) + _write_workspace_command( + staging, + command_id, + fingerprint, + snapshot["workspaceId"], + snapshot["workspaceRevision"] + 1, + ) + _rewrite_workspace_paths(staging, target) + _exchange_directories(target, staging) + try: + return _load_committed_workspace(target) + except Exception: + _exchange_directories(target, staging) + raise + finally: + shutil.rmtree(staging, ignore_errors=True) + + +def _pdk_config(pdk: PDK) -> dict[str, Any]: + return { + "name": pdk.name, + "version": pdk.version, + "root": str(pdk.root or ""), + "tech": str(pdk.tech or ""), + "lefs": [str(path) for path in pdk.lefs], + "libs": [str(path) for path in pdk.libs], + "mapping_file": str(pdk.mapping_file or ""), + "dont_use": pdk.dont_use, + "abc_load": pdk.abc_load, + } def _bound_pdk(spec: dict, binding: dict) -> tuple[PDK | str, str, dict | None]: @@ -271,3 +364,45 @@ def _write_workspace_command( } if not json_write(path / "home" / "workspace-commands.json", payload): raise OSError("Failed to persist Workspace command ledger") + + +def _copy_workspace_commands(source: Path, destination: Path) -> None: + source_path = source / "home" / "workspace-commands.json" + if source_path.is_file(): + shutil.copy2(source_path, destination / "home" / "workspace-commands.json") + + +def _rewrite_workspace_paths(source_root: Path, target_root: Path) -> None: + from chipcompiler.utility import json_write + + source = str(source_root) + target = str(target_root) + for path in source_root.rglob("*.json"): + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + rewritten = _replace_string_prefix(value, source, target) + if rewritten != value and not json_write(path, rewritten): + raise OSError(f"Failed to rewrite staged Workspace path: {path}") + + +def _replace_string_prefix(value: Any, source: str, target: str) -> Any: + if isinstance(value, str): + return target + value[len(source) :] if value.startswith(source) else value + if isinstance(value, list): + return [_replace_string_prefix(item, source, target) for item in value] + if isinstance(value, dict): + return {key: _replace_string_prefix(item, source, target) for key, item in value.items()} + return value + + +def _exchange_directories(left: Path, right: Path) -> None: + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = getattr(libc, "renameat2", None) + if os.name != "posix" or renameat2 is None: + raise OSError(errno.ENOTSUP, "atomic Workspace Update is unavailable") + if renameat2(-100, os.fsencode(left), -100, os.fsencode(right), 2) == 0: + return + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) diff --git a/test/engine/test_workspace_spec.py b/test/engine/test_workspace_spec.py index fbe833d2..d67d5a4f 100644 --- a/test/engine/test_workspace_spec.py +++ b/test/engine/test_workspace_spec.py @@ -2,6 +2,8 @@ from copy import deepcopy from pathlib import Path +import pytest + from chipcompiler.engine.workspace_spec import ( describe_workspace_spec, validate_workspace_spec, @@ -149,3 +151,101 @@ def test_validated_workspace_spec_creates_main_compatible_workspace( assert reopened.design.name == "gcd" assert reopened.parameters.data["frequency_max"] == 200.0 assert reopened.flow.steps()[0]["name"] == "Synthesis" + + +def test_manual_pdk_workspace_reopens_through_main_persistence(tmp_path): + from chipcompiler.data import load_workspace + from chipcompiler.engine import ( + assess_execution_readiness, + create_workspace_from_spec, + describe_workspace_binding_requirement, + ) + + rtl = tmp_path / "gcd.v" + rtl.write_text("module gcd(input clk); endmodule\n") + pdk_root = tmp_path / "pdk" + pdk_root.mkdir() + for name in ("tech.lef", "cells.lef", "typ.lib"): + (pdk_root / name).write_text(name) + spec = _valid_spec() + spec["flow"] = {"flowId": "syn_sta"} + spec["pdk"] = { + "familyId": "ics55", + "mode": "manual", + "files": [ + {"fileId": "tech", "role": "tech"}, + {"fileId": "cells", "role": "lef"}, + {"fileId": "lib", "role": "liberty"}, + ], + } + bindings = { + "inputs": {"rtl-main": str(rtl)}, + "pdk": { + "root": str(pdk_root), + "files": { + "tech": str(pdk_root / "tech.lef"), + "cells": str(pdk_root / "cells.lef"), + "lib": str(pdk_root / "typ.lib"), + }, + }, + } + target = tmp_path / "workspace" + + create_workspace_from_spec(target, spec, bindings) + reopened = load_workspace(target) + + assert reopened.pdk.tech == pdk_root / "tech.lef" + assert reopened.pdk.lefs == [pdk_root / "cells.lef"] + assert describe_workspace_binding_requirement(target)["mode"] == "manual" + assert assess_execution_readiness(target, bindings) == {"ready": True} + + +def test_workspace_spec_update_is_atomic_revisioned_and_idempotent( + tmp_path, minimal_ics55_pdk_factory +): + from chipcompiler.engine import ( + WorkspaceLifecycleError, + create_workspace_from_spec, + update_workspace_from_spec, + ) + from chipcompiler.engine.snapshot import read_engineering_snapshot + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + target = tmp_path / "workspace" + created = create_workspace_from_spec(target, payload["workspaceSpec"], bindings, "create-1") + before = read_engineering_snapshot(created) + updated_spec = deepcopy(payload["workspaceSpec"]) + updated_spec["parameters"]["design.frequency_mhz"] = 250.0 + + updated = update_workspace_from_spec( + target, + before["workspaceRevision"], + updated_spec, + bindings, + "update-1", + ) + after = read_engineering_snapshot(updated) + repeated = update_workspace_from_spec( + target, + before["workspaceRevision"], + updated_spec, + bindings, + "update-1", + ) + + assert after["workspaceId"] == before["workspaceId"] + assert after["workspaceRevision"] == before["workspaceRevision"] + 1 + assert read_engineering_snapshot(repeated) == after + assert repeated.parameters.data["frequency_max"] == 250.0 + + with pytest.raises(WorkspaceLifecycleError) as conflict: + update_workspace_from_spec( + target, + before["workspaceRevision"], + payload["workspaceSpec"], + bindings, + "update-2", + ) + assert conflict.value.code == "revision_conflict" + assert read_engineering_snapshot(repeated) == after From 96d286a2675de74181ab34c5b125492ebc7fa4d8 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 02:22:33 +0800 Subject: [PATCH 10/43] fix(workspace): preserve rollback evidence atomically --- chipcompiler/data/parameter_keys.py | 2 +- chipcompiler/data/workspace_config.py | 2 +- chipcompiler/docs/ecc-user-guide.cn.md | 1 - chipcompiler/docs/ecc-user-guide.en.md | 1 - chipcompiler/engine/snapshot.py | 4 ++-- chipcompiler/engine/workspace_configuration.py | 15 ++++++++++++++- docs/development.cn.md | 3 +-- docs/specification/cli-design.md | 13 +++++-------- test/data/test_workspace_config.py | 2 +- 9 files changed, 25 insertions(+), 18 deletions(-) diff --git a/chipcompiler/data/parameter_keys.py b/chipcompiler/data/parameter_keys.py index 8478d4bb..1cc2d9a3 100644 --- a/chipcompiler/data/parameter_keys.py +++ b/chipcompiler/data/parameter_keys.py @@ -6,7 +6,7 @@ ``top_module``, ``die``/``core`` subtrees, ...). Older workspaces persisted display-oriented keys (``"Frequency max [MHz]"``, ``"Top module"``, ...) and the GUI sends flat keys plus a handful of positional geometry aliases at the -project/RPC boundary. This module is the only place that knows the legacy +project/adapter boundary. This module is the only place that knows the legacy vocabulary; everything else consumes the canonical form. """ diff --git a/chipcompiler/data/workspace_config.py b/chipcompiler/data/workspace_config.py index 97df9411..941b8b38 100644 --- a/chipcompiler/data/workspace_config.py +++ b/chipcompiler/data/workspace_config.py @@ -140,7 +140,7 @@ def validate_flow_config(flow: object) -> dict[str, str]: if not isinstance(value, str): raise WorkspaceFlowTargetError(f"[flow] {key} must be a string: {value!r}") # Workspace files carry canonical step names only; display-name - # aliases are translated at the manifest/RPC boundary, never here. + # aliases are translated at the manifest/adapter boundary, never here. if value not in canonical_names: raise WorkspaceFlowTargetError(f"[flow] unknown step name: {value!r}") normalized[key] = value diff --git a/chipcompiler/docs/ecc-user-guide.cn.md b/chipcompiler/docs/ecc-user-guide.cn.md index 20d9ccce..948794e1 100644 --- a/chipcompiler/docs/ecc-user-guide.cn.md +++ b/chipcompiler/docs/ecc-user-guide.cn.md @@ -111,7 +111,6 @@ Commands: workspace Refresh managed workspaces from project configuration signoff Inspect and export signoff packages report Generate design-summary, QoR score, checklist, and step reports - rpc Run the private ECC JSON-RPC runtime ``` ## 1.5. doc — 在终端阅读内置指南 diff --git a/chipcompiler/docs/ecc-user-guide.en.md b/chipcompiler/docs/ecc-user-guide.en.md index 88772370..98161fa1 100644 --- a/chipcompiler/docs/ecc-user-guide.en.md +++ b/chipcompiler/docs/ecc-user-guide.en.md @@ -111,7 +111,6 @@ Commands: workspace Refresh managed workspaces from project configuration signoff Inspect and export signoff packages report Generate design-summary, QoR score, checklist, and step reports - rpc Run the private ECC JSON-RPC runtime ``` ## 1.5. doc — read the bundled guides in the terminal diff --git a/chipcompiler/engine/snapshot.py b/chipcompiler/engine/snapshot.py index 86984912..8ccd332a 100644 --- a/chipcompiler/engine/snapshot.py +++ b/chipcompiler/engine/snapshot.py @@ -80,9 +80,9 @@ def commit_engineering_snapshot( ] if remaining: snapshot["stalePredecessor"] = {**deepcopy(stale), "invalidatedStepIds": remaining} - else: - _stale_snapshot_path(workspace).unlink(missing_ok=True) _write_snapshot(_snapshot_path(workspace), snapshot) + if isinstance(stale, dict) and "stalePredecessor" not in snapshot: + _stale_snapshot_path(workspace).unlink(missing_ok=True) return snapshot diff --git a/chipcompiler/engine/workspace_configuration.py b/chipcompiler/engine/workspace_configuration.py index 196e62a6..dcd68470 100644 --- a/chipcompiler/engine/workspace_configuration.py +++ b/chipcompiler/engine/workspace_configuration.py @@ -1,3 +1,5 @@ +import os +import tempfile from copy import deepcopy from pathlib import Path from typing import Any @@ -425,4 +427,15 @@ def _restore_files(snapshot: dict[Path, bytes | None]) -> None: path.unlink(missing_ok=True) else: path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(content) + fd, temporary = tempfile.mkstemp( + dir=path.parent, prefix=f".{path.name}.", suffix=".tmp" + ) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except BaseException: + os.unlink(temporary) + raise diff --git a/docs/development.cn.md b/docs/development.cn.md index 0b14b99a..a6773d87 100644 --- a/docs/development.cn.md +++ b/docs/development.cn.md @@ -247,8 +247,7 @@ chipcompiler/cli/commands/ # typer 命令定义层（薄） ├── project_config.py # project 子应用（set/unset/add/remove/show） ├── workspace.py # workspace 子应用（refresh） ├── signoff.py # signoff 子应用（inspect/export） - ├── report.py # report 子应用（summary/qor/checklist/step） - └── rpc.py # rpc 子应用（serve） + └── report.py # report 子应用（summary/qor/checklist/step） chipcompiler/cli/command_handlers/ # 业务处理层（唯一的处理器包，有状态/重逻辑） ├── project.py # init / check / run / migrate / workspace refresh（含 preset 解析与环境预检） ├── inspect.py # status / log / config diff --git a/docs/specification/cli-design.md b/docs/specification/cli-design.md index d678d39c..0e845c01 100644 --- a/docs/specification/cli-design.md +++ b/docs/specification/cli-design.md @@ -188,8 +188,8 @@ agent-specific disclosure fields inside core flow APIs. ### Core Commands The current root surface is a Typer command graph. The project-first command -surface stays small, with version reporting and the private runtime sidecar -available as explicit root entries: +surface stays small, with version reporting and layout rendering as explicit +root entries: ```bash ecc --version @@ -208,7 +208,6 @@ ecc project ecc workspace ecc signoff ecc report -ecc rpc ecc layout-image ``` @@ -232,7 +231,6 @@ Responsibilities: | `ecc workspace` | Refresh a declared workspace from current `ecc.toml` without running it | | `ecc signoff` | Inspect package readiness and export the tar.gz package | | `ecc report` | Write design-summary, QoR, and checklist reports; show step evidence | -| `ecc rpc` | Serve the private JSON-RPC runtime sidecar over stdio | | `ecc layout-image` | Render a GDS file into an image | `ecc run` preflights the tools its preset needs (yosys for synthesis, @@ -277,7 +275,7 @@ The command graph follows these rules; new commands must follow them too: `check`, `run`, `status`, `log`, `config`, `doctor`, `migrate`, `version`) plus the frozen tool invocations (`layout-image`). Resource management and reporting live in noun groups (`param`, `pdk`, `project`, `workspace`, - `signoff`, `report`, `rpc`). + `signoff`, `report`). - **Subcommand verbs.** Mutable resources use the CRUD set (`list`, `show`, `set`, `unset`, `diff`). The `report` group names its artifacts instead (`summary`, `qor`, `checklist`, `step`) @@ -767,9 +765,8 @@ NAME` for a workspace declared in the project manifest. It persists a workspace-local override and refreshes the affected configuration without running the flow. PDK resources and input references are project-level values: edit `ecc.toml` through `ecc project` (or the matching `ecc pdk`/`ecc param` -commands), then use `ecc workspace refresh NAME`. Old workspace create/run -automation should use the private JSON-RPC runtime sidecar. The long-term -default is project-oriented and configuration-driven through `ecc.toml` and +commands), then use `ecc workspace refresh NAME`. The supported integration +surface is project-oriented and configuration-driven through `ecc.toml` and subcommands such as `ecc run --project `. The project-level Python APIs should remain compatible with existing Python diff --git a/test/data/test_workspace_config.py b/test/data/test_workspace_config.py index 94ec050f..790f5663 100644 --- a/test/data/test_workspace_config.py +++ b/test/data/test_workspace_config.py @@ -144,7 +144,7 @@ def test_flow_validation_accepts_canonical_names(): def test_flow_validation_rejects_display_name_aliases(): # Workspace files carry canonical names only; aliases translate at the - # manifest/RPC boundary. + # manifest/adapter boundary. with pytest.raises(WorkspaceFlowTargetError): validate_flow_config({"start": "Synth", "end": "Filler"}) From f58c45cf9a831d5a42a74ef460703c445bfd4ea9 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 03:45:24 +0800 Subject: [PATCH 11/43] refactor(data): centralize parameter catalog ownership Move the public parameter schema and workspace override helpers into the data layer while keeping CLI compatibility facades. --- chipcompiler/cli/project/params.py | 673 +----------------- chipcompiler/cli/project/workspace_params.py | 143 +--- chipcompiler/data/parameter_schema.py | 491 +++++++++++++ chipcompiler/data/workspace_parameters.py | 127 ++++ .../engine/workspace_configuration.py | 126 +++- chipcompiler/engine/workspace_lifecycle.py | 45 +- chipcompiler/engine/workspace_spec.py | 2 +- 7 files changed, 763 insertions(+), 844 deletions(-) create mode 100644 chipcompiler/data/parameter_schema.py create mode 100644 chipcompiler/data/workspace_parameters.py diff --git a/chipcompiler/cli/project/params.py b/chipcompiler/cli/project/params.py index b4db5c0b..6241f7d7 100644 --- a/chipcompiler/cli/project/params.py +++ b/chipcompiler/cli/project/params.py @@ -1,682 +1,27 @@ -import copy -import json -from dataclasses import dataclass +"""CLI compatibility facade for the canonical data parameter catalog.""" -from chipcompiler.data.config_params import CONFIG_PARAM_SCHEMAS -from chipcompiler.data.config_params.common import ParamSchema +from dataclasses import replace -_LEGACY_PARAM_REGISTRY: tuple[ParamSchema, ...] = ( - ParamSchema( - param="design.frequency_mhz", - group="design", - name="frequency_mhz", - type="float", - default=100.0, - applies="synthesis", - maps_to="frequency_max", - description="Target clock frequency in MHz", - range=(1e-6, 10000.0), - unit="MHz", - example="200.0", - ), - ParamSchema( - param="floorplan.core_util", - group="floorplan", - name="core_util", - type="float", - default=0.4, - applies="floorplan", - maps_to={"core": "utilitization"}, - description="Core utilization ratio", - range=(0.01, 1.0), - example="0.45", - ), - ParamSchema( - param="floorplan.core_margin", - group="floorplan", - name="core_margin", - type="list[int]", - default=[2, 2], - applies="floorplan", - maps_to={"core": "margin"}, - description="Core margin in micrometers [horizontal, vertical]", - example="[2, 2]", - ), - ParamSchema( - param="floorplan.aspect_ratio", - group="floorplan", - name="aspect_ratio", - type="float", - default=1.0, - applies="floorplan", - maps_to={"core": "aspect_ratio"}, - description="Core aspect ratio (width/height)", - range=(0.1, 10.0), - example="1.0", - ), - ParamSchema( - param="cts.max_fanout", - group="cts", - name="max_fanout", - type="int", - default=20, - applies="cts", - maps_to="max_fanout", - description="Maximum fanout for clock tree synthesis", - range=(1, 200), - example="16", - ), - ParamSchema( - param="place.target_density", - group="place", - name="target_density", - type="float", - default=0.2, - applies="placement", - maps_to={"dreamplace": "target_density"}, - description="Target placement density", - range=(0.1, 0.95), - example="0.65", - ), - ParamSchema( - param="place.target_overflow", - group="place", - name="target_overflow", - type="float", - default=0.1, - applies="placement", - maps_to={"dreamplace": "stop_overflow"}, - description="Target overflow for global placement", - range=(0.0, 1.0), - example="0.08", - ), - ParamSchema( - param="place.global_right_padding", - group="place", - name="global_right_padding", - type="int", - default=0, - applies="placement", - maps_to="global_right_padding", - description="Global right padding for placement sites", - range=(0, 100), - example="8", - ), - ParamSchema( - param="place.cell_padding_x", - group="place", - name="cell_padding_x", - type="int", - default=300, - applies="placement", - maps_to={"dreamplace": "cell_padding_x"}, - description="Cell padding in x-direction in database units", - range=(0, 10000), - example="400", - ), - ParamSchema( - param="place.routability_opt", - group="place", - name="routability_opt", - type="int", - default=1, - applies="placement", - maps_to={"dreamplace": "routability_opt_flag"}, - description="Enable routability-driven placement optimization", - choices=("0", "1"), - example="1", - ), - ParamSchema( - param="route.bottom_layer", - group="route", - name="bottom_layer", - type="str", - default="MET2", - applies="routing", - maps_to="bottom_layer", - description="Bottom routing layer", - choices=("MET1", "MET2", "MET3", "MET4", "MET5"), - example="MET2", - ), - ParamSchema( - param="route.top_layer", - group="route", - name="top_layer", - type="str", - default="MET5", - applies="routing", - maps_to="top_layer", - description="Top routing layer", - choices=("MET2", "MET3", "MET4", "MET5", "MET6"), - example="MET5", - ), - ParamSchema( - param="sta.max_paths", - group="sta", - name="max_paths", - type="int", - default=1000, - applies="sta", - maps_to="sta_max_paths", - description="Maximum number of paths in each STA timing report", - range=(1, 100000), - example="1000", - ), - ParamSchema( - param="flow.run_analysis", - group="flow", - name="run_analysis", - type="bool", - default=True, - applies="all", - maps_to="run_analysis", - description="Run per-step analysis (metrics, plots, checklist) after each step", - example="false", - ), -) +from chipcompiler.data.parameter_schema import * # noqa: F401,F403 +from chipcompiler.data.parameter_schema import validate_schema_type -PARAM_REGISTRY = _LEGACY_PARAM_REGISTRY + CONFIG_PARAM_SCHEMAS +_validate_schema_type = validate_schema_type -_REGISTRY_INDEX: dict[str, ParamSchema] = {s.param: s for s in PARAM_REGISTRY} -_REQUIRED_FIELDS = ( - "param", - "group", - "name", - "type", - "default", - "applies", - "description", -) - -def lookup_schema(key: str) -> ParamSchema | None: - return _REGISTRY_INDEX.get(key) - - -def list_schemas() -> tuple[ParamSchema, ...]: - return PARAM_REGISTRY - - -def list_groups() -> list[str]: - seen: list[str] = [] - for s in PARAM_REGISTRY: - if s.group not in seen: - seen.append(s.group) - return seen - - -def is_known_key(key: str) -> bool: - return key in _REGISTRY_INDEX - - -def validate_schema_record(schema: ParamSchema) -> list[str]: - return [ - f"missing required field: {f}" - for f in _REQUIRED_FIELDS - if getattr(schema, f, None) is None or (f != "default" and getattr(schema, f) == "") - ] - - -# --------------------------------------------------------------------------- -# Value parsing -# --------------------------------------------------------------------------- - - -def parse_value(raw: str, schema: ParamSchema) -> object: - ptype = schema.type - - if ptype == "int": - try: - return int(raw) - except ValueError as exc: - raise ValueError(f"expected int for {schema.param}, got '{raw}'") from exc - - if ptype == "float": - try: - return float(raw) - except ValueError as exc: - raise ValueError(f"expected float for {schema.param}, got '{raw}'") from exc - - if ptype == "bool": - low = raw.lower() - if low in ("true", "1", "yes"): - return True - if low in ("false", "0", "no"): - return False - raise ValueError(f"expected bool for {schema.param}, got '{raw}'") - - if ptype == "str": - return raw - - if ptype in ("list[int]", "list[float]", "list[str]"): - try: - parsed = json.loads(raw) - except json.JSONDecodeError: - parsed = None - if parsed is not None: - value, error = _validate_schema_type(parsed, schema) - if error is None: - return value - raise ValueError(error) - - stripped = raw.strip("[]() ") - if not stripped: - return [] - parts = [p.strip() for p in stripped.split(",")] - if ptype == "list[int]": - try: - return [int(p) for p in parts if p] - except ValueError as exc: - raise ValueError(f"expected list[int] for {schema.param}, got '{raw}'") from exc - if ptype == "list[float]": - try: - return [float(p) for p in parts if p] - except ValueError as exc: - raise ValueError(f"expected list[float] for {schema.param}, got '{raw}'") from exc - return [p for p in parts if p] - - if ptype == "json": - try: - return json.loads(raw) - except json.JSONDecodeError as exc: - raise ValueError(f"expected JSON for {schema.param}, got '{raw}'") from exc - - raise ValueError(f"unsupported type '{ptype}' for {schema.param}") - - -def validate_value(value: object, schema: ParamSchema) -> list[str]: - errors: list[str] = [] - - if schema.range is not None: - lo, hi = schema.range - if not isinstance(value, (int, float)): - # Raw layers (e.g. project.json parameters) reach this without - # the typed parsing --set/[params] get: a string must fail loud - # instead of silently skipping the range check. - errors.append(f"value {value!r} is not numeric for {schema.param}") - elif value < lo or value > hi: - errors.append(f"value {value} out of range [{lo}, {hi}] for {schema.param}") - - if schema.choices is not None: - str_val = str(value) - if str_val not in schema.choices: - errors.append( - f"value '{str_val}' not in allowed choices {schema.choices} for {schema.param}" - ) - - return errors - - -# --------------------------------------------------------------------------- -# Source-aware resolution -# --------------------------------------------------------------------------- - - -@dataclass -class ResolvedParam: - param: str - value: object - default: object - source: str - schema: ParamSchema - - @property - def is_explicit(self) -> bool: - return self.source != "default" - - -def _validate_schema_type(value: object, schema: ParamSchema) -> tuple[object, str | None]: - ptype = schema.type - key = schema.param - - if ptype == "int": - if isinstance(value, bool) or not isinstance(value, int): - return value, f"expected int for {key}, got {type(value).__name__}" - return value, None - - if ptype == "float": - if isinstance(value, bool): - return value, f"expected float for {key}, got bool" - if isinstance(value, (int, float)): - try: - return float(value), None - except OverflowError: - # A huge JSON integer cannot become a float; that is invalid - # input, not a crash. - return value, f"expected float for {key}, value too large to represent" - return value, f"expected float for {key}, got {type(value).__name__}" - - if ptype == "bool": - if isinstance(value, bool): - return value, None - if isinstance(value, str): - low = value.lower() - if low in ("true", "1", "yes"): - return True, None - if low in ("false", "0", "no"): - return False, None - return value, f"expected bool for {key}, got {type(value).__name__}" - - if ptype == "str": - if isinstance(value, str): - return value, None - return value, f"expected str for {key}, got {type(value).__name__}" - - if ptype == "list[int]": - if not isinstance(value, list): - return value, f"expected list for {key}, got {type(value).__name__}" - for i, v in enumerate(value): - if isinstance(v, bool) or not isinstance(v, int): - return value, f"expected list[int] for {key}, element {i} is {type(v).__name__}" - return value, None - - if ptype == "list[float]": - if not isinstance(value, list): - return value, f"expected list for {key}, got {type(value).__name__}" - for i, v in enumerate(value): - if isinstance(v, bool): - return value, f"expected list[float] for {key}, element {i} is bool" - if not isinstance(v, (int, float)): - return value, f"expected list[float] for {key}, element {i} is {type(v).__name__}" - try: - return [float(v) for v in value], None - except OverflowError: - return value, f"expected list[float] for {key}, value too large to represent" - - if ptype == "list[str]": - if not isinstance(value, list): - return value, f"expected list for {key}, got {type(value).__name__}" - for i, v in enumerate(value): - if not isinstance(v, str): - return value, f"expected list[str] for {key}, element {i} is {type(v).__name__}" - return value, None - - if ptype == "json": - if isinstance(value, (dict, list)): - return value, None - return value, f"expected JSON object or array for {key}, got {type(value).__name__}" - - return value, None - - -def resolve_parameters( - toml_overrides: dict[str, object] | None = None, - cli_overrides: dict[str, object] | None = None, - manifest_overrides: dict[str, object] | None = None, -) -> tuple[list[ResolvedParam], list[str]]: - toml_overrides = toml_overrides or {} - cli_overrides = cli_overrides or {} - manifest_overrides = manifest_overrides or {} - resolved: list[ResolvedParam] = [] - errors: list[str] = [] - - for schema in PARAM_REGISTRY: - key = schema.param - if key in cli_overrides: - value = cli_overrides[key] - val_errors = validate_value(value, schema) - if val_errors: - errors.extend(val_errors) - resolved.append( - ResolvedParam( - param=key, - value=value, - default=schema.default, - source="cli", - schema=schema, - ) - ) - elif key in toml_overrides: - value = toml_overrides[key] - value, coerce_err = _validate_schema_type(value, schema) - if coerce_err: - errors.append(coerce_err) - val_errors = validate_value(value, schema) - if val_errors: - errors.extend(val_errors) - resolved.append( - ResolvedParam( - param=key, - value=value, - default=schema.default, - source="ecc.toml", - schema=schema, - ) - ) - elif key in manifest_overrides: - value = manifest_overrides[key] - value, coerce_err = _validate_schema_type(value, schema) - if coerce_err: - errors.append(coerce_err) - val_errors = validate_value(value, schema) - if val_errors: - errors.extend(val_errors) - resolved.append( - ResolvedParam( - param=key, - value=value, - default=schema.default, - source="project.json", - schema=schema, - ) - ) - else: - resolved.append( - ResolvedParam( - param=key, - value=schema.default, - default=schema.default, - source="default", - schema=schema, - ) - ) - - return resolved, errors - - -# --------------------------------------------------------------------------- -# Semantic-to-backend mapping -# --------------------------------------------------------------------------- - - -def manifest_value_for(canonical: dict, maps_to: str | dict | None) -> tuple[object, bool]: - """Resolve a manifest-layer value via its schema target (presence-keyed). - - A string ``maps_to`` is a top-level canonical key; a dict ``maps_to`` is a - single-level (subtree, leaf) path. Returns (value, present): an explicit - JSON ``null`` counts as present, an absent key does not. - """ - if isinstance(maps_to, str): - if maps_to in canonical: - return canonical[maps_to], True - return None, False - if maps_to is None: - return None, False - for subtree, leaf in maps_to.items(): - node = canonical.get(subtree) - if isinstance(node, dict) and leaf in node: - return node[leaf], True - return None, False - - -def coerce_manifest_parameters( - canonical: dict, - registry: tuple[ParamSchema, ...] = PARAM_REGISTRY, - skip_params: frozenset | set = frozenset(), -) -> tuple[dict, list[str]]: - """Coerce manifest-layer parameter values to their schema types. - - Type check/coercion only — range/choice rules stay check-path concerns. - Works on a deep copy: coerced values are written back, uncoercible values - are left as-is (the caller decides what an error means), and keys outside - the registry pass through untouched (forward-compatible). - """ - coerced = copy.deepcopy(canonical) - errors: list[str] = [] - for schema in registry: - if schema.param in skip_params: - continue - value, present = manifest_value_for(coerced, schema.maps_to) - if not present: - continue - new_value, err = _validate_schema_type(value, schema) - if err: - errors.append(err) - continue - maps_to = schema.maps_to - if isinstance(maps_to, str): - coerced[maps_to] = new_value - elif isinstance(maps_to, dict): - for subtree, leaf in maps_to.items(): - node = coerced.get(subtree) - if isinstance(node, dict) and leaf in node: - node[leaf] = new_value - break - return coerced, errors - - -def build_backend_overrides(resolved: list[ResolvedParam]) -> dict: - overrides: dict = {} - for rp in resolved: - if rp.value == rp.default and rp.source == "default": - continue - maps_to = rp.schema.maps_to - value = rp.value - if isinstance(maps_to, str): - overrides[maps_to] = value - elif isinstance(maps_to, dict): - for parent_key, child_key in maps_to.items(): - if parent_key not in overrides: - overrides[parent_key] = {} - overrides[parent_key][child_key] = value - return overrides - - -def build_config_overrides(resolved: list[ResolvedParam]) -> dict[str, object]: - overrides: dict[str, object] = {} - for rp in resolved: - target = rp.schema.config_target - if target is None or not rp.is_explicit: - continue - config_overrides = overrides.setdefault(target.config_key, {}) - _set_nested_value(config_overrides, target.json_path, rp.value) - return overrides - - -def build_pdk_overrides(resolved: list[ResolvedParam]) -> dict[str, object]: - return { - rp.schema.pdk_target: rp.value - for rp in resolved - if rp.schema.pdk_target is not None and rp.is_explicit - } - - -def _set_nested_value(data: dict, path: tuple[str, ...], value: object) -> None: - current = data - for key in path[:-1]: - child = current.get(key) - if not isinstance(child, dict): - child = {} - current[key] = child - current = child - current[path[-1]] = value - - -def parse_cli_overrides(pairs: list[str]) -> tuple[dict[str, object], list[str]]: - result: dict[str, object] = {} - errors: list[str] = [] - - for pair in pairs: - if "=" not in pair: - errors.append(f"malformed override: '{pair}' (expected key=value)") - continue - - key, _, raw_value = pair.partition("=") - key = key.strip() - raw_value = raw_value.strip() - - schema = lookup_schema(key) - if schema is None: - errors.append(f"unknown parameter: '{key}'") - continue - - try: - value = parse_value(raw_value, schema) - except ValueError as exc: - errors.append(str(exc)) - continue - - val_errors = validate_value(value, schema) - if val_errors: - errors.extend(val_errors) - continue - - result[key] = value - - return result, errors - - -def validate_pdk_target(schema: ParamSchema, value: object, cfg) -> str | None: - """Validate a PDK-target value using the existing PDK resolver and checks.""" +def validate_pdk_target(schema, value, cfg) -> str | None: if schema.pdk_target is None: return None - - from dataclasses import replace - from chipcompiler.cli.project.config import ( _validate_pdk_contents, resolve_pdk_overrides, resolve_pdk_root, ) - raw_overrides = dict(cfg.pdk_overrides) - raw_overrides[schema.pdk_target] = value - candidate = replace(cfg, pdk_overrides=raw_overrides) + overrides = dict(cfg.pdk_overrides) + overrides[schema.pdk_target] = value + candidate = replace(cfg, pdk_overrides=overrides) return _validate_pdk_contents( candidate.pdk_name, resolve_pdk_root(candidate), resolve_pdk_overrides(candidate), ) - - -def parse_toml_params(params_table: dict) -> tuple[dict[str, object], list[str]]: - flat: dict[str, object] = {} - errors: list[str] = [] - - def visit(path: tuple[str, ...], value: object) -> None: - param_key = ".".join(path) - schema = lookup_schema(param_key) - if schema is not None: - try: - if isinstance(value, str): - parsed = parse_value(value, schema) - else: - parsed, type_err = _validate_schema_type(value, schema) - if type_err: - errors.append(type_err) - return - except ValueError as exc: - errors.append(str(exc)) - return - - val_errors = validate_value(parsed, schema) - if val_errors: - errors.extend(val_errors) - return - flat[param_key] = parsed - return - - if isinstance(value, dict): - for key, child in value.items(): - visit((*path, key), child) - return - - errors.append(f"unknown parameter in ecc.toml: '{param_key}'") - - for group_key, group_val in params_table.items(): - if not isinstance(group_val, dict): - errors.append(f"[params.{group_key}] must be a table, got {type(group_val).__name__}") - continue - visit((group_key,), group_val) - - return flat, errors diff --git a/chipcompiler/cli/project/workspace_params.py b/chipcompiler/cli/project/workspace_params.py index 3c48b6f2..da1e397f 100644 --- a/chipcompiler/cli/project/workspace_params.py +++ b/chipcompiler/cli/project/workspace_params.py @@ -1,142 +1,3 @@ -"""Workspace-local parameter persistence and replay helpers.""" +"""CLI compatibility facade for workspace parameter persistence.""" -from copy import deepcopy - -from chipcompiler.cli.project.params import ( - ResolvedParam, - build_backend_overrides, - build_config_overrides, -) -from chipcompiler.data.parameter import update_parameters -from chipcompiler.data.workspace.config_overrides import CONFIG_OVERRIDES_KEY -from chipcompiler.utility import json_read - -WORKSPACE_PARAM_OVERRIDES_KEY = "workspace_param_overrides" - -_APPLIES_TO_STEP = { - "synthesis": "Synthesis", - "floorplan": "preFloorplan", - "placement": "place", - "cts": "CTS", - "routing": "route", - "filler": "filler", - "rcx": "RCX", - "sta": "sta", -} -_MISSING = object() - - -def workspace_param_step(schema) -> str: - step = _APPLIES_TO_STEP.get(schema.applies) - if step is None: - raise ValueError(f"{schema.param} requires a full workspace refresh") - return step - - -def workspace_param_value(workspace, schema) -> object: - if schema.pdk_target is not None: - raise ValueError(f"{schema.param} requires a full workspace refresh") - if schema.maps_to is not None: - return _parameter_target_value(workspace.parameters.data, schema.maps_to, schema.default) - if schema.config_target is not None: - config_path = workspace.config.get(schema.config_target.config_key) - if config_path is None: - raise ValueError(f"workspace config missing target: {schema.config_target.config_key}") - value = _nested_value(json_read(config_path), schema.config_target.json_path) - return schema.default if value is _MISSING else value - raise ValueError(f"{schema.param} has no workspace configuration target") - - -def set_workspace_param(workspace, schema, value: object) -> tuple[object, str]: - baseline = update_workspace_param_value(workspace, schema, value) - return baseline, workspace_param_step(schema) - - -def update_workspace_param_value(workspace, schema, value: object) -> object: - """Apply and track one public parameter without choosing invalidation scope.""" - records = _override_records(workspace.parameters.data) - record = next((item for item in records if item["key"] == schema.param), None) - if record is None: - record = { - "key": schema.param, - "baseline": deepcopy(workspace_param_value(workspace, schema)), - } - records.append(record) - record["value"] = deepcopy(value) - _set_override_records(workspace.parameters.data, records) - _apply_workspace_value(workspace, schema, value) - return record["baseline"] - - -def unset_workspace_param(workspace, schema) -> tuple[object, str] | None: - records = _override_records(workspace.parameters.data) - record = next((item for item in records if item["key"] == schema.param), None) - if record is None: - return None - records.remove(record) - _set_override_records(workspace.parameters.data, records) - _apply_workspace_value(workspace, schema, record["baseline"]) - return record["baseline"], workspace_param_step(schema) - - -def workspace_param_diff(workspace) -> list[dict]: - return _override_records(workspace.parameters.data) - - -def _apply_workspace_value(workspace, schema, value: object) -> None: - resolved = ResolvedParam( - param=schema.param, - value=value, - default=schema.default, - source="workspace", - schema=schema, - ) - update_parameters(build_backend_overrides([resolved]), workspace.parameters.data) - config_overrides = build_config_overrides([resolved]) - if config_overrides: - update_parameters({CONFIG_OVERRIDES_KEY: config_overrides}, workspace.parameters.data) - - -def _override_records(parameters: dict) -> list[dict]: - raw = parameters.get(WORKSPACE_PARAM_OVERRIDES_KEY, []) - if not isinstance(raw, list): - return [] - records = [] - for item in raw: - if not isinstance(item, dict): - continue - key = item.get("key") - if not isinstance(key, str) or "baseline" not in item: - continue - records.append(deepcopy(item)) - return records - - -def _set_override_records(parameters: dict, records: list[dict]) -> None: - if records: - parameters[WORKSPACE_PARAM_OVERRIDES_KEY] = records - else: - parameters.pop(WORKSPACE_PARAM_OVERRIDES_KEY, None) - - -def _parameter_target_value(parameters: dict, target, default: object) -> object: - if isinstance(target, str): - return deepcopy(parameters.get(target, default)) - if isinstance(target, dict) and len(target) == 1: - parent, child = next(iter(target.items())) - nested = parameters.get(parent) - if isinstance(nested, dict): - return deepcopy(nested.get(child, default)) - return deepcopy(default) - - -def _nested_value(data: object, path: tuple[str, ...]) -> object: - current = data - for key in path: - if not isinstance(current, dict): - return _MISSING - payload: dict = dict(current) - if key not in payload: - return _MISSING - current = payload[key] - return deepcopy(current) +from chipcompiler.data.workspace_parameters import * # noqa: F401,F403 diff --git a/chipcompiler/data/parameter_schema.py b/chipcompiler/data/parameter_schema.py new file mode 100644 index 00000000..45bd1a53 --- /dev/null +++ b/chipcompiler/data/parameter_schema.py @@ -0,0 +1,491 @@ +"""Canonical public parameter catalog and schema operations. + +The catalog is shared by the headless Engine, CLI, and Studio adapter. CLI +specific PDK path validation remains in ``chipcompiler.cli.project.params``. +""" + +import copy +import json +from dataclasses import dataclass + +from chipcompiler.data.config_params import CONFIG_PARAM_SCHEMAS +from chipcompiler.data.config_params.common import ParamSchema + +_LEGACY_PARAM_REGISTRY: tuple[ParamSchema, ...] = ( + ParamSchema( + "design.frequency_mhz", + "design", + "frequency_mhz", + "float", + 100.0, + "synthesis", + "Target clock frequency in MHz", + "frequency_max", + range=(1e-6, 10000.0), + unit="MHz", + example="200.0", + ), + ParamSchema( + "floorplan.core_util", + "floorplan", + "core_util", + "float", + 0.4, + "floorplan", + "Core utilization ratio", + {"core": "utilitization"}, + range=(0.01, 1.0), + example="0.45", + ), + ParamSchema( + "floorplan.core_margin", + "floorplan", + "core_margin", + "list[int]", + [2, 2], + "floorplan", + "Core margin in micrometers [horizontal, vertical]", + {"core": "margin"}, + example="[2, 2]", + ), + ParamSchema( + "floorplan.aspect_ratio", + "floorplan", + "aspect_ratio", + "float", + 1.0, + "floorplan", + "Core aspect ratio (width/height)", + {"core": "aspect_ratio"}, + range=(0.1, 10.0), + example="1.0", + ), + ParamSchema( + "cts.max_fanout", + "cts", + "max_fanout", + "int", + 20, + "cts", + "Maximum fanout for clock tree synthesis", + "max_fanout", + range=(1, 200), + example="16", + ), + ParamSchema( + "place.target_density", + "place", + "target_density", + "float", + 0.2, + "placement", + "Target placement density", + {"dreamplace": "target_density"}, + range=(0.1, 0.95), + example="0.65", + ), + ParamSchema( + "place.target_overflow", + "place", + "target_overflow", + "float", + 0.1, + "placement", + "Target overflow for global placement", + {"dreamplace": "stop_overflow"}, + range=(0.0, 1.0), + example="0.08", + ), + ParamSchema( + "place.global_right_padding", + "place", + "global_right_padding", + "int", + 0, + "placement", + "Global right padding for placement sites", + "global_right_padding", + range=(0, 100), + example="8", + ), + ParamSchema( + "place.cell_padding_x", + "place", + "cell_padding_x", + "int", + 300, + "placement", + "Cell padding in x-direction in database units", + {"dreamplace": "cell_padding_x"}, + range=(0, 10000), + example="400", + ), + ParamSchema( + "place.routability_opt", + "place", + "routability_opt", + "int", + 1, + "placement", + "Enable routability-driven placement optimization", + {"dreamplace": "routability_opt_flag"}, + choices=("0", "1"), + example="1", + ), + ParamSchema( + "route.bottom_layer", + "route", + "bottom_layer", + "str", + "MET2", + "routing", + "Bottom routing layer", + "bottom_layer", + choices=("MET1", "MET2", "MET3", "MET4", "MET5"), + example="MET2", + ), + ParamSchema( + "route.top_layer", + "route", + "top_layer", + "str", + "MET5", + "routing", + "Top routing layer", + "top_layer", + choices=("MET2", "MET3", "MET4", "MET5", "MET6"), + example="MET5", + ), + ParamSchema( + "sta.max_paths", + "sta", + "max_paths", + "int", + 1000, + "sta", + "Maximum number of paths in each STA timing report", + "sta_max_paths", + range=(1, 100000), + example="1000", + ), + ParamSchema( + param="flow.run_analysis", + group="flow", + name="run_analysis", + type="bool", + default=True, + applies="all", + description="Run per-step analysis (metrics, plots, checklist) after each step", + maps_to="run_analysis", + example="false", + ), +) + +PARAM_REGISTRY = _LEGACY_PARAM_REGISTRY + CONFIG_PARAM_SCHEMAS +_REGISTRY_INDEX = {schema.param: schema for schema in PARAM_REGISTRY} +_REQUIRED_FIELDS = ("param", "group", "name", "type", "default", "applies", "description") + + +def lookup_schema(key: str) -> ParamSchema | None: + return _REGISTRY_INDEX.get(key) + + +def list_schemas() -> tuple[ParamSchema, ...]: + return PARAM_REGISTRY + + +def list_groups() -> list[str]: + return list(dict.fromkeys(schema.group for schema in PARAM_REGISTRY)) + + +def is_known_key(key: str) -> bool: + return key in _REGISTRY_INDEX + + +def validate_schema_record(schema: ParamSchema) -> list[str]: + return [ + f"missing required field: {field}" + for field in _REQUIRED_FIELDS + if getattr(schema, field, None) is None + or (field != "default" and getattr(schema, field) == "") + ] + + +def validate_schema_type(value: object, schema: ParamSchema) -> tuple[object, str | None]: + key, ptype = schema.param, schema.type + if ptype == "int": + return ( + (value, None) + if isinstance(value, int) and not isinstance(value, bool) + else (value, f"expected int for {key}, got {type(value).__name__}") + ) + if ptype == "float": + if isinstance(value, bool) or not isinstance(value, (int, float)): + return value, f"expected float for {key}, got {type(value).__name__}" + try: + return float(value), None + except OverflowError: + return value, f"expected float for {key}, value too large to represent" + if ptype == "bool": + if isinstance(value, bool): + return value, None + if isinstance(value, str) and value.lower() in {"true", "1", "yes", "false", "0", "no"}: + return value.lower() in {"true", "1", "yes"}, None + return value, f"expected bool for {key}, got {type(value).__name__}" + if ptype == "str": + return ( + (value, None) + if isinstance(value, str) + else (value, f"expected str for {key}, got {type(value).__name__}") + ) + if ptype.startswith("list["): + if not isinstance(value, list): + return value, f"expected list for {key}, got {type(value).__name__}" + element_type = ptype[5:-1] + for index, item in enumerate(value): + if element_type == "str": + valid = isinstance(item, str) + elif element_type == "int": + valid = isinstance(item, int) and not isinstance(item, bool) + else: + valid = isinstance(item, (int, float)) and not isinstance(item, bool) + if not valid: + return ( + value, + f"expected {ptype} for {key}, element {index} is {type(item).__name__}", + ) + if element_type == "float": + try: + return [float(item) for item in value], None + except OverflowError: + return value, f"expected list[float] for {key}, value too large to represent" + return value, None + if ptype == "json": + return ( + (value, None) + if isinstance(value, (dict, list)) + else (value, f"expected JSON object or array for {key}, got {type(value).__name__}") + ) + return value, None + + +def parse_value(raw: str, schema: ParamSchema) -> object: + if schema.type in {"int", "float"}: + try: + return int(raw) if schema.type == "int" else float(raw) + except ValueError as exc: + raise ValueError(f"expected {schema.type} for {schema.param}, got '{raw}'") from exc + if schema.type == "bool": + low = raw.lower() + if low in {"true", "1", "yes"}: + return True + if low in {"false", "0", "no"}: + return False + raise ValueError(f"expected bool for {schema.param}, got '{raw}'") + if schema.type == "str": + return raw + if schema.type.startswith("list["): + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parts = [part.strip() for part in raw.strip("[]() ").split(",") if part.strip()] + element_type = schema.type[5:-1] + try: + parsed = ( + [int(part) for part in parts] + if element_type == "int" + else [float(part) for part in parts] + if element_type == "float" + else parts + ) + except ValueError as exc: + raise ValueError(f"expected {schema.type} for {schema.param}, got '{raw}'") from exc + value, error = validate_schema_type(parsed, schema) + if error: + raise ValueError(error) + return value + if schema.type == "json": + try: + return json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"expected JSON for {schema.param}, got '{raw}'") from exc + raise ValueError(f"unsupported type '{schema.type}' for {schema.param}") + + +def validate_value(value: object, schema: ParamSchema) -> list[str]: + errors = [] + if schema.range is not None: + if not isinstance(value, (int, float)) or isinstance(value, bool): + errors.append(f"value {value!r} is not numeric for {schema.param}") + elif value < schema.range[0] or value > schema.range[1]: + errors.append( + f"value {value} out of range [{schema.range[0]}, {schema.range[1]}] " + f"for {schema.param}" + ) + if schema.choices is not None and str(value) not in schema.choices: + errors.append(f"value '{value}' not in allowed choices {schema.choices} for {schema.param}") + return errors + + +@dataclass +class ResolvedParam: + param: str + value: object + default: object + source: str + schema: ParamSchema + + @property + def is_explicit(self) -> bool: + return self.source != "default" + + +def resolve_parameters(toml_overrides=None, cli_overrides=None, manifest_overrides=None): + layers = ( + (cli_overrides or {}, "cli"), + (toml_overrides or {}, "ecc.toml"), + (manifest_overrides or {}, "project.json"), + ) + resolved, errors = [], [] + for schema in PARAM_REGISTRY: + value, source = schema.default, "default" + for layer, layer_source in layers: + if schema.param not in layer: + continue + value, source = layer[schema.param], layer_source + if source != "cli": + value, type_error = validate_schema_type(value, schema) + if type_error: + errors.append(type_error) + errors.extend(validate_value(value, schema)) + break + resolved.append(ResolvedParam(schema.param, value, schema.default, source, schema)) + return resolved, errors + + +def manifest_value_for(canonical: dict, maps_to) -> tuple[object, bool]: + if isinstance(maps_to, str): + return (canonical[maps_to], True) if maps_to in canonical else (None, False) + if isinstance(maps_to, dict): + for parent, child in maps_to.items(): + node = canonical.get(parent) + if isinstance(node, dict) and child in node: + return node[child], True + return None, False + + +def coerce_manifest_parameters(canonical: dict, registry=PARAM_REGISTRY, skip_params=frozenset()): + coerced, errors = copy.deepcopy(canonical), [] + for schema in registry: + if schema.param in skip_params: + continue + value, present = manifest_value_for(coerced, schema.maps_to) + if not present: + continue + value, error = validate_schema_type(value, schema) + if error: + errors.append(error) + continue + if isinstance(schema.maps_to, str): + coerced[schema.maps_to] = value + else: + for parent, child in schema.maps_to.items(): + if isinstance(coerced.get(parent), dict) and child in coerced[parent]: + coerced[parent][child] = value + break + return coerced, errors + + +def build_backend_overrides(resolved): + overrides = {} + for item in resolved: + if item.source == "default" and item.value == item.default: + continue + target, value = item.schema.maps_to, item.value + if isinstance(target, str): + overrides[target] = value + elif isinstance(target, dict): + for parent, child in target.items(): + overrides.setdefault(parent, {})[child] = value + return overrides + + +def build_config_overrides(resolved): + overrides = {} + for item in resolved: + target = item.schema.config_target + if target is None or not item.is_explicit: + continue + current = overrides.setdefault(target.config_key, {}) + for key in target.json_path[:-1]: + current = current.setdefault(key, {}) + current[target.json_path[-1]] = item.value + return overrides + + +def build_pdk_overrides(resolved): + return { + item.schema.pdk_target: item.value + for item in resolved + if item.schema.pdk_target and item.is_explicit + } + + +def parse_cli_overrides(pairs): + result, errors = {}, [] + for pair in pairs: + if "=" not in pair: + errors.append(f"malformed override: '{pair}' (expected key=value)") + continue + key, raw = (part.strip() for part in pair.split("=", 1)) + schema = lookup_schema(key) + if schema is None: + errors.append(f"unknown parameter: '{key}'") + continue + try: + value = parse_value(raw, schema) + except ValueError as exc: + errors.append(str(exc)) + continue + value_errors = validate_value(value, schema) + if value_errors: + errors.extend(value_errors) + continue + result[key] = value + return result, errors + + +def parse_toml_params(params_table): + flat, errors = {}, [] + + def visit(path, value): + key = ".".join(path) + schema = lookup_schema(key) + if schema: + try: + parsed = parse_value(value, schema) if isinstance(value, str) else value + parsed, type_error = validate_schema_type(parsed, schema) + if type_error: + errors.append(type_error) + return + except ValueError as exc: + errors.append(str(exc)) + return + value_errors = validate_value(parsed, schema) + if value_errors: + errors.extend(value_errors) + return + flat[key] = parsed + return + if isinstance(value, dict): + for child, child_value in value.items(): + visit((*path, child), child_value) + else: + errors.append(f"unknown parameter in ecc.toml: '{key}'") + + for group, value in params_table.items(): + if isinstance(value, dict): + visit((group,), value) + else: + errors.append(f"[params.{group}] must be a table, got {type(value).__name__}") + return flat, errors diff --git a/chipcompiler/data/workspace_parameters.py b/chipcompiler/data/workspace_parameters.py new file mode 100644 index 00000000..8beee995 --- /dev/null +++ b/chipcompiler/data/workspace_parameters.py @@ -0,0 +1,127 @@ +"""Workspace-local parameter persistence shared by Engine and CLI.""" + +from copy import deepcopy + +from chipcompiler.data.parameter import update_parameters +from chipcompiler.data.parameter_schema import ( + ResolvedParam, + build_backend_overrides, + build_config_overrides, +) +from chipcompiler.data.workspace.config_overrides import CONFIG_OVERRIDES_KEY +from chipcompiler.utility import json_read + +WORKSPACE_PARAM_OVERRIDES_KEY = "workspace_param_overrides" +_APPLIES_TO_STEP = { + "synthesis": "Synthesis", + "floorplan": "Floorplan", + "placement": "place", + "cts": "CTS", + "routing": "route", + "filler": "filler", + "rcx": "RCX", + "sta": "sta", +} +_MISSING = object() + + +def workspace_param_step(schema) -> str: + step = _APPLIES_TO_STEP.get(schema.applies) + if step is None: + raise ValueError(f"{schema.param} requires a full workspace refresh") + return step + + +def workspace_param_value(workspace, schema) -> object: + if schema.pdk_target is not None: + raise ValueError(f"{schema.param} requires a full workspace refresh") + if schema.maps_to is not None: + return _parameter_target_value(workspace.parameters.data, schema.maps_to, schema.default) + if schema.config_target is not None: + config_path = workspace.config.get(schema.config_target.config_key) + if config_path is None: + raise ValueError(f"workspace config missing target: {schema.config_target.config_key}") + value = _nested_value(json_read(config_path), schema.config_target.json_path) + return schema.default if value is _MISSING else value + raise ValueError(f"{schema.param} has no workspace configuration target") + + +def set_workspace_param(workspace, schema, value: object) -> tuple[object, str]: + baseline = update_workspace_param_value(workspace, schema, value) + return baseline, workspace_param_step(schema) + + +def update_workspace_param_value(workspace, schema, value: object) -> object: + records = _override_records(workspace.parameters.data) + record = next((item for item in records if item["key"] == schema.param), None) + if record is None: + record = { + "key": schema.param, + "baseline": deepcopy(workspace_param_value(workspace, schema)), + } + records.append(record) + record["value"] = deepcopy(value) + _set_override_records(workspace.parameters.data, records) + _apply_workspace_value(workspace, schema, value) + return record["baseline"] + + +def unset_workspace_param(workspace, schema) -> tuple[object, str] | None: + records = _override_records(workspace.parameters.data) + record = next((item for item in records if item["key"] == schema.param), None) + if record is None: + return None + records.remove(record) + _set_override_records(workspace.parameters.data, records) + _apply_workspace_value(workspace, schema, record["baseline"]) + return record["baseline"], workspace_param_step(schema) + + +def workspace_param_diff(workspace) -> list[dict]: + return _override_records(workspace.parameters.data) + + +def _apply_workspace_value(workspace, schema, value: object) -> None: + resolved = ResolvedParam(schema.param, value, schema.default, "workspace", schema) + update_parameters(build_backend_overrides([resolved]), workspace.parameters.data) + config_overrides = build_config_overrides([resolved]) + if config_overrides: + update_parameters({CONFIG_OVERRIDES_KEY: config_overrides}, workspace.parameters.data) + + +def _override_records(parameters: dict) -> list[dict]: + raw = parameters.get(WORKSPACE_PARAM_OVERRIDES_KEY, []) + if not isinstance(raw, list): + return [] + return [ + deepcopy(item) + for item in raw + if isinstance(item, dict) and isinstance(item.get("key"), str) and "baseline" in item + ] + + +def _set_override_records(parameters: dict, records: list[dict]) -> None: + if records: + parameters[WORKSPACE_PARAM_OVERRIDES_KEY] = records + else: + parameters.pop(WORKSPACE_PARAM_OVERRIDES_KEY, None) + + +def _parameter_target_value(parameters: dict, target, default: object) -> object: + if isinstance(target, str): + return deepcopy(parameters.get(target, default)) + if isinstance(target, dict) and len(target) == 1: + parent, child = next(iter(target.items())) + nested = parameters.get(parent) + if isinstance(nested, dict): + return deepcopy(nested.get(child, default)) + return deepcopy(default) + + +def _nested_value(data: object, path: tuple[str, ...]) -> object: + current = data + for key in path: + if not isinstance(current, dict) or key not in current: + return _MISSING + current = current[key] + return deepcopy(current) diff --git a/chipcompiler/engine/workspace_configuration.py b/chipcompiler/engine/workspace_configuration.py index dcd68470..10d3de6d 100644 --- a/chipcompiler/engine/workspace_configuration.py +++ b/chipcompiler/engine/workspace_configuration.py @@ -4,25 +4,27 @@ from pathlib import Path from typing import Any -from chipcompiler.cli.project.params import ( - _validate_schema_type, +from chipcompiler.data import load_workspace, save_parameter +from chipcompiler.data.parameter_schema import ( list_schemas, lookup_schema, + validate_schema_type, validate_value, ) -from chipcompiler.cli.project.workspace_params import ( +from chipcompiler.data.workspace import workspace_config_paths +from chipcompiler.data.workspace_parameters import ( update_workspace_param_value, workspace_param_value, ) -from chipcompiler.data import load_workspace, save_parameter -from chipcompiler.data.workspace import workspace_config_paths from chipcompiler.engine.flow import EngineFlow from chipcompiler.engine.rerun import invalidate_from from chipcompiler.engine.snapshot import ( SNAPSHOT_FILENAME, STALE_SNAPSHOT_FILENAME, + EngineeringSnapshotError, ensure_engineering_snapshot, invalidate_engineering_snapshot, + read_engineering_snapshot, ) from chipcompiler.rtl2gds import get_flow_builders, normalize_flow_step @@ -45,18 +47,21 @@ def update_workspace_configuration( target = Path(target_directory).expanduser().resolve() if not target.is_dir(): raise WorkspaceLifecycleError("workspace_missing", f"Workspace not found: {target}") - before = _snapshot_files(_workspace_file_paths(target)) - try: - return _update_workspace_configuration( - target, - expected_workspace_revision, - configuration, - workspace_bindings, - command_id, - ) - except BaseException: - _restore_files(before) - raise + from chipcompiler.engine.reconcile import _workspace_lock + + with _workspace_lock(target): + before = _snapshot_files(_workspace_file_paths(target)) + try: + return _update_workspace_configuration( + target, + expected_workspace_revision, + configuration, + workspace_bindings, + command_id, + ) + except BaseException: + _restore_files(before) + raise def _update_workspace_configuration( @@ -108,7 +113,7 @@ def _update_workspace_configuration( schema = lookup_schema(parameter) if schema is None or schema.pdk_target is not None: raise WorkspaceLifecycleError("unknown_parameter", f"Unknown parameter: {parameter}") - normalized, type_error = _validate_schema_type(value, schema) + normalized, type_error = validate_schema_type(value, schema) errors = [type_error] if type_error else validate_value(normalized, schema) if errors: raise WorkspaceLifecycleError("invalid_parameter", str(errors[0])) @@ -164,7 +169,14 @@ def read_workspace_configuration(workspace: Any) -> dict[str, Any]: if flow_names: flow.update({"fromStepId": flow_names[0], "throughStepId": flow_names[-1]}) inputs, input_bindings = _workspace_inputs(workspace) + pdk_mode = "manual" if workspace.parameters.data.get("pdk_config") else "default" + pdk_files, pdk_file_bindings = ( + _workspace_pdk_files(workspace) if pdk_mode == "manual" else ([], {}) + ) + snapshot = _read_snapshot_metadata(workspace) return { + "workspaceId": snapshot["workspaceId"], + "workspaceRevision": snapshot["workspaceRevision"], "workspaceSpec": { "schemaVersion": 1, "design": { @@ -172,12 +184,15 @@ def read_workspace_configuration(workspace: Any) -> dict[str, Any]: "topModule": workspace.design.top_module, "clockPort": str(workspace.parameters.data.get("clock", "")), }, - "inputMode": "postSynthesis" if workspace.design.origin_def is not None else "rtl", + "inputMode": workspace.parameters.data.get("_input_mode") + if workspace.parameters.data.get("_input_mode") in {"rtl", "postSynthesis"} + else ("postSynthesis" if workspace.design.origin_def is not None else "rtl"), "inputs": inputs, "pdk": { "familyId": workspace.pdk.name, "version": workspace.pdk.version, - "mode": "default", + "mode": pdk_mode, + **({"files": pdk_files} if pdk_files else {}), }, "flow": flow, "parameters": parameters, @@ -187,6 +202,7 @@ def read_workspace_configuration(workspace: Any) -> dict[str, Any]: "pdk": { "root": str(workspace.pdk.root or ""), "version": workspace.pdk.version, + **({"files": pdk_file_bindings} if pdk_file_bindings else {}), }, }, } @@ -194,7 +210,7 @@ def read_workspace_configuration(workspace: Any) -> dict[str, Any]: def read_step_configuration(workspace: Any, step_id: str) -> dict[str, Any]: step, schemas = _step_catalog(workspace, step_id) - snapshot = ensure_engineering_snapshot(workspace) + snapshot = _read_snapshot_metadata(workspace) if not schemas: raise WorkspaceLifecycleError( "step_configuration_unavailable", @@ -231,18 +247,21 @@ def update_workspace_step_configuration( target = Path(target_directory).expanduser().resolve() if not target.is_dir(): raise WorkspaceLifecycleError("workspace_missing", f"Workspace not found: {target}") - before = _snapshot_files(_workspace_file_paths(target)) - try: - return _update_workspace_step_configuration( - target, - expected_workspace_revision, - step_id, - parameters, - command_id, - ) - except BaseException: - _restore_files(before) - raise + from chipcompiler.engine.reconcile import _workspace_lock + + with _workspace_lock(target): + before = _snapshot_files(_workspace_file_paths(target)) + try: + return _update_workspace_step_configuration( + target, + expected_workspace_revision, + step_id, + parameters, + command_id, + ) + except BaseException: + _restore_files(before) + raise def _update_workspace_step_configuration( @@ -289,7 +308,7 @@ def _update_workspace_step_configuration( "parameter_not_applicable", f"Parameter {parameter} is not configurable at {step}", ) - normalized, type_error = _validate_schema_type(value, schema) + normalized, type_error = validate_schema_type(value, schema) errors = [type_error] if type_error else validate_value(normalized, schema) if errors: raise WorkspaceLifecycleError("invalid_parameter", str(errors[0])) @@ -378,7 +397,10 @@ def read_workspace_configuration_from_directory( def _workspace_inputs(workspace: Any) -> tuple[list[dict[str, str]], dict[str, str]]: values = [] - if workspace.design.input_filelist is not None: + input_mode = workspace.parameters.data.get("_input_mode") + if input_mode == "postSynthesis" and workspace.design.origin_verilog is not None: + values.append(("netlist", workspace.design.origin_verilog)) + elif workspace.design.input_filelist is not None: values.append(("filelist", workspace.design.input_filelist)) elif workspace.design.origin_verilog is not None: role = "netlist" if workspace.design.origin_def is not None else "rtl" @@ -397,11 +419,45 @@ def _workspace_inputs(workspace: Any) -> tuple[list[dict[str, str]], dict[str, s ) +def _workspace_pdk_files(workspace: Any) -> tuple[list[dict[str, str]], dict[str, str]]: + grouped = { + "tech": [workspace.pdk.tech] if workspace.pdk.tech else [], + "lef": list(workspace.pdk.lefs), + "liberty": list(workspace.pdk.libs), + "mapping": [workspace.pdk.mapping_file] if workspace.pdk.mapping_file else [], + } + refs = [] + bindings = {} + for role, paths in grouped.items(): + for index, path in enumerate(paths): + file_id = role if len(paths) == 1 else f"{role}-{index}" + refs.append({"fileId": file_id, "role": role}) + bindings[file_id] = str(path) + return refs, bindings + + +def _read_snapshot_metadata(workspace: Any) -> dict[str, Any]: + try: + snapshot = read_engineering_snapshot(workspace) + except (EngineeringSnapshotError, OSError): + return {"workspaceId": None, "workspaceRevision": None} + return { + "workspaceId": snapshot["workspaceId"], + "workspaceRevision": snapshot["workspaceRevision"], + } + + def _flow_id(names: list[str]) -> str: for flow_id, builder in get_flow_builders().items(): candidate = [str(getattr(step, "value", step)) for step, _tool, _state in builder()] if candidate == names: return flow_id + if flow_id == "rtl2gds" and names: + start = next( + (index for index, value in enumerate(candidate) if value == names[0]), None + ) + if start is not None and candidate[start : start + len(names)] == names: + return flow_id return "custom" diff --git a/chipcompiler/engine/workspace_lifecycle.py b/chipcompiler/engine/workspace_lifecycle.py index 97c826e0..e4db911c 100644 --- a/chipcompiler/engine/workspace_lifecycle.py +++ b/chipcompiler/engine/workspace_lifecycle.py @@ -8,8 +8,8 @@ from pathlib import Path from typing import Any -from chipcompiler.cli.project.params import build_backend_overrides, resolve_parameters from chipcompiler.data import PDK, create_workspace, get_pdk, load_workspace +from chipcompiler.data.parameter_schema import build_backend_overrides, resolve_parameters from chipcompiler.engine.snapshot import create_engineering_snapshot from chipcompiler.engine.workspace_spec import validate_workspace_spec from chipcompiler.rtl2gds import get_flow_builders @@ -82,6 +82,20 @@ def create_workspace_from_spec( spec: object, bindings: object, command_id: str = "", +): + """Create a Workspace Spec target while serializing sibling creators.""" + from chipcompiler.engine.reconcile import _workspace_lock + + target = Path(target_directory).expanduser().resolve() + with _workspace_lock(target): + return _create_workspace_from_spec(target, spec, bindings, command_id) + + +def _create_workspace_from_spec( + target_directory: str | Path, + spec: object, + bindings: object, + command_id: str = "", ): target = Path(target_directory).expanduser().resolve() fingerprint = _workspace_command_fingerprint("create", spec, bindings) @@ -122,9 +136,11 @@ def create_workspace_from_spec( } ) flow_steps = get_flow_builders()[resolved["flow"]["flowId"]]() + flow_start = resolved["flow"].get("fromStepId") + flow_end = resolved["flow"].get("throughStepId") flow_config = { - "start_step": str(getattr(flow_steps[0][0], "value", flow_steps[0][0])), - "end_step": str(getattr(flow_steps[-1][0], "value", flow_steps[-1][0])), + "start_step": flow_start or str(getattr(flow_steps[0][0], "value", flow_steps[0][0])), + "end_step": flow_end or str(getattr(flow_steps[-1][0], "value", flow_steps[-1][0])), } pdk, pdk_root, pdk_overrides = _bound_pdk( resolved["pdk"], _string_keyed_dict(binding_map["pdk"]) @@ -167,6 +183,11 @@ def create_workspace_from_spec( raise WorkspaceLifecycleError( "workspace_create_failed", f"Workspace creation failed: {target}" ) + workspace.parameters.data["_input_mode"] = resolved["inputMode"] + from chipcompiler.data import save_parameter + + if not save_parameter(workspace.parameters): + raise OSError(f"Failed to persist input mode: {workspace.parameters.path}") snapshot = create_engineering_snapshot(workspace) _write_workspace_command( target, @@ -192,6 +213,24 @@ def update_workspace_from_spec( spec: object, bindings: object, command_id: str = "", +): + target = Path(target_directory).expanduser().resolve() + if not target.is_dir(): + raise WorkspaceLifecycleError("workspace_missing", f"Workspace not found: {target}") + from chipcompiler.engine.reconcile import _workspace_lock + + with _workspace_lock(target): + return _update_workspace_from_spec( + target, expected_workspace_revision, spec, bindings, command_id + ) + + +def _update_workspace_from_spec( + target_directory: str | Path, + expected_workspace_revision: int, + spec: object, + bindings: object, + command_id: str = "", ): from chipcompiler.engine.snapshot import ensure_engineering_snapshot diff --git a/chipcompiler/engine/workspace_spec.py b/chipcompiler/engine/workspace_spec.py index 9facf0fd..f95f1c40 100644 --- a/chipcompiler/engine/workspace_spec.py +++ b/chipcompiler/engine/workspace_spec.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import Any -from chipcompiler.cli.project.params import list_schemas, resolve_parameters +from chipcompiler.data.parameter_schema import list_schemas, resolve_parameters from chipcompiler.engine.pdk_binding import pdk_binding_content_hash from chipcompiler.rtl2gds import get_flow_builders, normalize_flow_step from chipcompiler.utility.filelist import validate_filelist From b4bdb3057ab39761880dcf4136301969a30925db Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 03:45:52 +0800 Subject: [PATCH 12/43] fix(workspace): complete headless lifecycle contracts Preserve creator input modes, flow filters, read-only configuration and signoff assessment semantics across CLI and headless APIs. --- chipcompiler/cli/command_handlers/param.py | 16 ++++- chipcompiler/cli/project/run_prepare.py | 3 + chipcompiler/engine/signoff/collector.py | 1 + chipcompiler/engine/signoff_assessment.py | 9 +++ chipcompiler/engine/signoff_export.py | 37 ++++++++++- chipcompiler/tools/ecc/signoff_checklist.py | 19 +++++- test/cli/commands/test_workspace_range.py | 1 + test/cli/params/test_commands.py | 39 ----------- test/cli/params/test_flow_filters.py | 72 +++++++++++++++++++++ test/engine/test_signoff_assessment.py | 41 ++++++++++++ test/engine/test_workspace_spec.py | 21 ++++++ test/test_signoff_package.py | 12 ++++ 12 files changed, 225 insertions(+), 46 deletions(-) create mode 100644 test/cli/params/test_flow_filters.py create mode 100644 test/engine/test_signoff_assessment.py diff --git a/chipcompiler/cli/command_handlers/param.py b/chipcompiler/cli/command_handlers/param.py index 0fe62d6f..0e795c64 100644 --- a/chipcompiler/cli/command_handlers/param.py +++ b/chipcompiler/cli/command_handlers/param.py @@ -62,14 +62,28 @@ def param_list(args, ctx: CommandContext) -> CommandResult: selected_step = normalize_flow_step(getattr(args, "step", None) or "").casefold() show_all = bool(getattr(args, "all", False)) + preset = getattr(ctx.config, "flow_preset", "") + builder = get_flow_builders().get(preset) + flow_step_names = ( + [normalize_flow_step(step).casefold() for step, _tool, _state in builder()] + if builder is not None + else [] + ) + flow_steps = set(flow_step_names) + first_flow_step = flow_step_names[0] if flow_step_names else "" + if selected_step and flow_steps and selected_step not in flow_steps: + return CommandResult.ok([]) records = [] for rp in resolved: s = rp.schema + applies = normalize_flow_step(s.applies).casefold() + if flow_steps and s.applies != "all" and applies not in flow_steps: + continue schema_steps = { normalize_flow_step(s.group).casefold(), normalize_flow_step(s.applies).casefold(), } - global_at_first_step = s.applies == "all" and selected_step == "synthesis" + global_at_first_step = s.applies == "all" and selected_step == first_flow_step if selected_step and selected_step not in schema_steps and not global_at_first_step: continue if not selected_step and not show_all and s.has_direct_target and not rp.is_explicit: diff --git a/chipcompiler/cli/project/run_prepare.py b/chipcompiler/cli/project/run_prepare.py index bed0caf2..00fdb580 100644 --- a/chipcompiler/cli/project/run_prepare.py +++ b/chipcompiler/cli/project/run_prepare.py @@ -353,6 +353,9 @@ def failed_workspace(reason: str | None) -> CommandResult: input_filelist = generated_filelist origin_verilog = "" parameters = to_parameters(cfg) + if uses_netlist_input: + # Preserve the creator's input role for Studio/CLI reopen symmetry. + parameters["_input_mode"] = "postSynthesis" pdk_root = resolve_pdk_root(cfg) if base is not None: diff --git a/chipcompiler/engine/signoff/collector.py b/chipcompiler/engine/signoff/collector.py index ba4fac13..a34c7cd6 100644 --- a/chipcompiler/engine/signoff/collector.py +++ b/chipcompiler/engine/signoff/collector.py @@ -578,6 +578,7 @@ def add_file( checklist_data = rebuild_home_checklist( self.workspace, resource_issues=[*issues, *analysis_issues], + persist=options.materialize, ) add_file( "status.checklist", diff --git a/chipcompiler/engine/signoff_assessment.py b/chipcompiler/engine/signoff_assessment.py index 9b378ce2..e5018c0a 100644 --- a/chipcompiler/engine/signoff_assessment.py +++ b/chipcompiler/engine/signoff_assessment.py @@ -24,6 +24,15 @@ class _ReviewGroup(TypedDict): def build_signoff_assessment(workspace: Any) -> dict[str, Any]: + flow = getattr(workspace, "flow", None) + steps_fn = getattr(flow, "steps", None) + if callable(steps_fn): + steps = steps_fn() + else: + flow_data = getattr(flow, "data", None) + steps = flow_data.get("steps", []) if isinstance(flow_data, dict) else [] + if steps and any(str(step.get("state", "")) not in {"Success", "Skipped"} for step in steps): + return _unavailable_assessment() checklist = json_read(Path(workspace.directory) / "home" / "checklist.json") if checklist.get("schema_version") != 3 or checklist.get("kind") != "signoff_checklist": return _unavailable_assessment() diff --git a/chipcompiler/engine/signoff_export.py b/chipcompiler/engine/signoff_export.py index 1c7a19d9..fbeda19b 100644 --- a/chipcompiler/engine/signoff_export.py +++ b/chipcompiler/engine/signoff_export.py @@ -44,11 +44,23 @@ def export_signoff_package_archive( package_dir = Path(result.package_dir) + if additional_files is not None and not isinstance(additional_files, list): + raise SignoffExportError("additional_files must be a list") if additional_files: for file_info in additional_files: - p = package_dir / file_info["archivePath"] - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(file_info["content"], encoding="utf-8") + if not isinstance(file_info, dict): + raise SignoffExportError("additional file entries must be objects") + archive_path = file_info.get("archivePath") + content = file_info.get("content") + if not isinstance(archive_path, str) or not archive_path: + raise SignoffExportError("additional file entries require archivePath") + if not isinstance(content, str): + raise SignoffExportError("additional file entries require string content") + path = _additional_file_path(package_dir, archive_path) + path.parent.mkdir(parents=True, exist_ok=True) + if _has_symlink_parent(package_dir, path): + raise SignoffExportError("additional file path contains a symlink") + path.write_text(content, encoding="utf-8") archive = package_dir.with_suffix(".tar.gz") import tarfile @@ -70,3 +82,22 @@ def export_signoff_package_archive( staged_path.unlink(missing_ok=True) return str(destination) + + +def _additional_file_path(package_dir: Path, archive_path: str) -> Path: + relative = Path(archive_path) + if not archive_path or relative.is_absolute() or ".." in relative.parts: + raise SignoffExportError("additional file path must stay inside the signoff package") + destination = (package_dir / relative).resolve() + if not destination.is_relative_to(package_dir.resolve()): + raise SignoffExportError("additional file path must stay inside the signoff package") + return destination + + +def _has_symlink_parent(package_dir: Path, path: Path) -> bool: + current = package_dir + for component in path.relative_to(package_dir).parts: + current /= component + if current.is_symlink(): + return True + return False diff --git a/chipcompiler/tools/ecc/signoff_checklist.py b/chipcompiler/tools/ecc/signoff_checklist.py index 25cd0984..304eb23e 100644 --- a/chipcompiler/tools/ecc/signoff_checklist.py +++ b/chipcompiler/tools/ecc/signoff_checklist.py @@ -639,7 +639,9 @@ def _package_items(resource_issues) -> list[dict]: return items -def rebuild_home_checklist(workspace: Workspace, resource_issues=None) -> dict: +def rebuild_home_checklist( + workspace: Workspace, resource_issues=None, *, persist: bool = True +) -> dict: """Replace the aggregate workspace checklist from current step snapshots.""" workspace_directory = getattr(workspace, "directory", None) if not workspace_directory: @@ -692,6 +694,17 @@ def rebuild_home_checklist(workspace: Workspace, resource_issues=None) -> dict: checklist_path = workspace_dir / "home" / "checklist.json" if workspace.home.path is not None: workspace.home.set_checklist(checklist_path) - checklist = Checklist(checklist_path) - checklist.replace(list(deduplicated.values())) + if persist: + checklist = Checklist(checklist_path) + checklist.replace(list(deduplicated.values())) + else: + # Build the same normalized payload without Checklist.__init__ or + # replace(), both of which intentionally write to home/checklist.json. + checklist = object.__new__(Checklist) + checklist.path = Path(checklist_path) + checklist.data = checklist._default_data() + checklist.data["checklist"] = [ + checklist._normalize_item(item) for item in deduplicated.values() + ] + Checklist._refresh_summary(checklist.data) return checklist.data diff --git a/test/cli/commands/test_workspace_range.py b/test/cli/commands/test_workspace_range.py index 3cfce1a9..fe585e3b 100644 --- a/test/cli/commands/test_workspace_range.py +++ b/test/cli/commands/test_workspace_range.py @@ -53,6 +53,7 @@ def test_new_workspace_range_uses_ecc_toml_inputs_and_registers_before_execution assert create_kwargs["directory"] == str(Path(project_dir) / "cts-only") assert create_kwargs["origin_def"] == str(design_def) assert create_kwargs["origin_verilog"] == str(netlist) + assert create_kwargs["parameters"]["_input_mode"] == "postSynthesis" assert create_kwargs["flow_config"] == {"start_step": "CTS", "end_step": "CTS"} manifest = json.loads((Path(project_dir) / "project.json").read_text()) entry = manifest["workspaces"][0] diff --git a/test/cli/params/test_commands.py b/test/cli/params/test_commands.py index 015e7836..abeeac71 100644 --- a/test/cli/params/test_commands.py +++ b/test/cli/params/test_commands.py @@ -26,20 +26,6 @@ def test_param_list_step_includes_config_schemas(self, tmp_path, capsys, create_ assert rc == 0 assert "cts.skew_bound" in capsys.readouterr().out - def test_param_list_synthesis_includes_global_parameters( - self, tmp_path, capsys, create_cli_project, plain_records - ): - project_dir = create_cli_project() - rc = cli_main.run( - ["param", "list", "--step", "synthesis", "--project", project_dir, "--plain"] - ) - - assert rc == 0 - assert [record["param"] for record in plain_records(capsys.readouterr().out)] == [ - "design.frequency_mhz", - "flow.run_analysis", - ] - class TestParamShow: def test_param_show_known_key(self, tmp_path, capsys, create_cli_project): @@ -104,31 +90,6 @@ def test_param_set_rejects_invalid_value(self, tmp_path, capsys, create_cli_proj rc = cli_main.run(["param", "set", "place.target_density", "1.5", "--project", project_dir]) assert rc == 1 - def test_param_set_rejects_parameter_outside_project_flow( - self, tmp_path, capsys, create_cli_project, plain_records - ): - project_dir = create_cli_project() - config_path = tmp_path / "gcd" / "ecc.toml" - config_path.write_text( - config_path.read_text().replace('preset = "rtl2gds"', 'preset = "syn_sta"') - ) - - rc = cli_main.run( - [ - "param", - "set", - "place.target_density", - "0.65", - "--project", - project_dir, - "--plain", - ] - ) - - assert rc == 1 - assert plain_records(capsys.readouterr().out)[0]["error"] == "parameter_not_in_flow" - assert "target_density" not in config_path.read_text() - def test_param_set_accepts_negative_value( self, tmp_path, capsys, create_cli_project, plain_records ): diff --git a/test/cli/params/test_flow_filters.py b/test/cli/params/test_flow_filters.py new file mode 100644 index 00000000..e9486dc1 --- /dev/null +++ b/test/cli/params/test_flow_filters.py @@ -0,0 +1,72 @@ +from chipcompiler.cli import main as cli_main + + +def test_param_list_synthesis_includes_global_parameters(capsys, create_cli_project, plain_records): + project_dir = create_cli_project() + rc = cli_main.run(["param", "list", "--step", "synthesis", "--project", project_dir, "--plain"]) + + assert rc == 0 + assert [record["param"] for record in plain_records(capsys.readouterr().out)] == [ + "design.frequency_mhz", + "flow.run_analysis", + ] + + +def test_param_set_rejects_parameter_outside_project_flow( + tmp_path, capsys, create_cli_project, plain_records +): + project_dir = create_cli_project() + config_path = tmp_path / "gcd" / "ecc.toml" + config_path.write_text( + config_path.read_text().replace('preset = "rtl2gds"', 'preset = "syn_sta"') + ) + + rc = cli_main.run( + [ + "param", + "set", + "place.target_density", + "0.65", + "--project", + project_dir, + "--plain", + ] + ) + + assert rc == 1 + assert plain_records(capsys.readouterr().out)[0]["error"] == "parameter_not_in_flow" + assert "target_density" not in config_path.read_text() + + +def test_param_list_excludes_steps_absent_from_selected_flow( + tmp_path, capsys, create_cli_project, plain_records +): + project_dir = create_cli_project() + config_path = tmp_path / "gcd" / "ecc.toml" + config_path.write_text( + config_path.read_text().replace('preset = "rtl2gds"', 'preset = "synthesis_lec"') + ) + + rc = cli_main.run(["param", "list", "--step", "floorplan", "--project", project_dir, "--plain"]) + + assert rc == 0 + assert plain_records(capsys.readouterr().out) == [] + + +def test_param_list_default_excludes_explicit_absent_step_parameters( + tmp_path, capsys, create_cli_project, plain_records +): + project_dir = create_cli_project() + config_path = tmp_path / "gcd" / "ecc.toml" + config_path.write_text( + config_path.read_text().replace('preset = "rtl2gds"', 'preset = "synthesis_lec"') + + "\n[params.place]\ntarget_density = 0.65\n" + ) + + rc = cli_main.run(["param", "list", "--project", project_dir, "--plain"]) + + assert rc == 0 + assert all( + record["param"] != "place.target_density" + for record in plain_records(capsys.readouterr().out) + ) diff --git a/test/engine/test_signoff_assessment.py b/test/engine/test_signoff_assessment.py new file mode 100644 index 00000000..821144b8 --- /dev/null +++ b/test/engine/test_signoff_assessment.py @@ -0,0 +1,41 @@ +import json +from pathlib import Path +from types import SimpleNamespace + +from chipcompiler.engine.signoff_assessment import build_signoff_assessment +from chipcompiler.engine.signoff_export import SignoffExportError, _additional_file_path + + +def test_stale_checklist_cannot_make_incomplete_flow_ready(tmp_path): + home = tmp_path / "home" + home.mkdir() + (home / "checklist.json").write_text( + json.dumps( + { + "schema_version": 3, + "kind": "signoff_checklist", + "status": "ready", + "checklist": [], + } + ) + ) + workspace = SimpleNamespace( + directory=Path(tmp_path), + flow=SimpleNamespace(steps=lambda: [{"name": "Synthesis", "state": "Unstart"}]), + ) + + result = build_signoff_assessment(workspace) + + assert result["status"] == "blocked" + + +def test_signoff_additional_file_path_rejects_escape(tmp_path): + package = tmp_path / "package" + package.mkdir() + + for value in ("/tmp/escape", "../escape", ""): + try: + _additional_file_path(package, value) + except SignoffExportError: + continue + raise AssertionError(f"unsafe path accepted: {value!r}") diff --git a/test/engine/test_workspace_spec.py b/test/engine/test_workspace_spec.py index d67d5a4f..d0703bfd 100644 --- a/test/engine/test_workspace_spec.py +++ b/test/engine/test_workspace_spec.py @@ -200,6 +200,27 @@ def test_manual_pdk_workspace_reopens_through_main_persistence(tmp_path): assert assess_execution_readiness(target, bindings) == {"ready": True} +def test_partial_flow_spec_preserves_requested_boundaries(tmp_path, minimal_ics55_pdk_factory): + from chipcompiler.data import load_workspace + from chipcompiler.engine import create_workspace_from_spec + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + spec = deepcopy(payload["workspaceSpec"]) + spec["flow"] = { + "flowId": "rtl2gds", + "fromStepId": "Synthesis", + "throughStepId": "lec", + } + spec["parameters"] = {"design.frequency_mhz": 200.0} + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings) + + assert [step["name"] for step in load_workspace(workspace.directory).flow.steps()] == [ + "Synthesis", + "lec", + ] + + def test_workspace_spec_update_is_atomic_revisioned_and_idempotent( tmp_path, minimal_ics55_pdk_factory ): diff --git a/test/test_signoff_package.py b/test/test_signoff_package.py index c2c2bfec..1de074f1 100644 --- a/test/test_signoff_package.py +++ b/test/test_signoff_package.py @@ -301,6 +301,18 @@ def test_collect_signoff_package_requires_synthesis_verilog(tmp_path): ) +def test_read_only_signoff_collection_does_not_rewrite_home_checklist(tmp_path): + workspace_dir = _make_signoff_workspace(tmp_path) + checklist = workspace_dir / "home" / "checklist.json" + before = file_digest(checklist) + + _make_engine_flow(workspace_dir).collect_signoff_package( + SignoffPackageOptions(archive=False, materialize=False) + ) + + assert file_digest(checklist) == before + + def test_collect_signoff_package_rejects_stale_post_route_lec_proof(tmp_path): workspace_dir = _make_signoff_workspace(tmp_path) gate = workspace_dir / "lvs_ecc" / "output" / "gcd_lvs.v.gz" From 92e634bd24ce616c791207ff06004fa0b257a659 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 03:46:11 +0800 Subject: [PATCH 13/43] refactor(engine): share canonical QoR scoring Reuse the report scoring policy for headless assessments and expose the committed step callback contract. --- chipcompiler/engine/execution.py | 2 +- chipcompiler/engine/qor_scoring.py | 156 +++++++++-------------------- test/engine/test_execution.py | 27 +++++ 3 files changed, 73 insertions(+), 112 deletions(-) diff --git a/chipcompiler/engine/execution.py b/chipcompiler/engine/execution.py index e25a2cd0..24faa3dd 100644 --- a/chipcompiler/engine/execution.py +++ b/chipcompiler/engine/execution.py @@ -61,7 +61,7 @@ def __init__(self, workspace: Any): self.workspace = workspace self.snapshot = ensure_engineering_snapshot(workspace) - def commit_step(self, _step: Any, state: Any, _error: str | None = None) -> None: + def on_step_completed(self, _step: Any, state: Any, _error: str | None = None) -> None: from chipcompiler.engine.snapshot import commit_engineering_snapshot self.snapshot = commit_engineering_snapshot( diff --git a/chipcompiler/engine/qor_scoring.py b/chipcompiler/engine/qor_scoring.py index 00fc6283..01be13a6 100644 --- a/chipcompiler/engine/qor_scoring.py +++ b/chipcompiler/engine/qor_scoring.py @@ -1,58 +1,19 @@ from dataclasses import dataclass from typing import Any -QOR_SCORE_THRESHOLD = 60 - -DIMENSION_WEIGHTS = { - "timing": 0.35, - "power_integrity": 0.25, - "routability_physical": 0.2, - "area_cost": 0.1, - "clock_robustness_dfm": 0.1, - "runtime": 0.0, -} - -METRIC_FAIL_VALUES = { - "drc_count": 10, - "lvs_count": 10, - "route_wirelength": 6000, - "route_via_count": 2000, - "cts_buffer_count": 20, - "cts_buffer_area": 40, - "clock_wirelength": 400000, - "cts_clock_wirelength_max": 100000, - "cts_clock_tree_max_level": 20, - "die_area": 3000, - "core_area": 2500, - "core_utilization": 0.85, - "synthesis_cell_area": 3000, - "fanout_max": 100, - "place_hpwl": 10000, - "place_grwl": 12000, - "place_flute_wirelength": 10000, - "place_congestion_egr_overflow_total": 100, - "place_congestion_egr_overflow_max": 20, - "place_rudy_utilization_max": 1, - "place_lutrudy_utilization_max": 1, - "route_dr_total_violation_count": 50, - "route_dr_total_patch_count": 100, - "route_dr_total_wirelength": 6000, - "route_dr_total_via_count": 2000, - "route_la_total_overflow": 100, - "rcx_missing_corner_count": 9, - "sta_setup_wns": -0.2, - "sta_setup_tns": -1, - "sta_hold_wns": -0.2, - "sta_hold_tns": -1, - "sta_frequency_mhz": 100, - "sta_setup_violation_count": 1, - "sta_hold_violation_count": 1, - "sta_missing_corner_count": 1, - "harden_artifact_missing_count": 6, -} - -_SLACK_METRICS = {"sta_setup_wns", "sta_setup_tns", "sta_hold_wns", "sta_hold_tns"} -_ROLE_PRIORITY = {"final": 0, "gate": 1, "trend": 2, "none": 3} +from .qor_report import ( + _STEP_ENUM_TO_LABEL, + QorMetricRecord, + _select_project_records, + _weighted_overall, + score_record, +) +from .qor_report import ( + DIMENSION_WEIGHTS as DIMENSION_WEIGHTS, +) +from .qor_report import ( + QOR_SCORE_THRESHOLD as QOR_SCORE_THRESHOLD, +) @dataclass(frozen=True) @@ -71,7 +32,7 @@ class QorScoringMetric: @dataclass(frozen=True) class ScoredQorMetric: - metric: QorScoringMetric + metric: QorMetricRecord score: float | None @@ -84,80 +45,53 @@ class QorScoringResult: def score_qor(records: list[QorScoringMetric]) -> QorScoringResult: + canonical = tuple(_canonical_record(record) for record in records) area_step = next( ( record.step - for record in reversed(records) + for record in reversed(canonical) if record.dimension == "area_cost" and record.rating_score ), None, ) - selected: dict[tuple[str, str, str], tuple[int, int, QorScoringMetric]] = {} - for order, record in enumerate(records): - if record.project_role == "none": - continue - if record.dimension == "area_cost" and record.step != area_step: - continue - key = (record.metric_id, record.scope, record.corner or "") - candidate = (_ROLE_PRIORITY.get(record.project_role, 3), -order, record) - if key not in selected or candidate[:2] < selected[key][:2]: - selected[key] = candidate - + selected = _select_project_records(canonical, area_step) scored = tuple( - ScoredQorMetric(record, score_metric(record) if record.rating_score else None) - for _role, _order, record in sorted(selected.values(), key=lambda item: item[2].metric_id) + ScoredQorMetric(record, score_record(record) if record.rating_score else None) + for record in selected ) by_dimension: dict[str, list[float]] = {} for item in scored: if item.score is not None: by_dimension.setdefault(item.metric.dimension, []).append(item.score) dimensions = { - dimension: ( - round(sum(by_dimension[dimension]) / len(by_dimension[dimension]), 1), - len(by_dimension[dimension]), - ) - for dimension in DIMENSION_WEIGHTS - if dimension in by_dimension + dimension: (round(sum(values) / len(values), 1), len(values)) + for dimension, values in by_dimension.items() } - weighted = sum( - score * DIMENSION_WEIGHTS[dimension] - for dimension, (score, _count) in dimensions.items() - if DIMENSION_WEIGHTS[dimension] > 0 + return QorScoringResult( + area_step, + scored, + dimensions, + round(_weighted_overall({key: value[0] for key, value in dimensions.items()}), 1) + if dimensions + else None, ) - overall = ( - round(weighted, 1) - if any(DIMENSION_WEIGHTS[dimension] > 0 for dimension in dimensions) - else None - ) - return QorScoringResult(area_step, scored, dimensions, overall) def score_metric(record: QorScoringMetric) -> float | None: - if record.direction == "trend_only": - return None - fail = METRIC_FAIL_VALUES.get(record.metric_id) - if fail is None: - return None - if record.metric_id in _SLACK_METRICS: - if fail >= 0: - return None - return 100.0 if record.value >= 0 else _clamp(100 * (record.value - fail) / -fail) - if record.direction == "target_range": - if record.metric_id != "core_utilization": - return None - if 0.45 <= record.value <= 0.7: - return 100.0 - if record.value < 0.45: - return _clamp(100 * record.value / 0.45) - return _clamp(100 * (fail - record.value) / (fail - 0.7)) - if fail <= 0: - return None - if record.direction == "lower_is_better": - return _clamp(100 * (fail - record.value) / fail) - if record.direction == "higher_is_better": - return _clamp(100 * record.value / fail) - return None - - -def _clamp(value: float) -> float: - return max(0.0, min(100.0, value)) + return score_record(_canonical_record(record)) + + +def _canonical_record(record: QorScoringMetric) -> QorMetricRecord: + return QorMetricRecord( + step=_STEP_ENUM_TO_LABEL.get(record.step, record.step), + metric_name=record.metric_id, + display_name=record.metric_id, + value=record.value, + dimension=record.dimension, + polarity=record.direction, + scope=record.scope, + corner=record.corner, + project_role=record.project_role, + step_role="detail", + rating_score=record.rating_score, + ) diff --git a/test/engine/test_execution.py b/test/engine/test_execution.py index ba0851f0..0efef4ef 100644 --- a/test/engine/test_execution.py +++ b/test/engine/test_execution.py @@ -48,3 +48,30 @@ def run_steps(self, *, rerun=False, observer=None): assert not result.succeeded assert result.state == StateEnum.Imcomplete.value + + +def test_default_execution_observer_commits_completed_steps(monkeypatch, tmp_path): + from chipcompiler.engine import execution + + committed = [] + + class Flow: + workspace = SimpleNamespace(directory=tmp_path) + + def run_steps(self, *, rerun=False, observer=None): + observer.on_step_completed(SimpleNamespace(name="synthesis"), StateEnum.Success) + return True + + monkeypatch.setattr( + execution, + "_EngineeringCommitSink", + lambda workspace: SimpleNamespace( + snapshot={"workspaceId": "workspace-1", "workspaceRevision": 1}, + on_step_completed=lambda step, state, error=None: committed.append((step.name, state)), + ), + ) + + result = execution.execute(Flow(), execution.ExecutionPlan(intent="run")) + + assert result.succeeded + assert committed == [("synthesis", StateEnum.Success)] From 358bb3794a91c0974375ab47a318c7ff256953e5 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 04:09:01 +0800 Subject: [PATCH 14/43] fix(engine): harden query and signoff error boundaries Keep manifest audit timestamps, tolerate forward-version QoR facts and malformed checklists, and map collector failures to the signoff domain error. --- chipcompiler/engine/qor_report.py | 10 +++++++-- chipcompiler/engine/qor_scoring.py | 27 ++++++++++++++++++++--- chipcompiler/engine/signoff_assessment.py | 7 +++++- chipcompiler/engine/signoff_export.py | 19 +++++++++------- chipcompiler/project/api.py | 9 ++++---- test/engine/test_qor_scoring.py | 11 +++++++++ test/engine/test_signoff_assessment.py | 11 +++++++++ test/project/test_manifest.py | 24 ++++++++++++++++++++ 8 files changed, 100 insertions(+), 18 deletions(-) diff --git a/chipcompiler/engine/qor_report.py b/chipcompiler/engine/qor_report.py index dbfac606..95cfa97b 100644 --- a/chipcompiler/engine/qor_report.py +++ b/chipcompiler/engine/qor_report.py @@ -310,7 +310,13 @@ def _select_project_records(records, area_scoring_step) -> list[QorMetricRecord] def _selection_rank(record: QorMetricRecord) -> tuple: - return (_ROLE_PRIORITY[record.project_role], -FLOW_STEPS.index(record.step)) + try: + step_rank = FLOW_STEPS.index(record.step) + except ValueError: + # Forward-version/custom steps remain presentable but have no + # canonical ordering; prefer known flow evidence when deduplicating. + step_rank = -1 + return (_ROLE_PRIORITY.get(record.project_role, len(_ROLE_PRIORITY)), -step_rank) def _resolve_area_scoring_step(records, flow_steps_by_label) -> str | None: @@ -384,7 +390,7 @@ def _weighted_overall(dimension_scores: dict) -> float | None: weighted_total = 0.0 used_weight = 0.0 for dimension, score in dimension_scores.items(): - weight = DIMENSION_WEIGHTS[dimension] + weight = DIMENSION_WEIGHTS.get(dimension, 0.0) if weight <= 0: continue weighted_total += score * weight diff --git a/chipcompiler/engine/qor_scoring.py b/chipcompiler/engine/qor_scoring.py index 01be13a6..ce96787e 100644 --- a/chipcompiler/engine/qor_scoring.py +++ b/chipcompiler/engine/qor_scoring.py @@ -32,7 +32,7 @@ class QorScoringMetric: @dataclass(frozen=True) class ScoredQorMetric: - metric: QorMetricRecord + metric: QorScoringMetric score: float | None @@ -45,7 +45,8 @@ class QorScoringResult: def score_qor(records: list[QorScoringMetric]) -> QorScoringResult: - canonical = tuple(_canonical_record(record) for record in records) + canonical_pairs = tuple((record, _canonical_record(record)) for record in records) + canonical = tuple(item[1] for item in canonical_pairs) area_step = next( ( record.step @@ -55,8 +56,14 @@ def score_qor(records: list[QorScoringMetric]) -> QorScoringResult: None, ) selected = _select_project_records(canonical, area_step) + original_by_canonical_id = { + id(canonical_record): original for original, canonical_record in canonical_pairs + } scored = tuple( - ScoredQorMetric(record, score_record(record) if record.rating_score else None) + ScoredQorMetric( + original_by_canonical_id.get(id(record), _restore_record(record)), + score_record(record) if record.rating_score else None, + ) for record in selected ) by_dimension: dict[str, list[float]] = {} @@ -95,3 +102,17 @@ def _canonical_record(record: QorScoringMetric) -> QorMetricRecord: step_role="detail", rating_score=record.rating_score, ) + + +def _restore_record(record: QorMetricRecord) -> QorScoringMetric: + return QorScoringMetric( + step=record.step, + metric_id=record.metric_name, + value=record.value, + dimension=record.dimension, + direction=record.polarity, + scope=record.scope, + corner=record.corner, + project_role=record.project_role, + rating_score=record.rating_score, + ) diff --git a/chipcompiler/engine/signoff_assessment.py b/chipcompiler/engine/signoff_assessment.py index e5018c0a..35c71bf0 100644 --- a/chipcompiler/engine/signoff_assessment.py +++ b/chipcompiler/engine/signoff_assessment.py @@ -34,7 +34,12 @@ def build_signoff_assessment(workspace: Any) -> dict[str, Any]: if steps and any(str(step.get("state", "")) not in {"Success", "Skipped"} for step in steps): return _unavailable_assessment() checklist = json_read(Path(workspace.directory) / "home" / "checklist.json") - if checklist.get("schema_version") != 3 or checklist.get("kind") != "signoff_checklist": + if ( + not isinstance(checklist, dict) + or checklist.get("schema_version") != 3 + or checklist.get("kind") != "signoff_checklist" + or not isinstance(checklist.get("checklist"), list) + ): return _unavailable_assessment() groups: dict[str, _ReviewGroup] = {} diff --git a/chipcompiler/engine/signoff_export.py b/chipcompiler/engine/signoff_export.py index fbeda19b..84b23842 100644 --- a/chipcompiler/engine/signoff_export.py +++ b/chipcompiler/engine/signoff_export.py @@ -27,15 +27,18 @@ def export_signoff_package_archive( destination = raw_destination.parent.resolve() / raw_destination.name with tempfile.TemporaryDirectory(prefix="ecc-signoff-") as temporary_root: - result = EngineFlow(workspace).collect_signoff_package( - SignoffPackageOptions( - output_dir=temporary_root, - archive=False, - include_debug=include_debug, - materialize=False, - refresh_analysis=False, + try: + result = EngineFlow(workspace).collect_signoff_package( + SignoffPackageOptions( + output_dir=temporary_root, + archive=False, + include_debug=include_debug, + materialize=False, + refresh_analysis=False, + ) ) - ) + except (OSError, ValueError, TypeError, KeyError) as exc: + raise SignoffExportError(str(exc)) from exc if not result.ok: missing = ", ".join(result.missing_required) or "unknown required resources" raise SignoffExportError(f"signoff package is incomplete: {missing}") diff --git a/chipcompiler/project/api.py b/chipcompiler/project/api.py index 4662f38e..d2ddf918 100644 --- a/chipcompiler/project/api.py +++ b/chipcompiler/project/api.py @@ -226,14 +226,15 @@ def _select_workspace(document: dict, mutation: dict) -> None: raise ManifestError(f"Workspace is not active: {workspace_id}") field = "qor_baseline" if mutation["type"] == "select_qor_baseline" else "best_workspace" document[field] = {"workspace_id": workspace_id, "reason": str(mutation.get("reason") or "")} - document["updated_at"] = str(mutation.get("updated_at") or "") + document["updated_at"] = str(mutation.get("updated_at") or datetime.now(UTC).isoformat()) def _archive_workspace(document: dict, mutation: dict) -> None: workspace = _find_workspace(document, _required_string(mutation, "workspace_id")) workspace["status"] = "archived" - workspace["updated_at"] = str(mutation.get("updated_at") or "") - document["updated_at"] = workspace["updated_at"] + timestamp = str(mutation.get("updated_at") or datetime.now(UTC).isoformat()) + workspace["updated_at"] = timestamp + document["updated_at"] = timestamp def _delete_workspace(document: dict, mutation: dict) -> None: @@ -251,7 +252,7 @@ def _delete_workspace(document: dict, mutation: dict) -> None: selection = document.get(field) if isinstance(selection, dict) and selection.get("workspace_id") == workspace_id: document[field] = None - document["updated_at"] = str(mutation.get("updated_at") or "") + document["updated_at"] = str(mutation.get("updated_at") or datetime.now(UTC).isoformat()) def _find_workspace(document: dict, workspace_id: str) -> dict: diff --git a/test/engine/test_qor_scoring.py b/test/engine/test_qor_scoring.py index d61317db..f186b2f7 100644 --- a/test/engine/test_qor_scoring.py +++ b/test/engine/test_qor_scoring.py @@ -32,3 +32,14 @@ def test_qor_scoring_selects_latest_area_and_combines_dimension_weights(): "area_cost": (50.0, 1), } assert result.overall_score == 42.5 + + +def test_qor_scoring_ignores_forward_version_dimensions_and_steps(): + result = score_qor( + [ + _metric("future-step", "future_metric", 1, "future_dimension"), + _metric("Harden", "die_area", 1500, "area_cost"), + ] + ) + + assert result.dimensions == {"area_cost": (50.0, 1)} diff --git a/test/engine/test_signoff_assessment.py b/test/engine/test_signoff_assessment.py index 821144b8..e3ea8fb2 100644 --- a/test/engine/test_signoff_assessment.py +++ b/test/engine/test_signoff_assessment.py @@ -29,6 +29,17 @@ def test_stale_checklist_cannot_make_incomplete_flow_ready(tmp_path): assert result["status"] == "blocked" +def test_malformed_checklist_is_reported_as_unavailable(tmp_path): + home = tmp_path / "home" + home.mkdir() + (home / "checklist.json").write_text("[]") + workspace = SimpleNamespace(directory=Path(tmp_path), flow=None) + + result = build_signoff_assessment(workspace) + + assert result["status"] == "blocked" + + def test_signoff_additional_file_path_rejects_escape(tmp_path): package = tmp_path / "package" package.mkdir() diff --git a/test/project/test_manifest.py b/test/project/test_manifest.py index c93ed17e..e9f04b01 100644 --- a/test/project/test_manifest.py +++ b/test/project/test_manifest.py @@ -51,3 +51,27 @@ def test_register_workspace_uses_main_manifest_shape(tmp_path): assert updated == load_project_manifest(tmp_path) assert updated["workspaces"][0]["start_step"] == "Synth" assert updated["workspaces"][0]["end_step"] == "Synth" + + +def test_manifest_mutation_without_timestamp_keeps_audit_timestamp(tmp_path): + workspace = tmp_path / "experiment" + (workspace / "home").mkdir(parents=True) + (workspace / "home" / "flow.json").write_text(json.dumps({"steps": []})) + create_project_manifest(tmp_path, "Demo", "gcd", now="2026-01-01T00:00:00Z") + mutate_project_manifest( + tmp_path, + { + "type": "register_workspace", + "workspace_id": "experiment", + "workspace_path": str(workspace), + "name": "Experiment", + }, + ) + + updated = mutate_project_manifest( + tmp_path, + {"type": "archive_workspace", "workspace_id": "experiment"}, + ) + + assert updated["updated_at"] + assert updated["workspaces"][0]["updated_at"] From 77c75f365a48848b8e81aae5e6b0cb3755afc128 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 04:26:01 +0800 Subject: [PATCH 15/43] refactor(project): move manifest ownership to headless domain Keep the CLI import paths as module aliases while making Project APIs independent of CLI internals, add read-only workspace loading, and materialize config-target parameters. --- chipcompiler/cli/project/manifest.py | 428 +----------------- chipcompiler/cli/project/manifest_write.py | 397 +--------------- chipcompiler/data/parameter_schema.py | 4 +- chipcompiler/data/workspace/__init__.py | 38 +- .../engine/workspace_configuration.py | 2 +- chipcompiler/engine/workspace_lifecycle.py | 32 +- chipcompiler/project/api.py | 4 +- chipcompiler/project/locking.py | 14 + chipcompiler/project/manifest.py | 425 +++++++++++++++++ chipcompiler/project/manifest_write.py | 396 ++++++++++++++++ test/engine/test_workspace_configuration.py | 20 + test/engine/test_workspace_spec.py | 43 ++ 12 files changed, 963 insertions(+), 840 deletions(-) create mode 100644 chipcompiler/project/locking.py create mode 100644 chipcompiler/project/manifest.py create mode 100644 chipcompiler/project/manifest_write.py diff --git a/chipcompiler/cli/project/manifest.py b/chipcompiler/cli/project/manifest.py index a61239ec..57aac98b 100644 --- a/chipcompiler/cli/project/manifest.py +++ b/chipcompiler/cli/project/manifest.py @@ -1,427 +1,7 @@ -#!/usr/bin/env python +"""CLI compatibility alias for the Project Manifest domain module.""" -"""``project.json`` manifest support for the CLI. +import sys -The manifest is the GUI's project descriptor (schema v1). The CLI reads it -for configuration layering and run discovery, and projects it into -configuration payloads. Write operations (generation, status write-back, -registration) live in chipcompiler.cli.project.manifest_write, which -routes every write through one read-modify-write helper. +from chipcompiler.project import manifest as _domain_manifest -This module sits on the CLI startup path (imported by -cli/core/invocation.py): keep module-level imports cheap — no -chipcompiler.data imports here. -""" - -import json -import os -import re -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -MANIFEST_FILENAME = "project.json" - -# GUI display names for the canonical rtl2gds chain. -MANIFEST_FLOW_STEPS = ( - "Synth", - "LEC", - "PreFloorplan", - "MacroPlacement", - "PostFloorplan", - "Place", - "CTS", - "Legal", - "TimingOpt", - "Route", - "Filler", - "RCX", - "STA", - "LVS", - "PostRouteLEC", - "DRC", - "Harden", -) - -PRESET_MANIFEST_RANGE = { - "syn_sta": ("Synth", "Synth"), - "rtl2gds": ("Synth", "Harden"), - "synthesis_lec": ("Synth", "LEC"), - # Legacy presets removed from the builder; keep resolving them so - # persisted projects still load. - "rcx": ("Synth", "STA"), - "harden": ("Synth", "Harden"), -} - -_CANONICAL_TO_MANIFEST_STEP = { - "Synthesis": "Synth", - "lec": "LEC", - "preFloorplan": "PreFloorplan", - "macroPlacement": "MacroPlacement", - "postFloorplan": "PostFloorplan", - "place": "Place", - "CTS": "CTS", - "legalization": "Legal", - "Timing optimization": "TimingOpt", - "route": "Route", - "filler": "Filler", - "RCX": "RCX", - "sta": "STA", - "lvs": "LVS", - "postRouteLec": "PostRouteLEC", - "drc": "DRC", - "Harden": "Harden", -} - -_WORKSPACE_STATUSES = frozenset( - {"success", "failed", "running", "in_progress", "not_started", "archived"} -) - - -class ManifestError(ValueError): - """Raised when a project.json manifest cannot be used (manifest_invalid).""" - - -@dataclass(frozen=True) -class ManifestWorkspace: - workspace_id: str - workspace_path: str - start_step: str - end_step: str - status: str - parameter_patch: dict = field(default_factory=dict) - raw: dict = field(default_factory=dict) - - -@dataclass(frozen=True) -class ProjectManifest: - project_dir: str - path: str - project_id: str - name: str - design_name: str - base_design: dict - objectives: dict - workspaces: tuple[ManifestWorkspace, ...] - qor_baseline: dict | None - raw: dict - - def active_workspaces(self) -> list[ManifestWorkspace]: - return [w for w in self.workspaces if w.status != "archived"] - - def find_workspace(self, workspace_id: str) -> ManifestWorkspace | None: - """Match a managed workspace by its declared identifier only.""" - for workspace in self.workspaces: - if workspace.workspace_id == workspace_id: - return workspace - return None - - -def _optional_str(value: Any) -> str: - return value.strip() if isinstance(value, str) else "" - - -def _record(value: Any) -> dict: - return value if isinstance(value, dict) else {} - - -def _normalize_workspace_entry(value: Any, index: int, project_dir: str) -> ManifestWorkspace: - source = _record(value) - workspace_id = _optional_str(source.get("workspace_id")) - workspace_path = _optional_str(source.get("workspace_path")) - if not workspace_id or not workspace_path: - raise ManifestError(f"workspaces[{index}] requires workspace_id and workspace_path") - resolved = Path(workspace_path) - if not resolved.is_absolute(): - resolved = Path(project_dir) / resolved - try: - canonical = resolved.resolve() - canonical.relative_to(Path(project_dir).resolve()) - except ValueError: - raise ManifestError( - f"workspaces[{index}] workspace_path escapes the project root: {workspace_path}" - ) from None - except RuntimeError as exc: - # e.g. a symlink loop inside the spelled path: invalid manifest - # input, never a traceback. - raise ManifestError( - f"workspaces[{index}] workspace_path cannot be resolved: {workspace_path}" - ) from exc - status = source.get("status") - if not isinstance(status, str) or status not in _WORKSPACE_STATUSES: - status = "not_started" - start_step = _optional_str(source.get("start_step")) or "Synth" - end_step = _optional_str(source.get("end_step")) or "Harden" - for step_name, field_name in ((start_step, "start_step"), (end_step, "end_step")): - if step_name not in MANIFEST_FLOW_STEPS: - raise ManifestError( - f"workspaces[{index}] {field_name} is not on the canonical flow chain: {step_name}" - ) - if MANIFEST_FLOW_STEPS.index(start_step) > MANIFEST_FLOW_STEPS.index(end_step): - raise ManifestError( - f"workspaces[{index}] flow range is reversed: {start_step} -> {end_step}" - ) - return ManifestWorkspace( - workspace_id=workspace_id, - workspace_path=str(canonical), - start_step=start_step, - end_step=end_step, - status=status, - parameter_patch=_record(source.get("parameter_patch")), - raw=dict(source), - ) - - -def _validate_mpc(value: Any) -> None: - """Mirror the GUI parser's mpc rules: null or a well-formed MPC record.""" - if value is None: - return - source = _record(value) - if not source: - raise ManifestError("invalid project manifest: mpc must be an object or null") - resource_id = _optional_str(source.get("resource_id")) - if not resource_id.startswith("mpc:") or len(resource_id) == 4: - raise ManifestError("invalid project manifest: mpc.resource_id must be an MPC id") - for field_name in ("display_name", "installed_version", "path", "spec_path"): - if not _optional_str(source.get(field_name)): - raise ManifestError(f"invalid project manifest: mpc.{field_name} is required") - mpc_path = source["path"].rstrip("/") - if source["spec_path"] != f"{mpc_path}/spec/spec.json.in": - raise ManifestError( - "invalid project manifest: mpc.spec_path must reference spec/spec.json.in" - ) - design = _record(source.get("design")) - index = design.get("index") - if ( - not design - or isinstance(index, bool) # a JSON boolean is not an index (True == 1 in Python) - or not isinstance(index, (int, float)) - # The GUI accepts integral numbers (0.0). is_integer() applies to - # floats only: float(huge_int) raises OverflowError, and ints are - # integral by construction. - or (isinstance(index, float) and not index.is_integer()) - or index < 0 - or not _optional_str(design.get("design_name")) - ): - raise ManifestError( - "invalid project manifest: mpc.design requires a non-negative index and design_name" - ) - if not isinstance(source.get("core_template"), dict): - raise ManifestError("invalid project manifest: mpc.core_template must be an object") - - -def load_manifest(project_dir: str) -> ProjectManifest: - """Load and tolerantly normalize ``/project.json``. - - Mirrors the GUI parser's contract: schema_version 1 and a workspaces - array are required, everything else is default-filled. Raises - ManifestError on parse failure, root_path mismatch, or a workspace - path outside the project root. - """ - path = os.path.join(project_dir, MANIFEST_FILENAME) - try: - with open(path, encoding="utf-8") as f: - source = json.load(f) - except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc: - raise ManifestError(f"invalid project manifest: {path}: {exc}") from exc - - if not isinstance(source, dict): - raise ManifestError(f"invalid project manifest: {path}: top level must be an object") - schema_version = source.get("schema_version") - # A JSON boolean is not the schema version (True == 1 in Python); the - # GUI parser rejects it, and the CLI must not open what the GUI cannot. - if isinstance(schema_version, bool) or schema_version != 1: - raise ManifestError("invalid project manifest: schema_version 1 is required") - raw_workspaces = source.get("workspaces") - if not isinstance(raw_workspaces, list): - raise ManifestError("invalid project manifest: workspaces must be an array") - - root_path = _optional_str(source.get("root_path")) - if not root_path: - raise ManifestError("invalid project manifest: root_path is required") - if os.path.realpath(root_path) != os.path.realpath(project_dir): - raise ManifestError( - f"invalid project manifest: root_path {root_path} does not match {project_dir}" - ) - design_name = _optional_str(source.get("design_name")) - if not design_name: - raise ManifestError("invalid project manifest: design_name is required") - - name = _optional_str(source.get("name")) or os.path.basename(project_dir) or "project" - base_design = _record(source.get("base_design")) - base_design = {**base_design, "parameters": _record(base_design.get("parameters"))} - # Mirror the GUI parser: primary defaults to "timing", directions keep - # only maximize/minimize entries from the source (no default fill). - objectives_raw = _record(source.get("objectives")) - objectives = dict(objectives_raw) - objectives["primary"] = _optional_str(objectives_raw.get("primary")) or "timing" - objectives["directions"] = { - key: value - for key, value in _record(objectives_raw.get("directions")).items() - if value in ("maximize", "minimize") - } - qor_baseline_raw = _record(source.get("qor_baseline")) - qor_baseline = None - if _optional_str(qor_baseline_raw.get("workspace_id")): - qor_baseline = { - "workspace_id": qor_baseline_raw["workspace_id"], - "reason": _optional_str(qor_baseline_raw.get("reason")) or "Project QoR baseline", - } - - _validate_mpc(source.get("mpc")) - - return ProjectManifest( - project_dir=project_dir, - path=path, - project_id=_optional_str(source.get("project_id")) or f"proj_{_slugify(name)}", - name=name, - design_name=design_name, - base_design=base_design, - objectives=objectives, - workspaces=tuple( - _normalize_workspace_entry(entry, index, project_dir) - for index, entry in enumerate(raw_workspaces) - ), - qor_baseline=qor_baseline, - raw=source, - ) - - -def find_manifest(project_dir: str) -> str | None: - """The manifest path when the entry lexically exists. - - Lexical presence, not readability: a directory, symlink loop, or - dangling symlink still counts as PRESENT so ``load_manifest`` fails - loud (manifest_invalid) instead of silently demoting the project to - the virgin or legacy layout. - """ - path = os.path.join(project_dir, MANIFEST_FILENAME) - return path if os.path.lexists(path) else None - - -def has_legacy_runs_layout(project_dir: str) -> bool: - runs_dir = os.path.join(project_dir, "runs") - if not os.path.isdir(runs_dir): - return False - try: - return any(os.path.isdir(os.path.join(runs_dir, entry)) for entry in os.listdir(runs_dir)) - except OSError: - return False - - -def classify_project(project_dir: str) -> str: - """Classify a project directory: manifest | legacy | virgin.""" - if find_manifest(project_dir) is not None: - return "manifest" - if has_legacy_runs_layout(project_dir): - return "legacy" - return "virgin" - - -def assemble_config(manifest: ProjectManifest, workspace: ManifestWorkspace | None) -> dict: - """Flatten the manifest into a parameter payload (lowest precedence layer). - - ``base_design.parameters`` plus the workspace's ``parameter_patch`` form - the base layer beneath project ecc.toml and --set overrides. - """ - parameters = dict(manifest.base_design.get("parameters") or {}) - if workspace is not None: - for key, change in workspace.parameter_patch.items(): - parameters[key] = ( - change["to"] if isinstance(change, dict) and "to" in change else change - ) - if manifest.design_name and not _optional_str(parameters.get("design")): - parameters["design"] = manifest.design_name - rtl_list = manifest.base_design.get("rtl_list") - if not isinstance(rtl_list, list): - rtl_list = [] - return { - "pdk": _optional_str(manifest.base_design.get("pdk")), - "pdk_root": _optional_str(manifest.base_design.get("pdk_root")), - "design_name": manifest.design_name, - "top_module": _optional_str(manifest.base_design.get("top_module")), - "clock": _optional_str(manifest.base_design.get("clock")), - "rtl_list": [item for item in rtl_list if isinstance(item, str)], - "origin_verilog": _optional_str(manifest.base_design.get("origin_verilog")), - "origin_def": _optional_str(manifest.base_design.get("origin_def")), - "netlist": _optional_str(manifest.base_design.get("netlist")), - "golden_netlist": _optional_str(manifest.base_design.get("golden_netlist")), - "sdc": _optional_str(manifest.base_design.get("sdc")), - "spef": _optional_str(manifest.base_design.get("spef")), - "parameters": parameters, - } - - -def resolved_base_parameters(cfg) -> dict: - """The ecc.toml-resolved base_design.parameters for a generated manifest. - - GUI-flat vocabulary: identity fields plus the [params] overrides, - projected through the geometry converter so positional values surface - as the wizard's aliases (die_width, utilitization, margin, ...). - --set values are run-scoped and never included. - """ - canonical: dict = { - "design": cfg.design_name, - "top_module": cfg.design_top, - "clock": cfg.design_clock_port, - "frequency_max": cfg.design_frequency_mhz, - } - if cfg.params_overrides: - from chipcompiler.cli.project.params import ( - build_backend_overrides, - resolve_parameters, - ) - - resolved, _ = resolve_parameters(toml_overrides=cfg.params_overrides) - canonical.update(build_backend_overrides(resolved)) - - from chipcompiler.data.parameter_keys import parameters_to_geometry - - flat = parameters_to_geometry(canonical) - # Exclusive GUI-flat shape: geometry lives only in the aliases — - # the canonical die/core subtrees are consumed, not duplicated. - # Non-positional members (e.g. aspect_ratio) hoist to flat top-level - # keys; positional members are covered by the aliases. - for subtree_name in ("die", "core"): - subtree = flat.pop(subtree_name, None) - if not isinstance(subtree, dict): - continue - for member, value in subtree.items(): - if member in ("size", "utilitization", "margin"): - continue - flat.setdefault(member, value) - return flat - - -def base_design_from_config(cfg, pdk_root: str) -> dict: - """The base_design document for a generated manifest. - - Identity and sources come from the ecc.toml-resolved config with the - DECLARED project source spellings preserved: ``rtl_list`` verbatim, - and ``origin_verilog`` when the single source is plain RTL (empty for - a filelist source; the document builder drops empty keys). Parameters - are the GUI-flat projection. Shared by virgin generation and first - migration so the two writers cannot drift. - """ - from chipcompiler.cli.project.config import resolve_rtl - - _, origin_verilog, _ = resolve_rtl(cfg) - return { - "pdk": cfg.pdk_name, - "pdk_root": pdk_root, - "top_module": cfg.design_top, - "clock": cfg.design_clock_port, - "rtl_list": cfg.design_rtl, - "origin_verilog": cfg.design_rtl[0] if origin_verilog else "", - "origin_def": cfg.design_def, - "netlist": cfg.design_netlist, - "golden_netlist": cfg.design_golden_netlist, - "sdc": cfg.design_sdc, - "spef": cfg.design_spef, - "parameters": resolved_base_parameters(cfg), - } - - -def _slugify(value: str) -> str: - slug = re.sub(r"[^a-z0-9]+", "_", value.strip().lower()).strip("_") - return slug or "project" +sys.modules[__name__] = _domain_manifest diff --git a/chipcompiler/cli/project/manifest_write.py b/chipcompiler/cli/project/manifest_write.py index bc90e154..61eb0ca3 100644 --- a/chipcompiler/cli/project/manifest_write.py +++ b/chipcompiler/cli/project/manifest_write.py @@ -1,396 +1,7 @@ -#!/usr/bin/env python +"""CLI compatibility alias for Project Manifest mutations.""" -"""Manifest write, mutation, and registration operations for the CLI. +import sys -All ``project.json`` writes go through one read-modify-write helper so -status write-back and migration entry-append share the same atomicity -story. Loading and normalization live in -chipcompiler.cli.project.manifest; this module imports from it, never -the reverse. +from chipcompiler.project import manifest_write as _domain_manifest_write -Like manifest.py, this module sits on the CLI startup path (imported by -run dispatch and migration flows): keep module-level imports cheap — no -chipcompiler.data imports here. -""" - -import json -import logging -import os -import tempfile -from copy import deepcopy -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -from chipcompiler.cli.project.manifest import ( - _CANONICAL_TO_MANIFEST_STEP, - MANIFEST_FILENAME, - PRESET_MANIFEST_RANGE, - ManifestError, - _record, - _slugify, - base_design_from_config, -) - -logger = logging.getLogger(__name__) - -DEFAULT_OBJECTIVES = { - "primary": "timing", - "directions": { - "wns": "maximize", - "tns": "maximize", - "area": "minimize", - "drc_count": "minimize", - "lvs_count": "minimize", - "power": "minimize", - }, -} - - -def _now_iso() -> str: - return datetime.now(UTC).isoformat() - - -def manifest_workspace_entry( - workspace_id: str, - *, - name: str, - workspace_path: str, - start_step: str, - end_step: str, - status: str, - now: str, -) -> dict: - """One complete schema-v1 workspaces[] entry, every field materialized. - - The single builder for generated manifests and migration previews, so - the previewed entry and the applied entry are the same object shape. - """ - return { - "workspace_id": workspace_id, - "name": name, - "workspace_path": workspace_path, - "source_workspace_id": None, - "branch_from": None, - "start_step": start_step, - "end_step": end_step, - "status": status, - "created_at": now, - "updated_at": now, - "parameter_patch": {}, - "metrics_summary": {}, - "step_metrics": {}, - } - - -def build_manifest_document( - project_dir: str, - *, - design_name: str, - base_design: dict, - workspace_id: str, - workspace_path: str, - start_step: str, - end_step: str, - status: str = "running", -) -> dict: - """Assemble a schema-v1 manifest for a virgin project's first run.""" - now = _now_iso() - document = build_project_document( - project_dir, - design_name=design_name, - base_design=base_design, - now=now, - ) - document["workspaces"] = [ - manifest_workspace_entry( - workspace_id, - name=design_name, - workspace_path=workspace_path, - start_step=start_step, - end_step=end_step, - status=status, - now=now, - ) - ] - document["qor_baseline"] = { - "workspace_id": workspace_id, - "reason": "Default project QoR baseline", - } - return document - - -def build_project_document( - project_dir: str, - *, - design_name: str, - base_design: dict, - name: str | None = None, - now: str | None = None, - mpc: dict | None = None, -) -> dict[str, Any]: - """Assemble a schema-v1 Project Manifest without a Workspace.""" - timestamp = now or _now_iso() - project_name = name or os.path.basename(os.path.normpath(project_dir)) or "project" - return { - "schema_version": 1, - "project_id": f"proj_{_slugify(project_name)}", - "name": project_name, - "design_name": design_name, - "description": "", - "root_path": project_dir, - "created_at": timestamp, - "updated_at": timestamp, - "base_design": { - **{key: value for key, value in base_design.items() if key != "parameters" and value}, - "parameters": _record(base_design.get("parameters")), - "rtl_list": [ - item for item in base_design.get("rtl_list") or [] if isinstance(item, str) - ], - }, - "objectives": json.loads(json.dumps(DEFAULT_OBJECTIVES)), - "workspaces": [], - "mpc": deepcopy(mpc), - "best_workspace": None, - "qor_baseline": None, - } - - -def write_manifest_if_absent(project_dir: str, document: dict) -> bool: - """Write the manifest only when it does not exist (virgin generation race). - - Fully written and fsynced at a temp path, then linked into place: - readers never see a partial file, and a concurrent creator wins the - link — ours is discarded and the caller continues read-only. - """ - path = os.path.join(project_dir, MANIFEST_FILENAME) - content = json.dumps(document, indent=2) + "\n" - tmp_path = None - try: - with tempfile.NamedTemporaryFile( - "w", - dir=project_dir, - delete=False, - prefix=f".{MANIFEST_FILENAME}.", - suffix=".tmp", - encoding="utf-8", - ) as f: - tmp_path = f.name - f.write(content) - f.flush() - os.fsync(f.fileno()) - # Mode stays the tempfile default (0600), matching json_write's - # convention for newly created state files. - os.link(tmp_path, path) - return True - except FileExistsError: - return False - except OSError as exc: - logger.warning("manifest write failed: %s: %s", path, exc) - return False - finally: - if tmp_path is not None: - Path(tmp_path).unlink(missing_ok=True) - - -def _read_manifest_document(path: str): - try: - with open(path, encoding="utf-8") as f: - document = json.load(f) - except (OSError, json.JSONDecodeError, UnicodeDecodeError): - return None - return document if isinstance(document, dict) else None - - -def update_manifest(project_dir: str, mutator) -> bool: - """Read-modify-write the manifest atomically (locked re-read + patch + replace). - - The whole read-modify-replace runs under ``.manifest.lock`` (flock): - two cooperating writers can no longer both complete the fresh read - before either replaces, so neither loses the other's update. The - mutator receives the parsed document and edits it in place. When an - unrelated change lands between the read and the write, the mutator is - re-applied to the freshest document instead of overwriting the change. - Project-level fields (including updated_at) are owned by the mutator. - Returns False (with a warning) when the manifest is missing, unreadable, - the lock cannot be taken, or the write fails — callers degrade to a - warning, never a run failure. - """ - from chipcompiler.cli.project.migrate_fs import flock_file - - path = os.path.join(project_dir, MANIFEST_FILENAME) - try: - with flock_file(os.path.join(project_dir, ".manifest.lock"), exclusive=True): - return _update_manifest_locked(path, mutator) - except OSError as exc: - # An untakeable lock (e.g. a directory at the lock path) degrades - # like any write failure: a warning, never an uncaught exception — - # the migration registration path relies on False to roll back. - logger.warning("manifest update failed: %s: %s", path, exc) - return False - - -def _update_manifest_locked(path: str, mutator) -> bool: - base = _read_manifest_document(path) - if base is None: - logger.warning("manifest update skipped (unreadable): %s", path) - return False - - document = deepcopy(base) - mutator(document) - - fresh = _read_manifest_document(path) - if fresh is not None and fresh != base: - # An unrelated edit landed after our read: re-apply the mutator to - # the freshest document so the interleaved change survives. - document = fresh - mutator(document) - - target = Path(path) - tmp_path = None - try: - with tempfile.NamedTemporaryFile( - "w", - dir=target.parent, - delete=False, - prefix=f".{target.name}.", - suffix=".tmp", - encoding="utf-8", - ) as f: - tmp_path = Path(f.name) - json.dump(document, f, indent=2) - f.write("\n") - f.flush() - os.fsync(f.fileno()) - # Preserve the existing manifest's permissions: mkstemp's 0600 must - # not silently narrow a shared project.json. - if target.exists(): - os.chmod(tmp_path, target.stat().st_mode & 0o7777) - os.replace(tmp_path, target) - return True - except OSError as exc: - logger.warning("manifest update failed: %s: %s", path, exc) - if tmp_path is not None: - tmp_path.unlink(missing_ok=True) - return False - - -def write_back_workspace_status(project_dir: str, workspace_id: str, status: str) -> bool: - """Update one workspace entry's status (and updated_at) after a run.""" - - def mutate(document: dict) -> None: - for entry in document.get("workspaces", []): - if isinstance(entry, dict) and entry.get("workspace_id") == workspace_id: - entry["status"] = status - entry["updated_at"] = _now_iso() - - return update_manifest(project_dir, mutate) - - -def remove_workspace_registration(project_dir: str, workspace_id: str) -> bool: - """Roll back a pre-registration: drop the freshly added entry. - - Used when an overwrite run against an undeclared workspace fails before - the replacement is constructed: the restored previous workspace must not - be shadowed by a stale ``not_started`` entry this invocation created. - """ - - def mutate(document: dict) -> None: - workspaces = document.get("workspaces") - if isinstance(workspaces, list): - document["workspaces"] = [ - entry - for entry in workspaces - if not (isinstance(entry, dict) and entry.get("workspace_id") == workspace_id) - ] - document["updated_at"] = _now_iso() - - return update_manifest(project_dir, mutate) - - -def manifest_range_for_flow(cfg, flow_config: dict | None) -> tuple[str, str]: - """Return the GUI manifest range for a workspace's effective target.""" - if isinstance(flow_config, dict) and flow_config.get("start_step"): - from chipcompiler.rtl2gds import normalize_flow_step - - start = normalize_flow_step(flow_config["start_step"]) - end = normalize_flow_step(flow_config.get("end_step") or start) - try: - return (_CANONICAL_TO_MANIFEST_STEP[start], _CANONICAL_TO_MANIFEST_STEP[end]) - except KeyError as exc: - raise ManifestError(f"unknown workspace flow step: {exc.args[0]}") from None - return PRESET_MANIFEST_RANGE.get(cfg.flow_preset, ("Synth", "Harden")) - - -def pre_register_workspace( - project_dir: str, - *, - cfg, - pdk_root: str, - workspace_id: str, - workspace_path: str, - flow_config: dict | None, -) -> str: - """Atomically register a fresh managed workspace before filesystem creation. - - Returns ``registered``, ``existing``, ``conflict``, or ``failed``. A - workspace entry intentionally contains no input snapshot: copied files and - the workspace config are the reproducibility boundary. - """ - try: - start_step, end_step = manifest_range_for_flow(cfg, flow_config) - except ManifestError: - return "failed" - now = _now_iso() - manifest_path = os.path.join(project_dir, MANIFEST_FILENAME) - if not os.path.lexists(manifest_path): - document = build_manifest_document( - project_dir, - design_name=cfg.design_name, - base_design=base_design_from_config(cfg, pdk_root), - workspace_id=workspace_id, - workspace_path=workspace_path, - start_step=start_step, - end_step=end_step, - status="not_started", - ) - if write_manifest_if_absent(project_dir, document): - return "registered" - # A concurrent creator won the link race: fall through and apply the - # same registration mutation under the manifest lock instead of - # aborting this run. A genuine I/O failure fails again below. - - outcome = "registered" - - def mutate(document: dict) -> None: - nonlocal outcome - workspaces = document.get("workspaces") - if not isinstance(workspaces, list): - outcome = "failed" - return - for entry in workspaces: - if not isinstance(entry, dict) or entry.get("workspace_id") != workspace_id: - continue - if os.path.realpath(str(entry.get("workspace_path", ""))) == os.path.realpath( - workspace_path - ): - outcome = "existing" - else: - outcome = "conflict" - return - workspaces.append( - manifest_workspace_entry( - workspace_id, - name=cfg.design_name, - workspace_path=workspace_path, - start_step=start_step, - end_step=end_step, - status="not_started", - now=now, - ) - ) - document["updated_at"] = now - - if not update_manifest(project_dir, mutate): - return "failed" - return outcome +sys.modules[__name__] = _domain_manifest_write diff --git a/chipcompiler/data/parameter_schema.py b/chipcompiler/data/parameter_schema.py index 45bd1a53..3dc10876 100644 --- a/chipcompiler/data/parameter_schema.py +++ b/chipcompiler/data/parameter_schema.py @@ -396,10 +396,10 @@ def coerce_manifest_parameters(canonical: dict, registry=PARAM_REGISTRY, skip_pa return coerced, errors -def build_backend_overrides(resolved): +def build_backend_overrides(resolved, *, include_defaults: bool = False): overrides = {} for item in resolved: - if item.source == "default" and item.value == item.default: + if not include_defaults and item.source == "default" and item.value == item.default: continue target, value = item.schema.maps_to, item.value if isinstance(target, str): diff --git a/chipcompiler/data/workspace/__init__.py b/chipcompiler/data/workspace/__init__.py index 247adc4a..c6f0ff1d 100644 --- a/chipcompiler/data/workspace/__init__.py +++ b/chipcompiler/data/workspace/__init__.py @@ -1184,19 +1184,21 @@ def _persisted_golden_verilog(workspace_dir: Path) -> tuple[Path | None, bool]: return None, True -def load_workspace(directory: str | Path) -> Workspace: +def load_workspace(directory: str | Path, *, read_only: bool = False) -> Workspace: workspace_dir = Path(directory).expanduser().resolve() origin_dir = workspace_dir / "origin" home_dir = workspace_dir / "home" if not workspace_dir.exists(): return None - migrate_legacy_parameters(workspace_dir) + if not read_only: + migrate_legacy_parameters(workspace_dir) # create workspace instance workspace = Workspace() workspace.directory = workspace_dir - migrate_workspace_config_filenames(workspace_dir) + if not read_only: + migrate_workspace_config_filenames(workspace_dir) workspace.config = build_workspace_config_paths(workspace) config_path = workspace_config_toml_path(workspace_dir) @@ -1282,19 +1284,27 @@ def load_workspace(directory: str | Path) -> Workspace: workspace.design.input_filelist = filelist_path # set home data - home_dir.mkdir(parents=True, exist_ok=True) - workspace.config["dir"].mkdir(parents=True, exist_ok=True) workspace.flow.path = home_dir / "flow.json" - workspace.home.init(path=home_dir / "home.json") - workspace.home.set_flow(workspace.flow.path) - workspace.home.set_checklist(home_dir / "checklist.json") - workspace.home.set_parameters(workspace.parameters.path) - - # create logger first (needed for copy operations) - workspace.logger = create_logger(name=parameters.data["design"], log_dir=workspace_dir / "log") + if read_only: + workspace.home.path = home_dir / "home.json" + home_data = json_read(workspace.home.path) + workspace.home.data = home_data if isinstance(home_data, dict) else {} + workspace.logger = Logger(name=parameters.data["design"]) + else: + home_dir.mkdir(parents=True, exist_ok=True) + workspace.config["dir"].mkdir(parents=True, exist_ok=True) + workspace.home.init(path=home_dir / "home.json") + workspace.home.set_flow(workspace.flow.path) + workspace.home.set_checklist(home_dir / "checklist.json") + workspace.home.set_parameters(workspace.parameters.path) + + # create logger first (needed for copy operations) + workspace.logger = create_logger( + name=parameters.data["design"], log_dir=workspace_dir / "log" + ) - log_workspace(workspace) - log_parameters(workspace) + log_workspace(workspace) + log_parameters(workspace) return workspace diff --git a/chipcompiler/engine/workspace_configuration.py b/chipcompiler/engine/workspace_configuration.py index 10d3de6d..7964408b 100644 --- a/chipcompiler/engine/workspace_configuration.py +++ b/chipcompiler/engine/workspace_configuration.py @@ -389,7 +389,7 @@ def read_workspace_configuration_from_directory( target_directory: str | Path, ) -> dict[str, Any]: target = Path(target_directory).expanduser().resolve() - workspace = load_workspace(target) + workspace = load_workspace(target, read_only=True) if workspace is None: raise WorkspaceLifecycleError("workspace_missing", f"Workspace not found: {target}") return read_workspace_configuration(workspace) diff --git a/chipcompiler/engine/workspace_lifecycle.py b/chipcompiler/engine/workspace_lifecycle.py index e4db911c..b8c5e073 100644 --- a/chipcompiler/engine/workspace_lifecycle.py +++ b/chipcompiler/engine/workspace_lifecycle.py @@ -9,7 +9,12 @@ from typing import Any from chipcompiler.data import PDK, create_workspace, get_pdk, load_workspace -from chipcompiler.data.parameter_schema import build_backend_overrides, resolve_parameters +from chipcompiler.data.parameter_schema import ( + build_backend_overrides, + build_config_overrides, + resolve_parameters, +) +from chipcompiler.data.workspace.config_overrides import CONFIG_OVERRIDES_KEY from chipcompiler.engine.snapshot import create_engineering_snapshot from chipcompiler.engine.workspace_spec import validate_workspace_spec from chipcompiler.rtl2gds import get_flow_builders @@ -127,7 +132,10 @@ def _create_workspace_from_spec( "Workspace parameters are invalid", {"issues": errors}, ) - backend_parameters = build_backend_overrides(parameters) + backend_parameters = build_backend_overrides(parameters, include_defaults=True) + config_overrides = build_config_overrides(parameters) + if config_overrides: + backend_parameters[CONFIG_OVERRIDES_KEY] = config_overrides backend_parameters.update( { "design": resolved["design"]["name"], @@ -232,7 +240,11 @@ def _update_workspace_from_spec( bindings: object, command_id: str = "", ): - from chipcompiler.engine.snapshot import ensure_engineering_snapshot + from chipcompiler.engine.snapshot import ( + EngineeringSnapshotError, + ensure_engineering_snapshot, + read_engineering_snapshot, + ) target = Path(target_directory).expanduser().resolve() fingerprint = _workspace_command_fingerprint( @@ -241,7 +253,19 @@ def _update_workspace_from_spec( if command_id and _command_retry_matches(target, command_id, fingerprint): return _load_committed_workspace(target) current = _load_committed_workspace(target) - snapshot = ensure_engineering_snapshot(current) + try: + snapshot = read_engineering_snapshot(current) + except EngineeringSnapshotError as exc: + if expected_workspace_revision != 1: + raise WorkspaceLifecycleError( + "revision_conflict", + "Workspace Revision does not match", + { + "expectedWorkspaceRevision": expected_workspace_revision, + "actualWorkspaceRevision": 1, + }, + ) from exc + snapshot = ensure_engineering_snapshot(current) if snapshot["workspaceRevision"] != expected_workspace_revision: raise WorkspaceLifecycleError( "revision_conflict", diff --git a/chipcompiler/project/api.py b/chipcompiler/project/api.py index d2ddf918..d40abba2 100644 --- a/chipcompiler/project/api.py +++ b/chipcompiler/project/api.py @@ -4,12 +4,12 @@ from datetime import UTC, datetime from pathlib import Path -from chipcompiler.cli.project.manifest import ( +from chipcompiler.project.manifest import ( _CANONICAL_TO_MANIFEST_STEP, ManifestError, load_manifest, ) -from chipcompiler.cli.project.manifest_write import ( +from chipcompiler.project.manifest_write import ( build_project_document, manifest_workspace_entry, update_manifest, diff --git a/chipcompiler/project/locking.py b/chipcompiler/project/locking.py new file mode 100644 index 00000000..6a8f4082 --- /dev/null +++ b/chipcompiler/project/locking.py @@ -0,0 +1,14 @@ +"""Project-domain file lock helper.""" + +import fcntl +from contextlib import contextmanager + + +@contextmanager +def flock_file(path: str, *, exclusive: bool = True): + with open(path, "a") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) + try: + yield lock_file + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) diff --git a/chipcompiler/project/manifest.py b/chipcompiler/project/manifest.py new file mode 100644 index 00000000..99d51ba0 --- /dev/null +++ b/chipcompiler/project/manifest.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python + +"""``project.json`` manifest support for the CLI. + +The manifest is the GUI's project descriptor (schema v1). The CLI reads it +for configuration layering and run discovery, and projects it into +configuration payloads. Write operations (generation, status write-back, +registration) live in chipcompiler.project.manifest_write, which +routes every write through one read-modify-write helper. + +This module sits on the CLI startup path (imported by +cli/core/invocation.py): keep module-level imports cheap — no +chipcompiler.data imports here. +""" + +import json +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +MANIFEST_FILENAME = "project.json" + +# GUI display names for the canonical rtl2gds chain. +MANIFEST_FLOW_STEPS = ( + "Synth", + "LEC", + "Floor", + "Place", + "CTS", + "Legal", + "TimingOpt", + "Route", + "Filler", + "RCX", + "STA", + "LVS", + "PostRouteLEC", + "DRC", + "Harden", +) + +PRESET_MANIFEST_RANGE = { + "syn_sta": ("Synth", "Synth"), + "rtl2gds": ("Synth", "Harden"), + "synthesis_lec": ("Synth", "LEC"), + # Legacy presets removed from the builder; keep resolving them so + # persisted projects still load. + "rcx": ("Synth", "STA"), + "harden": ("Synth", "Harden"), +} + +_CANONICAL_TO_MANIFEST_STEP = { + "Synthesis": "Synth", + "lec": "LEC", + "Floorplan": "Floor", + "place": "Place", + "CTS": "CTS", + "legalization": "Legal", + "Timing optimization": "TimingOpt", + "route": "Route", + "filler": "Filler", + "RCX": "RCX", + "sta": "STA", + "lvs": "LVS", + "postRouteLec": "PostRouteLEC", + "drc": "DRC", + "Harden": "Harden", +} + +_WORKSPACE_STATUSES = frozenset( + {"success", "failed", "running", "in_progress", "not_started", "archived"} +) + + +class ManifestError(ValueError): + """Raised when a project.json manifest cannot be used (manifest_invalid).""" + + +@dataclass(frozen=True) +class ManifestWorkspace: + workspace_id: str + workspace_path: str + start_step: str + end_step: str + status: str + parameter_patch: dict = field(default_factory=dict) + raw: dict = field(default_factory=dict) + + +@dataclass(frozen=True) +class ProjectManifest: + project_dir: str + path: str + project_id: str + name: str + design_name: str + base_design: dict + objectives: dict + workspaces: tuple[ManifestWorkspace, ...] + qor_baseline: dict | None + raw: dict + + def active_workspaces(self) -> list[ManifestWorkspace]: + return [w for w in self.workspaces if w.status != "archived"] + + def find_workspace(self, workspace_id: str) -> ManifestWorkspace | None: + """Match a managed workspace by its declared identifier only.""" + for workspace in self.workspaces: + if workspace.workspace_id == workspace_id: + return workspace + return None + + +def _optional_str(value: Any) -> str: + return value.strip() if isinstance(value, str) else "" + + +def _record(value: Any) -> dict: + return value if isinstance(value, dict) else {} + + +def _normalize_workspace_entry(value: Any, index: int, project_dir: str) -> ManifestWorkspace: + source = _record(value) + workspace_id = _optional_str(source.get("workspace_id")) + workspace_path = _optional_str(source.get("workspace_path")) + if not workspace_id or not workspace_path: + raise ManifestError(f"workspaces[{index}] requires workspace_id and workspace_path") + resolved = Path(workspace_path) + if not resolved.is_absolute(): + resolved = Path(project_dir) / resolved + try: + canonical = resolved.resolve() + canonical.relative_to(Path(project_dir).resolve()) + except ValueError: + raise ManifestError( + f"workspaces[{index}] workspace_path escapes the project root: {workspace_path}" + ) from None + except RuntimeError as exc: + # e.g. a symlink loop inside the spelled path: invalid manifest + # input, never a traceback. + raise ManifestError( + f"workspaces[{index}] workspace_path cannot be resolved: {workspace_path}" + ) from exc + status = source.get("status") + if not isinstance(status, str) or status not in _WORKSPACE_STATUSES: + status = "not_started" + start_step = _optional_str(source.get("start_step")) or "Synth" + end_step = _optional_str(source.get("end_step")) or "Harden" + for step_name, field_name in ((start_step, "start_step"), (end_step, "end_step")): + if step_name not in MANIFEST_FLOW_STEPS: + raise ManifestError( + f"workspaces[{index}] {field_name} is not on the canonical flow chain: {step_name}" + ) + if MANIFEST_FLOW_STEPS.index(start_step) > MANIFEST_FLOW_STEPS.index(end_step): + raise ManifestError( + f"workspaces[{index}] flow range is reversed: {start_step} -> {end_step}" + ) + return ManifestWorkspace( + workspace_id=workspace_id, + workspace_path=str(canonical), + start_step=start_step, + end_step=end_step, + status=status, + parameter_patch=_record(source.get("parameter_patch")), + raw=dict(source), + ) + + +def _validate_mpc(value: Any) -> None: + """Mirror the GUI parser's mpc rules: null or a well-formed MPC record.""" + if value is None: + return + source = _record(value) + if not source: + raise ManifestError("invalid project manifest: mpc must be an object or null") + resource_id = _optional_str(source.get("resource_id")) + if not resource_id.startswith("mpc:") or len(resource_id) == 4: + raise ManifestError("invalid project manifest: mpc.resource_id must be an MPC id") + for field_name in ("display_name", "installed_version", "path", "spec_path"): + if not _optional_str(source.get(field_name)): + raise ManifestError(f"invalid project manifest: mpc.{field_name} is required") + mpc_path = source["path"].rstrip("/") + if source["spec_path"] != f"{mpc_path}/spec/spec.json.in": + raise ManifestError( + "invalid project manifest: mpc.spec_path must reference spec/spec.json.in" + ) + design = _record(source.get("design")) + index = design.get("index") + if ( + not design + or isinstance(index, bool) # a JSON boolean is not an index (True == 1 in Python) + or not isinstance(index, (int, float)) + # The GUI accepts integral numbers (0.0). is_integer() applies to + # floats only: float(huge_int) raises OverflowError, and ints are + # integral by construction. + or (isinstance(index, float) and not index.is_integer()) + or index < 0 + or not _optional_str(design.get("design_name")) + ): + raise ManifestError( + "invalid project manifest: mpc.design requires a non-negative index and design_name" + ) + if not isinstance(source.get("core_template"), dict): + raise ManifestError("invalid project manifest: mpc.core_template must be an object") + + +def load_manifest(project_dir: str) -> ProjectManifest: + """Load and tolerantly normalize ``/project.json``. + + Mirrors the GUI parser's contract: schema_version 1 and a workspaces + array are required, everything else is default-filled. Raises + ManifestError on parse failure, root_path mismatch, or a workspace + path outside the project root. + """ + path = os.path.join(project_dir, MANIFEST_FILENAME) + try: + with open(path, encoding="utf-8") as f: + source = json.load(f) + except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc: + raise ManifestError(f"invalid project manifest: {path}: {exc}") from exc + + if not isinstance(source, dict): + raise ManifestError(f"invalid project manifest: {path}: top level must be an object") + schema_version = source.get("schema_version") + # A JSON boolean is not the schema version (True == 1 in Python); the + # GUI parser rejects it, and the CLI must not open what the GUI cannot. + if isinstance(schema_version, bool) or schema_version != 1: + raise ManifestError("invalid project manifest: schema_version 1 is required") + raw_workspaces = source.get("workspaces") + if not isinstance(raw_workspaces, list): + raise ManifestError("invalid project manifest: workspaces must be an array") + + root_path = _optional_str(source.get("root_path")) + if not root_path: + raise ManifestError("invalid project manifest: root_path is required") + if os.path.realpath(root_path) != os.path.realpath(project_dir): + raise ManifestError( + f"invalid project manifest: root_path {root_path} does not match {project_dir}" + ) + design_name = _optional_str(source.get("design_name")) + if not design_name: + raise ManifestError("invalid project manifest: design_name is required") + + name = _optional_str(source.get("name")) or os.path.basename(project_dir) or "project" + base_design = _record(source.get("base_design")) + base_design = {**base_design, "parameters": _record(base_design.get("parameters"))} + # Mirror the GUI parser: primary defaults to "timing", directions keep + # only maximize/minimize entries from the source (no default fill). + objectives_raw = _record(source.get("objectives")) + objectives = dict(objectives_raw) + objectives["primary"] = _optional_str(objectives_raw.get("primary")) or "timing" + objectives["directions"] = { + key: value + for key, value in _record(objectives_raw.get("directions")).items() + if value in ("maximize", "minimize") + } + qor_baseline_raw = _record(source.get("qor_baseline")) + qor_baseline = None + if _optional_str(qor_baseline_raw.get("workspace_id")): + qor_baseline = { + "workspace_id": qor_baseline_raw["workspace_id"], + "reason": _optional_str(qor_baseline_raw.get("reason")) or "Project QoR baseline", + } + + _validate_mpc(source.get("mpc")) + + return ProjectManifest( + project_dir=project_dir, + path=path, + project_id=_optional_str(source.get("project_id")) or f"proj_{_slugify(name)}", + name=name, + design_name=design_name, + base_design=base_design, + objectives=objectives, + workspaces=tuple( + _normalize_workspace_entry(entry, index, project_dir) + for index, entry in enumerate(raw_workspaces) + ), + qor_baseline=qor_baseline, + raw=source, + ) + + +def find_manifest(project_dir: str) -> str | None: + """The manifest path when the entry lexically exists. + + Lexical presence, not readability: a directory, symlink loop, or + dangling symlink still counts as PRESENT so ``load_manifest`` fails + loud (manifest_invalid) instead of silently demoting the project to + the virgin or legacy layout. + """ + path = os.path.join(project_dir, MANIFEST_FILENAME) + return path if os.path.lexists(path) else None + + +def has_legacy_runs_layout(project_dir: str) -> bool: + runs_dir = os.path.join(project_dir, "runs") + if not os.path.isdir(runs_dir): + return False + try: + return any(os.path.isdir(os.path.join(runs_dir, entry)) for entry in os.listdir(runs_dir)) + except OSError: + return False + + +def classify_project(project_dir: str) -> str: + """Classify a project directory: manifest | legacy | virgin.""" + if find_manifest(project_dir) is not None: + return "manifest" + if has_legacy_runs_layout(project_dir): + return "legacy" + return "virgin" + + +def assemble_config(manifest: ProjectManifest, workspace: ManifestWorkspace | None) -> dict: + """Flatten the manifest into a parameter payload (lowest precedence layer). + + ``base_design.parameters`` plus the workspace's ``parameter_patch`` form + the base layer beneath project ecc.toml and --set overrides. + """ + parameters = dict(manifest.base_design.get("parameters") or {}) + if workspace is not None: + for key, change in workspace.parameter_patch.items(): + parameters[key] = ( + change["to"] if isinstance(change, dict) and "to" in change else change + ) + if manifest.design_name and not _optional_str(parameters.get("design")): + parameters["design"] = manifest.design_name + rtl_list = manifest.base_design.get("rtl_list") + if not isinstance(rtl_list, list): + rtl_list = [] + return { + "pdk": _optional_str(manifest.base_design.get("pdk")), + "pdk_root": _optional_str(manifest.base_design.get("pdk_root")), + "design_name": manifest.design_name, + "top_module": _optional_str(manifest.base_design.get("top_module")), + "clock": _optional_str(manifest.base_design.get("clock")), + "rtl_list": [item for item in rtl_list if isinstance(item, str)], + "origin_verilog": _optional_str(manifest.base_design.get("origin_verilog")), + "origin_def": _optional_str(manifest.base_design.get("origin_def")), + "netlist": _optional_str(manifest.base_design.get("netlist")), + "golden_netlist": _optional_str(manifest.base_design.get("golden_netlist")), + "sdc": _optional_str(manifest.base_design.get("sdc")), + "spef": _optional_str(manifest.base_design.get("spef")), + "parameters": parameters, + } + + +def resolved_base_parameters(cfg) -> dict: + """The ecc.toml-resolved base_design.parameters for a generated manifest. + + GUI-flat vocabulary: identity fields plus the [params] overrides, + projected through the geometry converter so positional values surface + as the wizard's aliases (die_width, utilitization, margin, ...). + --set values are run-scoped and never included. + """ + canonical: dict = { + "design": cfg.design_name, + "top_module": cfg.design_top, + "clock": cfg.design_clock_port, + "frequency_max": cfg.design_frequency_mhz, + } + if cfg.params_overrides: + from chipcompiler.data.parameter_schema import ( + build_backend_overrides, + resolve_parameters, + ) + + resolved, _ = resolve_parameters(toml_overrides=cfg.params_overrides) + canonical.update(build_backend_overrides(resolved)) + + from chipcompiler.data.parameter_keys import parameters_to_geometry + + flat = parameters_to_geometry(canonical) + # Exclusive GUI-flat shape: geometry lives only in the aliases — + # the canonical die/core subtrees are consumed, not duplicated. + # Non-positional members (e.g. aspect_ratio) hoist to flat top-level + # keys; positional members are covered by the aliases. + for subtree_name in ("die", "core"): + subtree = flat.pop(subtree_name, None) + if not isinstance(subtree, dict): + continue + for member, value in subtree.items(): + if member in ("size", "utilitization", "margin"): + continue + flat.setdefault(member, value) + return flat + + +def base_design_from_config(cfg, pdk_root: str) -> dict: + """The base_design document for a generated manifest. + + Identity and sources come from the ecc.toml-resolved config with the + DECLARED project source spellings preserved: ``rtl_list`` verbatim, + and ``origin_verilog`` when the single source is plain RTL (empty for + a filelist source; the document builder drops empty keys). Parameters + are the GUI-flat projection. Shared by virgin generation and first + migration so the two writers cannot drift. + """ + from chipcompiler.utility.filelist import FILELIST_SUFFIXES + + first_rtl = cfg.design_rtl[0] if cfg.design_rtl else "" + suffix = os.path.splitext(first_rtl)[1].lower() + origin_verilog = first_rtl if first_rtl and suffix not in FILELIST_SUFFIXES else "" + return { + "pdk": cfg.pdk_name, + "pdk_root": pdk_root, + "top_module": cfg.design_top, + "clock": cfg.design_clock_port, + "rtl_list": cfg.design_rtl, + "origin_verilog": cfg.design_rtl[0] if origin_verilog else "", + "origin_def": cfg.design_def, + "netlist": cfg.design_netlist, + "golden_netlist": cfg.design_golden_netlist, + "sdc": cfg.design_sdc, + "spef": cfg.design_spef, + "parameters": resolved_base_parameters(cfg), + } + + +def _slugify(value: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "_", value.strip().lower()).strip("_") + return slug or "project" diff --git a/chipcompiler/project/manifest_write.py b/chipcompiler/project/manifest_write.py new file mode 100644 index 00000000..cf1364d0 --- /dev/null +++ b/chipcompiler/project/manifest_write.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python + +"""Manifest write, mutation, and registration operations for the CLI. + +All ``project.json`` writes go through one read-modify-write helper so +status write-back and migration entry-append share the same atomicity +story. Loading and normalization live in +chipcompiler.project.manifest; this module imports from it, never +the reverse. + +Like manifest.py, this module sits on the CLI startup path (imported by +run dispatch and migration flows): keep module-level imports cheap — no +chipcompiler.data imports here. +""" + +import json +import logging +import os +import tempfile +from copy import deepcopy +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from chipcompiler.project.manifest import ( + _CANONICAL_TO_MANIFEST_STEP, + MANIFEST_FILENAME, + PRESET_MANIFEST_RANGE, + ManifestError, + _record, + _slugify, + base_design_from_config, +) + +logger = logging.getLogger(__name__) + +DEFAULT_OBJECTIVES = { + "primary": "timing", + "directions": { + "wns": "maximize", + "tns": "maximize", + "area": "minimize", + "drc_count": "minimize", + "lvs_count": "minimize", + "power": "minimize", + }, +} + + +def _now_iso() -> str: + return datetime.now(UTC).isoformat() + + +def manifest_workspace_entry( + workspace_id: str, + *, + name: str, + workspace_path: str, + start_step: str, + end_step: str, + status: str, + now: str, +) -> dict: + """One complete schema-v1 workspaces[] entry, every field materialized. + + The single builder for generated manifests and migration previews, so + the previewed entry and the applied entry are the same object shape. + """ + return { + "workspace_id": workspace_id, + "name": name, + "workspace_path": workspace_path, + "source_workspace_id": None, + "branch_from": None, + "start_step": start_step, + "end_step": end_step, + "status": status, + "created_at": now, + "updated_at": now, + "parameter_patch": {}, + "metrics_summary": {}, + "step_metrics": {}, + } + + +def build_manifest_document( + project_dir: str, + *, + design_name: str, + base_design: dict, + workspace_id: str, + workspace_path: str, + start_step: str, + end_step: str, + status: str = "running", +) -> dict: + """Assemble a schema-v1 manifest for a virgin project's first run.""" + now = _now_iso() + document = build_project_document( + project_dir, + design_name=design_name, + base_design=base_design, + now=now, + ) + document["workspaces"] = [ + manifest_workspace_entry( + workspace_id, + name=design_name, + workspace_path=workspace_path, + start_step=start_step, + end_step=end_step, + status=status, + now=now, + ) + ] + document["qor_baseline"] = { + "workspace_id": workspace_id, + "reason": "Default project QoR baseline", + } + return document + + +def build_project_document( + project_dir: str, + *, + design_name: str, + base_design: dict, + name: str | None = None, + now: str | None = None, + mpc: dict | None = None, +) -> dict[str, Any]: + """Assemble a schema-v1 Project Manifest without a Workspace.""" + timestamp = now or _now_iso() + project_name = name or os.path.basename(os.path.normpath(project_dir)) or "project" + return { + "schema_version": 1, + "project_id": f"proj_{_slugify(project_name)}", + "name": project_name, + "design_name": design_name, + "description": "", + "root_path": project_dir, + "created_at": timestamp, + "updated_at": timestamp, + "base_design": { + **{key: value for key, value in base_design.items() if key != "parameters" and value}, + "parameters": _record(base_design.get("parameters")), + "rtl_list": [ + item for item in base_design.get("rtl_list") or [] if isinstance(item, str) + ], + }, + "objectives": json.loads(json.dumps(DEFAULT_OBJECTIVES)), + "workspaces": [], + "mpc": deepcopy(mpc), + "best_workspace": None, + "qor_baseline": None, + } + + +def write_manifest_if_absent(project_dir: str, document: dict) -> bool: + """Write the manifest only when it does not exist (virgin generation race). + + Fully written and fsynced at a temp path, then linked into place: + readers never see a partial file, and a concurrent creator wins the + link — ours is discarded and the caller continues read-only. + """ + path = os.path.join(project_dir, MANIFEST_FILENAME) + content = json.dumps(document, indent=2) + "\n" + tmp_path = None + try: + with tempfile.NamedTemporaryFile( + "w", + dir=project_dir, + delete=False, + prefix=f".{MANIFEST_FILENAME}.", + suffix=".tmp", + encoding="utf-8", + ) as f: + tmp_path = f.name + f.write(content) + f.flush() + os.fsync(f.fileno()) + # Mode stays the tempfile default (0600), matching json_write's + # convention for newly created state files. + os.link(tmp_path, path) + return True + except FileExistsError: + return False + except OSError as exc: + logger.warning("manifest write failed: %s: %s", path, exc) + return False + finally: + if tmp_path is not None: + Path(tmp_path).unlink(missing_ok=True) + + +def _read_manifest_document(path: str): + try: + with open(path, encoding="utf-8") as f: + document = json.load(f) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return None + return document if isinstance(document, dict) else None + + +def update_manifest(project_dir: str, mutator) -> bool: + """Read-modify-write the manifest atomically (locked re-read + patch + replace). + + The whole read-modify-replace runs under ``.manifest.lock`` (flock): + two cooperating writers can no longer both complete the fresh read + before either replaces, so neither loses the other's update. The + mutator receives the parsed document and edits it in place. When an + unrelated change lands between the read and the write, the mutator is + re-applied to the freshest document instead of overwriting the change. + Project-level fields (including updated_at) are owned by the mutator. + Returns False (with a warning) when the manifest is missing, unreadable, + the lock cannot be taken, or the write fails — callers degrade to a + warning, never a run failure. + """ + from chipcompiler.project.locking import flock_file + + path = os.path.join(project_dir, MANIFEST_FILENAME) + try: + with flock_file(os.path.join(project_dir, ".manifest.lock"), exclusive=True): + return _update_manifest_locked(path, mutator) + except OSError as exc: + # An untakeable lock (e.g. a directory at the lock path) degrades + # like any write failure: a warning, never an uncaught exception — + # the migration registration path relies on False to roll back. + logger.warning("manifest update failed: %s: %s", path, exc) + return False + + +def _update_manifest_locked(path: str, mutator) -> bool: + base = _read_manifest_document(path) + if base is None: + logger.warning("manifest update skipped (unreadable): %s", path) + return False + + document = deepcopy(base) + mutator(document) + + fresh = _read_manifest_document(path) + if fresh is not None and fresh != base: + # An unrelated edit landed after our read: re-apply the mutator to + # the freshest document so the interleaved change survives. + document = fresh + mutator(document) + + target = Path(path) + tmp_path = None + try: + with tempfile.NamedTemporaryFile( + "w", + dir=target.parent, + delete=False, + prefix=f".{target.name}.", + suffix=".tmp", + encoding="utf-8", + ) as f: + tmp_path = Path(f.name) + json.dump(document, f, indent=2) + f.write("\n") + f.flush() + os.fsync(f.fileno()) + # Preserve the existing manifest's permissions: mkstemp's 0600 must + # not silently narrow a shared project.json. + if target.exists(): + os.chmod(tmp_path, target.stat().st_mode & 0o7777) + os.replace(tmp_path, target) + return True + except OSError as exc: + logger.warning("manifest update failed: %s: %s", path, exc) + if tmp_path is not None: + tmp_path.unlink(missing_ok=True) + return False + + +def write_back_workspace_status(project_dir: str, workspace_id: str, status: str) -> bool: + """Update one workspace entry's status (and updated_at) after a run.""" + + def mutate(document: dict) -> None: + for entry in document.get("workspaces", []): + if isinstance(entry, dict) and entry.get("workspace_id") == workspace_id: + entry["status"] = status + entry["updated_at"] = _now_iso() + + return update_manifest(project_dir, mutate) + + +def remove_workspace_registration(project_dir: str, workspace_id: str) -> bool: + """Roll back a pre-registration: drop the freshly added entry. + + Used when an overwrite run against an undeclared workspace fails before + the replacement is constructed: the restored previous workspace must not + be shadowed by a stale ``not_started`` entry this invocation created. + """ + + def mutate(document: dict) -> None: + workspaces = document.get("workspaces") + if isinstance(workspaces, list): + document["workspaces"] = [ + entry + for entry in workspaces + if not (isinstance(entry, dict) and entry.get("workspace_id") == workspace_id) + ] + document["updated_at"] = _now_iso() + + return update_manifest(project_dir, mutate) + + +def manifest_range_for_flow(cfg, flow_config: dict | None) -> tuple[str, str]: + """Return the GUI manifest range for a workspace's effective target.""" + if isinstance(flow_config, dict) and flow_config.get("start_step"): + from chipcompiler.rtl2gds import normalize_flow_step + + start = normalize_flow_step(flow_config["start_step"]) + end = normalize_flow_step(flow_config.get("end_step") or start) + try: + return (_CANONICAL_TO_MANIFEST_STEP[start], _CANONICAL_TO_MANIFEST_STEP[end]) + except KeyError as exc: + raise ManifestError(f"unknown workspace flow step: {exc.args[0]}") from None + return PRESET_MANIFEST_RANGE.get(cfg.flow_preset, ("Synth", "Harden")) + + +def pre_register_workspace( + project_dir: str, + *, + cfg, + pdk_root: str, + workspace_id: str, + workspace_path: str, + flow_config: dict | None, +) -> str: + """Atomically register a fresh managed workspace before filesystem creation. + + Returns ``registered``, ``existing``, ``conflict``, or ``failed``. A + workspace entry intentionally contains no input snapshot: copied files and + the workspace config are the reproducibility boundary. + """ + try: + start_step, end_step = manifest_range_for_flow(cfg, flow_config) + except ManifestError: + return "failed" + now = _now_iso() + manifest_path = os.path.join(project_dir, MANIFEST_FILENAME) + if not os.path.lexists(manifest_path): + document = build_manifest_document( + project_dir, + design_name=cfg.design_name, + base_design=base_design_from_config(cfg, pdk_root), + workspace_id=workspace_id, + workspace_path=workspace_path, + start_step=start_step, + end_step=end_step, + status="not_started", + ) + if write_manifest_if_absent(project_dir, document): + return "registered" + # A concurrent creator won the link race: fall through and apply the + # same registration mutation under the manifest lock instead of + # aborting this run. A genuine I/O failure fails again below. + + outcome = "registered" + + def mutate(document: dict) -> None: + nonlocal outcome + workspaces = document.get("workspaces") + if not isinstance(workspaces, list): + outcome = "failed" + return + for entry in workspaces: + if not isinstance(entry, dict) or entry.get("workspace_id") != workspace_id: + continue + if os.path.realpath(str(entry.get("workspace_path", ""))) == os.path.realpath( + workspace_path + ): + outcome = "existing" + else: + outcome = "conflict" + return + workspaces.append( + manifest_workspace_entry( + workspace_id, + name=cfg.design_name, + workspace_path=workspace_path, + start_step=start_step, + end_step=end_step, + status="not_started", + now=now, + ) + ) + document["updated_at"] = now + + if not update_manifest(project_dir, mutate): + return "failed" + return outcome diff --git a/test/engine/test_workspace_configuration.py b/test/engine/test_workspace_configuration.py index 2b7702cd..e9bb8e72 100644 --- a/test/engine/test_workspace_configuration.py +++ b/test/engine/test_workspace_configuration.py @@ -97,6 +97,26 @@ def fail_refresh(_workspace): assert read_engineering_snapshot(workspace) == before_snapshot +def test_read_workspace_configuration_does_not_materialize_missing_home_files( + tmp_path, minimal_ics55_pdk_factory +): + spec, bindings = _workspace_spec_fixture() + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings) + home_file = workspace.directory / "home" / "home.json" + log_dir = workspace.directory / "log" + home_file.unlink() + if log_dir.exists(): + import shutil + + shutil.rmtree(log_dir) + + read_workspace_configuration_from_directory(workspace.directory) + + assert not home_file.exists() + assert not log_dir.exists() + + def test_step_configuration_update_invalidates_only_target_suffix( tmp_path, minimal_ics55_pdk_factory ): diff --git a/test/engine/test_workspace_spec.py b/test/engine/test_workspace_spec.py index d0703bfd..7f057153 100644 --- a/test/engine/test_workspace_spec.py +++ b/test/engine/test_workspace_spec.py @@ -153,6 +153,22 @@ def test_validated_workspace_spec_creates_main_compatible_workspace( assert reopened.flow.steps()[0]["name"] == "Synthesis" +def test_workspace_spec_applies_config_target_parameters(tmp_path, minimal_ics55_pdk_factory): + from chipcompiler.data import load_workspace + from chipcompiler.engine import create_workspace_from_spec + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + spec = deepcopy(payload["workspaceSpec"]) + spec["parameters"]["cts.skew_bound"] = "0.12" + + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings) + reopened = load_workspace(workspace.directory) + cts = json.loads(reopened.config["CTS"].read_text(encoding="utf-8")) + + assert cts["skew_bound"] == "0.12" + + def test_manual_pdk_workspace_reopens_through_main_persistence(tmp_path): from chipcompiler.data import load_workspace from chipcompiler.engine import ( @@ -270,3 +286,30 @@ def test_workspace_spec_update_is_atomic_revisioned_and_idempotent( ) assert conflict.value.code == "revision_conflict" assert read_engineering_snapshot(repeated) == after + + +def test_workspace_spec_stale_revision_does_not_create_missing_snapshot( + tmp_path, minimal_ics55_pdk_factory +): + from chipcompiler.engine import WorkspaceLifecycleError, create_workspace_from_spec + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + target = tmp_path / "workspace" + create_workspace_from_spec(target, payload["workspaceSpec"], bindings) + snapshot = target / "home" / "engineering-snapshot.json" + snapshot.unlink() + + with pytest.raises(WorkspaceLifecycleError) as conflict: + from chipcompiler.engine import update_workspace_from_spec + + update_workspace_from_spec( + target, + 2, + payload["workspaceSpec"], + bindings, + "stale-update", + ) + + assert conflict.value.code == "revision_conflict" + assert not snapshot.exists() From 7bccaa934bd665595e946f4e9c4696b3b6fa0aaa Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 04:58:23 +0800 Subject: [PATCH 16/43] fix(workspace): preserve read-only and error boundaries --- chipcompiler/data/pdk.py | 21 +++++++++------- chipcompiler/data/workspace/__init__.py | 1 + chipcompiler/engine/signoff_export.py | 24 ++++++++++++------ chipcompiler/engine/workspace_lifecycle.py | 5 ++++ test/engine/test_signoff_assessment.py | 2 +- test/engine/test_workspace_configuration.py | 8 +++--- test/engine/test_workspace_spec.py | 27 +++++++++++++++++++++ 7 files changed, 68 insertions(+), 20 deletions(-) diff --git a/chipcompiler/data/pdk.py b/chipcompiler/data/pdk.py index febc744f..9d8278b1 100644 --- a/chipcompiler/data/pdk.py +++ b/chipcompiler/data/pdk.py @@ -249,6 +249,8 @@ def get_pdk( pdk_root: str | Path = "", pdk_config: str | Path = "", overrides: dict | None = None, + *, + validate: bool = True, ) -> PDK: """ Return the PDK instance based on the given pdk name. @@ -273,15 +275,16 @@ def get_pdk( pdk = _builtin_pdk(pdk_name_normalized, pdk_root=pdk_root) or PDK(name=pdk_name_normalized) overrides = overrides or {} pdk = apply_pdk_overrides(pdk, overrides) - pdk.validate() - errors = [] - for key, label in _OPTIONAL_PATH_LABELS.items(): - if key not in overrides: - continue - path = getattr(pdk, key) - if path and not path.is_file(): - errors.append(f"{label}: {path}") - _raise_pdk_validation_error(errors) + if validate: + pdk.validate() + errors = [] + for key, label in _OPTIONAL_PATH_LABELS.items(): + if key not in overrides: + continue + path = getattr(pdk, key) + if path and not path.is_file(): + errors.append(f"{label}: {path}") + _raise_pdk_validation_error(errors) return pdk diff --git a/chipcompiler/data/workspace/__init__.py b/chipcompiler/data/workspace/__init__.py index c6f0ff1d..8dc7d5ff 100644 --- a/chipcompiler/data/workspace/__init__.py +++ b/chipcompiler/data/workspace/__init__.py @@ -1228,6 +1228,7 @@ def load_workspace(directory: str | Path, *, read_only: bool = False) -> Workspa pdk_name=parameters.data.get("pdk", ""), pdk_root=parameters.data.get("pdk_root", ""), pdk_config=parameters.data.get("pdk_config", ""), + validate=not read_only, ) sdc_path = list(origin_dir.rglob("*.sdc")) if len(sdc_path) > 0: diff --git a/chipcompiler/engine/signoff_export.py b/chipcompiler/engine/signoff_export.py index 84b23842..19afb6b1 100644 --- a/chipcompiler/engine/signoff_export.py +++ b/chipcompiler/engine/signoff_export.py @@ -59,11 +59,16 @@ def export_signoff_package_archive( raise SignoffExportError("additional file entries require archivePath") if not isinstance(content, str): raise SignoffExportError("additional file entries require string content") - path = _additional_file_path(package_dir, archive_path) - path.parent.mkdir(parents=True, exist_ok=True) - if _has_symlink_parent(package_dir, path): - raise SignoffExportError("additional file path contains a symlink") - path.write_text(content, encoding="utf-8") + try: + path = _additional_file_path(package_dir, archive_path) + path.parent.mkdir(parents=True, exist_ok=True) + if _has_symlink_parent(package_dir, path): + raise SignoffExportError("additional file path contains a symlink") + path.write_text(content, encoding="utf-8") + except SignoffExportError: + raise + except (OSError, ValueError, TypeError) as exc: + raise SignoffExportError(str(exc)) from exc archive = package_dir.with_suffix(".tar.gz") import tarfile @@ -88,10 +93,15 @@ def export_signoff_package_archive( def _additional_file_path(package_dir: Path, archive_path: str) -> Path: + if not archive_path or "\x00" in archive_path: + raise SignoffExportError("additional file path must name a relative file") relative = Path(archive_path) - if not archive_path or relative.is_absolute() or ".." in relative.parts: + if relative == Path(".") or relative.is_absolute() or ".." in relative.parts: raise SignoffExportError("additional file path must stay inside the signoff package") - destination = (package_dir / relative).resolve() + try: + destination = (package_dir / relative).resolve() + except (OSError, ValueError) as exc: + raise SignoffExportError("additional file path must name a relative file") from exc if not destination.is_relative_to(package_dir.resolve()): raise SignoffExportError("additional file path must stay inside the signoff package") return destination diff --git a/chipcompiler/engine/workspace_lifecycle.py b/chipcompiler/engine/workspace_lifecycle.py index b8c5e073..726c7cd6 100644 --- a/chipcompiler/engine/workspace_lifecycle.py +++ b/chipcompiler/engine/workspace_lifecycle.py @@ -253,9 +253,14 @@ def _update_workspace_from_spec( if command_id and _command_retry_matches(target, command_id, fingerprint): return _load_committed_workspace(target) current = _load_committed_workspace(target) + snapshot_path = target / "home" / "engineering-snapshot.json" try: snapshot = read_engineering_snapshot(current) except EngineeringSnapshotError as exc: + if snapshot_path.exists() or snapshot_path.is_symlink(): + raise WorkspaceLifecycleError( + "workspace_invalid", "Invalid Engineering Snapshot" + ) from exc if expected_workspace_revision != 1: raise WorkspaceLifecycleError( "revision_conflict", diff --git a/test/engine/test_signoff_assessment.py b/test/engine/test_signoff_assessment.py index e3ea8fb2..d237b3a8 100644 --- a/test/engine/test_signoff_assessment.py +++ b/test/engine/test_signoff_assessment.py @@ -44,7 +44,7 @@ def test_signoff_additional_file_path_rejects_escape(tmp_path): package = tmp_path / "package" package.mkdir() - for value in ("/tmp/escape", "../escape", ""): + for value in ("/tmp/escape", "../escape", "", ".", "bad\x00path"): try: _additional_file_path(package, value) except SignoffExportError: diff --git a/test/engine/test_workspace_configuration.py b/test/engine/test_workspace_configuration.py index e9bb8e72..e98e3126 100644 --- a/test/engine/test_workspace_configuration.py +++ b/test/engine/test_workspace_configuration.py @@ -100,6 +100,8 @@ def fail_refresh(_workspace): def test_read_workspace_configuration_does_not_materialize_missing_home_files( tmp_path, minimal_ics55_pdk_factory ): + import shutil + spec, bindings = _workspace_spec_fixture() bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings) @@ -107,12 +109,12 @@ def test_read_workspace_configuration_does_not_materialize_missing_home_files( log_dir = workspace.directory / "log" home_file.unlink() if log_dir.exists(): - import shutil - shutil.rmtree(log_dir) + shutil.rmtree(Path(bindings["pdk"]["root"])) - read_workspace_configuration_from_directory(workspace.directory) + configuration = read_workspace_configuration_from_directory(workspace.directory) + assert configuration["workspaceSpec"]["pdk"]["familyId"] == "ics55" assert not home_file.exists() assert not log_dir.exists() diff --git a/test/engine/test_workspace_spec.py b/test/engine/test_workspace_spec.py index 7f057153..e8782024 100644 --- a/test/engine/test_workspace_spec.py +++ b/test/engine/test_workspace_spec.py @@ -313,3 +313,30 @@ def test_workspace_spec_stale_revision_does_not_create_missing_snapshot( assert conflict.value.code == "revision_conflict" assert not snapshot.exists() + + +def test_workspace_spec_rejects_invalid_existing_snapshot(tmp_path, minimal_ics55_pdk_factory): + from chipcompiler.engine import ( + WorkspaceLifecycleError, + create_workspace_from_spec, + update_workspace_from_spec, + ) + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + target = tmp_path / "workspace" + create_workspace_from_spec(target, payload["workspaceSpec"], bindings) + snapshot = target / "home" / "engineering-snapshot.json" + snapshot.write_text('{"schemaVersion": 2}', encoding="utf-8") + + with pytest.raises(WorkspaceLifecycleError) as invalid: + update_workspace_from_spec( + target, + 1, + payload["workspaceSpec"], + bindings, + "invalid-snapshot-update", + ) + + assert invalid.value.code == "workspace_invalid" + assert snapshot.read_text(encoding="utf-8") == '{"schemaVersion": 2}' From 2bfc2ffa1edddbbd08413fab6662d2442b2f3377 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 05:10:42 +0800 Subject: [PATCH 17/43] refactor(workspace): isolate hydration helpers --- chipcompiler/data/checklist.py | 12 ++ chipcompiler/data/workspace/__init__.py | 134 ++------------------ chipcompiler/data/workspace/loader.py | 129 +++++++++++++++++++ chipcompiler/tools/ecc/checklist_render.py | 13 ++ chipcompiler/tools/ecc/signoff_checklist.py | 17 +-- 5 files changed, 166 insertions(+), 139 deletions(-) create mode 100644 chipcompiler/data/workspace/loader.py create mode 100644 chipcompiler/tools/ecc/checklist_render.py diff --git a/chipcompiler/data/checklist.py b/chipcompiler/data/checklist.py index 2356a447..0f7b80d2 100644 --- a/chipcompiler/data/checklist.py +++ b/chipcompiler/data/checklist.py @@ -46,6 +46,18 @@ def __init__(self, path: Path | str): self.path = Path(path) self.data = self._load_current_data() + @classmethod + def from_items(cls, path: Path | str, items) -> dict: + """Build normalized checklist data without reading or writing a file.""" + checklist = cls.__new__(cls) + checklist.path = Path(path) + checklist.data = checklist._default_data() + checklist.data["checklist"] = [ + cls._normalize_item(item) for item in items if isinstance(item, dict) + ] + cls._refresh_summary(checklist.data) + return checklist.data + @staticmethod def _timestamp() -> str: return datetime.now(UTC).isoformat().replace("+00:00", "Z") diff --git a/chipcompiler/data/workspace/__init__.py b/chipcompiler/data/workspace/__init__.py index 8dc7d5ff..a62aea56 100644 --- a/chipcompiler/data/workspace/__init__.py +++ b/chipcompiler/data/workspace/__init__.py @@ -14,16 +14,20 @@ from ..parameter import ( Parameters, get_parameters, - load_parameter, reload_parameter, save_parameter, update_parameters, ) +from ..parameter import ( + load_parameter as load_parameter, +) from ..pdk import PDK, get_pdk from ..step import StateEnum, StepEnum from ..workspace_config import ( - legacy_parameters_fallback, - migrate_legacy_parameters, + legacy_parameters_fallback as legacy_parameters_fallback, +) +from ..workspace_config import ( + migrate_legacy_parameters as migrate_legacy_parameters, ) from ..workspace_config import ( workspace_config_path as workspace_config_toml_path, @@ -1185,129 +1189,9 @@ def _persisted_golden_verilog(workspace_dir: Path) -> tuple[Path | None, bool]: def load_workspace(directory: str | Path, *, read_only: bool = False) -> Workspace: - workspace_dir = Path(directory).expanduser().resolve() - origin_dir = workspace_dir / "origin" - home_dir = workspace_dir / "home" - if not workspace_dir.exists(): - return None - - if not read_only: - migrate_legacy_parameters(workspace_dir) - - # create workspace instance - workspace = Workspace() - workspace.directory = workspace_dir - if not read_only: - migrate_workspace_config_filenames(workspace_dir) - workspace.config = build_workspace_config_paths(workspace) - - config_path = workspace_config_toml_path(workspace_dir) - legacy_path = home_dir / "parameters.json" - if config_path.is_symlink(): - # A symlinked canonical config would make the workspace execute - # with external parameters it does not own: reject it the same way - # the save path refuses to write through a symlink. - from chipcompiler.data.workspace_config import WorkspaceConfigError - - raise WorkspaceConfigError(f"workspace config is a symlink: {config_path}") - parameters = load_parameter(workspace_config_toml_path(workspace_dir)) - if len(parameters.data) <= 0 and not config_path.exists() and legacy_path.exists(): - # Migration was deferred (e.g. read-only dir): fall back to the - # normalized in-memory copy so the workspace still opens. When the - # TOML exists it wins unconditionally — a malformed config never - # silently falls back to stale JSON. - fallback = legacy_parameters_fallback(workspace_dir) - if fallback: - parameters.data = fallback - if len(parameters.data) <= 0: - return None - - workspace.parameters = parameters + from .loader import load_workspace as hydrate_workspace - pdk = get_pdk( - pdk_name=parameters.data.get("pdk", ""), - pdk_root=parameters.data.get("pdk_root", ""), - pdk_config=parameters.data.get("pdk_config", ""), - validate=not read_only, - ) - sdc_path = list(origin_dir.rglob("*.sdc")) - if len(sdc_path) > 0: - pdk.sdc = sdc_path[0] - spef_path = list(origin_dir.rglob("*.spef")) - if len(spef_path) > 0: - pdk.spef = spef_path[0] - - # update lef and lib paths based on config - from chipcompiler.utility import json_read - - db_json = json_read(workspace.config.get("db", "")) - if db_json.get("INPUT", {}).get("tech_lef_path", "") != "": - pdk.tech = Path(db_json.get("INPUT", {}).get("tech_lef_path", "")) - if db_json.get("INPUT", {}).get("lef_paths", []) != []: - pdk.lefs = [Path(path) for path in db_json.get("INPUT", {}).get("lef_paths", [])] - if db_json.get("INPUT", {}).get("lib_path", []) != []: - pdk.libs = [Path(path) for path in db_json.get("INPUT", {}).get("lib_path", [])] - workspace.pdk = pdk - - # update config - workspace.design.name = parameters.data.get("design", "") - workspace.design.top_module = parameters.data.get("top_module", "") - def_path = list(origin_dir.rglob("*.def")) - def_gz_path = list(origin_dir.rglob("*.def.gz")) - if len(def_path) > 0: - workspace.design.origin_def = def_path[0] - if len(def_gz_path) > 0: - workspace.design.origin_def = def_gz_path[0] - - # The golden netlist path is persisted in the first flow step's info at - # creation; trust it over the golden_* filename convention so a primary - # netlist whose name merely starts with "golden_" keeps its role. Only - # legacy ledgers without step info fall back to the filename convention. - golden, golden_declared = _persisted_golden_verilog(workspace_dir) - if golden is None and not golden_declared: - golden_paths = list(origin_dir.rglob("golden_*.v")) + list( - origin_dir.rglob("golden_*.v.gz") - ) - golden = golden_paths[0] if golden_paths else None - - verilog_path = [path for path in origin_dir.rglob("*.v") if path != golden] - verilog_gz_path = [path for path in origin_dir.rglob("*.v.gz") if path != golden] - if len(verilog_path) > 0: - workspace.design.origin_verilog = verilog_path[0] - if len(verilog_gz_path) > 0: - workspace.design.origin_verilog = verilog_gz_path[0] - - if golden is not None: - workspace.design.golden_verilog = golden - - filelist_path = origin_dir / "filelist" - if filelist_path.exists(): - workspace.design.input_filelist = filelist_path - - # set home data - workspace.flow.path = home_dir / "flow.json" - if read_only: - workspace.home.path = home_dir / "home.json" - home_data = json_read(workspace.home.path) - workspace.home.data = home_data if isinstance(home_data, dict) else {} - workspace.logger = Logger(name=parameters.data["design"]) - else: - home_dir.mkdir(parents=True, exist_ok=True) - workspace.config["dir"].mkdir(parents=True, exist_ok=True) - workspace.home.init(path=home_dir / "home.json") - workspace.home.set_flow(workspace.flow.path) - workspace.home.set_checklist(home_dir / "checklist.json") - workspace.home.set_parameters(workspace.parameters.path) - - # create logger first (needed for copy operations) - workspace.logger = create_logger( - name=parameters.data["design"], log_dir=workspace_dir / "log" - ) - - log_workspace(workspace) - log_parameters(workspace) - - return workspace + return hydrate_workspace(directory, read_only=read_only) def log_workspace(workspace: Workspace): diff --git a/chipcompiler/data/workspace/loader.py b/chipcompiler/data/workspace/loader.py new file mode 100644 index 00000000..46bf1d2f --- /dev/null +++ b/chipcompiler/data/workspace/loader.py @@ -0,0 +1,129 @@ +"""Hydrate a Workspace from committed files without mixing migration policy.""" + +from pathlib import Path +from typing import Any + +from chipcompiler.utility import Logger, create_logger, json_read + +from ..parameter import load_parameter +from ..pdk import get_pdk +from ..workspace_config import ( + legacy_parameters_fallback, + migrate_legacy_parameters, +) +from ..workspace_config import ( + workspace_config_path as workspace_config_toml_path, +) + + +def load_workspace(directory: str | Path, *, read_only: bool = False) -> Any: + from . import ( + Workspace, + _persisted_golden_verilog, + build_workspace_config_paths, + log_parameters, + log_workspace, + migrate_workspace_config_filenames, + ) + + workspace_dir = Path(directory).expanduser().resolve() + origin_dir = workspace_dir / "origin" + home_dir = workspace_dir / "home" + if not workspace_dir.exists(): + return None + + if not read_only: + migrate_legacy_parameters(workspace_dir) + + workspace = Workspace() + workspace.directory = workspace_dir + if not read_only: + migrate_workspace_config_filenames(workspace_dir) + workspace.config = build_workspace_config_paths(workspace) + + config_path = workspace_config_toml_path(workspace_dir) + legacy_path = home_dir / "parameters.json" + if config_path.is_symlink(): + from chipcompiler.data.workspace_config import WorkspaceConfigError + + raise WorkspaceConfigError(f"workspace config is a symlink: {config_path}") + parameters = load_parameter(config_path) + if len(parameters.data) <= 0 and not config_path.exists() and legacy_path.exists(): + fallback = legacy_parameters_fallback(workspace_dir) + if fallback: + parameters.data = fallback + if len(parameters.data) <= 0: + return None + + workspace.parameters = parameters + pdk = get_pdk( + pdk_name=parameters.data.get("pdk", ""), + pdk_root=parameters.data.get("pdk_root", ""), + pdk_config=parameters.data.get("pdk_config", ""), + validate=not read_only, + ) + sdc_path = list(origin_dir.rglob("*.sdc")) + if sdc_path: + pdk.sdc = sdc_path[0] + spef_path = list(origin_dir.rglob("*.spef")) + if spef_path: + pdk.spef = spef_path[0] + + db_json = json_read(workspace.config.get("db", "")) + if db_json.get("INPUT", {}).get("tech_lef_path", "") != "": + pdk.tech = Path(db_json["INPUT"]["tech_lef_path"]) + if db_json.get("INPUT", {}).get("lef_paths", []) != []: + pdk.lefs = [Path(path) for path in db_json["INPUT"]["lef_paths"]] + if db_json.get("INPUT", {}).get("lib_path", []) != []: + pdk.libs = [Path(path) for path in db_json["INPUT"]["lib_path"]] + workspace.pdk = pdk + + workspace.design.name = parameters.data.get("design", "") + workspace.design.top_module = parameters.data.get("top_module", "") + def_path = list(origin_dir.rglob("*.def")) + def_gz_path = list(origin_dir.rglob("*.def.gz")) + if def_path: + workspace.design.origin_def = def_path[0] + if def_gz_path: + workspace.design.origin_def = def_gz_path[0] + + golden, golden_declared = _persisted_golden_verilog(workspace_dir) + if golden is None and not golden_declared: + golden_paths = list(origin_dir.rglob("golden_*.v")) + list( + origin_dir.rglob("golden_*.v.gz") + ) + golden = golden_paths[0] if golden_paths else None + + verilog_path = [path for path in origin_dir.rglob("*.v") if path != golden] + verilog_gz_path = [path for path in origin_dir.rglob("*.v.gz") if path != golden] + if verilog_path: + workspace.design.origin_verilog = verilog_path[0] + if verilog_gz_path: + workspace.design.origin_verilog = verilog_gz_path[0] + if golden is not None: + workspace.design.golden_verilog = golden + + filelist_path = origin_dir / "filelist" + if filelist_path.exists(): + workspace.design.input_filelist = filelist_path + + workspace.flow.path = home_dir / "flow.json" + if read_only: + workspace.home.path = home_dir / "home.json" + home_data = json_read(workspace.home.path) + workspace.home.data = home_data if isinstance(home_data, dict) else {} + workspace.logger = Logger(name=parameters.data["design"]) + else: + home_dir.mkdir(parents=True, exist_ok=True) + workspace.config["dir"].mkdir(parents=True, exist_ok=True) + workspace.home.init(path=home_dir / "home.json") + workspace.home.set_flow(workspace.flow.path) + workspace.home.set_checklist(home_dir / "checklist.json") + workspace.home.set_parameters(workspace.parameters.path) + workspace.logger = create_logger( + name=parameters.data["design"], log_dir=workspace_dir / "log" + ) + log_workspace(workspace) + log_parameters(workspace) + + return workspace diff --git a/chipcompiler/tools/ecc/checklist_render.py b/chipcompiler/tools/ecc/checklist_render.py new file mode 100644 index 00000000..03c381e6 --- /dev/null +++ b/chipcompiler/tools/ecc/checklist_render.py @@ -0,0 +1,13 @@ +"""Non-persisting checklist rendering for read-only package inspection.""" + +from pathlib import Path + +from chipcompiler.data import Checklist + + +def render_checklist(path: Path | str, items, *, persist: bool = False) -> dict: + if persist: + checklist = Checklist(path) + checklist.replace(list(items)) + return checklist.data + return Checklist.from_items(path, items) diff --git a/chipcompiler/tools/ecc/signoff_checklist.py b/chipcompiler/tools/ecc/signoff_checklist.py index 304eb23e..e69d6733 100644 --- a/chipcompiler/tools/ecc/signoff_checklist.py +++ b/chipcompiler/tools/ecc/signoff_checklist.py @@ -694,17 +694,6 @@ def rebuild_home_checklist( checklist_path = workspace_dir / "home" / "checklist.json" if workspace.home.path is not None: workspace.home.set_checklist(checklist_path) - if persist: - checklist = Checklist(checklist_path) - checklist.replace(list(deduplicated.values())) - else: - # Build the same normalized payload without Checklist.__init__ or - # replace(), both of which intentionally write to home/checklist.json. - checklist = object.__new__(Checklist) - checklist.path = Path(checklist_path) - checklist.data = checklist._default_data() - checklist.data["checklist"] = [ - checklist._normalize_item(item) for item in deduplicated.values() - ] - Checklist._refresh_summary(checklist.data) - return checklist.data + from chipcompiler.tools.ecc.checklist_render import render_checklist + + return render_checklist(checklist_path, deduplicated.values(), persist=persist) From 3060ebb710f6c5895552c7becb7d4f05c56fad9f Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Wed, 9 Sep 2026 05:17:19 +0800 Subject: [PATCH 18/43] fix(signoff): keep read-only inspection side-effect free --- chipcompiler/tools/ecc/signoff_checklist.py | 2 +- test/engine/test_workspace_configuration.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/chipcompiler/tools/ecc/signoff_checklist.py b/chipcompiler/tools/ecc/signoff_checklist.py index e69d6733..5c9ab93a 100644 --- a/chipcompiler/tools/ecc/signoff_checklist.py +++ b/chipcompiler/tools/ecc/signoff_checklist.py @@ -692,7 +692,7 @@ def rebuild_home_checklist( # Recover home.json files whose checklist path was cleared by an older # home.reset(); persist so later checklist updates resolve as well. checklist_path = workspace_dir / "home" / "checklist.json" - if workspace.home.path is not None: + if persist and workspace.home.path is not None: workspace.home.set_checklist(checklist_path) from chipcompiler.tools.ecc.checklist_render import render_checklist diff --git a/test/engine/test_workspace_configuration.py b/test/engine/test_workspace_configuration.py index e98e3126..a4ae5a8a 100644 --- a/test/engine/test_workspace_configuration.py +++ b/test/engine/test_workspace_configuration.py @@ -4,6 +4,7 @@ import pytest +from chipcompiler.data import load_workspace from chipcompiler.engine import ( EngineFlow, WorkspaceLifecycleError, @@ -106,16 +107,23 @@ def test_read_workspace_configuration_does_not_materialize_missing_home_files( bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings) home_file = workspace.directory / "home" / "home.json" + checklist_file = workspace.directory / "home" / "checklist.json" log_dir = workspace.directory / "log" home_file.unlink() + checklist_file.unlink() if log_dir.exists(): shutil.rmtree(log_dir) shutil.rmtree(Path(bindings["pdk"]["root"])) configuration = read_workspace_configuration_from_directory(workspace.directory) + readonly_workspace = load_workspace(workspace.directory, read_only=True) + from chipcompiler.tools.ecc.signoff_checklist import rebuild_home_checklist + + rebuild_home_checklist(readonly_workspace, persist=False) assert configuration["workspaceSpec"]["pdk"]["familyId"] == "ics55" assert not home_file.exists() + assert not checklist_file.exists() assert not log_dir.exists() From 9f3828851504aee782606bcfad0ca625f3d92efe Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Thu, 10 Sep 2026 12:24:29 +0800 Subject: [PATCH 19/43] refactor(workspace): replace in-memory restore with flock backups Keep configuration updates under an exclusive lock and recover from on-disk backups instead of restoring file bytes from process memory. --- chipcompiler/data/__init__.py | 6 + chipcompiler/data/workspace/loader.py | 3 + chipcompiler/data/workspace_transaction.py | 122 ++++++++++++++++++ .../engine/workspace_configuration.py | 88 +++++-------- test/engine/test_workspace_configuration.py | 81 ++++++++++++ 5 files changed, 244 insertions(+), 56 deletions(-) create mode 100644 chipcompiler/data/workspace_transaction.py diff --git a/chipcompiler/data/__init__.py b/chipcompiler/data/__init__.py index 11b3e9b1..e435c8fb 100644 --- a/chipcompiler/data/__init__.py +++ b/chipcompiler/data/__init__.py @@ -70,6 +70,10 @@ YosysReport, YosysStep, ) +from .workspace_transaction import ( + WorkspaceFileTransaction, + recover_workspace_file_transaction, +) __all__ = [ "create_workspace", @@ -109,6 +113,8 @@ "build_workspace_config_paths", "workspace_config_paths", "workspace_config_path", + "WorkspaceFileTransaction", + "recover_workspace_file_transaction", "step_config_keys", "step_config_paths", "init_workspace_config", diff --git a/chipcompiler/data/workspace/loader.py b/chipcompiler/data/workspace/loader.py index 46bf1d2f..e08b06e7 100644 --- a/chipcompiler/data/workspace/loader.py +++ b/chipcompiler/data/workspace/loader.py @@ -33,6 +33,9 @@ def load_workspace(directory: str | Path, *, read_only: bool = False) -> Any: return None if not read_only: + from ..workspace_transaction import recover_workspace_file_transaction + + recover_workspace_file_transaction(workspace_dir) migrate_legacy_parameters(workspace_dir) workspace = Workspace() diff --git a/chipcompiler/data/workspace_transaction.py b/chipcompiler/data/workspace_transaction.py new file mode 100644 index 00000000..96d23b14 --- /dev/null +++ b/chipcompiler/data/workspace_transaction.py @@ -0,0 +1,122 @@ +import fcntl +import os +import shutil +from collections.abc import Iterable +from pathlib import Path +from typing import BinaryIO + +_BACKUP_NAME = ".workspace-configuration-backup" +_DISCARD_NAME = f"{_BACKUP_NAME}.discard" +_ACTIVE: set[Path] = set() + + +class WorkspaceFileTransaction: + def __init__(self, workspace: Path, lock: BinaryIO): + self.workspace = workspace + self.lock = lock + self.backup = workspace / "home" / _BACKUP_NAME + self.finished = False + + @classmethod + def begin(cls, workspace: str | Path, paths: Iterable[Path]) -> "WorkspaceFileTransaction": + root = Path(workspace).expanduser().resolve() + lock = _open_lock(_lock_path(root)) + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + try: + _recover_locked(root) + transaction = cls(root, lock) + transaction._prepare(paths) + _ACTIVE.add(root) + return transaction + except BaseException: + fcntl.flock(lock.fileno(), fcntl.LOCK_UN) + lock.close() + raise + + def commit(self) -> None: + discard = self.workspace / "home" / _DISCARD_NAME + if discard.exists(): + shutil.rmtree(discard) + if self.backup.exists(): + self.backup.replace(discard) + shutil.rmtree(discard) + self._release() + + def rollback(self) -> None: + try: + _restore_backup(self.workspace, self.backup) + shutil.rmtree(self.backup, ignore_errors=True) + finally: + self._release() + + def _prepare(self, paths: Iterable[Path]) -> None: + home = self.workspace / "home" + if home.is_symlink(): + raise OSError(f"Workspace home directory is a symlink: {home}") + home.mkdir(parents=True, exist_ok=True) + shutil.rmtree(self.backup, ignore_errors=True) + self.backup.mkdir(parents=True) + for path in sorted(set(paths)): + target = Path(path).resolve() + relative = target.relative_to(self.workspace) + if not target.is_file(): + continue + destination = self.backup / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(target, destination) + + def _release(self) -> None: + if self.finished: + return + _ACTIVE.discard(self.workspace) + fcntl.flock(self.lock.fileno(), fcntl.LOCK_UN) + self.lock.close() + self.finished = True + + +def recover_workspace_file_transaction(workspace: str | Path) -> None: + root = Path(workspace).expanduser().resolve() + if root in _ACTIVE: + return + home = root / "home" + if not (home / _BACKUP_NAME).exists() and not (home / _DISCARD_NAME).exists(): + return + with _open_lock(_lock_path(root)) as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + try: + _recover_locked(root) + finally: + fcntl.flock(lock.fileno(), fcntl.LOCK_UN) + + +def _recover_locked(workspace: Path) -> None: + home = workspace / "home" + if home.is_symlink(): + raise OSError(f"Workspace home directory is a symlink: {home}") + discard = home / _DISCARD_NAME + if discard.exists(): + shutil.rmtree(discard) + backup = home / _BACKUP_NAME + if backup.exists(): + _restore_backup(workspace, backup) + shutil.rmtree(backup) + + +def _restore_backup(workspace: Path, backup: Path) -> None: + if not backup.is_dir(): + return + for source in backup.rglob("*"): + if not source.is_file(): + continue + target = workspace / source.relative_to(backup) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + + +def _lock_path(workspace: Path) -> Path: + return workspace.parent / f".{workspace.name}.configuration.lock" + + +def _open_lock(path: Path) -> BinaryIO: + descriptor = os.open(path, os.O_RDWR | os.O_CREAT, 0o600) + return os.fdopen(descriptor, "a+b") diff --git a/chipcompiler/engine/workspace_configuration.py b/chipcompiler/engine/workspace_configuration.py index 7964408b..d4e1eb8f 100644 --- a/chipcompiler/engine/workspace_configuration.py +++ b/chipcompiler/engine/workspace_configuration.py @@ -1,5 +1,3 @@ -import os -import tempfile from copy import deepcopy from pathlib import Path from typing import Any @@ -16,6 +14,7 @@ update_workspace_param_value, workspace_param_value, ) +from chipcompiler.data.workspace_transaction import WorkspaceFileTransaction from chipcompiler.engine.flow import EngineFlow from chipcompiler.engine.rerun import invalidate_from from chipcompiler.engine.snapshot import ( @@ -47,21 +46,16 @@ def update_workspace_configuration( target = Path(target_directory).expanduser().resolve() if not target.is_dir(): raise WorkspaceLifecycleError("workspace_missing", f"Workspace not found: {target}") - from chipcompiler.engine.reconcile import _workspace_lock - - with _workspace_lock(target): - before = _snapshot_files(_workspace_file_paths(target)) - try: - return _update_workspace_configuration( - target, - expected_workspace_revision, - configuration, - workspace_bindings, - command_id, - ) - except BaseException: - _restore_files(before) - raise + return _run_file_transaction( + target, + lambda: _update_workspace_configuration( + target, + expected_workspace_revision, + configuration, + workspace_bindings, + command_id, + ), + ) def _update_workspace_configuration( @@ -247,21 +241,16 @@ def update_workspace_step_configuration( target = Path(target_directory).expanduser().resolve() if not target.is_dir(): raise WorkspaceLifecycleError("workspace_missing", f"Workspace not found: {target}") - from chipcompiler.engine.reconcile import _workspace_lock - - with _workspace_lock(target): - before = _snapshot_files(_workspace_file_paths(target)) - try: - return _update_workspace_step_configuration( - target, - expected_workspace_revision, - step_id, - parameters, - command_id, - ) - except BaseException: - _restore_files(before) - raise + return _run_file_transaction( + target, + lambda: _update_workspace_step_configuration( + target, + expected_workspace_revision, + step_id, + parameters, + command_id, + ), + ) def _update_workspace_step_configuration( @@ -461,6 +450,17 @@ def _flow_id(names: list[str]) -> str: return "custom" +def _run_file_transaction(target: Path, apply): + transaction = WorkspaceFileTransaction.begin(target, _workspace_file_paths(target)) + try: + result = apply() + transaction.commit() + return result + except BaseException: + transaction.rollback() + raise + + def _workspace_file_paths(target: Path) -> list[Path]: paths = [ target / "home" / "params.toml", @@ -471,27 +471,3 @@ def _workspace_file_paths(target: Path) -> list[Path]: ] paths.extend(path for key, path in workspace_config_paths(target).items() if key != "dir") return paths - - -def _snapshot_files(paths: list[Path]) -> dict[Path, bytes | None]: - return {path: path.read_bytes() if path.is_file() else None for path in paths} - - -def _restore_files(snapshot: dict[Path, bytes | None]) -> None: - for path, content in snapshot.items(): - if content is None: - path.unlink(missing_ok=True) - else: - path.parent.mkdir(parents=True, exist_ok=True) - fd, temporary = tempfile.mkstemp( - dir=path.parent, prefix=f".{path.name}.", suffix=".tmp" - ) - try: - with os.fdopen(fd, "wb") as handle: - handle.write(content) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, path) - except BaseException: - os.unlink(temporary) - raise diff --git a/test/engine/test_workspace_configuration.py b/test/engine/test_workspace_configuration.py index a4ae5a8a..d1ef9a29 100644 --- a/test/engine/test_workspace_configuration.py +++ b/test/engine/test_workspace_configuration.py @@ -1,4 +1,5 @@ import json +import os from copy import deepcopy from pathlib import Path @@ -98,6 +99,86 @@ def fail_refresh(_workspace): assert read_engineering_snapshot(workspace) == before_snapshot +def test_configuration_update_recovers_after_process_exit_before_snapshot( + tmp_path, minimal_ics55_pdk_factory +): + spec, bindings = _workspace_spec_fixture() + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings, "create-1") + before = read_engineering_snapshot(workspace) + + child = os.fork() + if child == 0: + import chipcompiler.engine.workspace_configuration as configuration_module + + configuration_module.invalidate_engineering_snapshot = lambda *_args, **_kwargs: os._exit( + 23 + ) + update_workspace_configuration( + workspace.directory, + before["workspaceRevision"], + {"parameters": {"design.frequency_mhz": 250.0}}, + bindings, + "configuration-crash", + ) + os._exit(99) + + _pid, status = os.waitpid(child, 0) + assert os.waitstatus_to_exitcode(status) == 23 + + updated = update_workspace_configuration( + workspace.directory, + before["workspaceRevision"], + {"parameters": {"design.frequency_mhz": 250.0}}, + bindings, + "configuration-crash", + ) + + assert updated.parameters.data["frequency_max"] == 250.0 + assert read_engineering_snapshot(updated)["workspaceRevision"] == ( + before["workspaceRevision"] + 1 + ) + + +def test_configuration_update_recovers_after_process_exit_during_commit_cleanup( + tmp_path, minimal_ics55_pdk_factory +): + spec, bindings = _workspace_spec_fixture() + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings, "create-1") + before = read_engineering_snapshot(workspace) + + child = os.fork() + if child == 0: + import chipcompiler.data.workspace_transaction as transaction_module + + original_rmtree = transaction_module.shutil.rmtree + + def crash_cleanup(path, *args, **kwargs): + if Path(path).name == ".workspace-configuration-backup.discard": + os._exit(24) + return original_rmtree(path, *args, **kwargs) + + transaction_module.shutil.rmtree = crash_cleanup + update_workspace_configuration( + workspace.directory, + before["workspaceRevision"], + {"parameters": {"design.frequency_mhz": 250.0}}, + bindings, + "configuration-cleanup-crash", + ) + os._exit(99) + + _pid, status = os.waitpid(child, 0) + assert os.waitstatus_to_exitcode(status) == 24 + + recovered = load_workspace(workspace.directory) + assert recovered.parameters.data["frequency_max"] == 250.0 + assert read_engineering_snapshot(recovered)["workspaceRevision"] == ( + before["workspaceRevision"] + 1 + ) + + def test_read_workspace_configuration_does_not_materialize_missing_home_files( tmp_path, minimal_ics55_pdk_factory ): From e4111d8674ff1703a7f4e9a3b6648ee235db0c5b Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Thu, 10 Sep 2026 12:24:41 +0800 Subject: [PATCH 20/43] docs(qor): point scoring rules at the shared qor_scoring module Move fail thresholds, weights, and metric selection into qor_scoring. CLI reports and Snapshot assessment call that module; stop documenting a GUI port and a second threshold table. --- chipcompiler/cli/commands/report.py | 2 +- chipcompiler/docs/ecc-tutorial.cn.md | 2 +- chipcompiler/docs/ecc-tutorial.en.md | 2 +- chipcompiler/docs/ecc-user-guide.cn.md | 2 +- chipcompiler/docs/ecc-user-guide.en.md | 2 +- chipcompiler/engine/qor.py | 6 + chipcompiler/engine/qor_report.py | 245 ++++++------------------- chipcompiler/engine/qor_scoring.py | 222 +++++++++++++++------- docs/development.cn.md | 8 +- docs/development.md | 27 ++- test/engine/test_qor_scoring.py | 23 ++- 11 files changed, 263 insertions(+), 278 deletions(-) diff --git a/chipcompiler/cli/commands/report.py b/chipcompiler/cli/commands/report.py index 2aa6f3bf..015ad1e8 100644 --- a/chipcompiler/cli/commands/report.py +++ b/chipcompiler/cli/commands/report.py @@ -31,7 +31,7 @@ def _finish(subcommand: str, command_input, handler) -> None: execute_command("report", command_input, handler, render_key=f"report:{subcommand}") -@report_app.command("qor", help="Show the overall QoR score report (GUI scoring rules)") +@report_app.command("qor", help="Show the overall QoR score report") def qor_cmd( *, output_path: OutputPathOption = None, diff --git a/chipcompiler/docs/ecc-tutorial.cn.md b/chipcompiler/docs/ecc-tutorial.cn.md index 384d4e1d..49259e44 100644 --- a/chipcompiler/docs/ecc-tutorial.cn.md +++ b/chipcompiler/docs/ecc-tutorial.cn.md @@ -527,7 +527,7 @@ $ ecc report summary ### 5.4 QoR 总分：ecc report qor -按 GUI 项目看板同一套规则打分：每条指标折算 0–100 分，按维度加权（Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1），60 分为通过线；缺项维度不重归一化（缺项会拉低总分）： +用 ECC 共用的 `qor_scoring` 规则打分（Studio Snapshot 也用这一套）：每条指标折算 0–100 分，按维度加权（Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1），60 分为通过线；缺项维度不重归一化（缺项会拉低总分）： ```console $ ecc report qor diff --git a/chipcompiler/docs/ecc-tutorial.en.md b/chipcompiler/docs/ecc-tutorial.en.md index 5e2ff8a0..f5412e9b 100644 --- a/chipcompiler/docs/ecc-tutorial.en.md +++ b/chipcompiler/docs/ecc-tutorial.en.md @@ -528,7 +528,7 @@ Excerpts from this gcd run (full report: `cat` the file above): ### 5.4 QoR score: ecc report qor -Scores the workspace with the same rules as the GUI project dashboard: each metric maps to 0–100, dimensions are weighted (Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1), 60 is the pass line; absent dimensions are not renormalized (absence drags the overall score down): +Scores the workspace with ECC's shared `qor_scoring` rules (the same table Studio Snapshot uses): each metric maps to 0–100, dimensions are weighted (Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1), 60 is the pass line; absent dimensions are not renormalized (absence drags the overall score down): ```console $ ecc report qor diff --git a/chipcompiler/docs/ecc-user-guide.cn.md b/chipcompiler/docs/ecc-user-guide.cn.md index 948794e1..f42d9ecb 100644 --- a/chipcompiler/docs/ecc-user-guide.cn.md +++ b/chipcompiler/docs/ecc-user-guide.cn.md @@ -893,7 +893,7 @@ PDK / Node : ics55 ### 12.2 qor — QoR 总体计分报告 -按 GUI 项目看板的计分规则给当前 workspace 打分：每条 v3 `qor_metrics.json` 指标按固定失败阈值折算 0-100 分（slack 类线性、core_utilization 目标区间 [0.45,0.70]、lower/higher_is_better 比例），维度内取平均，再按权重（Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1）加权出总分——**缺项维度不重归一化**（与 GUI 一致，缺项会拉低总分）；60 分为通过线。默认写 `/signoff/_qor_report.txt`： +用 ECC 共用的 `qor_scoring` 规则给当前 workspace 打分（Studio Snapshot 也用这一套）：每条 v3 `qor_metrics.json` 指标按固定失败阈值折算 0-100 分（slack 类线性、core_utilization 目标区间 [0.45,0.70]、lower/higher_is_better 比例），维度内取平均，再按权重（Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1）加权出总分——**缺项维度不重归一化**（缺项会拉低总分）；60 分为通过线。默认写 `/signoff/_qor_report.txt`： ```console $ ecc report qor --project gcd --plain diff --git a/chipcompiler/docs/ecc-user-guide.en.md b/chipcompiler/docs/ecc-user-guide.en.md index 98161fa1..cb170570 100644 --- a/chipcompiler/docs/ecc-user-guide.en.md +++ b/chipcompiler/docs/ecc-user-guide.en.md @@ -941,7 +941,7 @@ the report extracts the current state by default (the engine API ### 12.2 qor — overall QoR score report -Scores the current workspace by the GUI project-dashboard rules: every v3 `qor_metrics.json` metric is converted to 0-100 against fixed fail thresholds (slack metrics linearly, core_utilization against the [0.45, 0.70] target window, lower/higher_is_better proportionally), averaged per dimension, then combined with the dimension weights (Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1) into the overall score — **absent dimensions are not renormalized** (matching the GUI; missing dimensions lower the score); 60 is the pass line. By default written to `/signoff/_qor_report.txt`: +Scores the current workspace with ECC's shared `qor_scoring` rules (the same table Studio Snapshot uses): every v3 `qor_metrics.json` metric is converted to 0-100 against fixed fail thresholds (slack metrics linearly, core_utilization against the [0.45, 0.70] target window, lower/higher_is_better proportionally), averaged per dimension, then combined with the dimension weights (Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1) into the overall score — **absent dimensions are not renormalized** (missing dimensions lower the score); 60 is the pass line. By default written to `/signoff/_qor_report.txt`: ```console $ ecc report qor --project gcd --plain diff --git a/chipcompiler/engine/qor.py b/chipcompiler/engine/qor.py index 6cef28c2..c54788cb 100644 --- a/chipcompiler/engine/qor.py +++ b/chipcompiler/engine/qor.py @@ -1,3 +1,9 @@ +"""Assemble Snapshot `qorAssessment` from committed analysis. + +Scoring rules live in `qor_scoring`; this module only validates metric +records, calls `score_qor`, and attaches gate status from step summaries. +""" + import math from typing import Any diff --git a/chipcompiler/engine/qor_report.py b/chipcompiler/engine/qor_report.py index 95cfa97b..15703179 100644 --- a/chipcompiler/engine/qor_report.py +++ b/chipcompiler/engine/qor_report.py @@ -1,18 +1,18 @@ """Overall QoR score report for one workspace. -Python port of the ECOS Studio GUI scoring pipeline -(ecos/gui/apps/renderer/src/utils/projectQorTrend.ts), restricted to the -single-workspace view the CLI needs: normalize the per-step schema-v3 -``analysis/qor_metrics.json`` records, select project-level records, score -each metric against the GUI fail thresholds, average per dimension, and -combine with the GUI dimension weights (deliberately NOT renormalized over -missing dimensions, matching GUI behavior). Trimmed relative to the GUI: -cross-workspace trend/regression analysis, summary blocking-issue gates, and -signoff-readiness score eligibility are project-dashboard concerns. - -Report source of truth: metrics files already carry dimension (category), -polarity (direction), and the rating gate, written by -tools/ecc/metrics.py::build_qor_metrics_payload. +Reads current per-step ``analysis/qor_metrics.json`` files and scores them +with ``chipcompiler.engine.qor_scoring``. That module is the only scoring +rule table (fail thresholds, dimension weights, pass line, metric +selection). Studio uses the same scorer via the Engineering Snapshot +``qorAssessment``; this report does not keep a second copy of the rules. + +This file owns CLI collection and presentation only: metric-file +normalization, DRC/LVS/RCX/STA flow-state gates, and the text report. +Cross-workspace trend, summary blocking-issue gates, and signoff-readiness +score eligibility stay on the Studio project dashboard. + +Metrics already carry dimension (category), polarity (direction), and the +rating gate from tools/ecc/metrics.py::build_qor_metrics_payload. """ import dataclasses @@ -21,6 +21,13 @@ from chipcompiler.data import StateEnum, StepEnum from chipcompiler.data.step_dirs import STEP_DIRECTORIES +from chipcompiler.engine.qor_scoring import ( + DIMENSION_WEIGHTS, + QOR_SCORE_THRESHOLD, + QorScoringMetric, + score_metric, + score_qor, +) from chipcompiler.utility.json import json_read # GUI FlowStep label for each canonical step that owns a scored directory. @@ -50,15 +57,6 @@ FLOW_STEPS = tuple(FLOW_STEP_DIRS) -DIMENSION_WEIGHTS = { - "timing": 0.35, - "power_integrity": 0.25, - "routability_physical": 0.2, - "area_cost": 0.1, - "clock_robustness_dfm": 0.1, - "runtime": 0.0, -} - DIMENSION_LABELS = { "timing": "Timing", "power_integrity": "Power / IR / EM", @@ -68,54 +66,9 @@ "runtime": "Runtime", } -METRIC_FAIL_VALUES = { - "drc_count": 10, - "lvs_count": 10, - "route_wirelength": 6000, - "route_via_count": 2000, - "cts_buffer_count": 20, - "cts_buffer_area": 40, - "clock_wirelength": 400000, - "cts_clock_wirelength_max": 100000, - "cts_clock_tree_max_level": 20, - "die_area": 3000, - "core_area": 2500, - "core_utilization": 0.85, - "synthesis_cell_area": 3000, - "fanout_max": 100, - "place_hpwl": 10000, - "place_grwl": 12000, - "place_flute_wirelength": 10000, - "place_congestion_egr_overflow_total": 100, - "place_congestion_egr_overflow_max": 20, - "place_rudy_utilization_max": 1, - "place_lutrudy_utilization_max": 1, - "route_dr_total_violation_count": 50, - "route_dr_total_patch_count": 100, - "route_dr_total_wirelength": 6000, - "route_dr_total_via_count": 2000, - "route_la_total_overflow": 100, - "rcx_missing_corner_count": 9, - "sta_setup_wns": -0.2, - "sta_setup_tns": -1, - "sta_hold_wns": -0.2, - "sta_hold_tns": -1, - "sta_frequency_mhz": 100, - "sta_setup_violation_count": 1, - "sta_hold_violation_count": 1, - "sta_missing_corner_count": 1, - "harden_artifact_missing_count": 6, -} - -SLACK_METRICS = {"sta_setup_wns", "sta_setup_tns", "sta_hold_wns", "sta_hold_tns"} -CORE_UTILIZATION_TARGET = (0.45, 0.70) - -#: The 0-100 line separating the GUI pass/fail presentation. -QOR_SCORE_THRESHOLD = 60 - GATE_STEPS = ("DRC", "LVS", "RCX", "STA") -_ROLE_PRIORITY = {"final": 0, "gate": 1, "trend": 2, "none": 3} +_ROLE_PRIORITY = {"final", "gate", "trend", "none"} @dataclasses.dataclass(frozen=True) @@ -242,90 +195,34 @@ def _normalize_metrics(step: str, payload: dict) -> list[QorMetricRecord]: return records -# --------------------------------------------------------------------------- -# Scoring (port of scoreRecord / buildDimensionScores / weightedOverallScore) -# --------------------------------------------------------------------------- - - -def _clamp_score(score: float) -> float: - return max(0.0, min(100.0, score)) - - -def _round_score(score: float) -> float: - return round(score, 1) - - -def _score_target_range(value: float, min_target: float, max_target: float, fail: float) -> float: - if min_target <= value <= max_target: - return 100.0 - if value < min_target: - return _clamp_score(100 * value / min_target) - return _clamp_score(100 * (fail - value) / (fail - max_target)) - - def score_record(record: QorMetricRecord) -> float | None: - if record.polarity == "trend_only": - return None - if record.metric_name not in METRIC_FAIL_VALUES: - return None - - if record.metric_name in SLACK_METRICS: - fail = METRIC_FAIL_VALUES[record.metric_name] - if fail >= 0: - return None - if record.value >= 0: - return 100.0 - return _clamp_score(100 * (record.value - fail) / -fail) - - if record.polarity == "target_range": - if record.metric_name == "core_utilization": - return _score_target_range( - record.value, *CORE_UTILIZATION_TARGET, METRIC_FAIL_VALUES["core_utilization"] - ) - return None - - fail = METRIC_FAIL_VALUES[record.metric_name] - if fail <= 0: - return None - if record.polarity == "lower_is_better": - return _clamp_score(100 * (fail - record.value) / fail) - return _clamp_score(100 * record.value / fail) - - -def _record_key(record: QorMetricRecord) -> tuple: - return (record.metric_name, record.scope, record.corner or "") - - -def _select_project_records(records, area_scoring_step) -> list[QorMetricRecord]: - selected: dict[tuple, QorMetricRecord] = {} - for record in records: - if record.project_role == "none": - continue - if record.dimension == "area_cost" and record.step != area_scoring_step: - continue - current = selected.get(_record_key(record)) - if current is None or _selection_rank(record) < _selection_rank(current): - selected[_record_key(record)] = record - return sorted(selected.values(), key=lambda r: r.metric_name) - - -def _selection_rank(record: QorMetricRecord) -> tuple: - try: - step_rank = FLOW_STEPS.index(record.step) - except ValueError: - # Forward-version/custom steps remain presentable but have no - # canonical ordering; prefer known flow evidence when deduplicating. - step_rank = -1 - return (_ROLE_PRIORITY.get(record.project_role, len(_ROLE_PRIORITY)), -step_rank) + return score_metric(_scoring_metric(record)) + + +def _scoring_metric(record: QorMetricRecord) -> QorScoringMetric: + return QorScoringMetric( + step=record.step, + metric_id=record.metric_name, + value=record.value, + dimension=record.dimension, + direction=record.polarity, + scope=record.scope, + corner=record.corner, + project_role=record.project_role, + rating_score=record.rating_score, + payload=record, + ) def _resolve_area_scoring_step(records, flow_steps_by_label) -> str | None: - for step in reversed(FLOW_STEPS): - if flow_steps_by_label.get(step) != StateEnum.Success.value: - continue - if any(r.step == step and r.dimension == "area_cost" and r.rating_score for r in records): - return step - return None + scored = [ + _scoring_metric(record) + for record in records + if record.rating_score + and record.dimension == "area_cost" + and flow_steps_by_label.get(record.step) == StateEnum.Success.value + ] + return score_qor(scored, flow_order=FLOW_STEPS).area_scoring_step def _gate_status(flow_steps_by_label) -> str: @@ -386,21 +283,6 @@ def _workspace_status(flow_state: str, score: float | None, gate: str) -> str: return "Red" -def _weighted_overall(dimension_scores: dict) -> float | None: - weighted_total = 0.0 - used_weight = 0.0 - for dimension, score in dimension_scores.items(): - weight = DIMENSION_WEIGHTS.get(dimension, 0.0) - if weight <= 0: - continue - weighted_total += score * weight - used_weight += weight - if used_weight == 0: - return None - # GUI behavior: no renormalization over missing dimensions. - return weighted_total - - # --------------------------------------------------------------------------- # Workspace collection and rendering # --------------------------------------------------------------------------- @@ -452,25 +334,10 @@ def build_qor_report(workspace) -> QorScoreReport: analyzed_steps.append(step) records.extend(_normalize_metrics(step, payload)) - area_scoring_step = _resolve_area_scoring_step(records, flow_steps_by_label) - project_records = _select_project_records(records, area_scoring_step) - - scored: list[QorMetricRecord] = [] - by_dimension: dict[str, list[float]] = {} - for record in project_records: - # GUI gate: only rating.score records feed dimension averages; the - # rest stay in the table marked as trend-only. - score = score_record(record) if record.rating_score else None - scored.append(dataclasses.replace(record, score=score)) - if score is not None: - by_dimension.setdefault(record.dimension, []).append(score) - - dimension_averages = { - dimension: _round_score(sum(scores) / len(scores)) - for dimension, scores in by_dimension.items() - } - overall = _weighted_overall(dimension_averages) - overall_score = _round_score(overall) if overall is not None else None + scoring = score_qor([_scoring_metric(record) for record in records], flow_order=FLOW_STEPS) + scored = [ + dataclasses.replace(item.metric.payload, score=item.score) for item in scoring.metrics + ] gate = _gate_status(flow_steps_by_label) flow_state = _flow_completion_state(flow_steps_by_label.values()) @@ -490,25 +357,25 @@ def build_qor_report(workspace) -> QorScoreReport: dimension=dimension, label=DIMENSION_LABELS[dimension], weight=DIMENSION_WEIGHTS[dimension], - score=dimension_averages[dimension], - metric_count=len(by_dimension[dimension]), + score=scoring.dimensions[dimension][0], + metric_count=scoring.dimensions[dimension][1], ) for dimension in DIMENSION_WEIGHTS - if dimension in dimension_averages + if dimension in scoring.dimensions ] absent = [ DIMENSION_LABELS[dimension] for dimension in DIMENSION_WEIGHTS - if dimension not in dimension_averages and DIMENSION_WEIGHTS[dimension] > 0 + if dimension not in scoring.dimensions and DIMENSION_WEIGHTS[dimension] > 0 ] return QorScoreReport( workspace=str(workspace_root), design=design, - overall_score=overall_score, - status=_workspace_status(flow_state, overall_score, gate), + overall_score=scoring.overall_score, + status=_workspace_status(flow_state, scoring.overall_score, gate), gate_status=gate, - area_scoring_step=area_scoring_step, + area_scoring_step=scoring.area_scoring_step, dimension_scores=dimension_scores, metrics=scored, absent_dimensions=absent, diff --git a/chipcompiler/engine/qor_scoring.py b/chipcompiler/engine/qor_scoring.py index ce96787e..d3de78d5 100644 --- a/chipcompiler/engine/qor_scoring.py +++ b/chipcompiler/engine/qor_scoring.py @@ -1,19 +1,67 @@ +"""Shared QoR scoring rules. + +Owns fail thresholds, dimension weights, metric selection, and overall +score. Callers: Snapshot assessment (`qor.py`) and CLI report +(`qor_report.py`). Studio GUI consumes the Snapshot result; it does not +keep a second rule table. +""" + from dataclasses import dataclass from typing import Any -from .qor_report import ( - _STEP_ENUM_TO_LABEL, - QorMetricRecord, - _select_project_records, - _weighted_overall, - score_record, -) -from .qor_report import ( - DIMENSION_WEIGHTS as DIMENSION_WEIGHTS, -) -from .qor_report import ( - QOR_SCORE_THRESHOLD as QOR_SCORE_THRESHOLD, -) +QOR_SCORE_THRESHOLD = 60 + +DIMENSION_WEIGHTS = { + "timing": 0.35, + "power_integrity": 0.25, + "routability_physical": 0.2, + "area_cost": 0.1, + "clock_robustness_dfm": 0.1, + "runtime": 0.0, +} + +METRIC_FAIL_VALUES = { + "drc_count": 10, + "lvs_count": 10, + "route_wirelength": 6000, + "route_via_count": 2000, + "cts_buffer_count": 20, + "cts_buffer_area": 40, + "clock_wirelength": 400000, + "cts_clock_wirelength_max": 100000, + "cts_clock_tree_max_level": 20, + "die_area": 3000, + "core_area": 2500, + "core_utilization": 0.85, + "synthesis_cell_area": 3000, + "fanout_max": 100, + "place_hpwl": 10000, + "place_grwl": 12000, + "place_flute_wirelength": 10000, + "place_congestion_egr_overflow_total": 100, + "place_congestion_egr_overflow_max": 20, + "place_rudy_utilization_max": 1, + "place_lutrudy_utilization_max": 1, + "route_dr_total_violation_count": 50, + "route_dr_total_patch_count": 100, + "route_dr_total_wirelength": 6000, + "route_dr_total_via_count": 2000, + "route_la_total_overflow": 100, + "rcx_missing_corner_count": 9, + "sta_setup_wns": -0.2, + "sta_setup_tns": -1, + "sta_hold_wns": -0.2, + "sta_hold_tns": -1, + "sta_frequency_mhz": 100, + "sta_setup_violation_count": 1, + "sta_hold_violation_count": 1, + "sta_missing_corner_count": 1, + "harden_artifact_missing_count": 6, +} + +_SLACK_METRICS = {"sta_setup_wns", "sta_setup_tns", "sta_hold_wns", "sta_hold_tns"} +_ROLE_PRIORITY = {"final": 0, "gate": 1, "trend": 2, "none": 3} +CORE_UTILIZATION_TARGET = (0.45, 0.70) @dataclass(frozen=True) @@ -44,75 +92,109 @@ class QorScoringResult: overall_score: float | None -def score_qor(records: list[QorScoringMetric]) -> QorScoringResult: - canonical_pairs = tuple((record, _canonical_record(record)) for record in records) - canonical = tuple(item[1] for item in canonical_pairs) - area_step = next( - ( - record.step - for record in reversed(canonical) - if record.dimension == "area_cost" and record.rating_score - ), - None, - ) - selected = _select_project_records(canonical, area_step) - original_by_canonical_id = { - id(canonical_record): original for original, canonical_record in canonical_pairs - } - scored = tuple( - ScoredQorMetric( - original_by_canonical_id.get(id(record), _restore_record(record)), - score_record(record) if record.rating_score else None, +def score_qor( + records: list[QorScoringMetric], *, flow_order: tuple[str, ...] = () +) -> QorScoringResult: + area_step = _area_scoring_step(records, flow_order) + selected: dict[tuple[str, str, str], tuple[int, int, QorScoringMetric]] = {} + for record in records: + if record.project_role == "none": + continue + if record.dimension == "area_cost" and record.step != area_step: + continue + key = (record.metric_id, record.scope, record.corner or "") + candidate = ( + _ROLE_PRIORITY.get(record.project_role, 3), + -_step_rank(record.step, flow_order), + record, ) - for record in selected + if key not in selected or candidate[:2] < selected[key][:2]: + selected[key] = candidate + + scored = tuple( + ScoredQorMetric(record, score_metric(record) if record.rating_score else None) + for _role, _rank, record in sorted(selected.values(), key=lambda item: item[2].metric_id) ) by_dimension: dict[str, list[float]] = {} for item in scored: if item.score is not None: by_dimension.setdefault(item.metric.dimension, []).append(item.score) dimensions = { - dimension: (round(sum(values) / len(values), 1), len(values)) - for dimension, values in by_dimension.items() + dimension: ( + round(sum(by_dimension[dimension]) / len(by_dimension[dimension]), 1), + len(by_dimension[dimension]), + ) + for dimension in DIMENSION_WEIGHTS + if dimension in by_dimension } - return QorScoringResult( - area_step, - scored, - dimensions, - round(_weighted_overall({key: value[0] for key, value in dimensions.items()}), 1) - if dimensions - else None, + weighted = sum( + score * DIMENSION_WEIGHTS[dimension] + for dimension, (score, _count) in dimensions.items() + if DIMENSION_WEIGHTS[dimension] > 0 + ) + overall = ( + round(weighted, 1) + if any(DIMENSION_WEIGHTS[dimension] > 0 for dimension in dimensions) + else None ) + return QorScoringResult(area_step, scored, dimensions, overall) def score_metric(record: QorScoringMetric) -> float | None: - return score_record(_canonical_record(record)) - - -def _canonical_record(record: QorScoringMetric) -> QorMetricRecord: - return QorMetricRecord( - step=_STEP_ENUM_TO_LABEL.get(record.step, record.step), - metric_name=record.metric_id, - display_name=record.metric_id, - value=record.value, - dimension=record.dimension, - polarity=record.direction, - scope=record.scope, - corner=record.corner, - project_role=record.project_role, - step_role="detail", - rating_score=record.rating_score, - ) + if record.direction == "trend_only": + return None + fail = METRIC_FAIL_VALUES.get(record.metric_id) + if fail is None: + return None + if record.metric_id in _SLACK_METRICS: + if fail >= 0: + return None + return 100.0 if record.value >= 0 else _clamp(100 * (record.value - fail) / -fail) + if record.direction == "target_range": + if record.metric_id != "core_utilization": + return None + low, high = CORE_UTILIZATION_TARGET + if low <= record.value <= high: + return 100.0 + if record.value < low: + return _clamp(100 * record.value / low) + return _clamp(100 * (fail - record.value) / (fail - high)) + if fail <= 0: + return None + if record.direction == "lower_is_better": + return _clamp(100 * (fail - record.value) / fail) + if record.direction == "higher_is_better": + return _clamp(100 * record.value / fail) + return None -def _restore_record(record: QorMetricRecord) -> QorScoringMetric: - return QorScoringMetric( - step=record.step, - metric_id=record.metric_name, - value=record.value, - dimension=record.dimension, - direction=record.polarity, - scope=record.scope, - corner=record.corner, - project_role=record.project_role, - rating_score=record.rating_score, +def _area_scoring_step(records: list[QorScoringMetric], flow_order: tuple[str, ...]) -> str | None: + if flow_order: + by_step = { + record.step + for record in records + if record.dimension == "area_cost" and record.rating_score + } + for step in reversed(flow_order): + if step in by_step: + return step + return None + return next( + ( + record.step + for record in reversed(records) + if record.dimension == "area_cost" and record.rating_score + ), + None, ) + + +def _step_rank(step: str, flow_order: tuple[str, ...]) -> int: + try: + return flow_order.index(step) + except ValueError: + return -1 + + +def _clamp(value: float) -> float: + return max(0.0, min(100.0, value)) diff --git a/docs/development.cn.md b/docs/development.cn.md index a6773d87..23f4d3ce 100644 --- a/docs/development.cn.md +++ b/docs/development.cn.md @@ -273,7 +273,8 @@ chipcompiler/cli/inspection/ # 只读探查逻辑 chipcompiler/cli/project/ # config.py（ecc.toml 解析校验）/ config_fields.py（`ecc project` 的项目声明 schema）/ params.py（参数注册表）/ workspace_params.py（workspace 局部覆盖记录）/ manifest.py（项目形态分类）/ effective_config.py / config_params/（直配参数 schema）/ migrate*.py（旧布局迁移）/ run_*.py（run 目标解析与分发） chipcompiler/cli/rendering/ # 输出渲染（render / renderers / pretty / progress） chipcompiler/engine/signoff/ # 签核收集器 + 设计/checklist 报告（包，见下文） -chipcompiler/engine/qor_report.py # QoR 总分计分（GUI 规则移植） +chipcompiler/engine/qor_scoring.py # QoR 评分规则（阈值/权重/选指标，Studio Snapshot 与 CLI 共用） +chipcompiler/engine/qor_report.py # CLI QoR 报告（读当前 analysis，调用 qor_scoring） ``` 模块归属由 `test/cli/test_cli_module_layout.py` 强制：核心框架必须在 `cli/core/`、命令注册在 `cli/commands/`、全部处理器在唯一的 `cli/command_handlers/` 包、只读探查在 `cli/inspection/`、渲染在 `cli/rendering/`；旧的 `chipcompiler/cli/*.py` 平铺模块必须不可导入。新增文件时放进对应子包，不要在 `cli/` 根下新建模块。 @@ -427,7 +428,8 @@ config_param( #### 扩展报告（`ecc report summary/qor/checklist/step`） - **设计总结**：`ecc report summary` 调用 `chipcompiler.engine.signoff.generate_text_report`。其实现按职责分模块（`report.py` 编排 / `report_data.py` 数据契约 / `report_extract.py` 解析器+workspace 收集 / `report_sections.py` 分区抽取 / `report_timing.py` timing 链 / `report_text.py` 格式化），全部经包 `__init__` 对外暴露。新增报告分区时，在 `report_sections.py`（或 timing 链）增加 `_extract_(q)`，并在 `report.py` 编排处注册。 -- `engine/qor_report.py`：GUI `projectQorTrend.ts` 的单 workspace 移植——常量表（`METRIC_FAIL_VALUES`/`DIMENSION_WEIGHTS`/`QOR_SCORE_THRESHOLD`）+ 归一化 + 项目级记录选择（role 优先级 final>gate>trend、area_cost 只取最后成功的 area 步）+ `score_record` 计分公式 + 维度加权（不重归一化）。新增可计分指标 = 在 GUI 与 `METRIC_FAIL_VALUES` 同步加阈值。 +- `engine/qor_scoring.py`：唯一的 QoR 评分规则。无 I/O；负责 metric 选择（role 优先级 final>gate>trend、area_cost 只取最后成功的 area 步）、单指标阈值（`METRIC_FAIL_VALUES`）、维度平均、权重（缺项不重归一化）和 overall score。Studio 经 Snapshot `qorAssessment`（`engine/qor.py`）消费同一套规则；不要在 GUI 或 CLI 再抄一份阈值表。新增可计分指标 = 只在 `METRIC_FAIL_VALUES` 加阈值。 +- `engine/qor_report.py`：CLI `ecc report qor` 的采集与文本。读取当前 workspace 的 v3 `qor_metrics.json`，交给 `score_qor`，再渲染总分、维度表和逐指标明细。不拥有评分公式。 - `engine/signoff/report_checklist.py`：只读渲染 `home/checklist.json`（不合法时报 unavailable，绝不回写文件）。 - CLI：`cli/commands/report.py` + `cli/command_handlers/report.py`；workspace 解析复用 `inspection/discovery.py`（`resolve_workspace_path` 是无副作用核心，`resolve_command_workspace` 是核心加 `load_workspace`；signoff、report 与只读的 status/log/config 共用）。 @@ -489,7 +491,7 @@ uv run ecc run --project gcd --preset rtl2gds ### 报告 -`ecc report qor` 按 GUI 项目看板相同的方式给 workspace 打分（每指标对固定 fail 阈值计分、维度求均值、加权总分——缺失维度不做权重重归一化）；`ecc report checklist` 渲染签核清单状态；`ecc report summary` 写出与 GUI 一致的文本设计总结。三者默认写入 `/signoff/`，接受 `-o` 以及常规的 `--project` 和可选的受管 `--workspace NAME` 选择器： +`ecc report qor` 用共享的 `qor_scoring` 规则给 workspace 打分（每指标对固定 fail 阈值计分、维度求均值、加权总分——缺失维度不做权重重归一化）；Studio Snapshot `qorAssessment` 用同一套计分器。`ecc report checklist` 渲染签核清单状态；`ecc report summary` 写出与 GUI 一致的文本设计总结。三者默认写入 `/signoff/`，接受 `-o` 以及常规的 `--project` 和可选的受管 `--workspace NAME` 选择器： ```bash uv run ecc report qor --project gcd diff --git a/docs/development.md b/docs/development.md index bb23cba1..ef00240d 100644 --- a/docs/development.md +++ b/docs/development.md @@ -303,7 +303,8 @@ chipcompiler/cli/inspection/ # read-only probing logic chipcompiler/cli/project/ # config.py (ecc.toml parsing and validation) / config_fields.py (project declaration schema for `ecc project`) / params.py (parameter registry) / workspace_params.py (workspace-local override records) / manifest.py (project-state classification) / effective_config.py / config_params/ (direct-config schemas) / migrate*.py (legacy-layout migration) / run_*.py (workspace target resolution and dispatch) chipcompiler/cli/rendering/ # output rendering (render / renderers / pretty / progress) chipcompiler/engine/signoff/ # signoff collector + design/checklist reports (package, see below) -chipcompiler/engine/qor_report.py # overall QoR scoring (port of the GUI rules) +chipcompiler/engine/qor_scoring.py # QoR scoring rules (thresholds/weights/selection; shared with Studio Snapshot) +chipcompiler/engine/qor_report.py # CLI QoR report (reads current analysis, calls qor_scoring) ``` Module placement is enforced by `test/cli/test_cli_module_layout.py`: the core @@ -571,13 +572,18 @@ required by doctor. / `report_text.py` formatting), all exposed through the package `__init__`. Add a report section through an `_extract_(q)` in `report_sections.py` (or the timing chain) and register it from `report.py`. -- `engine/qor_report.py`: the single-workspace port of the GUI's - `projectQorTrend.ts` — constant tables - (`METRIC_FAIL_VALUES`/`DIMENSION_WEIGHTS`/`QOR_SCORE_THRESHOLD`) + - normalization + project-level record selection (role priority - final>gate>trend; area_cost only from the last successful area step) + the - `score_record` formulas + dimension weighting (no renormalization). Adding a - scoreable metric = adding its threshold here and in the GUI. +- `engine/qor_scoring.py`: the only QoR scoring rules. No I/O; owns metric + selection (role priority final>gate>trend; area_cost only from the last + successful area step), per-metric thresholds (`METRIC_FAIL_VALUES`), + dimension averages, weights (absent dimensions are not renormalized), and + the overall score. Studio consumes the same rules through Snapshot + `qorAssessment` (`engine/qor.py`); do not copy the threshold table into the + GUI or CLI. Adding a scoreable metric = adding its threshold in + `METRIC_FAIL_VALUES` only. +- `engine/qor_report.py`: collection and text for `ecc report qor`. Reads the + workspace's current v3 `qor_metrics.json` files, calls `score_qor`, then + renders the overall score, dimension table, and per-metric detail. It does + not own the scoring formulas. - `engine/signoff/report_checklist.py`: read-only rendering of `home/checklist.json` (reports unavailable on an invalid file; never writes back). @@ -677,9 +683,10 @@ uv run ecc run --project gcd --preset rtl2gds ### Reports -`ecc report qor` scores the workspace the same way the GUI project dashboard -does (per-metric scores against fixed fail thresholds, dimension averages, +`ecc report qor` scores the workspace with the shared `qor_scoring` rules +(per-metric scores against fixed fail thresholds, dimension averages, weighted overall — weights are not renormalized over missing dimensions); +Studio Snapshot `qorAssessment` uses the same scorer. `ecc report checklist` renders the signoff checklist status, and `ecc report summary` writes the GUI-parity text design summary. All three write to `/signoff/` by default and accept `-o` plus the usual diff --git a/test/engine/test_qor_scoring.py b/test/engine/test_qor_scoring.py index f186b2f7..26b98410 100644 --- a/test/engine/test_qor_scoring.py +++ b/test/engine/test_qor_scoring.py @@ -1,4 +1,4 @@ -from chipcompiler.engine.qor_scoring import QorScoringMetric, score_qor +from chipcompiler.engine.qor_scoring import QorScoringMetric, score_metric, score_qor def _metric(step, metric_id, value, dimension, direction="lower_is_better", **kwargs): @@ -43,3 +43,24 @@ def test_qor_scoring_ignores_forward_version_dimensions_and_steps(): ) assert result.dimensions == {"area_cost": (50.0, 1)} + + +def test_qor_scoring_uses_flow_order_for_area_step_when_provided(): + result = score_qor( + [ + _metric("Harden", "die_area", 1500, "area_cost"), + _metric("DRC", "die_area", 300, "area_cost"), + ], + flow_order=("STA", "DRC", "Harden"), + ) + + assert result.area_scoring_step == "Harden" + assert result.dimensions == {"area_cost": (50.0, 1)} + + +def test_score_metric_matches_fail_threshold_formulas(): + slack = _metric("STA", "sta_setup_wns", -0.1, "timing", "higher_is_better") + assert score_metric(slack) == 50.0 + assert score_metric(_metric("DRC", "drc_count", 0, "routability_physical")) == 100.0 + utilization = _metric("Harden", "core_utilization", 0.55, "area_cost", "target_range") + assert score_metric(utilization) == 100.0 From 66fca39733a84fd9529c162c39a91df0ea49dd6c Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Thu, 10 Sep 2026 14:46:49 +0800 Subject: [PATCH 21/43] fix(qor): map synthesis power metrics to power_integrity Studio rejects the whole snapshot QoR section when any analysis metric uses category "power". Restore the canonical power_integrity mapping and remap existing on-disk qor_metrics.json when assembling snapshots. --- chipcompiler/engine/analysis.py | 28 ++++++++++++++++++++++++++++ chipcompiler/tools/ecc/metrics.py | 8 ++++---- test/engine/test_qor_scoring.py | 28 ++++++++++++++++++++++++++++ test/tools/yosys/test_metrics.py | 2 +- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/chipcompiler/engine/analysis.py b/chipcompiler/engine/analysis.py index 2e1c58c2..6df7763d 100644 --- a/chipcompiler/engine/analysis.py +++ b/chipcompiler/engine/analysis.py @@ -4,9 +4,12 @@ from typing import Any, TypeGuard from chipcompiler.data.step_dirs import STEP_DIRECTORIES +from chipcompiler.engine.qor_scoring import DIMENSION_WEIGHTS from chipcompiler.tools.ecc.sta_qor import STA_POWER_REPORT_FILENAME, STA_REPORT_FILENAMES from chipcompiler.utility import JsonReadError, file_digest, json_read_strict +_LEGACY_METRIC_CATEGORIES = {"power": "power_integrity"} + _ANALYSIS_FILES = ( ("metrics", "qor_metrics", "qor_metrics.json", 3), ("summary", "qor_summary", "qor_summary.json", 4), @@ -310,9 +313,34 @@ def _analysis_file(path: Path, artifact_id: str, schema_version: int, root: Path "reasonCode": "ANALYSIS_SCHEMA_UNSUPPORTED", "data": None, } + if schema_version == 3 and isinstance(data.get("metrics"), list): + data = _canonical_metrics_payload(data) return {"artifactId": artifact_id, "status": "available", "data": data} +def _canonical_metrics_payload(data: dict[str, Any]) -> dict[str, Any]: + records = data.get("metrics") + if not isinstance(records, list): + return data + updated = [] + changed = False + for record in records: + if not isinstance(record, dict): + updated.append(record) + continue + category = record.get("category") + mapped = _LEGACY_METRIC_CATEGORIES.get(category, category) + if mapped != category and mapped in DIMENSION_WEIGHTS: + record = {**record, "category": mapped} + changed = True + updated.append(record) + if not changed: + return data + payload = dict(data) + payload["metrics"] = updated + return payload + + def _subflow_summary(path: Path, root: Path) -> dict[str, Any]: if _has_symlink(path, root): return {"status": "unsafe", "steps": []} diff --git a/chipcompiler/tools/ecc/metrics.py b/chipcompiler/tools/ecc/metrics.py index efa854cf..03a109f7 100644 --- a/chipcompiler/tools/ecc/metrics.py +++ b/chipcompiler/tools/ecc/metrics.py @@ -54,7 +54,7 @@ "name": "synthesis_power_internal_uw", "display_name": "Synthesis Internal Power", "unit": "uW", - "dimension": "power", + "dimension": "power_integrity", "polarity": "trend_only", "confidence": "medium", }, @@ -62,7 +62,7 @@ "name": "synthesis_power_switching_uw", "display_name": "Synthesis Switching Power", "unit": "uW", - "dimension": "power", + "dimension": "power_integrity", "polarity": "trend_only", "confidence": "medium", }, @@ -70,7 +70,7 @@ "name": "synthesis_power_dynamic_uw", "display_name": "Synthesis Dynamic Power", "unit": "uW", - "dimension": "power", + "dimension": "power_integrity", "polarity": "trend_only", "confidence": "medium", }, @@ -78,7 +78,7 @@ "name": "synthesis_power_leakage_uw", "display_name": "Synthesis Leakage Power", "unit": "uW", - "dimension": "power", + "dimension": "power_integrity", "polarity": "trend_only", "confidence": "medium", }, diff --git a/test/engine/test_qor_scoring.py b/test/engine/test_qor_scoring.py index 26b98410..2f01abc2 100644 --- a/test/engine/test_qor_scoring.py +++ b/test/engine/test_qor_scoring.py @@ -58,6 +58,34 @@ def test_qor_scoring_uses_flow_order_for_area_step_when_provided(): assert result.dimensions == {"area_cost": (50.0, 1)} +def test_analysis_maps_legacy_power_category_to_power_integrity(tmp_path): + from chipcompiler.engine.analysis import _analysis_file + from chipcompiler.utility import json_write + + path = tmp_path / "qor_metrics.json" + json_write( + path, + { + "schema_version": 3, + "metrics": [ + { + "id": "synthesis_power_dynamic_uw", + "display_name": "Synthesis Dynamic Power", + "value": 18.5, + "category": "power", + "direction": "trend_only", + "scope": "synthesis", + "rating": {"gate": False, "score": False, "trend": True}, + } + ], + }, + ) + + payload = _analysis_file(path, "artifact-metrics", 3, tmp_path) + assert payload["status"] == "available" + assert payload["data"]["metrics"][0]["category"] == "power_integrity" + + def test_score_metric_matches_fail_threshold_formulas(): slack = _metric("STA", "sta_setup_wns", -0.1, "timing", "higher_is_better") assert score_metric(slack) == 50.0 diff --git a/test/tools/yosys/test_metrics.py b/test/tools/yosys/test_metrics.py index f9185a23..5e0900ef 100644 --- a/test/tools/yosys/test_metrics.py +++ b/test/tools/yosys/test_metrics.py @@ -98,7 +98,7 @@ def test_synthesis_metrics_write_v2_qor_files_without_legacy_metrics(tmp_path): "display_name": "Synthesis Dynamic Power", "value": 62.7968, "unit": "uW", - "category": "power", + "category": "power_integrity", "direction": "trend_only", "scope": "synthesis", "corner": None, From 5d7ab03345c758c2a4e0792cbeab8c228f93fa28 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Thu, 10 Sep 2026 15:16:02 +0800 Subject: [PATCH 22/43] fix(qor): restore instance metrics and dashboard detail facts Studio reads std_cell/macro/io-pad metrics plus database_facts and lvs_connectivity_summary from the snapshot. Emit those from existing feature JSON, and reject STA when pdk.sdc is unset instead of crashing. --- chipcompiler/tools/ecc/metrics.py | 123 +++++++++++ chipcompiler/tools/ecc/qor_detail_facts.py | 210 ++++++++++++++++++ chipcompiler/tools/ecc/runner.py | 2 +- test/tools/ecc/test_qor_detail_facts.py | 240 +++++++++++++++++++++ test/tools/ecc/test_runner.py | 44 ++++ 5 files changed, 618 insertions(+), 1 deletion(-) create mode 100644 chipcompiler/tools/ecc/qor_detail_facts.py create mode 100644 test/tools/ecc/test_qor_detail_facts.py diff --git a/chipcompiler/tools/ecc/metrics.py b/chipcompiler/tools/ecc/metrics.py index 03a109f7..68b58a45 100644 --- a/chipcompiler/tools/ecc/metrics.py +++ b/chipcompiler/tools/ecc/metrics.py @@ -5,6 +5,7 @@ from pathlib import Path from chipcompiler.data import EccStep, StateEnum, StepEnum, StepMetrics, Workspace, WorkspaceStep +from chipcompiler.tools.ecc.qor_detail_facts import database_fact_summary, lvs_detail_summary from chipcompiler.tools.ecc.sta_qor import ( POST_SYNTHESIS_STA_CORNER, STA_POWER_SUMMARY_FILENAME, @@ -138,6 +139,13 @@ "dimension": "area_cost", "polarity": "trend_only", }, + "instance_area": { + "name": "instance_area", + "display_name": "Instance Area", + "unit": "um^2", + "dimension": "area_cost", + "polarity": "trend_only", + }, "Total nets": { "name": "net_count", "display_name": "Net Count", @@ -145,6 +153,62 @@ "dimension": "routability_physical", "polarity": "trend_only", }, + "macro_count": { + "name": "macro_count", + "display_name": "Macro Count", + "unit": "count", + "dimension": "area_cost", + "polarity": "trend_only", + }, + "macro_area": { + "name": "macro_area", + "display_name": "Macro Area", + "unit": "um^2", + "dimension": "area_cost", + "polarity": "trend_only", + }, + "std_cell_count": { + "name": "std_cell_count", + "display_name": "Standard Cell Count", + "unit": "count", + "dimension": "area_cost", + "polarity": "trend_only", + }, + "std_cell_area": { + "name": "std_cell_area", + "display_name": "Standard Cell Area", + "unit": "um^2", + "dimension": "area_cost", + "polarity": "trend_only", + }, + "clock_count": { + "name": "clock_count", + "display_name": "Clock Cell Count", + "unit": "count", + "dimension": "area_cost", + "polarity": "trend_only", + }, + "clock_area": { + "name": "clock_area", + "display_name": "Clock Cell Area", + "unit": "um^2", + "dimension": "area_cost", + "polarity": "trend_only", + }, + "io_pad_count": { + "name": "io_pad_count", + "display_name": "IO Pad Count", + "unit": "count", + "dimension": "routability_physical", + "polarity": "trend_only", + }, + "io_pad_area": { + "name": "io_pad_area", + "display_name": "IO Pad Area", + "unit": "um^2", + "dimension": "area_cost", + "polarity": "trend_only", + }, "GP HPWL": { "name": "place_hpwl", "display_name": "Place HPWL", @@ -1856,7 +1920,16 @@ def _sta_corner_context(step: WorkspaceStep, corner: str | None) -> dict | None: "core_utilization": "/Design Layout/core_usage", "io_pin_count": "/Design Statis/num_iopins", "instance_count": "/Design Statis/num_instances", + "instance_area": "/Instances/total/area", "net_count": "/Design Statis/num_nets", + "macro_count": "/Instances/macros/num", + "macro_area": "/Instances/macros/area", + "std_cell_count": "/Instances/logic/num", + "std_cell_area": "/Instances/logic/area", + "clock_count": "/Instances/clock/num", + "clock_area": "/Instances/clock/area", + "io_pad_count": "/Instances/iopads/num", + "io_pad_area": "/Instances/iopads/area", } @@ -2099,6 +2172,22 @@ def _metric_feature_source( def _qor_detail_records(step: WorkspaceStep, step_metrics: StepMetrics) -> list[dict]: details = [] + database_path = getattr(step.feature, "db", None) + database_summary = database_fact_summary(database_path) + database_source = _relative_step_path(step, database_path) + if database_summary is not None and database_source is not None: + details.append( + { + "id": "database_facts", + "presentation": "database_facts", + "summary": database_summary, + "feature_source": { + "kind": "feature", + "path": database_source, + "selector": "", + }, + } + ) detail_specs = ( ("place_map_metrics", "place_map_summary", getattr(step.feature, "map", None)), ("cts_clock_skew_metrics", "cts_clock_skew_table", getattr(step.feature, "step", None)), @@ -2165,6 +2254,23 @@ def _qor_detail_records(step: WorkspaceStep, step_metrics: StepMetrics) -> list[ }, } ) + if step.name == StepEnum.LVS.value: + feature_path = getattr(step.feature, "step", None) + source_path = _relative_step_path(step, feature_path) + summary = lvs_detail_summary(feature_path) + if source_path is not None and summary is not None: + details.append( + { + "id": "lvs_connectivity_summary", + "presentation": "lvs_connectivity_tables", + "summary": summary, + "feature_source": { + "kind": "feature", + "path": source_path, + "selector": "/lvs", + }, + } + ) return details @@ -3505,6 +3611,23 @@ def build_metrics_db(workspace: Workspace, step: EccStep) -> dict: ): _add_number_metric(metrics, label, statistics.get(key)) + instances = data.get("Instances", {}) + instances = instances if isinstance(instances, dict) else {} + for metric_id, instance_kind, key in ( + ("instance_area", "total", "area"), + ("macro_count", "macros", "num"), + ("macro_area", "macros", "area"), + ("std_cell_count", "logic", "num"), + ("std_cell_area", "logic", "area"), + ("clock_count", "clock", "num"), + ("clock_area", "clock", "area"), + ("io_pad_count", "iopads", "num"), + ("io_pad_area", "iopads", "area"), + ): + values = instances.get(instance_kind, {}) + if isinstance(values, dict): + _add_number_metric(metrics, metric_id, values.get(key)) + metrics.update(build_metrics_timing(workspace=workspace, step=step)) return metrics diff --git a/chipcompiler/tools/ecc/qor_detail_facts.py b/chipcompiler/tools/ecc/qor_detail_facts.py new file mode 100644 index 00000000..7777d134 --- /dev/null +++ b/chipcompiler/tools/ecc/qor_detail_facts.py @@ -0,0 +1,210 @@ +"""Bounded dashboard detail facts derived from step feature JSON.""" + +from math import isfinite +from typing import Any + +from chipcompiler.utility import json_read + +_INSTANCE_CLASS_LIMIT = 32 +_LAYER_RECORD_LIMIT = 64 +_PIN_DISTRIBUTION_LIMIT = 64 +_LVS_RECORD_LIMIT = 100 + + +def qor_number(value: Any) -> int | float | None: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) if value.is_integer() else value if isfinite(value) else None + if isinstance(value, str): + text = value.strip() + if not text: + return None + percent = text.endswith("%") + if percent: + text = text[:-1].strip() + text = text.replace(",", "") + try: + number = float(text) + except ValueError: + return None + if percent: + number = number / 100.0 + if not isfinite(number): + return None + return int(number) if number.is_integer() else number + return None + + +def database_fact_summary(feature_path) -> dict | None: + data = json_read(feature_path or "") + if not isinstance(data, dict): + return None + layout = data.get("Design Layout") + statistics = data.get("Design Statis") + instances = data.get("Instances") + pins = data.get("Pins") + layers = data.get("Layers") + nets = data.get("Nets") + layout = layout if isinstance(layout, dict) else {} + statistics = statistics if isinstance(statistics, dict) else {} + instances = instances if isinstance(instances, dict) else {} + pins = pins if isinstance(pins, dict) else {} + layers = layers if isinstance(layers, dict) else {} + nets = nets if isinstance(nets, dict) else {} + + def instance_record(kind, value): + value = value if isinstance(value, dict) else {} + return { + "kind": kind, + "count": qor_number(value.get("num")), + "area": qor_number(value.get("area")), + "pin_count": qor_number(value.get("pin_num")), + } + + def layer_records(value, source_metric, output_metric): + if not isinstance(value, list): + return [] + records = [] + for index, item in enumerate(value[:_LAYER_RECORD_LIMIT]): + if not isinstance(item, dict): + continue + layer = item.get("layer_name") + records.append( + { + "layer": layer if isinstance(layer, str) and layer else f"Layer {index + 1}", + output_metric: qor_number(item.get(source_metric)), + } + ) + return records + + pin_distribution = pins.get("pin_distribution") + summary = { + "schema_version": 1, + "layout": { + "die_area": qor_number(layout.get("die_area")), + "die_usage": qor_number(layout.get("die_usage")), + "die_width": qor_number(layout.get("die_bounding_width")), + "die_height": qor_number(layout.get("die_bounding_height")), + "core_area": qor_number(layout.get("core_area")), + "core_usage": qor_number(layout.get("core_usage")), + "core_width": qor_number(layout.get("core_bounding_width")), + "core_height": qor_number(layout.get("core_bounding_height")), + "dbu": qor_number(layout.get("design_dbu")), + }, + "statistics": { + "io_pins": qor_number(statistics.get("num_iopins")), + "instances": qor_number(statistics.get("num_instances")), + "nets": qor_number(statistics.get("num_nets")), + "pdn": qor_number(statistics.get("num_pdn")), + }, + "instance_classes": [ + instance_record(kind, value) + for kind, value in sorted(instances.items()) + if kind != "total" and isinstance(kind, str) and isinstance(value, dict) + ][:_INSTANCE_CLASS_LIMIT], + "instance_total": instance_record("total", instances.get("total")), + "pin_distribution": [ + { + "pin_count": int(pin_count), + "instance_count": qor_number(item.get("inst_num")), + "net_count": qor_number(item.get("net_num")), + } + for item in ( + pin_distribution[:_PIN_DISTRIBUTION_LIMIT] + if isinstance(pin_distribution, list) + else [] + ) + if isinstance(item, dict) + and (pin_count := qor_number(item.get("pin_num"))) is not None + and pin_count >= 0 + and float(pin_count).is_integer() + ], + "cut_layers": layer_records(layers.get("cut_layers"), "via_num", "via_count"), + "routing_layers": layer_records(layers.get("routing_layers"), "wire_len", "wire_length"), + "wire_length": qor_number(nets.get("wire_len")), + "via_count": qor_number(nets.get("num_via")), + } + scalar_groups = (summary["layout"], summary["statistics"]) + has_instance_total = any( + summary["instance_total"][key] is not None for key in ("count", "area", "pin_count") + ) + if ( + not has_instance_total + and not any(value is not None for group in scalar_groups for value in group.values()) + and not any( + summary[key] + for key in ("instance_classes", "pin_distribution", "cut_layers", "routing_layers") + ) + and summary["wire_length"] is None + and summary["via_count"] is None + ): + return None + return summary + + +def lvs_text(value) -> str: + if isinstance(value, list): + return ", ".join(str(item).strip() for item in value if str(item).strip()) + return value.strip() if isinstance(value, str) else "" + + +def lvs_detail_summary(feature_path) -> dict | None: + feature = json_read(feature_path or "") + if not isinstance(feature, dict): + return None + section = feature.get("lvs", feature) + if not isinstance(section, dict): + return None + entities = [] + raw_entities = section.get("entity", []) + for item in raw_entities[:_LVS_RECORD_LIMIT] if isinstance(raw_entities, list) else []: + if not isinstance(item, dict) or not lvs_text(item.get("entity")): + continue + entities.append( + { + "entity": lvs_text(item.get("entity")), + "netlist": qor_number(item.get("netlist")), + "def": qor_number(item.get("def")), + "difference": qor_number(item.get("difference")), + } + ) + connectivity = [] + raw_connectivity = section.get("connectivity", []) + for item in raw_connectivity[:_LVS_RECORD_LIMIT] if isinstance(raw_connectivity, list) else []: + if not isinstance(item, dict) or not lvs_text(item.get("connectivity")): + continue + connectivity.append( + { + "connectivity": lvs_text(item.get("connectivity")), + "open": qor_number(item.get("open")), + "short": qor_number(item.get("short")), + "connected": qor_number(item.get("connected")), + "total": qor_number(item.get("total")), + } + ) + violations = [] + raw_violations = section.get("violations", []) + for item in raw_violations[:_LVS_RECORD_LIMIT] if isinstance(raw_violations, list) else []: + if not isinstance(item, dict): + continue + violation_type = lvs_text(item.get("type")) + if not violation_type: + continue + violations.append( + { + "type": violation_type, + "net": lvs_text(item.get("net")), + "instance": lvs_text(item.get("instance")), + "terminals": lvs_text(item.get("terminals")), + "components": lvs_text(item.get("components")), + } + ) + return { + "schema_version": 1, + "entities": entities, + "connectivity": connectivity, + "violations": violations, + } diff --git a/chipcompiler/tools/ecc/runner.py b/chipcompiler/tools/ecc/runner.py index e5714012..8c622b64 100644 --- a/chipcompiler/tools/ecc/runner.py +++ b/chipcompiler/tools/ecc/runner.py @@ -935,7 +935,7 @@ def run_sta(workspace: Workspace, step: EccStep, ecc_module: ECCToolsModule | No sub_flow.update_step(step_name=EccSubFlowEnum.run_sta.value, state=StateEnum.Imcomplete) return False - if not os.path.exists(workspace.pdk.sdc): + if not workspace.pdk.sdc or not os.path.exists(workspace.pdk.sdc): workspace.logger.error("STA SDC does not exist: %s", workspace.pdk.sdc) sub_flow.update_step(step_name=EccSubFlowEnum.run_sta.value, state=StateEnum.Imcomplete) return False diff --git a/test/tools/ecc/test_qor_detail_facts.py b/test/tools/ecc/test_qor_detail_facts.py new file mode 100644 index 00000000..722ff0d8 --- /dev/null +++ b/test/tools/ecc/test_qor_detail_facts.py @@ -0,0 +1,240 @@ +import json + +from chipcompiler.data import OriginDesign, StepEnum, Workspace +from chipcompiler.tools.ecc.builder import build_step, build_step_space +from chipcompiler.tools.ecc.metrics import build_metrics_floorplan, build_metrics_lvs +from chipcompiler.tools.ecc.qor_detail_facts import database_fact_summary + + +def test_database_fact_summary_extracts_layout_and_instance_classes(tmp_path): + path = tmp_path / "Floorplan.db.json" + path.write_text( + json.dumps( + { + "Design Layout": { + "die_area": 2259.861, + "core_area": 1778.432, + "die_bounding_width": 47.538, + "die_bounding_height": 47.538, + "die_usage": 0.34, + "core_usage": 0.42, + }, + "Design Statis": { + "num_iopins": 58, + "num_instances": 615, + "num_nets": 361, + }, + "Instances": { + "total": {"area": 1200.0, "num": 615}, + "clock": {"area": 20.0, "num": 10}, + "iopads": {"area": 10.0, "num": 2}, + "logic": {"area": 709.25, "num": 286}, + "macros": {"area": 401.5, "num": 3}, + }, + } + ), + encoding="utf-8", + ) + + summary = database_fact_summary(path) + + assert summary is not None + assert summary["layout"]["core_area"] == 1778.432 + assert summary["instance_total"]["count"] == 615 + assert {item["kind"] for item in summary["instance_classes"]} == { + "clock", + "iopads", + "logic", + "macros", + } + + +def test_floorplan_qor_metrics_include_instance_classes_and_database_facts(tmp_path): + workspace = Workspace( + directory=tmp_path, + design=OriginDesign(name="gcd", top_module="gcd"), + ) + step = build_step( + workspace=workspace, + step_name=StepEnum.FLOORPLAN.value, + input_def=tmp_path / "input.def", + input_verilog=tmp_path / "input.v", + ) + build_step_space(step) + assert step.feature.db is not None + step.feature.db.write_text( + json.dumps( + { + "Design Layout": { + "die_area": 2259.861, + "core_area": 1778.432, + "die_bounding_width": 47.538, + "die_bounding_height": 47.538, + "die_usage": 0.34, + "core_usage": 0.42, + }, + "Design Statis": { + "num_iopins": 58, + "num_instances": 615, + "num_nets": 361, + }, + "Instances": { + "total": {"area": 1200.0, "num": 615}, + "clock": {"area": 20.0, "num": 10}, + "iopads": {"area": 10.0, "num": 2}, + "logic": {"area": 709.25, "num": 286}, + "macros": {"area": 401.5, "num": 3}, + }, + } + ), + encoding="utf-8", + ) + + metrics = build_metrics_floorplan(workspace, step) + + assert metrics is not None + assert step.analysis.qor_metrics is not None + assert step.analysis.qor_metrics.exists() + qor_metrics = json.loads(step.analysis.qor_metrics.read_text(encoding="utf-8")) + assert qor_metrics["schema_version"] == 3 + assert qor_metrics["tool"] == "ecc" + assert qor_metrics["step"] == StepEnum.FLOORPLAN.value + assert qor_metrics["design"] == "gcd" + + records = {record["id"]: record for record in qor_metrics["metrics"]} + assert records["core_utilization"]["value"] == 0.42 + assert records["core_utilization"]["direction"] == "target_range" + assert records["core_area"]["value"] == 1778.432 + assert records["die_area"]["unit"] == "um^2" + assert { + metric_id: ( + records[metric_id]["value"], + records[metric_id]["source"]["selector"], + records[metric_id]["rating"]["score"], + ) + for metric_id in ( + "macro_count", + "macro_area", + "std_cell_count", + "std_cell_area", + "clock_count", + "clock_area", + "io_pad_count", + "io_pad_area", + "instance_area", + ) + } == { + "macro_count": (3, "/Instances/macros/num", False), + "macro_area": (401.5, "/Instances/macros/area", False), + "std_cell_count": (286, "/Instances/logic/num", False), + "std_cell_area": (709.25, "/Instances/logic/area", False), + "clock_count": (10, "/Instances/clock/num", False), + "clock_area": (20, "/Instances/clock/area", False), + "io_pad_count": (2, "/Instances/iopads/num", False), + "io_pad_area": (10, "/Instances/iopads/area", False), + "instance_area": (1200, "/Instances/total/area", False), + } + details = {detail["id"]: detail["summary"] for detail in qor_metrics["details"]} + assert details["database_facts"]["layout"]["core_area"] == 1778.432 + assert details["database_facts"]["instance_total"]["count"] == 615 + assert {item["kind"] for item in details["database_facts"]["instance_classes"]} == { + "clock", + "iopads", + "logic", + "macros", + } + + +def test_lvs_qor_metrics_include_connectivity_summary(tmp_path): + workspace = Workspace( + directory=tmp_path, + design=OriginDesign(name="gcd", top_module="gcd"), + ) + step = build_step( + workspace=workspace, + step_name=StepEnum.LVS.value, + input_def=tmp_path / "input.def", + input_verilog=tmp_path / "input.v", + ) + build_step_space(step) + assert step.feature.step is not None + step.feature.step.write_text( + json.dumps( + { + "entity": [{"entity": "nets", "netlist": 10, "def": 9, "difference": 1}], + "connectivity": [ + {"connectivity": "signals", "open": 1, "short": 0, "connected": 9, "total": 10} + ], + "violations": [ + {"type": "open", "net": ["n1"], "terminals": ["A", "B"]}, + {"type": "short", "net": "n2", "components": ["u1", "u2"]}, + ], + } + ), + encoding="utf-8", + ) + + metrics = build_metrics_lvs(workspace, step) + + assert metrics is not None + assert metrics.data["lvs_count"] == 2 + assert step.analysis.qor_summary is not None + summary = json.loads(step.analysis.qor_summary.read_text(encoding="utf-8")) + assert summary["quality_status"] == "blocked" + assert summary["gates"] == [ + { + "id": "qor.lvs.clean", + "title": "Final LVS clean", + "state": "failed", + "blocking": True, + "metrics": [ + { + "id": "lvs_count", + "actual": 2, + "operator": "==", + "expected": 0, + "source": { + "kind": "feature", + "path": "feature/lvs.step.json", + "selector": "/violations", + }, + } + ], + "evidence": [ + { + "kind": "feature", + "path": "feature/lvs.step.json", + "selector": "/violations", + } + ], + } + ] + assert step.analysis.qor_metrics is not None + details = { + detail["id"]: detail + for detail in json.loads(step.analysis.qor_metrics.read_text(encoding="utf-8"))["details"] + } + assert details["lvs_connectivity_summary"]["summary"] == { + "schema_version": 1, + "entities": [{"entity": "nets", "netlist": 10, "def": 9, "difference": 1}], + "connectivity": [ + {"connectivity": "signals", "open": 1, "short": 0, "connected": 9, "total": 10} + ], + "violations": [ + { + "type": "open", + "net": "n1", + "instance": "", + "terminals": "A, B", + "components": "", + }, + { + "type": "short", + "net": "n2", + "instance": "", + "terminals": "", + "components": "u1, u2", + }, + ], + } + diff --git a/test/tools/ecc/test_runner.py b/test/tools/ecc/test_runner.py index 0518e01e..4c98f506 100644 --- a/test/tools/ecc/test_runner.py +++ b/test/tools/ecc/test_runner.py @@ -953,6 +953,50 @@ def test_run_sta_uses_matched_report_and_feature_corner_directories(tmp_path, mo ] +def test_run_sta_returns_false_when_sdc_is_missing(tmp_path, monkeypatch): + config_dir = tmp_path / "config" + config_dir.mkdir() + max_lib = tmp_path / "pdk" / "max.lib" + spef = tmp_path / "RCX_ecc" / "output" / "gcd_RCworst_125C.spef" + for path in (max_lib, spef): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("", encoding="utf-8") + sta_config = config_dir / "sta_ecc.json" + sta_config.write_text( + json.dumps( + { + "liberty": [{"corner": "MAX", "temperature": 125, "path": [str(max_lib)]}], + "signoff": [{"MAX": ["RCworst"]}], + } + ), + encoding="utf-8", + ) + rcx_config = config_dir / "rcx_ecc.json" + rcx_config.write_text( + json.dumps({"output": str(tmp_path / "RCX_ecc" / "data")}), + encoding="utf-8", + ) + logger = FakeLogger() + workspace = Workspace( + directory=tmp_path, + design=OriginDesign(name="gcd", top_module="gcd"), + pdk=PDK(libs=[max_lib], sdc=None), + config={StepEnum.STA.value: sta_config, StepEnum.RCX.value: rcx_config}, + logger=logger, + ) + step = EccStep( + name=StepEnum.STA.value, + data=EccData(steps={StepEnum.STA.value: tmp_path / "sta_ecc" / "data" / "sta"}), + report=EccReport(dir=tmp_path / "sta_ecc" / "report"), + feature=EccFeature(dir=tmp_path / "sta_ecc" / "feature"), + ) + monkeypatch.setattr(ecc_runner, "EccSubFlow", FakeSubFlow) + monkeypatch.setattr(ecc_runner, "get_eda_instance", lambda **kwargs: FakeSynthesisStaModule()) + + assert ecc_runner.run_sta(workspace, step) is False + assert logger.errors[0][0] == "STA SDC does not exist: %s" + + def test_rcx_checklist_strips_top_module_from_spef_corner(tmp_path): checklist = EccRcxChecklist.__new__(EccRcxChecklist) checklist.workspace = Workspace( From a58307c25440efb644fa1122d4f2998451494f7c Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Thu, 10 Sep 2026 17:18:12 +0800 Subject: [PATCH 23/43] fix(snapshot): resolve sizer Timing Opt directory names Sizer stores Timing Opt as timing_optimization_sizer. Snapshot analysis fell back to Timing optimization_sizer, so Studio showed empty QoR, layout, reports, and subflow for a successful step. --- chipcompiler/data/__init__.py | 2 + chipcompiler/data/step.py | 11 +++ chipcompiler/engine/analysis.py | 5 +- chipcompiler/tools/ecc_sizer/builder.py | 4 +- test/data/test_step_storage_name.py | 7 ++ test/engine/test_sizer_snapshot_directory.py | 89 ++++++++++++++++++++ 6 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 test/data/test_step_storage_name.py create mode 100644 test/engine/test_sizer_snapshot_directory.py diff --git a/chipcompiler/data/__init__.py b/chipcompiler/data/__init__.py index e435c8fb..638b2e29 100644 --- a/chipcompiler/data/__init__.py +++ b/chipcompiler/data/__init__.py @@ -18,6 +18,7 @@ is_finished_step_state, load_metrics, save_metrics, + step_storage_name, ) from .workspace import ( OriginDesign, @@ -138,6 +139,7 @@ "get_design_parameters", "get_pdk", "StepEnum", + "step_storage_name", "StateEnum", "CheckState", "StepMetrics", diff --git a/chipcompiler/data/step.py b/chipcompiler/data/step.py index c1251287..c532ed37 100644 --- a/chipcompiler/data/step.py +++ b/chipcompiler/data/step.py @@ -48,6 +48,17 @@ class StateEnum(Enum): FINISHED_STEP_STATES = frozenset({StateEnum.Success.value}) +def step_storage_name(step_name: str, tool_name: str) -> str: + """Directory stem for ``{stem}_{tool}`` workspace step folders. + + Sizer stores Timing Opt as ``timing_optimization_sizer`` instead of + ``Timing optimization_sizer``. Other tools keep the flow step name. + """ + if tool_name.lower() == "sizer": + return "_".join(step_name.split()).lower() + return step_name + + def is_finished_step_state(state: object) -> bool: """Whether a persisted step state counts as done for selection and skipping. diff --git a/chipcompiler/engine/analysis.py b/chipcompiler/engine/analysis.py index 6df7763d..06ddc03e 100644 --- a/chipcompiler/engine/analysis.py +++ b/chipcompiler/engine/analysis.py @@ -3,6 +3,7 @@ from pathlib import Path from typing import Any, TypeGuard +from chipcompiler.data.step import step_storage_name from chipcompiler.data.step_dirs import STEP_DIRECTORIES from chipcompiler.engine.qor_scoring import DIMENSION_WEIGHTS from chipcompiler.tools.ecc.sta_qor import STA_POWER_REPORT_FILENAME, STA_REPORT_FILENAMES @@ -52,7 +53,9 @@ def build_workspace_analysis( identity = "".join(character for character in step_id.casefold() if character.isalnum()) if identity == "fixfanout": continue - step_dir = root / STEP_DIRECTORIES.get(step_id, f"{step_id}_{tool_id}") + step_dir = root / STEP_DIRECTORIES.get( + step_id, f"{step_storage_name(step_id, tool_id)}_{tool_id}" + ) step: dict[str, Any] = { "stepId": step_id, "toolId": tool_id, diff --git a/chipcompiler/tools/ecc_sizer/builder.py b/chipcompiler/tools/ecc_sizer/builder.py index 6009ebeb..0723038a 100644 --- a/chipcompiler/tools/ecc_sizer/builder.py +++ b/chipcompiler/tools/ecc_sizer/builder.py @@ -2,7 +2,7 @@ import shutil from pathlib import Path -from chipcompiler.data import EccStep, Workspace +from chipcompiler.data import EccStep, Workspace, step_storage_name from chipcompiler.tools.ecc import builder as ecc_builder from .utility import find_sizer_root @@ -22,7 +22,7 @@ def step_shape( Dependency-free (no rosettakit import at module scope of this module's callers) so deferred creation and selected creation share one shape. """ - safe_step_name = "_".join(step_name.split()).lower() + safe_step_name = step_storage_name(step_name, "sizer") step_directory = Path(workspace.directory) / f"{safe_step_name}_sizer" if output_def is None: output_def = step_directory / "output" / f"{workspace.design.name}_{safe_step_name}.def.gz" diff --git a/test/data/test_step_storage_name.py b/test/data/test_step_storage_name.py new file mode 100644 index 00000000..1785b1ad --- /dev/null +++ b/test/data/test_step_storage_name.py @@ -0,0 +1,7 @@ +from chipcompiler.data import StepEnum, step_storage_name + + +def test_step_storage_name_sanitizes_sizer_timing_opt(): + assert step_storage_name(StepEnum.TIMING_OPT.value, "sizer") == "timing_optimization" + assert step_storage_name(StepEnum.TIMING_OPT.value, "Sizer") == "timing_optimization" + assert step_storage_name(StepEnum.FLOORPLAN.value, "ecc") == StepEnum.FLOORPLAN.value diff --git a/test/engine/test_sizer_snapshot_directory.py b/test/engine/test_sizer_snapshot_directory.py new file mode 100644 index 00000000..e52c120a --- /dev/null +++ b/test/engine/test_sizer_snapshot_directory.py @@ -0,0 +1,89 @@ +import json +from types import SimpleNamespace + +from chipcompiler.engine.snapshot import create_engineering_snapshot + + +def test_snapshot_resolves_canonical_sizer_step_directory(tmp_path): + step_dir = tmp_path / "timing_optimization_sizer" + analysis = step_dir / "analysis" + output = step_dir / "output" + report = step_dir / "report" + analysis.mkdir(parents=True) + output.mkdir() + report.mkdir() + metric = { + "id": "instance_count", + "display_name": "Instance Count", + "value": 298, + "unit": "count", + "category": "area_cost", + "direction": "trend_only", + "scope": "timing_optimization", + "corner": None, + "analysis_group": "timing optimization_metrics", + "rating": {"gate": False, "score": False, "trend": True}, + "project_role": "trend", + "step_role": "primary", + "confidence": "high", + "source": {}, + } + (analysis / "qor_metrics.json").write_text( + json.dumps({"schema_version": 3, "metrics": [metric]}), encoding="utf-8" + ) + (analysis / "qor_summary.json").write_text( + json.dumps( + { + "schema_version": 4, + "analysis_status": "valid", + "quality_status": "pass", + "gates": [], + "missing_metrics": [], + } + ), + encoding="utf-8", + ) + (analysis / "qor_hotspots.json").write_text( + json.dumps({"schema_version": 3, "hotspots": []}), encoding="utf-8" + ) + (output / "gcd_Timing optimization.png").write_bytes(b"layout") + (report / "Timing optimization.db.rpt").write_text("report", encoding="utf-8") + (step_dir / "subflow.json").write_text( + json.dumps({"steps": [{"name": "run sizer", "state": "Success"}]}), + encoding="utf-8", + ) + workspace = SimpleNamespace( + directory=tmp_path, + design=SimpleNamespace(name="gcd"), + flow=SimpleNamespace( + data={ + "steps": [ + { + "name": "Timing optimization", + "tool": "sizer", + "state": "Success", + } + ] + } + ), + parameters=SimpleNamespace(data={}), + home=SimpleNamespace(data={}), + ) + + snapshot = create_engineering_snapshot(workspace, workspace_id="engineering-a") + + step = snapshot["analysis"]["steps"][0] + assert step["stepId"] == "Timing optimization" + assert step["metrics"]["data"]["metrics"] == [metric] + assert step["subflow"]["status"] == "available" + artifacts = {artifact["kind"]: artifact for artifact in snapshot["artifacts"]} + assert artifacts["qor_metrics"]["reference"] == ( + "timing_optimization_sizer/analysis/qor_metrics.json" + ) + assert artifacts["qor_metrics"]["availability"] == "available" + assert artifacts["layout_image"]["availability"] == "available" + assert artifacts["layout_image"]["reference"] == ( + "timing_optimization_sizer/output/gcd_Timing optimization.png" + ) + reports = [artifact for artifact in snapshot["artifacts"] if artifact["kind"] == "report_text"] + assert reports[0]["availability"] == "available" From 32e03660e1cec6a352efd222d6c79b3365ab80a9 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Thu, 10 Sep 2026 21:38:43 +0800 Subject: [PATCH 24/43] style(test): drop extra trailing newline in qor facts fixture --- test/tools/ecc/test_qor_detail_facts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/tools/ecc/test_qor_detail_facts.py b/test/tools/ecc/test_qor_detail_facts.py index 722ff0d8..f4fe8537 100644 --- a/test/tools/ecc/test_qor_detail_facts.py +++ b/test/tools/ecc/test_qor_detail_facts.py @@ -237,4 +237,3 @@ def test_lvs_qor_metrics_include_connectivity_summary(tmp_path): }, ], } - From 3037e3e02e126535f51b59fed0d6104164c20104 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Sat, 12 Sep 2026 14:22:19 +0800 Subject: [PATCH 25/43] feat(runtime): consolidate ECC workspace runtime --- .../cli/command_handlers/workspace_params.py | 53 ++ chipcompiler/cli/inspection/discovery.py | 9 +- chipcompiler/cli/project/run_existing.py | 10 +- chipcompiler/cli/project/run_prepare.py | 8 +- chipcompiler/engine/execution.py | 117 ++++- chipcompiler/engine/flow.py | 2 + chipcompiler/engine/reconcile.py | 12 +- chipcompiler/engine/rerun.py | 97 +++- chipcompiler/engine/workspace_spec.py | 28 +- chipcompiler/rtl2gds/builder.py | 1 + chipcompiler/runtime/errors.py | 6 + chipcompiler/runtime/methods.py | 90 ++++ chipcompiler/runtime/operations.py | 161 +++++- chipcompiler/runtime/requests.py | 116 ++++- chipcompiler/runtime/server.py | 68 ++- chipcompiler/runtime/sessions.py | 65 ++- chipcompiler/runtime/workspace_api.py | 344 +++++++++++-- chipcompiler/runtime/workspace_spec_api.py | 481 ++++++++++++++++++ chipcompiler/utility/workspace_lock.py | 54 ++ pyproject.toml | 2 +- test/engine/test_execution.py | 24 +- test/runtime/test_methods.py | 13 +- test/runtime/test_operations.py | 46 +- test/runtime/test_server.py | 2 +- uv.lock | 2 +- 25 files changed, 1680 insertions(+), 131 deletions(-) create mode 100644 chipcompiler/runtime/errors.py create mode 100644 chipcompiler/runtime/workspace_spec_api.py create mode 100644 chipcompiler/utility/workspace_lock.py diff --git a/chipcompiler/cli/command_handlers/workspace_params.py b/chipcompiler/cli/command_handlers/workspace_params.py index 673de650..7977ed40 100644 --- a/chipcompiler/cli/command_handlers/workspace_params.py +++ b/chipcompiler/cli/command_handlers/workspace_params.py @@ -172,6 +172,8 @@ def _mutate( workspace, workspace_error = _load_workspace(ctx) if workspace_error is not None: return workspace_error + if (Path(ctx.run_dir) / "home" / "engineering-snapshot.json").is_file(): + return _mutate_via_engine(ctx, workspace, schema, requested_value, status) try: result = mutation(workspace) except ValueError as exc: @@ -228,6 +230,57 @@ def _mutate( return CommandResult.ok([record]) +def _mutate_via_engine(ctx, workspace, schema, requested_value: object, status: str): + from chipcompiler.data.workspace_parameters import workspace_param_diff, workspace_param_step + from chipcompiler.engine import update_workspace_step_configuration + from chipcompiler.engine.snapshot import read_engineering_snapshot + from chipcompiler.engine.workspace_lifecycle import WorkspaceLifecycleError + + overrides = workspace_param_diff(workspace) + existing = next((item for item in overrides if item["key"] == schema.param), None) + if status == "unset" and existing is None: + return CommandResult.ok([_record(ctx, schema.param, None, "no_override")]) + value = requested_value if status == "set" else existing["baseline"] + step = workspace_param_step(schema) + try: + revision = read_engineering_snapshot(workspace)["workspaceRevision"] + updated = update_workspace_step_configuration( + ctx.run_dir, + revision, + step, + {schema.param: value}, + command_id="", + ) + except WorkspaceLifecycleError as exc: + return CommandResult.err( + [error_record(exc.code, param=schema.param, reason=str(exc), **exc.details)] + ) + except Exception as exc: + return CommandResult.err( + [error_record("workspace_param_refresh_failed", param=schema.param, reason=str(exc))] + ) + + updated_steps = updated.flow.steps() + target = normalize_flow_step(step).casefold() + start = next( + ( + index + for index, item in enumerate(updated_steps) + if normalize_flow_step(item.get("name", "")).casefold() == target + ), + len(updated_steps), + ) + invalidated = [ + str(item.get("name", "")) + for item in updated_steps[start:] + if item.get("name") and item.get("state") == "Unstart" + ] + record = _record(ctx, schema.param, value, status) + record["from_step"] = step + record["invalidated_steps"] = invalidated + return CommandResult.ok([record]) + + def _schema_and_workspace_error(key: str, ctx: CommandContext): schema = lookup_schema(key) if schema is None: diff --git a/chipcompiler/cli/inspection/discovery.py b/chipcompiler/cli/inspection/discovery.py index f1a1ad55..573ae386 100644 --- a/chipcompiler/cli/inspection/discovery.py +++ b/chipcompiler/cli/inspection/discovery.py @@ -185,13 +185,20 @@ def resolve_command_workspace(workspace_arg, project, workspace_id, run_dir): instead. Returns (workspace, error-record-or-None); the caller maps a non-None record to a CommandResult.err. """ + import inspect + from chipcompiler.data import load_workspace path, error = resolve_workspace_path(workspace_arg, project, workspace_id, run_dir) if error is not None: return None, error try: - workspace = load_workspace(path) + load_kwargs = ( + {"read_only": True} + if "read_only" in inspect.signature(load_workspace).parameters + else {} + ) + workspace = load_workspace(path, **load_kwargs) except Exception as exc: return None, error_record("invalid_workspace", workspace=path, reason=str(exc)) if workspace is None: diff --git a/chipcompiler/cli/project/run_existing.py b/chipcompiler/cli/project/run_existing.py index afeda0b3..43cdc370 100644 --- a/chipcompiler/cli/project/run_existing.py +++ b/chipcompiler/cli/project/run_existing.py @@ -229,7 +229,15 @@ def mismatch_error(reason: str) -> CommandResult: # The persisted ledger may be wider than the reconciled # target by design (workspace_steps is bound above), so # the full-ledger completeness check does not apply. - flow_ok = engine_flow.run_steps(require_full_ledger=False) + from chipcompiler.engine import ExecutionPlan, execute + + flow_ok = execute( + engine_flow, + ExecutionPlan( + intent="run", + step_ids=tuple(step.name for step in engine_flow.workspace_steps), + ), + ).succeeded except Exception as exc: if workspace_registered: _write_back_status(project_dir, run_name, "failed", warnings) diff --git a/chipcompiler/cli/project/run_prepare.py b/chipcompiler/cli/project/run_prepare.py index 00fdb580..32fddc16 100644 --- a/chipcompiler/cli/project/run_prepare.py +++ b/chipcompiler/cli/project/run_prepare.py @@ -476,6 +476,10 @@ def failed_workspace(reason: str | None) -> CommandResult: # The replacement is fully constructed and verified: commit it. # The previous workspace's backup is obsolete, the new tree owns # the target, and later failures are a normal failed run. + from chipcompiler.engine.snapshot import create_engineering_snapshot + + if getattr(workspace, "directory", None): + create_engineering_snapshot(workspace) commit_replacement() if workspace_registered and execute_flow: @@ -504,7 +508,9 @@ def failed_workspace(reason: str | None) -> CommandResult: if should_enable_run_progress(ctx, sys.stderr): flow_ok = run_flow_with_progress(engine_flow, ctx, project, sys.stderr) else: - flow_ok = engine_flow.run_steps() + from chipcompiler.engine import ExecutionPlan, execute + + flow_ok = execute(engine_flow, ExecutionPlan(intent="run")).succeeded if not flow_ok: if workspace_registered: diff --git a/chipcompiler/engine/execution.py b/chipcompiler/engine/execution.py index 24faa3dd..b09ee03f 100644 --- a/chipcompiler/engine/execution.py +++ b/chipcompiler/engine/execution.py @@ -2,13 +2,14 @@ from dataclasses import dataclass from typing import Any, Literal -from chipcompiler.data import StateEnum +from chipcompiler.data import StateEnum, is_finished_step_state @dataclass(frozen=True) class ExecutionPlan: intent: Literal["run", "rerun"] step_id: str | None = None + step_ids: tuple[str, ...] = () @dataclass(frozen=True) @@ -16,45 +17,109 @@ class ExecutionResult: succeeded: bool state: str step_id: str | None = None + executed_steps: tuple[str, ...] = () + failed_step: str | None = None + no_op: bool = False def execute(flow: Any, plan: ExecutionPlan, *, event_sink: Any = None) -> ExecutionResult: if plan.intent not in {"run", "rerun"}: raise ValueError(f"unsupported execution intent: {plan.intent}") - if event_sink is None and getattr(flow.workspace, "directory", None): - event_sink = _EngineeringCommitSink(flow.workspace) - rerun = plan.intent == "rerun" - if plan.step_id is None: - succeeded = bool(_invoke(flow.run_steps, rerun=rerun, observer=event_sink)) + selected_ids = tuple(plan.step_ids) + if plan.step_id is not None: + if selected_ids: + raise ValueError("ExecutionPlan cannot set both step_id and step_ids") + selected_ids = (plan.step_id,) + if plan.intent == "run" and not selected_ids and _is_completed(flow): return ExecutionResult( - succeeded=succeeded, - state=StateEnum.Success.value if succeeded else StateEnum.Imcomplete.value, + succeeded=True, + state=StateEnum.Success.value, + executed_steps=(), + no_op=True, ) + workspace = getattr(flow, "workspace", None) + if event_sink is None and getattr(workspace, "directory", None): + event_sink = _EngineeringCommitSink(workspace) + rerun = plan.intent == "rerun" + observer = _ExecutionObserver(event_sink) + if not selected_ids: + succeeded = bool(_invoke(flow.run_steps, rerun=rerun, observer=observer)) + else: + succeeded = True + for step_id in selected_ids: + step = _find_step(flow, step_id) + if step is None: + raise ValueError(f"step not found: {step_id}") + state = _invoke(flow.run_step, step, rerun=rerun, observer=observer) + if ( + state is not StateEnum.Success + and getattr(state, "value", state) != StateEnum.Success.value + ): + succeeded = False + break + executed_steps = tuple(getattr(observer, "executed_steps", ())) + failed_step = getattr(observer, "failed_step", None) + state = StateEnum.Success.value if succeeded else StateEnum.Imcomplete.value + return ExecutionResult( + succeeded=succeeded, + state=state, + step_id=selected_ids[0] if len(selected_ids) == 1 else None, + executed_steps=executed_steps, + failed_step=failed_step, + ) + +def _find_step(flow: Any, step_id: str) -> Any | None: get_workspace_step = getattr(flow, "get_workspace_step", None) - step = ( - get_workspace_step(plan.step_id) - if callable(get_workspace_step) - else next( - ( - candidate - for candidate in getattr(flow, "workspace_steps", []) - if getattr(candidate, "name", None) == plan.step_id - ), - None, - ) + if callable(get_workspace_step): + return get_workspace_step(step_id) + return next( + ( + candidate + for candidate in getattr(flow, "workspace_steps", []) + if getattr(candidate, "name", None) == step_id + ), + None, ) - if step is None: - raise ValueError(f"step not found: {plan.step_id}") - state = _invoke(flow.run_step, step, rerun=rerun, observer=event_sink) - return ExecutionResult( - succeeded=state is StateEnum.Success, - state=getattr(state, "value", str(state)), - step_id=plan.step_id, + + +def _is_completed(flow: Any) -> bool: + data = getattr(getattr(getattr(flow, "workspace", None), "flow", None), "data", {}) + steps = data.get("steps", []) if isinstance(data, dict) else [] + return bool(steps) and all( + isinstance(step, dict) and is_finished_step_state(step.get("state")) for step in steps ) +class _ExecutionObserver: + def __init__(self, delegate: Any): + self.delegate = delegate + self.fatal_observer = bool(getattr(delegate, "fatal_observer", False)) + self.executed_steps: list[str] = [] + self.failed_step: str | None = None + + def on_step_completed(self, step: Any, state: Any, error: str | None = None) -> None: + name = getattr(step, "name", None) + if isinstance(name, str) and name: + if getattr(state, "value", state) == StateEnum.Success.value: + self.executed_steps.append(name) + elif self.failed_step is None: + self.failed_step = name + callback = getattr(self.delegate, "on_step_completed", None) + if callable(callback): + callback(step, state, error) + + def __getattr__(self, name: str): + if self.delegate is not None: + callback = getattr(self.delegate, name, None) + if callback is not None: + return callback + return lambda *_args, **_kwargs: None + + class _EngineeringCommitSink: + fatal_observer = True + def __init__(self, workspace: Any): from chipcompiler.engine.snapshot import ensure_engineering_snapshot diff --git a/chipcompiler/engine/flow.py b/chipcompiler/engine/flow.py index dd31da52..3ee911d1 100644 --- a/chipcompiler/engine/flow.py +++ b/chipcompiler/engine/flow.py @@ -850,6 +850,8 @@ def _notify_flow_observer(observer, method_name: str, *args) -> None: try: callback(*args) except (Exception, SystemExit): + if getattr(observer, "fatal_observer", False): + raise # Runtime observers must never turn a completed tool execution into a # failed flow. The coordinator records transport failures separately. logging.getLogger(__name__).exception("flow observer callback failed: %s", method_name) diff --git a/chipcompiler/engine/reconcile.py b/chipcompiler/engine/reconcile.py index addf79db..1eb77e99 100644 --- a/chipcompiler/engine/reconcile.py +++ b/chipcompiler/engine/reconcile.py @@ -33,7 +33,6 @@ when the flow is compatible but an append/adopt is due. """ -import fcntl import logging from contextlib import contextmanager from dataclasses import dataclass, field @@ -123,15 +122,10 @@ def _persisted_flow_data(workspace_dir: Path, json_read) -> dict: @contextmanager def _workspace_lock(workspace_dir: Path): - # The lock lives NEXT TO the workspace (never inside it): an overwrite - # deleting the tree cannot invalidate the lock's inode, so a waiter - # always serializes against the run that replaces the directory. - lock_path = workspace_dir.parent / f"{workspace_dir.name}.lock" - workspace_dir.parent.mkdir(parents=True, exist_ok=True) - with open(lock_path, "a") as lock_file: - fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + from chipcompiler.utility.workspace_lock import workspace_lock + + with workspace_lock(workspace_dir): yield - fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) def resolve_target_section(project_flow: dict | None, workspace_flow: dict | None) -> dict: diff --git a/chipcompiler/engine/rerun.py b/chipcompiler/engine/rerun.py index 4de57a69..080d5667 100644 --- a/chipcompiler/engine/rerun.py +++ b/chipcompiler/engine/rerun.py @@ -177,29 +177,88 @@ def _invalidate_suffix(flow: "EngineFlow", index: int, last_index: int | None = def _run_selected(flow: "EngineFlow", selected: list[tuple[WorkspaceStep, Path]]) -> StepRunResult: - """Run the selected steps in order, each with a fresh output directory.""" + """Run selected steps through the shared Engine execution contract.""" # a new selection must not inherit a DB positioned by an earlier run if flow.engine_db is not None: flow.engine_db.close() - executed = [] - for workspace_step, output_dir in selected: - flow.workspace.logger.log_section( - f"{workspace_step.tool} - begin step - {workspace_step.name}" - ) - _reset_output_dir(output_dir) - _redirect_to_step_log(workspace_step) - flow.init_db_engine_for_step(workspace_step) - state = flow.run_step(workspace_step, rerun=True) - log_flow(workspace=flow.workspace) - flow.workspace.logger.log_section( - f"{workspace_step.tool} - end step - {workspace_step.name}" - ) - if state is not StateEnum.Success: - # A persisted Incomplete blocks the rerun. - return StepRunResult(ok=False, executed=tuple(executed), failed=workspace_step.name) - executed.append(workspace_step.name) - return StepRunResult(ok=True, executed=tuple(executed)) + output_dirs = {workspace_step.name: output_dir for workspace_step, output_dir in selected} + + class PreparedFlow: + workspace = flow.workspace + + def get_workspace_step(self, step_id): + return flow.get_workspace_step(step_id) + + def run_step(self, workspace_step, *, rerun=False, observer=None): + output_dir = output_dirs[workspace_step.name] + flow.workspace.logger.log_section( + f"{workspace_step.tool} - begin step - {workspace_step.name}" + ) + _reset_output_dir(output_dir) + _redirect_to_step_log(workspace_step) + flow.init_db_engine_for_step(workspace_step) + if _callable_accepts_keyword(flow.run_step, "observer"): + state = flow.run_step(workspace_step, rerun=rerun, observer=observer) + else: + state = flow.run_step(workspace_step, rerun=rerun) + log_flow(workspace=flow.workspace) + flow.workspace.logger.log_section( + f"{workspace_step.tool} - end step - {workspace_step.name}" + ) + return state + + from chipcompiler.engine.execution import ExecutionPlan, execute + + result = execute( + PreparedFlow(), + ExecutionPlan(intent="rerun", step_ids=tuple(output_dirs)), + ) + executed = result.executed_steps or _successful_prefix( + flow, + selected, + succeeded=result.succeeded, + ) + return StepRunResult( + ok=result.succeeded, + executed=executed, + failed=result.failed_step + or ( + None + if result.succeeded + else tuple(output_dirs)[min(len(executed), len(output_dirs) - 1)] + ), + ) + + +def _callable_accepts_keyword(callback, keyword: str) -> bool: + import inspect + + try: + parameters = inspect.signature(callback).parameters.values() + except (TypeError, ValueError): + return False + return any( + parameter.name == keyword or parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + + +def _successful_prefix(flow, selected, *, succeeded: bool) -> tuple[str, ...]: + if succeeded: + return tuple(step.name for step, _output_dir in selected) + persisted = getattr(getattr(flow.workspace, "flow", None), "data", {}) + states = { + str(step.get("name")): step.get("state") + for step in persisted.get("steps", []) + if isinstance(step, dict) + } + completed = [] + for step, _output_dir in selected: + if states.get(step.name) != StateEnum.Success.value: + break + completed.append(step.name) + return tuple(completed) def _validated_output_dirs(workspace: Workspace, steps: list[WorkspaceStep]) -> list[Path]: diff --git a/chipcompiler/engine/workspace_spec.py b/chipcompiler/engine/workspace_spec.py index f95f1c40..c25cf232 100644 --- a/chipcompiler/engine/workspace_spec.py +++ b/chipcompiler/engine/workspace_spec.py @@ -309,22 +309,12 @@ def _validate_parameter_applicability( flow_steps: set[str], issues: list[dict[str, Any]], ) -> None: - normalized_steps = {step.lower() for step in flow_steps} - aliases = { - "synthesis": "synth", - "floorplan": "floor", - "fixfanout": "fanout", - "placement": "place", - "routing": "route", - } - normalized_steps |= {aliases.get(step, step) for step in normalized_steps} for schema in list_schemas(): if schema.param not in explicit: continue if schema.applies == "all": continue - applies = aliases.get(schema.applies.lower(), schema.applies.lower()) - if flow_steps and applies not in normalized_steps: + if flow_steps and not _parameter_applies_to_flow(schema.applies, flow_steps): issues.append( _issue( "inapplicable_parameter", @@ -335,18 +325,30 @@ def _validate_parameter_applicability( def _effective_parameter_values(resolved, steps: set[str]) -> dict[str, object]: - available = {normalize_flow_step(step).casefold() for step in steps} return { parameter.param: deepcopy(parameter.value) for parameter in resolved if parameter.schema.pdk_target is None and ( parameter.schema.applies == "all" - or normalize_flow_step(parameter.schema.applies).casefold() in available + or _parameter_applies_to_flow(parameter.schema.applies, steps) ) } +def _parameter_applies_to_flow(applies: str, steps: set[str]) -> bool: + normalized = {normalize_flow_step(step).casefold() for step in steps} + aliases = { + "synthesis": {"synthesis"}, + "floorplan": {"prefloorplan", "macroplacement", "postfloorplan"}, + "placement": {"placement", "place", "macroplacement"}, + "routing": {"routing", "route"}, + "fixfanout": {"fixfanout"}, + } + candidates = aliases.get(applies.casefold(), {normalize_flow_step(applies).casefold()}) + return bool(normalized & candidates) + + def _refs( value: object, id_key: str, diff --git a/chipcompiler/rtl2gds/builder.py b/chipcompiler/rtl2gds/builder.py index a3020ac3..32e7ff23 100644 --- a/chipcompiler/rtl2gds/builder.py +++ b/chipcompiler/rtl2gds/builder.py @@ -40,6 +40,7 @@ def normalize_flow_step(value: str | StepEnum) -> str: "synth": StepEnum.SYNTHESIS.value, "synthesis": StepEnum.SYNTHESIS.value, "prefloorplan": StepEnum.PRE_FLOORPLAN.value, + "floorplan": StepEnum.PRE_FLOORPLAN.value, "macroplace": StepEnum.MACRO_PLACEMENT.value, "macroplacement": StepEnum.MACRO_PLACEMENT.value, "postfloorplan": StepEnum.POST_FLOORPLAN.value, diff --git a/chipcompiler/runtime/errors.py b/chipcompiler/runtime/errors.py new file mode 100644 index 00000000..64ace25c --- /dev/null +++ b/chipcompiler/runtime/errors.py @@ -0,0 +1,6 @@ +class RuntimeApiError(RuntimeError): + def __init__(self, code: str, message: str, data: dict | None = None): + super().__init__(message) + self.code = code + self.message = message + self.data = data or {} diff --git a/chipcompiler/runtime/methods.py b/chipcompiler/runtime/methods.py index f17f4e50..340aa8f1 100644 --- a/chipcompiler/runtime/methods.py +++ b/chipcompiler/runtime/methods.py @@ -4,6 +4,7 @@ from chipcompiler.runtime.requests import ( DbEnsureRequest, DbReleaseRequest, + EmptyRequest, FloorplanEditInspectRequest, FloorplanEditRunAutoRequest, FloorplanEditValidateRequest, @@ -17,7 +18,11 @@ OperationIdRequest, OperationStartFlowRequest, OperationStartStepRequest, + ProjectManifestDiscoverRequest, + ProjectManifestLoadRequest, + ProjectManifestMutationRequest, WorkspaceCloseRequest, + WorkspaceConfigurationUpdateRequest, WorkspaceCreateRequest, WorkspaceExportSignoffRequest, WorkspaceIdRequest, @@ -25,7 +30,12 @@ WorkspaceInspectSignoffRequest, WorkspaceOpenRequest, WorkspaceRecoverInterruptedRequest, + WorkspaceSpecOpenRequest, + WorkspaceSpecValidateRequest, + WorkspaceStepConfigurationReadRequest, + WorkspaceStepConfigurationUpdateRequest, WorkspaceSyncConfigRequest, + WorkspaceUpdateRequest, ) RequestT = TypeVar("RequestT") @@ -39,6 +49,31 @@ class RuntimeMethodSpec(Generic[RequestT]): RUNTIME_METHODS: Final[tuple[RuntimeMethodSpec[Any], ...]] = ( + RuntimeMethodSpec( + method_name="workspace_spec.describe", + request_model=EmptyRequest, + handler_name="describe_workspace_spec", + ), + RuntimeMethodSpec( + method_name="workspace_spec.validate", + request_model=WorkspaceSpecValidateRequest, + handler_name="validate_workspace_spec", + ), + RuntimeMethodSpec( + method_name="project.discover", + request_model=ProjectManifestDiscoverRequest, + handler_name="discover_project", + ), + RuntimeMethodSpec( + method_name="project.manifest.load", + request_model=ProjectManifestLoadRequest, + handler_name="load_project_manifest", + ), + RuntimeMethodSpec( + method_name="project.manifest.mutate", + request_model=ProjectManifestMutationRequest, + handler_name="mutate_project_manifest", + ), RuntimeMethodSpec( method_name="workspace.create", request_model=WorkspaceCreateRequest, @@ -49,6 +84,36 @@ class RuntimeMethodSpec(Generic[RequestT]): request_model=WorkspaceOpenRequest, handler_name="open_workspace", ), + RuntimeMethodSpec( + method_name="workspace.binding_requirement", + request_model=WorkspaceSpecOpenRequest, + handler_name="workspace_binding_requirement", + ), + RuntimeMethodSpec( + method_name="workspace.update", + request_model=WorkspaceUpdateRequest, + handler_name="update_workspace", + ), + RuntimeMethodSpec( + method_name="workspace.configuration.update", + request_model=WorkspaceConfigurationUpdateRequest, + handler_name="update_workspace_configuration", + ), + RuntimeMethodSpec( + method_name="workspace.configuration.read", + request_model=WorkspaceOpenRequest, + handler_name="read_workspace_configuration", + ), + RuntimeMethodSpec( + method_name="workspace.step_configuration.update", + request_model=WorkspaceStepConfigurationUpdateRequest, + handler_name="update_workspace_step_configuration", + ), + RuntimeMethodSpec( + method_name="workspace.step_configuration.read", + request_model=WorkspaceStepConfigurationReadRequest, + handler_name="read_workspace_step_configuration", + ), RuntimeMethodSpec( method_name="workspace.close", request_model=WorkspaceCloseRequest, @@ -129,6 +194,11 @@ class RuntimeMethodSpec(Generic[RequestT]): request_model=WorkspaceIdRequest, handler_name="workspace_snapshot", ), + RuntimeMethodSpec( + method_name="workspace.engineering_snapshot", + request_model=WorkspaceIdRequest, + handler_name="engineering_snapshot", + ), RuntimeMethodSpec( method_name="workspace.recover_interrupted", request_model=WorkspaceRecoverInterruptedRequest, @@ -136,6 +206,26 @@ class RuntimeMethodSpec(Generic[RequestT]): ), ) +# These methods were added with the v1 workspace contract. Older embedders may +# provide only the legacy runtime API; expose a stable invalid-request result +# until they upgrade instead of making the whole RPC server unstartable. +OPTIONAL_RUNTIME_METHOD_NAMES: Final[frozenset[str]] = frozenset( + { + "workspace_spec.describe", + "workspace_spec.validate", + "project.discover", + "project.manifest.load", + "project.manifest.mutate", + "workspace.binding_requirement", + "workspace.update", + "workspace.configuration.update", + "workspace.configuration.read", + "workspace.step_configuration.update", + "workspace.step_configuration.read", + "workspace.engineering_snapshot", + } +) + PERSISTENT_DB_METHODS: Final[tuple[RuntimeMethodSpec[Any], ...]] = ( RuntimeMethodSpec( diff --git a/chipcompiler/runtime/operations.py b/chipcompiler/runtime/operations.py index eed9292a..9e29c34c 100644 --- a/chipcompiler/runtime/operations.py +++ b/chipcompiler/runtime/operations.py @@ -14,7 +14,9 @@ _RENDER_ACK_RETRY_SECONDS = 5.0 _RENDER_ACK_PAUSE_SECONDS = 30.0 _RENDER_ACK_ABORT_SECONDS = 300.0 -_TERMINAL_OPERATION_STATES = frozenset({"succeeded", "failed", "cancelled"}) +_TERMINAL_OPERATION_STATES = frozenset({"succeeded", "failed", "cancelled", "interrupted"}) +_MAX_TERMINAL_OPERATIONS = 256 +_LEDGER_SCHEMA_VERSION = 1 @dataclass @@ -84,6 +86,7 @@ def __init__(self, publisher: Callable[[dict[str, Any]], None] | None = None): self._step_log_tails: dict[str, _StepLogTail] = {} self._runtime_instance_id = uuid4().hex self._workspace_sequences: dict[str, int] = {} + self._ledger_paths: dict[str, Path] = {} def set_publisher(self, publisher: Callable[[dict[str, Any]], None] | None) -> None: with self._lock: @@ -99,6 +102,7 @@ def start( step: str, idempotency_key: str, runner: Callable[[RuntimeFlowObserver], dict[str, Any]], + workspace_revision: int = 0, ) -> dict[str, Any]: with self._lock: if idempotency_key: @@ -126,11 +130,13 @@ def start( rerun=rerun, step=step, idempotency_key=idempotency_key, + workspace_revision=workspace_revision, ) self._operations[operation.operation_id] = operation self._active_by_workspace[workspace_id] = operation.operation_id if idempotency_key: self._idempotency[(workspace_id, idempotency_key)] = operation.operation_id + self._persist_workspace_locked(workspace_id) queued_event = self._new_event_locked(operation, "operation.queued", {}) self._publish(queued_event) @@ -155,6 +161,55 @@ def is_active(self, operation_id: str) -> bool: operation = self._operations.get(operation_id) return operation is not None and operation.state not in _TERMINAL_OPERATION_STATES + def has_active_workspace(self, workspace_id: str) -> bool: + with self._lock: + operation_id = self._active_by_workspace.get(workspace_id) + if operation_id is None: + return False + operation = self._operations.get(operation_id) + return operation is not None and operation.state not in _TERMINAL_OPERATION_STATES + + def load_workspace_ledger(self, workspace_id: str, ledger_path: str | Path) -> list[str]: + """Restore the bounded execution ledger after a Runtime restart.""" + from chipcompiler.utility import JsonReadError, json_read_strict + + path = Path(ledger_path).expanduser().resolve() + with self._lock: + self._ledger_paths[workspace_id] = path + try: + payload = json_read_strict(path) + except (OSError, JsonReadError): + return [] + entries = payload.get("operations", []) if isinstance(payload, dict) else payload + if not isinstance(entries, list): + return [] + restored: list[str] = [] + for entry in entries: + if not isinstance(entry, dict) or entry.get("workspaceId") != workspace_id: + continue + operation = _operation_from_payload(entry, self._runtime_instance_id) + if operation is None: + continue + if operation.state not in _TERMINAL_OPERATION_STATES: + operation.state = "interrupted" + operation.error = { + "code": "interrupted", + "message": "Runtime process ended before the Operation completed", + } + operation.updated_at = time.time() + self._operations[operation.operation_id] = operation + if operation.idempotency_key: + self._idempotency[(workspace_id, operation.idempotency_key)] = ( + operation.operation_id + ) + self._workspace_sequences[workspace_id] = max( + self._workspace_sequences.get(workspace_id, 0), operation.sequence + ) + restored.append(operation.operation_id) + self._prune_terminal_locked() + self._persist_workspace_locked(workspace_id) + return restored + def workspace_snapshot(self, workspace_id: str) -> dict[str, Any]: with self._lock: operations = [ @@ -222,6 +277,7 @@ def acknowledge_step_rendered( operation.render_wait_started_at = None operation.last_render_ack_at = time.time() operation.updated_at = time.time() + self._persist_workspace_locked(operation.workspace_id) self._render_gate.notify_all() return { "accepted": True, @@ -239,6 +295,7 @@ def request_cancel(self, operation_id: str) -> dict[str, Any]: return {"accepted": False, "operationId": operation_id, "state": operation.state} operation.cancel_requested = True operation.updated_at = time.time() + self._persist_workspace_locked(operation.workspace_id) event = self._new_event_locked(operation, "operation.cancel_requested", {}) self._render_gate.notify_all() self._publish(event) @@ -286,6 +343,8 @@ def _run( "operation.completed", {"result": result}, ) + self._prune_terminal_locked() + self._persist_workspace_locked(operation.workspace_id) except RuntimeOperationCancelled as exc: with self._lock: operation = self._operations[operation_id] @@ -305,6 +364,8 @@ def _run( event_type, {"error": operation.error}, ) + self._prune_terminal_locked() + self._persist_workspace_locked(operation.workspace_id) except Exception as exc: with self._lock: operation = self._operations[operation_id] @@ -330,6 +391,8 @@ def _run( } ) event = self._new_event_locked(operation, event_type, payload) + self._prune_terminal_locked() + self._persist_workspace_locked(operation.workspace_id) self._publish(event) finally: try: @@ -365,6 +428,7 @@ def step_started(self, operation_id: str, workspace_step: Any) -> None: ) if log_tail is not None: self._step_log_tails[operation_id] = log_tail + self._persist_workspace_locked(operation.workspace_id) self._publish(event) if log_tail is not None: thread = threading.Thread( @@ -445,6 +509,7 @@ def step_completed( operation.render_retry_count = 0 operation.render_wait_started_at = time.monotonic() operation.state = "waiting_for_gui_sync" + self._persist_workspace_locked(operation.workspace_id) self._publish(event) def subflow_stage( @@ -470,6 +535,7 @@ def subflow_stage( "tool": tool, }, ) + self._persist_workspace_locked(operation.workspace_id) self._publish(event) def step_skipped(self, operation_id: str, workspace_step: Any) -> None: @@ -642,6 +708,45 @@ def _new_event_locked( "payload": payload, } + def _prune_terminal_locked(self) -> None: + terminal = [ + operation + for operation in self._operations.values() + if operation.state in _TERMINAL_OPERATION_STATES + ] + if len(terminal) <= _MAX_TERMINAL_OPERATIONS: + return + terminal.sort(key=lambda operation: (operation.updated_at, operation.operation_id)) + removed = terminal[: len(terminal) - _MAX_TERMINAL_OPERATIONS] + removed_ids = {operation.operation_id for operation in removed} + for operation_id in removed_ids: + self._operations.pop(operation_id, None) + self._idempotency = { + key: operation_id + for key, operation_id in self._idempotency.items() + if operation_id not in removed_ids + } + + def _persist_workspace_locked(self, workspace_id: str) -> None: + path = self._ledger_paths.get(workspace_id) + if path is None: + return + from chipcompiler.utility import json_write + + operations = [ + self._operation_payload(operation) + for operation in self._operations.values() + if operation.workspace_id == workspace_id + ] + json_write( + path, + { + "schemaVersion": _LEDGER_SCHEMA_VERSION, + "workspaceId": workspace_id, + "operations": operations, + }, + ) + @staticmethod def _operation_payload(operation: RuntimeOperation) -> dict[str, Any]: return { @@ -653,6 +758,7 @@ def _operation_payload(operation: RuntimeOperation) -> dict[str, Any]: "origin": operation.origin, "rerun": operation.rerun, "step": operation.step, + "idempotencyKey": operation.idempotency_key, "state": operation.state, "currentStep": operation.current_step, "currentTool": operation.current_tool, @@ -670,6 +776,7 @@ def _operation_payload(operation: RuntimeOperation) -> dict[str, Any]: "shutdownBarrier": operation.state not in _TERMINAL_OPERATION_STATES, "createdAt": operation.created_at, "updatedAt": operation.updated_at, + "sequence": operation.sequence, } def _publish(self, event: dict[str, Any]) -> None: @@ -761,3 +868,55 @@ def _step_log_tail_for( tool=tool, cursor=cursor, ) + + +def _operation_from_payload( + payload: dict[str, Any], runtime_instance_id: str +) -> RuntimeOperation | None: + operation_id = payload.get("operationId") + run_session_id = payload.get("runSessionId") + workspace_id = payload.get("workspaceId") + kind = payload.get("kind") + origin = payload.get("origin") + if not all( + isinstance(value, str) and value + for value in (operation_id, run_session_id, workspace_id, kind, origin) + ): + return None + return RuntimeOperation( + operation_id=operation_id, + run_session_id=run_session_id, + runtime_instance_id=runtime_instance_id, + workspace_id=workspace_id, + kind=kind, + origin=origin, + rerun=bool(payload.get("rerun", False)), + step=str(payload.get("step", "")), + idempotency_key=str(payload.get("idempotencyKey", "")), + state=str(payload.get("state", "interrupted")), + current_step=str(payload.get("currentStep", "")), + current_tool=str(payload.get("currentTool", "")), + error=payload.get("error") if isinstance(payload.get("error"), dict) else None, + result=payload.get("result") if isinstance(payload.get("result"), dict) else None, + created_at=_number(payload.get("createdAt")), + updated_at=_number(payload.get("updatedAt")), + sequence=int(payload.get("sequence", 0) or 0), + workspace_revision=int(payload.get("workspaceRevision", 0) or 0), + render_sync_state=str(payload.get("renderSyncState", "idle")), + render_retry_count=int(payload.get("renderRetryCount", 0) or 0), + last_render_ack_at=_optional_number(payload.get("lastRenderAckAt")), + cancel_requested=bool(payload.get("cancelRequested", False)), + interruptibility=str(payload.get("interruptibility", "deferred")), + ) + + +def _number(value: Any) -> float: + return ( + float(value) + if isinstance(value, (int, float)) and not isinstance(value, bool) + else time.time() + ) + + +def _optional_number(value: Any) -> float | None: + return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None diff --git a/chipcompiler/runtime/requests.py b/chipcompiler/runtime/requests.py index 8eb6116d..490388af 100644 --- a/chipcompiler/runtime/requests.py +++ b/chipcompiler/runtime/requests.py @@ -4,7 +4,7 @@ @dataclass(frozen=True) class WorkspaceCreateRequest: - directory: str + directory: str = "" pdk: str = "" pdk_root: str = "" pdk_json: Any = None @@ -15,16 +15,112 @@ class WorkspaceCreateRequest: rtl_list: list[str] | None = None sdc: str = "" flow_config: dict[str, Any] | None = None + command_id: str = "" + target_directory: str = "" + workspace_spec: dict[str, Any] | None = None + workspace_bindings: dict[str, Any] | None = None + project_id: str = "" + project_root: str = "" @dataclass(frozen=True) class WorkspaceOpenRequest: directory: str + workspace_bindings: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class EmptyRequest: + pass + + +@dataclass(frozen=True) +class WorkspaceSpecOpenRequest: + directory: str + workspace_bindings: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class WorkspaceSpecValidateRequest: + workspace_spec: dict[str, Any] + workspace_bindings: dict[str, Any] + + +@dataclass(frozen=True) +class ProjectManifestLoadRequest: + project_root: str + + +@dataclass(frozen=True) +class ProjectManifestDiscoverRequest: + directory: str + + +@dataclass(frozen=True) +class ProjectManifestMutationRequest: + project_root: str + mutation: dict[str, Any] + + +@dataclass(frozen=True) +class WorkspaceSpecCreateRequest: + command_id: str = "" + target_directory: str = "" + workspace_spec: dict[str, Any] | None = None + workspace_bindings: dict[str, Any] | None = None + project_id: str = "" + project_root: str = "" + directory: str = "" + pdk: str = "" + pdk_root: str = "" + pdk_json: Any = None + parameters: dict[str, Any] | None = None + origin_def: str = "" + origin_verilog: str = "" + filelist: str = "" + rtl_list: list[str] | None = None + sdc: str = "" + flow_config: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class WorkspaceUpdateRequest: + command_id: str + workspace_id: str + expected_workspace_revision: int + workspace_spec: dict[str, Any] + workspace_bindings: dict[str, Any] + + +@dataclass(frozen=True) +class WorkspaceConfigurationUpdateRequest: + command_id: str + workspace_id: str + expected_workspace_revision: int + configuration: dict[str, Any] + workspace_bindings: dict[str, Any] + + +@dataclass(frozen=True) +class WorkspaceStepConfigurationUpdateRequest: + command_id: str + workspace_id: str + expected_workspace_revision: int + step_id: str + parameters: dict[str, Any] + + +@dataclass(frozen=True) +class WorkspaceStepConfigurationReadRequest: + step: str + workspace_id: str = "" + directory: str = "" @dataclass(frozen=True) class WorkspaceIdRequest: workspace_id: str + expected_workspace_revision: int = 1 @dataclass(frozen=True) @@ -66,6 +162,7 @@ class WorkspaceInfoRequest: @dataclass(frozen=True) class FlowRunRequest: workspace_id: str + expected_workspace_revision: int = 1 rerun: bool = False @@ -73,12 +170,14 @@ class FlowRunRequest: class FlowRunStepRequest: workspace_id: str step: str + expected_workspace_revision: int = 1 rerun: bool = False @dataclass(frozen=True) class OperationStartFlowRequest: workspace_id: str + expected_workspace_revision: int = 1 rerun: bool = False origin: str = "gui" idempotency_key: str = "" @@ -88,6 +187,7 @@ class OperationStartFlowRequest: class OperationStartStepRequest: workspace_id: str step: str + expected_workspace_revision: int = 1 rerun: bool = False reset_dependents: bool = False origin: str = "gui" @@ -137,6 +237,13 @@ class LayoutEditApplyRequest: class LayoutEditSaveRequest: edit_session_id: str expected_revision: int + expected_workspace_revision: int = 1 + + +@dataclass(frozen=True) +class WorkspaceMutationRequest: + workspace_id: str + expected_workspace_revision: int = 1 @dataclass(frozen=True) @@ -178,6 +285,7 @@ def __init__(self, reason: str): "paramJson": "parameters", "rtlList": "rtl_list", "workspaceId": "workspace_id", + "expectedWorkspaceRevision": "expected_workspace_revision", "operationId": "operation_id", "eventId": "event_id", "stepCommitId": "step_commit_id", @@ -194,6 +302,12 @@ def __init__(self, reason: str): "expectedSourceFingerprint": "expected_source_fingerprint", "id": "info_id", "additionalFiles": "additional_files", + "workspaceSpec": "workspace_spec", + "workspaceBindings": "workspace_bindings", + "targetDirectory": "target_directory", + "projectId": "project_id", + "projectRoot": "project_root", + "stepId": "step_id", } diff --git a/chipcompiler/runtime/server.py b/chipcompiler/runtime/server.py index 70167ea3..98f35f91 100644 --- a/chipcompiler/runtime/server.py +++ b/chipcompiler/runtime/server.py @@ -60,7 +60,7 @@ def set_notification_sink(self, sink: Callable[[str, dict], None] | None) -> Non def _publish_runtime_event(self, event: dict) -> None: sink = self._notification_sink if sink is not None: - sink("runtime.event", event) + sink("runtime.event", _project_runtime_event(event)) def _register_base_methods(self) -> None: self.dispatcher.add_method("rpc.hello", self._hello) @@ -76,6 +76,7 @@ def _hello(self, version: int): ) return { "version": PROTOCOL_VERSION, + "protocolVersion": PROTOCOL_VERSION, "eccVersion": getattr(chipcompiler, "__version__", "unknown"), "capabilities": list(self.capabilities), } @@ -101,9 +102,13 @@ def _register_runtime_methods(self) -> None: ): api_method = getattr(self.api, spec.handler_name, None) if not callable(api_method): - raise TypeError( - f"runtime method {spec.method_name} handler {spec.handler_name} is not callable" + if spec.method_name not in methods.OPTIONAL_RUNTIME_METHOD_NAMES: + raise TypeError(f"runtime API handler is not callable: {spec.handler_name}") + self.dispatcher.add_method( + spec.method_name, + self._missing_optional_method_handler(spec), ) + continue self.dispatcher.add_method( spec.method_name, self._runtime_method_handler(spec, api_method), @@ -129,6 +134,16 @@ def handler(**params): {"message": exc.message, **exc.data}, ) except Exception as exc: + stable_code = getattr(exc, "code", None) + if isinstance(stable_code, str): + details = getattr(exc, "details", None) + if not isinstance(details, dict): + details = getattr(exc, "data", None) + return Error( + ERROR_CODES.get(stable_code, -32000), + stable_code, + {"message": str(exc), **(details if isinstance(details, dict) else {})}, + ) return Error( ERROR_CODES["command_failed"], "command_failed", @@ -136,3 +151,50 @@ def handler(**params): ) return handler + + @staticmethod + def _missing_optional_method_handler(spec): + def handler(**_params): + return Error( + -32602, + "invalid_request", + {"message": f"runtime API does not support {spec.method_name}"}, + ) + + return handler + + +def _project_runtime_event(event: dict) -> dict: + source_type = str(event.get("type", "")) + payload = {**event.get("payload", {}), "sourceType": source_type} + if source_type == "step.completed": + event_type = "workspace.committed" + elif source_type in { + "step.started", + "step.log", + "subflow.stage", + "operation.rerun_prepared", + }: + event_type = "execution.progress" + else: + event_type = "operation.changed" + state = { + "operation.queued": "queued", + "operation.started": "running", + "operation.completed": "succeeded", + "operation.failed": "failed", + "operation.cancelled": "cancelled", + "operation.interrupted": "interrupted", + }.get(source_type) + if state: + payload["state"] = state + return { + **event, + "type": event_type, + "payload": payload, + **( + {"workspaceRevision": payload["workspaceRevision"]} + if "workspaceRevision" in payload + else {} + ), + } diff --git a/chipcompiler/runtime/sessions.py b/chipcompiler/runtime/sessions.py index c975ac10..1bfeda45 100644 --- a/chipcompiler/runtime/sessions.py +++ b/chipcompiler/runtime/sessions.py @@ -11,6 +11,9 @@ class WorkspaceSession: workspace_id: str directory: Path workspace: Any + workspace_revision: int = 0 + execution_readiness: dict[str, Any] = field(default_factory=lambda: {"ready": True}) + workspace_bindings: dict[str, Any] | None = None db_handle: Any = None layout_edit_session: "LayoutEditSession | None" = None mutation_lock: threading.Lock = field(default_factory=threading.Lock) @@ -38,6 +41,7 @@ class LayoutEditSession: requires_verilog: bool = False used_floorplan_editor: bool = False validation_diagnostics: list[dict[str, Any]] = field(default_factory=list) + ownership_lock: Any = None class WorkspaceSessionNotFound(KeyError): @@ -58,22 +62,54 @@ def __init__(self, db_releaser: Callable[[Any], None] | None = _close_db_handle) self._db_releaser = db_releaser self._lock = threading.Lock() - def create_session(self, directory: str | Path, *, workspace: Any) -> WorkspaceSession: + def create_session( + self, + directory: str | Path, + *, + workspace: Any, + workspace_id: str | None = None, + workspace_revision: int = 0, + execution_readiness: dict[str, Any] | None = None, + workspace_bindings: dict[str, Any] | None = None, + ) -> WorkspaceSession: resolved_directory = Path(directory).resolve() with self._lock: existing_id = self._sessions_by_directory.get(resolved_directory) if existing_id is not None: self._remove_session(existing_id) - return self._create_session(resolved_directory, workspace=workspace) - - def open_session(self, directory: str | Path, *, workspace: Any) -> WorkspaceSession: + return self._create_session( + resolved_directory, + workspace=workspace, + workspace_id=workspace_id, + workspace_revision=workspace_revision, + execution_readiness=execution_readiness, + workspace_bindings=workspace_bindings, + ) + + def open_session( + self, + directory: str | Path, + *, + workspace: Any, + workspace_id: str | None = None, + workspace_revision: int = 0, + execution_readiness: dict[str, Any] | None = None, + workspace_bindings: dict[str, Any] | None = None, + ) -> WorkspaceSession: resolved_directory = Path(directory).resolve() with self._lock: existing_id = self._sessions_by_directory.get(resolved_directory) if existing_id is not None: return self._sessions[existing_id] - return self._create_session(resolved_directory, workspace=workspace) + return self._create_session( + resolved_directory, + workspace=workspace, + workspace_id=workspace_id, + workspace_revision=workspace_revision, + execution_readiness=execution_readiness, + workspace_bindings=workspace_bindings, + ) def get_session(self, workspace_id: str) -> WorkspaceSession: try: @@ -96,13 +132,26 @@ def close_all(self) -> None: def release_session_db(self, session: WorkspaceSession) -> bool: return self._release_session_db(session) - def _create_session(self, directory: Path, *, workspace: Any) -> WorkspaceSession: - workspace_id = f"workspace-{self._next_id}" - self._next_id += 1 + def _create_session( + self, + directory: Path, + *, + workspace: Any, + workspace_id: str | None, + workspace_revision: int, + execution_readiness: dict[str, Any] | None, + workspace_bindings: dict[str, Any] | None, + ) -> WorkspaceSession: + if workspace_id is None: + workspace_id = f"workspace-{self._next_id}" + self._next_id += 1 session = WorkspaceSession( workspace_id=workspace_id, directory=directory, workspace=workspace, + workspace_revision=workspace_revision, + execution_readiness=execution_readiness or {"ready": True}, + workspace_bindings=workspace_bindings, ) self._sessions[workspace_id] = session self._sessions_by_directory[directory] = workspace_id diff --git a/chipcompiler/runtime/workspace_api.py b/chipcompiler/runtime/workspace_api.py index 5c256c22..5e23513b 100644 --- a/chipcompiler/runtime/workspace_api.py +++ b/chipcompiler/runtime/workspace_api.py @@ -6,11 +6,18 @@ import tempfile import threading from collections.abc import Callable +from contextlib import contextmanager from copy import copy, deepcopy from dataclasses import replace from pathlib import Path from typing import Any, TypeVar +try: + import fcntl +except ImportError: # pragma: no cover - Windows uses the in-process lock. + fcntl = None + +from chipcompiler.runtime.errors import RuntimeApiError from chipcompiler.runtime.operations import ( RuntimeOperationConflict, RuntimeOperationManager, @@ -38,6 +45,9 @@ WorkspaceInspectSignoffRequest, WorkspaceOpenRequest, WorkspaceRecoverInterruptedRequest, + WorkspaceSpecCreateRequest, + WorkspaceSpecOpenRequest, + WorkspaceStepConfigurationReadRequest, WorkspaceSyncConfigRequest, ) from chipcompiler.runtime.sessions import ( @@ -56,15 +66,10 @@ _T = TypeVar("_T") -class RuntimeApiError(RuntimeError): - def __init__(self, code: str, message: str, data: dict | None = None): - super().__init__(message) - self.code = code - self.message = message - self.data = data or {} +from chipcompiler.runtime.workspace_spec_api import WorkspaceSpecRuntimeMixin # noqa: E402 -class WorkspaceRuntimeApi: +class WorkspaceRuntimeApi(WorkspaceSpecRuntimeMixin): def __init__( self, sessions: WorkspaceSessionRegistry | None = None, @@ -86,7 +91,40 @@ def __init__( def set_event_publisher(self, publisher: Callable[[dict[str, Any]], None] | None) -> None: self.operations.set_publisher(publisher) - def create_workspace(self, request: WorkspaceCreateRequest) -> dict: + def create_workspace( + self, request: WorkspaceCreateRequest | WorkspaceSpecCreateRequest + ) -> dict: + if getattr(request, "workspace_spec", None) is not None: + spec_request = ( + request + if isinstance(request, WorkspaceSpecCreateRequest) + else WorkspaceSpecCreateRequest( + command_id=request.command_id, + target_directory=request.target_directory, + workspace_spec=request.workspace_spec, + workspace_bindings=request.workspace_bindings or {}, + project_id=request.project_id, + project_root=request.project_root, + ) + ) + return self._create_workspace_from_spec(spec_request) + if isinstance(request, WorkspaceSpecCreateRequest): + request = WorkspaceCreateRequest( + directory=request.directory, + pdk=request.pdk, + pdk_root=request.pdk_root, + pdk_json=request.pdk_json, + parameters=request.parameters, + origin_def=request.origin_def, + origin_verilog=request.origin_verilog, + filelist=request.filelist, + rtl_list=request.rtl_list, + sdc=request.sdc, + flow_config=request.flow_config, + ) + return self._create_legacy_workspace(request) + + def _create_legacy_workspace(self, request: WorkspaceCreateRequest) -> dict: if not request.directory: raise RuntimeApiError("invalid_request", "missing required field: directory") @@ -127,24 +165,52 @@ def create_workspace(self, request: WorkspaceCreateRequest) -> dict: build_flow_for_workspace(workspace) session = self.sessions.create_session(workspace.directory, workspace=workspace) + self.operations.load_workspace_ledger( + session.workspace_id, + session.directory / "home" / "runtime-commands.json", + ) return _workspace_session_result(session) - def open_workspace(self, request: WorkspaceOpenRequest) -> dict: + def open_workspace(self, request: WorkspaceOpenRequest | WorkspaceSpecOpenRequest) -> dict: + if isinstance(request, WorkspaceSpecOpenRequest) or request.workspace_bindings is not None: + spec_request = ( + request + if isinstance(request, WorkspaceSpecOpenRequest) + else WorkspaceSpecOpenRequest( + directory=request.directory, + workspace_bindings=request.workspace_bindings, + ) + ) + return WorkspaceSpecRuntimeMixin.open_workspace(self, spec_request) + return self._open_legacy_workspace(request) + + def _open_legacy_workspace(self, request: WorkspaceOpenRequest) -> dict: workspace = self._load_workspace(request.directory) build_flow_for_workspace(workspace, create_step_workspaces=False) session = self.sessions.open_session(workspace.directory, workspace=workspace) + self.operations.load_workspace_ledger( + session.workspace_id, + session.directory / "home" / "runtime-commands.json", + ) return _workspace_session_result(session) def recover_interrupted(self, request: WorkspaceRecoverInterruptedRequest) -> dict: from chipcompiler.runtime.recovery import recover_interrupted_operation - return self._with_session_mutation_lock( - request.workspace_id, - lambda session: recover_interrupted_operation( + def recover(session: WorkspaceSession) -> dict: + result = recover_interrupted_operation( session.workspace, self.operations, request.operation_id, - ), + ) + snapshot_path = Path(session.workspace.directory) / "home" / "engineering-snapshot.json" + if result["recovered"] and snapshot_path.is_file(): + self._commit_workspace_snapshot(session, "operation.recovered") + return result + + return self._with_session_mutation_lock( + request.workspace_id, + recover, ) def workspace_home(self, request: WorkspaceIdRequest) -> dict: @@ -173,6 +239,47 @@ def workspace_info(self, request: WorkspaceInfoRequest) -> dict: "info": stringify_paths(info or {}), } + def read_workspace_step_configuration( + self, request: WorkspaceStepConfigurationReadRequest + ) -> dict: + if bool(request.workspace_id) == bool(request.directory): + raise RuntimeApiError( + "invalid_request", + "Exactly one of workspaceId or directory is required", + ) + + from chipcompiler.engine import ( + WorkspaceLifecycleError, + read_step_configuration, + read_step_configuration_from_directory, + ) + + try: + if request.directory: + result = read_step_configuration_from_directory(request.directory, request.step) + else: + session = self._get_session(request.workspace_id) + result = read_step_configuration(session.workspace, request.step) + except WorkspaceLifecycleError as exc: + if exc.code == "step_configuration_unavailable": + return { + "status": "unavailable", + "step": request.step, + "reason": exc.code, + **exc.details, + **( + { + "workspaceId": session.workspace_id, + "workspaceRevision": session.workspace_revision, + } + if request.workspace_id + else {} + ), + } + raise RuntimeApiError(exc.code, str(exc), exc.details) from exc + + return {"status": "available", **result} + def refresh_config(self, request: WorkspaceIdRequest) -> dict: def refresh(session: WorkspaceSession) -> dict: self._release_session_db(session) @@ -262,6 +369,7 @@ def _flow_run( preserve_user_inputs: bool = False, ) -> dict: def run(session: WorkspaceSession) -> dict: + self._validate_workspace_revision(session, request.expected_workspace_revision) should_capture = self._should_capture_session_db(session) previous_db = session.db_handle if should_capture else None if request.rerun and should_capture: @@ -285,7 +393,12 @@ def run(session: WorkspaceSession) -> dict: scope="flow", ) try: - ok = _run_engine_flow_steps(engine_flow, rerun=request.rerun, observer=observer) + ok = _run_engine_flow_steps( + engine_flow, + rerun=request.rerun, + observer=observer, + session=session, + ) finally: if should_capture: self._capture_flow_db( @@ -316,6 +429,7 @@ def _flow_run_step( reset_dependents: bool = False, ) -> dict: def run_step(session: WorkspaceSession) -> dict: + self._validate_workspace_revision(session, request.expected_workspace_revision) should_capture = self._should_capture_session_db(session) previous_db = session.db_handle if should_capture else None if request.rerun and should_capture: @@ -368,6 +482,7 @@ def run_step(session: WorkspaceSession) -> dict: workspace_step, rerun=request.rerun, observer=observer, + session=session, ) finally: if should_capture: @@ -393,7 +508,8 @@ def run_step(session: WorkspaceSession) -> dict: def start_flow_operation(self, request: OperationStartFlowRequest) -> dict: self._require_gui_operation_origin(request.origin) - self._get_session(request.workspace_id) + session = self._get_session(request.workspace_id) + self._validate_workspace_revision(session, request.expected_workspace_revision) try: return self.operations.start( workspace_id=request.workspace_id, @@ -402,18 +518,24 @@ def start_flow_operation(self, request: OperationStartFlowRequest) -> dict: rerun=request.rerun, step="", idempotency_key=request.idempotency_key, + workspace_revision=request.expected_workspace_revision, runner=lambda observer: self._flow_run( - FlowRunRequest(workspace_id=request.workspace_id, rerun=request.rerun), + FlowRunRequest( + workspace_id=request.workspace_id, + expected_workspace_revision=request.expected_workspace_revision, + rerun=request.rerun, + ), observer=observer, preserve_user_inputs=request.rerun, ), ) except RuntimeOperationConflict as exc: - raise RuntimeApiError("command_failed", str(exc)) from exc + raise RuntimeApiError("operation_conflict", str(exc)) from exc def start_step_operation(self, request: OperationStartStepRequest) -> dict: self._require_gui_operation_origin(request.origin) - self._get_session(request.workspace_id) + session = self._get_session(request.workspace_id) + self._validate_workspace_revision(session, request.expected_workspace_revision) try: return self.operations.start( workspace_id=request.workspace_id, @@ -422,10 +544,12 @@ def start_step_operation(self, request: OperationStartStepRequest) -> dict: rerun=request.rerun, step=request.step, idempotency_key=request.idempotency_key, + workspace_revision=request.expected_workspace_revision, runner=lambda observer: self._flow_run_step( FlowRunStepRequest( workspace_id=request.workspace_id, step=request.step, + expected_workspace_revision=request.expected_workspace_revision, rerun=request.rerun, ), observer=observer, @@ -433,7 +557,7 @@ def start_step_operation(self, request: OperationStartStepRequest) -> dict: ), ) except RuntimeOperationConflict as exc: - raise RuntimeApiError("command_failed", str(exc)) from exc + raise RuntimeApiError("operation_conflict", str(exc)) from exc def operation_status(self, request: OperationIdRequest) -> dict: try: @@ -512,6 +636,20 @@ def workspace_snapshot(self, request: WorkspaceIdRequest) -> dict: "parameters": stringify_paths(deepcopy(parameters_data)), } + def engineering_snapshot(self, request: WorkspaceIdRequest) -> dict: + """Read the committed Snapshot for the Phase 1 wire contract.""" + session = self._get_session(request.workspace_id) + from chipcompiler.engine.snapshot import EngineeringSnapshotError, read_engineering_snapshot + + try: + return read_engineering_snapshot(session.workspace) + except EngineeringSnapshotError as exc: + raise RuntimeApiError( + "engineering_snapshot_unavailable", + str(exc), + {"workspaceId": request.workspace_id}, + ) from exc + def db_ensure(self, request: DbEnsureRequest) -> dict: self._require_persistent_db() @@ -563,6 +701,11 @@ def release(session: WorkspaceSession) -> dict: def layout_edit_begin(self, request: LayoutEditBeginRequest) -> dict: self._require_persistent_db() + session = self._get_session(request.workspace_id) + ownership_lock = None + if session.layout_edit_session is None: + ownership_lock = _workspace_ownership_lock(session.directory) + ownership_lock.__enter__() def begin(session: WorkspaceSession) -> dict: with self._layout_edit_lock: @@ -669,7 +812,19 @@ def begin(session: WorkspaceSession) -> dict: self._layout_edit_sessions[edit_session.edit_session_id] = edit_session return _layout_edit_begin_result(edit_session, reused=False) - return self._with_session_mutation_lock(request.workspace_id, begin) + try: + result = self._with_session_mutation_lock(request.workspace_id, begin) + except BaseException as exc: + if ownership_lock is not None: + ownership_lock.__exit__(type(exc), exc, exc.__traceback__) + raise + if ownership_lock is not None: + edit_session = session.layout_edit_session + if edit_session is not None and result.get("reused") is not True: + edit_session.ownership_lock = ownership_lock + else: + ownership_lock.__exit__(None, None, None) + return result def layout_edit_apply(self, request: LayoutEditApplyRequest) -> dict: def apply(session: WorkspaceSession, edit_session: LayoutEditSession) -> dict: @@ -744,7 +899,9 @@ def save(session: WorkspaceSession, edit_session: LayoutEditSession) -> dict: }, ) if not edit_session.dirty: - return _layout_edit_save_result(edit_session, saved=False) + result = _layout_edit_save_result(edit_session, saved=False) + _release_layout_edit_ownership_lock(edit_session) + return result current_fingerprint = _artifact_fingerprint(edit_session.source_paths) if current_fingerprint != edit_session.source_fingerprint: @@ -780,8 +937,19 @@ def save(session: WorkspaceSession, edit_session: LayoutEditSession) -> dict: edit_session.source_kind = "db" edit_session.source_paths = (output_db,) edit_session.source_fingerprint = _artifact_fingerprint(edit_session.source_paths) + workspace_revision = None + snapshot_path = Path(session.workspace.directory) / "home" / "engineering-snapshot.json" + if snapshot_path.is_file(): + workspace_revision = WorkspaceSpecRuntimeMixin._commit_workspace_snapshot( + session, + "layout.edit.save", + ) edit_session.dirty = False - return _layout_edit_save_result(edit_session, saved=True, artifacts=artifacts) + result = _layout_edit_save_result(edit_session, saved=True, artifacts=artifacts) + if workspace_revision is not None: + result["workspaceRevision"] = workspace_revision + _release_layout_edit_ownership_lock(edit_session) + return result return self._with_layout_edit_session_mutation_lock(request.edit_session_id, save) @@ -862,7 +1030,9 @@ def _discard_layout_edit_session(self, session: WorkspaceSession) -> bool: if edit_session is None: return False self._layout_edit_sessions.pop(edit_session.edit_session_id, None) - return self.sessions.release_layout_edit_session(session) + released = self.sessions.release_layout_edit_session(session) + _release_layout_edit_ownership_lock(edit_session) + return released def _release_session_db(self, session: WorkspaceSession) -> bool: return self.sessions.release_session_db(session) @@ -906,10 +1076,43 @@ def _with_session_mutation_lock( self, workspace_id: str, operation: Callable[[WorkspaceSession], _T], + *, + reject_active_operation: bool = False, ) -> _T: session = self._get_session(workspace_id) + if reject_active_operation: + self._ensure_no_active_operation(session) with session.mutation_lock: - return operation(session) + if reject_active_operation: + self._ensure_no_active_operation(session) + with _workspace_ownership_lock(session.directory): + return operation(session) + + @staticmethod + def _validate_workspace_revision( + session: WorkspaceSession, + expected_workspace_revision: int, + ) -> None: + if ( + session.workspace_revision == 0 + or expected_workspace_revision == session.workspace_revision + ): + return + raise RuntimeApiError( + "revision_conflict", + "Workspace Revision does not match", + { + "expectedRevision": expected_workspace_revision, + "actualRevision": session.workspace_revision, + }, + ) + + def _ensure_no_active_operation(self, session: WorkspaceSession) -> None: + if self.operations.has_active_workspace(session.workspace_id): + raise RuntimeApiError( + "operation_conflict", + "Workspace has an active Operation", + ) def _refresh_workspace_config(self, workspace) -> None: import chipcompiler.data as data_api @@ -1113,6 +1316,32 @@ def _validate_step_artifact_dir( return resolved +@contextmanager +def _workspace_ownership_lock(directory: Path): + """Acquire the CLI-compatible sibling lock without waiting in Runtime.""" + if fcntl is None: + yield + return + from chipcompiler.utility.workspace_lock import workspace_lock + + try: + with workspace_lock(directory, blocking=False): + yield + except BlockingIOError as exc: + raise RuntimeApiError( + "workspace_busy", + f"Workspace is busy: {directory}", + {"directory": str(directory)}, + ) from exc + + +def _release_layout_edit_ownership_lock(edit_session: LayoutEditSession) -> None: + ownership_lock = edit_session.ownership_lock + edit_session.ownership_lock = None + if ownership_lock is not None: + ownership_lock.__exit__(None, None, None) + + def _layout_edit_begin_result(edit_session: LayoutEditSession, *, reused: bool) -> dict: return { "editSessionId": edit_session.edit_session_id, @@ -2231,26 +2460,57 @@ def _state_value(state: Any) -> str: return getattr(state, "value", str(state)) -def _run_engine_flow_steps(engine_flow, *, rerun: bool, observer) -> bool: - run_steps = engine_flow.run_steps - if observer is not None and _callable_accepts_keyword(run_steps, "observer"): - return run_steps(rerun=rerun, observer=observer) - return run_steps(rerun=rerun) +def _run_engine_flow_steps(engine_flow, *, rerun: bool, observer, session) -> bool: + from chipcompiler.engine import ExecutionPlan, execute + result = execute( + engine_flow, + ExecutionPlan(intent="rerun" if rerun else "run"), + event_sink=_snapshot_observer(engine_flow.workspace, session, observer), + ) + return result.succeeded -def _run_engine_flow_step(engine_flow, workspace_step, *, rerun: bool, observer): - run_step = engine_flow.run_step - if observer is not None and _callable_accepts_keyword(run_step, "observer"): - return run_step(workspace_step, rerun=rerun, observer=observer) - return run_step(workspace_step, rerun=rerun) +def _run_engine_flow_step(engine_flow, workspace_step, *, rerun: bool, observer, session): + from chipcompiler.engine import ExecutionPlan, execute -def _callable_accepts_keyword(callback, keyword: str) -> bool: - try: - parameters = inspect.signature(callback).parameters.values() - except (TypeError, ValueError): - return False - return any( - parameter.name == keyword or parameter.kind == inspect.Parameter.VAR_KEYWORD - for parameter in parameters + result = execute( + engine_flow, + ExecutionPlan( + intent="rerun" if rerun else "run", + step_id=str(getattr(workspace_step, "name", "")), + ), + event_sink=_snapshot_observer(engine_flow.workspace, session, observer), ) + return _success_state() if result.succeeded else result.state + + +def _snapshot_observer(workspace, session, observer): + if observer is None: + return None + return _RuntimeSnapshotObserver(workspace, session, observer) + + +class _RuntimeSnapshotObserver: + fatal_observer = True + + def __init__(self, workspace, session, delegate): + self._workspace = workspace + self._session = session + self._delegate = delegate + + def on_step_completed(self, step, state, error=None): + from chipcompiler.engine.snapshot import commit_engineering_snapshot + + snapshot = commit_engineering_snapshot( + self._workspace, + workspace_id=self._session.workspace_id, + cause=f"flow_step.{_state_value(state).lower()}", + ) + self._session.workspace_revision = snapshot["workspaceRevision"] + callback = getattr(self._delegate, "on_step_completed", None) + if callable(callback): + callback(step, state, error) + + def __getattr__(self, name): + return getattr(self._delegate, name) diff --git a/chipcompiler/runtime/workspace_spec_api.py b/chipcompiler/runtime/workspace_spec_api.py new file mode 100644 index 00000000..9f5a7fbd --- /dev/null +++ b/chipcompiler/runtime/workspace_spec_api.py @@ -0,0 +1,481 @@ +import os +from datetime import UTC, datetime +from pathlib import Path + +from chipcompiler.runtime.errors import RuntimeApiError +from chipcompiler.runtime.requests import ( + ProjectManifestDiscoverRequest, + ProjectManifestLoadRequest, + ProjectManifestMutationRequest, + WorkspaceConfigurationUpdateRequest, + WorkspaceOpenRequest, + WorkspaceSpecCreateRequest, + WorkspaceSpecOpenRequest, + WorkspaceSpecValidateRequest, + WorkspaceStepConfigurationUpdateRequest, + WorkspaceUpdateRequest, +) +from chipcompiler.runtime.sessions import ( + WorkspaceSession, + WorkspaceSessionNotFound, +) + + +class WorkspaceSpecRuntimeMixin: + def describe_workspace_spec(self, _request) -> dict: + from chipcompiler.engine import describe_workspace_spec + + return describe_workspace_spec() + + def validate_workspace_spec(self, request: WorkspaceSpecValidateRequest) -> dict: + from chipcompiler.engine import validate_workspace_spec + + return validate_workspace_spec(request.workspace_spec, request.workspace_bindings) + + def workspace_binding_requirement(self, request: WorkspaceSpecOpenRequest) -> dict: + from chipcompiler.engine import describe_workspace_binding_requirement + + try: + return describe_workspace_binding_requirement(request.directory) + except (OSError, ValueError) as exc: + raise RuntimeApiError("workspace_descriptor_invalid", str(exc)) from exc + + def read_workspace_configuration(self, request: WorkspaceOpenRequest) -> dict: + from chipcompiler.engine import ( + WorkspaceLifecycleError, + read_workspace_configuration_from_directory, + ) + + try: + return read_workspace_configuration_from_directory(request.directory) + except WorkspaceLifecycleError as exc: + raise RuntimeApiError(exc.code, str(exc), exc.details) from exc + + def discover_project(self, request: ProjectManifestDiscoverRequest) -> dict | None: + from chipcompiler.project import discover_project_manifest + + try: + discovered = discover_project_manifest(request.directory) + except (OSError, ValueError) as exc: + raise RuntimeApiError("project_manifest_invalid", str(exc)) from exc + if discovered is None: + return None + project_root, manifest = discovered + return { + "projectRoot": str(project_root), + "projectId": manifest["project_id"], + } + + def load_project_manifest(self, request: ProjectManifestLoadRequest) -> dict: + from chipcompiler.project import load_project_manifest + + try: + return load_project_manifest(request.project_root) + except (OSError, ValueError) as exc: + raise RuntimeApiError("project_manifest_invalid", str(exc)) from exc + + def mutate_project_manifest(self, request: ProjectManifestMutationRequest) -> dict: + from chipcompiler.project import ( + create_project_manifest, + mutate_project_manifest, + ) + + mutation = request.mutation + kind = mutation.get("type") + if kind == "create": + try: + return create_project_manifest( + request.project_root, + str(mutation.get("name") or "project"), + str(mutation.get("designName") or ""), + mpc=(mutation.get("mpc") if isinstance(mutation.get("mpc"), dict) else None), + ) + except (OSError, ValueError) as exc: + raise RuntimeApiError("project_manifest_invalid", str(exc)) from exc + now = str(mutation.get("now") or datetime.now(UTC).isoformat()) + source = mutation.get("input") if isinstance(mutation.get("input"), dict) else mutation + translated = { + "type": str(kind).replace("-", "_"), + **{ + target: source[key] + for key, target in ( + ("workspaceId", "workspace_id"), + ("workspacePath", "workspace_path"), + ("sourceWorkspaceId", "source_workspace_id"), + ("startStep", "start_step"), + ("endStep", "end_step"), + ("lifecycle", "lifecycle"), + ("name", "name"), + ("reason", "reason"), + ) + if key in source + }, + "updated_at": now, + } + if source.get("sourceStep"): + translated["branch_from"] = { + "source_workspace_id": str(source.get("sourceWorkspaceId") or ""), + "source_step": str(source["sourceStep"]), + **( + {"source_output_path": str(source["sourceOutputPath"])} + if source.get("sourceOutputPath") + else {} + ), + **( + {"source_output_type": str(source["sourceOutputType"])} + if source.get("sourceOutputType") + else {} + ), + } + if translated["type"] == "register_workspace": + workspace_path = str(translated.get("workspace_path") or "") + workspace_id = str(translated.get("workspace_id") or Path(workspace_path).name) + translated.update( + { + "workspace_id": workspace_id, + "name": str(translated.get("name") or workspace_id), + "created_at": now, + } + ) + try: + return mutate_project_manifest(request.project_root, translated) + except (OSError, ValueError) as exc: + raise RuntimeApiError("project_manifest_invalid", str(exc)) from exc + + def create_workspace( + self, + request: WorkspaceSpecCreateRequest, + ) -> dict: + return self._create_workspace_from_spec(request) + + def _create_workspace_from_spec(self, request: WorkspaceSpecCreateRequest) -> dict: + from chipcompiler.engine import ( + WorkspaceLifecycleError, + create_workspace_from_spec, + ) + + try: + if request.project_root: + from chipcompiler.project import create_project_workspace + + workspace = create_project_workspace( + request.project_root, + request.target_directory, + request.workspace_spec, + request.workspace_bindings, + command_id=request.command_id, + expected_project_id=request.project_id or None, + ) + else: + workspace = create_workspace_from_spec( + request.target_directory, + request.workspace_spec, + request.workspace_bindings, + request.command_id, + ) + except WorkspaceLifecycleError as exc: + raise RuntimeApiError(exc.code, str(exc), exc.details) from exc + except (OSError, ValueError) as exc: + raise RuntimeApiError("project_manifest_invalid", str(exc)) from exc + snapshot = self._read_engineering_snapshot(workspace) + session = self.sessions.create_session( + workspace.directory, + workspace=workspace, + workspace_id=snapshot["workspaceId"], + workspace_revision=snapshot["workspaceRevision"], + execution_readiness={"ready": True}, + workspace_bindings=request.workspace_bindings, + ) + self.operations.load_workspace_ledger( + session.workspace_id, + session.directory / "home" / "runtime-commands.json", + ) + return _workspace_session_result(session) + + def open_workspace( + self, + request: WorkspaceOpenRequest | WorkspaceSpecOpenRequest, + ) -> dict: + from chipcompiler.runtime.workspace_api import _workspace_ownership_lock + + with _workspace_ownership_lock(Path(request.directory)): + return self._open_workspace_unlocked(request) + + def _open_workspace_unlocked( + self, + request: WorkspaceOpenRequest | WorkspaceSpecOpenRequest, + ) -> dict: + workspace = self._load_workspace(request.directory) + snapshot = self._ensure_engineering_snapshot(workspace) + bindings = ( + request.workspace_bindings if isinstance(request, WorkspaceSpecOpenRequest) else None + ) + if isinstance(request, WorkspaceSpecOpenRequest): + from chipcompiler.engine import assess_execution_readiness + + readiness = assess_execution_readiness(workspace.directory, bindings) + if readiness.get("ready") is True: + from chipcompiler.engine import apply_workspace_bindings + + apply_workspace_bindings(workspace, bindings) + else: + readiness = {"ready": True} + + from chipcompiler.runtime.workspace_api import build_flow_for_workspace + + build_flow_for_workspace(workspace, create_step_workspaces=False) + session = self.sessions.open_session( + workspace.directory, + workspace=workspace, + workspace_id=snapshot["workspaceId"], + workspace_revision=snapshot["workspaceRevision"], + execution_readiness=readiness, + workspace_bindings=bindings, + ) + session.execution_readiness = readiness + session.workspace_bindings = bindings + self.operations.load_workspace_ledger( + session.workspace_id, + session.directory / "home" / "runtime-commands.json", + ) + return _workspace_session_result(session) + + def update_workspace(self, request: WorkspaceUpdateRequest) -> dict: + from chipcompiler.engine import ( + WorkspaceLifecycleError, + update_workspace_from_spec, + ) + + def update(session: WorkspaceSession) -> dict: + self._validate_workspace_revision( + session, + request.expected_workspace_revision, + ) + self._release_session_db(session) + try: + workspace = update_workspace_from_spec( + session.directory, + request.expected_workspace_revision, + request.workspace_spec, + request.workspace_bindings, + request.command_id, + ) + except WorkspaceLifecycleError as exc: + raise RuntimeApiError(exc.code, str(exc), exc.details) from exc + except (OSError, ValueError) as exc: + raise RuntimeApiError("workspace_update_failed", str(exc)) from exc + snapshot = self._read_engineering_snapshot(workspace) + session.workspace = workspace + session.workspace_revision = snapshot["workspaceRevision"] + session.workspace_bindings = request.workspace_bindings + session.execution_readiness = {"ready": True} + return _workspace_session_result(session) + + return self._with_session_mutation_lock( + request.workspace_id, + update, + reject_active_operation=True, + ) + + def update_workspace_configuration(self, request: WorkspaceConfigurationUpdateRequest) -> dict: + from chipcompiler.engine import ( + WorkspaceLifecycleError, + update_workspace_configuration, + ) + + def update(session: WorkspaceSession) -> dict: + self._validate_workspace_revision( + session, + request.expected_workspace_revision, + ) + self._release_session_db(session) + try: + workspace = update_workspace_configuration( + session.directory, + request.expected_workspace_revision, + request.configuration, + request.workspace_bindings, + request.command_id, + ) + except WorkspaceLifecycleError as exc: + raise RuntimeApiError(exc.code, str(exc), exc.details) from exc + except (OSError, ValueError) as exc: + raise RuntimeApiError("workspace_configuration_update_failed", str(exc)) from exc + snapshot = self._read_engineering_snapshot(workspace) + session.workspace = workspace + session.workspace_revision = snapshot["workspaceRevision"] + session.workspace_bindings = request.workspace_bindings + session.execution_readiness = {"ready": True} + return _workspace_session_result(session) + + return self._with_session_mutation_lock( + request.workspace_id, + update, + reject_active_operation=True, + ) + + def update_workspace_step_configuration( + self, request: WorkspaceStepConfigurationUpdateRequest + ) -> dict: + from chipcompiler.engine import ( + WorkspaceLifecycleError, + update_workspace_step_configuration, + ) + + def update(session: WorkspaceSession) -> dict: + self._validate_workspace_revision( + session, + request.expected_workspace_revision, + ) + self._release_session_db(session) + try: + workspace = update_workspace_step_configuration( + session.directory, + request.expected_workspace_revision, + request.step_id, + request.parameters, + request.command_id, + ) + except WorkspaceLifecycleError as exc: + raise RuntimeApiError(exc.code, str(exc), exc.details) from exc + except (OSError, ValueError) as exc: + raise RuntimeApiError( + "workspace_step_configuration_update_failed", str(exc) + ) from exc + snapshot = self._read_engineering_snapshot(workspace) + session.workspace = workspace + session.workspace_revision = snapshot["workspaceRevision"] + return _workspace_session_result(session) + + return self._with_session_mutation_lock( + request.workspace_id, + update, + reject_active_operation=True, + ) + + def _load_workspace(self, directory: str): + if not directory: + raise RuntimeApiError("invalid_request", "missing required field: directory") + if not os.path.isdir(directory): + raise RuntimeApiError( + "invalid_request", + f"invalid workspace directory: {directory}", + ) + + import chipcompiler.data as data_api + + workspace = data_api.load_workspace(directory=directory) + if workspace is None: + raise RuntimeApiError( + "command_failed", + f"load workspace failed : {directory}", + ) + return workspace + + def _get_session(self, workspace_id: str) -> WorkspaceSession: + try: + return self.sessions.get_session(workspace_id) + except WorkspaceSessionNotFound as exc: + raise RuntimeApiError( + "workspace_session_not_found", + f"workspace session not found: {workspace_id}", + ) from exc + + @staticmethod + def _ensure_execution_ready(session: WorkspaceSession) -> None: + readiness = session.execution_readiness + if readiness.get("ready") is True: + return + code = str(readiness.get("code") or "pdk_binding_missing") + raise RuntimeApiError(code, "Workspace is not ready for execution", readiness) + + @staticmethod + def _ensure_engineering_snapshot(workspace) -> dict: + from chipcompiler.engine.snapshot import ( + EngineeringSnapshotError, + read_engineering_snapshot, + ) + + try: + return read_engineering_snapshot(workspace) + except EngineeringSnapshotError as exc: + raise RuntimeApiError("engineering_snapshot_unavailable", str(exc)) from exc + + @staticmethod + def _create_engineering_snapshot(workspace) -> dict: + from chipcompiler.engine.snapshot import ( + EngineeringSnapshotError, + create_engineering_snapshot, + ) + + try: + return create_engineering_snapshot(workspace) + except EngineeringSnapshotError as exc: + raise RuntimeApiError("engineering_snapshot_commit_failed", str(exc)) from exc + + @staticmethod + def _read_engineering_snapshot(owner) -> dict: + from chipcompiler.engine.snapshot import ( + EngineeringSnapshotError, + read_engineering_snapshot, + ) + + try: + return read_engineering_snapshot(getattr(owner, "workspace", owner)) + except EngineeringSnapshotError as exc: + raise RuntimeApiError("engineering_snapshot_unavailable", str(exc)) from exc + + @staticmethod + def _commit_step_snapshot( + session: WorkspaceSession, + workspace_step, + state, + error: str | None, + ) -> int: + state_value = str(getattr(state, "value", state)).lower() + try: + return WorkspaceSpecRuntimeMixin._commit_workspace_snapshot( + session, + f"flow_step.{state_value}", + ) + except RuntimeApiError as exc: + raise RuntimeApiError( + "engineering_snapshot_commit_failed", + str(exc), + { + "step": str(getattr(workspace_step, "name", "")), + "error": error or "", + }, + ) from exc + + @staticmethod + def _commit_workspace_snapshot(session: WorkspaceSession, cause: str) -> int: + from chipcompiler.engine.snapshot import ( + EngineeringSnapshotError, + commit_engineering_snapshot, + ) + + try: + snapshot = commit_engineering_snapshot( + session.workspace, + workspace_id=session.workspace_id, + cause=cause, + ) + except EngineeringSnapshotError as exc: + raise RuntimeApiError("engineering_snapshot_commit_failed", str(exc)) from exc + session.workspace_revision = snapshot["workspaceRevision"] + return session.workspace_revision + + +def _workspace_session_result(session: WorkspaceSession) -> dict: + return { + "workspaceId": session.workspace_id, + "workspaceRevision": session.workspace_revision, + "directory": str(session.directory), + **( + {"executionReadiness": session.execution_readiness} + if session.workspace_bindings is not None + or session.execution_readiness.get("ready") is not True + else {} + ), + } diff --git a/chipcompiler/utility/workspace_lock.py b/chipcompiler/utility/workspace_lock.py new file mode 100644 index 00000000..84a841ae --- /dev/null +++ b/chipcompiler/utility/workspace_lock.py @@ -0,0 +1,54 @@ +import threading +from contextlib import contextmanager +from pathlib import Path + +try: + import fcntl +except ImportError: # pragma: no cover - Windows has no sibling flock. + fcntl = None + +_local = threading.local() +_locks_guard = threading.Lock() +_locks: dict[str, threading.RLock] = {} + + +@contextmanager +def workspace_lock(directory: str | Path, *, blocking: bool = True): + """Share the sibling Workspace lock across Engine and Runtime callers.""" + path = Path(directory).expanduser().resolve() + key = str(path) + held = getattr(_local, "held", set()) + if key in held: + yield + return + + with _locks_guard: + lock = _locks.setdefault(key, threading.RLock()) + if not lock.acquire(blocking=blocking): + raise BlockingIOError(f"Workspace lock is busy: {path}") + + lock_file = None + file_locked = False + try: + path.parent.mkdir(parents=True, exist_ok=True) + if fcntl is not None: + lock_file = (path.parent / f"{path.name}.lock").open("a") + flags = fcntl.LOCK_EX if blocking else fcntl.LOCK_EX | fcntl.LOCK_NB + fcntl.flock(lock_file.fileno(), flags) + file_locked = True + held = set(held) + held.add(key) + _local.held = held + yield + finally: + if key in getattr(_local, "held", set()): + remaining = set(_local.held) + remaining.remove(key) + _local.held = remaining + if lock_file is not None: + try: + if file_locked: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + finally: + lock_file.close() + lock.release() diff --git a/pyproject.toml b/pyproject.toml index 514f5ab9..7e7a65ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "ecc-dreamplace==0.1.0a7", "ecc-tools-bin==0.1.0a13", "fastapi>=0.109", - "jsonrpcserver>=5.0.9", + "jsonrpcserver>=5.0.9,<6", "klayout>=0.30.2", "matplotlib>=3.4", "numpy>=1.21", diff --git a/test/engine/test_execution.py b/test/engine/test_execution.py index 0efef4ef..7af3fbf2 100644 --- a/test/engine/test_execution.py +++ b/test/engine/test_execution.py @@ -34,7 +34,8 @@ def run_step(self, step, *, rerun=False, observer=None): state=StateEnum.Success.value, step_id="Floorplan", ) - assert calls == [("flow", False, observer), ("Floorplan", True, observer)] + assert [call[:2] for call in calls] == [("flow", False), ("Floorplan", True)] + assert all(call[2].delegate is observer for call in calls) def test_execution_failure_keeps_main_blocking_semantics(): @@ -75,3 +76,24 @@ def run_steps(self, *, rerun=False, observer=None): assert result.succeeded assert committed == [("synthesis", StateEnum.Success)] + + +def test_execution_reports_ordered_steps_and_completed_default_is_noop(): + calls = [] + + class Flow: + workspace = SimpleNamespace( + flow=SimpleNamespace(data={"steps": [{"name": "synthesis", "state": "Success"}]}) + ) + + def run_steps(self, **_kwargs): + calls.append("run") + return True + + result = execute(Flow(), ExecutionPlan(intent="run")) + + assert result.succeeded + assert result.no_op + assert result.executed_steps == () + assert result.failed_step is None + assert calls == [] diff --git a/test/runtime/test_methods.py b/test/runtime/test_methods.py index b3fb0d12..b5a0a27d 100644 --- a/test/runtime/test_methods.py +++ b/test/runtime/test_methods.py @@ -16,8 +16,19 @@ def test_runtime_method_registry_contains_current_methods_once(): from chipcompiler.runtime.methods import RUNTIME_METHODS, runtime_method_names expected_methods = ( + "workspace_spec.describe", + "workspace_spec.validate", + "project.discover", + "project.manifest.load", + "project.manifest.mutate", "workspace.create", "workspace.open", + "workspace.binding_requirement", + "workspace.update", + "workspace.configuration.update", + "workspace.configuration.read", + "workspace.step_configuration.update", + "workspace.step_configuration.read", "workspace.close", "workspace.home", "workspace.info", @@ -34,9 +45,9 @@ def test_runtime_method_registry_contains_current_methods_once(): "operation.cancel", "operation.ack_step_rendered", "workspace.snapshot", + "workspace.engineering_snapshot", "workspace.recover_interrupted", ) - assert runtime_method_names() == expected_methods assert len(runtime_method_names()) == len(set(runtime_method_names())) assert len(RUNTIME_METHODS) == len(expected_methods) diff --git a/test/runtime/test_operations.py b/test/runtime/test_operations.py index 70c9e77e..ef7a44b2 100644 --- a/test/runtime/test_operations.py +++ b/test/runtime/test_operations.py @@ -500,6 +500,50 @@ def runner(observer): } +def test_operation_manager_keeps_only_latest_terminal_window(): + manager = RuntimeOperationManager() + + for index in range(257): + started = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=False, + step="step", + idempotency_key=f"command-{index}", + runner=lambda _observer: {"ok": True}, + ) + assert _wait_for_terminal(manager, started["operationId"])["state"] == "succeeded" + + assert len(manager.workspace_snapshot("workspace-1")["operations"]) == 256 + + +def test_operation_ledger_recovers_unfinished_operations_as_interrupted(tmp_path): + ledger = tmp_path / "runtime-commands.json" + manager = RuntimeOperationManager() + manager.load_workspace_ledger("workspace-1", ledger) + release = threading.Event() + started = manager.start( + workspace_id="workspace-1", + kind="flow", + origin="gui", + rerun=False, + step="", + idempotency_key="running-command", + runner=lambda _observer: release.wait(timeout=2), + ) + for _ in range(100): + if ledger.exists() and started["operationId"] in ledger.read_text(): + break + threading.Event().wait(0.01) + + restored = RuntimeOperationManager() + restored_ids = restored.load_workspace_ledger("workspace-1", ledger) + assert restored_ids == [started["operationId"]] + assert restored.operation_status(started["operationId"])["state"] == "interrupted" + release.set() + + def _wait_for_event(events: list[dict], event_type: str) -> dict: for _ in range(200): for event in events: @@ -512,7 +556,7 @@ def _wait_for_event(events: list[dict], event_type: str) -> dict: def _wait_for_terminal(manager: RuntimeOperationManager, operation_id: str) -> dict: for _ in range(100): status = manager.operation_status(operation_id) - if status["state"] in {"succeeded", "failed", "cancelled"}: + if status["state"] in {"succeeded", "failed", "cancelled", "interrupted"}: return status threading.Event().wait(0.01) return manager.operation_status(operation_id) diff --git a/test/runtime/test_server.py b/test/runtime/test_server.py index 34393f85..2d1db5fa 100644 --- a/test/runtime/test_server.py +++ b/test/runtime/test_server.py @@ -383,7 +383,7 @@ def test_first_slice_methods_are_registered(method): response = _dispatch(server, f'{{"jsonrpc":"2.0","method":"{method}","id":1}}') - assert response["error"]["code"] != -32601 + assert response.get("error", {}).get("code") != -32601 @pytest.mark.parametrize( diff --git a/uv.lock b/uv.lock index aedf194b..1c2ee175 100644 --- a/uv.lock +++ b/uv.lock @@ -493,7 +493,7 @@ requires-dist = [ { name = "ecc-dreamplace", editable = "chipcompiler/thirdparty/ecc-dreamplace" }, { name = "ecc-tools-bin", editable = "chipcompiler/thirdparty/ecc-tools" }, { name = "fastapi", specifier = ">=0.109" }, - { name = "jsonrpcserver", specifier = ">=5.0.9" }, + { name = "jsonrpcserver", specifier = ">=5.0.9,<6" }, { name = "klayout", specifier = ">=0.30.2" }, { name = "matplotlib", specifier = ">=3.4" }, { name = "numpy", specifier = ">=1.21" }, From 23493fda436806d5bb4e2dfa0e958482221cb96a Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Sat, 12 Sep 2026 16:52:56 +0800 Subject: [PATCH 26/43] fix(runtime): keep GUI sync out of flow execution --- chipcompiler/engine/flow.py | 21 ---- chipcompiler/runtime/operations.py | 106 ++-------------- test/runtime/test_operations.py | 195 +++++++---------------------- 3 files changed, 55 insertions(+), 267 deletions(-) diff --git a/chipcompiler/engine/flow.py b/chipcompiler/engine/flow.py index 3ee911d1..c5fc8259 100644 --- a/chipcompiler/engine/flow.py +++ b/chipcompiler/engine/flow.py @@ -778,13 +778,6 @@ def run_step( runtime, peak_memory_mb, ) - if state == StateEnum.Success and not _wait_for_step_rendered( - observer, - workspace_step, - state, - ): - return StateEnum.Invalid - return state def init_db_engine_for_step(self, workspace_step: WorkspaceStep) -> bool: @@ -855,17 +848,3 @@ def _notify_flow_observer(observer, method_name: str, *args) -> None: # Runtime observers must never turn a completed tool execution into a # failed flow. The coordinator records transport failures separately. logging.getLogger(__name__).exception("flow observer callback failed: %s", method_name) - - -def _wait_for_step_rendered(observer, workspace_step: WorkspaceStep, state: StateEnum) -> bool: - if observer is None or state != StateEnum.Success: - return True - callback = getattr(observer, "wait_for_step_rendered", None) - if not callable(callback): - return True - try: - return bool(callback(workspace_step, state)) - except Exception: - # Fail-open: observer bugs must not invalidate successful tool results. - logging.getLogger(__name__).exception("flow observer render gate failed") - return True diff --git a/chipcompiler/runtime/operations.py b/chipcompiler/runtime/operations.py index 9e29c34c..73d523de 100644 --- a/chipcompiler/runtime/operations.py +++ b/chipcompiler/runtime/operations.py @@ -11,9 +11,6 @@ _LOG_POLL_INTERVAL_SECONDS = 0.25 _MAX_LOG_CHUNK_BYTES = 16 * 1024 _MAX_FINAL_LOG_BYTES = 64 * 1024 -_RENDER_ACK_RETRY_SECONDS = 5.0 -_RENDER_ACK_PAUSE_SECONDS = 30.0 -_RENDER_ACK_ABORT_SECONDS = 300.0 _TERMINAL_OPERATION_STATES = frozenset({"succeeded", "failed", "cancelled", "interrupted"}) _MAX_TERMINAL_OPERATIONS = 256 _LEDGER_SCHEMA_VERSION = 1 @@ -65,9 +62,7 @@ class RuntimeOperation: workspace_revision: int = 0 render_sync_state: str = "idle" render_retry_count: int = 0 - render_wait_started_at: float | None = None last_render_ack_at: float | None = None - render_sync_degraded: bool = False acked_event_ids: set[str] = field(default_factory=set) cancel_requested: bool = False interruptibility: str = "deferred" @@ -79,7 +74,6 @@ class RuntimeOperationManager: def __init__(self, publisher: Callable[[dict[str, Any]], None] | None = None): self._publisher = publisher self._lock = threading.RLock() - self._render_gate = threading.Condition(self._lock) self._operations: dict[str, RuntimeOperation] = {} self._active_by_workspace: dict[str, str] = {} self._idempotency: dict[tuple[str, str], str] = {} @@ -197,6 +191,12 @@ def load_workspace_ledger(self, workspace_id: str, ledger_path: str | Path) -> l "message": "Runtime process ended before the Operation completed", } operation.updated_at = time.time() + # Render ACKs are retained only as a wire-compatibility field. They + # never gate execution, so old waiting records must not resurrect it. + operation.awaiting_event_id = None + operation.awaiting_event = None + operation.awaiting_step_commit_id = None + operation.render_sync_state = "idle" self._operations[operation.operation_id] = operation if operation.idempotency_key: self._idempotency[(workspace_id, operation.idempotency_key)] = ( @@ -234,7 +234,7 @@ def acknowledge_step_rendered( step_commit_id: str = "", workspace_revision: int | None = None, ) -> dict[str, Any]: - with self._render_gate: + with self._lock: operation = self._operations.get(operation_id) if operation is None: raise KeyError(operation_id) @@ -274,11 +274,9 @@ def acknowledge_step_rendered( operation.awaiting_event = None operation.awaiting_step_commit_id = None operation.render_sync_state = "idle" - operation.render_wait_started_at = None operation.last_render_ack_at = time.time() operation.updated_at = time.time() self._persist_workspace_locked(operation.workspace_id) - self._render_gate.notify_all() return { "accepted": True, "duplicate": False, @@ -287,7 +285,7 @@ def acknowledge_step_rendered( } def request_cancel(self, operation_id: str) -> dict[str, Any]: - with self._render_gate: + with self._lock: operation = self._operations.get(operation_id) if operation is None: raise KeyError(operation_id) @@ -297,7 +295,6 @@ def request_cancel(self, operation_id: str) -> dict[str, Any]: operation.updated_at = time.time() self._persist_workspace_locked(operation.workspace_id) event = self._new_event_locked(operation, "operation.cancel_requested", {}) - self._render_gate.notify_all() self._publish(event) return {"accepted": True, "operationId": operation_id, "state": operation.state} @@ -473,7 +470,7 @@ def step_completed( self._stop_step_log_tail(operation_id) state_value = str(getattr(state, "value", state)) final_log = _read_final_log(getattr(workspace_step, "log", None)) - with self._render_gate: + with self._lock: operation = self._operations[operation_id] operation.current_step = str(getattr(workspace_step, "name", "")) operation.current_tool = str(getattr(workspace_step, "tool", "")) @@ -501,14 +498,6 @@ def step_completed( step_commit_id = f"{operation.operation_id}:step:{operation.workspace_revision}" payload["stepCommitId"] = step_commit_id payload["workspaceRevision"] = operation.workspace_revision - if not operation.render_sync_degraded: - operation.awaiting_event_id = event["eventId"] - operation.awaiting_event = event - operation.awaiting_step_commit_id = step_commit_id - operation.render_sync_state = "waiting_for_gui_sync" - operation.render_retry_count = 0 - operation.render_wait_started_at = time.monotonic() - operation.state = "waiting_for_gui_sync" self._persist_workspace_locked(operation.workspace_id) self._publish(event) @@ -557,81 +546,8 @@ def step_skipped(self, operation_id: str, workspace_step: Any) -> None: self._publish(event) def wait_for_step_rendered(self, operation_id: str) -> bool: - while True: - degraded_event: dict[str, Any] | None = None - replay_event: dict[str, Any] | None = None - pause_event: dict[str, Any] | None = None - with self._render_gate: - operation = self._operations[operation_id] - if operation.cancel_requested: - return False - if not operation.awaiting_event_id: - if operation.state in { - "waiting_for_gui_sync", - "paused_for_gui_recovery", - }: - operation.state = "running" - operation.updated_at = time.time() - return True - - started_at = operation.render_wait_started_at or time.monotonic() - elapsed = time.monotonic() - started_at - if elapsed >= _RENDER_ACK_ABORT_SECONDS: - awaiting_event_id = operation.awaiting_event_id - awaiting_step_commit_id = operation.awaiting_step_commit_id - operation.awaiting_event_id = None - operation.awaiting_event = None - operation.awaiting_step_commit_id = None - operation.render_sync_state = "gui_sync_degraded" - operation.render_sync_degraded = True - operation.state = "running" - operation.updated_at = time.time() - degraded_event = self._new_event_locked( - operation, - "operation.gui_sync_degraded", - { - "eventId": awaiting_event_id, - "stepCommitId": awaiting_step_commit_id, - "workspaceRevision": operation.workspace_revision, - }, - ) - elif ( - elapsed >= _RENDER_ACK_PAUSE_SECONDS - and operation.render_sync_state != "paused_for_gui_recovery" - ): - operation.render_sync_state = "paused_for_gui_recovery" - operation.state = "paused_for_gui_recovery" - operation.updated_at = time.time() - pause_event = self._new_event_locked( - operation, - "operation.gui_sync_paused", - { - "eventId": operation.awaiting_event_id, - "stepCommitId": operation.awaiting_step_commit_id, - "workspaceRevision": operation.workspace_revision, - }, - ) - - if degraded_event is None: - operation.render_retry_count += 1 - if operation.awaiting_event is not None: - replay_event = { - **operation.awaiting_event, - "payload": { - **operation.awaiting_event["payload"], - "replayed": True, - "retryCount": operation.render_retry_count, - }, - } - self._render_gate.wait(timeout=_RENDER_ACK_RETRY_SECONDS) - - if degraded_event is not None: - self._publish(degraded_event) - return True - if pause_event is not None: - self._publish(pause_event) - if replay_event is not None: - self._publish(replay_event) + with self._lock: + return not self._operations[operation_id].cancel_requested def _tail_step_log(self, log_tail: _StepLogTail) -> None: while not log_tail.stopped.is_set(): diff --git a/test/runtime/test_operations.py b/test/runtime/test_operations.py index ef7a44b2..2bc98189 100644 --- a/test/runtime/test_operations.py +++ b/test/runtime/test_operations.py @@ -1,14 +1,13 @@ +import json import threading from types import SimpleNamespace from chipcompiler.data import StateEnum -from chipcompiler.runtime import operations from chipcompiler.runtime.operations import RuntimeOperationManager -def test_successful_step_waits_for_matching_render_ack_before_completing(): +def test_successful_step_does_not_wait_for_render_ack_before_completing(): events = [] - entered_render_gate = threading.Event() completed = threading.Event() manager = RuntimeOperationManager(events.append) step = SimpleNamespace(name="Synthesis", tool="yosys", log=SimpleNamespace(file="")) @@ -16,7 +15,6 @@ def test_successful_step_waits_for_matching_render_ack_before_completing(): def runner(observer): observer.on_step_started(step) observer.on_step_completed(step, StateEnum.Success) - entered_render_gate.set() assert observer.wait_for_step_rendered(step, StateEnum.Success) completed.set() return {"rerun": False} @@ -31,32 +29,15 @@ def runner(observer): runner=runner, ) - assert started["state"] in {"queued", "running", "waiting_for_gui_sync"} - assert entered_render_gate.wait(timeout=1) - assert not completed.wait(timeout=0.05) + assert started["state"] in {"queued", "running", "succeeded"} + assert completed.wait(timeout=1) step_completed = next(event for event in events if event["type"] == "step.completed") assert step_completed["payload"]["stepCommitId"] assert step_completed["payload"]["workspaceRevision"] == 1 - assert not manager.acknowledge_step_rendered( - started["operationId"], - step_completed["eventId"], - "wrong-step-commit", - step_completed["payload"]["workspaceRevision"], - )["accepted"] - - assert manager.acknowledge_step_rendered( - started["operationId"], - step_completed["eventId"], - step_completed["payload"]["stepCommitId"], - step_completed["payload"]["workspaceRevision"], - ) == { - "accepted": True, - "duplicate": False, - "operationId": started["operationId"], - "eventId": step_completed["eventId"], - } - assert completed.wait(timeout=1) - assert manager.operation_status(started["operationId"])["state"] == "succeeded" + status = manager.operation_status(started["operationId"]) + assert status["state"] == "succeeded" + assert status["renderSyncState"] == "idle" + assert status["awaitingEventId"] is None assert events[-1]["type"] == "operation.completed" @@ -104,56 +85,6 @@ def runner(observer): assert _wait_for_terminal(manager, started["operationId"])["state"] == "succeeded" -def test_render_ack_replays_one_commit_then_pauses_before_a_bounded_timeout(monkeypatch): - monkeypatch.setattr(operations, "_RENDER_ACK_RETRY_SECONDS", 0.01) - monkeypatch.setattr(operations, "_RENDER_ACK_PAUSE_SECONDS", 0.02) - monkeypatch.setattr(operations, "_RENDER_ACK_ABORT_SECONDS", 0.5) - events = [] - entered_render_gate = threading.Event() - manager = RuntimeOperationManager(events.append) - step = SimpleNamespace(name="Synthesis", tool="yosys", log=SimpleNamespace(file="")) - - def runner(observer): - observer.on_step_started(step) - observer.on_step_completed(step, StateEnum.Success) - entered_render_gate.set() - assert observer.wait_for_step_rendered(step, StateEnum.Success) - return {"rerun": False} - - started = manager.start( - workspace_id="workspace-1", - kind="flow", - origin="gui", - rerun=False, - step="", - idempotency_key="request-replay", - runner=runner, - ) - assert entered_render_gate.wait(timeout=1) - step_completed = _wait_for_event(events, "step.completed") - paused = _wait_for_event(events, "operation.gui_sync_paused") - replays = [ - event - for event in events - if event["type"] == "step.completed" and event["payload"].get("replayed") - ] - assert paused["payload"]["stepCommitId"] == step_completed["payload"]["stepCommitId"] - assert replays - assert all(event["eventId"] == step_completed["eventId"] for event in replays) - assert ( - manager.operation_status(started["operationId"])["renderSyncState"] - == "paused_for_gui_recovery" - ) - - assert manager.acknowledge_step_rendered( - started["operationId"], - step_completed["eventId"], - step_completed["payload"]["stepCommitId"], - step_completed["payload"]["workspaceRevision"], - )["accepted"] - assert _wait_for_terminal(manager, started["operationId"])["state"] == "succeeded" - - def test_ack_and_start_requests_are_idempotent(): events = [] release = threading.Event() @@ -269,49 +200,6 @@ def runner(observer): assert len([event for event in events if event["type"] == "operation.rerun_prepared"]) == 1 -def test_render_ack_timeout_degrades_and_allows_the_flow_to_continue(monkeypatch): - monkeypatch.setattr(operations, "_RENDER_ACK_RETRY_SECONDS", 0.01) - monkeypatch.setattr(operations, "_RENDER_ACK_PAUSE_SECONDS", 0.02) - monkeypatch.setattr(operations, "_RENDER_ACK_ABORT_SECONDS", 0.04) - events = [] - manager = RuntimeOperationManager(events.append) - step = SimpleNamespace(name="Synthesis", tool="yosys", log=SimpleNamespace(file="")) - - def runner(observer): - observer.on_step_started(step) - observer.on_step_completed(step, StateEnum.Success) - assert observer.wait_for_step_rendered(step, StateEnum.Success) - observer.on_step_started(step) - observer.on_step_completed(step, StateEnum.Success) - assert observer.wait_for_step_rendered(step, StateEnum.Success) - return {"rerun": False} - - started = manager.start( - workspace_id="workspace-1", - kind="flow", - origin="gui", - rerun=False, - step="", - idempotency_key="request-degraded-sync", - runner=runner, - ) - - degraded = _wait_for_event(events, "operation.gui_sync_degraded") - assert degraded["payload"]["stepCommitId"] - assert _wait_for_terminal(manager, started["operationId"])["state"] == "succeeded" - completed_events = [ - event - for event in events - if event["type"] == "step.completed" and not event["payload"].get("replayed") - ] - assert len(completed_events) == 2 - assert completed_events[1]["payload"]["stepCommitId"] - assert ( - manager.operation_status(started["operationId"])["renderSyncState"] == "gui_sync_degraded" - ) - assert not any(event["type"] == "operation.failed" for event in events) - - def test_active_operation_reports_a_shutdown_barrier_and_safe_boundary(): release = threading.Event() manager = RuntimeOperationManager() @@ -336,34 +224,6 @@ def test_active_operation_reports_a_shutdown_barrier_and_safe_boundary(): release.set() -def test_cancel_at_render_ack_boundary_releases_the_waiting_flow(): - entered_render_gate = threading.Event() - manager = RuntimeOperationManager() - step = SimpleNamespace(name="Synthesis", tool="yosys", log=SimpleNamespace(file="")) - - def runner(observer): - observer.on_step_started(step) - observer.on_step_completed(step, StateEnum.Success) - entered_render_gate.set() - if not observer.wait_for_step_rendered(step, StateEnum.Success): - raise RuntimeError("operation cancelled at a render boundary") - return {"rerun": False} - - started = manager.start( - workspace_id="workspace-1", - kind="flow", - origin="gui", - rerun=False, - step="", - idempotency_key="request-cancel-at-gate", - runner=runner, - ) - assert entered_render_gate.wait(timeout=1) - - assert manager.request_cancel(started["operationId"])["accepted"] is True - assert _wait_for_terminal(manager, started["operationId"])["state"] == "cancelled" - - def test_step_log_events_stream_only_new_log_bytes_and_keep_final_tail(tmp_path): events = [] step_started = threading.Event() @@ -405,9 +265,6 @@ def runner(observer): complete_step.set() step_complete = _wait_for_event(events, "step.completed") assert step_complete["payload"]["finalLog"] == ("previous run\nlive line one\nlive line two\n") - assert manager.acknowledge_step_rendered(started["operationId"], step_complete["eventId"])[ - "accepted" - ] assert _wait_for_terminal(manager, started["operationId"])["state"] == "succeeded" @@ -544,6 +401,42 @@ def test_operation_ledger_recovers_unfinished_operations_as_interrupted(tmp_path release.set() +def test_operation_ledger_clears_legacy_render_wait_state(tmp_path): + ledger = tmp_path / "runtime-commands.json" + ledger.write_text( + json.dumps( + { + "schemaVersion": 1, + "workspaceId": "workspace-1", + "operations": [ + { + "operationId": "operation-legacy", + "runSessionId": "session-legacy", + "runtimeInstanceId": "runtime-legacy", + "workspaceId": "workspace-1", + "kind": "flow", + "origin": "gui", + "state": "waiting_for_gui_sync", + "awaitingEventId": "event-legacy", + "awaitingStepCommitId": "commit-legacy", + "renderSyncState": "waiting_for_gui_sync", + } + ], + } + ), + encoding="utf-8", + ) + + manager = RuntimeOperationManager() + manager.load_workspace_ledger("workspace-1", ledger) + + status = manager.operation_status("operation-legacy") + assert status["state"] == "interrupted" + assert status["renderSyncState"] == "idle" + assert status["awaitingEventId"] is None + assert status["awaitingStepCommitId"] is None + + def _wait_for_event(events: list[dict], event_type: str) -> dict: for _ in range(200): for event in events: From 07a3189928663ec4e4a418c7f77080bb9ab23bfa Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Sat, 12 Sep 2026 17:30:36 +0800 Subject: [PATCH 27/43] fix(runtime): include workspace config in snapshot --- chipcompiler/runtime/workspace_api.py | 17 ++++++++++++++ test/runtime/test_workspace_api.py | 32 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/chipcompiler/runtime/workspace_api.py b/chipcompiler/runtime/workspace_api.py index 5e23513b..0433dce0 100644 --- a/chipcompiler/runtime/workspace_api.py +++ b/chipcompiler/runtime/workspace_api.py @@ -593,6 +593,8 @@ def acknowledge_step_rendered(self, request: OperationAckStepRenderedRequest) -> def workspace_snapshot(self, request: WorkspaceIdRequest) -> dict: session = self._get_session(request.workspace_id) + from chipcompiler.engine import read_workspace_configuration + flow_data = getattr(getattr(session.workspace, "flow", None), "data", {}) raw_steps = flow_data.get("steps", []) if isinstance(flow_data, dict) else [] steps = [ @@ -628,12 +630,27 @@ def workspace_snapshot(self, request: WorkspaceIdRequest) -> dict: parameter_path = workspace_config_path(session.directory) home_data["parameters"] = str(parameter_path) + engineering_snapshot = self._read_engineering_snapshot(session) + try: + configuration = read_workspace_configuration(session.workspace) + except (OSError, ValueError): + configuration = None + if ( + not isinstance(configuration, dict) + or configuration.get("workspaceId") != session.workspace_id + or not isinstance(configuration.get("workspaceSpec"), dict) + or not isinstance(configuration.get("workspaceBindings"), dict) + ): + configuration = None + return { **self.operations.workspace_snapshot(request.workspace_id), + "engineeringSnapshot": engineering_snapshot, "directory": str(session.directory), "flow": {"steps": steps}, "home": stringify_paths(home_data), "parameters": stringify_paths(deepcopy(parameters_data)), + "configuration": stringify_paths(configuration) if configuration else None, } def engineering_snapshot(self, request: WorkspaceIdRequest) -> dict: diff --git a/test/runtime/test_workspace_api.py b/test/runtime/test_workspace_api.py index be63e65d..5bc3b7c3 100644 --- a/test/runtime/test_workspace_api.py +++ b/test/runtime/test_workspace_api.py @@ -923,6 +923,38 @@ def test_runtime_modules_do_not_import_typer_or_click(): assert "import click" not in source +def test_workspace_snapshot_includes_configuration_and_engineering_snapshot(monkeypatch, tmp_path): + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + configuration = { + "workspaceId": "workspace-1", + "workspaceRevision": 2, + "workspaceSpec": {"design": {"name": "gcd"}}, + "workspaceBindings": {}, + } + monkeypatch.setattr( + "chipcompiler.engine.read_workspace_configuration", + lambda _workspace: configuration, + ) + api = WorkspaceRuntimeApi() + monkeypatch.setattr( + api, + "_read_engineering_snapshot", + lambda _owner: {"workspaceId": "workspace-1", "workspaceRevision": 2}, + ) + workspace_id = api.open_workspace(WorkspaceOpenRequest(directory=str(ws)))["workspaceId"] + session = api.sessions.get_session(workspace_id) + session.workspace.parameters = SimpleNamespace(data={}, path=ws / "home" / "params.toml") + session.workspace.home.data = {} + + snapshot = api.workspace_snapshot(WorkspaceIdRequest(workspace_id)) + + assert snapshot["configuration"] == configuration + assert snapshot["engineeringSnapshot"] == { + "workspaceId": "workspace-1", + "workspaceRevision": 2, + } + + def test_flow_run_uses_run_steps_and_prepare_on_rerun(monkeypatch, tmp_path): _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) prepared = [] From 17fdc72daaed4882b48e98085ca446e0c63adcaa Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Sat, 12 Sep 2026 23:35:46 +0800 Subject: [PATCH 28/43] fix(runtime): preserve committed snapshot revisions --- chipcompiler/runtime/operations.py | 226 ++++++++++++++++++--- chipcompiler/runtime/workspace_api.py | 205 +++++++++++++------ test/runtime/test_operation_parity.py | 278 ++++++++++++++++++++++++++ test/runtime/test_workspace_api.py | 38 ++++ 4 files changed, 662 insertions(+), 85 deletions(-) create mode 100644 test/runtime/test_operation_parity.py diff --git a/chipcompiler/runtime/operations.py b/chipcompiler/runtime/operations.py index 73d523de..f9321a3e 100644 --- a/chipcompiler/runtime/operations.py +++ b/chipcompiler/runtime/operations.py @@ -1,5 +1,8 @@ from __future__ import annotations +import hashlib +import json +import logging import threading import time from collections.abc import Callable @@ -14,6 +17,7 @@ _TERMINAL_OPERATION_STATES = frozenset({"succeeded", "failed", "cancelled", "interrupted"}) _MAX_TERMINAL_OPERATIONS = 256 _LEDGER_SCHEMA_VERSION = 1 +logger = logging.getLogger(__name__) @dataclass @@ -37,6 +41,10 @@ class RuntimeOperationCancelled(RuntimeError): """Cancellation was accepted at a safe step boundary.""" +class RuntimeOperationIdempotencyConflict(RuntimeError): + """A command ID was reused with different immutable input.""" + + @dataclass class RuntimeOperation: operation_id: str @@ -48,6 +56,7 @@ class RuntimeOperation: rerun: bool step: str = "" idempotency_key: str = "" + command_fingerprint: str = "" state: str = "queued" current_step: str = "" current_tool: str = "" @@ -76,11 +85,12 @@ def __init__(self, publisher: Callable[[dict[str, Any]], None] | None = None): self._lock = threading.RLock() self._operations: dict[str, RuntimeOperation] = {} self._active_by_workspace: dict[str, str] = {} - self._idempotency: dict[tuple[str, str], str] = {} + self._idempotency: dict[tuple[str, str], tuple[str, str]] = {} self._step_log_tails: dict[str, _StepLogTail] = {} self._runtime_instance_id = uuid4().hex self._workspace_sequences: dict[str, int] = {} self._ledger_paths: dict[str, Path] = {} + self._loaded_ledgers: set[Path] = set() def set_publisher(self, publisher: Callable[[dict[str, Any]], None] | None) -> None: with self._lock: @@ -97,16 +107,38 @@ def start( idempotency_key: str, runner: Callable[[RuntimeFlowObserver], dict[str, Any]], workspace_revision: int = 0, + snapshot_committer: Callable[[Any, Any, str | None], int] | None = None, + command_input: dict[str, Any] | None = None, + ledger_path: str | Path | None = None, + precondition: Callable[[], None] | None = None, ) -> dict[str, Any]: + fingerprint = _command_fingerprint( + kind=kind, + origin=origin, + rerun=rerun, + step=step, + workspace_revision=workspace_revision, + command_input=command_input, + ) + if ledger_path is not None: + self.load_workspace_ledger(workspace_id, ledger_path) with self._lock: if idempotency_key: - known_id = self._idempotency.get((workspace_id, idempotency_key)) - if known_id is not None: + known = self._idempotency.get((workspace_id, idempotency_key)) + if known is not None: + known_id, known_fingerprint = known + if known_fingerprint and known_fingerprint != fingerprint: + raise RuntimeOperationIdempotencyConflict( + f"command id reused with different input: {idempotency_key}" + ) return { **self._operation_payload(self._operations[known_id]), "deduplicated": True, } + if precondition is not None: + precondition() + active_id = self._active_by_workspace.get(workspace_id) if active_id is not None: active = self._operations[active_id] @@ -124,19 +156,23 @@ def start( rerun=rerun, step=step, idempotency_key=idempotency_key, + command_fingerprint=fingerprint, workspace_revision=workspace_revision, ) self._operations[operation.operation_id] = operation self._active_by_workspace[workspace_id] = operation.operation_id if idempotency_key: - self._idempotency[(workspace_id, idempotency_key)] = operation.operation_id + self._idempotency[(workspace_id, idempotency_key)] = ( + operation.operation_id, + fingerprint, + ) self._persist_workspace_locked(workspace_id) queued_event = self._new_event_locked(operation, "operation.queued", {}) self._publish(queued_event) thread = threading.Thread( target=self._run, - args=(operation.operation_id, runner), + args=(operation.operation_id, runner, snapshot_committer), name=f"ecc-runtime-{operation.operation_id}", daemon=True, ) @@ -169,6 +205,9 @@ def load_workspace_ledger(self, workspace_id: str, ledger_path: str | Path) -> l path = Path(ledger_path).expanduser().resolve() with self._lock: + if path in self._loaded_ledgers: + return [] + self._loaded_ledgers.add(path) self._ledger_paths[workspace_id] = path try: payload = json_read_strict(path) @@ -200,7 +239,8 @@ def load_workspace_ledger(self, workspace_id: str, ledger_path: str | Path) -> l self._operations[operation.operation_id] = operation if operation.idempotency_key: self._idempotency[(workspace_id, operation.idempotency_key)] = ( - operation.operation_id + operation.operation_id, + operation.command_fingerprint, ) self._workspace_sequences[workspace_id] = max( self._workspace_sequences.get(workspace_id, 0), operation.sequence @@ -298,6 +338,12 @@ def request_cancel(self, operation_id: str) -> dict[str, Any]: self._publish(event) return {"accepted": True, "operationId": operation_id, "state": operation.state} + def raise_if_cancel_requested(self, operation_id: str) -> None: + with self._lock: + operation = self._operations[operation_id] + if operation.cancel_requested: + raise RuntimeOperationCancelled("operation cancelled at a step boundary") + def shutdown_barrier(self) -> dict[str, Any] | None: with self._lock: for operation_id in self._active_by_workspace.values(): @@ -317,13 +363,14 @@ def _run( self, operation_id: str, runner: Callable[[RuntimeFlowObserver], dict[str, Any]], + snapshot_committer: Callable[[Any, Any, str | None], int] | None, ) -> None: with self._lock: operation = self._operations[operation_id] operation.state = "running" operation.updated_at = time.time() started_event = self._new_event_locked(operation, "operation.started", {}) - observer = RuntimeFlowObserver(self, operation_id) + observer = RuntimeFlowObserver(self, operation_id, snapshot_committer) try: self._publish(started_event) try: @@ -372,10 +419,7 @@ def _run( event_type = "operation.cancelled" else: operation.state = "failed" - operation.error = operation.error or { - "message": str(exc), - "code": "command_failed", - } + operation.error = operation.error or _operation_error_from_exception(exc) event_type = "operation.failed" operation.updated_at = time.time() payload = {"error": operation.error} @@ -444,20 +488,27 @@ def rerun_prepared( affected_steps: list[str], scope: str, target_step: str = "", + workspace_revision: int | None = None, ) -> None: """Publish the idempotent GUI reset boundary before a rerun starts.""" with self._lock: operation = self._operations[operation_id] + if workspace_revision is not None: + operation.workspace_revision = workspace_revision operation.updated_at = time.time() + payload = { + "affectedSteps": affected_steps, + "scope": scope, + "targetStep": target_step, + } + if workspace_revision is not None: + payload["workspaceRevision"] = workspace_revision event = self._new_event_locked( operation, "operation.rerun_prepared", - { - "affectedSteps": affected_steps, - "scope": scope, - "targetStep": target_step, - }, + payload, ) + self._persist_workspace_locked(operation.workspace_id) self._publish(event) def step_completed( @@ -466,6 +517,7 @@ def step_completed( workspace_step: Any, state: Any, error: str | None = None, + workspace_revision: int | None = None, ) -> None: self._stop_step_log_tail(operation_id) state_value = str(getattr(state, "value", state)) @@ -483,7 +535,7 @@ def step_completed( } if error: log_file = str(getattr(getattr(workspace_step, "log", None), "file", "") or "") - operation.error = { + operation.error = operation.error or { "code": "tool_failed", "message": error, "step": operation.current_step, @@ -491,13 +543,58 @@ def step_completed( "logFile": log_file, } payload["error"] = operation.error - payload["logFile"] = log_file + payload["logFile"] = str(operation.error.get("logFile", log_file)) event = self._new_event_locked(operation, "step.completed", payload) - if state_value == "Success": + if workspace_revision is None: operation.workspace_revision += 1 - step_commit_id = f"{operation.operation_id}:step:{operation.workspace_revision}" - payload["stepCommitId"] = step_commit_id - payload["workspaceRevision"] = operation.workspace_revision + else: + operation.workspace_revision = workspace_revision + step_commit_id = f"{operation.operation_id}:step:{operation.workspace_revision}" + payload["stepCommitId"] = step_commit_id + payload["workspaceRevision"] = operation.workspace_revision + self._persist_workspace_locked(operation.workspace_id) + self._publish(event) + + def step_diagnostic( + self, + operation_id: str, + workspace_step: Any, + diagnostic: dict[str, Any], + ) -> None: + message = str(diagnostic.get("message", "tool failed")) + log_file = str(getattr(getattr(workspace_step, "log", None), "file", "") or "") + with self._lock: + operation = self._operations[operation_id] + operation.error = { + "code": "tool_failed", + "message": message, + "step": str(getattr(workspace_step, "name", "")), + "tool": str(getattr(workspace_step, "tool", "")), + "logFile": log_file, + **diagnostic, + } + operation.updated_at = time.time() + revision = operation.workspace_revision + operation.error.update( + { + "snapshotRevision": revision, + "eventRevision": revision, + "operationRevision": revision, + } + ) + event = self._new_event_locked( + operation, + "step.diagnostic", + { + "step": str(getattr(workspace_step, "name", "")), + "tool": str(getattr(workspace_step, "tool", "")), + "diagnostic": operation.error, + "snapshotRevision": revision, + "eventRevision": revision, + "operationRevision": revision, + "workspaceRevision": revision, + }, + ) self._persist_workspace_locked(operation.workspace_id) self._publish(event) @@ -638,9 +735,7 @@ def _prune_terminal_locked(self) -> None: for operation_id in removed_ids: self._operations.pop(operation_id, None) self._idempotency = { - key: operation_id - for key, operation_id in self._idempotency.items() - if operation_id not in removed_ids + key: record for key, record in self._idempotency.items() if record[0] not in removed_ids } def _persist_workspace_locked(self, workspace_id: str) -> None: @@ -675,6 +770,7 @@ def _operation_payload(operation: RuntimeOperation) -> dict[str, Any]: "rerun": operation.rerun, "step": operation.step, "idempotencyKey": operation.idempotency_key, + "commandFingerprint": operation.command_fingerprint, "state": operation.state, "currentStep": operation.current_step, "currentTool": operation.current_tool, @@ -698,13 +794,25 @@ def _operation_payload(operation: RuntimeOperation) -> dict[str, Any]: def _publish(self, event: dict[str, Any]) -> None: publisher = self._publisher if publisher is not None: - publisher(event) + try: + publisher(event) + except Exception: + logger.exception("runtime event consumer failed: %s", event.get("type")) class RuntimeFlowObserver: - def __init__(self, manager: RuntimeOperationManager, operation_id: str): + fatal_observer = True + + def __init__( + self, + manager: RuntimeOperationManager, + operation_id: str, + snapshot_committer: Callable[[Any, Any, str | None], int] | None = None, + ): self._manager = manager self._operation_id = operation_id + self._snapshot_committer = snapshot_committer + self._committed_revision: int | None = None @property def runtime_operation(self) -> dict[str, Any]: @@ -723,21 +831,46 @@ def on_rerun_prepared( affected_steps: list[str], scope: str, target_step: str = "", + workspace_revision: int | None = None, ) -> None: self._manager.rerun_prepared( self._operation_id, affected_steps=affected_steps, scope=scope, target_step=target_step, + workspace_revision=workspace_revision, ) + def raise_if_cancelled(self) -> None: + self._manager.raise_if_cancel_requested(self._operation_id) + def on_step_completed( self, workspace_step: Any, state: Any, error: str | None = None, ) -> None: - self._manager.step_completed(self._operation_id, workspace_step, state, error) + if self._committed_revision is None and self._snapshot_committer is not None: + self.commit_step(workspace_step, state, error) + self._manager.step_completed( + self._operation_id, + workspace_step, + state, + error, + self._committed_revision, + ) + self._committed_revision = None + + def on_step_diagnostic( + self, + workspace_step: Any, + diagnostic: dict[str, Any], + ) -> None: + self._manager.step_diagnostic(self._operation_id, workspace_step, diagnostic) + + def commit_step(self, workspace_step: Any, state: Any, error: str | None = None) -> None: + if self._snapshot_committer is not None: + self._committed_revision = self._snapshot_committer(workspace_step, state, error) def on_subflow_stage(self, workspace_step: Any, subflow_step: dict[str, Any]) -> None: self._manager.subflow_stage(self._operation_id, workspace_step, subflow_step) @@ -809,6 +942,7 @@ def _operation_from_payload( rerun=bool(payload.get("rerun", False)), step=str(payload.get("step", "")), idempotency_key=str(payload.get("idempotencyKey", "")), + command_fingerprint=str(payload.get("commandFingerprint", "")), state=str(payload.get("state", "interrupted")), current_step=str(payload.get("currentStep", "")), current_tool=str(payload.get("currentTool", "")), @@ -834,5 +968,41 @@ def _number(value: Any) -> float: ) +def _command_fingerprint( + *, + kind: str, + origin: str, + rerun: bool, + step: str, + workspace_revision: int, + command_input: dict[str, Any] | None = None, +) -> str: + encoded = json.dumps( + { + "kind": kind, + "origin": origin, + "rerun": rerun, + "step": step, + "workspaceRevision": workspace_revision, + "commandInput": command_input or {}, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _operation_error_from_exception(exc: Exception) -> dict[str, Any]: + code = getattr(exc, "code", None) + data = getattr(exc, "data", None) + error: dict[str, Any] = { + "message": str(exc), + "code": code if isinstance(code, str) and code else "command_failed", + } + if isinstance(data, dict): + error.update(data) + return error + + def _optional_number(value: Any) -> float | None: return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None diff --git a/chipcompiler/runtime/workspace_api.py b/chipcompiler/runtime/workspace_api.py index 0433dce0..cc04a481 100644 --- a/chipcompiler/runtime/workspace_api.py +++ b/chipcompiler/runtime/workspace_api.py @@ -20,6 +20,7 @@ from chipcompiler.runtime.errors import RuntimeApiError from chipcompiler.runtime.operations import ( RuntimeOperationConflict, + RuntimeOperationIdempotencyConflict, RuntimeOperationManager, ) from chipcompiler.runtime.requests import ( @@ -369,16 +370,19 @@ def _flow_run( preserve_user_inputs: bool = False, ) -> dict: def run(session: WorkspaceSession) -> dict: + self._ensure_execution_ready(session) self._validate_workspace_revision(session, request.expected_workspace_revision) + stale_step_ids = self._stale_step_ids(session.workspace) + requires_preparation = request.rerun or bool(stale_step_ids) should_capture = self._should_capture_session_db(session) previous_db = session.db_handle if should_capture else None - if request.rerun and should_capture: + if requires_preparation and should_capture: self._release_session_db(session) previous_db = None engine_flow = self._build_flow_for_session( session, - attach_session_db=should_capture and not request.rerun, + attach_session_db=should_capture and not requires_preparation, ) if request.rerun: affected_steps = list(getattr(engine_flow, "workspace_steps", [])) @@ -387,17 +391,43 @@ def run(session: WorkspaceSession) -> dict: engine_flow, preserve_user_inputs=preserve_user_inputs, ) + reset_revision = self._commit_rerun_snapshot( + session, + "flow.rerun_prepared", + ) + self._notify_rerun_prepared( + observer, + affected_steps, + scope="flow", + workspace_revision=reset_revision, + ) + elif stale_step_ids: + affected_steps = [ + step + for step in getattr(engine_flow, "workspace_steps", []) + if str(getattr(step, "name", "")) in stale_step_ids + ] + self._refresh_workspace_config(session.workspace) + self._prepare_steps_for_rerun( + session.workspace, + engine_flow, + affected_steps, + ) + reset_revision = self._commit_rerun_snapshot( + session, + "flow.rerun_prepared", + ) self._notify_rerun_prepared( observer, affected_steps, scope="flow", + workspace_revision=reset_revision, ) try: ok = _run_engine_flow_steps( engine_flow, rerun=request.rerun, observer=observer, - session=session, ) finally: if should_capture: @@ -429,18 +459,30 @@ def _flow_run_step( reset_dependents: bool = False, ) -> dict: def run_step(session: WorkspaceSession) -> dict: + self._ensure_execution_ready(session) self._validate_workspace_revision(session, request.expected_workspace_revision) + stale_step_ids = self._stale_step_ids(session.workspace) should_capture = self._should_capture_session_db(session) previous_db = session.db_handle if should_capture else None - if request.rerun and should_capture: + stale_target_index = -1 + if request.step in stale_step_ids: + stale_target_index = stale_step_ids.index(request.step) + requires_preparation = request.rerun or stale_target_index == 0 + if stale_target_index > 0: + raise RuntimeApiError( + "stale_dependency", + f"rerun {stale_step_ids[0]} before {request.step}", + {"requiredStep": stale_step_ids[0]}, + ) + if requires_preparation and should_capture: self._release_session_db(session) previous_db = None engine_flow = self._build_flow_for_session( session, - attach_session_db=should_capture and not request.rerun, + attach_session_db=should_capture and not requires_preparation, ) - if request.rerun: + if requires_preparation: if session.layout_edit_session is not None: raise RuntimeApiError( "layout_edit_active", @@ -451,26 +493,39 @@ def run_step(session: WorkspaceSession) -> dict: workspace_step = engine_flow.get_workspace_step(request.step) if workspace_step is None: raise RuntimeApiError("command_failed", f"step not found: {request.step}") - if request.rerun: - affected_steps = self._rerun_affected_steps( - engine_flow, - workspace_step, - reset_dependents=reset_dependents, + if requires_preparation: + affected_steps = ( + [ + step + for step in getattr(engine_flow, "workspace_steps", []) + if str(getattr(step, "name", "")) in stale_step_ids + ] + if stale_target_index == 0 + else self._rerun_affected_steps( + engine_flow, + workspace_step, + reset_dependents=reset_dependents, + ) ) self._prepare_steps_for_rerun( session.workspace, engine_flow, affected_steps, ) + reset_revision = self._commit_rerun_snapshot( + session, + "flow.rerun_prepared", + ) self._notify_rerun_prepared( observer, affected_steps, scope="step", target_step=workspace_step.name, + workspace_revision=reset_revision, ) try: - step_already_succeeded = not request.rerun and engine_flow.check_state( + step_already_succeeded = not requires_preparation and engine_flow.check_state( name=workspace_step.name, tool=workspace_step.tool, state=_success_state(), @@ -480,9 +535,8 @@ def run_step(session: WorkspaceSession) -> dict: state = _run_engine_flow_step( engine_flow, workspace_step, - rerun=request.rerun, + rerun=requires_preparation, observer=observer, - session=session, ) finally: if should_capture: @@ -509,8 +563,12 @@ def run_step(session: WorkspaceSession) -> dict: def start_flow_operation(self, request: OperationStartFlowRequest) -> dict: self._require_gui_operation_origin(request.origin) session = self._get_session(request.workspace_id) - self._validate_workspace_revision(session, request.expected_workspace_revision) try: + + def validate_start() -> None: + self._ensure_execution_ready(session) + self._validate_workspace_revision(session, request.expected_workspace_revision) + return self.operations.start( workspace_id=request.workspace_id, kind="flow", @@ -519,6 +577,14 @@ def start_flow_operation(self, request: OperationStartFlowRequest) -> dict: step="", idempotency_key=request.idempotency_key, workspace_revision=request.expected_workspace_revision, + snapshot_committer=lambda step, state, error: self._commit_step_snapshot( + session, + step, + state, + error, + ), + ledger_path=session.directory / "home" / "runtime-commands.json", + precondition=validate_start, runner=lambda observer: self._flow_run( FlowRunRequest( workspace_id=request.workspace_id, @@ -529,14 +595,20 @@ def start_flow_operation(self, request: OperationStartFlowRequest) -> dict: preserve_user_inputs=request.rerun, ), ) + except RuntimeOperationIdempotencyConflict as exc: + raise RuntimeApiError("idempotency_conflict", str(exc)) from exc except RuntimeOperationConflict as exc: raise RuntimeApiError("operation_conflict", str(exc)) from exc def start_step_operation(self, request: OperationStartStepRequest) -> dict: self._require_gui_operation_origin(request.origin) session = self._get_session(request.workspace_id) - self._validate_workspace_revision(session, request.expected_workspace_revision) try: + + def validate_start() -> None: + self._ensure_execution_ready(session) + self._validate_workspace_revision(session, request.expected_workspace_revision) + return self.operations.start( workspace_id=request.workspace_id, kind="step", @@ -545,6 +617,15 @@ def start_step_operation(self, request: OperationStartStepRequest) -> dict: step=request.step, idempotency_key=request.idempotency_key, workspace_revision=request.expected_workspace_revision, + command_input={"resetDependents": request.reset_dependents}, + snapshot_committer=lambda step, state, error: self._commit_step_snapshot( + session, + step, + state, + error, + ), + ledger_path=session.directory / "home" / "runtime-commands.json", + precondition=validate_start, runner=lambda observer: self._flow_run_step( FlowRunStepRequest( workspace_id=request.workspace_id, @@ -556,6 +637,8 @@ def start_step_operation(self, request: OperationStartStepRequest) -> dict: reset_dependents=request.reset_dependents, ), ) + except RuntimeOperationIdempotencyConflict as exc: + raise RuntimeApiError("idempotency_conflict", str(exc)) from exc except RuntimeOperationConflict as exc: raise RuntimeApiError("operation_conflict", str(exc)) from exc @@ -595,8 +678,13 @@ def workspace_snapshot(self, request: WorkspaceIdRequest) -> dict: session = self._get_session(request.workspace_id) from chipcompiler.engine import read_workspace_configuration - flow_data = getattr(getattr(session.workspace, "flow", None), "data", {}) + flow = getattr(session.workspace, "flow", None) + flow_data = getattr(flow, "data", {}) raw_steps = flow_data.get("steps", []) if isinstance(flow_data, dict) else [] + if not raw_steps: + loader = getattr(flow, "steps", None) + if callable(loader): + raw_steps = loader() steps = [ { "name": str(step.get("name", "")), @@ -1131,6 +1219,36 @@ def _ensure_no_active_operation(self, session: WorkspaceSession) -> None: "Workspace has an active Operation", ) + @staticmethod + def _stale_step_ids(workspace) -> list[str]: + from chipcompiler.engine.snapshot import ( + EngineeringSnapshotError, + read_engineering_snapshot, + ) + + try: + snapshot = read_engineering_snapshot(workspace) + except EngineeringSnapshotError: + snapshot_path = Path(workspace.directory) / "home" / "engineering-snapshot.json" + if snapshot_path.is_file(): + raise + return [] + predecessor = snapshot.get("stalePredecessor") + if not isinstance(predecessor, dict): + return [] + return [ + step_id + for step_id in predecessor.get("invalidatedStepIds", []) + if isinstance(step_id, str) and step_id + ] + + @staticmethod + def _commit_rerun_snapshot(session: WorkspaceSession, cause: str) -> int | None: + snapshot_path = Path(session.directory) / "home" / "engineering-snapshot.json" + if not snapshot_path.is_file(): + return None + return WorkspaceSpecRuntimeMixin._commit_workspace_snapshot(session, cause) + def _refresh_workspace_config(self, workspace) -> None: import chipcompiler.data as data_api @@ -1168,16 +1286,20 @@ def _notify_rerun_prepared( workspace_steps, *, scope: str, + workspace_revision: int | None, target_step: str = "", ) -> None: callback = getattr(observer, "on_rerun_prepared", None) if callback is None: return - callback( - affected_steps=[str(getattr(step, "name", "")) for step in workspace_steps], - scope=scope, - target_step=target_step, - ) + payload = { + "affected_steps": [str(getattr(step, "name", "")) for step in workspace_steps], + "scope": scope, + "target_step": target_step, + } + if workspace_revision is not None: + payload["workspace_revision"] = workspace_revision + callback(**payload) @staticmethod def _prepare_step_for_rerun(workspace, engine_flow, workspace_step) -> None: @@ -2477,18 +2599,18 @@ def _state_value(state: Any) -> str: return getattr(state, "value", str(state)) -def _run_engine_flow_steps(engine_flow, *, rerun: bool, observer, session) -> bool: +def _run_engine_flow_steps(engine_flow, *, rerun: bool, observer) -> bool: from chipcompiler.engine import ExecutionPlan, execute result = execute( engine_flow, ExecutionPlan(intent="rerun" if rerun else "run"), - event_sink=_snapshot_observer(engine_flow.workspace, session, observer), + event_sink=observer, ) return result.succeeded -def _run_engine_flow_step(engine_flow, workspace_step, *, rerun: bool, observer, session): +def _run_engine_flow_step(engine_flow, workspace_step, *, rerun: bool, observer): from chipcompiler.engine import ExecutionPlan, execute result = execute( @@ -2497,37 +2619,6 @@ def _run_engine_flow_step(engine_flow, workspace_step, *, rerun: bool, observer, intent="rerun" if rerun else "run", step_id=str(getattr(workspace_step, "name", "")), ), - event_sink=_snapshot_observer(engine_flow.workspace, session, observer), + event_sink=observer, ) return _success_state() if result.succeeded else result.state - - -def _snapshot_observer(workspace, session, observer): - if observer is None: - return None - return _RuntimeSnapshotObserver(workspace, session, observer) - - -class _RuntimeSnapshotObserver: - fatal_observer = True - - def __init__(self, workspace, session, delegate): - self._workspace = workspace - self._session = session - self._delegate = delegate - - def on_step_completed(self, step, state, error=None): - from chipcompiler.engine.snapshot import commit_engineering_snapshot - - snapshot = commit_engineering_snapshot( - self._workspace, - workspace_id=self._session.workspace_id, - cause=f"flow_step.{_state_value(state).lower()}", - ) - self._session.workspace_revision = snapshot["workspaceRevision"] - callback = getattr(self._delegate, "on_step_completed", None) - if callable(callback): - callback(step, state, error) - - def __getattr__(self, name): - return getattr(self._delegate, name) diff --git a/test/runtime/test_operation_parity.py b/test/runtime/test_operation_parity.py new file mode 100644 index 00000000..de8f1261 --- /dev/null +++ b/test/runtime/test_operation_parity.py @@ -0,0 +1,278 @@ +import threading +from types import SimpleNamespace + +import pytest + +from chipcompiler.data import StateEnum +from chipcompiler.runtime.errors import RuntimeApiError +from chipcompiler.runtime.operations import ( + RuntimeOperationIdempotencyConflict, + RuntimeOperationManager, +) + + +def test_terminal_step_uses_the_committed_snapshot_revision_for_failures(): + events = [] + commits = [] + manager = RuntimeOperationManager(events.append) + step = SimpleNamespace(name="LEC", tool="yosys_lec", log=SimpleNamespace(file="")) + + def runner(observer): + observer.on_step_completed(step, StateEnum.Imcomplete, "missing output") + raise RuntimeError("run step LEC failed with state Incomplete") + + started = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=False, + step="LEC", + idempotency_key="failed-lec-revision", + workspace_revision=6, + snapshot_committer=lambda *_args: commits.append(7) or 7, + runner=runner, + ) + + status = _wait_for_terminal(manager, started["operationId"]) + completed = _wait_for_event(events, "step.completed") + + assert status["workspaceRevision"] == 7 + assert commits == [7] + assert completed["payload"]["workspaceRevision"] == 7 + assert completed["payload"]["stepCommitId"].endswith(":step:7") + + +def test_rerun_prepared_adopts_and_publishes_the_reset_revision(): + events = [] + manager = RuntimeOperationManager(events.append) + + def runner(observer): + observer.on_rerun_prepared( + affected_steps=["preFloorplan", "place"], + scope="flow", + workspace_revision=8, + ) + return {"rerun": True} + + started = manager.start( + workspace_id="workspace-1", + kind="flow", + origin="gui", + rerun=True, + step="", + idempotency_key="rerun-revision", + workspace_revision=7, + runner=runner, + ) + + status = _wait_for_terminal(manager, started["operationId"]) + prepared = _wait_for_event(events, "operation.rerun_prepared") + + assert prepared["payload"]["workspaceRevision"] == 8 + assert status["workspaceRevision"] == 8 + + +def test_reusing_a_command_id_with_different_parameters_conflicts(): + release = threading.Event() + manager = RuntimeOperationManager() + first = manager.start( + workspace_id="workspace-1", + kind="flow", + origin="gui", + rerun=False, + step="", + idempotency_key="request-1", + runner=lambda _observer: release.wait(timeout=1) or {}, + ) + + try: + with pytest.raises(RuntimeOperationIdempotencyConflict): + manager.start( + workspace_id="workspace-1", + kind="flow", + origin="gui", + rerun=True, + step="", + idempotency_key="request-1", + runner=lambda _observer: {}, + ) + finally: + release.set() + _wait_for_terminal(manager, first["operationId"]) + + +def test_command_fingerprint_includes_command_specific_input(): + release = threading.Event() + manager = RuntimeOperationManager() + first = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=True, + step="Place", + idempotency_key="request-1", + command_input={"resetDependents": False}, + runner=lambda _observer: release.wait(timeout=1) or {}, + ) + + try: + with pytest.raises(RuntimeOperationIdempotencyConflict): + manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=True, + step="Place", + idempotency_key="request-1", + command_input={"resetDependents": True}, + runner=lambda _observer: {}, + ) + finally: + release.set() + _wait_for_terminal(manager, first["operationId"]) + + +def test_step_diagnostic_persists_structured_tool_error(): + events = [] + manager = RuntimeOperationManager(events.append) + step = SimpleNamespace(name="place", tool="dreamplace", log=SimpleNamespace(file="place.log")) + + def runner(observer): + observer.on_step_diagnostic( + step, + {"message": "overflow exceeded", "details": {"overflow": 1.2}}, + ) + return {"rerun": False} + + started = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=False, + step="place", + idempotency_key="diagnostic", + runner=runner, + ) + + status = _wait_for_terminal(manager, started["operationId"]) + + assert status["error"] == { + "code": "tool_failed", + "message": "overflow exceeded", + "step": "place", + "tool": "dreamplace", + "logFile": "place.log", + "details": {"overflow": 1.2}, + "snapshotRevision": 0, + "eventRevision": 0, + "operationRevision": 0, + } + diagnostic = next(event for event in events if event["type"] == "step.diagnostic") + assert diagnostic["payload"]["workspaceRevision"] == status["workspaceRevision"] == 0 + assert diagnostic["payload"]["diagnostic"]["operationRevision"] == 0 + + +def test_terminal_completion_preserves_a_prior_structured_diagnostic(): + manager = RuntimeOperationManager() + step = SimpleNamespace(name="place", tool="dreamplace", log=SimpleNamespace(file="place.log")) + + def runner(observer): + observer.on_step_diagnostic(step, {"message": "overflow exceeded", "overflow": 1.2}) + observer.on_step_completed(step, StateEnum.Imcomplete, "tool exited with failure") + + started = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=False, + step="place", + idempotency_key="diagnostic-terminal", + runner=runner, + ) + + status = _wait_for_terminal(manager, started["operationId"]) + + assert status["error"] == { + "code": "tool_failed", + "message": "overflow exceeded", + "step": "place", + "tool": "dreamplace", + "logFile": "place.log", + "overflow": 1.2, + "snapshotRevision": 0, + "eventRevision": 0, + "operationRevision": 0, + } + + +def test_snapshot_commit_failure_keeps_a_stable_operation_error_code(): + manager = RuntimeOperationManager() + step = SimpleNamespace(name="Synthesis", tool="yosys", log=SimpleNamespace(file="")) + + def fail_commit(*_args): + raise RuntimeApiError( + "engineering_snapshot_commit_failed", + "snapshot write failed", + {"step": "Synthesis"}, + ) + + started = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=False, + step="Synthesis", + idempotency_key="snapshot-failure", + snapshot_committer=fail_commit, + runner=lambda observer: observer.on_step_completed(step, StateEnum.Success), + ) + + status = _wait_for_terminal(manager, started["operationId"]) + + assert status["state"] == "failed" + assert status["error"] == { + "code": "engineering_snapshot_commit_failed", + "message": "snapshot write failed", + "step": "Synthesis", + } + + +def test_event_consumer_failure_does_not_rollback_a_durable_step_commit(): + manager = RuntimeOperationManager( + lambda _event: (_ for _ in ()).throw(RuntimeError("IPC down")) + ) + step = SimpleNamespace(name="Synthesis", tool="yosys", log=SimpleNamespace(file="")) + + started = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=False, + step="Synthesis", + idempotency_key="publisher-failure", + snapshot_committer=lambda *_args: 12, + runner=lambda observer: observer.on_step_completed(step, StateEnum.Success), + ) + + status = _wait_for_terminal(manager, started["operationId"]) + + assert status["state"] == "succeeded" + assert status["workspaceRevision"] == 12 + + +def _wait_for_event(events: list[dict], event_type: str) -> dict: + for _ in range(200): + for event in events: + if event["type"] == event_type: + return event + threading.Event().wait(0.01) + raise AssertionError(f"event not received: {event_type}") + + +def _wait_for_terminal(manager: RuntimeOperationManager, operation_id: str) -> dict: + for _ in range(100): + status = manager.operation_status(operation_id) + if status["state"] in {"succeeded", "failed", "cancelled", "interrupted"}: + return status + threading.Event().wait(0.01) + return manager.operation_status(operation_id) diff --git a/test/runtime/test_workspace_api.py b/test/runtime/test_workspace_api.py index 5bc3b7c3..bfb3dce9 100644 --- a/test/runtime/test_workspace_api.py +++ b/test/runtime/test_workspace_api.py @@ -955,6 +955,44 @@ def test_workspace_snapshot_includes_configuration_and_engineering_snapshot(monk } +def test_workspace_snapshot_falls_back_to_persisted_flow_steps(monkeypatch, tmp_path): + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + api = WorkspaceRuntimeApi() + workspace_id = api.open_workspace(WorkspaceOpenRequest(directory=str(ws)))["workspaceId"] + workspace = api.sessions.get_session(workspace_id).workspace + flow = workspace.flow + flow.data = {} + workspace.parameters = SimpleNamespace(data={}, path=ws / "home" / "parameters.json") + workspace.home = SimpleNamespace(data={}) + monkeypatch.setattr( + flow, + "steps", + lambda: [{"name": "Synthesis", "tool": "yosys", "state": "Success"}], + raising=False, + ) + monkeypatch.setattr( + api, + "_read_engineering_snapshot", + lambda _owner: {"workspaceId": "workspace-1", "workspaceRevision": 1}, + ) + monkeypatch.setattr( + "chipcompiler.engine.read_workspace_configuration", + lambda _workspace: {}, + ) + + snapshot = api.workspace_snapshot(WorkspaceIdRequest(workspace_id)) + + assert snapshot["flow"]["steps"] == [ + { + "name": "Synthesis", + "tool": "yosys", + "state": "Success", + "runtime": "", + "peakMemory": 0, + } + ] + + def test_flow_run_uses_run_steps_and_prepare_on_rerun(monkeypatch, tmp_path): _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) prepared = [] From 2fbf8f425f8940ac6847fee1fabbdf91159d0cb6 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Sun, 13 Sep 2026 23:29:14 +0800 Subject: [PATCH 29/43] fix(runtime): preserve floorplan overrides during step setup --- chipcompiler/data/workspace/__init__.py | 3 +++ test/data/test_workspace.py | 34 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/chipcompiler/data/workspace/__init__.py b/chipcompiler/data/workspace/__init__.py index a62aea56..75a5a743 100644 --- a/chipcompiler/data/workspace/__init__.py +++ b/chipcompiler/data/workspace/__init__.py @@ -952,6 +952,9 @@ def update_step_config(workspace: Workspace, step: WorkspaceStep) -> None: if step.name in {StepEnum.PRE_FLOORPLAN.value, StepEnum.POST_FLOORPLAN.value}: _refresh_floorplan_config(workspace, step=step) + from .config_overrides import apply_config_overrides + + apply_config_overrides(workspace.config, workspace.parameters.data) if step.name == StepEnum.ROUTING.value and isinstance(step.data, EccData): router = json_read(workspace.config[f"{StepEnum.ROUTING.value}"]) diff --git a/test/data/test_workspace.py b/test/data/test_workspace.py index 4a0d9d33..61f6a97c 100644 --- a/test/data/test_workspace.py +++ b/test/data/test_workspace.py @@ -19,7 +19,9 @@ prepare_workspace_for_rerun, refresh_workspace_config, sync_workspace_config_to_parameters, + update_step_config, ) +from chipcompiler.data.workspace.layout import EccData, EccOutput, EccStep, StepInput from chipcompiler.utility import json_read, json_write EXPECTED_WORKSPACE_CONFIG_FILENAMES = { @@ -1173,6 +1175,38 @@ def test_refresh_workspace_config_reapplies_direct_config_overrides( assert json_read(workspace.config["dreamplace"])["num_threads"] == 12 +def test_update_step_config_preserves_floorplan_mode_override_after_result_backfill( + tmp_path, minimal_ics55_pdk_factory, default_ics55_parameters +): + workspace_dir, workspace = _create_loaded_ics55_workspace( + tmp_path, + "workspace_floorplan_mode_override", + minimal_ics55_pdk_factory, + default_ics55_parameters, + ) + parameters = _read_parameters(workspace_dir / "home" / "params.toml") + parameters["die"] = {"size": [31.8, 32.0], "area": 1017.6} + parameters["config_overrides"] = { + "Floorplan": {"die_builder": {"mode": "die_util"}}, + } + _write_parameters(workspace_dir / "home" / "params.toml", parameters) + workspace.parameters.data = parameters + + step = EccStep( + name=StepEnum.POST_FLOORPLAN.value, + input=StepInput(), + output=EccOutput(dir=workspace_dir / "postFloorplan_ecc" / "output"), + data=EccData( + steps={StepEnum.POST_FLOORPLAN.value: workspace_dir / "postFloorplan_ecc" / "data"} + ), + ) + + update_step_config(workspace, step) + + floorplan = json_read(workspace.config[StepEnum.FLOORPLAN.value]) + assert floorplan["die_builder"]["mode"] == "die_util" + + def test_sync_workspace_config_to_parameters_updates_routing_layers_and_refreshes_peers( tmp_path, minimal_ics55_pdk_factory, default_ics55_parameters ): From 6b702c3b940084e8e419da140a908689e35c7a86 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Mon, 14 Sep 2026 10:06:42 +0800 Subject: [PATCH 30/43] fix(runtime): stop cancelled flows at step boundaries --- chipcompiler/engine/execution.py | 1 + chipcompiler/engine/flow.py | 1 + test/engine/test_execution.py | 40 +++++++++++++++++++++ test/runtime/test_operations.py | 61 ++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+) diff --git a/chipcompiler/engine/execution.py b/chipcompiler/engine/execution.py index b09ee03f..5ce285a9 100644 --- a/chipcompiler/engine/execution.py +++ b/chipcompiler/engine/execution.py @@ -47,6 +47,7 @@ def execute(flow: Any, plan: ExecutionPlan, *, event_sink: Any = None) -> Execut else: succeeded = True for step_id in selected_ids: + observer.raise_if_cancelled() step = _find_step(flow, step_id) if step is None: raise ValueError(f"step not found: {step_id}") diff --git a/chipcompiler/engine/flow.py b/chipcompiler/engine/flow.py index c5fc8259..3ece30ca 100644 --- a/chipcompiler/engine/flow.py +++ b/chipcompiler/engine/flow.py @@ -516,6 +516,7 @@ def run_steps( """ for workspace_step in self.workspace_steps: + _notify_flow_observer(observer, "raise_if_cancelled") self.workspace.logger.log_section( f"{workspace_step.tool} - begin step - {workspace_step.name}" ) diff --git a/test/engine/test_execution.py b/test/engine/test_execution.py index 7af3fbf2..734fe2b6 100644 --- a/test/engine/test_execution.py +++ b/test/engine/test_execution.py @@ -1,5 +1,7 @@ from types import SimpleNamespace +import pytest + from chipcompiler.data import StateEnum from chipcompiler.engine.execution import ExecutionPlan, execute @@ -38,6 +40,44 @@ def run_step(self, step, *, rerun=False, observer=None): assert all(call[2].delegate is observer for call in calls) +def test_execution_plan_checks_cancel_before_each_selected_step(): + calls = [] + + class Cancelled(RuntimeError): + pass + + class Observer: + fatal_observer = True + + def __init__(self): + self.completed = 0 + + def on_step_completed(self, _step, _state, _error=None): + self.completed += 1 + + def raise_if_cancelled(self): + if self.completed: + raise Cancelled + + class Flow: + def get_workspace_step(self, step_id): + return SimpleNamespace(name=step_id) + + def run_step(self, step, *, rerun=False, observer=None): + calls.append(step.name) + observer.on_step_completed(step, StateEnum.Success) + return StateEnum.Success + + with pytest.raises(Cancelled): + execute( + Flow(), + ExecutionPlan(intent="run", step_ids=("Synthesis", "Floorplan")), + event_sink=Observer(), + ) + + assert calls == ["Synthesis"] + + def test_execution_failure_keeps_main_blocking_semantics(): class Flow: workspace = SimpleNamespace(flow=SimpleNamespace(data={"steps": []})) diff --git a/test/runtime/test_operations.py b/test/runtime/test_operations.py index 2bc98189..8127c27a 100644 --- a/test/runtime/test_operations.py +++ b/test/runtime/test_operations.py @@ -3,6 +3,7 @@ from types import SimpleNamespace from chipcompiler.data import StateEnum +from chipcompiler.engine.flow import EngineFlow from chipcompiler.runtime.operations import RuntimeOperationManager @@ -41,6 +42,66 @@ def runner(observer): assert events[-1]["type"] == "operation.completed" +def test_cancel_stops_before_next_engine_flow_step(monkeypatch): + events = [] + first_step_running = threading.Event() + release_first_step = threading.Event() + manager = RuntimeOperationManager(events.append) + steps = [ + SimpleNamespace(name=name, tool="mock", log=SimpleNamespace(file="")) + for name in ("Synthesis", "Floorplan") + ] + workspace = SimpleNamespace( + flow=SimpleNamespace(data={"steps": [{}, {}]}), + logger=SimpleNamespace(log_section=lambda *_args: None, error=lambda *_args: None), + ) + flow = EngineFlow(workspace=None) + flow.workspace = workspace + flow.workspace_steps = steps + flow.init_db_engine = lambda: True + executed = [] + + def run_step(step, *, rerun=False, observer=None): + executed.append(step.name) + observer.on_step_started(step) + if step.name == "Synthesis": + first_step_running.set() + assert release_first_step.wait(timeout=2) + observer.on_step_completed(step, StateEnum.Success) + return StateEnum.Success + + flow.run_step = run_step + monkeypatch.setattr("chipcompiler.engine.flow.log_flow", lambda **_kwargs: None) + revisions = [] + + def commit_step(*_args): + revisions.append(len(revisions) + 1) + return revisions[-1] + + started = manager.start( + workspace_id="workspace-1", + kind="flow", + origin="gui", + rerun=False, + step="", + idempotency_key="cancel-at-step-boundary", + snapshot_committer=commit_step, + runner=lambda observer: {"succeeded": flow.run_steps(observer=observer)}, + ) + assert first_step_running.wait(timeout=1) + assert manager.request_cancel(started["operationId"])["accepted"] is True + release_first_step.set() + + status = _wait_for_terminal(manager, started["operationId"]) + + assert status["state"] == "cancelled" + assert status["workspaceRevision"] == 1 + assert executed == ["Synthesis"] + assert revisions == [1] + event_types = [event["type"] for event in events] + assert event_types.index("step.completed") < event_types.index("operation.cancelled") + + def test_subflow_stage_is_emitted_for_the_active_workspace_step(): events = [] released = threading.Event() From 77dc1f729c63fc63dce2f230750d5331724a5e86 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Mon, 14 Sep 2026 11:08:03 +0800 Subject: [PATCH 31/43] docs: move integration specs to parent repository --- docs/index.md | 1 - .../adr-headless-engine-contract.md | 80 ----- .../ecc-runtime-consolidation-plan.zh-CN.md | 221 ------------- .../specification/glossary-headless-engine.md | 89 ------ .../specification/headless-engine-contract.md | 290 ------------------ 5 files changed, 681 deletions(-) delete mode 100644 docs/specification/adr-headless-engine-contract.md delete mode 100644 docs/specification/ecc-runtime-consolidation-plan.zh-CN.md delete mode 100644 docs/specification/glossary-headless-engine.md delete mode 100644 docs/specification/headless-engine-contract.md diff --git a/docs/index.md b/docs/index.md index 24afee4a..efe18b71 100644 --- a/docs/index.md +++ b/docs/index.md @@ -43,7 +43,6 @@ ChipCompiler supports various EDA file formats. Technical specifications for par ### CLI Specifications -- **[ECC Runtime Consolidation Plan](specification/ecc-runtime-consolidation-plan.zh-CN.md)** - ECC-owned Runtime, shared CLI/Engine writes, Studio Snapshot boundary, phased migration and protocol cleanup - **[CLI Design](specification/cli-design.md)** - Progressive-disclosure CLI design and roadmap - Grep-friendly summary lines with disclosure commands - Project, run, step, metric, artifact, issue, and config object model diff --git a/docs/specification/adr-headless-engine-contract.md b/docs/specification/adr-headless-engine-contract.md deleted file mode 100644 index 687c814a..00000000 --- a/docs/specification/adr-headless-engine-contract.md +++ /dev/null @@ -1,80 +0,0 @@ -# ADR: Headless Engine Contract Scope (PR 275) - -> **Superseded.** The accepted PR 275 scope was Studio-only. The current -> cross-repository decision is documented in -> [ECC Runtime Consolidation v1](ecc-runtime-consolidation-plan.zh-CN.md) and -> moves all CLI Workspace writes onto the same ECC Engine commit paths. - -Status: accepted during review of -[PR 275](https://github.com/openecos-projects/ecc/pull/275) -(`ekko/refactor-ecc-cli-integration`). Implementation spec: -[headless-engine-contract.md](headless-engine-contract.md). - -## Context - -This PR removes the in-process RPC runtime and adds headless workspace APIs -under `chipcompiler.engine`. Two interpretations were available: - -- Treat the new APIs as a shared CLI + Studio contract in this PR. -- Treat them as a Studio-owned headless contract, and leave CLI on its - existing `create_workspace` / `EngineFlow` path. - -`main` already had two products, not one transaction model: - -- CLI serialized runs with a sibling `{workspace}.lock` and rolled back - `ecc param --workspace` from in-memory file bytes. -- RPC/Studio serialized mutations with a process-local session lock and - had no file-backed configuration transaction. - -## Decision - -This PR delivers a **Studio-only** headless contract. CLI does not adopt -the new create / configure / execute APIs in this round. - -Consequences that follow from that scope: - -1. Studio create, structural update, and configuration update share one - sibling `{workspace}.lock`. Configuration backup remains a Studio - transaction implementation and does **not** own - `.{workspace}.configuration.lock` (that file is new in this PR and is - removed). Backup and crash recovery run only while the ownership lock - is already held. -2. CLI keeps `{workspace}.lock` and `_snapshot_transaction`. Those are - the pre-existing CLI design; this PR does not merge them into - `WorkspaceFileTransaction`. -3. Studio execution goes through `chipcompiler.engine.execute`, which - commits the Engineering Snapshot. CLI continues to call `EngineFlow` - directly and does not write snapshots. -4. Studio query APIs (`describe`, `assess`, `read_*`) load workspaces - `read_only=True`. They must not migrate, create home files, or append - logs. Mutating APIs may load writable workspaces. -5. Unused runtime extraction in `workspace_flow.py` is deleted. `execute` - calls `EngineFlow` directly. -6. `create_workspace_from_spec` may delete `target` only if this call - created that directory. A pre-existing Workspace is never removed on - create failure (`workspace_exists` instead). -7. `chipcompiler.engine` exports the Studio contract: lifecycle, - configuration, `execute`, Engineering Snapshot reads, and signoff - inspect/export. It does not export analysis builders or `score_qor`; - those stay internal to snapshot/execute. -8. Workspace Spec parameter applicability uses `normalize_flow_step()`. - The handwritten step-alias table in spec validation is deleted. - -## Non-goals for this PR - -- Wiring CLI `ecc run`, `ecc workspace refresh`, or `ecc param --workspace` - through `create_workspace_from_spec` / `update_workspace_configuration` - / `execute`. -- Making CLI and Studio lock files mutually exclusive. -- Replacing CLI in-memory parameter rollback with the Studio file - transaction. -- Changing CLI `sys.modules` manifest aliases. -- Making CLI `ecc signoff inspect` a read-only load. -- Splitting `metrics.py`. -- Treating full-suite / PyInstaller / Nix runs as a contract change - (they remain a merge checklist, not a scope item). - -## Follow-up - -A later PR may adopt the headless contract in CLI. That change is -explicitly out of scope here and would be a separate decision. diff --git a/docs/specification/ecc-runtime-consolidation-plan.zh-CN.md b/docs/specification/ecc-runtime-consolidation-plan.zh-CN.md deleted file mode 100644 index ea5a863a..00000000 --- a/docs/specification/ecc-runtime-consolidation-plan.zh-CN.md +++ /dev/null @@ -1,221 +0,0 @@ -# ECC Runtime Consolidation v1 - -状态：已确认，准备实施 - -本文是 ECC 与 ECOS Studio 的联合实施方案。目标是删除 -`ecos-studio-runtime`/`ecos-ecc-runtime-adapter` 的独立实现，同时保留 -ECOS Studio 的全部能力，包括 Workspace Snapshot、Project Comparison、QoR、 -Signoff、Layout/Floorplan 编辑、取消、恢复和后台执行。 - -## 1. 最终边界 - -```text -ECC Engine 领域事务、配置、Flow 执行、Revision、Engineering Snapshot -ECC Runtime ECC 内部 JSON-RPC、Session、Operation、Ledger、事件、取消、恢复 -ECC CLI CLI 解析、输出和只读查询；写操作调用 ECC Engine -Electron 产品策略、权限、路径授权、进程监督、Snapshot 读取和展示 -``` - -- 只发布一个 `ecc` 可执行文件。普通 CLI 与隐藏的 `ecc rpc serve` 使用同一 - 可执行文件、同一 Python 环境和同一 ECC 版本。 -- ECC Runtime 是 ECC 的私有 Studio 集成接口，不承诺第三方协议兼容性。 -- 每个有执行或编辑工作的 Workspace 使用独立 Runtime 进程和故障域。 -- Workspace 无活动 Operation、Layout Edit 或 finalization 工作后，Electron - 释放该 Workspace Runtime；Workspace 无关请求使用一个应用生命周期内的懒加载 - Control Runtime，但它仍是同一 `ecc` 进程模型中的路由细节。 -- Electron Agent、Quick Start 和后台产品命令必须走 - `Electron Product Command -> ECC Runtime -> ECC Engine`，不能直接导入 ECC - 或绕过 Runtime 调用 Engine。 - -## 2. Snapshot 决策 - -### Engineering Snapshot - -`home/engineering-snapshot.json` 是 Studio 的 committed engineering read -model，不是 CLI 查询 API。 - -- ECC 是唯一生成者和写入者。 -- Workspace 创建、结构更新、配置更新、rerun preparation、保存 Layout 和每个 - Flow Step Commit 都按统一 Revision 提交 Snapshot。 -- 文件保持现有 schema v2 和文件名；`engineering-snapshot.stale.json` 继续表示 - 单个 stale predecessor。 -- Flow 状态、Snapshot 和 commit metadata 必须以 Atomic Engineering Commit - 一起暂存、验证并原子替换；任何失败都保留前一组 committed 文件且不发 - `workspace.committed`。 -- Electron 直接读取并校验 Snapshot（路径、大小、JSON、schema、Workspace ID、 - Revision、section 和 Artifact 指纹），Runtime Event 只作为失效提示。 -- Snapshot 缺失、损坏或不支持时不回退到旧文件、不自动转换、不在查询时物化；旧 - Workspace 使用显式 `ecc workspace refresh`、重新创建或已有 `--overwrite` 流程。 - -### Execution Snapshot - -`workspace.snapshot` 是 Runtime 的运行态视图，由 live Session、Active -Operation 和有界 Operation Ledger 重建；它不替代 Engineering Snapshot。 - -- Ledger 始终保留 Active Operation 和最近 256 个 terminal Operation。 -- 更早记录及其 command-id 去重结果可以删除，不提供用户历史记录功能。 -- 进程重启后无法确认的非终态 Operation 统一恢复为 `interrupted`。 -- Event 可以重复、延迟或丢失；序列缺口或重启后必须重新查询 Execution Snapshot - 和 Engineering Snapshot。 - -## 3. 一致性、锁和 Revision - -- Workspace Revision 是唯一单调序列：创建为 1，配置/结构更新、rerun preparation、 - Layout 保存和每个 Step Commit 各推进一次。 -- Runtime 与 CLI 使用同一个 Workspace Ownership Lock（Workspace 同级的 - `.lock`）。Runtime fail-fast 返回 `workspace_busy`；同一 Runtime - 已有冲突 Operation 返回 `operation_conflict`。CLI 保持现有阻塞等待行为。 -- 同时触及 Project Manifest 和 Workspace 的 API 固定锁顺序： - `Project Manifest Lock -> Workspace Ownership Lock`。 -- Workspace-only 只取得 Workspace Lock；Manifest-only 只取得 Manifest Lock。 -- Runtime `workspace.open` 是短暂的 Open Transaction：校验 Snapshot、Bindings、 - Ledger 和恢复状态后释放锁。Operation 或 Layout Edit 才持有长生命周期锁。 -- Layout Edit 从 `layout.edit.begin` 持锁到 save/discard/close/process loss；save - 先提交 Revision 与 Snapshot，再释放锁；进程丢失不发布未提交编辑。 -- Bindings 只保存为机器本地绝对路径，不能写入 Descriptor/Manifest、推进 Revision - 或单独生成 Snapshot。 - -## 4. CLI 合同 - -### 写路径 - -下列 CLI 写操作全部改为直接调用 ECC Engine 的事务和提交路径： - -- Workspace 创建、Project 管理的 create/update 和 `ecc workspace refresh`； -- `ecc param set/unset --workspace`； -- rerun preparation、Layout 保存和 Flow 执行； -- `ecc run --overwrite`。 - -CLI 继续负责参数优先级、命令名称、阻塞锁、`--overwrite` 临时备份、输出记录和 -错误退出码。CLI 不调用 ECC Runtime，也不保留自己的 Snapshot/参数回滚实现。 -Project Manifest 与 Workspace 的联合更新使用现有 ECC API staging、Manifest 原子 -更新和失败回滚，不引入 two-phase commit。 - -CLI 解析 `--resume`、`--from`、`--to`、`--only`、`--force` 和 preset，解析为规范化 -有序 `step_ids` 后传给 Engine `ExecutionPlan`。Engine 负责依赖验证、执行、提交和 -Revision；Engine 不认识 CLI flag 名称。 - -Engine `ExecutionResult` 至少提供 `succeeded`、最终状态、可选 `step_id`、按执行顺序 -排列的 `executed_steps` 和可选 `failed_step`。CLI 只把它映射为现有 -`StepRunResult`/`CommandResult`，不重新扫描 `flow.json` 推断结果。 - -没有显式 rerun selector 且目标已经完成时，Engine 返回成功 no-op：不创建 Operation、 -不写 Workspace、不推进 Revision、不刷新 Engineering Snapshot。显式 `--force`、 -`--from`、`--only` 等仍然是 mutation。 - -### 读路径 - -第一阶段保留现有查询输出和实现边界： - -- `status` 直接读取 `home/flow.json`； -- `report step` 直接读取 Flow 和声明的 Step artifacts； -- `signoff inspect` 保留现有 Signoff 查询结果； -- `report qor/checklist/summary` 保留现有报告构建和写入默认或显式的 - `signoff/` 报告文件。 - -这些命令都不能创建或刷新 Engineering Snapshot。需要加载 Workspace 对象的报告和 -Signoff 查询必须使用 `load_workspace(read_only=True)`，不能触发迁移、事务恢复、 -配置文件名重写或 Workspace 日志写入；报告目标文件是唯一有意写入。 - -## 5. ECC 实施步骤 - -### Phase 0：基线与冻结 - -1. 固定 ECC git commit、Studio gitlink、当前磁盘文件 schema 和现有 CLI/RPC 测试。 -2. 建立跨组件 fixture：ECC 创建的 Workspace、Studio 创建的 Workspace、已有 - Snapshot、Snapshot-less Workspace、无效 Snapshot、revision 冲突和 lock 占用。 -3. 记录现有 `ecos/runtime-adapter` 的模块、测试、依赖、启动脚本和打包资源清单。 - -### Phase 1：ECC ownership migration（保持线协议兼容） - -1. 将 `ecos/runtime-adapter/ecos_runtime_adapter/*.py` 直接移动到 ECC 的 - `chipcompiler/runtime/`，按现有模块拆分重写 import；测试移动到 `test/runtime/`。 -2. 把 `jsonrpcserver>=5.0.9,<6` 移入 ECC `pyproject.toml`，删除 Adapter - requirements、wrapper、重复 package、重复测试和独立 Adapter binary。 -3. 保留 JSON-RPC method、字段、错误码、事件和内部旧 aliases，确保旧 Studio 与 - 新 ECC 的 parity；普通 CLI help 隐藏 `rpc`。 -4. 在 Engine 中实现统一 Workspace Ownership Lock、Manifest->Workspace 锁顺序、 - Atomic Engineering Commit、统一 Revision 和 Stable Domain Error 映射。 -5. 让 Engine 的 create/update/configuration/rerun/layout/execute 共享同一提交路径； - Flow Step Commit 必须先完成 Flow + Snapshot，再通知 Runtime。 -6. 将所有 CLI Workspace 写操作接入 Engine；保留只读命令输出和报告文件输出。 -7. 将 report/Signoff 的 Workspace hydration 改为 `read_only=True`，补充无副作用断言。 -8. 为 `ExecutionPlan`、`ExecutionResult`、no-op、`workspace_busy`、 - `operation_conflict`、revision conflict、overwrite rollback 和 Snapshot failure - 补齐 Engine/CLI/runtime 测试。 -9. 运行 ECC Ruff、全量 Python tests、PyInstaller 和 framed-stdio Runtime smoke。 - -### Phase 1.5：Studio 切换与跨仓库验证 - -1. 先合并并发布 ECC Phase 1，再更新 Studio ECC gitlink。 -2. Electron 启动命令切换为 pinned `ecc rpc serve --stdio --persistent-db`，删除 - Adapter binary staging、resource assertions 和独立 requirements 安装。 -3. 保留现有 Workspace Runtime/Control Runtime 路由、事件、取消、恢复、Layout 编辑 - 和 Project Snapshot reader；只替换后端进程来源。 -4. 验证 ECC 生成 Workspace 可被 Studio 打开，Studio 生成 Workspace 可继续被 CLI - 执行；验证 Project Comparison 直接读取持久化 Snapshot。 -5. 运行 Studio frozen install、typecheck、lint/format、Electron/Renderer tests、 - build、desktop smoke 和 parent packaging build。 - -### Phase 2：协议清理（必须完成） - -Phase 2 只有在 Phase 1 parity、打包和跨组件 smoke 全部通过后开始，不能无限期保留 -兼容分支： - -1. 强制 Control Runtime 和每个 Workspace Runtime 在任何业务命令前完成 `rpc.hello`。 -2. 使用 private protocol v2，返回 `protocolVersion`、ECC package version 和明确 - capabilities；缺失或不匹配在任何写操作前返回 `runtime_incompatible`。 -3. 删除过渡性的 `workspace.engineering_snapshot` RPC，统一为 Electron 直接读取 - `home/engineering-snapshot.json`。 -4. 删除 `adapterVersion`、`runtime.adapter.v1`、Adapter-named capability、旧 DTO、 - 旧字段 aliases 和兼容错误映射。 -5. 删除 `ecos-ecc-runtime-adapter` 名称、wrapper、资源断言和所有重复实现；CI 以 - grep/contract test 阻止旧名称回归。 -6. 保留一个 Snapshot validator、一个 Runtime capability vocabulary、一个 private - protocol version 和一个 canonical read path；ECC 与 Studio 作为一个 pinned update - 发布。 - -## 6. 失败与恢复语义 - -- Snapshot 提交失败：返回 `engineering_snapshot_commit_failed`，保留前一 Snapshot， - Flow Operation 失败，不发送 committed event。 -- Snapshot 缺失/损坏/版本不支持：查询返回稳定的 unavailable/invalid reason，不回退 - 为 `unstart`，不自动写文件；显式 refresh、run 或 overwrite 才能修复。 -- Runtime 无法确认进程被终止时：Ledger 恢复为 `interrupted`，不能伪装成 - `cancelled` 或 succeeded。 -- 普通 cancel 只在 Flow Step 边界生效；运行中的 EDA 工具继续完成当前 Step。 -- Force Quit 才会硬终止 Runtime，最多进行 3 秒清理，然后依靠 Ledger 恢复。 -- 已有 `--overwrite` 的临时备份只用于失败恢复；成功后删除，不形成用户历史归档。 - -## 7. 验收与合并门禁 - -必须同时满足： - -- ECC：Ruff、全量 Python suite、Engine/Runtime/CLI 聚焦测试、PyInstaller、framed - stdio smoke（hello、create/open、Execution Snapshot、Flow、cancel、recovery）。 -- Studio：frozen install、typecheck、lint/format、Electron/Renderer tests、build、 - desktop smoke、parent packaging build。 -- 跨组件：ECC->Studio Workspace、Studio->CLI Workspace、overwrite rollback、 - Snapshot missing/invalid、revision conflict、Workspace Busy、background run、 - Force Quit recovery、Layout Edit lock lifetime、报告输出兼容。 -- 打包产物：只包含一个 `ecc` executable，不包含 Adapter binary、wrapper 或第二套 - Python requirements。 -- 只读门禁：status/report/signoff 查询不迁移、不恢复、不追加 Workspace 日志；报告 - 命令只写其声明的报告目标。 -- 事件门禁：事件重复、延迟、丢失和序列缺口都能通过重新查询 Snapshot 恢复，正确性不 - 依赖 Renderer ACK。 - -## 8. 不在本方案内 - -- 不做旧 Workspace 或旧 Snapshot schema 的隐式迁移和 raw-file fallback。 -- 不保留第二个 Adapter forwarding package、symlink 或独立 runtime binary。 -- 不把 CLI 只读命令整体迁移为 Snapshot reader；后续若统一 read model，另立变更。 -- 不实现用户可见的 Operation 历史、无限 Ledger、Snapshot 历史时间旅行或通用 Artifact - 浏览器。 -- 不重做 Studio UI、QoR 算法、Signoff 判定、Project Manifest 领域归属或 ECC-FE。 - -## 9. 决策记录 - -本方案对应 Studio 架构 ADR-0033（ECC owns the private Studio runtime）以及 ECC -Headless Engine 既有 glossary。旧的 Studio-only Headless Engine Contract 文档只保留 -作历史记录，不能覆盖本文确认的 CLI 共用 Engine、Snapshot 边界和 Phase 2 清理要求。 diff --git a/docs/specification/glossary-headless-engine.md b/docs/specification/glossary-headless-engine.md deleted file mode 100644 index e8c3af42..00000000 --- a/docs/specification/glossary-headless-engine.md +++ /dev/null @@ -1,89 +0,0 @@ -# Glossary: Headless Engine and Workspace Lifecycle - -> **Historical vocabulary.** This glossary was written for the earlier -> Studio-only Headless Engine Contract. The current CLI/Studio ownership, -> Snapshot boundary and protocol phases are defined in -> [ECC Runtime Consolidation v1](ecc-runtime-consolidation-plan.zh-CN.md). - -Terms used by the Studio-owned Headless Engine Contract introduced in -PR 275. Implementation spec: -[headless-engine-contract.md](headless-engine-contract.md). -CLI vocabulary (`ecc.toml`, `--workspace`, `ecc run`) is unchanged. - -## Workspace - -A directory that holds one flow execution: `origin/`, `home/`, `config/`, -step directories, and logs. Identified on disk by its path. Studio also -identifies it by `workspaceId` inside the Engineering Snapshot. - -## Workspace Spec - -The Studio input that describes what a Workspace should contain: design -inputs, PDK binding, flow range, and parameters. Validated by -`validate_workspace_spec` before create or structural update. Parameter -applicability uses `normalize_flow_step()`, the same canonical step names -as flow construction. - -## Bindings - -Resolved host paths for Spec roles (RTL, filelist, PDK files, and so on). -The Spec names roles; Bindings point at files on this machine. - -## Engineering Snapshot - -`home/engineering-snapshot.json`. Studio's committed view of a Workspace: -`workspaceId`, `workspaceRevision`, analysis, QoR assessment, signoff -assessment, and artifact index. CLI-created workspaces do not have this -file in this PR. - -## Workspace Revision - -Monotonic integer in the Engineering Snapshot. Studio configuration and -structural updates are revision-aware: a stale expected revision is a -conflict, not a silent overwrite. - -## Headless Engine Contract - -The Studio Python APIs exported by `chipcompiler.engine`: create, -structural update, configuration update, query, `execute`, Engineering -Snapshot reads, and signoff inspect/export. Analysis builders and QoR -scoring stay internal. This PR does not make the CLI a client of that -contract. - -## Workspace Ownership Lock - -Sibling file `{workspace-name}.lock` next to the Workspace directory. It -survives directory replacement. Studio create / update / configuration -and CLI run / CLI workspace-param all use this filename, but this PR does -not treat CLI and Studio as one locking domain. - -## Configuration Transaction - -Studio-only disk backup used by `update_workspace_configuration`. It -restores params, flow, snapshot, and related files if a configuration -update fails or the process dies mid-commit. It runs only while the -Workspace Ownership Lock is already held. It is not a second lock file, -and it is not CLI parameter rollback. - -## CLI Parameter Rollback - -In-memory bytes snapshot used by `ecc param --workspace`. Process-local -only. Pre-existing CLI behavior; not merged into the Configuration -Transaction in this PR. - -## `execute` - -Studio entry point that runs an `EngineFlow` with an `ExecutionPlan` and -commits the Engineering Snapshot after completed steps. CLI does not call -it in this PR. - -## `EngineFlow` - -The existing flow runner. CLI `ecc run` calls it directly. Studio -reaches it through `execute`. - -## Read-only Workspace Load - -`load_workspace(..., read_only=True)`. Studio query APIs use this so -inspection cannot migrate configs, create `home/` files, or write logs. -Mutating Studio APIs load a writable Workspace. diff --git a/docs/specification/headless-engine-contract.md b/docs/specification/headless-engine-contract.md deleted file mode 100644 index 9c9154f8..00000000 --- a/docs/specification/headless-engine-contract.md +++ /dev/null @@ -1,290 +0,0 @@ -# Headless Engine Contract (PR 275 follow-through) - -> **Historical document.** This earlier spec described a Studio-only Engine -> contract and explicitly left CLI writes on the old path. It is superseded by -> [ECC Runtime Consolidation v1](ecc-runtime-consolidation-plan.zh-CN.md), which -> makes ECC Engine the shared mutation path for CLI and Studio while keeping CLI -> read commands and report output compatible. Do not use the old CLI scope or -> Snapshot ownership statements below as the current contract. - -Status: ready for implementation. Scope is the Studio-owned Headless Engine -Contract already sketched in this branch; this spec closes the gaps decided -during review. Vocabulary follows -[glossary-headless-engine.md](glossary-headless-engine.md). Binding ADR: -[adr-headless-engine-contract.md](adr-headless-engine-contract.md). - -## Problem Statement - -Studio needs a process-local Python contract to create, inspect, configure, -and run a Workspace after the in-process RPC runtime is gone. The branch -already exposes lifecycle and configuration APIs, but they are not yet a -single contract: configuration uses a second lock file, query APIs can -migrate the Workspace, create can delete a directory it did not create, -execution is not the documented Studio entry, and the public engine surface -does not export snapshot or signoff reads. - -CLI already has its own ownership lock and in-memory parameter rollback. -Those must keep working. This spec does not ask CLI to adopt the Headless -Engine Contract. - -## Solution - -Finish the Studio-only Headless Engine Contract on this branch: - -- One Workspace Ownership Lock for every Studio mutation. -- Configuration Transaction as disk backup only, never a second lock. -- Query APIs load a Workspace read-only and never migrate. -- Studio execution goes through `execute` and commits the Engineering - Snapshot. -- Create failure deletes only a directory this call created. -- The engine package exports snapshot reads and signoff inspect/export, not - scoring internals. -- Workspace Spec parameter applicability uses canonical flow step names. - -CLI `ecc run` and `ecc param --workspace` stay on their existing paths. - -## User Stories - -1. As a Studio engineer, I want to create a Workspace from a Workspace Spec - and Bindings, so that the GUI does not speak RPC. -2. As a Studio engineer, I want a successful create to write an Engineering - Snapshot at revision 1, so that later updates have a committed identity. -3. As a Studio engineer, I want repeating the same create command id to - return the existing Workspace, so that retries are safe. -4. As a Studio engineer, I want create to fail with `workspace_exists` when - the directory already belongs to someone else, so that another Workspace - is never deleted. -5. As a Studio engineer, I want create to remove only the directory this - call created if setup fails after mkdir, so that half-written trees are - cleaned up without touching pre-existing data. -6. As a Studio engineer, I want structural update to replace design, PDK, - flow range, and parameters atomically, so that a failed update does not - leave a mixed Workspace. -7. As a Studio engineer, I want structural update to require the expected - Workspace Revision, so that two Studio sessions cannot silently overwrite - each other. -8. As a Studio engineer, I want a matching command id on structural update - to be idempotent, so that a retried request does not bump the revision - twice. -9. As a Studio engineer, I want a stale expected revision to fail with - `revision_conflict` without creating a missing snapshot, so that conflict - handling cannot invent identity. -10. As a Studio engineer, I want configuration update to change only - parameters (not design or PDK structure), so that geometry and PDK - edits stay on the structural-update path. -11. As a Studio engineer, I want configuration update to take the same - Workspace Ownership Lock as create and structural update, so that - Studio cannot configure a Workspace while another Studio mutation - replaces it. -12. As a Studio engineer, I want configuration update to keep a disk backup - and restore it if refresh or invalidate fails, so that params, flow, - and snapshot stay consistent. -13. As a Studio engineer, I want configuration crash recovery to run only - while the Workspace Ownership Lock is already held, so that restore - cannot race another Studio mutation. -14. As a Studio engineer, I want no `.configuration.lock` sibling file, so - that lock ownership is one filename and one story. -15. As a Studio engineer, I want `describe_workspace_binding_requirement` - to report family, version, and mode without writing files, so that - opening a Workspace in the GUI is inspection, not migration. -16. As a Studio engineer, I want `assess_execution_readiness` to say whether - Bindings are enough to run, without creating `home/` files or logs. -17. As a Studio engineer, I want `read_workspace_configuration` and - `read_step_configuration` from a directory to be read-only, so that - viewing settings cannot rewrite configs. -18. As a Studio engineer, I want a missing `home.json` or checklist to stay - missing after a query, so that inspection never materializes defaults. -19. As a Studio engineer, I want `execute` to run a full range or a single - step, so that the GUI has one run entry. -20. As a Studio engineer, I want `execute` to commit the Engineering - Snapshot after completed steps, so that QoR and artifacts in the GUI - match disk. -21. As a Studio engineer, I want a failed `execute` to keep the same - blocking semantics as `EngineFlow` (stop vs continue), so that run - behavior does not fork. -22. As a Studio engineer, I want to read the committed Engineering Snapshot - from the engine package, so that the GUI does not import snapshot - internals. -23. As a Studio engineer, I want to inspect the current Signoff Assessment - without persisting a checklist, so that signoff views are safe. -24. As a Studio engineer, I want to export a signoff archive through the - engine package, so that package creation stays on the same contract. -25. As a Studio engineer, I want analysis builders and QoR scoring to stay - internal, so that the GUI reads committed snapshot data instead of - recomputing scores. -26. As a Studio engineer, I want Workspace Spec validation to accept the - same step spellings as flow construction, so that `synth` and - `Synthesis` are not two applicability worlds. -27. As a Studio engineer, I want a parameter that does not apply to the - requested flow range to be a structured spec issue, so that the GUI can - show the field path. -28. As a Studio engineer, I want unused runtime flow helpers gone, so that - the only Studio run path is `execute` over `EngineFlow`. -29. As a CLI user, I want `ecc run` to keep calling `EngineFlow` directly, - so that CLI workspaces do not suddenly grow an Engineering Snapshot. -30. As a CLI user, I want `ecc param --workspace` to keep in-memory - rollback and the sibling ownership lock, so that parameter edits still - undo on refresh failure in the same process. -31. As a CLI user, I want CLI and Studio to share a lock filename without - this change promising cross-product exclusion, so that CLI behavior - stays what it was on main. -32. As a CLI user, I want `ecc signoff inspect` unchanged, so that CLI - inspection is not silently rewritten as a Studio query. -33. As a reviewer, I want the engine package export list to be the Studio - contract, so that deep imports are not required for snapshot or - signoff. -34. As a reviewer, I want tests to speak only through public engine APIs - and on-disk Workspace files, so that lock-file names inside the - transaction helper can change without rewriting the suite. -35. As a Studio engineer, I want Bindings applied at execute/update time, - so that a Workspace can be copied to another machine and rebound. -36. As a Studio engineer, I want a PDK root mismatch to be - `pdk_binding_mismatch`, so that the GUI can ask the user to rebind - instead of running the wrong PDK. -37. As a Studio engineer, I want step configuration update to invalidate - only from the target step, so that earlier successful steps are not - thrown away. -38. As a Studio engineer, I want configuration update to reject design or - PDK sections, so that the GUI cannot hide a structural change inside - a parameter save. -39. As a Studio engineer, I want `execute` to apply Bindings before - running, so that a default-PDK Workspace can resolve files on this - host. -40. As a maintainer, I want `execute` to call `EngineFlow` methods - directly, so that runtime signature probing is not part of the - contract. - -## Implementation Decisions - -- The Headless Engine Contract is Studio-only for this change. CLI does not - call create-from-spec, configuration update, or `execute`. -- Studio mutations (create, structural update, configuration update) all - acquire the Workspace Ownership Lock: the sibling `{workspace-name}.lock` - next to the Workspace directory. That filename already exists for CLI - runs; this change does not add a second Studio ownership lock. -- The Configuration Transaction keeps its disk backup under the Workspace - `home/` directory and still restores on failure or crash. It must not - open or flock a `.configuration.lock` file. Backup, commit, rollback, and - crash recovery run only while the caller already holds the Workspace - Ownership Lock. -- Writable `load_workspace` (and therefore transaction crash recovery - hooked off load) is allowed on mutating Studio paths. Query paths must - not go through that load. -- Query APIs are `describe_workspace_binding_requirement`, - `assess_execution_readiness`, `read_workspace_configuration_from_directory`, - `read_step_configuration_from_directory`, Engineering Snapshot reads from - a directory, and signoff inspect. Each loads with `read_only=True` and - must not migrate filenames, create `home/` files, rebuild a persisted - checklist, or append logs. -- In-memory Workspace objects passed into `read_workspace_configuration` / - `read_step_configuration` are already loaded; those helpers must not - persist side effects either. -- `create_workspace_from_spec` treats an existing target as - `workspace_exists`, except when `command_id` matches a completed create - (idempotent retry). Failure cleanup may `rmtree` only if this invocation - created the target directory. A pre-existing directory is never removed. -- Structural update may keep using a staging directory that this call - created; that staging directory is owned by the call and may be deleted - on failure. -- Studio execution uses `execute` with an `ExecutionPlan`. The default - observer commits the Engineering Snapshot after completed steps. CLI - continues to construct `EngineFlow` and call `run_steps` / `run_step`. -- `execute` calls `EngineFlow.run_steps` / `run_step` directly with `rerun` - and observer arguments. No `inspect.signature` probing. -- Delete unused runtime-extraction helpers that have no production callers - (`workspace_step_from_flow`, DB-init-for-step, success/state adapters). - Tests that only needed `build_flow_for_workspace` should construct - `EngineFlow` themselves. -- `chipcompiler.engine` exports, in addition to the current lifecycle / - configuration / `execute` surface: Engineering Snapshot read helpers and - signoff inspect/export. It does not export analysis builders or - `score_qor`. -- Snapshot *writes* stay internal to create, update, configuration, and - `execute`. Studio reads committed snapshots; it does not call create / - ensure / invalidate snapshot helpers as public API. -- Workspace Spec parameter applicability uses `normalize_flow_step()` for - both the requested flow steps and each schema's `applies` value. The - handwritten alias map (`synthesis→synth`, and the rest) is deleted. - Parameters with `applies == "all"` remain applicable to any range. -- Keep CLI in-memory parameter rollback and CLI use of the sibling lock. - Do not route CLI param through the Configuration Transaction. -- Keep CLI `sys.modules` manifest aliases and CLI signoff inspect as they - are. -- Do not split the metrics module in this change. -- Typed errors stay `WorkspaceLifecycleError` with stable `code` values - already used by create/update/configuration (`workspace_exists`, - `revision_conflict`, `workspace_spec_invalid`, `pdk_binding_missing`, - `pdk_binding_mismatch`, `workspace_structure_change_requires_update`, - and the existing snapshot/config invalid codes). - -## Testing Decisions - -Good tests assert observable contract behavior: return values, exception -codes, Engineering Snapshot identity/revision, and whether files on disk -appeared, changed, or stayed absent. They do not assert the Configuration -Transaction's internal lock filename, backup directory name, or helper -class. They do not import analysis/scoring modules to prove the public -contract. - -Highest seam: `chipcompiler.engine` public APIs plus the Workspace -directory they mutate. That is the existing `test/engine/` seam -(`test_workspace_spec.py`, `test_workspace_configuration.py`, -`test_execution.py`, `test_signoff_assessment.py`). Prefer extending those -modules over adding a new test package. - -Cover: - -- Create idempotent retry vs `workspace_exists`; create failure does not - delete a pre-existing directory (the call did not create it). -- Configuration update and create/structural update cannot interleave: a - holder of the Workspace Ownership Lock blocks the other Studio mutation - (same style as existing CLI lock-wait tests, but through engine APIs). -- After removing the independent configuration lock, crash-recovery tests - still restore params/snapshot when the process dies mid-commit; recovery - is observed via a later mutating API or a read of committed files, not - via the transaction lock path. -- Query APIs with missing `home.json` / checklist / log dir / missing PDK - root do not recreate those paths. Extend the existing read-only - configuration test to `describe`, `assess`, and step-configuration read. -- `execute` commits snapshot after a completed step; already present in - `test_execution.py` and remains required. -- Spec applicability: a parameter whose canonical `applies` step is - outside the requested range is a structured issue; a non-canonical - spelling that `normalize_flow_step()` maps into the range is allowed. -- Engine package export list includes snapshot reads and signoff - inspect/export, and does not include `score_qor` or analysis builders. -- Existing revision/idempotency/rollback tests in workspace spec and - configuration suites still pass. - -CLI suites are regression guards, not the place to add Studio contract -tests. Do not add tests that require CLI to write an Engineering Snapshot. - -## Out of Scope - -- Wiring CLI create, refresh, param, or run through the Headless Engine - Contract. -- Making CLI and Studio one locking domain (cross-product mutual - exclusion). -- Replacing CLI Parameter Rollback with the Configuration Transaction. -- Changing CLI manifest import aliases. -- Making CLI signoff inspect a read-only load. -- Splitting the metrics module. -- Exporting analysis builders or QoR scoring from the engine package. -- Teaching CLI `ecc run` to write an Engineering Snapshot. -- Full-suite, PyInstaller, and Nix runs as spec items (merge checklist - only). - -## Further Notes - -`main` already had two products. CLI serialized with the sibling ownership -lock and rolled back workspace params from in-memory bytes. RPC/Studio -serialized with a process-local session lock and had no file-backed -configuration transaction. This spec does not merge those histories; it -gives Studio a file-backed contract after RPC removal. - -The independent `.configuration.lock` file is new on this branch, not -inherited from `main`. Removing it is restoring a single ownership lock, -not deleting CLI behavior. - -A later change may adopt the Headless Engine Contract in CLI. That is a -separate decision. From 14e1025cdf6a96327f5cd35803b49fd1a197ebdd Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Mon, 14 Sep 2026 15:05:51 +0800 Subject: [PATCH 32/43] feat(snapshot): prepare explicit QoR v3 migration --- chipcompiler/engine/__init__.py | 6 + chipcompiler/engine/snapshot.py | 79 +++++++++- chipcompiler/engine/snapshot_qor.py | 208 +++++++++++++++++++++++++ test/engine/test_snapshot_migration.py | 83 ++++++++++ 4 files changed, 374 insertions(+), 2 deletions(-) create mode 100644 chipcompiler/engine/snapshot_qor.py create mode 100644 test/engine/test_snapshot_migration.py diff --git a/chipcompiler/engine/__init__.py b/chipcompiler/engine/__init__.py index c523bb8e..bff9d382 100644 --- a/chipcompiler/engine/__init__.py +++ b/chipcompiler/engine/__init__.py @@ -3,6 +3,10 @@ from .flow import EngineFlow from .rerun import StepRunResult from .signoff import SignoffPackageCollector, SignoffPackageOptions +from .snapshot import ( + migrate_engineering_snapshot, + migrate_engineering_snapshot_v2_to_v3, +) from .workspace_configuration import ( read_step_configuration, read_step_configuration_from_directory, @@ -24,6 +28,8 @@ __all__ = [ "EngineDB", "EngineFlow", + "migrate_engineering_snapshot", + "migrate_engineering_snapshot_v2_to_v3", "ExecutionPlan", "ExecutionResult", "StepRunResult", diff --git a/chipcompiler/engine/snapshot.py b/chipcompiler/engine/snapshot.py index 8ccd332a..aa16fde5 100644 --- a/chipcompiler/engine/snapshot.py +++ b/chipcompiler/engine/snapshot.py @@ -3,11 +3,21 @@ from typing import Any from uuid import uuid4 +from chipcompiler.engine.snapshot_qor import ( + build_qor_snapshot_extension, + unavailable_qor_snapshot_extension, + validate_qor_snapshot_extension, +) from chipcompiler.utility import JsonReadError, json_read, json_read_strict, json_write SNAPSHOT_SCHEMA_VERSION = 2 +SNAPSHOT_V3_SCHEMA_VERSION = 3 +SUPPORTED_SNAPSHOT_SCHEMA_VERSIONS = frozenset( + {SNAPSHOT_SCHEMA_VERSION, SNAPSHOT_V3_SCHEMA_VERSION} +) SNAPSHOT_FILENAME = "engineering-snapshot.json" STALE_SNAPSHOT_FILENAME = "engineering-snapshot.stale.json" +SNAPSHOT_V2_TO_V3_MIGRATION_CAUSE = "snapshot.migrated.v2_to_v3" class EngineeringSnapshotError(RuntimeError): @@ -51,6 +61,49 @@ def read_stale_engineering_snapshot(workspace: Any) -> dict[str, Any] | None: return _read_snapshot(path) if path.is_file() else None +def migrate_engineering_snapshot( + workspace: Any, + *, + expected_workspace_revision: int | None = None, + cause: str = SNAPSHOT_V2_TO_V3_MIGRATION_CAUSE, +) -> dict[str, Any]: + """Explicitly migrate one v2 Snapshot to the prepared v3 contract. + + Normal Snapshot producers remain pinned to v2. This write-only seam is the + only path that emits v3 until ECC and Studio switch the production contract. + """ + current = _read_snapshot(_snapshot_path(workspace)) + if current["schemaVersion"] != SNAPSHOT_SCHEMA_VERSION: + raise EngineeringSnapshotError("Snapshot migration requires schemaVersion 2") + if ( + expected_workspace_revision is not None + and current["workspaceRevision"] != expected_workspace_revision + ): + raise EngineeringSnapshotError( + "Workspace Revision does not match before Snapshot migration" + ) + try: + snapshot = _build_snapshot( + workspace, + workspace_id=current["workspaceId"], + workspace_revision=current["workspaceRevision"] + 1, + cause=cause, + schema_version=SNAPSHOT_V3_SCHEMA_VERSION, + strict_qor=True, + ) + except Exception as exc: + raise EngineeringSnapshotError( + "failed to regenerate QoR facts for Snapshot migration" + ) from exc + if isinstance(current.get("stalePredecessor"), dict): + snapshot["stalePredecessor"] = deepcopy(current["stalePredecessor"]) + _write_snapshot(_snapshot_path(workspace), snapshot) + return snapshot + + +migrate_engineering_snapshot_v2_to_v3 = migrate_engineering_snapshot + + def commit_engineering_snapshot( workspace: Any, *, @@ -58,6 +111,8 @@ def commit_engineering_snapshot( cause: str, ) -> dict[str, Any]: current = read_engineering_snapshot(workspace) + if current["schemaVersion"] != SNAPSHOT_SCHEMA_VERSION: + raise EngineeringSnapshotError("production Snapshot schema is still v2") if current["workspaceId"] != workspace_id: raise EngineeringSnapshotError("Workspace identity changed before commit") snapshot = _build_snapshot( @@ -94,6 +149,8 @@ def invalidate_engineering_snapshot( first_invalidated_step: str | None = None, ) -> dict[str, Any]: current = read_engineering_snapshot(workspace) + if current["schemaVersion"] != SNAPSHOT_SCHEMA_VERSION: + raise EngineeringSnapshotError("production Snapshot schema is still v2") if current["workspaceId"] != workspace_id: raise EngineeringSnapshotError("Workspace identity changed before invalidation") flow = deepcopy(current.get("flow", {})) @@ -145,6 +202,8 @@ def _build_snapshot( workspace_id: str, workspace_revision: int, cause: str, + schema_version: int = SNAPSHOT_SCHEMA_VERSION, + strict_qor: bool = False, ) -> dict[str, Any]: flow_owner = getattr(workspace, "flow", None) flow = _data_mapping(flow_owner) @@ -160,8 +219,19 @@ def _build_snapshot( analysis, artifacts = build_workspace_analysis(workspace, workspace_id) qor_assessment = build_workspace_qor_assessment(analysis) + try: + from chipcompiler.analysis.qor import build_qor_analysis + + qor_extension = build_qor_snapshot_extension( + build_qor_analysis(workspace), + artifacts, + ) + except Exception as exc: + if strict_qor: + raise + qor_extension = unavailable_qor_snapshot_extension(str(exc)) return { - "schemaVersion": SNAPSHOT_SCHEMA_VERSION, + "schemaVersion": schema_version, "workspaceId": workspace_id, "workspaceRevision": workspace_revision, "cause": cause, @@ -171,6 +241,7 @@ def _build_snapshot( "analysis": analysis, "metrics": deepcopy(qor_assessment["metrics"]), "qorAssessment": qor_assessment, + "qorSnapshotExtension": qor_extension, "signoffAssessment": build_signoff_assessment(workspace), "artifacts": artifacts, } @@ -202,12 +273,16 @@ def _read_snapshot(path: Path) -> dict[str, Any]: raise EngineeringSnapshotError(f"invalid Engineering Snapshot: {path}") from exc if ( not isinstance(snapshot, dict) - or snapshot.get("schemaVersion") != SNAPSHOT_SCHEMA_VERSION + or snapshot.get("schemaVersion") not in SUPPORTED_SNAPSHOT_SCHEMA_VERSIONS or not isinstance(snapshot.get("workspaceId"), str) or not snapshot["workspaceId"] or isinstance(snapshot.get("workspaceRevision"), bool) or not isinstance(snapshot.get("workspaceRevision"), int) or snapshot["workspaceRevision"] < 1 + or ( + snapshot.get("schemaVersion") == SNAPSHOT_V3_SCHEMA_VERSION + and not validate_qor_snapshot_extension(snapshot.get("qorSnapshotExtension")) + ) ): raise EngineeringSnapshotError(f"invalid Engineering Snapshot: {path}") return snapshot diff --git a/chipcompiler/engine/snapshot_qor.py b/chipcompiler/engine/snapshot_qor.py new file mode 100644 index 00000000..9b42d025 --- /dev/null +++ b/chipcompiler/engine/snapshot_qor.py @@ -0,0 +1,208 @@ +"""Bounded QoR v3 facts embedded in an Engineering Snapshot.""" + +from math import isfinite +from typing import Any + +QOR_SNAPSHOT_EXTENSION_SCHEMA_VERSION = 1 +_MAX_DIAGNOSES = 64 +_MAX_INTERVENTIONS = 4 +_MAX_ARTIFACT_IDS = 512 +_MAX_TEXT = 512 + + +def build_qor_snapshot_extension(analysis: Any, artifacts: list[dict[str, Any]]) -> dict[str, Any]: + """Project the report into bounded, path-free committed QoR facts.""" + report = analysis.to_dict() if hasattr(analysis, "to_dict") else analysis + if not isinstance(report, dict): + raise ValueError("QoR analysis is not an object") + + dimensions = {} + for key, dimension in report.get("qor_record", {}).items(): + if not isinstance(key, str) or not isinstance(dimension, dict): + continue + features = dimension.get("features", []) + dimensions[key] = { + "value": _number_or_none(dimension.get("value")), + "state": _text(dimension.get("state"), "UNKNOWN"), + "featureIds": [ + feature["feature_id"] + for feature in features[:32] + if isinstance(feature, dict) and isinstance(feature.get("feature_id"), str) + ], + } + + feasibility = report.get("feasibility") + feasibility = feasibility if isinstance(feasibility, dict) else {} + gates = [] + for gate in feasibility.get("gates", []): + if not isinstance(gate, dict) or not isinstance(gate.get("id"), str): + continue + gates.append( + { + "id": gate["id"], + "stage": _text(gate.get("stage")), + "state": _text(gate.get("state"), "unavailable"), + "blocksTapeout": bool(gate.get("blocks_tapeout")), + "metrics": [ + metric for metric in gate.get("metrics", [])[:32] if isinstance(metric, str) + ], + "availability": gate.get("availability") + if isinstance(gate.get("availability"), str) + else None, + } + ) + + summary = report.get("scalar_summary") + summary = summary if isinstance(summary, dict) else {} + evidence = report.get("evidence") + evidence = evidence if isinstance(evidence, dict) else {} + inflation = report.get("inflation") + inflation = inflation if isinstance(inflation, dict) else {} + power = report.get("power") + power = power if isinstance(power, dict) else {} + + return { + "schemaVersion": QOR_SNAPSHOT_EXTENSION_SCHEMA_VERSION, + "scoringEngine": "qor-v3", + "status": "available", + "score": _number_or_none(summary.get("score")), + "scalarStatus": _text(summary.get("status"), "NOT_RATED"), + "profile": _text(summary.get("profile"), "balanced"), + "qphys": dimensions, + "feasibility": { + "status": _text(feasibility.get("status"), "UNKNOWN"), + "gates": gates[:32], + }, + "evidence": { + "index": _number_or_none(evidence.get("index")), + "state": _text(evidence.get("state"), "NOT_VERIFIED"), + "integrity": _number_or_none(evidence.get("integrity")), + "coverage": _number_or_none(evidence.get("coverage")), + "consistency": _number_or_none(evidence.get("consistency")), + }, + "diagnoses": [_diagnosis(item) for item in report.get("diagnoses", [])[:_MAX_DIAGNOSES]], + "inflation": { + "iPlace": _number_or_none(inflation.get("i_place")), + "iRoute": _number_or_none(inflation.get("i_route")), + "iTotal": _number_or_none(inflation.get("i_total")), + "congestionSeverity": _number_or_none(inflation.get("congestion_severity")), + "compatibilityStatus": _text(inflation.get("compatibility_status")), + }, + "power": { + "totalUw": _number_or_none(power.get("total_uw")), + "budgetUw": _number_or_none(power.get("budget_uw")), + "sourceKind": power.get("source_kind") + if isinstance(power.get("source_kind"), str) + else None, + "corner": power.get("corner") if isinstance(power.get("corner"), str) else None, + }, + "artifactIds": _artifact_ids(artifacts), + } + + +def unavailable_qor_snapshot_extension(reason: str) -> dict[str, Any]: + """Return an explicit unavailable extension without inventing QoR values.""" + return { + "schemaVersion": QOR_SNAPSHOT_EXTENSION_SCHEMA_VERSION, + "scoringEngine": "qor-v3", + "status": "unavailable", + "reason": _text(reason, "QoR analysis unavailable"), + "score": None, + "scalarStatus": "NOT_RATED", + "profile": "balanced", + "qphys": {}, + "feasibility": {"status": "UNKNOWN", "gates": []}, + "evidence": { + "index": None, + "state": "NOT_VERIFIED", + "integrity": None, + "coverage": None, + "consistency": None, + }, + "diagnoses": [], + "inflation": { + "iPlace": None, + "iRoute": None, + "iTotal": None, + "congestionSeverity": None, + "compatibilityStatus": "UNAVAILABLE", + }, + "power": {"totalUw": None, "budgetUw": None, "sourceKind": None, "corner": None}, + "artifactIds": [], + } + + +def validate_qor_snapshot_extension(value: object) -> bool: + """Validate the migration-only extension boundary.""" + if not isinstance(value, dict): + return False + return ( + value.get("schemaVersion") == QOR_SNAPSHOT_EXTENSION_SCHEMA_VERSION + and value.get("scoringEngine") == "qor-v3" + and value.get("status") in {"available", "unavailable"} + and isinstance(value.get("qphys"), dict) + and isinstance(value.get("feasibility"), dict) + and isinstance(value["feasibility"].get("gates"), list) + and isinstance(value.get("evidence"), dict) + and isinstance(value.get("diagnoses"), list) + and isinstance(value.get("artifactIds"), list) + and all(isinstance(item, str) and item for item in value["artifactIds"]) + ) + + +def _diagnosis(value: object) -> dict[str, Any]: + if not isinstance(value, dict): + return {"diagnosisId": "unknown", "state": "UNKNOWN", "severity": None} + interventions = [] + for item in value.get("interventions", [])[:_MAX_INTERVENTIONS]: + if not isinstance(item, dict): + continue + interventions.append( + { + "hypothesis": _text(item.get("hypothesis")), + "tier": _text(item.get("tier")), + "confidence": _text(item.get("confidence"), "LOW"), + "parameterKnob": item.get("parameter_knob") + if isinstance(item.get("parameter_knob"), str) + else None, + "validationProcedure": item.get("validation_procedure") + if isinstance(item.get("validation_procedure"), str) + else None, + } + ) + return { + "diagnosisId": _text(value.get("diagnosis_id"), "unknown"), + "state": _text(value.get("state"), "UNKNOWN"), + "severity": _number_or_none(value.get("severity")), + "confidence": _text(value.get("diagnosis_confidence"), "LOW"), + "triggerFeatures": [ + item for item in value.get("trigger_features", [])[:32] if isinstance(item, str) + ], + "affectedDimensions": [ + item for item in value.get("affected_dimensions", [])[:16] if isinstance(item, str) + ], + "interventions": interventions, + "interventionConfidence": _text(value.get("intervention_confidence"), "LOW"), + "validationRequired": value.get("validation_required") + if isinstance(value.get("validation_required"), str) + else None, + } + + +def _artifact_ids(artifacts: list[dict[str, Any]]) -> list[str]: + ids = { + item["artifactId"] + for item in artifacts + if isinstance(item, dict) and isinstance(item.get("artifactId"), str) + } + return sorted(ids)[:_MAX_ARTIFACT_IDS] + + +def _number_or_none(value: object) -> float | int | None: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value): + return None + return value + + +def _text(value: object, default: str = "") -> str: + return value[:_MAX_TEXT] if isinstance(value, str) and value else default diff --git a/test/engine/test_snapshot_migration.py b/test/engine/test_snapshot_migration.py new file mode 100644 index 00000000..e117feee --- /dev/null +++ b/test/engine/test_snapshot_migration.py @@ -0,0 +1,83 @@ +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from chipcompiler.engine.snapshot import ( + SNAPSHOT_V3_SCHEMA_VERSION, + EngineeringSnapshotError, + create_engineering_snapshot, + migrate_engineering_snapshot, + read_engineering_snapshot, +) + + +def _workspace(tmp_path): + root = tmp_path / "workspace" + (root / "home").mkdir(parents=True) + steps = [{"name": "Synthesis", "tool": "yosys", "state": "Success"}] + (root / "home" / "flow.json").write_text(json.dumps({"steps": steps}), encoding="utf-8") + return SimpleNamespace( + directory=root, + flow=SimpleNamespace(data={"steps": steps}), + home=SimpleNamespace(data={}), + parameters=SimpleNamespace(data={"design": "gcd"}), + design=SimpleNamespace(name="gcd"), + ) + + +def test_migration_preserves_identity_advances_revision_and_projects_qor(tmp_path): + workspace = _workspace(tmp_path) + current = create_engineering_snapshot(workspace, workspace_id="engineering-gcd") + + migrated = migrate_engineering_snapshot( + workspace, + expected_workspace_revision=current["workspaceRevision"], + ) + + assert migrated["schemaVersion"] == SNAPSHOT_V3_SCHEMA_VERSION + assert migrated["workspaceId"] == "engineering-gcd" + assert migrated["workspaceRevision"] == current["workspaceRevision"] + 1 + assert migrated["cause"] == "snapshot.migrated.v2_to_v3" + assert migrated["qorSnapshotExtension"]["scoringEngine"] == "qor-v3" + assert read_engineering_snapshot(workspace)["schemaVersion"] == SNAPSHOT_V3_SCHEMA_VERSION + + +def test_migration_preserves_stale_predecessor_metadata(tmp_path): + workspace = _workspace(tmp_path) + current = create_engineering_snapshot(workspace, workspace_id="engineering-gcd") + path = Path(workspace.directory) / "home" / "engineering-snapshot.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["stalePredecessor"] = {"workspaceRevision": 1, "invalidatedStepIds": ["route"]} + path.write_text(json.dumps(payload), encoding="utf-8") + + migrated = migrate_engineering_snapshot(workspace) + + assert migrated["workspaceRevision"] == current["workspaceRevision"] + 1 + assert migrated["stalePredecessor"] == payload["stalePredecessor"] + + +def test_migration_failure_keeps_previous_snapshot(tmp_path, monkeypatch): + workspace = _workspace(tmp_path) + create_engineering_snapshot(workspace, workspace_id="engineering-gcd") + path = Path(workspace.directory) / "home" / "engineering-snapshot.json" + before = path.read_bytes() + + def fail(_workspace): + raise RuntimeError("broken analysis") + + monkeypatch.setattr("chipcompiler.analysis.qor.build_qor_analysis", fail) + with pytest.raises(EngineeringSnapshotError, match="regenerate QoR facts"): + migrate_engineering_snapshot(workspace) + + assert path.read_bytes() == before + + +def test_unsupported_schema_is_not_migrated(tmp_path): + workspace = _workspace(tmp_path) + path = Path(workspace.directory) / "home" / "engineering-snapshot.json" + path.write_text(json.dumps({"schemaVersion": 99}), encoding="utf-8") + + with pytest.raises(EngineeringSnapshotError, match="invalid Engineering Snapshot"): + migrate_engineering_snapshot(workspace) From 311976f4578ec8e01fbcfb2012f7c8c35ffb307d Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Mon, 14 Sep 2026 15:12:30 +0800 Subject: [PATCH 33/43] fix(snapshot): block v3 writes before execution --- chipcompiler/engine/snapshot.py | 5 ++++- test/engine/test_snapshot_migration.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/chipcompiler/engine/snapshot.py b/chipcompiler/engine/snapshot.py index aa16fde5..979e2ba5 100644 --- a/chipcompiler/engine/snapshot.py +++ b/chipcompiler/engine/snapshot.py @@ -44,7 +44,10 @@ def create_engineering_snapshot( def ensure_engineering_snapshot(workspace: Any) -> dict[str, Any]: path = _snapshot_path(workspace) if path.is_file(): - return _read_snapshot(path) + snapshot = _read_snapshot(path) + if snapshot["schemaVersion"] != SNAPSHOT_SCHEMA_VERSION: + raise EngineeringSnapshotError("production Snapshot schema is still v2") + return snapshot return create_engineering_snapshot(workspace, cause="workspace.migrated") diff --git a/test/engine/test_snapshot_migration.py b/test/engine/test_snapshot_migration.py index e117feee..ffad1390 100644 --- a/test/engine/test_snapshot_migration.py +++ b/test/engine/test_snapshot_migration.py @@ -8,6 +8,7 @@ SNAPSHOT_V3_SCHEMA_VERSION, EngineeringSnapshotError, create_engineering_snapshot, + ensure_engineering_snapshot, migrate_engineering_snapshot, read_engineering_snapshot, ) @@ -81,3 +82,12 @@ def test_unsupported_schema_is_not_migrated(tmp_path): with pytest.raises(EngineeringSnapshotError, match="invalid Engineering Snapshot"): migrate_engineering_snapshot(workspace) + + +def test_production_write_paths_reject_migrated_v3_snapshot(tmp_path): + workspace = _workspace(tmp_path) + create_engineering_snapshot(workspace, workspace_id="engineering-gcd") + migrate_engineering_snapshot(workspace) + + with pytest.raises(EngineeringSnapshotError, match="production Snapshot schema is still v2"): + ensure_engineering_snapshot(workspace) From 11e306d1f1bd9dc6c0c1a975dbb9c3feb91f2e30 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Mon, 14 Sep 2026 15:16:58 +0800 Subject: [PATCH 34/43] test(snapshot): assert v2 rollout remains path-free --- test/engine/test_snapshot_migration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/engine/test_snapshot_migration.py b/test/engine/test_snapshot_migration.py index ffad1390..13728909 100644 --- a/test/engine/test_snapshot_migration.py +++ b/test/engine/test_snapshot_migration.py @@ -31,6 +31,8 @@ def _workspace(tmp_path): def test_migration_preserves_identity_advances_revision_and_projects_qor(tmp_path): workspace = _workspace(tmp_path) current = create_engineering_snapshot(workspace, workspace_id="engineering-gcd") + assert current["schemaVersion"] == 2 + assert str(workspace.directory) not in json.dumps(current["qorSnapshotExtension"]) migrated = migrate_engineering_snapshot( workspace, From 0ebd2f8f7ae053fc2fba9371f7a5f41336d98eb7 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Mon, 14 Sep 2026 15:33:10 +0800 Subject: [PATCH 35/43] fix(project): align manifest flow with current stages --- chipcompiler/project/manifest.py | 8 ++++++-- test/engine/test_workspace_configuration.py | 4 ++-- test/engine/test_workspace_spec.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/chipcompiler/project/manifest.py b/chipcompiler/project/manifest.py index 99d51ba0..22f68fc2 100644 --- a/chipcompiler/project/manifest.py +++ b/chipcompiler/project/manifest.py @@ -26,7 +26,9 @@ MANIFEST_FLOW_STEPS = ( "Synth", "LEC", - "Floor", + "PreFloorplan", + "MacroPlacement", + "PostFloorplan", "Place", "CTS", "Legal", @@ -54,7 +56,9 @@ _CANONICAL_TO_MANIFEST_STEP = { "Synthesis": "Synth", "lec": "LEC", - "Floorplan": "Floor", + "preFloorplan": "PreFloorplan", + "macroPlacement": "MacroPlacement", + "postFloorplan": "PostFloorplan", "place": "Place", "CTS": "CTS", "legalization": "Legal", diff --git a/test/engine/test_workspace_configuration.py b/test/engine/test_workspace_configuration.py index d1ef9a29..96b9f9dd 100644 --- a/test/engine/test_workspace_configuration.py +++ b/test/engine/test_workspace_configuration.py @@ -250,8 +250,8 @@ def test_step_configuration_update_invalidates_only_target_suffix( } states = {step["name"]: step["state"] for step in updated.flow.steps()} assert states["Synthesis"] == "Success" - assert states["lec"] == "Success" - assert states["Floorplan"] == "Unstart" + assert states["preFloorplan"] == "Unstart" + assert states["macroPlacement"] == "Unstart" assert read_engineering_snapshot(updated)["workspaceRevision"] == 2 with pytest.raises(WorkspaceLifecycleError) as inapplicable: diff --git a/test/engine/test_workspace_spec.py b/test/engine/test_workspace_spec.py index e8782024..d9dd47d3 100644 --- a/test/engine/test_workspace_spec.py +++ b/test/engine/test_workspace_spec.py @@ -226,14 +226,14 @@ def test_partial_flow_spec_preserves_requested_boundaries(tmp_path, minimal_ics5 spec["flow"] = { "flowId": "rtl2gds", "fromStepId": "Synthesis", - "throughStepId": "lec", + "throughStepId": "preFloorplan", } spec["parameters"] = {"design.frequency_mhz": 200.0} workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings) assert [step["name"] for step in load_workspace(workspace.directory).flow.steps()] == [ "Synthesis", - "lec", + "preFloorplan", ] From a53d39df72002da41e027be4a4858c53f23cbc55 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Mon, 14 Sep 2026 15:33:10 +0800 Subject: [PATCH 36/43] test: remove stale environment assumptions --- test/cli/test_help_rendering.py | 2 ++ test/data/test_workspace_filelist.py | 20 ++++++-------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/test/cli/test_help_rendering.py b/test/cli/test_help_rendering.py index c8fc586e..ee7a0347 100644 --- a/test/cli/test_help_rendering.py +++ b/test/cli/test_help_rendering.py @@ -28,6 +28,8 @@ def test_help_renders_for_every_command(path, capsys): def test_help_keeps_styles_when_color_is_forced(monkeypatch, capsys): monkeypatch.setattr("typer.rich_utils.FORCE_TERMINAL", True) + monkeypatch.delenv("NO_COLOR", raising=False) + monkeypatch.setenv("TERM", "xterm-256color") rc = cli_main.run(["--help"]) diff --git a/test/data/test_workspace_filelist.py b/test/data/test_workspace_filelist.py index 1fad257f..4c01a107 100644 --- a/test/data/test_workspace_filelist.py +++ b/test/data/test_workspace_filelist.py @@ -86,9 +86,7 @@ def test_workspace_with_nested_filelist(self, tmp_path, test_parameters, pdk): assert (origin_dir / "rtl" / "core" / "alu.v").exists() assert (origin_dir / "rtl" / "core" / "ctrl.v").exists() - def test_filelist_absolute_entries_rewritten_to_frozen_sources( - self, tmp_path, test_parameters, pdk - ): + def test_filelist_absolute_entries_rewritten_to_frozen_sources(self, tmp_path): from chipcompiler.data.workspace import copy_filelist_with_sources project_dir = tmp_path / "project" @@ -111,9 +109,7 @@ def test_filelist_absolute_entries_rewritten_to_frozen_sources( assert (workspace_dir / "origin" / "rtl" / "b.v").exists() assert installed == str(workspace_dir / "origin" / "design.f") - def test_filelist_rewrite_handles_quoted_and_commented_entries( - self, tmp_path, test_parameters, pdk - ): + def test_filelist_rewrite_handles_quoted_and_commented_entries(self, tmp_path): from chipcompiler.data.workspace import copy_filelist_with_sources project_dir = tmp_path / "project" @@ -129,9 +125,7 @@ def test_filelist_rewrite_handles_quoted_and_commented_entries( assert lines == ['"a.v" # top'] assert (workspace_dir / "origin" / "a.v").exists() - def test_filelist_absolute_duplicate_basenames_are_disambiguated( - self, tmp_path, test_parameters, pdk - ): + def test_filelist_absolute_duplicate_basenames_are_disambiguated(self, tmp_path): from chipcompiler.data.workspace import copy_filelist_with_sources dir_a = tmp_path / "a" @@ -152,7 +146,7 @@ def test_filelist_absolute_duplicate_basenames_are_disambiguated( assert (workspace_dir / "origin" / "foo.v").read_text() == "module foo_a; endmodule\n" assert (workspace_dir / "origin" / "b_foo.v").read_text() == "module foo_b; endmodule\n" - def test_filelist_absolute_incdir_is_frozen_inside_origin(self, tmp_path, test_parameters, pdk): + def test_filelist_absolute_incdir_is_frozen_inside_origin(self, tmp_path): from chipcompiler.data.workspace.filelist_copy import copy_filelist_with_sources include_dir = tmp_path / "proj" / "include" @@ -169,7 +163,7 @@ def test_filelist_absolute_incdir_is_frozen_inside_origin(self, tmp_path, test_p assert lines[0] == "+incdir+include" assert (workspace_dir / "origin" / "include" / "defs.svh").exists() - def test_load_workspace_rejects_symlinked_params_toml(self, tmp_path, test_parameters, pdk): + def test_load_workspace_rejects_symlinked_params_toml(self, tmp_path): from chipcompiler.data.workspace import load_workspace from chipcompiler.data.workspace_config import WorkspaceConfigError, save_workspace_config @@ -185,9 +179,7 @@ def test_load_workspace_rejects_symlinked_params_toml(self, tmp_path, test_param with pytest.raises(WorkspaceConfigError): load_workspace(str(workspace_dir)) - def test_filelist_absolute_incdirs_with_same_basename_are_disambiguated( - self, tmp_path, test_parameters, pdk - ): + def test_filelist_absolute_incdirs_with_same_basename_are_disambiguated(self, tmp_path): from chipcompiler.data.workspace.filelist_copy import copy_filelist_with_sources dir_a = tmp_path / "a" / "include" From fdf23b24303ff5a283bb0b13531cd0a4ccaaba14 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Mon, 14 Sep 2026 16:11:42 +0800 Subject: [PATCH 37/43] fix(flow): preserve atomic completion contracts --- agent/engine.py | 3 +- chipcompiler/engine/flow.py | 126 +++----------- chipcompiler/engine/flow_completion.py | 96 +++++++++++ chipcompiler/engine/snapshot_qor.py | 228 +++++++++++++++++++++++-- chipcompiler/project/manifest.py | 8 + chipcompiler/rtl2gds/builder.py | 1 + docs/development.cn.md | 11 +- docs/development.md | 34 ++-- test/cli/project/test_manifest.py | 21 +++ test/engine/test_snapshot_migration.py | 22 +++ test/test_engine_flow.py | 33 ++++ 11 files changed, 447 insertions(+), 136 deletions(-) create mode 100644 chipcompiler/engine/flow_completion.py diff --git a/agent/engine.py b/agent/engine.py index ebf9232e..44d725f2 100644 --- a/agent/engine.py +++ b/agent/engine.py @@ -5,6 +5,7 @@ from chipcompiler.data import StateEnum, WorkspaceStep from chipcompiler.engine.flow import EngineFlow +from chipcompiler.engine.flow_completion import normalize_legacy_terminal_state from chipcompiler.engine.step_execution import get_process_rss_mb, track_current_process_memory from chipcompiler.utility.log import redirect_stdio_to_file @@ -34,7 +35,7 @@ def run_step(self, workspace_step: WorkspaceStep | str, *, rerun: bool = False) self.clear_db_engine_after_step(workspace_step, StateEnum.Success) return StateEnum.Success - self._normalize_legacy_terminal_state(workspace_step, step_tag) + normalize_legacy_terminal_state(self, workspace_step, step_tag) start_time = time.time() timing_constraints = self.timing_constraint_facts() diff --git a/chipcompiler/engine/flow.py b/chipcompiler/engine/flow.py index df9b540e..faf74896 100644 --- a/chipcompiler/engine/flow.py +++ b/chipcompiler/engine/flow.py @@ -16,6 +16,24 @@ log_flow, ) from chipcompiler.engine import EngineDB +from chipcompiler.engine.flow_completion import ( + finalize_interrupted_subflow as _finalize_interrupted_subflow, +) +from chipcompiler.engine.flow_completion import ( + normalize_legacy_terminal_state as _normalize_legacy_terminal_state, +) +from chipcompiler.engine.flow_completion import ( + notify_flow_observer as _notify_flow_observer, +) +from chipcompiler.engine.flow_completion import ( + notify_step_completed, +) +from chipcompiler.engine.flow_completion import ( + refresh_qor_report as _refresh_qor_report, +) +from chipcompiler.engine.flow_completion import ( + refresh_signoff_checklist as _refresh_signoff_checklist, +) from chipcompiler.engine.signoff import ( SignoffPackageCollector, SignoffPackageOptions, @@ -559,33 +577,6 @@ def run_steps( return True - def _normalize_legacy_terminal_state(self, workspace_step, step_tag): - """Reset stuck terminal states from pre-guard workspaces to Unstart. - - Pre-guard workspaces may have steps stuck in Incomplete/Invalid from - earlier runs, or in the removed terminal Warning state of the - synthesis LEC. Batch resets (_invalidate_suffix, clear_states) handle - rerun paths; this handles the rerun=False resume path. - """ - old_step = self.get_step(name=workspace_step.name, tool=workspace_step.tool) - if old_step is None: - return - persisted = old_step.get("state") - if persisted in { - StateEnum.Imcomplete.value, - StateEnum.Invalid.value, - "Warning", - }: - logger.warning( - "Normalizing legacy %s state '%s' → Unstart before rerun", - step_tag, - persisted, - ) - old_step["state"] = StateEnum.Unstart.value - old_step["runtime"] = "" - old_step["peak memory (mb)"] = 0 - # No self.save() — set_state(Ongoing) below saves. - def run_step( self, workspace_step: WorkspaceStep | str, @@ -609,17 +600,10 @@ def run_step( self.workspace.logger.info("[SKIP] %s already succeeded", step_tag) self.clear_db_engine_after_step(workspace_step, StateEnum.Success) _notify_flow_observer(observer, "on_step_skipped", workspace_step) - try: - from chipcompiler.analysis.qor import refresh_workspace_qor_report - - refresh_workspace_qor_report(self.workspace) - except Exception: - self.workspace.logger.exception( - "[QOR] %s failed to refresh the workspace QoR report after skip", step_tag - ) + _refresh_qor_report(self.workspace, step_tag) return StateEnum.Success - self._normalize_legacy_terminal_state(workspace_step, step_tag) + _normalize_legacy_terminal_state(self, workspace_step, step_tag) # set state ongoing start_time = time.time() @@ -654,6 +638,7 @@ def run_step( ): raise RuntimeError(f"failed to persist ongoing state for {step_tag}") _notify_flow_observer(observer, "on_step_started", workspace_step) + previous_step = deepcopy(flow_step) if flow_step is not None else None execution = execute_tool_step( self.workspace, @@ -749,14 +734,7 @@ def run_step( # artifacts refreshed above, so it runs after they exist; a # failure degrades to a warning like the facts refresh. if state == StateEnum.Success: - try: - from chipcompiler.analysis.qor import refresh_workspace_qor_report - - refresh_workspace_qor_report(self.workspace) - except Exception: - self.workspace.logger.exception( - "[QOR] %s failed to refresh the workspace QoR report", step_tag - ) + _refresh_qor_report(self.workspace, step_tag) except (Exception, SystemExit) as exc: failure_message = record_tool_failure(self.workspace.logger, step_tag, exc) step_error = step_error or failure_message @@ -785,12 +763,13 @@ def run_step( except (Exception, SystemExit): logger.exception("Failed to release DB engine after %s", step_tag) if terminal_persisted: - _notify_flow_observer( + notify_step_completed( + self, observer, - "on_step_completed", workspace_step, state, step_error, + previous_step, ) self.workspace.logger.info( @@ -815,58 +794,3 @@ def init_db_engine_for_step(self, workspace_step: WorkspaceStep) -> bool: return True return self.engine_db.create_db_engine(step=workspace_step) - - -def _finalize_interrupted_subflow( - observer, - workspace_step: WorkspaceStep, - runtime: str, - peak_memory_mb: float, -) -> None: - try: - from chipcompiler.runtime.subflow_events import finalize_interrupted_subflow - - for subflow_step in finalize_interrupted_subflow( - workspace_step, - runtime, - peak_memory_mb, - ): - _notify_flow_observer( - observer, - "on_subflow_stage", - workspace_step, - subflow_step, - ) - except (Exception, SystemExit): - logger.exception("Failed to finalize subflow after %s", workspace_step.name) - - -def _refresh_signoff_checklist(workspace: Workspace, workspace_step: WorkspaceStep) -> None: - """Replace step/home checklists after the step's terminal flow state is saved.""" - try: - from chipcompiler.tools.ecc.signoff_checklist import refresh_step_checklist - - refresh_step_checklist(workspace, workspace_step) - except (Exception, SystemExit): - logger.exception( - "Failed to refresh signoff checklist after %s/%s", - workspace_step.name, - workspace_step.tool, - ) - - -def _notify_flow_observer(observer, method_name: str, *args) -> None: - """Keep optional GUI observers outside the flow engine's failure domain.""" - if observer is None: - return - callback = getattr(observer, method_name, None) - if not callable(callback): - return - try: - callback(*args) - except (Exception, SystemExit): - if getattr(observer, "fatal_observer", False): - raise - # Runtime observers must never turn a completed tool execution into a - # failed flow. The coordinator records transport failures separately. - logging.getLogger(__name__).exception("flow observer callback failed: %s", method_name) diff --git a/chipcompiler/engine/flow_completion.py b/chipcompiler/engine/flow_completion.py new file mode 100644 index 00000000..d99dc962 --- /dev/null +++ b/chipcompiler/engine/flow_completion.py @@ -0,0 +1,96 @@ +"""Shared EngineFlow completion hooks.""" + +import logging +from copy import deepcopy + +logger = logging.getLogger(__name__) + + +def normalize_legacy_terminal_state(flow, workspace_step, step_tag) -> None: + """Reset terminal states from pre-guard workspaces before rerun.""" + old_step = flow.get_step(name=workspace_step.name, tool=workspace_step.tool) + if old_step is None: + return + persisted = old_step.get("state") + if persisted in {"Incomplete", "Invalid", "Warning"}: + logger.warning( + "Normalizing legacy %s state '%s' -> Unstart", + step_tag, + persisted, + ) + old_step["state"] = "Unstart" + old_step["runtime"] = "" + old_step["peak memory (mb)"] = 0 + + +def finalize_interrupted_subflow(observer, workspace_step, runtime, peak_memory_mb) -> None: + try: + from chipcompiler.runtime.subflow_events import finalize_interrupted_subflow as finalize + + for subflow_step in finalize(workspace_step, runtime, peak_memory_mb): + notify_flow_observer(observer, "on_subflow_stage", workspace_step, subflow_step) + except (Exception, SystemExit): + logger.exception("Failed to finalize subflow after %s", workspace_step.name) + + +def refresh_signoff_checklist(workspace, workspace_step) -> None: + """Replace step/home checklists after the step's terminal flow state is saved.""" + try: + from chipcompiler.tools.ecc.signoff_checklist import refresh_step_checklist + + refresh_step_checklist(workspace, workspace_step) + except (Exception, SystemExit): + logger.exception( + "Failed to refresh signoff checklist after %s/%s", + workspace_step.name, + workspace_step.tool, + ) + + +def refresh_qor_report(workspace, step_tag: str) -> None: + """Refresh the inspection report without changing flow completion policy.""" + try: + from chipcompiler.analysis.qor import refresh_workspace_qor_report + + refresh_workspace_qor_report(workspace) + except Exception: + workspace.logger.exception("[QOR] %s failed to refresh the workspace QoR report", step_tag) + + +def notify_flow_observer(observer, method_name: str, *args) -> None: + """Keep optional GUI observers outside the flow engine's failure domain.""" + if observer is None: + return + callback = getattr(observer, method_name, None) + if not callable(callback): + return + try: + callback(*args) + except (Exception, SystemExit): + if getattr(observer, "fatal_observer", False): + raise + logger.exception("flow observer callback failed: %s", method_name) + + +def notify_step_completed( + flow, + observer, + workspace_step, + state, + error, + previous_step: dict | None, +) -> None: + """Publish completion and restore Ongoing state when a fatal commit fails.""" + try: + notify_flow_observer(observer, "on_step_completed", workspace_step, state, error) + except (Exception, SystemExit): + if previous_step is not None: + current = flow.get_step(workspace_step.name, workspace_step.tool) + if current is not None: + current.clear() + current.update(deepcopy(previous_step)) + if not flow.save(): + raise RuntimeError( + "failed to roll back flow state after completion commit failure" + ) from None + raise diff --git a/chipcompiler/engine/snapshot_qor.py b/chipcompiler/engine/snapshot_qor.py index 9b42d025..d4ac7883 100644 --- a/chipcompiler/engine/snapshot_qor.py +++ b/chipcompiler/engine/snapshot_qor.py @@ -3,6 +3,17 @@ from math import isfinite from typing import Any +from chipcompiler.analysis.qor.schema import ( + CONFIDENCES, + DIMENSION_KEYS, + DIMENSION_STATES, + EVIDENCE_STATES, + FEASIBILITY_STATUSES, + GATE_STATES, + SCALAR_STATUSES, + TIERS, +) + QOR_SNAPSHOT_EXTENSION_SCHEMA_VERSION = 1 _MAX_DIAGNOSES = 64 _MAX_INTERVENTIONS = 4 @@ -133,20 +144,215 @@ def unavailable_qor_snapshot_extension(reason: str) -> dict[str, Any]: def validate_qor_snapshot_extension(value: object) -> bool: - """Validate the migration-only extension boundary.""" + """Validate the bounded, path-free QoR extension fail-closed.""" if not isinstance(value, dict): return False + status = value.get("status") + required = { + "schemaVersion", + "scoringEngine", + "status", + "score", + "scalarStatus", + "profile", + "qphys", + "feasibility", + "evidence", + "diagnoses", + "inflation", + "power", + "artifactIds", + } + if status == "unavailable": + required.add("reason") + if ( + set(value) != required + or value.get("schemaVersion") != QOR_SNAPSHOT_EXTENSION_SCHEMA_VERSION + or value.get("scoringEngine") != "qor-v3" + or status not in {"available", "unavailable"} + or not _bounded_text( + value.get("profile"), + values=("balanced", "timing_critical", "low_power", "area_optimized"), + ) + or not _bounded_number(value.get("score"), 0, 100, nullable=True) + or value.get("scalarStatus") not in SCALAR_STATUSES + ): + return False + if status == "unavailable" and not _bounded_text(value.get("reason")): + return False + + qphys = value.get("qphys") + if not isinstance(qphys, dict) or len(qphys) > len(DIMENSION_KEYS): + return False + for key, dimension in qphys.items(): + if key not in DIMENSION_KEYS or not isinstance(dimension, dict): + return False + if set(dimension) != {"value", "state", "featureIds"}: + return False + if not _bounded_number(dimension["value"], 0, 100, nullable=True): + return False + if dimension["state"] not in DIMENSION_STATES or not _bounded_strings( + dimension["featureIds"], 32 + ): + return False + + feasibility = value.get("feasibility") + if not isinstance(feasibility, dict) or set(feasibility) != {"status", "gates"}: + return False + if ( + feasibility["status"] not in FEASIBILITY_STATUSES + or not isinstance(feasibility["gates"], list) + or len(feasibility["gates"]) > 32 + ): + return False + for gate in feasibility["gates"]: + if not isinstance(gate, dict) or set(gate) != { + "id", + "stage", + "state", + "blocksTapeout", + "metrics", + "availability", + }: + return False + if ( + not _bounded_text(gate["id"]) + or not _bounded_text(gate["stage"]) + or gate["state"] not in GATE_STATES + or not isinstance(gate["blocksTapeout"], bool) + or not _bounded_strings(gate["metrics"], 32) + or not _bounded_text(gate["availability"], nullable=True) + ): + return False + + evidence = value.get("evidence") + if not isinstance(evidence, dict) or set(evidence) != { + "index", + "state", + "integrity", + "coverage", + "consistency", + }: + return False + if ( + evidence["state"] not in EVIDENCE_STATES + or not _bounded_number(evidence["index"], 0, 100, nullable=True) + or not all( + _bounded_number(evidence[field], 0, 1, nullable=True) + for field in ("integrity", "coverage", "consistency") + ) + ): + return False + + diagnoses = value.get("diagnoses") + if not isinstance(diagnoses, list) or len(diagnoses) > _MAX_DIAGNOSES: + return False + for diagnosis in diagnoses: + if not isinstance(diagnosis, dict) or set(diagnosis) != { + "diagnosisId", + "state", + "severity", + "confidence", + "triggerFeatures", + "affectedDimensions", + "interventions", + "interventionConfidence", + "validationRequired", + }: + return False + if ( + not _bounded_text(diagnosis["diagnosisId"]) + or not _bounded_text(diagnosis["state"]) + or not _bounded_number(diagnosis["severity"], 0, 1) + or diagnosis["confidence"] not in CONFIDENCES + or not _bounded_strings(diagnosis["triggerFeatures"], 32) + or not _bounded_strings(diagnosis["affectedDimensions"], 16) + or not isinstance(diagnosis["interventions"], list) + or len(diagnosis["interventions"]) > _MAX_INTERVENTIONS + or diagnosis["interventionConfidence"] not in CONFIDENCES + or not _bounded_text(diagnosis["validationRequired"], nullable=True) + ): + return False + for intervention in diagnosis["interventions"]: + if not isinstance(intervention, dict) or set(intervention) != { + "hypothesis", + "tier", + "confidence", + "parameterKnob", + "validationProcedure", + }: + return False + if ( + not _bounded_text(intervention["hypothesis"]) + or intervention["tier"] not in TIERS + or intervention["confidence"] not in CONFIDENCES + or not _bounded_text(intervention["parameterKnob"], nullable=True) + or not _bounded_text(intervention["validationProcedure"], nullable=True) + ): + return False + + inflation = value.get("inflation") + if not isinstance(inflation, dict) or set(inflation) != { + "iPlace", + "iRoute", + "iTotal", + "congestionSeverity", + "compatibilityStatus", + }: + return False + if not all( + _bounded_number(inflation[field], 0, None, nullable=True) + for field in ("iPlace", "iRoute", "iTotal", "congestionSeverity") + ) or inflation["compatibilityStatus"] not in { + "EXACT_COMPATIBLE", + "MAPPED_COMPATIBLE", + "INCOMPATIBLE", + "UNAVAILABLE", + }: + return False + + power = value.get("power") + if not isinstance(power, dict) or set(power) != {"totalUw", "budgetUw", "sourceKind", "corner"}: + return False + if ( + not _bounded_number(power["totalUw"], 0, None, nullable=True) + or not _bounded_number(power["budgetUw"], 0, None, nullable=True) + or power["sourceKind"] not in {None, "signoff", "synthesis"} + or not _bounded_text(power["corner"], nullable=True) + ): + return False + + return _bounded_strings(value["artifactIds"], _MAX_ARTIFACT_IDS) + + +def _bounded_number( + value: object, low: float | None, high: float | None, *, nullable: bool +) -> bool: + if value is None: + return nullable + if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value): + return False + return (low is None or value >= low) and (high is None or value <= high) + + +def _bounded_text( + value: object, *, nullable: bool = False, values: tuple[str, ...] | None = None +) -> bool: + if value is None: + return nullable + return ( + isinstance(value, str) + and bool(value) + and len(value) <= _MAX_TEXT + and (values is None or value in values) + ) + + +def _bounded_strings(value: object, maximum: int) -> bool: return ( - value.get("schemaVersion") == QOR_SNAPSHOT_EXTENSION_SCHEMA_VERSION - and value.get("scoringEngine") == "qor-v3" - and value.get("status") in {"available", "unavailable"} - and isinstance(value.get("qphys"), dict) - and isinstance(value.get("feasibility"), dict) - and isinstance(value["feasibility"].get("gates"), list) - and isinstance(value.get("evidence"), dict) - and isinstance(value.get("diagnoses"), list) - and isinstance(value.get("artifactIds"), list) - and all(isinstance(item, str) and item for item in value["artifactIds"]) + isinstance(value, list) + and len(value) <= maximum + and all(_bounded_text(item) for item in value) ) diff --git a/chipcompiler/project/manifest.py b/chipcompiler/project/manifest.py index 22f68fc2..1df09e9b 100644 --- a/chipcompiler/project/manifest.py +++ b/chipcompiler/project/manifest.py @@ -43,6 +43,11 @@ "Harden", ) +# ``Floor`` was the public manifest value before floorplanning was split into +# pre/macro/post stages. Read it as the completed handoff stage so old ranges +# keep their original start/end meaning without rewriting project.json. +_MANIFEST_STEP_ALIASES = {"Floor": "PostFloorplan"} + PRESET_MANIFEST_RANGE = { "syn_sta": ("Synth", "Synth"), "rtl2gds": ("Synth", "Harden"), @@ -59,6 +64,7 @@ "preFloorplan": "PreFloorplan", "macroPlacement": "MacroPlacement", "postFloorplan": "PostFloorplan", + "Floorplan": "PostFloorplan", "place": "Place", "CTS": "CTS", "legalization": "Legal", @@ -152,6 +158,8 @@ def _normalize_workspace_entry(value: Any, index: int, project_dir: str) -> Mani status = "not_started" start_step = _optional_str(source.get("start_step")) or "Synth" end_step = _optional_str(source.get("end_step")) or "Harden" + start_step = _MANIFEST_STEP_ALIASES.get(start_step, start_step) + end_step = _MANIFEST_STEP_ALIASES.get(end_step, end_step) for step_name, field_name in ((start_step, "start_step"), (end_step, "end_step")): if step_name not in MANIFEST_FLOW_STEPS: raise ManifestError( diff --git a/chipcompiler/rtl2gds/builder.py b/chipcompiler/rtl2gds/builder.py index 2092af34..a320bc11 100644 --- a/chipcompiler/rtl2gds/builder.py +++ b/chipcompiler/rtl2gds/builder.py @@ -42,6 +42,7 @@ def normalize_flow_step(value: str | StepEnum) -> str: "synthesis": StepEnum.SYNTHESIS.value, "prefloorplan": StepEnum.PRE_FLOORPLAN.value, "floorplan": StepEnum.PRE_FLOORPLAN.value, + "floor": StepEnum.POST_FLOORPLAN.value, "macroplace": StepEnum.MACRO_PLACEMENT.value, "macroplacement": StepEnum.MACRO_PLACEMENT.value, "postfloorplan": StepEnum.POST_FLOORPLAN.value, diff --git a/docs/development.cn.md b/docs/development.cn.md index 23f4d3ce..28c538cc 100644 --- a/docs/development.cn.md +++ b/docs/development.cn.md @@ -273,8 +273,8 @@ chipcompiler/cli/inspection/ # 只读探查逻辑 chipcompiler/cli/project/ # config.py（ecc.toml 解析校验）/ config_fields.py（`ecc project` 的项目声明 schema）/ params.py（参数注册表）/ workspace_params.py（workspace 局部覆盖记录）/ manifest.py（项目形态分类）/ effective_config.py / config_params/（直配参数 schema）/ migrate*.py（旧布局迁移）/ run_*.py（run 目标解析与分发） chipcompiler/cli/rendering/ # 输出渲染（render / renderers / pretty / progress） chipcompiler/engine/signoff/ # 签核收集器 + 设计/checklist 报告（包，见下文） -chipcompiler/engine/qor_scoring.py # QoR 评分规则（阈值/权重/选指标，Studio Snapshot 与 CLI 共用） -chipcompiler/engine/qor_report.py # CLI QoR 报告（读当前 analysis，调用 qor_scoring） +chipcompiler/analysis/qor/ # QoR v3 唯一分析、评分与报告契约 +chipcompiler/engine/qor_report.py # CLI QoR facade，委托 analysis.qor ``` 模块归属由 `test/cli/test_cli_module_layout.py` 强制：核心框架必须在 `cli/core/`、命令注册在 `cli/commands/`、全部处理器在唯一的 `cli/command_handlers/` 包、只读探查在 `cli/inspection/`、渲染在 `cli/rendering/`；旧的 `chipcompiler/cli/*.py` 平铺模块必须不可导入。新增文件时放进对应子包，不要在 `cli/` 根下新建模块。 @@ -428,8 +428,9 @@ config_param( #### 扩展报告（`ecc report summary/qor/checklist/step`） - **设计总结**：`ecc report summary` 调用 `chipcompiler.engine.signoff.generate_text_report`。其实现按职责分模块（`report.py` 编排 / `report_data.py` 数据契约 / `report_extract.py` 解析器+workspace 收集 / `report_sections.py` 分区抽取 / `report_timing.py` timing 链 / `report_text.py` 格式化），全部经包 `__init__` 对外暴露。新增报告分区时，在 `report_sections.py`（或 timing 链）增加 `_extract_(q)`，并在 `report.py` 编排处注册。 -- `engine/qor_scoring.py`：唯一的 QoR 评分规则。无 I/O；负责 metric 选择（role 优先级 final>gate>trend、area_cost 只取最后成功的 area 步）、单指标阈值（`METRIC_FAIL_VALUES`）、维度平均、权重（缺项不重归一化）和 overall score。Studio 经 Snapshot `qorAssessment`（`engine/qor.py`）消费同一套规则；不要在 GUI 或 CLI 再抄一份阈值表。新增可计分指标 = 只在 `METRIC_FAIL_VALUES` 加阈值。 -- `engine/qor_report.py`：CLI `ecc report qor` 的采集与文本。读取当前 workspace 的 v3 `qor_metrics.json`，交给 `score_qor`，再渲染总分、维度表和逐指标明细。不拥有评分公式。 +- `analysis/qor/`：QoR v3 唯一分析引擎，负责指标加载、feature/维度计算、feasibility gates、evidence、评分、diagnosis、intervention、有界报告 schema 和文本渲染。新的工程结论只能在这里实现，GUI/CLI 不得复制阈值或公式。 +- `engine/qor_report.py`：CLI `ecc report qor` facade，委托 `analysis.qor`，不拥有第二套评分实现。 +- `engine/qor_scoring.py` 与 `engine/qor.py`：仅为生产 Snapshot v2 兼容保留；ECC-only 阶段不要让 QoR v3 消费这条路径。 - `engine/signoff/report_checklist.py`：只读渲染 `home/checklist.json`（不合法时报 unavailable，绝不回写文件）。 - CLI：`cli/commands/report.py` + `cli/command_handlers/report.py`；workspace 解析复用 `inspection/discovery.py`（`resolve_workspace_path` 是无副作用核心，`resolve_command_workspace` 是核心加 `load_workspace`；signoff、report 与只读的 status/log/config 共用）。 @@ -491,7 +492,7 @@ uv run ecc run --project gcd --preset rtl2gds ### 报告 -`ecc report qor` 用共享的 `qor_scoring` 规则给 workspace 打分（每指标对固定 fail 阈值计分、维度求均值、加权总分——缺失维度不做权重重归一化）；Studio Snapshot `qorAssessment` 用同一套计分器。`ecc report checklist` 渲染签核清单状态；`ecc report summary` 写出与 GUI 一致的文本设计总结。三者默认写入 `/signoff/`，接受 `-o` 以及常规的 `--project` 和可选的受管 `--workspace NAME` 选择器： +`ecc report qor` 委托 `chipcompiler.analysis.qor` QoR v3 引擎。生产 Snapshot v2 的 `qorAssessment` 仅为当前 GUI 兼容保留，不是 v3 工程结论的第二来源。`ecc report checklist` 渲染签核清单状态；`ecc report summary` 写出与 GUI 一致的文本设计总结。三者默认写入 `/signoff/`，接受 `-o` 以及常规的 `--project` 和可选的受管 `--workspace NAME` 选择器： ```bash uv run ecc report qor --project gcd diff --git a/docs/development.md b/docs/development.md index ef00240d..a6d8b107 100644 --- a/docs/development.md +++ b/docs/development.md @@ -303,8 +303,8 @@ chipcompiler/cli/inspection/ # read-only probing logic chipcompiler/cli/project/ # config.py (ecc.toml parsing and validation) / config_fields.py (project declaration schema for `ecc project`) / params.py (parameter registry) / workspace_params.py (workspace-local override records) / manifest.py (project-state classification) / effective_config.py / config_params/ (direct-config schemas) / migrate*.py (legacy-layout migration) / run_*.py (workspace target resolution and dispatch) chipcompiler/cli/rendering/ # output rendering (render / renderers / pretty / progress) chipcompiler/engine/signoff/ # signoff collector + design/checklist reports (package, see below) -chipcompiler/engine/qor_scoring.py # QoR scoring rules (thresholds/weights/selection; shared with Studio Snapshot) -chipcompiler/engine/qor_report.py # CLI QoR report (reads current analysis, calls qor_scoring) +chipcompiler/analysis/qor/ # canonical QoR v3 analysis, scoring, and report contract +chipcompiler/engine/qor_report.py # CLI QoR facade delegating to analysis.qor ``` Module placement is enforced by `test/cli/test_cli_module_layout.py`: the core @@ -572,18 +572,16 @@ required by doctor. / `report_text.py` formatting), all exposed through the package `__init__`. Add a report section through an `_extract_(q)` in `report_sections.py` (or the timing chain) and register it from `report.py`. -- `engine/qor_scoring.py`: the only QoR scoring rules. No I/O; owns metric - selection (role priority final>gate>trend; area_cost only from the last - successful area step), per-metric thresholds (`METRIC_FAIL_VALUES`), - dimension averages, weights (absent dimensions are not renormalized), and - the overall score. Studio consumes the same rules through Snapshot - `qorAssessment` (`engine/qor.py`); do not copy the threshold table into the - GUI or CLI. Adding a scoreable metric = adding its threshold in - `METRIC_FAIL_VALUES` only. -- `engine/qor_report.py`: collection and text for `ecc report qor`. Reads the - workspace's current v3 `qor_metrics.json` files, calls `score_qor`, then - renders the overall score, dimension table, and per-metric detail. It does - not own the scoring formulas. +- `analysis/qor/`: the canonical QoR v3 Engine. It owns metric loading, + feature and dimension evaluation, feasibility gates, evidence, scoring, + diagnoses, interventions, the bounded report schema, and text rendering. + New QoR conclusions belong here; do not copy thresholds or formulas into + the GUI or CLI. +- `engine/qor_report.py`: the CLI `ecc report qor` facade. It delegates to + `analysis.qor` and does not own a second scoring implementation. +- `engine/qor_scoring.py` and `engine/qor.py`: legacy v2 Snapshot assessment + kept only while production Snapshot writes remain v2. Do not extend this + path for QoR v3 consumers. - `engine/signoff/report_checklist.py`: read-only rendering of `home/checklist.json` (reports unavailable on an invalid file; never writes back). @@ -683,10 +681,10 @@ uv run ecc run --project gcd --preset rtl2gds ### Reports -`ecc report qor` scores the workspace with the shared `qor_scoring` rules -(per-metric scores against fixed fail thresholds, dimension averages, -weighted overall — weights are not renormalized over missing dimensions); -Studio Snapshot `qorAssessment` uses the same scorer. +`ecc report qor` delegates to the canonical QoR v3 Engine in +`chipcompiler.analysis.qor`. The production v2 Snapshot `qorAssessment` +projection remains available for GUI compatibility during the ECC-only +rollout; it is not a second source for v3 conclusions. `ecc report checklist` renders the signoff checklist status, and `ecc report summary` writes the GUI-parity text design summary. All three write to `/signoff/` by default and accept `-o` plus the usual diff --git a/test/cli/project/test_manifest.py b/test/cli/project/test_manifest.py index 4832d8ab..40a82326 100644 --- a/test/cli/project/test_manifest.py +++ b/test/cli/project/test_manifest.py @@ -73,6 +73,27 @@ def test_load_manifest_normalizes_workspace_entries(tmp_path): assert manifest.active_workspaces() == [entry] +def test_load_manifest_maps_legacy_floor_range_to_post_floorplan(tmp_path): + _write_manifest( + tmp_path, + _minimal_document( + tmp_path, + workspaces=[ + { + "workspace_id": "legacy", + "workspace_path": str(tmp_path / "legacy"), + "start_step": "Floor", + "end_step": "Place", + } + ], + ), + ) + + entry = load_manifest(str(tmp_path)).workspaces[0] + + assert (entry.start_step, entry.end_step) == ("PostFloorplan", "Place") + + def test_load_manifest_relative_workspace_path_resolves_inside_root(tmp_path): _write_manifest( tmp_path, diff --git a/test/engine/test_snapshot_migration.py b/test/engine/test_snapshot_migration.py index 13728909..da1d7c05 100644 --- a/test/engine/test_snapshot_migration.py +++ b/test/engine/test_snapshot_migration.py @@ -1,4 +1,5 @@ import json +from copy import deepcopy from pathlib import Path from types import SimpleNamespace @@ -12,6 +13,10 @@ migrate_engineering_snapshot, read_engineering_snapshot, ) +from chipcompiler.engine.snapshot_qor import ( + unavailable_qor_snapshot_extension, + validate_qor_snapshot_extension, +) def _workspace(tmp_path): @@ -93,3 +98,20 @@ def test_production_write_paths_reject_migrated_v3_snapshot(tmp_path): with pytest.raises(EngineeringSnapshotError, match="production Snapshot schema is still v2"): ensure_engineering_snapshot(workspace) + + +def test_qor_extension_validator_rejects_missing_and_invalid_nested_fields(): + extension = unavailable_qor_snapshot_extension("analysis unavailable") + assert validate_qor_snapshot_extension(extension) + + missing_score = deepcopy(extension) + missing_score.pop("score") + assert not validate_qor_snapshot_extension(missing_score) + + invalid_power = deepcopy(extension) + invalid_power["power"]["sourceKind"] = "raw_file" + assert not validate_qor_snapshot_extension(invalid_power) + + invalid_gate = deepcopy(extension) + invalid_gate["feasibility"]["gates"].append({"id": "broken"}) + assert not validate_qor_snapshot_extension(invalid_gate) diff --git a/test/test_engine_flow.py b/test/test_engine_flow.py index ad7b9101..2dd4b326 100644 --- a/test/test_engine_flow.py +++ b/test/test_engine_flow.py @@ -522,6 +522,39 @@ class Observer: assert step["state"] == StateEnum.Ongoing.value assert step["info"]["runtime_operation"]["operation_id"] == "operation-1" + def test_fatal_completion_commit_restores_ongoing_flow_marker(self, monkeypatch, tmp_path): + workspace = Workspace() + workspace.flow.path = tmp_path / "flow.json" + workspace.flow.data = { + "steps": [{"name": "place", "tool": "dreamplace", "state": "Unstart", "info": {}}], + } + workspace.flow.path.write_text(json.dumps(workspace.flow.data), encoding="utf-8") + engine_flow = EngineFlow(workspace) + workspace_step = EccStep(name="place", directory=tmp_path, tool="dreamplace") + engine_flow.workspace_steps = [workspace_step] + engine_flow.engine_db = SimpleNamespace(engine=None) + + class Observer: + fatal_observer = True + runtime_operation = { + "schema": 1, + "operation_id": "operation-1", + "runtime_instance_id": "runtime-1", + } + + def on_step_completed(self, _step, _state, _error=None): + raise RuntimeError("snapshot commit failed") + + monkeypatch.setattr(tools, "run_step", lambda **_kwargs: True) + monkeypatch.setattr(engine_flow, "check_step_result", lambda **_kwargs: True) + + with pytest.raises(RuntimeError, match="snapshot commit failed"): + engine_flow.run_step(workspace_step, observer=Observer()) + + persisted = json.loads(workspace.flow.path.read_text(encoding="utf-8"))["steps"][0] + assert persisted["state"] == StateEnum.Ongoing.value + assert persisted["info"]["runtime_operation"]["operation_id"] == "operation-1" + def test_result_check_system_exit_still_finalizes_step(self, monkeypatch, tmp_path): workspace = Workspace() workspace.flow.path = tmp_path / "flow.json" From e1cf51c7e2ffaa49722ab0a168573caae139a888 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Mon, 14 Sep 2026 21:42:57 +0800 Subject: [PATCH 38/43] fix(cli): commit snapshots during interactive ecc run Attach the Engineering Snapshot observer to TTY progress runs so each completed step updates home/engineering-snapshot.json, matching execute(). --- chipcompiler/cli/rendering/progress.py | 13 +++++- chipcompiler/engine/execution.py | 24 ++++++++-- docs/development.cn.md | 2 +- docs/development.md | 5 +- test/cli/rendering/test_progress.py | 63 ++++++++++++++++++++++++++ test/engine/test_execution.py | 7 ++- test/test_engine_flow.py | 21 +++++++++ 7 files changed, 127 insertions(+), 8 deletions(-) diff --git a/chipcompiler/cli/rendering/progress.py b/chipcompiler/cli/rendering/progress.py index a33f9be2..7bdf6d99 100644 --- a/chipcompiler/cli/rendering/progress.py +++ b/chipcompiler/cli/rendering/progress.py @@ -19,6 +19,11 @@ from chipcompiler.cli.rendering.pretty import BOLD, CYAN, DIM, GREEN, RED, RESET from chipcompiler.cli.rendering.pretty import style as _style from chipcompiler.data import StateEnum, log_flow +from chipcompiler.engine.execution import ( + event_sink_for_workspace, + execution_observer, + invoke_engine, +) from chipcompiler.utility.log import flush_cstdio, redirect_stdio_to_file @@ -434,6 +439,7 @@ def run_flow_with_progress(engine_flow, ctx, project, stderr): run_dir = engine_flow.workspace.directory run_name = ctx.run_id or "default" renderer.start_run(run_name, run_dir) + observer = execution_observer(event_sink_for_workspace(workspace)) for workspace_step in engine_flow.workspace_steps: step_token = normalize_step_name(workspace_step.name) @@ -476,7 +482,12 @@ def run_flow_with_progress(engine_flow, ctx, project, stderr): finally: if init_log_stream is not None: init_log_stream.close() - state = engine_flow.run_step(workspace_step) + state = invoke_engine( + engine_flow.run_step, + workspace_step, + rerun=False, + observer=observer, + ) finally: _stop_log_monitor(stop_event, monitor) renderer.clear() diff --git a/chipcompiler/engine/execution.py b/chipcompiler/engine/execution.py index 5ce285a9..e406e7b7 100644 --- a/chipcompiler/engine/execution.py +++ b/chipcompiler/engine/execution.py @@ -38,10 +38,10 @@ def execute(flow: Any, plan: ExecutionPlan, *, event_sink: Any = None) -> Execut no_op=True, ) workspace = getattr(flow, "workspace", None) - if event_sink is None and getattr(workspace, "directory", None): - event_sink = _EngineeringCommitSink(workspace) + if event_sink is None: + event_sink = event_sink_for_workspace(workspace) rerun = plan.intent == "rerun" - observer = _ExecutionObserver(event_sink) + observer = execution_observer(event_sink) if not selected_ids: succeeded = bool(_invoke(flow.run_steps, rerun=rerun, observer=observer)) else: @@ -92,7 +92,7 @@ def _is_completed(flow: Any) -> bool: ) -class _ExecutionObserver: +class ExecutionObserver: def __init__(self, delegate: Any): self.delegate = delegate self.fatal_observer = bool(getattr(delegate, "fatal_observer", False)) @@ -115,6 +115,8 @@ def __getattr__(self, name: str): callback = getattr(self.delegate, name, None) if callback is not None: return callback + if name == "runtime_operation": + return None return lambda *_args, **_kwargs: None @@ -137,6 +139,20 @@ def on_step_completed(self, _step: Any, state: Any, _error: str | None = None) - ) +def event_sink_for_workspace(workspace: Any) -> _EngineeringCommitSink | None: + if getattr(workspace, "directory", None): + return _EngineeringCommitSink(workspace) + return None + + +def execution_observer(event_sink: Any) -> ExecutionObserver: + return ExecutionObserver(event_sink) + + +def invoke_engine(callback, *args, rerun: bool = False, observer: Any = None): + return _invoke(callback, *args, rerun=rerun, observer=observer) + + def _invoke(callback, *args, rerun: bool, observer: Any): parameters = inspect.signature(callback).parameters.values() accepts_kwargs = any( diff --git a/docs/development.cn.md b/docs/development.cn.md index 28c538cc..afeed60c 100644 --- a/docs/development.cn.md +++ b/docs/development.cn.md @@ -414,7 +414,7 @@ config_param( - **新建 workspace**：解析 `[design]` 输入声明、PDK、参数与请求入口步骤；只校验入口步骤所需文件；先原子登记受管名称到 `project.json`（`not_started`）；预检工具；在 `/` 调用 `create_workspace`。`create_workspace` 将输入复制到 `origin/` 并产出全部步骤配置，CLI 后续不改写配置。正常新建 flow 用 preset；`--from A --to B` 改用 `rtl2gds.build_flow_range(A, B)` 动态构建包含式规范范围。新范围不能与 `--preset`、`--overwrite`、`--resume`、`--only`、`--force` 组合。 - **已有 workspace**：先由 `chipcompiler/engine/reconcile.py` 把持久化 flow 与目标对齐（前缀 → 追加扩展；超集且全成 → `no_op`；分叉 → `flow_mismatch`），再 `load_workspace` 后由 `chipcompiler.engine.rerun` 的 `run_resume`、`run_from` 或 `run_only` 原地复跑。`--from A --to B` 是已有 flow 的包含式范围，会将其后的步骤状态失效但保留其输出文件。已有 workspace 不会重新预检输入，也不会改写已复制输入或配置。 -项目 preset 的步骤序列定义在 `chipcompiler/rtl2gds/builder.py`（`build_*_flow()` / `get_flow_builders()`），不在 CLI 层。`build_flow_range()` 对规范的 `build_rtl2gds_flow()` 结果切片，步骤别名和顺序只有一份来源。修改序列时须同步引擎默认 flow、`StepEnum` 与 manifest 范围映射；CLI 只负责参数解析、输入契约、进度渲染选择与结果映射。 +项目 preset 的步骤序列定义在 `chipcompiler/rtl2gds/builder.py`（`build_*_flow()` / `get_flow_builders()`），不在 CLI 层。`build_flow_range()` 对规范的 `build_rtl2gds_flow()` 结果切片，步骤别名和顺序只有一份来源。修改序列时须同步引擎默认 flow、`StepEnum` 与 manifest 范围映射；CLI 只负责参数解析、输入契约、进度渲染选择与结果映射。交互式 TTY 的 `ecc run` 走 `run_flow_with_progress()`，`--plain` 与 GUI 走 `execute()`；两条路径挂同一套 Engineering Snapshot 提交 observer，每完成一步都会更新 `home/engineering-snapshot.json`。 #### 扩展环境探查（doctor / 预检） diff --git a/docs/development.md b/docs/development.md index a6d8b107..5e09dfe2 100644 --- a/docs/development.md +++ b/docs/development.md @@ -534,7 +534,10 @@ Project preset sequences are defined in `chipcompiler/rtl2gds/builder.py` step aliases and ordering have one source of truth. Keep a sequence change coordinated with the engine's default flow, `StepEnum`, and manifest range mappings; the CLI only handles argument parsing, input contracts, -progress-renderer selection, and result mapping. +progress-renderer selection, and result mapping. Interactive TTY +`ecc run` uses `run_flow_with_progress()`; `--plain` and GUI use +`execute()`. Both attach the same Engineering Snapshot commit observer +so each completed step updates `home/engineering-snapshot.json`. #### Extending environment probing (doctor / preflight) diff --git a/test/cli/rendering/test_progress.py b/test/cli/rendering/test_progress.py index e3aa90f5..9384ff5e 100644 --- a/test/cli/rendering/test_progress.py +++ b/test/cli/rendering/test_progress.py @@ -658,6 +658,10 @@ def check_state_fn(self, name, tool, state): class TestRunFlowWithProgress: + @pytest.fixture(autouse=True) + def _disable_snapshot_sink(self, monkeypatch): + monkeypatch.setattr(progress, "event_sink_for_workspace", lambda workspace: None) + def test_success_summary_format(self, tmp_path): flow = _make_flow( _make_ws(str(tmp_path)), @@ -1125,6 +1129,65 @@ def test_color_disabled_for_non_tty(self): for code in (BOLD, CYAN, GREEN, RED, DIM): assert code not in output + def test_progress_run_commits_snapshot_after_each_step(self, tmp_path, monkeypatch): + committed = [] + sink = SimpleNamespace( + on_step_completed=lambda step, state, error=None: committed.append( + (step.name, state, error) + ) + ) + monkeypatch.setattr(progress, "event_sink_for_workspace", lambda workspace: sink) + + def fake_run_step(self, step, *, rerun=False, observer=None): + assert rerun is False + observer.on_step_completed(step, StateEnum.Success) + return StateEnum.Success + + flow = _make_flow( + _make_ws(str(tmp_path)), + [ + _make_step("Synthesis", "yosys"), + _make_step("Floorplan", "ecc"), + ], + fake_run_step, + ) + + buf = FakeTTYStderr(isatty_value=True) + result = run_flow_with_progress(flow, _make_ctx(), None, buf) + + assert result is True + assert committed == [ + ("Synthesis", StateEnum.Success, None), + ("Floorplan", StateEnum.Success, None), + ] + + def test_progress_run_commits_snapshot_for_failed_step(self, monkeypatch): + committed = [] + sink = SimpleNamespace( + on_step_completed=lambda step, state, error=None: committed.append((step.name, state)) + ) + monkeypatch.setattr(progress, "event_sink_for_workspace", lambda workspace: sink) + + def fake_run_step(self, step, *, rerun=False, observer=None): + state = StateEnum.Success if step.name == "Synthesis" else StateEnum.Imcomplete + observer.on_step_completed(step, state) + return state + + flow = _make_flow( + _make_ws(), + [_make_step("Synthesis", "yosys"), _make_step("Floorplan", "ecc")], + fake_run_step, + ) + + buf = FakeTTYStderr(isatty_value=True) + result = run_flow_with_progress(flow, _make_ctx(), None, buf) + + assert result is False + assert committed == [ + ("Synthesis", StateEnum.Success), + ("Floorplan", StateEnum.Imcomplete), + ] + # --------------------------------------------------------------------------- # Failure context block formatting (AC-5) diff --git a/test/engine/test_execution.py b/test/engine/test_execution.py index 734fe2b6..5ec7ee53 100644 --- a/test/engine/test_execution.py +++ b/test/engine/test_execution.py @@ -3,7 +3,7 @@ import pytest from chipcompiler.data import StateEnum -from chipcompiler.engine.execution import ExecutionPlan, execute +from chipcompiler.engine.execution import ExecutionPlan, event_sink_for_workspace, execute def test_execution_plan_dispatches_full_flow_and_single_step(): @@ -137,3 +137,8 @@ def run_steps(self, **_kwargs): assert result.executed_steps == () assert result.failed_step is None assert calls == [] + + +def test_event_sink_for_workspace_is_none_without_directory(): + assert event_sink_for_workspace(SimpleNamespace()) is None + assert event_sink_for_workspace(SimpleNamespace(directory=None)) is None diff --git a/test/test_engine_flow.py b/test/test_engine_flow.py index 2dd4b326..802b07b2 100644 --- a/test/test_engine_flow.py +++ b/test/test_engine_flow.py @@ -41,6 +41,27 @@ def test_engine_flow_missing_path_is_not_initialized(): assert engine_flow.has_init() is False +def test_run_step_without_runtime_operation_marker_does_not_expand_function( + monkeypatch, tmp_path +): + from chipcompiler.engine.execution import ExecutionObserver + + workspace = Workspace() + workspace.flow.data = { + "steps": [{"name": "route", "tool": "ecc", "state": "Unstart"}], + } + engine_flow = EngineFlow(workspace) + workspace_step = EccStep(name="route", directory=tmp_path, tool="ecc") + engine_flow.workspace_steps = [workspace_step] + engine_flow.engine_db = SimpleNamespace(engine=None) + + monkeypatch.setattr(tools, "run_step", lambda **_kwargs: True) + monkeypatch.setattr(engine_flow, "check_step_result", lambda **_kwargs: True) + + observer = ExecutionObserver(object()) + assert engine_flow.run_step(workspace_step, observer=observer) == StateEnum.Success + + def test_engine_flow_default_steps_include_synthesis_lec(tmp_path): workspace = Workspace() workspace.flow.path = tmp_path / "flow.json" From ae4308eb553ba79db623f888d8604db95dfefcf2 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Mon, 14 Sep 2026 23:33:42 +0800 Subject: [PATCH 39/43] fix(ecc): close runtime consolidation contract gaps --- chipcompiler/engine/execution.py | 43 ++++-- chipcompiler/engine/snapshot.py | 159 ++++++++++++++++++++- chipcompiler/project/api.py | 68 +++++---- chipcompiler/project/manifest_write.py | 22 ++- chipcompiler/runtime/operations.py | 89 +++++++++--- chipcompiler/runtime/requests.py | 8 +- chipcompiler/runtime/server.py | 1 + chipcompiler/runtime/workspace_api.py | 30 +++- chipcompiler/runtime/workspace_spec_api.py | 11 +- test/engine/test_execution.py | 7 + test/engine/test_snapshot_migration.py | 57 ++++++++ test/project/test_manifest.py | 33 +++++ test/runtime/test_operations.py | 71 ++++++++- test/runtime/test_server.py | 14 +- test/runtime/test_workspace_api.py | 16 ++- test/test_engine_flow.py | 4 +- 16 files changed, 543 insertions(+), 90 deletions(-) diff --git a/chipcompiler/engine/execution.py b/chipcompiler/engine/execution.py index e406e7b7..3d81960f 100644 --- a/chipcompiler/engine/execution.py +++ b/chipcompiler/engine/execution.py @@ -106,18 +106,39 @@ def on_step_completed(self, step: Any, state: Any, error: str | None = None) -> self.executed_steps.append(name) elif self.failed_step is None: self.failed_step = name - callback = getattr(self.delegate, "on_step_completed", None) + self._delegate_call("on_step_completed", step, state, error) + + @property + def runtime_operation(self): + return getattr(self.delegate, "runtime_operation", None) + + def raise_if_cancelled(self) -> None: + self._delegate_call("raise_if_cancelled") + + def on_step_started(self, step: Any) -> None: + self._delegate_call("on_step_started", step) + + def on_step_skipped(self, step: Any) -> None: + self._delegate_call("on_step_skipped", step) + + def on_subflow_stage(self, step: Any, subflow_step: Any) -> None: + self._delegate_call("on_subflow_stage", step, subflow_step) + + def on_step_diagnostic(self, step: Any, diagnostic: dict[str, Any]) -> None: + self._delegate_call("on_step_diagnostic", step, diagnostic) + + def on_rerun_prepared(self, *args, **kwargs) -> None: + self._delegate_call("on_rerun_prepared", *args, **kwargs) + + def wait_for_step_rendered(self, step: Any, state: Any) -> bool: + result = self._delegate_call("wait_for_step_rendered", step, state) + return True if result is None else bool(result) + + def _delegate_call(self, name: str, *args, **kwargs): + callback = getattr(self.delegate, name, None) if callable(callback): - callback(step, state, error) - - def __getattr__(self, name: str): - if self.delegate is not None: - callback = getattr(self.delegate, name, None) - if callback is not None: - return callback - if name == "runtime_operation": - return None - return lambda *_args, **_kwargs: None + return callback(*args, **kwargs) + return None class _EngineeringCommitSink: diff --git a/chipcompiler/engine/snapshot.py b/chipcompiler/engine/snapshot.py index 979e2ba5..a1c55b58 100644 --- a/chipcompiler/engine/snapshot.py +++ b/chipcompiler/engine/snapshot.py @@ -1,3 +1,4 @@ +import hashlib from copy import deepcopy from pathlib import Path from typing import Any @@ -8,7 +9,7 @@ unavailable_qor_snapshot_extension, validate_qor_snapshot_extension, ) -from chipcompiler.utility import JsonReadError, json_read, json_read_strict, json_write +from chipcompiler.utility import JsonReadError, file_digest, json_read, json_read_strict, json_write SNAPSHOT_SCHEMA_VERSION = 2 SNAPSHOT_V3_SCHEMA_VERSION = 3 @@ -51,8 +52,22 @@ def ensure_engineering_snapshot(workspace: Any) -> dict[str, Any]: return create_engineering_snapshot(workspace, cause="workspace.migrated") -def read_engineering_snapshot(workspace: Any) -> dict[str, Any]: - return _read_snapshot(_snapshot_path(workspace)) +def read_engineering_snapshot( + workspace: Any, + *, + expected_workspace_id: str | None = None, + expected_workspace_revision: int | None = None, + validate_artifacts: bool = True, +) -> dict[str, Any]: + snapshot = _read_snapshot(_snapshot_path(workspace), validate_artifacts=validate_artifacts) + if expected_workspace_id is not None and snapshot["workspaceId"] != expected_workspace_id: + raise EngineeringSnapshotError("Engineering Snapshot workspace identity mismatch") + if ( + expected_workspace_revision is not None + and snapshot["workspaceRevision"] != expected_workspace_revision + ): + raise EngineeringSnapshotError("Engineering Snapshot Workspace Revision mismatch") + return snapshot def read_engineering_snapshot_from_directory(directory: str | Path) -> dict[str, Any]: @@ -61,7 +76,7 @@ def read_engineering_snapshot_from_directory(directory: str | Path) -> dict[str, def read_stale_engineering_snapshot(workspace: Any) -> dict[str, Any] | None: path = _stale_snapshot_path(workspace) - return _read_snapshot(path) if path.is_file() else None + return _read_snapshot(path, validate_artifacts=False) if path.is_file() else None def migrate_engineering_snapshot( @@ -113,7 +128,7 @@ def commit_engineering_snapshot( workspace_id: str, cause: str, ) -> dict[str, Any]: - current = read_engineering_snapshot(workspace) + current = read_engineering_snapshot(workspace, validate_artifacts=False) if current["schemaVersion"] != SNAPSHOT_SCHEMA_VERSION: raise EngineeringSnapshotError("production Snapshot schema is still v2") if current["workspaceId"] != workspace_id: @@ -151,7 +166,7 @@ def invalidate_engineering_snapshot( cause: str, first_invalidated_step: str | None = None, ) -> dict[str, Any]: - current = read_engineering_snapshot(workspace) + current = read_engineering_snapshot(workspace, validate_artifacts=False) if current["schemaVersion"] != SNAPSHOT_SCHEMA_VERSION: raise EngineeringSnapshotError("production Snapshot schema is still v2") if current["workspaceId"] != workspace_id: @@ -269,7 +284,7 @@ def _write_snapshot(path: Path, snapshot: dict[str, Any]) -> None: raise EngineeringSnapshotError(f"failed to persist Engineering Snapshot: {path}") -def _read_snapshot(path: Path) -> dict[str, Any]: +def _read_snapshot(path: Path, *, validate_artifacts: bool = True) -> dict[str, Any]: try: snapshot = json_read_strict(path) except (OSError, JsonReadError) as exc: @@ -288,4 +303,134 @@ def _read_snapshot(path: Path) -> dict[str, Any]: ) ): raise EngineeringSnapshotError(f"invalid Engineering Snapshot: {path}") + _validate_snapshot_sections( + snapshot, + path.parent.parent, + validate_artifacts=validate_artifacts, + ) return snapshot + + +def _validate_snapshot_sections( + snapshot: dict[str, Any], + workspace_root: Path, + *, + validate_artifacts: bool, +) -> None: + for key in ( + "flow", + "parameters", + "checklist", + "analysis", + "qorAssessment", + "signoffAssessment", + ): + if not isinstance(snapshot.get(key), dict): + raise EngineeringSnapshotError(f"invalid Engineering Snapshot section: {key}") + if "steps" in snapshot["flow"] and not isinstance(snapshot["flow"]["steps"], list): + raise EngineeringSnapshotError("invalid Engineering Snapshot section: flow.steps") + if "steps" in snapshot["analysis"] and not isinstance(snapshot["analysis"]["steps"], list): + raise EngineeringSnapshotError("invalid Engineering Snapshot section: analysis.steps") + artifacts = snapshot.get("artifacts") + if not isinstance(artifacts, list) or len(artifacts) > 4096: + raise EngineeringSnapshotError("invalid Engineering Snapshot section: artifacts") + workspace_root = workspace_root.resolve() + metadata_id = _workspace_metadata_id(workspace_root) + if metadata_id is not None and metadata_id != snapshot["workspaceId"]: + raise EngineeringSnapshotError("Engineering Snapshot workspace identity mismatch") + for artifact in artifacts: + _validate_snapshot_artifact( + artifact, + snapshot["workspaceId"], + workspace_root, + validate_artifacts=validate_artifacts, + ) + stale = snapshot.get("stalePredecessor") + if stale is not None and ( + not isinstance(stale, dict) + or type(stale.get("workspaceRevision")) is not int + or stale["workspaceRevision"] < 1 + or not isinstance(stale.get("invalidatedStepIds"), list) + or not all(isinstance(step_id, str) and step_id for step_id in stale["invalidatedStepIds"]) + ): + raise EngineeringSnapshotError("invalid Engineering Snapshot stale predecessor") + + +def _validate_snapshot_artifact( + artifact: object, + workspace_id: str, + workspace_root: Path, + *, + validate_artifacts: bool, +) -> None: + if not isinstance(artifact, dict): + raise EngineeringSnapshotError("invalid Engineering Snapshot artifact") + artifact_id = artifact.get("artifactId") + reference = artifact.get("reference") + availability = artifact.get("availability") + if ( + not isinstance(artifact_id, str) + or not isinstance(reference, str) + or not reference + or Path(reference).is_absolute() + or ".." in Path(reference).parts + or artifact_id != _artifact_id(workspace_id, reference) + or availability not in {"missing", "available", "stale"} + ): + raise EngineeringSnapshotError("invalid Engineering Snapshot artifact reference") + if availability != "available" or not validate_artifacts: + return + digest = artifact.get("sha256") + size = artifact.get("sizeBytes") + if ( + not isinstance(digest, str) + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest.lower()) + or type(size) is not int + or size < 0 + ): + raise EngineeringSnapshotError("invalid Engineering Snapshot artifact fingerprint") + candidate = workspace_root / reference + try: + candidate.relative_to(workspace_root) + except ValueError as exc: + raise EngineeringSnapshotError("invalid Engineering Snapshot artifact path") from exc + if _contains_symlink(candidate, workspace_root): + raise EngineeringSnapshotError("invalid Engineering Snapshot artifact path") + if file_digest(candidate) != (digest, size): + raise EngineeringSnapshotError("Engineering Snapshot artifact fingerprint mismatch") + + +def _contains_symlink(path: Path, root: Path) -> bool: + current = path + while current != root: + if current.is_symlink() or current.parent == current: + return True + current = current.parent + return root.is_symlink() + + +def _workspace_metadata_id(workspace_root: Path) -> str | None: + command_path = workspace_root / "home" / "workspace-commands.json" + try: + payload = json_read_strict(command_path) + except (OSError, JsonReadError): + return None + commands = payload.get("commands") if isinstance(payload, dict) else None + if not isinstance(commands, dict): + return None + workspace_ids = { + result.get("workspaceId") + for command in commands.values() + if isinstance(command, dict) + and isinstance(result := command.get("result"), dict) + and isinstance(result.get("workspaceId"), str) + } + if len(workspace_ids) > 1: + raise EngineeringSnapshotError("invalid Workspace command identity metadata") + return next(iter(workspace_ids), None) + + +def _artifact_id(workspace_id: str, reference: str) -> str: + digest = hashlib.sha256(f"{workspace_id}\0{reference}".encode()).hexdigest() + return f"artifact-{digest[:32]}" diff --git a/chipcompiler/project/api.py b/chipcompiler/project/api.py index d40abba2..7dfc6782 100644 --- a/chipcompiler/project/api.py +++ b/chipcompiler/project/api.py @@ -11,8 +11,10 @@ ) from chipcompiler.project.manifest_write import ( build_project_document, + manifest_lock, manifest_workspace_entry, update_manifest, + update_manifest_locked, write_manifest_if_absent, ) @@ -77,6 +79,14 @@ def mutate_project_manifest(project_dir: str | Path, mutation: dict) -> dict: project = Path(project_dir).expanduser().resolve() load_project_manifest(project) + apply = _project_manifest_mutator(project, mutation) + + if not update_manifest(str(project), apply): + raise ManifestError("Project Manifest update failed") + return load_project_manifest(project) + + +def _project_manifest_mutator(project: Path, mutation: dict): def apply(document: dict) -> None: kind = mutation.get("type") if kind == "register_workspace": @@ -90,9 +100,7 @@ def apply(document: dict) -> None: else: raise ManifestError(f"unsupported Project mutation: {kind}") - if not update_manifest(str(project), apply): - raise ManifestError("Project Manifest update failed") - return load_project_manifest(project) + return apply def create_project_workspace( @@ -108,7 +116,8 @@ def create_project_workspace( expected_project_id: str | None = None, now: str | None = None, ): - from chipcompiler.engine import create_workspace_from_spec + from chipcompiler.engine.reconcile import _workspace_lock + from chipcompiler.engine.workspace_lifecycle import _create_workspace_from_spec project = Path(project_dir).expanduser().resolve() target = Path(target_directory).expanduser().resolve() @@ -116,31 +125,34 @@ def create_project_workspace( target.relative_to(project) except ValueError as exc: raise ManifestError("Workspace must be inside the Project root") from exc - manifest = load_project_manifest(project) - if expected_project_id is not None and manifest["project_id"] != expected_project_id: - raise ManifestError("Project identity does not match") existed = target.exists() - workspace = create_workspace_from_spec(str(target), spec, bindings, command_id) - if workspace is None: - raise ManifestError("Workspace creation returned no Workspace") - identity = workspace_id or target.name - try: - mutate_project_manifest( - project, - { - "type": "register_workspace", - "workspace_id": identity, - "name": name or identity, - "workspace_path": str(target), - "source_workspace_id": source_workspace_id, - "created_at": now, - "updated_at": now, - }, - ) - except Exception: - if not existed: - shutil.rmtree(target, ignore_errors=True) - raise + with manifest_lock(project): + manifest = load_project_manifest(project) + if expected_project_id is not None and manifest["project_id"] != expected_project_id: + raise ManifestError("Project identity does not match") + try: + with _workspace_lock(target): + workspace = _create_workspace_from_spec(str(target), spec, bindings, command_id) + if workspace is None: + raise ManifestError("Workspace creation returned no Workspace") + identity = workspace_id or target.name + mutation = { + "type": "register_workspace", + "workspace_id": identity, + "name": name or identity, + "workspace_path": str(target), + "source_workspace_id": source_workspace_id, + "created_at": now, + "updated_at": now, + } + if not update_manifest_locked( + project, _project_manifest_mutator(project, mutation) + ): + raise ManifestError("Project Manifest update failed") + except Exception: + if not existed: + shutil.rmtree(target, ignore_errors=True) + raise return workspace diff --git a/chipcompiler/project/manifest_write.py b/chipcompiler/project/manifest_write.py index cf1364d0..71714bb1 100644 --- a/chipcompiler/project/manifest_write.py +++ b/chipcompiler/project/manifest_write.py @@ -17,6 +17,7 @@ import logging import os import tempfile +from contextlib import contextmanager from copy import deepcopy from datetime import UTC, datetime from pathlib import Path @@ -216,12 +217,10 @@ def update_manifest(project_dir: str, mutator) -> bool: the lock cannot be taken, or the write fails — callers degrade to a warning, never a run failure. """ - from chipcompiler.project.locking import flock_file - path = os.path.join(project_dir, MANIFEST_FILENAME) try: - with flock_file(os.path.join(project_dir, ".manifest.lock"), exclusive=True): - return _update_manifest_locked(path, mutator) + with manifest_lock(project_dir): + return update_manifest_locked(project_dir, mutator) except OSError as exc: # An untakeable lock (e.g. a directory at the lock path) degrades # like any write failure: a warning, never an uncaught exception — @@ -230,6 +229,21 @@ def update_manifest(project_dir: str, mutator) -> bool: return False +@contextmanager +def manifest_lock(project_dir: str | Path): + from chipcompiler.project.locking import flock_file + + yield_lock_path = os.path.join(str(project_dir), ".manifest.lock") + with flock_file(yield_lock_path, exclusive=True): + yield + + +def update_manifest_locked(project_dir: str | Path, mutator) -> bool: + """Apply a manifest mutation while the caller owns ``manifest_lock``.""" + path = os.path.join(str(project_dir), MANIFEST_FILENAME) + return _update_manifest_locked(path, mutator) + + def _update_manifest_locked(path: str, mutator) -> bool: base = _read_manifest_document(path) if base is None: diff --git a/chipcompiler/runtime/operations.py b/chipcompiler/runtime/operations.py index f9321a3e..c18653dc 100644 --- a/chipcompiler/runtime/operations.py +++ b/chipcompiler/runtime/operations.py @@ -189,7 +189,11 @@ def operation_status(self, operation_id: str) -> dict[str, Any]: def is_active(self, operation_id: str) -> bool: with self._lock: operation = self._operations.get(operation_id) - return operation is not None and operation.state not in _TERMINAL_OPERATION_STATES + return ( + operation is not None + and operation.state not in _TERMINAL_OPERATION_STATES + and self._active_by_workspace.get(operation.workspace_id) == operation_id + ) def has_active_workspace(self, workspace_id: str) -> bool: with self._lock: @@ -199,13 +203,21 @@ def has_active_workspace(self, workspace_id: str) -> bool: operation = self._operations.get(operation_id) return operation is not None and operation.state not in _TERMINAL_OPERATION_STATES - def load_workspace_ledger(self, workspace_id: str, ledger_path: str | Path) -> list[str]: - """Restore the bounded execution ledger after a Runtime restart.""" + def load_workspace_ledger( + self, + workspace_id: str, + ledger_path: str | Path, + *, + recover: bool = True, + ) -> list[str]: + """Load the bounded execution ledger, optionally marking live work interrupted.""" from chipcompiler.utility import JsonReadError, json_read_strict path = Path(ledger_path).expanduser().resolve() with self._lock: if path in self._loaded_ledgers: + if recover: + return self._recover_loaded_workspace_locked(workspace_id) return [] self._loaded_ledgers.add(path) self._ledger_paths[workspace_id] = path @@ -223,19 +235,6 @@ def load_workspace_ledger(self, workspace_id: str, ledger_path: str | Path) -> l operation = _operation_from_payload(entry, self._runtime_instance_id) if operation is None: continue - if operation.state not in _TERMINAL_OPERATION_STATES: - operation.state = "interrupted" - operation.error = { - "code": "interrupted", - "message": "Runtime process ended before the Operation completed", - } - operation.updated_at = time.time() - # Render ACKs are retained only as a wire-compatibility field. They - # never gate execution, so old waiting records must not resurrect it. - operation.awaiting_event_id = None - operation.awaiting_event = None - operation.awaiting_step_commit_id = None - operation.render_sync_state = "idle" self._operations[operation.operation_id] = operation if operation.idempotency_key: self._idempotency[(workspace_id, operation.idempotency_key)] = ( @@ -246,10 +245,34 @@ def load_workspace_ledger(self, workspace_id: str, ledger_path: str | Path) -> l self._workspace_sequences.get(workspace_id, 0), operation.sequence ) restored.append(operation.operation_id) - self._prune_terminal_locked() - self._persist_workspace_locked(workspace_id) + if recover: + self._recover_loaded_workspace_locked(workspace_id) return restored + def _recover_loaded_workspace_locked(self, workspace_id: str) -> list[str]: + recovered: list[str] = [] + for operation in self._operations.values(): + if ( + operation.workspace_id != workspace_id + or operation.state in _TERMINAL_OPERATION_STATES + ): + continue + operation.state = "interrupted" + operation.error = { + "code": "interrupted", + "message": "Runtime process ended before the Operation completed", + } + operation.updated_at = time.time() + operation.awaiting_event_id = None + operation.awaiting_event = None + operation.awaiting_step_commit_id = None + operation.render_sync_state = "idle" + self._active_by_workspace.pop(workspace_id, None) + recovered.append(operation.operation_id) + self._prune_terminal_locked() + self._persist_workspace_locked(workspace_id) + return recovered + def workspace_snapshot(self, workspace_id: str) -> dict[str, Any]: with self._lock: operations = [ @@ -332,6 +355,7 @@ def request_cancel(self, operation_id: str) -> dict[str, Any]: if operation.state in _TERMINAL_OPERATION_STATES: return {"accepted": False, "operationId": operation_id, "state": operation.state} operation.cancel_requested = True + operation.state = "cancelling" operation.updated_at = time.time() self._persist_workspace_locked(operation.workspace_id) event = self._new_event_locked(operation, "operation.cancel_requested", {}) @@ -367,12 +391,33 @@ def _run( ) -> None: with self._lock: operation = self._operations[operation_id] - operation.state = "running" - operation.updated_at = time.time() - started_event = self._new_event_locked(operation, "operation.started", {}) + if operation.cancel_requested: + operation.state = "cancelled" + operation.error = { + "message": "operation cancelled before execution", + "code": "cancelled", + } + operation.updated_at = time.time() + event = self._new_event_locked( + operation, + "operation.cancelled", + {"error": operation.error}, + ) + self._prune_terminal_locked() + self._persist_workspace_locked(operation.workspace_id) + self._active_by_workspace.pop(operation.workspace_id, None) + publish_before_return = True + else: + operation.state = "running" + operation.updated_at = time.time() + event = self._new_event_locked(operation, "operation.started", {}) + publish_before_return = False + if publish_before_return: + self._publish(event) + return observer = RuntimeFlowObserver(self, operation_id, snapshot_committer) try: - self._publish(started_event) + self._publish(event) try: result = runner(observer) with self._lock: diff --git a/chipcompiler/runtime/requests.py b/chipcompiler/runtime/requests.py index 490388af..b648684c 100644 --- a/chipcompiler/runtime/requests.py +++ b/chipcompiler/runtime/requests.py @@ -162,7 +162,7 @@ class WorkspaceInfoRequest: @dataclass(frozen=True) class FlowRunRequest: workspace_id: str - expected_workspace_revision: int = 1 + expected_workspace_revision: int | None = None rerun: bool = False @@ -170,14 +170,14 @@ class FlowRunRequest: class FlowRunStepRequest: workspace_id: str step: str - expected_workspace_revision: int = 1 + expected_workspace_revision: int | None = None rerun: bool = False @dataclass(frozen=True) class OperationStartFlowRequest: workspace_id: str - expected_workspace_revision: int = 1 + expected_workspace_revision: int | None = None rerun: bool = False origin: str = "gui" idempotency_key: str = "" @@ -187,7 +187,7 @@ class OperationStartFlowRequest: class OperationStartStepRequest: workspace_id: str step: str - expected_workspace_revision: int = 1 + expected_workspace_revision: int | None = None rerun: bool = False reset_dependents: bool = False origin: str = "gui" diff --git a/chipcompiler/runtime/server.py b/chipcompiler/runtime/server.py index 98f35f91..5589b11a 100644 --- a/chipcompiler/runtime/server.py +++ b/chipcompiler/runtime/server.py @@ -181,6 +181,7 @@ def _project_runtime_event(event: dict) -> dict: state = { "operation.queued": "queued", "operation.started": "running", + "operation.cancel_requested": "cancelling", "operation.completed": "succeeded", "operation.failed": "failed", "operation.cancelled": "cancelled", diff --git a/chipcompiler/runtime/workspace_api.py b/chipcompiler/runtime/workspace_api.py index cc04a481..a2c6a53d 100644 --- a/chipcompiler/runtime/workspace_api.py +++ b/chipcompiler/runtime/workspace_api.py @@ -192,6 +192,7 @@ def _open_legacy_workspace(self, request: WorkspaceOpenRequest) -> dict: self.operations.load_workspace_ledger( session.workspace_id, session.directory / "home" / "runtime-commands.json", + recover=False, ) return _workspace_session_result(session) @@ -563,6 +564,10 @@ def run_step(session: WorkspaceSession) -> dict: def start_flow_operation(self, request: OperationStartFlowRequest) -> dict: self._require_gui_operation_origin(request.origin) session = self._get_session(request.workspace_id) + expected_revision = request.expected_workspace_revision + operation_revision = ( + session.workspace_revision if expected_revision is None else expected_revision + ) try: def validate_start() -> None: @@ -576,7 +581,7 @@ def validate_start() -> None: rerun=request.rerun, step="", idempotency_key=request.idempotency_key, - workspace_revision=request.expected_workspace_revision, + workspace_revision=operation_revision, snapshot_committer=lambda step, state, error: self._commit_step_snapshot( session, step, @@ -588,7 +593,7 @@ def validate_start() -> None: runner=lambda observer: self._flow_run( FlowRunRequest( workspace_id=request.workspace_id, - expected_workspace_revision=request.expected_workspace_revision, + expected_workspace_revision=expected_revision, rerun=request.rerun, ), observer=observer, @@ -603,6 +608,10 @@ def validate_start() -> None: def start_step_operation(self, request: OperationStartStepRequest) -> dict: self._require_gui_operation_origin(request.origin) session = self._get_session(request.workspace_id) + expected_revision = request.expected_workspace_revision + operation_revision = ( + session.workspace_revision if expected_revision is None else expected_revision + ) try: def validate_start() -> None: @@ -616,7 +625,7 @@ def validate_start() -> None: rerun=request.rerun, step=request.step, idempotency_key=request.idempotency_key, - workspace_revision=request.expected_workspace_revision, + workspace_revision=operation_revision, command_input={"resetDependents": request.reset_dependents}, snapshot_committer=lambda step, state, error: self._commit_step_snapshot( session, @@ -630,7 +639,7 @@ def validate_start() -> None: FlowRunStepRequest( workspace_id=request.workspace_id, step=request.step, - expected_workspace_revision=request.expected_workspace_revision, + expected_workspace_revision=expected_revision, rerun=request.rerun, ), observer=observer, @@ -747,7 +756,13 @@ def engineering_snapshot(self, request: WorkspaceIdRequest) -> dict: from chipcompiler.engine.snapshot import EngineeringSnapshotError, read_engineering_snapshot try: - return read_engineering_snapshot(session.workspace) + return read_engineering_snapshot( + session.workspace, + expected_workspace_id=session.workspace_id, + expected_workspace_revision=( + session.workspace_revision if session.workspace_revision > 0 else None + ), + ) except EngineeringSnapshotError as exc: raise RuntimeApiError( "engineering_snapshot_unavailable", @@ -1196,10 +1211,11 @@ def _with_session_mutation_lock( @staticmethod def _validate_workspace_revision( session: WorkspaceSession, - expected_workspace_revision: int, + expected_workspace_revision: int | None, ) -> None: if ( - session.workspace_revision == 0 + expected_workspace_revision is None + or session.workspace_revision == 0 or expected_workspace_revision == session.workspace_revision ): return diff --git a/chipcompiler/runtime/workspace_spec_api.py b/chipcompiler/runtime/workspace_spec_api.py index 9f5a7fbd..ba22530d 100644 --- a/chipcompiler/runtime/workspace_spec_api.py +++ b/chipcompiler/runtime/workspace_spec_api.py @@ -237,6 +237,7 @@ def _open_workspace_unlocked( self.operations.load_workspace_ledger( session.workspace_id, session.directory / "home" / "runtime-commands.json", + recover=False, ) return _workspace_session_result(session) @@ -421,7 +422,15 @@ def _read_engineering_snapshot(owner) -> dict: ) try: - return read_engineering_snapshot(getattr(owner, "workspace", owner)) + workspace = getattr(owner, "workspace", owner) + revision = getattr(owner, "workspace_revision", None) + return read_engineering_snapshot( + workspace, + expected_workspace_id=getattr(owner, "workspace_id", None), + expected_workspace_revision=revision + if isinstance(revision, int) and revision > 0 + else None, + ) except EngineeringSnapshotError as exc: raise RuntimeApiError("engineering_snapshot_unavailable", str(exc)) from exc diff --git a/test/engine/test_execution.py b/test/engine/test_execution.py index 5ec7ee53..ad8d705e 100644 --- a/test/engine/test_execution.py +++ b/test/engine/test_execution.py @@ -40,6 +40,13 @@ def run_step(self, step, *, rerun=False, observer=None): assert all(call[2].delegate is observer for call in calls) +def test_execution_observer_does_not_hide_unknown_callbacks(): + from chipcompiler.engine.execution import ExecutionObserver + + with pytest.raises(AttributeError): + _ = ExecutionObserver(object()).unknown_callback + + def test_execution_plan_checks_cancel_before_each_selected_step(): calls = [] diff --git a/test/engine/test_snapshot_migration.py b/test/engine/test_snapshot_migration.py index da1d7c05..b0a23fc7 100644 --- a/test/engine/test_snapshot_migration.py +++ b/test/engine/test_snapshot_migration.py @@ -1,3 +1,4 @@ +import hashlib import json from copy import deepcopy from pathlib import Path @@ -115,3 +116,59 @@ def test_qor_extension_validator_rejects_missing_and_invalid_nested_fields(): invalid_gate = deepcopy(extension) invalid_gate["feasibility"]["gates"].append({"id": "broken"}) assert not validate_qor_snapshot_extension(invalid_gate) + + +def test_snapshot_read_validates_expected_identity_and_revision(tmp_path): + workspace = _workspace(tmp_path) + create_engineering_snapshot(workspace, workspace_id="engineering-gcd") + + with pytest.raises(EngineeringSnapshotError, match="identity mismatch"): + read_engineering_snapshot(workspace, expected_workspace_id="other") + with pytest.raises(EngineeringSnapshotError, match="Revision mismatch"): + read_engineering_snapshot(workspace, expected_workspace_revision=2) + + (workspace.directory / "home" / "workspace-commands.json").write_text( + json.dumps( + { + "schemaVersion": 1, + "commands": { + "create-1": {"result": {"workspaceId": "other"}}, + }, + } + ), + encoding="utf-8", + ) + with pytest.raises(EngineeringSnapshotError, match="identity mismatch"): + read_engineering_snapshot(workspace) + + +def test_snapshot_read_rejects_invalid_sections_and_artifact_fingerprints(tmp_path): + workspace = _workspace(tmp_path) + create_engineering_snapshot(workspace, workspace_id="engineering-gcd") + path = Path(workspace.directory) / "home" / "engineering-snapshot.json" + payload = json.loads(path.read_text(encoding="utf-8")) + + invalid_section = deepcopy(payload) + invalid_section["flow"] = [] + path.write_text(json.dumps(invalid_section), encoding="utf-8") + with pytest.raises(EngineeringSnapshotError, match="section: flow"): + read_engineering_snapshot(workspace) + + path.write_text(json.dumps(payload), encoding="utf-8") + artifact = payload["artifacts"][0] + artifact_path = workspace.directory / artifact["reference"] + artifact_path.parent.mkdir(parents=True, exist_ok=True) + artifact_path.write_bytes(b"before") + artifact.update( + { + "availability": "available", + "sizeBytes": 6, + "sha256": hashlib.sha256(b"before").hexdigest(), + } + ) + path.write_text(json.dumps(payload), encoding="utf-8") + assert read_engineering_snapshot(workspace)["artifacts"][0]["availability"] == "available" + + artifact_path.write_bytes(b"after") + with pytest.raises(EngineeringSnapshotError, match="fingerprint mismatch"): + read_engineering_snapshot(workspace) diff --git a/test/project/test_manifest.py b/test/project/test_manifest.py index e9f04b01..0d768760 100644 --- a/test/project/test_manifest.py +++ b/test/project/test_manifest.py @@ -1,5 +1,7 @@ import json +from contextlib import contextmanager +import chipcompiler.project.api as project_api from chipcompiler.cli.project.manifest import load_manifest from chipcompiler.project import ( create_project_manifest, @@ -75,3 +77,34 @@ def test_manifest_mutation_without_timestamp_keeps_audit_timestamp(tmp_path): assert updated["updated_at"] assert updated["workspaces"][0]["updated_at"] + + +def test_project_workspace_acquires_manifest_lock_before_workspace_lock(tmp_path, monkeypatch): + create_project_manifest(tmp_path, "Demo", "gcd", now="2026-01-01T00:00:00Z") + events = [] + + @contextmanager + def marked_lock(name): + events.append(f"{name}:acquire") + yield + + monkeypatch.setattr(project_api, "manifest_lock", lambda _project: marked_lock("manifest")) + monkeypatch.setattr( + "chipcompiler.engine.reconcile._workspace_lock", + lambda _target: marked_lock("workspace"), + ) + monkeypatch.setattr( + "chipcompiler.engine.workspace_lifecycle._create_workspace_from_spec", + lambda *_args: object(), + ) + monkeypatch.setattr(project_api, "update_manifest_locked", lambda *_args: True) + + project_api.create_project_workspace( + tmp_path, + tmp_path / "experiment", + {}, + {}, + command_id="create-1", + ) + + assert events == ["manifest:acquire", "workspace:acquire"] diff --git a/test/runtime/test_operations.py b/test/runtime/test_operations.py index 8127c27a..7ca9d415 100644 --- a/test/runtime/test_operations.py +++ b/test/runtime/test_operations.py @@ -89,7 +89,12 @@ def commit_step(*_args): runner=lambda observer: {"succeeded": flow.run_steps(observer=observer)}, ) assert first_step_running.wait(timeout=1) - assert manager.request_cancel(started["operationId"])["accepted"] is True + cancellation = manager.request_cancel(started["operationId"]) + assert cancellation == { + "accepted": True, + "operationId": started["operationId"], + "state": "cancelling", + } release_first_step.set() status = _wait_for_terminal(manager, started["operationId"]) @@ -102,6 +107,34 @@ def commit_step(*_args): assert event_types.index("step.completed") < event_types.index("operation.cancelled") +def test_queued_cancel_finishes_without_leaving_active_workspace(monkeypatch): + class DeferredThread: + def __init__(self, target, args, **_kwargs): + self._target = target + self._args = args + + def start(self): + return None + + monkeypatch.setattr("chipcompiler.runtime.operations.threading.Thread", DeferredThread) + manager = RuntimeOperationManager() + started = manager.start( + workspace_id="workspace-1", + kind="flow", + origin="gui", + rerun=False, + step="", + idempotency_key="queued-cancel", + runner=lambda _observer: {"ok": True}, + ) + + assert manager.request_cancel(started["operationId"])["state"] == "cancelling" + manager._run(started["operationId"], lambda _observer: {"ok": True}, None) + + assert manager.operation_status(started["operationId"])["state"] == "cancelled" + assert manager.shutdown_barrier() is None + + def test_subflow_stage_is_emitted_for_the_active_workspace_step(): events = [] released = threading.Event() @@ -462,6 +495,42 @@ def test_operation_ledger_recovers_unfinished_operations_as_interrupted(tmp_path release.set() +def test_read_only_ledger_load_does_not_recover_or_rewrite(tmp_path): + ledger = tmp_path / "runtime-commands.json" + ledger.write_text( + json.dumps( + { + "schemaVersion": 1, + "workspaceId": "workspace-1", + "operations": [ + { + "operationId": "operation-running", + "runSessionId": "session-running", + "runtimeInstanceId": "runtime-old", + "workspaceId": "workspace-1", + "kind": "flow", + "origin": "gui", + "state": "running", + } + ], + } + ), + encoding="utf-8", + ) + before = ledger.read_bytes() + + manager = RuntimeOperationManager() + loaded = manager.load_workspace_ledger("workspace-1", ledger, recover=False) + + assert loaded == ["operation-running"] + assert manager.operation_status("operation-running")["state"] == "running" + assert not manager.is_active("operation-running") + assert ledger.read_bytes() == before + + assert manager.load_workspace_ledger("workspace-1", ledger) == ["operation-running"] + assert manager.operation_status("operation-running")["state"] == "interrupted" + + def test_operation_ledger_clears_legacy_render_wait_state(tmp_path): ledger = tmp_path / "runtime-commands.json" ledger.write_text( diff --git a/test/runtime/test_server.py b/test/runtime/test_server.py index 2d1db5fa..7b275bcf 100644 --- a/test/runtime/test_server.py +++ b/test/runtime/test_server.py @@ -10,7 +10,7 @@ WorkspaceInspectSignoffRequest, WorkspaceOpenRequest, ) -from chipcompiler.runtime.server import RuntimeServer +from chipcompiler.runtime.server import RuntimeServer, _project_runtime_event from chipcompiler.runtime.workspace_api import RuntimeApiError @@ -136,6 +136,18 @@ def test_rpc_hello_reports_persistent_db_capabilities_when_enabled(): assert "db.release" in response["result"]["capabilities"] +def test_cancel_requested_event_projects_cancelling_state(): + projected = _project_runtime_event( + { + "type": "operation.cancel_requested", + "payload": {}, + } + ) + + assert projected["type"] == "operation.changed" + assert projected["payload"]["state"] == "cancelling" + + def test_rpc_hello_rejects_incompatible_version(): server = RuntimeServer() diff --git a/test/runtime/test_workspace_api.py b/test/runtime/test_workspace_api.py index bfb3dce9..c77bbea9 100644 --- a/test/runtime/test_workspace_api.py +++ b/test/runtime/test_workspace_api.py @@ -20,10 +20,24 @@ WorkspaceRecoverInterruptedRequest, WorkspaceSyncConfigRequest, ) -from chipcompiler.runtime.sessions import WorkspaceSessionRegistry +from chipcompiler.runtime.sessions import WorkspaceSession, WorkspaceSessionRegistry from chipcompiler.runtime.workspace_api import RuntimeApiError, WorkspaceRuntimeApi +def test_legacy_flow_request_without_revision_remains_compatible(tmp_path): + session = WorkspaceSession( + workspace_id="workspace-1", + directory=tmp_path, + workspace=object(), + workspace_revision=2, + ) + + WorkspaceRuntimeApi._validate_workspace_revision( + session, + FlowRunRequest(workspace_id="workspace-1").expected_workspace_revision, + ) + + class DummyEngineDB: def __init__(self, flow): self.flow = flow diff --git a/test/test_engine_flow.py b/test/test_engine_flow.py index 802b07b2..068dc017 100644 --- a/test/test_engine_flow.py +++ b/test/test_engine_flow.py @@ -41,9 +41,7 @@ def test_engine_flow_missing_path_is_not_initialized(): assert engine_flow.has_init() is False -def test_run_step_without_runtime_operation_marker_does_not_expand_function( - monkeypatch, tmp_path -): +def test_run_step_without_runtime_operation_marker_does_not_expand_function(monkeypatch, tmp_path): from chipcompiler.engine.execution import ExecutionObserver workspace = Workspace() From 8406944327ba488dc15c5cc6f3dbeecd0eefcc54 Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Tue, 15 Sep 2026 10:33:32 +0800 Subject: [PATCH 40/43] fix(runtime): preserve CLI workspace identity on open --- chipcompiler/runtime/workspace_api.py | 18 ++++++++-- test/runtime/test_workspace_api.py | 48 +++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/chipcompiler/runtime/workspace_api.py b/chipcompiler/runtime/workspace_api.py index a2c6a53d..ebb26ec7 100644 --- a/chipcompiler/runtime/workspace_api.py +++ b/chipcompiler/runtime/workspace_api.py @@ -188,7 +188,18 @@ def open_workspace(self, request: WorkspaceOpenRequest | WorkspaceSpecOpenReques def _open_legacy_workspace(self, request: WorkspaceOpenRequest) -> dict: workspace = self._load_workspace(request.directory) build_flow_for_workspace(workspace, create_step_workspaces=False) - session = self.sessions.open_session(workspace.directory, workspace=workspace) + from chipcompiler.engine.snapshot import EngineeringSnapshotError, read_engineering_snapshot + + try: + snapshot = read_engineering_snapshot(workspace) + except EngineeringSnapshotError: + snapshot = None + session = self.sessions.open_session( + workspace.directory, + workspace=workspace, + workspace_id=snapshot["workspaceId"] if snapshot else None, + workspace_revision=snapshot["workspaceRevision"] if snapshot else 0, + ) self.operations.load_workspace_ledger( session.workspace_id, session.directory / "home" / "runtime-commands.json", @@ -2460,7 +2471,10 @@ def build_flow_for_workspace(workspace, *, create_step_workspaces: bool = True): def _workspace_session_result(session: WorkspaceSession) -> dict: - return {"workspaceId": session.workspace_id, "directory": str(session.directory)} + result = {"workspaceId": session.workspace_id, "directory": str(session.directory)} + if session.workspace_revision > 0: + result["workspaceRevision"] = session.workspace_revision + return result def _db_ensure_result( diff --git a/test/runtime/test_workspace_api.py b/test/runtime/test_workspace_api.py index c77bbea9..427fdc7c 100644 --- a/test/runtime/test_workspace_api.py +++ b/test/runtime/test_workspace_api.py @@ -18,6 +18,7 @@ WorkspaceInfoRequest, WorkspaceOpenRequest, WorkspaceRecoverInterruptedRequest, + WorkspaceStepConfigurationReadRequest, WorkspaceSyncConfigRequest, ) from chipcompiler.runtime.sessions import WorkspaceSession, WorkspaceSessionRegistry @@ -417,6 +418,53 @@ def test_open_workspace_loads_without_creating_step_workspaces(monkeypatch, tmp_ assert not DummyFlow.instances[0].created +def test_open_workspace_reuses_engineering_snapshot_identity(monkeypatch, tmp_path): + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + monkeypatch.setattr( + "chipcompiler.engine.snapshot.read_engineering_snapshot", + lambda _workspace: {"workspaceId": "cli-workspace", "workspaceRevision": 7}, + ) + api = WorkspaceRuntimeApi() + + result = api.open_workspace(WorkspaceOpenRequest(directory=str(ws))) + + assert result == { + "workspaceId": "cli-workspace", + "workspaceRevision": 7, + "directory": str(ws.resolve()), + } + + +def test_step_configuration_keeps_cli_workspace_identity_after_open(monkeypatch, tmp_path): + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + monkeypatch.setattr( + "chipcompiler.engine.snapshot.read_engineering_snapshot", + lambda _workspace: {"workspaceId": "cli-workspace", "workspaceRevision": 7}, + ) + monkeypatch.setattr( + "chipcompiler.engine.read_step_configuration", + lambda _workspace, _step: { + "step": "Synthesis", + "stepId": "Synthesis", + "parameters": [], + "workspaceId": "cli-workspace", + "workspaceRevision": 7, + }, + ) + api = WorkspaceRuntimeApi() + opened = api.open_workspace(WorkspaceOpenRequest(directory=str(ws))) + + result = api.read_workspace_step_configuration( + WorkspaceStepConfigurationReadRequest( + step="Synthesis", + workspace_id=opened["workspaceId"], + ) + ) + + assert result["workspaceId"] == opened["workspaceId"] + assert result["workspaceRevision"] == opened["workspaceRevision"] + + def test_recover_interrupted_is_marker_scoped_and_idempotent(monkeypatch, tmp_path): _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) api = WorkspaceRuntimeApi() From 4787c24d783ab19804b4433d7865f2b99138742d Mon Sep 17 00:00:00 2001 From: Yell-walkalone <12112088@qq.com> Date: Tue, 15 Sep 2026 13:57:02 +0800 Subject: [PATCH 41/43] fix(data): persist workspace filelist across reloads create_workspace only rebinding design.input_filelist in memory meant the filelist input was lost whenever the workspace was reloaded from disk (step subprocesses, later GUI sessions); synthesis then fell back to the single origin verilog and multi-file designs failed hierarchy with "Module not found". Record the frozen filelist path in params.toml (file_list) at creation and restore design.input_filelist from it in the loader, keeping the origin/filelist convention as fallback. --- chipcompiler/data/workspace/__init__.py | 6 ++++ chipcompiler/data/workspace/loader.py | 9 +++++- test/data/test_workspace_filelist.py | 38 +++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/chipcompiler/data/workspace/__init__.py b/chipcompiler/data/workspace/__init__.py index dd0627f9..319102a7 100644 --- a/chipcompiler/data/workspace/__init__.py +++ b/chipcompiler/data/workspace/__init__.py @@ -1124,6 +1124,12 @@ def create_workspace( input_filelist=input_filelist, golden_verilog=golden_verilog, ) + if workspace.design.input_filelist is not None: + # The in-memory rebind dies with this process; persist the frozen copy + # so reloaded workspaces (step subprocesses, later sessions) keep the + # filelist input instead of falling back to the single origin verilog. + workspace.parameters.data["file_list"] = str(workspace.design.input_filelist) + init_workspace_config(workspace) # set home data diff --git a/chipcompiler/data/workspace/loader.py b/chipcompiler/data/workspace/loader.py index e08b06e7..e4bd204b 100644 --- a/chipcompiler/data/workspace/loader.py +++ b/chipcompiler/data/workspace/loader.py @@ -106,8 +106,15 @@ def load_workspace(directory: str | Path, *, read_only: bool = False) -> Any: if golden is not None: workspace.design.golden_verilog = golden + persisted_filelist = parameters.data.get("file_list", "") + if persisted_filelist and Path(persisted_filelist).is_file(): + workspace.design.input_filelist = Path(persisted_filelist) + filelist_path = origin_dir / "filelist" - if filelist_path.exists(): + if ( + workspace.design.input_filelist is None + and filelist_path.exists() + ): workspace.design.input_filelist = filelist_path workspace.flow.path = home_dir / "flow.json" diff --git a/test/data/test_workspace_filelist.py b/test/data/test_workspace_filelist.py index 4c01a107..b178dd33 100644 --- a/test/data/test_workspace_filelist.py +++ b/test/data/test_workspace_filelist.py @@ -86,6 +86,44 @@ def test_workspace_with_nested_filelist(self, tmp_path, test_parameters, pdk): assert (origin_dir / "rtl" / "core" / "alu.v").exists() assert (origin_dir / "rtl" / "core" / "ctrl.v").exists() + def test_filelist_survives_workspace_reload(self, tmp_path, minimal_ics55_pdk_factory): + from chipcompiler.data import load_workspace + from chipcompiler.data.parameter import load_parameter + from chipcompiler.data.workspace_config import workspace_config_path + + project_dir = tmp_path / "project" + project_dir.mkdir() + _write_rtl_file(project_dir / "gcd.v", "gcd") + + filelist = project_dir / "design.f" + _create_filelist(filelist, "gcd.v") + + pdk_root = minimal_ics55_pdk_factory(tmp_path / "ics55") + workspace_dir = tmp_path / "workspace" + create_workspace( + directory=str(workspace_dir), + origin_def="", + origin_verilog="", + pdk="ics55", + parameters={ + "design": "gcd", + "top_module": "gcd", + "clock": "clk", + "frequency_max": 100, + }, + input_filelist=str(filelist), + pdk_root=pdk_root, + ) + + frozen = workspace_dir / "origin" / "design.f" + assert frozen.is_file() + + parameters = load_parameter(workspace_config_path(workspace_dir)) + assert parameters.data["file_list"] == str(frozen) + + reloaded = load_workspace(workspace_dir) + assert reloaded.design.input_filelist == frozen + def test_filelist_absolute_entries_rewritten_to_frozen_sources(self, tmp_path): from chipcompiler.data.workspace import copy_filelist_with_sources From 850acadbd625e73f6cb134148136097f8d532ce5 Mon Sep 17 00:00:00 2001 From: Yell-walkalone <12112088@qq.com> Date: Tue, 15 Sep 2026 14:11:57 +0800 Subject: [PATCH 42/43] style: apply ruff format to workspace loader --- chipcompiler/data/workspace/loader.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/chipcompiler/data/workspace/loader.py b/chipcompiler/data/workspace/loader.py index e4bd204b..a5d0abaf 100644 --- a/chipcompiler/data/workspace/loader.py +++ b/chipcompiler/data/workspace/loader.py @@ -111,10 +111,7 @@ def load_workspace(directory: str | Path, *, read_only: bool = False) -> Any: workspace.design.input_filelist = Path(persisted_filelist) filelist_path = origin_dir / "filelist" - if ( - workspace.design.input_filelist is None - and filelist_path.exists() - ): + if workspace.design.input_filelist is None and filelist_path.exists(): workspace.design.input_filelist = filelist_path workspace.flow.path = home_dir / "flow.json" From d32f8f2a3487310b55f8ba65b2f2f4afea1256fc Mon Sep 17 00:00:00 2001 From: KoEkko <2251930460@qq.com> Date: Tue, 15 Sep 2026 16:02:00 +0800 Subject: [PATCH 43/43] fix(data): keep workspace filelists relocatable --- chipcompiler/data/workspace/__init__.py | 4 ++- chipcompiler/data/workspace/loader.py | 8 ++++-- test/data/test_workspace_filelist.py | 8 ++++-- test/engine/test_workspace_spec.py | 35 +++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/chipcompiler/data/workspace/__init__.py b/chipcompiler/data/workspace/__init__.py index 319102a7..20c50c4b 100644 --- a/chipcompiler/data/workspace/__init__.py +++ b/chipcompiler/data/workspace/__init__.py @@ -1128,7 +1128,9 @@ def create_workspace( # The in-memory rebind dies with this process; persist the frozen copy # so reloaded workspaces (step subprocesses, later sessions) keep the # filelist input instead of falling back to the single origin verilog. - workspace.parameters.data["file_list"] = str(workspace.design.input_filelist) + workspace.parameters.data["file_list"] = str( + workspace.design.input_filelist.relative_to(workspace_dir) + ) init_workspace_config(workspace) diff --git a/chipcompiler/data/workspace/loader.py b/chipcompiler/data/workspace/loader.py index a5d0abaf..e95a7e44 100644 --- a/chipcompiler/data/workspace/loader.py +++ b/chipcompiler/data/workspace/loader.py @@ -107,8 +107,12 @@ def load_workspace(directory: str | Path, *, read_only: bool = False) -> Any: workspace.design.golden_verilog = golden persisted_filelist = parameters.data.get("file_list", "") - if persisted_filelist and Path(persisted_filelist).is_file(): - workspace.design.input_filelist = Path(persisted_filelist) + if persisted_filelist: + persisted_filelist = Path(persisted_filelist) + if not persisted_filelist.is_absolute(): + persisted_filelist = workspace_dir / persisted_filelist + if persisted_filelist.is_file(): + workspace.design.input_filelist = persisted_filelist filelist_path = origin_dir / "filelist" if workspace.design.input_filelist is None and filelist_path.exists(): diff --git a/test/data/test_workspace_filelist.py b/test/data/test_workspace_filelist.py index b178dd33..749507b4 100644 --- a/test/data/test_workspace_filelist.py +++ b/test/data/test_workspace_filelist.py @@ -88,7 +88,7 @@ def test_workspace_with_nested_filelist(self, tmp_path, test_parameters, pdk): def test_filelist_survives_workspace_reload(self, tmp_path, minimal_ics55_pdk_factory): from chipcompiler.data import load_workspace - from chipcompiler.data.parameter import load_parameter + from chipcompiler.data.parameter import load_parameter, save_parameter from chipcompiler.data.workspace_config import workspace_config_path project_dir = tmp_path / "project" @@ -119,11 +119,15 @@ def test_filelist_survives_workspace_reload(self, tmp_path, minimal_ics55_pdk_fa assert frozen.is_file() parameters = load_parameter(workspace_config_path(workspace_dir)) - assert parameters.data["file_list"] == str(frozen) + assert parameters.data["file_list"] == "origin/design.f" reloaded = load_workspace(workspace_dir) assert reloaded.design.input_filelist == frozen + parameters.data["file_list"] = str(frozen) + assert save_parameter(parameters) + assert load_workspace(workspace_dir).design.input_filelist == frozen + def test_filelist_absolute_entries_rewritten_to_frozen_sources(self, tmp_path): from chipcompiler.data.workspace import copy_filelist_with_sources diff --git a/test/engine/test_workspace_spec.py b/test/engine/test_workspace_spec.py index d9dd47d3..d3cffaa1 100644 --- a/test/engine/test_workspace_spec.py +++ b/test/engine/test_workspace_spec.py @@ -288,6 +288,41 @@ def test_workspace_spec_update_is_atomic_revisioned_and_idempotent( assert read_engineering_snapshot(repeated) == after +def test_workspace_spec_update_keeps_generated_filelist_relocatable( + tmp_path, minimal_ics55_pdk_factory +): + from chipcompiler.data import load_workspace + from chipcompiler.data.parameter import load_parameter + from chipcompiler.engine import create_workspace_from_spec, update_workspace_from_spec + from chipcompiler.engine.snapshot import read_engineering_snapshot + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + target = tmp_path / "workspace" + created = create_workspace_from_spec(target, payload["workspaceSpec"], bindings) + + second_rtl = tmp_path / "helper.v" + second_rtl.write_text("module helper; endmodule\n", encoding="utf-8") + updated_spec = deepcopy(payload["workspaceSpec"]) + updated_spec["inputs"].append({"inputId": "rtl-helper", "role": "rtl"}) + updated_bindings = deepcopy(bindings) + updated_bindings["inputs"]["rtl-helper"] = str(second_rtl) + + update_workspace_from_spec( + target, + read_engineering_snapshot(created)["workspaceRevision"], + updated_spec, + updated_bindings, + ) + + persisted = Path(load_parameter(target / "home" / "params.toml").data["file_list"]) + reopened = load_workspace(target) + + assert not persisted.is_absolute() + assert (target / persisted).is_file() + assert reopened.design.input_filelist == target / persisted + + def test_workspace_spec_stale_revision_does_not_create_missing_snapshot( tmp_path, minimal_ics55_pdk_factory ):