diff --git a/agent/engine.py b/agent/engine.py index ebf9232ea..44d725f2b 100644 --- a/agent/engine.py +++ b/agent/engine.py @@ -5,6 +5,7 @@ from chipcompiler.data import StateEnum, WorkspaceStep from chipcompiler.engine.flow import EngineFlow +from chipcompiler.engine.flow_completion import normalize_legacy_terminal_state from chipcompiler.engine.step_execution import get_process_rss_mb, track_current_process_memory from chipcompiler.utility.log import redirect_stdio_to_file @@ -34,7 +35,7 @@ def run_step(self, workspace_step: WorkspaceStep | str, *, rerun: bool = False) self.clear_db_engine_after_step(workspace_step, StateEnum.Success) return StateEnum.Success - self._normalize_legacy_terminal_state(workspace_step, step_tag) + normalize_legacy_terminal_state(self, workspace_step, step_tag) start_time = time.time() timing_constraints = self.timing_constraint_facts() diff --git a/chipcompiler/cli/command_handlers/param.py b/chipcompiler/cli/command_handlers/param.py index 8a56b08fd..0e795c64e 100644 --- a/chipcompiler/cli/command_handlers/param.py +++ b/chipcompiler/cli/command_handlers/param.py @@ -11,6 +11,7 @@ validate_pdk_target, validate_value, ) +from chipcompiler.rtl2gds import get_flow_builders, normalize_flow_step from chipcompiler.utility.file import write_text_atomic @@ -29,6 +30,20 @@ def _manifest_mode_error(ctx: CommandContext) -> CommandResult | None: return None +def _parameter_flow_error(schema, ctx: CommandContext) -> CommandResult | None: + preset = getattr(ctx.config, "flow_preset", "") + builder = get_flow_builders().get(preset) + if builder is None or schema.applies == "all" or schema.pdk_target is not None: + return None + flow_steps = {normalize_flow_step(step).casefold() for step, _tool, _state in builder()} + if normalize_flow_step(schema.applies).casefold() in flow_steps: + return None + return CommandResult.err( + [error_record("parameter_not_in_flow", param=schema.param, preset=preset)], + exit_code=1, + ) + + def param_list(args, ctx: CommandContext) -> CommandResult: if getattr(args, "workspace", None) is not None: from chipcompiler.cli.command_handlers import workspace_params @@ -45,12 +60,31 @@ def param_list(args, ctx: CommandContext) -> CommandResult: resolved, _ = resolve_parameters(toml_overrides=toml_overrides) project = ctx.project - selected_step = (getattr(args, "step", None) or "").casefold() + selected_step = normalize_flow_step(getattr(args, "step", None) or "").casefold() show_all = bool(getattr(args, "all", False)) + preset = getattr(ctx.config, "flow_preset", "") + builder = get_flow_builders().get(preset) + flow_step_names = ( + [normalize_flow_step(step).casefold() for step, _tool, _state in builder()] + if builder is not None + else [] + ) + flow_steps = set(flow_step_names) + first_flow_step = flow_step_names[0] if flow_step_names else "" + if selected_step and flow_steps and selected_step not in flow_steps: + return CommandResult.ok([]) records = [] for rp in resolved: s = rp.schema - if selected_step and selected_step not in {s.group.casefold(), s.applies.casefold()}: + applies = normalize_flow_step(s.applies).casefold() + if flow_steps and s.applies != "all" and applies not in flow_steps: + continue + schema_steps = { + normalize_flow_step(s.group).casefold(), + normalize_flow_step(s.applies).casefold(), + } + global_at_first_step = s.applies == "all" and selected_step == first_flow_step + if selected_step and selected_step not in schema_steps and not global_at_first_step: continue if not selected_step and not show_all and s.has_direct_target and not rp.is_explicit: continue @@ -160,6 +194,9 @@ def param_set(args, ctx: CommandContext) -> CommandResult: ], exit_code=1, ) + flow_error = _parameter_flow_error(schema, ctx) + if flow_error is not None: + return flow_error try: value = parse_value(raw_value, schema) @@ -248,6 +285,9 @@ def param_unset(args, ctx: CommandContext) -> CommandResult: ], exit_code=1, ) + flow_error = _parameter_flow_error(schema, ctx) + if flow_error is not None: + return flow_error config_path = _find_config_path(ctx.project_dir) if config_path is None: diff --git a/chipcompiler/cli/command_handlers/project.py b/chipcompiler/cli/command_handlers/project.py index 6ea8a5973..23d301175 100644 --- a/chipcompiler/cli/command_handlers/project.py +++ b/chipcompiler/cli/command_handlers/project.py @@ -444,6 +444,20 @@ def error(kind: str, **fields) -> CommandResult: return error("flow_range_requires_pair") if fresh_target and (command_input.resume or command_input.only is not None): return error("selector_requires_workspace") + if not fresh_target and ctx.project_state == "manifest" and cfg.manifest_driven: + return run_dispatch.dispatch_project_run( + command_input, + ctx, + cfg, + run_dir, + run_name, + cli_overrides, + flow_config, + project_state, + warning_records, + workspace_registered=workspace_registered, + execute_flow=execute_flow, + ) errors = effective_config.validate_effective( ctx, cfg, diff --git a/chipcompiler/cli/command_handlers/signoff.py b/chipcompiler/cli/command_handlers/signoff.py index aa55c376e..80f53e73e 100644 --- a/chipcompiler/cli/command_handlers/signoff.py +++ b/chipcompiler/cli/command_handlers/signoff.py @@ -9,7 +9,7 @@ def inspect(command_input, ctx: CommandContext) -> CommandResult: if failure is not None: return failure - from chipcompiler.runtime.signoff_export import inspect_signoff_package + from chipcompiler.engine.signoff_export import inspect_signoff_package review = inspect_signoff_package(workspace) project = ctx.project @@ -56,17 +56,18 @@ def export(command_input, ctx: CommandContext) -> CommandResult: if failure is not None: return failure - from chipcompiler.runtime.workspace_api import RuntimeApiError - try: - from chipcompiler.runtime.signoff_export import export_signoff_package_archive + from chipcompiler.engine.signoff_export import ( + SignoffExportError, + export_signoff_package_archive, + ) output_path = export_signoff_package_archive( workspace, command_input.output_path, include_debug=command_input.include_debug, ) - except RuntimeApiError as exc: + except SignoffExportError as exc: return CommandResult.err( [ error_record( diff --git a/chipcompiler/cli/command_handlers/workspace_params.py b/chipcompiler/cli/command_handlers/workspace_params.py index 8c576c77e..7977ed402 100644 --- a/chipcompiler/cli/command_handlers/workspace_params.py +++ b/chipcompiler/cli/command_handlers/workspace_params.py @@ -12,6 +12,7 @@ workspace_param_step, workspace_param_value, ) +from chipcompiler.rtl2gds import normalize_flow_step def param_set(args, ctx: CommandContext) -> CommandResult: @@ -62,15 +63,19 @@ def param_list(args, ctx: CommandContext) -> CommandResult: if workspace_error is not None: return workspace_error overrides = {record["key"] for record in workspace_param_diff(workspace)} - selected_step = (args.step or "").casefold() + selected_step = normalize_flow_step(args.step or "").casefold() + flow_steps = workspace.flow.data.get("steps", []) + first_step = normalize_flow_step(flow_steps[0]["name"]).casefold() if flow_steps else "" records = [] for schema in list_schemas(): if schema.pdk_target is not None or (not args.all and schema.param not in overrides): continue - if selected_step and selected_step not in { - schema.group.casefold(), - schema.applies.casefold(), - }: + schema_steps = { + normalize_flow_step(schema.group).casefold(), + normalize_flow_step(schema.applies).casefold(), + } + global_at_first_step = schema.applies == "all" and selected_step == first_step + if selected_step and selected_step not in schema_steps and not global_at_first_step: continue try: value = workspace_param_value(workspace, schema) @@ -167,6 +172,8 @@ def _mutate( workspace, workspace_error = _load_workspace(ctx) if workspace_error is not None: return workspace_error + if (Path(ctx.run_dir) / "home" / "engineering-snapshot.json").is_file(): + return _mutate_via_engine(ctx, workspace, schema, requested_value, status) try: result = mutation(workspace) except ValueError as exc: @@ -223,6 +230,57 @@ def _mutate( return CommandResult.ok([record]) +def _mutate_via_engine(ctx, workspace, schema, requested_value: object, status: str): + from chipcompiler.data.workspace_parameters import workspace_param_diff, workspace_param_step + from chipcompiler.engine import update_workspace_step_configuration + from chipcompiler.engine.snapshot import read_engineering_snapshot + from chipcompiler.engine.workspace_lifecycle import WorkspaceLifecycleError + + overrides = workspace_param_diff(workspace) + existing = next((item for item in overrides if item["key"] == schema.param), None) + if status == "unset" and existing is None: + return CommandResult.ok([_record(ctx, schema.param, None, "no_override")]) + value = requested_value if status == "set" else existing["baseline"] + step = workspace_param_step(schema) + try: + revision = read_engineering_snapshot(workspace)["workspaceRevision"] + updated = update_workspace_step_configuration( + ctx.run_dir, + revision, + step, + {schema.param: value}, + command_id="", + ) + except WorkspaceLifecycleError as exc: + return CommandResult.err( + [error_record(exc.code, param=schema.param, reason=str(exc), **exc.details)] + ) + except Exception as exc: + return CommandResult.err( + [error_record("workspace_param_refresh_failed", param=schema.param, reason=str(exc))] + ) + + updated_steps = updated.flow.steps() + target = normalize_flow_step(step).casefold() + start = next( + ( + index + for index, item in enumerate(updated_steps) + if normalize_flow_step(item.get("name", "")).casefold() == target + ), + len(updated_steps), + ) + invalidated = [ + str(item.get("name", "")) + for item in updated_steps[start:] + if item.get("name") and item.get("state") == "Unstart" + ] + record = _record(ctx, schema.param, value, status) + record["from_step"] = step + record["invalidated_steps"] = invalidated + return CommandResult.ok([record]) + + def _schema_and_workspace_error(key: str, ctx: CommandContext): schema = lookup_schema(key) if schema is None: diff --git a/chipcompiler/cli/commands/report.py b/chipcompiler/cli/commands/report.py index 2aa6f3bfd..015ad1e82 100644 --- a/chipcompiler/cli/commands/report.py +++ b/chipcompiler/cli/commands/report.py @@ -31,7 +31,7 @@ def _finish(subcommand: str, command_input, handler) -> None: execute_command("report", command_input, handler, render_key=f"report:{subcommand}") -@report_app.command("qor", help="Show the overall QoR score report (GUI scoring rules)") +@report_app.command("qor", help="Show the overall QoR score report") def qor_cmd( *, output_path: OutputPathOption = None, diff --git a/chipcompiler/cli/inspection/discovery.py b/chipcompiler/cli/inspection/discovery.py index f1a1ad557..573ae386c 100644 --- a/chipcompiler/cli/inspection/discovery.py +++ b/chipcompiler/cli/inspection/discovery.py @@ -185,13 +185,20 @@ def resolve_command_workspace(workspace_arg, project, workspace_id, run_dir): instead. Returns (workspace, error-record-or-None); the caller maps a non-None record to a CommandResult.err. """ + import inspect + from chipcompiler.data import load_workspace path, error = resolve_workspace_path(workspace_arg, project, workspace_id, run_dir) if error is not None: return None, error try: - workspace = load_workspace(path) + load_kwargs = ( + {"read_only": True} + if "read_only" in inspect.signature(load_workspace).parameters + else {} + ) + workspace = load_workspace(path, **load_kwargs) except Exception as exc: return None, error_record("invalid_workspace", workspace=path, reason=str(exc)) if workspace is None: diff --git a/chipcompiler/cli/project/manifest.py b/chipcompiler/cli/project/manifest.py index a61239ec2..57aac98b5 100644 --- a/chipcompiler/cli/project/manifest.py +++ b/chipcompiler/cli/project/manifest.py @@ -1,427 +1,7 @@ -#!/usr/bin/env python +"""CLI compatibility alias for the Project Manifest domain module.""" -"""``project.json`` manifest support for the CLI. +import sys -The manifest is the GUI's project descriptor (schema v1). The CLI reads it -for configuration layering and run discovery, and projects it into -configuration payloads. Write operations (generation, status write-back, -registration) live in chipcompiler.cli.project.manifest_write, which -routes every write through one read-modify-write helper. +from chipcompiler.project import manifest as _domain_manifest -This module sits on the CLI startup path (imported by -cli/core/invocation.py): keep module-level imports cheap — no -chipcompiler.data imports here. -""" - -import json -import os -import re -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -MANIFEST_FILENAME = "project.json" - -# GUI display names for the canonical rtl2gds chain. -MANIFEST_FLOW_STEPS = ( - "Synth", - "LEC", - "PreFloorplan", - "MacroPlacement", - "PostFloorplan", - "Place", - "CTS", - "Legal", - "TimingOpt", - "Route", - "Filler", - "RCX", - "STA", - "LVS", - "PostRouteLEC", - "DRC", - "Harden", -) - -PRESET_MANIFEST_RANGE = { - "syn_sta": ("Synth", "Synth"), - "rtl2gds": ("Synth", "Harden"), - "synthesis_lec": ("Synth", "LEC"), - # Legacy presets removed from the builder; keep resolving them so - # persisted projects still load. - "rcx": ("Synth", "STA"), - "harden": ("Synth", "Harden"), -} - -_CANONICAL_TO_MANIFEST_STEP = { - "Synthesis": "Synth", - "lec": "LEC", - "preFloorplan": "PreFloorplan", - "macroPlacement": "MacroPlacement", - "postFloorplan": "PostFloorplan", - "place": "Place", - "CTS": "CTS", - "legalization": "Legal", - "Timing optimization": "TimingOpt", - "route": "Route", - "filler": "Filler", - "RCX": "RCX", - "sta": "STA", - "lvs": "LVS", - "postRouteLec": "PostRouteLEC", - "drc": "DRC", - "Harden": "Harden", -} - -_WORKSPACE_STATUSES = frozenset( - {"success", "failed", "running", "in_progress", "not_started", "archived"} -) - - -class ManifestError(ValueError): - """Raised when a project.json manifest cannot be used (manifest_invalid).""" - - -@dataclass(frozen=True) -class ManifestWorkspace: - workspace_id: str - workspace_path: str - start_step: str - end_step: str - status: str - parameter_patch: dict = field(default_factory=dict) - raw: dict = field(default_factory=dict) - - -@dataclass(frozen=True) -class ProjectManifest: - project_dir: str - path: str - project_id: str - name: str - design_name: str - base_design: dict - objectives: dict - workspaces: tuple[ManifestWorkspace, ...] - qor_baseline: dict | None - raw: dict - - def active_workspaces(self) -> list[ManifestWorkspace]: - return [w for w in self.workspaces if w.status != "archived"] - - def find_workspace(self, workspace_id: str) -> ManifestWorkspace | None: - """Match a managed workspace by its declared identifier only.""" - for workspace in self.workspaces: - if workspace.workspace_id == workspace_id: - return workspace - return None - - -def _optional_str(value: Any) -> str: - return value.strip() if isinstance(value, str) else "" - - -def _record(value: Any) -> dict: - return value if isinstance(value, dict) else {} - - -def _normalize_workspace_entry(value: Any, index: int, project_dir: str) -> ManifestWorkspace: - source = _record(value) - workspace_id = _optional_str(source.get("workspace_id")) - workspace_path = _optional_str(source.get("workspace_path")) - if not workspace_id or not workspace_path: - raise ManifestError(f"workspaces[{index}] requires workspace_id and workspace_path") - resolved = Path(workspace_path) - if not resolved.is_absolute(): - resolved = Path(project_dir) / resolved - try: - canonical = resolved.resolve() - canonical.relative_to(Path(project_dir).resolve()) - except ValueError: - raise ManifestError( - f"workspaces[{index}] workspace_path escapes the project root: {workspace_path}" - ) from None - except RuntimeError as exc: - # e.g. a symlink loop inside the spelled path: invalid manifest - # input, never a traceback. - raise ManifestError( - f"workspaces[{index}] workspace_path cannot be resolved: {workspace_path}" - ) from exc - status = source.get("status") - if not isinstance(status, str) or status not in _WORKSPACE_STATUSES: - status = "not_started" - start_step = _optional_str(source.get("start_step")) or "Synth" - end_step = _optional_str(source.get("end_step")) or "Harden" - for step_name, field_name in ((start_step, "start_step"), (end_step, "end_step")): - if step_name not in MANIFEST_FLOW_STEPS: - raise ManifestError( - f"workspaces[{index}] {field_name} is not on the canonical flow chain: {step_name}" - ) - if MANIFEST_FLOW_STEPS.index(start_step) > MANIFEST_FLOW_STEPS.index(end_step): - raise ManifestError( - f"workspaces[{index}] flow range is reversed: {start_step} -> {end_step}" - ) - return ManifestWorkspace( - workspace_id=workspace_id, - workspace_path=str(canonical), - start_step=start_step, - end_step=end_step, - status=status, - parameter_patch=_record(source.get("parameter_patch")), - raw=dict(source), - ) - - -def _validate_mpc(value: Any) -> None: - """Mirror the GUI parser's mpc rules: null or a well-formed MPC record.""" - if value is None: - return - source = _record(value) - if not source: - raise ManifestError("invalid project manifest: mpc must be an object or null") - resource_id = _optional_str(source.get("resource_id")) - if not resource_id.startswith("mpc:") or len(resource_id) == 4: - raise ManifestError("invalid project manifest: mpc.resource_id must be an MPC id") - for field_name in ("display_name", "installed_version", "path", "spec_path"): - if not _optional_str(source.get(field_name)): - raise ManifestError(f"invalid project manifest: mpc.{field_name} is required") - mpc_path = source["path"].rstrip("/") - if source["spec_path"] != f"{mpc_path}/spec/spec.json.in": - raise ManifestError( - "invalid project manifest: mpc.spec_path must reference spec/spec.json.in" - ) - design = _record(source.get("design")) - index = design.get("index") - if ( - not design - or isinstance(index, bool) # a JSON boolean is not an index (True == 1 in Python) - or not isinstance(index, (int, float)) - # The GUI accepts integral numbers (0.0). is_integer() applies to - # floats only: float(huge_int) raises OverflowError, and ints are - # integral by construction. - or (isinstance(index, float) and not index.is_integer()) - or index < 0 - or not _optional_str(design.get("design_name")) - ): - raise ManifestError( - "invalid project manifest: mpc.design requires a non-negative index and design_name" - ) - if not isinstance(source.get("core_template"), dict): - raise ManifestError("invalid project manifest: mpc.core_template must be an object") - - -def load_manifest(project_dir: str) -> ProjectManifest: - """Load and tolerantly normalize ``/project.json``. - - Mirrors the GUI parser's contract: schema_version 1 and a workspaces - array are required, everything else is default-filled. Raises - ManifestError on parse failure, root_path mismatch, or a workspace - path outside the project root. - """ - path = os.path.join(project_dir, MANIFEST_FILENAME) - try: - with open(path, encoding="utf-8") as f: - source = json.load(f) - except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc: - raise ManifestError(f"invalid project manifest: {path}: {exc}") from exc - - if not isinstance(source, dict): - raise ManifestError(f"invalid project manifest: {path}: top level must be an object") - schema_version = source.get("schema_version") - # A JSON boolean is not the schema version (True == 1 in Python); the - # GUI parser rejects it, and the CLI must not open what the GUI cannot. - if isinstance(schema_version, bool) or schema_version != 1: - raise ManifestError("invalid project manifest: schema_version 1 is required") - raw_workspaces = source.get("workspaces") - if not isinstance(raw_workspaces, list): - raise ManifestError("invalid project manifest: workspaces must be an array") - - root_path = _optional_str(source.get("root_path")) - if not root_path: - raise ManifestError("invalid project manifest: root_path is required") - if os.path.realpath(root_path) != os.path.realpath(project_dir): - raise ManifestError( - f"invalid project manifest: root_path {root_path} does not match {project_dir}" - ) - design_name = _optional_str(source.get("design_name")) - if not design_name: - raise ManifestError("invalid project manifest: design_name is required") - - name = _optional_str(source.get("name")) or os.path.basename(project_dir) or "project" - base_design = _record(source.get("base_design")) - base_design = {**base_design, "parameters": _record(base_design.get("parameters"))} - # Mirror the GUI parser: primary defaults to "timing", directions keep - # only maximize/minimize entries from the source (no default fill). - objectives_raw = _record(source.get("objectives")) - objectives = dict(objectives_raw) - objectives["primary"] = _optional_str(objectives_raw.get("primary")) or "timing" - objectives["directions"] = { - key: value - for key, value in _record(objectives_raw.get("directions")).items() - if value in ("maximize", "minimize") - } - qor_baseline_raw = _record(source.get("qor_baseline")) - qor_baseline = None - if _optional_str(qor_baseline_raw.get("workspace_id")): - qor_baseline = { - "workspace_id": qor_baseline_raw["workspace_id"], - "reason": _optional_str(qor_baseline_raw.get("reason")) or "Project QoR baseline", - } - - _validate_mpc(source.get("mpc")) - - return ProjectManifest( - project_dir=project_dir, - path=path, - project_id=_optional_str(source.get("project_id")) or f"proj_{_slugify(name)}", - name=name, - design_name=design_name, - base_design=base_design, - objectives=objectives, - workspaces=tuple( - _normalize_workspace_entry(entry, index, project_dir) - for index, entry in enumerate(raw_workspaces) - ), - qor_baseline=qor_baseline, - raw=source, - ) - - -def find_manifest(project_dir: str) -> str | None: - """The manifest path when the entry lexically exists. - - Lexical presence, not readability: a directory, symlink loop, or - dangling symlink still counts as PRESENT so ``load_manifest`` fails - loud (manifest_invalid) instead of silently demoting the project to - the virgin or legacy layout. - """ - path = os.path.join(project_dir, MANIFEST_FILENAME) - return path if os.path.lexists(path) else None - - -def has_legacy_runs_layout(project_dir: str) -> bool: - runs_dir = os.path.join(project_dir, "runs") - if not os.path.isdir(runs_dir): - return False - try: - return any(os.path.isdir(os.path.join(runs_dir, entry)) for entry in os.listdir(runs_dir)) - except OSError: - return False - - -def classify_project(project_dir: str) -> str: - """Classify a project directory: manifest | legacy | virgin.""" - if find_manifest(project_dir) is not None: - return "manifest" - if has_legacy_runs_layout(project_dir): - return "legacy" - return "virgin" - - -def assemble_config(manifest: ProjectManifest, workspace: ManifestWorkspace | None) -> dict: - """Flatten the manifest into a parameter payload (lowest precedence layer). - - ``base_design.parameters`` plus the workspace's ``parameter_patch`` form - the base layer beneath project ecc.toml and --set overrides. - """ - parameters = dict(manifest.base_design.get("parameters") or {}) - if workspace is not None: - for key, change in workspace.parameter_patch.items(): - parameters[key] = ( - change["to"] if isinstance(change, dict) and "to" in change else change - ) - if manifest.design_name and not _optional_str(parameters.get("design")): - parameters["design"] = manifest.design_name - rtl_list = manifest.base_design.get("rtl_list") - if not isinstance(rtl_list, list): - rtl_list = [] - return { - "pdk": _optional_str(manifest.base_design.get("pdk")), - "pdk_root": _optional_str(manifest.base_design.get("pdk_root")), - "design_name": manifest.design_name, - "top_module": _optional_str(manifest.base_design.get("top_module")), - "clock": _optional_str(manifest.base_design.get("clock")), - "rtl_list": [item for item in rtl_list if isinstance(item, str)], - "origin_verilog": _optional_str(manifest.base_design.get("origin_verilog")), - "origin_def": _optional_str(manifest.base_design.get("origin_def")), - "netlist": _optional_str(manifest.base_design.get("netlist")), - "golden_netlist": _optional_str(manifest.base_design.get("golden_netlist")), - "sdc": _optional_str(manifest.base_design.get("sdc")), - "spef": _optional_str(manifest.base_design.get("spef")), - "parameters": parameters, - } - - -def resolved_base_parameters(cfg) -> dict: - """The ecc.toml-resolved base_design.parameters for a generated manifest. - - GUI-flat vocabulary: identity fields plus the [params] overrides, - projected through the geometry converter so positional values surface - as the wizard's aliases (die_width, utilitization, margin, ...). - --set values are run-scoped and never included. - """ - canonical: dict = { - "design": cfg.design_name, - "top_module": cfg.design_top, - "clock": cfg.design_clock_port, - "frequency_max": cfg.design_frequency_mhz, - } - if cfg.params_overrides: - from chipcompiler.cli.project.params import ( - build_backend_overrides, - resolve_parameters, - ) - - resolved, _ = resolve_parameters(toml_overrides=cfg.params_overrides) - canonical.update(build_backend_overrides(resolved)) - - from chipcompiler.data.parameter_keys import parameters_to_geometry - - flat = parameters_to_geometry(canonical) - # Exclusive GUI-flat shape: geometry lives only in the aliases — - # the canonical die/core subtrees are consumed, not duplicated. - # Non-positional members (e.g. aspect_ratio) hoist to flat top-level - # keys; positional members are covered by the aliases. - for subtree_name in ("die", "core"): - subtree = flat.pop(subtree_name, None) - if not isinstance(subtree, dict): - continue - for member, value in subtree.items(): - if member in ("size", "utilitization", "margin"): - continue - flat.setdefault(member, value) - return flat - - -def base_design_from_config(cfg, pdk_root: str) -> dict: - """The base_design document for a generated manifest. - - Identity and sources come from the ecc.toml-resolved config with the - DECLARED project source spellings preserved: ``rtl_list`` verbatim, - and ``origin_verilog`` when the single source is plain RTL (empty for - a filelist source; the document builder drops empty keys). Parameters - are the GUI-flat projection. Shared by virgin generation and first - migration so the two writers cannot drift. - """ - from chipcompiler.cli.project.config import resolve_rtl - - _, origin_verilog, _ = resolve_rtl(cfg) - return { - "pdk": cfg.pdk_name, - "pdk_root": pdk_root, - "top_module": cfg.design_top, - "clock": cfg.design_clock_port, - "rtl_list": cfg.design_rtl, - "origin_verilog": cfg.design_rtl[0] if origin_verilog else "", - "origin_def": cfg.design_def, - "netlist": cfg.design_netlist, - "golden_netlist": cfg.design_golden_netlist, - "sdc": cfg.design_sdc, - "spef": cfg.design_spef, - "parameters": resolved_base_parameters(cfg), - } - - -def _slugify(value: str) -> str: - slug = re.sub(r"[^a-z0-9]+", "_", value.strip().lower()).strip("_") - return slug or "project" +sys.modules[__name__] = _domain_manifest diff --git a/chipcompiler/cli/project/manifest_write.py b/chipcompiler/cli/project/manifest_write.py index c1d422dbf..61eb0ca32 100644 --- a/chipcompiler/cli/project/manifest_write.py +++ b/chipcompiler/cli/project/manifest_write.py @@ -1,372 +1,7 @@ -#!/usr/bin/env python +"""CLI compatibility alias for Project Manifest mutations.""" -"""Manifest write, mutation, and registration operations for the CLI. +import sys -All ``project.json`` writes go through one read-modify-write helper so -status write-back and migration entry-append share the same atomicity -story. Loading and normalization live in -chipcompiler.cli.project.manifest; this module imports from it, never -the reverse. +from chipcompiler.project import manifest_write as _domain_manifest_write -Like manifest.py, this module sits on the CLI startup path (imported by -run dispatch and migration flows): keep module-level imports cheap — no -chipcompiler.data imports here. -""" - -import json -import logging -import os -import tempfile -from copy import deepcopy -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -from chipcompiler.cli.project.manifest import ( - _CANONICAL_TO_MANIFEST_STEP, - MANIFEST_FILENAME, - PRESET_MANIFEST_RANGE, - ManifestError, - _record, - _slugify, - base_design_from_config, -) - -logger = logging.getLogger(__name__) - -DEFAULT_OBJECTIVES = { - "primary": "timing", - "directions": { - "wns": "maximize", - "tns": "maximize", - "area": "minimize", - "drc_count": "minimize", - "lvs_count": "minimize", - "power": "minimize", - }, -} - - -def _now_iso() -> str: - return datetime.now(UTC).isoformat() - - -def manifest_workspace_entry( - workspace_id: str, - *, - name: str, - workspace_path: str, - start_step: str, - end_step: str, - status: str, - now: str, -) -> dict: - """One complete schema-v1 workspaces[] entry, every field materialized. - - The single builder for generated manifests and migration previews, so - the previewed entry and the applied entry are the same object shape. - """ - return { - "workspace_id": workspace_id, - "name": name, - "workspace_path": workspace_path, - "source_workspace_id": None, - "branch_from": None, - "start_step": start_step, - "end_step": end_step, - "status": status, - "created_at": now, - "updated_at": now, - "parameter_patch": {}, - "metrics_summary": {}, - "step_metrics": {}, - } - - -def build_manifest_document( - project_dir: str, - *, - design_name: str, - base_design: dict, - workspace_id: str, - workspace_path: str, - start_step: str, - end_step: str, - status: str = "running", -) -> dict: - """Assemble a schema-v1 manifest for a virgin project's first run.""" - now = _now_iso() - name = os.path.basename(os.path.normpath(project_dir)) or "project" - document: dict[str, Any] = { - "schema_version": 1, - "project_id": f"proj_{_slugify(name)}", - "name": name, - "design_name": design_name, - "description": "", - "root_path": project_dir, - "created_at": now, - "updated_at": now, - "base_design": { - **{key: value for key, value in base_design.items() if key != "parameters" and value}, - "parameters": _record(base_design.get("parameters")), - "rtl_list": [ - item for item in base_design.get("rtl_list") or [] if isinstance(item, str) - ], - }, - "objectives": json.loads(json.dumps(DEFAULT_OBJECTIVES)), - "workspaces": [ - manifest_workspace_entry( - workspace_id, - name=design_name, - workspace_path=workspace_path, - start_step=start_step, - end_step=end_step, - status=status, - now=now, - ) - ], - "mpc": None, - "best_workspace": None, - "qor_baseline": {"workspace_id": workspace_id, "reason": "Default project QoR baseline"}, - } - return document - - -def write_manifest_if_absent(project_dir: str, document: dict) -> bool: - """Write the manifest only when it does not exist (virgin generation race). - - Fully written and fsynced at a temp path, then linked into place: - readers never see a partial file, and a concurrent creator wins the - link — ours is discarded and the caller continues read-only. - """ - path = os.path.join(project_dir, MANIFEST_FILENAME) - content = json.dumps(document, indent=2) + "\n" - tmp_path = None - try: - with tempfile.NamedTemporaryFile( - "w", - dir=project_dir, - delete=False, - prefix=f".{MANIFEST_FILENAME}.", - suffix=".tmp", - encoding="utf-8", - ) as f: - tmp_path = f.name - f.write(content) - f.flush() - os.fsync(f.fileno()) - # Mode stays the tempfile default (0600), matching json_write's - # convention for newly created state files. - os.link(tmp_path, path) - return True - except FileExistsError: - return False - except OSError as exc: - logger.warning("manifest write failed: %s: %s", path, exc) - return False - finally: - if tmp_path is not None: - Path(tmp_path).unlink(missing_ok=True) - - -def _read_manifest_document(path: str): - try: - with open(path, encoding="utf-8") as f: - document = json.load(f) - except (OSError, json.JSONDecodeError, UnicodeDecodeError): - return None - return document if isinstance(document, dict) else None - - -def update_manifest(project_dir: str, mutator) -> bool: - """Read-modify-write the manifest atomically (locked re-read + patch + replace). - - The whole read-modify-replace runs under ``.manifest.lock`` (flock): - two cooperating writers can no longer both complete the fresh read - before either replaces, so neither loses the other's update. The - mutator receives the parsed document and edits it in place. When an - unrelated change lands between the read and the write, the mutator is - re-applied to the freshest document instead of overwriting the change. - Project-level fields (including updated_at) are owned by the mutator. - Returns False (with a warning) when the manifest is missing, unreadable, - the lock cannot be taken, or the write fails — callers degrade to a - warning, never a run failure. - """ - from chipcompiler.cli.project.migrate_fs import flock_file - - path = os.path.join(project_dir, MANIFEST_FILENAME) - try: - with flock_file(os.path.join(project_dir, ".manifest.lock"), exclusive=True): - return _update_manifest_locked(path, mutator) - except OSError as exc: - # An untakeable lock (e.g. a directory at the lock path) degrades - # like any write failure: a warning, never an uncaught exception — - # the migration registration path relies on False to roll back. - logger.warning("manifest update failed: %s: %s", path, exc) - return False - - -def _update_manifest_locked(path: str, mutator) -> bool: - base = _read_manifest_document(path) - if base is None: - logger.warning("manifest update skipped (unreadable): %s", path) - return False - - document = deepcopy(base) - mutator(document) - - fresh = _read_manifest_document(path) - if fresh is not None and fresh != base: - # An unrelated edit landed after our read: re-apply the mutator to - # the freshest document so the interleaved change survives. - document = fresh - mutator(document) - - target = Path(path) - tmp_path = None - try: - with tempfile.NamedTemporaryFile( - "w", - dir=target.parent, - delete=False, - prefix=f".{target.name}.", - suffix=".tmp", - encoding="utf-8", - ) as f: - tmp_path = Path(f.name) - json.dump(document, f, indent=2) - f.write("\n") - f.flush() - os.fsync(f.fileno()) - # Preserve the existing manifest's permissions: mkstemp's 0600 must - # not silently narrow a shared project.json. - if target.exists(): - os.chmod(tmp_path, target.stat().st_mode & 0o7777) - os.replace(tmp_path, target) - return True - except OSError as exc: - logger.warning("manifest update failed: %s: %s", path, exc) - if tmp_path is not None: - tmp_path.unlink(missing_ok=True) - return False - - -def write_back_workspace_status(project_dir: str, workspace_id: str, status: str) -> bool: - """Update one workspace entry's status (and updated_at) after a run.""" - - def mutate(document: dict) -> None: - for entry in document.get("workspaces", []): - if isinstance(entry, dict) and entry.get("workspace_id") == workspace_id: - entry["status"] = status - entry["updated_at"] = _now_iso() - - return update_manifest(project_dir, mutate) - - -def remove_workspace_registration(project_dir: str, workspace_id: str) -> bool: - """Roll back a pre-registration: drop the freshly added entry. - - Used when an overwrite run against an undeclared workspace fails before - the replacement is constructed: the restored previous workspace must not - be shadowed by a stale ``not_started`` entry this invocation created. - """ - - def mutate(document: dict) -> None: - workspaces = document.get("workspaces") - if isinstance(workspaces, list): - document["workspaces"] = [ - entry - for entry in workspaces - if not (isinstance(entry, dict) and entry.get("workspace_id") == workspace_id) - ] - document["updated_at"] = _now_iso() - - return update_manifest(project_dir, mutate) - - -def manifest_range_for_flow(cfg, flow_config: dict | None) -> tuple[str, str]: - """Return the GUI manifest range for a workspace's effective target.""" - if isinstance(flow_config, dict) and flow_config.get("start_step"): - from chipcompiler.rtl2gds import normalize_flow_step - - start = normalize_flow_step(flow_config["start_step"]) - end = normalize_flow_step(flow_config.get("end_step") or start) - try: - return (_CANONICAL_TO_MANIFEST_STEP[start], _CANONICAL_TO_MANIFEST_STEP[end]) - except KeyError as exc: - raise ManifestError(f"unknown workspace flow step: {exc.args[0]}") from None - return PRESET_MANIFEST_RANGE.get(cfg.flow_preset, ("Synth", "Harden")) - - -def pre_register_workspace( - project_dir: str, - *, - cfg, - pdk_root: str, - workspace_id: str, - workspace_path: str, - flow_config: dict | None, -) -> str: - """Atomically register a fresh managed workspace before filesystem creation. - - Returns ``registered``, ``existing``, ``conflict``, or ``failed``. A - workspace entry intentionally contains no input snapshot: copied files and - the workspace config are the reproducibility boundary. - """ - try: - start_step, end_step = manifest_range_for_flow(cfg, flow_config) - except ManifestError: - return "failed" - now = _now_iso() - manifest_path = os.path.join(project_dir, MANIFEST_FILENAME) - if not os.path.lexists(manifest_path): - document = build_manifest_document( - project_dir, - design_name=cfg.design_name, - base_design=base_design_from_config(cfg, pdk_root), - workspace_id=workspace_id, - workspace_path=workspace_path, - start_step=start_step, - end_step=end_step, - status="not_started", - ) - if write_manifest_if_absent(project_dir, document): - return "registered" - # A concurrent creator won the link race: fall through and apply the - # same registration mutation under the manifest lock instead of - # aborting this run. A genuine I/O failure fails again below. - - outcome = "registered" - - def mutate(document: dict) -> None: - nonlocal outcome - workspaces = document.get("workspaces") - if not isinstance(workspaces, list): - outcome = "failed" - return - for entry in workspaces: - if not isinstance(entry, dict) or entry.get("workspace_id") != workspace_id: - continue - if os.path.realpath(str(entry.get("workspace_path", ""))) == os.path.realpath( - workspace_path - ): - outcome = "existing" - else: - outcome = "conflict" - return - workspaces.append( - manifest_workspace_entry( - workspace_id, - name=cfg.design_name, - workspace_path=workspace_path, - start_step=start_step, - end_step=end_step, - status="not_started", - now=now, - ) - ) - document["updated_at"] = now - - if not update_manifest(project_dir, mutate): - return "failed" - return outcome +sys.modules[__name__] = _domain_manifest_write diff --git a/chipcompiler/cli/project/params.py b/chipcompiler/cli/project/params.py index 5521a3933..6241f7d70 100644 --- a/chipcompiler/cli/project/params.py +++ b/chipcompiler/cli/project/params.py @@ -1,683 +1,27 @@ -import copy -import json -from dataclasses import dataclass +"""CLI compatibility facade for the canonical data parameter catalog.""" -from chipcompiler.data.config_params import CONFIG_PARAM_SCHEMAS -from chipcompiler.data.config_params.common import ParamSchema -from chipcompiler.data.config_params.macro import SCHEMAS as MACRO_SCHEMAS +from dataclasses import replace -_LEGACY_PARAM_REGISTRY: tuple[ParamSchema, ...] = ( - ParamSchema( - param="design.frequency_mhz", - group="design", - name="frequency_mhz", - type="float", - default=100.0, - applies="synthesis", - maps_to="frequency_max", - description="Target clock frequency in MHz", - range=(1e-6, 10000.0), - unit="MHz", - example="200.0", - ), - ParamSchema( - param="floorplan.core_util", - group="floorplan", - name="core_util", - type="float", - default=0.4, - applies="floorplan", - maps_to={"core": "utilitization"}, - description="Core utilization ratio", - range=(0.01, 1.0), - example="0.45", - ), - ParamSchema( - param="floorplan.core_margin", - group="floorplan", - name="core_margin", - type="list[int]", - default=[2, 2], - applies="floorplan", - maps_to={"core": "margin"}, - description="Core margin in micrometers [horizontal, vertical]", - example="[2, 2]", - ), - ParamSchema( - param="floorplan.aspect_ratio", - group="floorplan", - name="aspect_ratio", - type="float", - default=1.0, - applies="floorplan", - maps_to={"core": "aspect_ratio"}, - description="Core aspect ratio (width/height)", - range=(0.1, 10.0), - example="1.0", - ), - ParamSchema( - param="cts.max_fanout", - group="cts", - name="max_fanout", - type="int", - default=20, - applies="cts", - maps_to="max_fanout", - description="Maximum fanout for clock tree synthesis", - range=(1, 200), - example="16", - ), - ParamSchema( - param="place.target_density", - group="place", - name="target_density", - type="float", - default=0.2, - applies="placement", - maps_to={"dreamplace": "target_density"}, - description="Target placement density", - range=(0.1, 0.95), - example="0.65", - ), - ParamSchema( - param="place.target_overflow", - group="place", - name="target_overflow", - type="float", - default=0.1, - applies="placement", - maps_to={"dreamplace": "stop_overflow"}, - description="Target overflow for global placement", - range=(0.0, 1.0), - example="0.08", - ), - ParamSchema( - param="place.global_right_padding", - group="place", - name="global_right_padding", - type="int", - default=0, - applies="placement", - maps_to="global_right_padding", - description="Global right padding for placement sites", - range=(0, 100), - example="8", - ), - ParamSchema( - param="place.cell_padding_x", - group="place", - name="cell_padding_x", - type="int", - default=300, - applies="placement", - maps_to={"dreamplace": "cell_padding_x"}, - description="Cell padding in x-direction in database units", - range=(0, 10000), - example="400", - ), - ParamSchema( - param="place.routability_opt", - group="place", - name="routability_opt", - type="int", - default=1, - applies="placement", - maps_to={"dreamplace": "routability_opt_flag"}, - description="Enable routability-driven placement optimization", - choices=("0", "1"), - example="1", - ), - ParamSchema( - param="route.bottom_layer", - group="route", - name="bottom_layer", - type="str", - default="MET2", - applies="routing", - maps_to="bottom_layer", - description="Bottom routing layer", - choices=("MET1", "MET2", "MET3", "MET4", "MET5"), - example="MET2", - ), - ParamSchema( - param="route.top_layer", - group="route", - name="top_layer", - type="str", - default="MET5", - applies="routing", - maps_to="top_layer", - description="Top routing layer", - choices=("MET2", "MET3", "MET4", "MET5", "MET6"), - example="MET5", - ), - ParamSchema( - param="sta.max_paths", - group="sta", - name="max_paths", - type="int", - default=1000, - applies="sta", - maps_to="sta_max_paths", - description="Maximum number of paths in each STA timing report", - range=(1, 100000), - example="1000", - ), - ParamSchema( - param="flow.run_analysis", - group="flow", - name="run_analysis", - type="bool", - default=True, - applies="all", - maps_to="run_analysis", - description="Run per-step analysis (metrics, plots, checklist) after each step", - example="false", - ), -) +from chipcompiler.data.parameter_schema import * # noqa: F401,F403 +from chipcompiler.data.parameter_schema import validate_schema_type -PARAM_REGISTRY = _LEGACY_PARAM_REGISTRY + CONFIG_PARAM_SCHEMAS + MACRO_SCHEMAS +_validate_schema_type = validate_schema_type -_REGISTRY_INDEX: dict[str, ParamSchema] = {s.param: s for s in PARAM_REGISTRY} -_REQUIRED_FIELDS = ( - "param", - "group", - "name", - "type", - "default", - "applies", - "description", -) - -def lookup_schema(key: str) -> ParamSchema | None: - return _REGISTRY_INDEX.get(key) - - -def list_schemas() -> tuple[ParamSchema, ...]: - return PARAM_REGISTRY - - -def list_groups() -> list[str]: - seen: list[str] = [] - for s in PARAM_REGISTRY: - if s.group not in seen: - seen.append(s.group) - return seen - - -def is_known_key(key: str) -> bool: - return key in _REGISTRY_INDEX - - -def validate_schema_record(schema: ParamSchema) -> list[str]: - return [ - f"missing required field: {f}" - for f in _REQUIRED_FIELDS - if getattr(schema, f, None) is None or (f != "default" and getattr(schema, f) == "") - ] - - -# --------------------------------------------------------------------------- -# Value parsing -# --------------------------------------------------------------------------- - - -def parse_value(raw: str, schema: ParamSchema) -> object: - ptype = schema.type - - if ptype == "int": - try: - return int(raw) - except ValueError as exc: - raise ValueError(f"expected int for {schema.param}, got '{raw}'") from exc - - if ptype == "float": - try: - return float(raw) - except ValueError as exc: - raise ValueError(f"expected float for {schema.param}, got '{raw}'") from exc - - if ptype == "bool": - low = raw.lower() - if low in ("true", "1", "yes"): - return True - if low in ("false", "0", "no"): - return False - raise ValueError(f"expected bool for {schema.param}, got '{raw}'") - - if ptype == "str": - return raw - - if ptype in ("list[int]", "list[float]", "list[str]"): - try: - parsed = json.loads(raw) - except json.JSONDecodeError: - parsed = None - if parsed is not None: - value, error = _validate_schema_type(parsed, schema) - if error is None: - return value - raise ValueError(error) - - stripped = raw.strip("[]() ") - if not stripped: - return [] - parts = [p.strip() for p in stripped.split(",")] - if ptype == "list[int]": - try: - return [int(p) for p in parts if p] - except ValueError as exc: - raise ValueError(f"expected list[int] for {schema.param}, got '{raw}'") from exc - if ptype == "list[float]": - try: - return [float(p) for p in parts if p] - except ValueError as exc: - raise ValueError(f"expected list[float] for {schema.param}, got '{raw}'") from exc - return [p for p in parts if p] - - if ptype == "json": - try: - return json.loads(raw) - except json.JSONDecodeError as exc: - raise ValueError(f"expected JSON for {schema.param}, got '{raw}'") from exc - - raise ValueError(f"unsupported type '{ptype}' for {schema.param}") - - -def validate_value(value: object, schema: ParamSchema) -> list[str]: - errors: list[str] = [] - - if schema.range is not None: - lo, hi = schema.range - if not isinstance(value, (int, float)): - # Raw layers (e.g. project.json parameters) reach this without - # the typed parsing --set/[params] get: a string must fail loud - # instead of silently skipping the range check. - errors.append(f"value {value!r} is not numeric for {schema.param}") - elif value < lo or value > hi: - errors.append(f"value {value} out of range [{lo}, {hi}] for {schema.param}") - - if schema.choices is not None: - str_val = str(value) - if str_val not in schema.choices: - errors.append( - f"value '{str_val}' not in allowed choices {schema.choices} for {schema.param}" - ) - - return errors - - -# --------------------------------------------------------------------------- -# Source-aware resolution -# --------------------------------------------------------------------------- - - -@dataclass -class ResolvedParam: - param: str - value: object - default: object - source: str - schema: ParamSchema - - @property - def is_explicit(self) -> bool: - return self.source != "default" - - -def _validate_schema_type(value: object, schema: ParamSchema) -> tuple[object, str | None]: - ptype = schema.type - key = schema.param - - if ptype == "int": - if isinstance(value, bool) or not isinstance(value, int): - return value, f"expected int for {key}, got {type(value).__name__}" - return value, None - - if ptype == "float": - if isinstance(value, bool): - return value, f"expected float for {key}, got bool" - if isinstance(value, (int, float)): - try: - return float(value), None - except OverflowError: - # A huge JSON integer cannot become a float; that is invalid - # input, not a crash. - return value, f"expected float for {key}, value too large to represent" - return value, f"expected float for {key}, got {type(value).__name__}" - - if ptype == "bool": - if isinstance(value, bool): - return value, None - if isinstance(value, str): - low = value.lower() - if low in ("true", "1", "yes"): - return True, None - if low in ("false", "0", "no"): - return False, None - return value, f"expected bool for {key}, got {type(value).__name__}" - - if ptype == "str": - if isinstance(value, str): - return value, None - return value, f"expected str for {key}, got {type(value).__name__}" - - if ptype == "list[int]": - if not isinstance(value, list): - return value, f"expected list for {key}, got {type(value).__name__}" - for i, v in enumerate(value): - if isinstance(v, bool) or not isinstance(v, int): - return value, f"expected list[int] for {key}, element {i} is {type(v).__name__}" - return value, None - - if ptype == "list[float]": - if not isinstance(value, list): - return value, f"expected list for {key}, got {type(value).__name__}" - for i, v in enumerate(value): - if isinstance(v, bool): - return value, f"expected list[float] for {key}, element {i} is bool" - if not isinstance(v, (int, float)): - return value, f"expected list[float] for {key}, element {i} is {type(v).__name__}" - try: - return [float(v) for v in value], None - except OverflowError: - return value, f"expected list[float] for {key}, value too large to represent" - - if ptype == "list[str]": - if not isinstance(value, list): - return value, f"expected list for {key}, got {type(value).__name__}" - for i, v in enumerate(value): - if not isinstance(v, str): - return value, f"expected list[str] for {key}, element {i} is {type(v).__name__}" - return value, None - - if ptype == "json": - if isinstance(value, (dict, list)): - return value, None - return value, f"expected JSON object or array for {key}, got {type(value).__name__}" - - return value, None - - -def resolve_parameters( - toml_overrides: dict[str, object] | None = None, - cli_overrides: dict[str, object] | None = None, - manifest_overrides: dict[str, object] | None = None, -) -> tuple[list[ResolvedParam], list[str]]: - toml_overrides = toml_overrides or {} - cli_overrides = cli_overrides or {} - manifest_overrides = manifest_overrides or {} - resolved: list[ResolvedParam] = [] - errors: list[str] = [] - - for schema in PARAM_REGISTRY: - key = schema.param - if key in cli_overrides: - value = cli_overrides[key] - val_errors = validate_value(value, schema) - if val_errors: - errors.extend(val_errors) - resolved.append( - ResolvedParam( - param=key, - value=value, - default=schema.default, - source="cli", - schema=schema, - ) - ) - elif key in toml_overrides: - value = toml_overrides[key] - value, coerce_err = _validate_schema_type(value, schema) - if coerce_err: - errors.append(coerce_err) - val_errors = validate_value(value, schema) - if val_errors: - errors.extend(val_errors) - resolved.append( - ResolvedParam( - param=key, - value=value, - default=schema.default, - source="ecc.toml", - schema=schema, - ) - ) - elif key in manifest_overrides: - value = manifest_overrides[key] - value, coerce_err = _validate_schema_type(value, schema) - if coerce_err: - errors.append(coerce_err) - val_errors = validate_value(value, schema) - if val_errors: - errors.extend(val_errors) - resolved.append( - ResolvedParam( - param=key, - value=value, - default=schema.default, - source="project.json", - schema=schema, - ) - ) - else: - resolved.append( - ResolvedParam( - param=key, - value=schema.default, - default=schema.default, - source="default", - schema=schema, - ) - ) - - return resolved, errors - - -# --------------------------------------------------------------------------- -# Semantic-to-backend mapping -# --------------------------------------------------------------------------- - - -def manifest_value_for(canonical: dict, maps_to: str | dict | None) -> tuple[object, bool]: - """Resolve a manifest-layer value via its schema target (presence-keyed). - - A string ``maps_to`` is a top-level canonical key; a dict ``maps_to`` is a - single-level (subtree, leaf) path. Returns (value, present): an explicit - JSON ``null`` counts as present, an absent key does not. - """ - if isinstance(maps_to, str): - if maps_to in canonical: - return canonical[maps_to], True - return None, False - if maps_to is None: - return None, False - for subtree, leaf in maps_to.items(): - node = canonical.get(subtree) - if isinstance(node, dict) and leaf in node: - return node[leaf], True - return None, False - - -def coerce_manifest_parameters( - canonical: dict, - registry: tuple[ParamSchema, ...] = PARAM_REGISTRY, - skip_params: frozenset | set = frozenset(), -) -> tuple[dict, list[str]]: - """Coerce manifest-layer parameter values to their schema types. - - Type check/coercion only — range/choice rules stay check-path concerns. - Works on a deep copy: coerced values are written back, uncoercible values - are left as-is (the caller decides what an error means), and keys outside - the registry pass through untouched (forward-compatible). - """ - coerced = copy.deepcopy(canonical) - errors: list[str] = [] - for schema in registry: - if schema.param in skip_params: - continue - value, present = manifest_value_for(coerced, schema.maps_to) - if not present: - continue - new_value, err = _validate_schema_type(value, schema) - if err: - errors.append(err) - continue - maps_to = schema.maps_to - if isinstance(maps_to, str): - coerced[maps_to] = new_value - elif isinstance(maps_to, dict): - for subtree, leaf in maps_to.items(): - node = coerced.get(subtree) - if isinstance(node, dict) and leaf in node: - node[leaf] = new_value - break - return coerced, errors - - -def build_backend_overrides(resolved: list[ResolvedParam]) -> dict: - overrides: dict = {} - for rp in resolved: - if rp.value == rp.default and rp.source == "default": - continue - maps_to = rp.schema.maps_to - value = rp.value - if isinstance(maps_to, str): - overrides[maps_to] = value - elif isinstance(maps_to, dict): - for parent_key, child_key in maps_to.items(): - if parent_key not in overrides: - overrides[parent_key] = {} - overrides[parent_key][child_key] = value - return overrides - - -def build_config_overrides(resolved: list[ResolvedParam]) -> dict[str, object]: - overrides: dict[str, object] = {} - for rp in resolved: - target = rp.schema.config_target - if target is None or not rp.is_explicit: - continue - config_overrides = overrides.setdefault(target.config_key, {}) - _set_nested_value(config_overrides, target.json_path, rp.value) - return overrides - - -def build_pdk_overrides(resolved: list[ResolvedParam]) -> dict[str, object]: - return { - rp.schema.pdk_target: rp.value - for rp in resolved - if rp.schema.pdk_target is not None and rp.is_explicit - } - - -def _set_nested_value(data: dict, path: tuple[str, ...], value: object) -> None: - current = data - for key in path[:-1]: - child = current.get(key) - if not isinstance(child, dict): - child = {} - current[key] = child - current = child - current[path[-1]] = value - - -def parse_cli_overrides(pairs: list[str]) -> tuple[dict[str, object], list[str]]: - result: dict[str, object] = {} - errors: list[str] = [] - - for pair in pairs: - if "=" not in pair: - errors.append(f"malformed override: '{pair}' (expected key=value)") - continue - - key, _, raw_value = pair.partition("=") - key = key.strip() - raw_value = raw_value.strip() - - schema = lookup_schema(key) - if schema is None: - errors.append(f"unknown parameter: '{key}'") - continue - - try: - value = parse_value(raw_value, schema) - except ValueError as exc: - errors.append(str(exc)) - continue - - val_errors = validate_value(value, schema) - if val_errors: - errors.extend(val_errors) - continue - - result[key] = value - - return result, errors - - -def validate_pdk_target(schema: ParamSchema, value: object, cfg) -> str | None: - """Validate a PDK-target value using the existing PDK resolver and checks.""" +def validate_pdk_target(schema, value, cfg) -> str | None: if schema.pdk_target is None: return None - - from dataclasses import replace - from chipcompiler.cli.project.config import ( _validate_pdk_contents, resolve_pdk_overrides, resolve_pdk_root, ) - raw_overrides = dict(cfg.pdk_overrides) - raw_overrides[schema.pdk_target] = value - candidate = replace(cfg, pdk_overrides=raw_overrides) + overrides = dict(cfg.pdk_overrides) + overrides[schema.pdk_target] = value + candidate = replace(cfg, pdk_overrides=overrides) return _validate_pdk_contents( candidate.pdk_name, resolve_pdk_root(candidate), resolve_pdk_overrides(candidate), ) - - -def parse_toml_params(params_table: dict) -> tuple[dict[str, object], list[str]]: - flat: dict[str, object] = {} - errors: list[str] = [] - - def visit(path: tuple[str, ...], value: object) -> None: - param_key = ".".join(path) - schema = lookup_schema(param_key) - if schema is not None: - try: - if isinstance(value, str): - parsed = parse_value(value, schema) - else: - parsed, type_err = _validate_schema_type(value, schema) - if type_err: - errors.append(type_err) - return - except ValueError as exc: - errors.append(str(exc)) - return - - val_errors = validate_value(parsed, schema) - if val_errors: - errors.extend(val_errors) - return - flat[param_key] = parsed - return - - if isinstance(value, dict): - for key, child in value.items(): - visit((*path, key), child) - return - - errors.append(f"unknown parameter in ecc.toml: '{param_key}'") - - for group_key, group_val in params_table.items(): - if not isinstance(group_val, dict): - errors.append(f"[params.{group_key}] must be a table, got {type(group_val).__name__}") - continue - visit((group_key,), group_val) - - return flat, errors diff --git a/chipcompiler/cli/project/run_existing.py b/chipcompiler/cli/project/run_existing.py index afeda0b3f..43cdc3704 100644 --- a/chipcompiler/cli/project/run_existing.py +++ b/chipcompiler/cli/project/run_existing.py @@ -229,7 +229,15 @@ def mismatch_error(reason: str) -> CommandResult: # The persisted ledger may be wider than the reconciled # target by design (workspace_steps is bound above), so # the full-ledger completeness check does not apply. - flow_ok = engine_flow.run_steps(require_full_ledger=False) + from chipcompiler.engine import ExecutionPlan, execute + + flow_ok = execute( + engine_flow, + ExecutionPlan( + intent="run", + step_ids=tuple(step.name for step in engine_flow.workspace_steps), + ), + ).succeeded except Exception as exc: if workspace_registered: _write_back_status(project_dir, run_name, "failed", warnings) diff --git a/chipcompiler/cli/project/run_prepare.py b/chipcompiler/cli/project/run_prepare.py index bed0caf21..32fddc160 100644 --- a/chipcompiler/cli/project/run_prepare.py +++ b/chipcompiler/cli/project/run_prepare.py @@ -353,6 +353,9 @@ def failed_workspace(reason: str | None) -> CommandResult: input_filelist = generated_filelist origin_verilog = "" parameters = to_parameters(cfg) + if uses_netlist_input: + # Preserve the creator's input role for Studio/CLI reopen symmetry. + parameters["_input_mode"] = "postSynthesis" pdk_root = resolve_pdk_root(cfg) if base is not None: @@ -473,6 +476,10 @@ def failed_workspace(reason: str | None) -> CommandResult: # The replacement is fully constructed and verified: commit it. # The previous workspace's backup is obsolete, the new tree owns # the target, and later failures are a normal failed run. + from chipcompiler.engine.snapshot import create_engineering_snapshot + + if getattr(workspace, "directory", None): + create_engineering_snapshot(workspace) commit_replacement() if workspace_registered and execute_flow: @@ -501,7 +508,9 @@ def failed_workspace(reason: str | None) -> CommandResult: if should_enable_run_progress(ctx, sys.stderr): flow_ok = run_flow_with_progress(engine_flow, ctx, project, sys.stderr) else: - flow_ok = engine_flow.run_steps() + from chipcompiler.engine import ExecutionPlan, execute + + flow_ok = execute(engine_flow, ExecutionPlan(intent="run")).succeeded if not flow_ok: if workspace_registered: diff --git a/chipcompiler/cli/project/workspace_params.py b/chipcompiler/cli/project/workspace_params.py index cc56cfa07..da1e397fe 100644 --- a/chipcompiler/cli/project/workspace_params.py +++ b/chipcompiler/cli/project/workspace_params.py @@ -1,137 +1,3 @@ -"""Workspace-local parameter persistence and replay helpers.""" +"""CLI compatibility facade for workspace parameter persistence.""" -from copy import deepcopy - -from chipcompiler.cli.project.params import ( - ResolvedParam, - build_backend_overrides, - build_config_overrides, -) -from chipcompiler.data.parameter import update_parameters -from chipcompiler.data.workspace.config_overrides import CONFIG_OVERRIDES_KEY -from chipcompiler.utility import json_read - -WORKSPACE_PARAM_OVERRIDES_KEY = "workspace_param_overrides" - -_APPLIES_TO_STEP = { - "synthesis": "Synthesis", - "floorplan": "preFloorplan", - "placement": "place", - "macro": "macroPlacement", - "cts": "CTS", - "routing": "route", - "filler": "filler", - "rcx": "RCX", - "sta": "sta", -} -_MISSING = object() - - -def workspace_param_step(schema) -> str: - step = _APPLIES_TO_STEP.get(schema.applies) - if step is None: - raise ValueError(f"{schema.param} requires a full workspace refresh") - return step - - -def workspace_param_value(workspace, schema) -> object: - if schema.pdk_target is not None: - raise ValueError(f"{schema.param} requires a full workspace refresh") - if schema.maps_to is not None: - return _parameter_target_value(workspace.parameters.data, schema.maps_to, schema.default) - if schema.config_target is not None: - config_path = workspace.config.get(schema.config_target.config_key) - if config_path is None: - raise ValueError(f"workspace config missing target: {schema.config_target.config_key}") - value = _nested_value(json_read(config_path), schema.config_target.json_path) - return schema.default if value is _MISSING else value - raise ValueError(f"{schema.param} has no workspace configuration target") - - -def set_workspace_param(workspace, schema, value: object) -> tuple[object, str]: - records = _override_records(workspace.parameters.data) - record = next((item for item in records if item["key"] == schema.param), None) - if record is None: - record = { - "key": schema.param, - "baseline": deepcopy(workspace_param_value(workspace, schema)), - } - records.append(record) - record["value"] = deepcopy(value) - _set_override_records(workspace.parameters.data, records) - _apply_workspace_value(workspace, schema, value) - return record["baseline"], workspace_param_step(schema) - - -def unset_workspace_param(workspace, schema) -> tuple[object, str] | None: - records = _override_records(workspace.parameters.data) - record = next((item for item in records if item["key"] == schema.param), None) - if record is None: - return None - records.remove(record) - _set_override_records(workspace.parameters.data, records) - _apply_workspace_value(workspace, schema, record["baseline"]) - return record["baseline"], workspace_param_step(schema) - - -def workspace_param_diff(workspace) -> list[dict]: - return _override_records(workspace.parameters.data) - - -def _apply_workspace_value(workspace, schema, value: object) -> None: - resolved = ResolvedParam( - param=schema.param, - value=value, - default=schema.default, - source="workspace", - schema=schema, - ) - update_parameters(build_backend_overrides([resolved]), workspace.parameters.data) - config_overrides = build_config_overrides([resolved]) - if config_overrides: - update_parameters({CONFIG_OVERRIDES_KEY: config_overrides}, workspace.parameters.data) - - -def _override_records(parameters: dict) -> list[dict]: - raw = parameters.get(WORKSPACE_PARAM_OVERRIDES_KEY, []) - if not isinstance(raw, list): - return [] - records = [] - for item in raw: - if not isinstance(item, dict): - continue - key = item.get("key") - if not isinstance(key, str) or "baseline" not in item: - continue - records.append(deepcopy(item)) - return records - - -def _set_override_records(parameters: dict, records: list[dict]) -> None: - if records: - parameters[WORKSPACE_PARAM_OVERRIDES_KEY] = records - else: - parameters.pop(WORKSPACE_PARAM_OVERRIDES_KEY, None) - - -def _parameter_target_value(parameters: dict, target, default: object) -> object: - if isinstance(target, str): - return deepcopy(parameters.get(target, default)) - if isinstance(target, dict) and len(target) == 1: - parent, child = next(iter(target.items())) - nested = parameters.get(parent) - if isinstance(nested, dict): - return deepcopy(nested.get(child, default)) - return deepcopy(default) - - -def _nested_value(data: object, path: tuple[str, ...]) -> object: - current = data - for key in path: - if not isinstance(current, dict): - return _MISSING - payload: dict = dict(current) - if key not in payload: - return _MISSING - current = payload[key] - return deepcopy(current) +from chipcompiler.data.workspace_parameters import * # noqa: F401,F403 diff --git a/chipcompiler/cli/rendering/progress.py b/chipcompiler/cli/rendering/progress.py index a33f9be24..7bdf6d99e 100644 --- a/chipcompiler/cli/rendering/progress.py +++ b/chipcompiler/cli/rendering/progress.py @@ -19,6 +19,11 @@ from chipcompiler.cli.rendering.pretty import BOLD, CYAN, DIM, GREEN, RED, RESET from chipcompiler.cli.rendering.pretty import style as _style from chipcompiler.data import StateEnum, log_flow +from chipcompiler.engine.execution import ( + event_sink_for_workspace, + execution_observer, + invoke_engine, +) from chipcompiler.utility.log import flush_cstdio, redirect_stdio_to_file @@ -434,6 +439,7 @@ def run_flow_with_progress(engine_flow, ctx, project, stderr): run_dir = engine_flow.workspace.directory run_name = ctx.run_id or "default" renderer.start_run(run_name, run_dir) + observer = execution_observer(event_sink_for_workspace(workspace)) for workspace_step in engine_flow.workspace_steps: step_token = normalize_step_name(workspace_step.name) @@ -476,7 +482,12 @@ def run_flow_with_progress(engine_flow, ctx, project, stderr): finally: if init_log_stream is not None: init_log_stream.close() - state = engine_flow.run_step(workspace_step) + state = invoke_engine( + engine_flow.run_step, + workspace_step, + rerun=False, + observer=observer, + ) finally: _stop_log_monitor(stop_event, monitor) renderer.clear() diff --git a/chipcompiler/data/__init__.py b/chipcompiler/data/__init__.py index 11b3e9b10..638b2e296 100644 --- a/chipcompiler/data/__init__.py +++ b/chipcompiler/data/__init__.py @@ -18,6 +18,7 @@ is_finished_step_state, load_metrics, save_metrics, + step_storage_name, ) from .workspace import ( OriginDesign, @@ -70,6 +71,10 @@ YosysReport, YosysStep, ) +from .workspace_transaction import ( + WorkspaceFileTransaction, + recover_workspace_file_transaction, +) __all__ = [ "create_workspace", @@ -109,6 +114,8 @@ "build_workspace_config_paths", "workspace_config_paths", "workspace_config_path", + "WorkspaceFileTransaction", + "recover_workspace_file_transaction", "step_config_keys", "step_config_paths", "init_workspace_config", @@ -132,6 +139,7 @@ "get_design_parameters", "get_pdk", "StepEnum", + "step_storage_name", "StateEnum", "CheckState", "StepMetrics", diff --git a/chipcompiler/data/checklist.py b/chipcompiler/data/checklist.py index 2356a4479..0f7b80d28 100644 --- a/chipcompiler/data/checklist.py +++ b/chipcompiler/data/checklist.py @@ -46,6 +46,18 @@ def __init__(self, path: Path | str): self.path = Path(path) self.data = self._load_current_data() + @classmethod + def from_items(cls, path: Path | str, items) -> dict: + """Build normalized checklist data without reading or writing a file.""" + checklist = cls.__new__(cls) + checklist.path = Path(path) + checklist.data = checklist._default_data() + checklist.data["checklist"] = [ + cls._normalize_item(item) for item in items if isinstance(item, dict) + ] + cls._refresh_summary(checklist.data) + return checklist.data + @staticmethod def _timestamp() -> str: return datetime.now(UTC).isoformat().replace("+00:00", "Z") diff --git a/chipcompiler/data/parameter_keys.py b/chipcompiler/data/parameter_keys.py index 8478d4bb7..1cc2d9a37 100644 --- a/chipcompiler/data/parameter_keys.py +++ b/chipcompiler/data/parameter_keys.py @@ -6,7 +6,7 @@ ``top_module``, ``die``/``core`` subtrees, ...). Older workspaces persisted display-oriented keys (``"Frequency max [MHz]"``, ``"Top module"``, ...) and the GUI sends flat keys plus a handful of positional geometry aliases at the -project/RPC boundary. This module is the only place that knows the legacy +project/adapter boundary. This module is the only place that knows the legacy vocabulary; everything else consumes the canonical form. """ diff --git a/chipcompiler/data/parameter_schema.py b/chipcompiler/data/parameter_schema.py new file mode 100644 index 000000000..8a26d3ff3 --- /dev/null +++ b/chipcompiler/data/parameter_schema.py @@ -0,0 +1,492 @@ +"""Canonical public parameter catalog and schema operations. + +The catalog is shared by the headless Engine, CLI, and Studio adapter. CLI +specific PDK path validation remains in ``chipcompiler.cli.project.params``. +""" + +import copy +import json +from dataclasses import dataclass + +from chipcompiler.data.config_params import CONFIG_PARAM_SCHEMAS +from chipcompiler.data.config_params.common import ParamSchema +from chipcompiler.data.config_params.macro import SCHEMAS as MACRO_SCHEMAS + +_LEGACY_PARAM_REGISTRY: tuple[ParamSchema, ...] = ( + ParamSchema( + "design.frequency_mhz", + "design", + "frequency_mhz", + "float", + 100.0, + "synthesis", + "Target clock frequency in MHz", + "frequency_max", + range=(1e-6, 10000.0), + unit="MHz", + example="200.0", + ), + ParamSchema( + "floorplan.core_util", + "floorplan", + "core_util", + "float", + 0.4, + "floorplan", + "Core utilization ratio", + {"core": "utilitization"}, + range=(0.01, 1.0), + example="0.45", + ), + ParamSchema( + "floorplan.core_margin", + "floorplan", + "core_margin", + "list[int]", + [2, 2], + "floorplan", + "Core margin in micrometers [horizontal, vertical]", + {"core": "margin"}, + example="[2, 2]", + ), + ParamSchema( + "floorplan.aspect_ratio", + "floorplan", + "aspect_ratio", + "float", + 1.0, + "floorplan", + "Core aspect ratio (width/height)", + {"core": "aspect_ratio"}, + range=(0.1, 10.0), + example="1.0", + ), + ParamSchema( + "cts.max_fanout", + "cts", + "max_fanout", + "int", + 20, + "cts", + "Maximum fanout for clock tree synthesis", + "max_fanout", + range=(1, 200), + example="16", + ), + ParamSchema( + "place.target_density", + "place", + "target_density", + "float", + 0.2, + "placement", + "Target placement density", + {"dreamplace": "target_density"}, + range=(0.1, 0.95), + example="0.65", + ), + ParamSchema( + "place.target_overflow", + "place", + "target_overflow", + "float", + 0.1, + "placement", + "Target overflow for global placement", + {"dreamplace": "stop_overflow"}, + range=(0.0, 1.0), + example="0.08", + ), + ParamSchema( + "place.global_right_padding", + "place", + "global_right_padding", + "int", + 0, + "placement", + "Global right padding for placement sites", + "global_right_padding", + range=(0, 100), + example="8", + ), + ParamSchema( + "place.cell_padding_x", + "place", + "cell_padding_x", + "int", + 300, + "placement", + "Cell padding in x-direction in database units", + {"dreamplace": "cell_padding_x"}, + range=(0, 10000), + example="400", + ), + ParamSchema( + "place.routability_opt", + "place", + "routability_opt", + "int", + 1, + "placement", + "Enable routability-driven placement optimization", + {"dreamplace": "routability_opt_flag"}, + choices=("0", "1"), + example="1", + ), + ParamSchema( + "route.bottom_layer", + "route", + "bottom_layer", + "str", + "MET2", + "routing", + "Bottom routing layer", + "bottom_layer", + choices=("MET1", "MET2", "MET3", "MET4", "MET5"), + example="MET2", + ), + ParamSchema( + "route.top_layer", + "route", + "top_layer", + "str", + "MET5", + "routing", + "Top routing layer", + "top_layer", + choices=("MET2", "MET3", "MET4", "MET5", "MET6"), + example="MET5", + ), + ParamSchema( + "sta.max_paths", + "sta", + "max_paths", + "int", + 1000, + "sta", + "Maximum number of paths in each STA timing report", + "sta_max_paths", + range=(1, 100000), + example="1000", + ), + ParamSchema( + param="flow.run_analysis", + group="flow", + name="run_analysis", + type="bool", + default=True, + applies="all", + description="Run per-step analysis (metrics, plots, checklist) after each step", + maps_to="run_analysis", + example="false", + ), +) + +PARAM_REGISTRY = _LEGACY_PARAM_REGISTRY + CONFIG_PARAM_SCHEMAS + MACRO_SCHEMAS +_REGISTRY_INDEX = {schema.param: schema for schema in PARAM_REGISTRY} +_REQUIRED_FIELDS = ("param", "group", "name", "type", "default", "applies", "description") + + +def lookup_schema(key: str) -> ParamSchema | None: + return _REGISTRY_INDEX.get(key) + + +def list_schemas() -> tuple[ParamSchema, ...]: + return PARAM_REGISTRY + + +def list_groups() -> list[str]: + return list(dict.fromkeys(schema.group for schema in PARAM_REGISTRY)) + + +def is_known_key(key: str) -> bool: + return key in _REGISTRY_INDEX + + +def validate_schema_record(schema: ParamSchema) -> list[str]: + return [ + f"missing required field: {field}" + for field in _REQUIRED_FIELDS + if getattr(schema, field, None) is None + or (field != "default" and getattr(schema, field) == "") + ] + + +def validate_schema_type(value: object, schema: ParamSchema) -> tuple[object, str | None]: + key, ptype = schema.param, schema.type + if ptype == "int": + return ( + (value, None) + if isinstance(value, int) and not isinstance(value, bool) + else (value, f"expected int for {key}, got {type(value).__name__}") + ) + if ptype == "float": + if isinstance(value, bool) or not isinstance(value, (int, float)): + return value, f"expected float for {key}, got {type(value).__name__}" + try: + return float(value), None + except OverflowError: + return value, f"expected float for {key}, value too large to represent" + if ptype == "bool": + if isinstance(value, bool): + return value, None + if isinstance(value, str) and value.lower() in {"true", "1", "yes", "false", "0", "no"}: + return value.lower() in {"true", "1", "yes"}, None + return value, f"expected bool for {key}, got {type(value).__name__}" + if ptype == "str": + return ( + (value, None) + if isinstance(value, str) + else (value, f"expected str for {key}, got {type(value).__name__}") + ) + if ptype.startswith("list["): + if not isinstance(value, list): + return value, f"expected list for {key}, got {type(value).__name__}" + element_type = ptype[5:-1] + for index, item in enumerate(value): + if element_type == "str": + valid = isinstance(item, str) + elif element_type == "int": + valid = isinstance(item, int) and not isinstance(item, bool) + else: + valid = isinstance(item, (int, float)) and not isinstance(item, bool) + if not valid: + return ( + value, + f"expected {ptype} for {key}, element {index} is {type(item).__name__}", + ) + if element_type == "float": + try: + return [float(item) for item in value], None + except OverflowError: + return value, f"expected list[float] for {key}, value too large to represent" + return value, None + if ptype == "json": + return ( + (value, None) + if isinstance(value, (dict, list)) + else (value, f"expected JSON object or array for {key}, got {type(value).__name__}") + ) + return value, None + + +def parse_value(raw: str, schema: ParamSchema) -> object: + if schema.type in {"int", "float"}: + try: + return int(raw) if schema.type == "int" else float(raw) + except ValueError as exc: + raise ValueError(f"expected {schema.type} for {schema.param}, got '{raw}'") from exc + if schema.type == "bool": + low = raw.lower() + if low in {"true", "1", "yes"}: + return True + if low in {"false", "0", "no"}: + return False + raise ValueError(f"expected bool for {schema.param}, got '{raw}'") + if schema.type == "str": + return raw + if schema.type.startswith("list["): + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parts = [part.strip() for part in raw.strip("[]() ").split(",") if part.strip()] + element_type = schema.type[5:-1] + try: + parsed = ( + [int(part) for part in parts] + if element_type == "int" + else [float(part) for part in parts] + if element_type == "float" + else parts + ) + except ValueError as exc: + raise ValueError(f"expected {schema.type} for {schema.param}, got '{raw}'") from exc + value, error = validate_schema_type(parsed, schema) + if error: + raise ValueError(error) + return value + if schema.type == "json": + try: + return json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"expected JSON for {schema.param}, got '{raw}'") from exc + raise ValueError(f"unsupported type '{schema.type}' for {schema.param}") + + +def validate_value(value: object, schema: ParamSchema) -> list[str]: + errors = [] + if schema.range is not None: + if not isinstance(value, (int, float)) or isinstance(value, bool): + errors.append(f"value {value!r} is not numeric for {schema.param}") + elif value < schema.range[0] or value > schema.range[1]: + errors.append( + f"value {value} out of range [{schema.range[0]}, {schema.range[1]}] " + f"for {schema.param}" + ) + if schema.choices is not None and str(value) not in schema.choices: + errors.append(f"value '{value}' not in allowed choices {schema.choices} for {schema.param}") + return errors + + +@dataclass +class ResolvedParam: + param: str + value: object + default: object + source: str + schema: ParamSchema + + @property + def is_explicit(self) -> bool: + return self.source != "default" + + +def resolve_parameters(toml_overrides=None, cli_overrides=None, manifest_overrides=None): + layers = ( + (cli_overrides or {}, "cli"), + (toml_overrides or {}, "ecc.toml"), + (manifest_overrides or {}, "project.json"), + ) + resolved, errors = [], [] + for schema in PARAM_REGISTRY: + value, source = schema.default, "default" + for layer, layer_source in layers: + if schema.param not in layer: + continue + value, source = layer[schema.param], layer_source + if source != "cli": + value, type_error = validate_schema_type(value, schema) + if type_error: + errors.append(type_error) + errors.extend(validate_value(value, schema)) + break + resolved.append(ResolvedParam(schema.param, value, schema.default, source, schema)) + return resolved, errors + + +def manifest_value_for(canonical: dict, maps_to) -> tuple[object, bool]: + if isinstance(maps_to, str): + return (canonical[maps_to], True) if maps_to in canonical else (None, False) + if isinstance(maps_to, dict): + for parent, child in maps_to.items(): + node = canonical.get(parent) + if isinstance(node, dict) and child in node: + return node[child], True + return None, False + + +def coerce_manifest_parameters(canonical: dict, registry=PARAM_REGISTRY, skip_params=frozenset()): + coerced, errors = copy.deepcopy(canonical), [] + for schema in registry: + if schema.param in skip_params: + continue + value, present = manifest_value_for(coerced, schema.maps_to) + if not present: + continue + value, error = validate_schema_type(value, schema) + if error: + errors.append(error) + continue + if isinstance(schema.maps_to, str): + coerced[schema.maps_to] = value + else: + for parent, child in schema.maps_to.items(): + if isinstance(coerced.get(parent), dict) and child in coerced[parent]: + coerced[parent][child] = value + break + return coerced, errors + + +def build_backend_overrides(resolved, *, include_defaults: bool = False): + overrides = {} + for item in resolved: + if not include_defaults and item.source == "default" and item.value == item.default: + continue + target, value = item.schema.maps_to, item.value + if isinstance(target, str): + overrides[target] = value + elif isinstance(target, dict): + for parent, child in target.items(): + overrides.setdefault(parent, {})[child] = value + return overrides + + +def build_config_overrides(resolved): + overrides = {} + for item in resolved: + target = item.schema.config_target + if target is None or not item.is_explicit: + continue + current = overrides.setdefault(target.config_key, {}) + for key in target.json_path[:-1]: + current = current.setdefault(key, {}) + current[target.json_path[-1]] = item.value + return overrides + + +def build_pdk_overrides(resolved): + return { + item.schema.pdk_target: item.value + for item in resolved + if item.schema.pdk_target and item.is_explicit + } + + +def parse_cli_overrides(pairs): + result, errors = {}, [] + for pair in pairs: + if "=" not in pair: + errors.append(f"malformed override: '{pair}' (expected key=value)") + continue + key, raw = (part.strip() for part in pair.split("=", 1)) + schema = lookup_schema(key) + if schema is None: + errors.append(f"unknown parameter: '{key}'") + continue + try: + value = parse_value(raw, schema) + except ValueError as exc: + errors.append(str(exc)) + continue + value_errors = validate_value(value, schema) + if value_errors: + errors.extend(value_errors) + continue + result[key] = value + return result, errors + + +def parse_toml_params(params_table): + flat, errors = {}, [] + + def visit(path, value): + key = ".".join(path) + schema = lookup_schema(key) + if schema: + try: + parsed = parse_value(value, schema) if isinstance(value, str) else value + parsed, type_error = validate_schema_type(parsed, schema) + if type_error: + errors.append(type_error) + return + except ValueError as exc: + errors.append(str(exc)) + return + value_errors = validate_value(parsed, schema) + if value_errors: + errors.extend(value_errors) + return + flat[key] = parsed + return + if isinstance(value, dict): + for child, child_value in value.items(): + visit((*path, child), child_value) + else: + errors.append(f"unknown parameter in ecc.toml: '{key}'") + + for group, value in params_table.items(): + if isinstance(value, dict): + visit((group,), value) + else: + errors.append(f"[params.{group}] must be a table, got {type(value).__name__}") + return flat, errors diff --git a/chipcompiler/data/pdk.py b/chipcompiler/data/pdk.py index febc744fb..9d8278b19 100644 --- a/chipcompiler/data/pdk.py +++ b/chipcompiler/data/pdk.py @@ -249,6 +249,8 @@ def get_pdk( pdk_root: str | Path = "", pdk_config: str | Path = "", overrides: dict | None = None, + *, + validate: bool = True, ) -> PDK: """ Return the PDK instance based on the given pdk name. @@ -273,15 +275,16 @@ def get_pdk( pdk = _builtin_pdk(pdk_name_normalized, pdk_root=pdk_root) or PDK(name=pdk_name_normalized) overrides = overrides or {} pdk = apply_pdk_overrides(pdk, overrides) - pdk.validate() - errors = [] - for key, label in _OPTIONAL_PATH_LABELS.items(): - if key not in overrides: - continue - path = getattr(pdk, key) - if path and not path.is_file(): - errors.append(f"{label}: {path}") - _raise_pdk_validation_error(errors) + if validate: + pdk.validate() + errors = [] + for key, label in _OPTIONAL_PATH_LABELS.items(): + if key not in overrides: + continue + path = getattr(pdk, key) + if path and not path.is_file(): + errors.append(f"{label}: {path}") + _raise_pdk_validation_error(errors) return pdk diff --git a/chipcompiler/data/step.py b/chipcompiler/data/step.py index c12512871..c532ed370 100644 --- a/chipcompiler/data/step.py +++ b/chipcompiler/data/step.py @@ -48,6 +48,17 @@ class StateEnum(Enum): FINISHED_STEP_STATES = frozenset({StateEnum.Success.value}) +def step_storage_name(step_name: str, tool_name: str) -> str: + """Directory stem for ``{stem}_{tool}`` workspace step folders. + + Sizer stores Timing Opt as ``timing_optimization_sizer`` instead of + ``Timing optimization_sizer``. Other tools keep the flow step name. + """ + if tool_name.lower() == "sizer": + return "_".join(step_name.split()).lower() + return step_name + + def is_finished_step_state(state: object) -> bool: """Whether a persisted step state counts as done for selection and skipping. diff --git a/chipcompiler/data/workspace/__init__.py b/chipcompiler/data/workspace/__init__.py index e41227674..20c50c4b4 100644 --- a/chipcompiler/data/workspace/__init__.py +++ b/chipcompiler/data/workspace/__init__.py @@ -14,16 +14,20 @@ from ..parameter import ( Parameters, get_parameters, - load_parameter, reload_parameter, save_parameter, update_parameters, ) +from ..parameter import ( + load_parameter as load_parameter, +) from ..pdk import PDK, get_pdk from ..step import StateEnum, StepEnum from ..workspace_config import ( - legacy_parameters_fallback, - migrate_legacy_parameters, + legacy_parameters_fallback as legacy_parameters_fallback, +) +from ..workspace_config import ( + migrate_legacy_parameters as migrate_legacy_parameters, ) from ..workspace_config import ( workspace_config_path as workspace_config_toml_path, @@ -954,6 +958,9 @@ def update_step_config(workspace: Workspace, step: WorkspaceStep) -> None: if step.name in {StepEnum.PRE_FLOORPLAN.value, StepEnum.POST_FLOORPLAN.value}: _refresh_floorplan_config(workspace, step=step) + from .config_overrides import apply_config_overrides + + apply_config_overrides(workspace.config, workspace.parameters.data) if step.name == StepEnum.ROUTING.value and isinstance(step.data, EccData): router = json_read(workspace.config[f"{StepEnum.ROUTING.value}"]) @@ -1117,6 +1124,14 @@ def create_workspace( input_filelist=input_filelist, golden_verilog=golden_verilog, ) + if workspace.design.input_filelist is not None: + # The in-memory rebind dies with this process; persist the frozen copy + # so reloaded workspaces (step subprocesses, later sessions) keep the + # filelist input instead of falling back to the single origin verilog. + workspace.parameters.data["file_list"] = str( + workspace.design.input_filelist.relative_to(workspace_dir) + ) + init_workspace_config(workspace) # set home data @@ -1190,119 +1205,10 @@ def _persisted_golden_verilog(workspace_dir: Path) -> tuple[Path | None, bool]: return None, True -def load_workspace(directory: str | Path) -> Workspace: - workspace_dir = Path(directory).expanduser().resolve() - origin_dir = workspace_dir / "origin" - home_dir = workspace_dir / "home" - if not workspace_dir.exists(): - return None - - migrate_legacy_parameters(workspace_dir) - - # create workspace instance - workspace = Workspace() - workspace.directory = workspace_dir - migrate_workspace_config_filenames(workspace_dir) - workspace.config = build_workspace_config_paths(workspace) - - config_path = workspace_config_toml_path(workspace_dir) - legacy_path = home_dir / "parameters.json" - if config_path.is_symlink(): - # A symlinked canonical config would make the workspace execute - # with external parameters it does not own: reject it the same way - # the save path refuses to write through a symlink. - from chipcompiler.data.workspace_config import WorkspaceConfigError - - raise WorkspaceConfigError(f"workspace config is a symlink: {config_path}") - parameters = load_parameter(workspace_config_toml_path(workspace_dir)) - if len(parameters.data) <= 0 and not config_path.exists() and legacy_path.exists(): - # Migration was deferred (e.g. read-only dir): fall back to the - # normalized in-memory copy so the workspace still opens. When the - # TOML exists it wins unconditionally — a malformed config never - # silently falls back to stale JSON. - fallback = legacy_parameters_fallback(workspace_dir) - if fallback: - parameters.data = fallback - if len(parameters.data) <= 0: - return None - - workspace.parameters = parameters - - pdk = get_pdk( - pdk_name=parameters.data.get("pdk", ""), - pdk_root=parameters.data.get("pdk_root", ""), - pdk_config=parameters.data.get("pdk_config", ""), - ) - sdc_path = list(origin_dir.rglob("*.sdc")) - if len(sdc_path) > 0: - pdk.sdc = sdc_path[0] - spef_path = list(origin_dir.rglob("*.spef")) - if len(spef_path) > 0: - pdk.spef = spef_path[0] - - # update lef and lib paths based on config - from chipcompiler.utility import json_read - - db_json = json_read(workspace.config.get("db", "")) - if db_json.get("INPUT", {}).get("tech_lef_path", "") != "": - pdk.tech = Path(db_json.get("INPUT", {}).get("tech_lef_path", "")) - if db_json.get("INPUT", {}).get("lef_paths", []) != []: - pdk.lefs = [Path(path) for path in db_json.get("INPUT", {}).get("lef_paths", [])] - if db_json.get("INPUT", {}).get("lib_path", []) != []: - pdk.libs = [Path(path) for path in db_json.get("INPUT", {}).get("lib_path", [])] - workspace.pdk = pdk - - # update config - workspace.design.name = parameters.data.get("design", "") - workspace.design.top_module = parameters.data.get("top_module", "") - def_path = list(origin_dir.rglob("*.def")) - def_gz_path = list(origin_dir.rglob("*.def.gz")) - if len(def_path) > 0: - workspace.design.origin_def = def_path[0] - if len(def_gz_path) > 0: - workspace.design.origin_def = def_gz_path[0] - - # The golden netlist path is persisted in the first flow step's info at - # creation; trust it over the golden_* filename convention so a primary - # netlist whose name merely starts with "golden_" keeps its role. Only - # legacy ledgers without step info fall back to the filename convention. - golden, golden_declared = _persisted_golden_verilog(workspace_dir) - if golden is None and not golden_declared: - golden_paths = list(origin_dir.rglob("golden_*.v")) + list( - origin_dir.rglob("golden_*.v.gz") - ) - golden = golden_paths[0] if golden_paths else None - - verilog_path = [path for path in origin_dir.rglob("*.v") if path != golden] - verilog_gz_path = [path for path in origin_dir.rglob("*.v.gz") if path != golden] - if len(verilog_path) > 0: - workspace.design.origin_verilog = verilog_path[0] - if len(verilog_gz_path) > 0: - workspace.design.origin_verilog = verilog_gz_path[0] +def load_workspace(directory: str | Path, *, read_only: bool = False) -> Workspace: + from .loader import load_workspace as hydrate_workspace - if golden is not None: - workspace.design.golden_verilog = golden - - filelist_path = origin_dir / "filelist" - if filelist_path.exists(): - workspace.design.input_filelist = filelist_path - - # set home data - home_dir.mkdir(parents=True, exist_ok=True) - workspace.config["dir"].mkdir(parents=True, exist_ok=True) - workspace.flow.path = home_dir / "flow.json" - workspace.home.init(path=home_dir / "home.json") - workspace.home.set_flow(workspace.flow.path) - workspace.home.set_checklist(home_dir / "checklist.json") - workspace.home.set_parameters(workspace.parameters.path) - - # create logger first (needed for copy operations) - workspace.logger = create_logger(name=parameters.data["design"], log_dir=workspace_dir / "log") - - log_workspace(workspace) - log_parameters(workspace) - - return workspace + return hydrate_workspace(directory, read_only=read_only) def log_workspace(workspace: Workspace): diff --git a/chipcompiler/data/workspace/loader.py b/chipcompiler/data/workspace/loader.py new file mode 100644 index 000000000..e95a7e440 --- /dev/null +++ b/chipcompiler/data/workspace/loader.py @@ -0,0 +1,140 @@ +"""Hydrate a Workspace from committed files without mixing migration policy.""" + +from pathlib import Path +from typing import Any + +from chipcompiler.utility import Logger, create_logger, json_read + +from ..parameter import load_parameter +from ..pdk import get_pdk +from ..workspace_config import ( + legacy_parameters_fallback, + migrate_legacy_parameters, +) +from ..workspace_config import ( + workspace_config_path as workspace_config_toml_path, +) + + +def load_workspace(directory: str | Path, *, read_only: bool = False) -> Any: + from . import ( + Workspace, + _persisted_golden_verilog, + build_workspace_config_paths, + log_parameters, + log_workspace, + migrate_workspace_config_filenames, + ) + + workspace_dir = Path(directory).expanduser().resolve() + origin_dir = workspace_dir / "origin" + home_dir = workspace_dir / "home" + if not workspace_dir.exists(): + return None + + if not read_only: + from ..workspace_transaction import recover_workspace_file_transaction + + recover_workspace_file_transaction(workspace_dir) + migrate_legacy_parameters(workspace_dir) + + workspace = Workspace() + workspace.directory = workspace_dir + if not read_only: + migrate_workspace_config_filenames(workspace_dir) + workspace.config = build_workspace_config_paths(workspace) + + config_path = workspace_config_toml_path(workspace_dir) + legacy_path = home_dir / "parameters.json" + if config_path.is_symlink(): + from chipcompiler.data.workspace_config import WorkspaceConfigError + + raise WorkspaceConfigError(f"workspace config is a symlink: {config_path}") + parameters = load_parameter(config_path) + if len(parameters.data) <= 0 and not config_path.exists() and legacy_path.exists(): + fallback = legacy_parameters_fallback(workspace_dir) + if fallback: + parameters.data = fallback + if len(parameters.data) <= 0: + return None + + workspace.parameters = parameters + pdk = get_pdk( + pdk_name=parameters.data.get("pdk", ""), + pdk_root=parameters.data.get("pdk_root", ""), + pdk_config=parameters.data.get("pdk_config", ""), + validate=not read_only, + ) + sdc_path = list(origin_dir.rglob("*.sdc")) + if sdc_path: + pdk.sdc = sdc_path[0] + spef_path = list(origin_dir.rglob("*.spef")) + if spef_path: + pdk.spef = spef_path[0] + + db_json = json_read(workspace.config.get("db", "")) + if db_json.get("INPUT", {}).get("tech_lef_path", "") != "": + pdk.tech = Path(db_json["INPUT"]["tech_lef_path"]) + if db_json.get("INPUT", {}).get("lef_paths", []) != []: + pdk.lefs = [Path(path) for path in db_json["INPUT"]["lef_paths"]] + if db_json.get("INPUT", {}).get("lib_path", []) != []: + pdk.libs = [Path(path) for path in db_json["INPUT"]["lib_path"]] + workspace.pdk = pdk + + workspace.design.name = parameters.data.get("design", "") + workspace.design.top_module = parameters.data.get("top_module", "") + def_path = list(origin_dir.rglob("*.def")) + def_gz_path = list(origin_dir.rglob("*.def.gz")) + if def_path: + workspace.design.origin_def = def_path[0] + if def_gz_path: + workspace.design.origin_def = def_gz_path[0] + + golden, golden_declared = _persisted_golden_verilog(workspace_dir) + if golden is None and not golden_declared: + golden_paths = list(origin_dir.rglob("golden_*.v")) + list( + origin_dir.rglob("golden_*.v.gz") + ) + golden = golden_paths[0] if golden_paths else None + + verilog_path = [path for path in origin_dir.rglob("*.v") if path != golden] + verilog_gz_path = [path for path in origin_dir.rglob("*.v.gz") if path != golden] + if verilog_path: + workspace.design.origin_verilog = verilog_path[0] + if verilog_gz_path: + workspace.design.origin_verilog = verilog_gz_path[0] + if golden is not None: + workspace.design.golden_verilog = golden + + persisted_filelist = parameters.data.get("file_list", "") + if persisted_filelist: + persisted_filelist = Path(persisted_filelist) + if not persisted_filelist.is_absolute(): + persisted_filelist = workspace_dir / persisted_filelist + if persisted_filelist.is_file(): + workspace.design.input_filelist = persisted_filelist + + filelist_path = origin_dir / "filelist" + if workspace.design.input_filelist is None and filelist_path.exists(): + workspace.design.input_filelist = filelist_path + + workspace.flow.path = home_dir / "flow.json" + if read_only: + workspace.home.path = home_dir / "home.json" + home_data = json_read(workspace.home.path) + workspace.home.data = home_data if isinstance(home_data, dict) else {} + workspace.logger = Logger(name=parameters.data["design"]) + else: + home_dir.mkdir(parents=True, exist_ok=True) + workspace.config["dir"].mkdir(parents=True, exist_ok=True) + workspace.home.init(path=home_dir / "home.json") + workspace.home.set_flow(workspace.flow.path) + workspace.home.set_checklist(home_dir / "checklist.json") + workspace.home.set_parameters(workspace.parameters.path) + workspace.logger = create_logger( + name=parameters.data["design"], log_dir=workspace_dir / "log" + ) + log_workspace(workspace) + log_parameters(workspace) + + return workspace diff --git a/chipcompiler/data/workspace_config.py b/chipcompiler/data/workspace_config.py index 97df94111..941b8b38b 100644 --- a/chipcompiler/data/workspace_config.py +++ b/chipcompiler/data/workspace_config.py @@ -140,7 +140,7 @@ def validate_flow_config(flow: object) -> dict[str, str]: if not isinstance(value, str): raise WorkspaceFlowTargetError(f"[flow] {key} must be a string: {value!r}") # Workspace files carry canonical step names only; display-name - # aliases are translated at the manifest/RPC boundary, never here. + # aliases are translated at the manifest/adapter boundary, never here. if value not in canonical_names: raise WorkspaceFlowTargetError(f"[flow] unknown step name: {value!r}") normalized[key] = value diff --git a/chipcompiler/data/workspace_parameters.py b/chipcompiler/data/workspace_parameters.py new file mode 100644 index 000000000..0b525c0e7 --- /dev/null +++ b/chipcompiler/data/workspace_parameters.py @@ -0,0 +1,128 @@ +"""Workspace-local parameter persistence shared by Engine and CLI.""" + +from copy import deepcopy + +from chipcompiler.data.parameter import update_parameters +from chipcompiler.data.parameter_schema import ( + ResolvedParam, + build_backend_overrides, + build_config_overrides, +) +from chipcompiler.data.workspace.config_overrides import CONFIG_OVERRIDES_KEY +from chipcompiler.utility import json_read + +WORKSPACE_PARAM_OVERRIDES_KEY = "workspace_param_overrides" +_APPLIES_TO_STEP = { + "synthesis": "Synthesis", + "floorplan": "Floorplan", + "placement": "place", + "macro": "macroPlacement", + "cts": "CTS", + "routing": "route", + "filler": "filler", + "rcx": "RCX", + "sta": "sta", +} +_MISSING = object() + + +def workspace_param_step(schema) -> str: + step = _APPLIES_TO_STEP.get(schema.applies) + if step is None: + raise ValueError(f"{schema.param} requires a full workspace refresh") + return step + + +def workspace_param_value(workspace, schema) -> object: + if schema.pdk_target is not None: + raise ValueError(f"{schema.param} requires a full workspace refresh") + if schema.maps_to is not None: + return _parameter_target_value(workspace.parameters.data, schema.maps_to, schema.default) + if schema.config_target is not None: + config_path = workspace.config.get(schema.config_target.config_key) + if config_path is None: + raise ValueError(f"workspace config missing target: {schema.config_target.config_key}") + value = _nested_value(json_read(config_path), schema.config_target.json_path) + return schema.default if value is _MISSING else value + raise ValueError(f"{schema.param} has no workspace configuration target") + + +def set_workspace_param(workspace, schema, value: object) -> tuple[object, str]: + baseline = update_workspace_param_value(workspace, schema, value) + return baseline, workspace_param_step(schema) + + +def update_workspace_param_value(workspace, schema, value: object) -> object: + records = _override_records(workspace.parameters.data) + record = next((item for item in records if item["key"] == schema.param), None) + if record is None: + record = { + "key": schema.param, + "baseline": deepcopy(workspace_param_value(workspace, schema)), + } + records.append(record) + record["value"] = deepcopy(value) + _set_override_records(workspace.parameters.data, records) + _apply_workspace_value(workspace, schema, value) + return record["baseline"] + + +def unset_workspace_param(workspace, schema) -> tuple[object, str] | None: + records = _override_records(workspace.parameters.data) + record = next((item for item in records if item["key"] == schema.param), None) + if record is None: + return None + records.remove(record) + _set_override_records(workspace.parameters.data, records) + _apply_workspace_value(workspace, schema, record["baseline"]) + return record["baseline"], workspace_param_step(schema) + + +def workspace_param_diff(workspace) -> list[dict]: + return _override_records(workspace.parameters.data) + + +def _apply_workspace_value(workspace, schema, value: object) -> None: + resolved = ResolvedParam(schema.param, value, schema.default, "workspace", schema) + update_parameters(build_backend_overrides([resolved]), workspace.parameters.data) + config_overrides = build_config_overrides([resolved]) + if config_overrides: + update_parameters({CONFIG_OVERRIDES_KEY: config_overrides}, workspace.parameters.data) + + +def _override_records(parameters: dict) -> list[dict]: + raw = parameters.get(WORKSPACE_PARAM_OVERRIDES_KEY, []) + if not isinstance(raw, list): + return [] + return [ + deepcopy(item) + for item in raw + if isinstance(item, dict) and isinstance(item.get("key"), str) and "baseline" in item + ] + + +def _set_override_records(parameters: dict, records: list[dict]) -> None: + if records: + parameters[WORKSPACE_PARAM_OVERRIDES_KEY] = records + else: + parameters.pop(WORKSPACE_PARAM_OVERRIDES_KEY, None) + + +def _parameter_target_value(parameters: dict, target, default: object) -> object: + if isinstance(target, str): + return deepcopy(parameters.get(target, default)) + if isinstance(target, dict) and len(target) == 1: + parent, child = next(iter(target.items())) + nested = parameters.get(parent) + if isinstance(nested, dict): + return deepcopy(nested.get(child, default)) + return deepcopy(default) + + +def _nested_value(data: object, path: tuple[str, ...]) -> object: + current = data + for key in path: + if not isinstance(current, dict) or key not in current: + return _MISSING + current = current[key] + return deepcopy(current) diff --git a/chipcompiler/data/workspace_transaction.py b/chipcompiler/data/workspace_transaction.py new file mode 100644 index 000000000..96d23b14c --- /dev/null +++ b/chipcompiler/data/workspace_transaction.py @@ -0,0 +1,122 @@ +import fcntl +import os +import shutil +from collections.abc import Iterable +from pathlib import Path +from typing import BinaryIO + +_BACKUP_NAME = ".workspace-configuration-backup" +_DISCARD_NAME = f"{_BACKUP_NAME}.discard" +_ACTIVE: set[Path] = set() + + +class WorkspaceFileTransaction: + def __init__(self, workspace: Path, lock: BinaryIO): + self.workspace = workspace + self.lock = lock + self.backup = workspace / "home" / _BACKUP_NAME + self.finished = False + + @classmethod + def begin(cls, workspace: str | Path, paths: Iterable[Path]) -> "WorkspaceFileTransaction": + root = Path(workspace).expanduser().resolve() + lock = _open_lock(_lock_path(root)) + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + try: + _recover_locked(root) + transaction = cls(root, lock) + transaction._prepare(paths) + _ACTIVE.add(root) + return transaction + except BaseException: + fcntl.flock(lock.fileno(), fcntl.LOCK_UN) + lock.close() + raise + + def commit(self) -> None: + discard = self.workspace / "home" / _DISCARD_NAME + if discard.exists(): + shutil.rmtree(discard) + if self.backup.exists(): + self.backup.replace(discard) + shutil.rmtree(discard) + self._release() + + def rollback(self) -> None: + try: + _restore_backup(self.workspace, self.backup) + shutil.rmtree(self.backup, ignore_errors=True) + finally: + self._release() + + def _prepare(self, paths: Iterable[Path]) -> None: + home = self.workspace / "home" + if home.is_symlink(): + raise OSError(f"Workspace home directory is a symlink: {home}") + home.mkdir(parents=True, exist_ok=True) + shutil.rmtree(self.backup, ignore_errors=True) + self.backup.mkdir(parents=True) + for path in sorted(set(paths)): + target = Path(path).resolve() + relative = target.relative_to(self.workspace) + if not target.is_file(): + continue + destination = self.backup / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(target, destination) + + def _release(self) -> None: + if self.finished: + return + _ACTIVE.discard(self.workspace) + fcntl.flock(self.lock.fileno(), fcntl.LOCK_UN) + self.lock.close() + self.finished = True + + +def recover_workspace_file_transaction(workspace: str | Path) -> None: + root = Path(workspace).expanduser().resolve() + if root in _ACTIVE: + return + home = root / "home" + if not (home / _BACKUP_NAME).exists() and not (home / _DISCARD_NAME).exists(): + return + with _open_lock(_lock_path(root)) as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + try: + _recover_locked(root) + finally: + fcntl.flock(lock.fileno(), fcntl.LOCK_UN) + + +def _recover_locked(workspace: Path) -> None: + home = workspace / "home" + if home.is_symlink(): + raise OSError(f"Workspace home directory is a symlink: {home}") + discard = home / _DISCARD_NAME + if discard.exists(): + shutil.rmtree(discard) + backup = home / _BACKUP_NAME + if backup.exists(): + _restore_backup(workspace, backup) + shutil.rmtree(backup) + + +def _restore_backup(workspace: Path, backup: Path) -> None: + if not backup.is_dir(): + return + for source in backup.rglob("*"): + if not source.is_file(): + continue + target = workspace / source.relative_to(backup) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + + +def _lock_path(workspace: Path) -> Path: + return workspace.parent / f".{workspace.name}.configuration.lock" + + +def _open_lock(path: Path) -> BinaryIO: + descriptor = os.open(path, os.O_RDWR | os.O_CREAT, 0o600) + return os.fdopen(descriptor, "a+b") diff --git a/chipcompiler/docs/ecc-tutorial.cn.md b/chipcompiler/docs/ecc-tutorial.cn.md index ae821d4b3..88c8cda20 100644 --- a/chipcompiler/docs/ecc-tutorial.cn.md +++ b/chipcompiler/docs/ecc-tutorial.cn.md @@ -527,7 +527,7 @@ $ ecc report summary ### 5.4 QoR 总分：ecc report qor -按 GUI 项目看板同一套规则打分：每条指标折算 0–100 分，按维度加权（Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1），60 分为通过线；缺项维度不重归一化（缺项会拉低总分）： +用 ECC 共用的 `qor_scoring` 规则打分（Studio Snapshot 也用这一套）：每条指标折算 0–100 分，按维度加权（Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1），60 分为通过线；缺项维度不重归一化（缺项会拉低总分）： ```console $ ecc report qor diff --git a/chipcompiler/docs/ecc-tutorial.en.md b/chipcompiler/docs/ecc-tutorial.en.md index da6f7cbff..d6763f888 100644 --- a/chipcompiler/docs/ecc-tutorial.en.md +++ b/chipcompiler/docs/ecc-tutorial.en.md @@ -528,7 +528,7 @@ Excerpts from this gcd run (full report: `cat` the file above): ### 5.4 QoR score: ecc report qor -Scores the workspace with the same rules as the GUI project dashboard: each metric maps to 0–100, dimensions are weighted (Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1), 60 is the pass line; absent dimensions are not renormalized (absence drags the overall score down): +Scores the workspace with ECC's shared `qor_scoring` rules (the same table Studio Snapshot uses): each metric maps to 0–100, dimensions are weighted (Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1), 60 is the pass line; absent dimensions are not renormalized (absence drags the overall score down): ```console $ ecc report qor diff --git a/chipcompiler/docs/ecc-user-guide.cn.md b/chipcompiler/docs/ecc-user-guide.cn.md index e755a2660..2f1d067fb 100644 --- a/chipcompiler/docs/ecc-user-guide.cn.md +++ b/chipcompiler/docs/ecc-user-guide.cn.md @@ -111,7 +111,6 @@ Commands: workspace Refresh managed workspaces from project configuration signoff Inspect and export signoff packages report Generate design-summary, QoR score, checklist, and step reports - rpc Run the private ECC JSON-RPC runtime ``` ## 1.5. doc — 在终端阅读内置指南 @@ -947,7 +946,7 @@ PDK / Node : ics55 ### 12.2 qor — QoR 总体计分报告 -按 GUI 项目看板的计分规则给当前 workspace 打分：每条 v3 `qor_metrics.json` 指标按固定失败阈值折算 0-100 分（slack 类线性、core_utilization 目标区间 [0.45,0.70]、lower/higher_is_better 比例），维度内取平均，再按权重（Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1）加权出总分——**缺项维度不重归一化**（与 GUI 一致，缺项会拉低总分）；60 分为通过线。默认写 `/signoff/_qor_report.txt`： +用 ECC 共用的 `qor_scoring` 规则给当前 workspace 打分（Studio Snapshot 也用这一套）：每条 v3 `qor_metrics.json` 指标按固定失败阈值折算 0-100 分（slack 类线性、core_utilization 目标区间 [0.45,0.70]、lower/higher_is_better 比例），维度内取平均，再按权重（Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1）加权出总分——**缺项维度不重归一化**（缺项会拉低总分）；60 分为通过线。默认写 `/signoff/_qor_report.txt`： ```console $ ecc report qor --project gcd --plain diff --git a/chipcompiler/docs/ecc-user-guide.en.md b/chipcompiler/docs/ecc-user-guide.en.md index cfa69b0de..a17c297de 100644 --- a/chipcompiler/docs/ecc-user-guide.en.md +++ b/chipcompiler/docs/ecc-user-guide.en.md @@ -111,7 +111,6 @@ Commands: workspace Refresh managed workspaces from project configuration signoff Inspect and export signoff packages report Generate design-summary, QoR score, checklist, and step reports - rpc Run the private ECC JSON-RPC runtime ``` ## 1.5. doc — read the bundled guides in the terminal @@ -995,7 +994,7 @@ the report extracts the current state by default (the engine API ### 12.2 qor — overall QoR score report -Scores the current workspace by the GUI project-dashboard rules: every v3 `qor_metrics.json` metric is converted to 0-100 against fixed fail thresholds (slack metrics linearly, core_utilization against the [0.45, 0.70] target window, lower/higher_is_better proportionally), averaged per dimension, then combined with the dimension weights (Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1) into the overall score — **absent dimensions are not renormalized** (matching the GUI; missing dimensions lower the score); 60 is the pass line. By default written to `/signoff/_qor_report.txt`: +Scores the current workspace with ECC's shared `qor_scoring` rules (the same table Studio Snapshot uses): every v3 `qor_metrics.json` metric is converted to 0-100 against fixed fail thresholds (slack metrics linearly, core_utilization against the [0.45, 0.70] target window, lower/higher_is_better proportionally), averaged per dimension, then combined with the dimension weights (Timing 0.35 / Power 0.25 / Routability 0.2 / Area 0.1 / Clock-DFM 0.1) into the overall score — **absent dimensions are not renormalized** (missing dimensions lower the score); 60 is the pass line. By default written to `/signoff/_qor_report.txt`: ```console $ ecc report qor --project gcd --plain diff --git a/chipcompiler/engine/__init__.py b/chipcompiler/engine/__init__.py index f75ec0efe..bff9d382c 100644 --- a/chipcompiler/engine/__init__.py +++ b/chipcompiler/engine/__init__.py @@ -1,12 +1,53 @@ from .db import EngineDB +from .execution import ExecutionPlan, ExecutionResult, execute from .flow import EngineFlow from .rerun import StepRunResult from .signoff import SignoffPackageCollector, SignoffPackageOptions +from .snapshot import ( + migrate_engineering_snapshot, + migrate_engineering_snapshot_v2_to_v3, +) +from .workspace_configuration import ( + read_step_configuration, + read_step_configuration_from_directory, + read_workspace_configuration, + read_workspace_configuration_from_directory, + update_workspace_configuration, + update_workspace_step_configuration, +) +from .workspace_lifecycle import ( + WorkspaceLifecycleError, + apply_workspace_bindings, + assess_execution_readiness, + create_workspace_from_spec, + describe_workspace_binding_requirement, + update_workspace_from_spec, +) +from .workspace_spec import describe_workspace_spec, validate_workspace_spec __all__ = [ "EngineDB", "EngineFlow", + "migrate_engineering_snapshot", + "migrate_engineering_snapshot_v2_to_v3", + "ExecutionPlan", + "ExecutionResult", "StepRunResult", "SignoffPackageCollector", "SignoffPackageOptions", + "WorkspaceLifecycleError", + "apply_workspace_bindings", + "assess_execution_readiness", + "create_workspace_from_spec", + "describe_workspace_spec", + "describe_workspace_binding_requirement", + "execute", + "read_step_configuration", + "read_step_configuration_from_directory", + "read_workspace_configuration", + "read_workspace_configuration_from_directory", + "update_workspace_from_spec", + "update_workspace_step_configuration", + "update_workspace_configuration", + "validate_workspace_spec", ] diff --git a/chipcompiler/engine/analysis.py b/chipcompiler/engine/analysis.py new file mode 100644 index 000000000..06ddc03e0 --- /dev/null +++ b/chipcompiler/engine/analysis.py @@ -0,0 +1,462 @@ +import hashlib +import math +from pathlib import Path +from typing import Any, TypeGuard + +from chipcompiler.data.step import step_storage_name +from chipcompiler.data.step_dirs import STEP_DIRECTORIES +from chipcompiler.engine.qor_scoring import DIMENSION_WEIGHTS +from chipcompiler.tools.ecc.sta_qor import STA_POWER_REPORT_FILENAME, STA_REPORT_FILENAMES +from chipcompiler.utility import JsonReadError, file_digest, json_read_strict + +_LEGACY_METRIC_CATEGORIES = {"power": "power_integrity"} + +_ANALYSIS_FILES = ( + ("metrics", "qor_metrics", "qor_metrics.json", 3), + ("summary", "qor_summary", "qor_summary.json", 4), + ("hotspots", "qor_hotspots", "qor_hotspots.json", 3), +) +_TIMING_FILE = ("timingIssues", "sta_timing_issues", "sta_timing_issues.json", 1) +_SUBFLOW_MAX_BYTES = 1024 * 1024 +_SUBFLOW_MAX_STEPS = 256 +_ARTIFACT_HASH_MAX_BYTES = 16 * 1024 * 1024 +_CONGESTION_IMAGES = ( + ("egr_congestion_map", "{step}_egr_horizontal_overflow.png"), + ("egr_congestion_map", "{step}_egr_vertical_overflow.png"), + ("egr_congestion_map", "{step}_egr_union_overflow.png"), + ("RUDY_map", "{step}_rudy_horizontal.png"), + ("RUDY_map", "{step}_rudy_vertical.png"), + ("RUDY_map", "{step}_rudy_union.png"), + ("RUDY_map", "{step}_lut_rudy_horizontal.png"), + ("RUDY_map", "{step}_lut_rudy_vertical.png"), + ("RUDY_map", "{step}_lut_rudy_union.png"), + ("density_map", "{step}_allcell_density.png"), +) + + +def build_workspace_analysis( + workspace: Any, workspace_id: str +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + root = Path(workspace.directory).resolve() + flow = getattr(getattr(workspace, "flow", None), "data", {}) + raw_steps = flow.get("steps", []) if isinstance(flow, dict) else [] + design = str(getattr(getattr(workspace, "design", None), "name", "")).strip() + steps: list[dict[str, Any]] = [] + artifacts: list[dict[str, Any]] = [] + for order, raw_step in enumerate(raw_steps): + if not isinstance(raw_step, dict): + continue + step_id = raw_step.get("name") + tool_id = raw_step.get("tool") + if not _safe_segment(step_id) or not _safe_segment(tool_id): + continue + identity = "".join(character for character in step_id.casefold() if character.isalnum()) + if identity == "fixfanout": + continue + step_dir = root / STEP_DIRECTORIES.get( + step_id, f"{step_storage_name(step_id, tool_id)}_{tool_id}" + ) + step: dict[str, Any] = { + "stepId": step_id, + "toolId": tool_id, + "order": order, + "flowState": str(raw_step.get("state", "")), + } + declared = list(_ANALYSIS_FILES) + if step_id.lower() == "sta": + declared.append(_TIMING_FILE) + for field, kind, filename, schema_version in declared: + path = step_dir / "analysis" / filename + reference = path.relative_to(root).as_posix() + artifact = _artifact_ref( + path, + workspace_id=workspace_id, + reference=reference, + step_id=step_id, + kind=kind, + root=root, + ) + artifacts.append(artifact) + step[field] = _analysis_file(path, artifact["artifactId"], schema_version, root) + if "timingIssues" not in step: + step["timingIssues"] = None + if tool_id.lower() == "yosys_lec" and design: + result = step_dir / "output" / f"{design}_{step_id}_result.json" + artifact = _artifact_ref( + result, + workspace_id=workspace_id, + reference=result.relative_to(root).as_posix(), + step_id=step_id, + kind="lec_result", + root=root, + ) + artifacts.append(artifact) + step["lecResult"] = _lec_result_file(result, artifact["artifactId"], root) + step["subflow"] = _subflow_summary(step_dir / "subflow.json", root) + if design: + layout = step_dir / "output" / f"{design}_{step_id}.png" + artifacts.append( + _artifact_ref( + layout, + workspace_id=workspace_id, + reference=layout.relative_to(root).as_posix(), + step_id=step_id, + kind="layout_image", + root=root, + ) + ) + if step_id.lower() == "harden": + for suffix in ("gds", "lef", "lib"): + output = step_dir / "output" / f"{design}_{step_id}.{suffix}" + artifacts.append( + _artifact_ref( + output, + workspace_id=workspace_id, + reference=output.relative_to(root).as_posix(), + step_id=step_id, + kind="harden_output", + root=root, + ) + ) + geometry = step_dir / "output" / "geometry" / "geometry.manifest" + artifacts.append( + _artifact_ref( + geometry, + workspace_id=workspace_id, + reference=geometry.relative_to(root).as_posix(), + step_id=step_id, + kind="layout_geometry", + root=root, + ) + ) + report_names = ( + (f"{step_id}_stat.json", f"{step_id}_check.rpt") + if tool_id.lower() == "yosys" + else (f"{step_id}.db.rpt", f"{step_id}.rpt") + ) + for report_name in report_names: + report = step_dir / "report" / report_name + artifacts.append( + _artifact_ref( + report, + workspace_id=workspace_id, + reference=report.relative_to(root).as_posix(), + step_id=step_id, + kind="report_text", + root=root, + ) + ) + timing_files: list[tuple[str, Path]] = [] + if step_id.lower() == "synthesis": + timing_root = step_dir / "feature" / "post_synthesis" + timing_files.extend( + ( + ("timing_summary", timing_root / "qor_summary.json"), + ("timing_paths", timing_root / "timing_paths.json"), + ) + ) + timing_data = step.get("timingIssues") + timing_payload = timing_data.get("data") if isinstance(timing_data, dict) else None + timing_paths = ( + timing_payload.get("artifact_paths") if isinstance(timing_payload, dict) else None + ) + if step_id.lower() == "sta": + for item in timing_paths[:32] if isinstance(timing_paths, list) else []: + if not isinstance(item, dict): + continue + report_dir = _safe_step_artifact_path(step_dir, item.get("report_dir"), root) + if report_dir is None: + continue + try: + relative_report_dir = report_dir.relative_to(step_dir / "report") + except ValueError: + continue + report_names = list(STA_REPORT_FILENAMES) + if (report_dir / STA_POWER_REPORT_FILENAME).is_file(): + report_names.append(STA_POWER_REPORT_FILENAME) + for report_name in report_names: + report = report_dir / report_name + artifact = _artifact_ref( + report, + workspace_id=workspace_id, + reference=report.relative_to(root).as_posix(), + step_id=step_id, + kind="report_text", + root=root, + ) + artifact["name"] = (relative_report_dir / report_name).as_posix() + artifacts.append(artifact) + for item in timing_paths[:32] if isinstance(timing_paths, list) else []: + if not isinstance(item, dict): + continue + for kind, field in ( + ("timing_summary", "qor_summary_file"), + ("timing_paths", "timing_paths_file"), + ): + timing_path = _safe_step_artifact_path(step_dir, item.get(field), root) + if timing_path is not None: + timing_files.append((kind, timing_path)) + for kind, path in timing_files: + artifacts.append( + _artifact_ref( + path, + workspace_id=workspace_id, + reference=path.relative_to(root).as_posix(), + step_id=step_id, + kind=kind, + root=root, + ) + ) + if step_id.lower() in {"place", "cts"}: + for directory, filename in _CONGESTION_IMAGES: + image = step_dir / "feature" / directory / filename.format(step=step_id) + artifacts.append( + _artifact_ref( + image, + workspace_id=workspace_id, + reference=image.relative_to(root).as_posix(), + step_id=step_id, + kind="congestion_image", + root=root, + ) + ) + steps.append(step) + return {"steps": steps}, artifacts + + +def _lec_result_file(path: Path, artifact_id: str, root: Path) -> dict[str, Any]: + if _has_symlink(path, root): + return { + "artifactId": artifact_id, + "status": "unsafe", + "reasonCode": "LEC_RESULT_UNSAFE", + "data": None, + } + if not path.is_file(): + return { + "artifactId": artifact_id, + "status": "missing", + "reasonCode": "LEC_RESULT_MISSING", + "data": None, + } + try: + data = json_read_strict(path) + except (OSError, JsonReadError): + return { + "artifactId": artifact_id, + "status": "invalid", + "reasonCode": "LEC_RESULT_INVALID", + "data": None, + } + if not isinstance(data, dict): + return { + "artifactId": artifact_id, + "status": "invalid", + "reasonCode": "LEC_RESULT_INVALID", + "data": None, + } + result = dict(data) + result["freshness_status"] = _lec_freshness_status(data, root) + return {"artifactId": artifact_id, "status": "available", "data": result} + + +def _lec_freshness_status(data: dict[str, Any], root: Path) -> str: + if data.get("status") != "proven": + return "incomplete" + for role in ("golden", "gate"): + path = data.get(f"{role}_verilog") + digest = data.get(f"{role}_sha256") + size = data.get(f"{role}_size_bytes") + if not isinstance(path, str) or not isinstance(digest, str) or type(size) is not int: + return "stale" + candidate = Path(path).resolve() + if not candidate.is_relative_to(root) or not candidate.is_file(): + return "stale" + actual = file_digest(candidate) + if actual is None or actual != (digest, size): + return "stale" + return "proven" + + +def _analysis_file(path: Path, artifact_id: str, schema_version: int, root: Path) -> dict[str, Any]: + if _has_symlink(path, root): + return { + "artifactId": artifact_id, + "status": "unsafe", + "reasonCode": "ANALYSIS_REFERENCE_UNSAFE", + "data": None, + } + if not path.is_file(): + return { + "artifactId": artifact_id, + "status": "missing", + "reasonCode": "ANALYSIS_FILE_MISSING", + "data": None, + } + try: + data = json_read_strict(path) + except (OSError, JsonReadError): + return { + "artifactId": artifact_id, + "status": "invalid", + "reasonCode": "ANALYSIS_FILE_INVALID", + "data": None, + } + if not isinstance(data, dict): + return { + "artifactId": artifact_id, + "status": "invalid", + "reasonCode": "ANALYSIS_FILE_INVALID", + "data": None, + } + if data.get("schema_version") != schema_version: + return { + "artifactId": artifact_id, + "status": "unsupported", + "reasonCode": "ANALYSIS_SCHEMA_UNSUPPORTED", + "data": None, + } + if schema_version == 3 and isinstance(data.get("metrics"), list): + data = _canonical_metrics_payload(data) + return {"artifactId": artifact_id, "status": "available", "data": data} + + +def _canonical_metrics_payload(data: dict[str, Any]) -> dict[str, Any]: + records = data.get("metrics") + if not isinstance(records, list): + return data + updated = [] + changed = False + for record in records: + if not isinstance(record, dict): + updated.append(record) + continue + category = record.get("category") + mapped = _LEGACY_METRIC_CATEGORIES.get(category, category) + if mapped != category and mapped in DIMENSION_WEIGHTS: + record = {**record, "category": mapped} + changed = True + updated.append(record) + if not changed: + return data + payload = dict(data) + payload["metrics"] = updated + return payload + + +def _subflow_summary(path: Path, root: Path) -> dict[str, Any]: + if _has_symlink(path, root): + return {"status": "unsafe", "steps": []} + if not path.is_file(): + return {"status": "missing", "steps": []} + try: + if path.stat().st_size > _SUBFLOW_MAX_BYTES: + return {"status": "oversized", "steps": []} + data = json_read_strict(path) + except (OSError, JsonReadError): + return {"status": "invalid", "steps": []} + raw_steps = data.get("steps") if isinstance(data, dict) else None + if not isinstance(raw_steps, list) or len(raw_steps) > _SUBFLOW_MAX_STEPS: + return {"status": "invalid", "steps": []} + steps: list[dict[str, Any]] = [] + for item in raw_steps: + if not isinstance(item, dict): + return {"status": "invalid", "steps": []} + name = item.get("name") + state = item.get("state") + if not isinstance(name, str) or not name or not isinstance(state, str): + return {"status": "invalid", "steps": []} + runtime = item.get("runtime") + peak_memory = item.get("peak memory (mb)") + step: dict[str, Any] = {"name": name, "state": state} + if isinstance(runtime, str): + step["runtime"] = runtime + if ( + isinstance(peak_memory, (int, float)) + and not isinstance(peak_memory, bool) + and math.isfinite(peak_memory) + ): + step["peakMemoryMb"] = peak_memory + steps.append(step) + return {"status": "available", "steps": steps} + + +def _artifact_ref( + path: Path, + *, + workspace_id: str, + reference: str, + step_id: str, + kind: str, + root: Path, +) -> dict[str, Any]: + artifact: dict[str, Any] = { + "artifactId": _artifact_id(workspace_id, reference), + "kind": kind, + "name": path.name, + "stepId": step_id, + "availability": "missing", + "reference": reference, + } + try: + if path.is_file() and not _has_symlink(path, root): + size = path.stat().st_size + if size <= _ARTIFACT_HASH_MAX_BYTES: + artifact.update( + availability="available", + sizeBytes=size, + sha256=_sha256(path), + ) + else: + artifact.update(availability="stale", sizeBytes=size) + except OSError: + pass + return artifact + + +def _safe_step_artifact_path(step_dir: Path, value: object, root: Path) -> Path | None: + if not isinstance(value, str) or not value: + return None + candidate = Path(value) + if candidate.is_absolute() or ".." in candidate.parts: + return None + resolved = step_dir / candidate + try: + resolved.relative_to(root) + except ValueError: + return None + return resolved + + +def _safe_segment(value: object) -> TypeGuard[str]: + return ( + isinstance(value, str) + and bool(value) + and value not in {".", ".."} + and "/" not in value + and "\\" not in value + ) + + +def _has_symlink(path: Path, root: Path) -> bool: + current = path + while current != root: + if current.is_symlink(): + return True + if current.parent == current: + return True + current = current.parent + return False + + +def _artifact_id(workspace_id: str, reference: str) -> str: + digest = hashlib.sha256(f"{workspace_id}\0{reference}".encode()).hexdigest() + return f"artifact-{digest[:32]}" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as artifact: + for chunk in iter(lambda: artifact.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/chipcompiler/engine/execution.py b/chipcompiler/engine/execution.py new file mode 100644 index 000000000..3d81960fc --- /dev/null +++ b/chipcompiler/engine/execution.py @@ -0,0 +1,188 @@ +import inspect +from dataclasses import dataclass +from typing import Any, Literal + +from chipcompiler.data import StateEnum, is_finished_step_state + + +@dataclass(frozen=True) +class ExecutionPlan: + intent: Literal["run", "rerun"] + step_id: str | None = None + step_ids: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ExecutionResult: + succeeded: bool + state: str + step_id: str | None = None + executed_steps: tuple[str, ...] = () + failed_step: str | None = None + no_op: bool = False + + +def execute(flow: Any, plan: ExecutionPlan, *, event_sink: Any = None) -> ExecutionResult: + if plan.intent not in {"run", "rerun"}: + raise ValueError(f"unsupported execution intent: {plan.intent}") + selected_ids = tuple(plan.step_ids) + if plan.step_id is not None: + if selected_ids: + raise ValueError("ExecutionPlan cannot set both step_id and step_ids") + selected_ids = (plan.step_id,) + if plan.intent == "run" and not selected_ids and _is_completed(flow): + return ExecutionResult( + succeeded=True, + state=StateEnum.Success.value, + executed_steps=(), + no_op=True, + ) + workspace = getattr(flow, "workspace", None) + if event_sink is None: + event_sink = event_sink_for_workspace(workspace) + rerun = plan.intent == "rerun" + observer = execution_observer(event_sink) + if not selected_ids: + succeeded = bool(_invoke(flow.run_steps, rerun=rerun, observer=observer)) + else: + succeeded = True + for step_id in selected_ids: + observer.raise_if_cancelled() + step = _find_step(flow, step_id) + if step is None: + raise ValueError(f"step not found: {step_id}") + state = _invoke(flow.run_step, step, rerun=rerun, observer=observer) + if ( + state is not StateEnum.Success + and getattr(state, "value", state) != StateEnum.Success.value + ): + succeeded = False + break + executed_steps = tuple(getattr(observer, "executed_steps", ())) + failed_step = getattr(observer, "failed_step", None) + state = StateEnum.Success.value if succeeded else StateEnum.Imcomplete.value + return ExecutionResult( + succeeded=succeeded, + state=state, + step_id=selected_ids[0] if len(selected_ids) == 1 else None, + executed_steps=executed_steps, + failed_step=failed_step, + ) + + +def _find_step(flow: Any, step_id: str) -> Any | None: + get_workspace_step = getattr(flow, "get_workspace_step", None) + if callable(get_workspace_step): + return get_workspace_step(step_id) + return next( + ( + candidate + for candidate in getattr(flow, "workspace_steps", []) + if getattr(candidate, "name", None) == step_id + ), + None, + ) + + +def _is_completed(flow: Any) -> bool: + data = getattr(getattr(getattr(flow, "workspace", None), "flow", None), "data", {}) + steps = data.get("steps", []) if isinstance(data, dict) else [] + return bool(steps) and all( + isinstance(step, dict) and is_finished_step_state(step.get("state")) for step in steps + ) + + +class ExecutionObserver: + def __init__(self, delegate: Any): + self.delegate = delegate + self.fatal_observer = bool(getattr(delegate, "fatal_observer", False)) + self.executed_steps: list[str] = [] + self.failed_step: str | None = None + + def on_step_completed(self, step: Any, state: Any, error: str | None = None) -> None: + name = getattr(step, "name", None) + if isinstance(name, str) and name: + if getattr(state, "value", state) == StateEnum.Success.value: + self.executed_steps.append(name) + elif self.failed_step is None: + self.failed_step = name + self._delegate_call("on_step_completed", step, state, error) + + @property + def runtime_operation(self): + return getattr(self.delegate, "runtime_operation", None) + + def raise_if_cancelled(self) -> None: + self._delegate_call("raise_if_cancelled") + + def on_step_started(self, step: Any) -> None: + self._delegate_call("on_step_started", step) + + def on_step_skipped(self, step: Any) -> None: + self._delegate_call("on_step_skipped", step) + + def on_subflow_stage(self, step: Any, subflow_step: Any) -> None: + self._delegate_call("on_subflow_stage", step, subflow_step) + + def on_step_diagnostic(self, step: Any, diagnostic: dict[str, Any]) -> None: + self._delegate_call("on_step_diagnostic", step, diagnostic) + + def on_rerun_prepared(self, *args, **kwargs) -> None: + self._delegate_call("on_rerun_prepared", *args, **kwargs) + + def wait_for_step_rendered(self, step: Any, state: Any) -> bool: + result = self._delegate_call("wait_for_step_rendered", step, state) + return True if result is None else bool(result) + + def _delegate_call(self, name: str, *args, **kwargs): + callback = getattr(self.delegate, name, None) + if callable(callback): + return callback(*args, **kwargs) + return None + + +class _EngineeringCommitSink: + fatal_observer = True + + def __init__(self, workspace: Any): + from chipcompiler.engine.snapshot import ensure_engineering_snapshot + + self.workspace = workspace + self.snapshot = ensure_engineering_snapshot(workspace) + + def on_step_completed(self, _step: Any, state: Any, _error: str | None = None) -> None: + from chipcompiler.engine.snapshot import commit_engineering_snapshot + + self.snapshot = commit_engineering_snapshot( + self.workspace, + workspace_id=self.snapshot["workspaceId"], + cause=f"flow_step.{getattr(state, 'value', str(state)).lower()}", + ) + + +def event_sink_for_workspace(workspace: Any) -> _EngineeringCommitSink | None: + if getattr(workspace, "directory", None): + return _EngineeringCommitSink(workspace) + return None + + +def execution_observer(event_sink: Any) -> ExecutionObserver: + return ExecutionObserver(event_sink) + + +def invoke_engine(callback, *args, rerun: bool = False, observer: Any = None): + return _invoke(callback, *args, rerun=rerun, observer=observer) + + +def _invoke(callback, *args, rerun: bool, observer: Any): + parameters = inspect.signature(callback).parameters.values() + accepts_kwargs = any( + parameter.kind == inspect.Parameter.VAR_KEYWORD for parameter in parameters + ) + kwargs = {} + names = {parameter.name for parameter in parameters} + if accepts_kwargs or "rerun" in names: + kwargs["rerun"] = rerun + if observer is not None and (accepts_kwargs or "observer" in names): + kwargs["observer"] = observer + return callback(*args, **kwargs) diff --git a/chipcompiler/engine/flow.py b/chipcompiler/engine/flow.py index 6e13ac7bc..faf748966 100644 --- a/chipcompiler/engine/flow.py +++ b/chipcompiler/engine/flow.py @@ -16,6 +16,24 @@ log_flow, ) from chipcompiler.engine import EngineDB +from chipcompiler.engine.flow_completion import ( + finalize_interrupted_subflow as _finalize_interrupted_subflow, +) +from chipcompiler.engine.flow_completion import ( + normalize_legacy_terminal_state as _normalize_legacy_terminal_state, +) +from chipcompiler.engine.flow_completion import ( + notify_flow_observer as _notify_flow_observer, +) +from chipcompiler.engine.flow_completion import ( + notify_step_completed, +) +from chipcompiler.engine.flow_completion import ( + refresh_qor_report as _refresh_qor_report, +) +from chipcompiler.engine.flow_completion import ( + refresh_signoff_checklist as _refresh_signoff_checklist, +) from chipcompiler.engine.signoff import ( SignoffPackageCollector, SignoffPackageOptions, @@ -516,6 +534,7 @@ def run_steps( """ for workspace_step in self.workspace_steps: + _notify_flow_observer(observer, "raise_if_cancelled") self.workspace.logger.log_section( f"{workspace_step.tool} - begin step - {workspace_step.name}" ) @@ -558,33 +577,6 @@ def run_steps( return True - def _normalize_legacy_terminal_state(self, workspace_step, step_tag): - """Reset stuck terminal states from pre-guard workspaces to Unstart. - - Pre-guard workspaces may have steps stuck in Incomplete/Invalid from - earlier runs, or in the removed terminal Warning state of the - synthesis LEC. Batch resets (_invalidate_suffix, clear_states) handle - rerun paths; this handles the rerun=False resume path. - """ - old_step = self.get_step(name=workspace_step.name, tool=workspace_step.tool) - if old_step is None: - return - persisted = old_step.get("state") - if persisted in { - StateEnum.Imcomplete.value, - StateEnum.Invalid.value, - "Warning", - }: - logger.warning( - "Normalizing legacy %s state '%s' → Unstart before rerun", - step_tag, - persisted, - ) - old_step["state"] = StateEnum.Unstart.value - old_step["runtime"] = "" - old_step["peak memory (mb)"] = 0 - # No self.save() — set_state(Ongoing) below saves. - def run_step( self, workspace_step: WorkspaceStep | str, @@ -608,17 +600,10 @@ def run_step( self.workspace.logger.info("[SKIP] %s already succeeded", step_tag) self.clear_db_engine_after_step(workspace_step, StateEnum.Success) _notify_flow_observer(observer, "on_step_skipped", workspace_step) - try: - from chipcompiler.analysis.qor import refresh_workspace_qor_report - - refresh_workspace_qor_report(self.workspace) - except Exception: - self.workspace.logger.exception( - "[QOR] %s failed to refresh the workspace QoR report after skip", step_tag - ) + _refresh_qor_report(self.workspace, step_tag) return StateEnum.Success - self._normalize_legacy_terminal_state(workspace_step, step_tag) + _normalize_legacy_terminal_state(self, workspace_step, step_tag) # set state ongoing start_time = time.time() @@ -653,6 +638,7 @@ def run_step( ): raise RuntimeError(f"failed to persist ongoing state for {step_tag}") _notify_flow_observer(observer, "on_step_started", workspace_step) + previous_step = deepcopy(flow_step) if flow_step is not None else None execution = execute_tool_step( self.workspace, @@ -748,14 +734,7 @@ def run_step( # artifacts refreshed above, so it runs after they exist; a # failure degrades to a warning like the facts refresh. if state == StateEnum.Success: - try: - from chipcompiler.analysis.qor import refresh_workspace_qor_report - - refresh_workspace_qor_report(self.workspace) - except Exception: - self.workspace.logger.exception( - "[QOR] %s failed to refresh the workspace QoR report", step_tag - ) + _refresh_qor_report(self.workspace, step_tag) except (Exception, SystemExit) as exc: failure_message = record_tool_failure(self.workspace.logger, step_tag, exc) step_error = step_error or failure_message @@ -784,12 +763,13 @@ def run_step( except (Exception, SystemExit): logger.exception("Failed to release DB engine after %s", step_tag) if terminal_persisted: - _notify_flow_observer( + notify_step_completed( + self, observer, - "on_step_completed", workspace_step, state, step_error, + previous_step, ) self.workspace.logger.info( @@ -799,13 +779,6 @@ def run_step( runtime, peak_memory_mb, ) - if state == StateEnum.Success and not _wait_for_step_rendered( - observer, - workspace_step, - state, - ): - return StateEnum.Invalid - return state def init_db_engine_for_step(self, workspace_step: WorkspaceStep) -> bool: @@ -821,70 +794,3 @@ def init_db_engine_for_step(self, workspace_step: WorkspaceStep) -> bool: return True return self.engine_db.create_db_engine(step=workspace_step) - - -def _finalize_interrupted_subflow( - observer, - workspace_step: WorkspaceStep, - runtime: str, - peak_memory_mb: float, -) -> None: - try: - from chipcompiler.runtime.subflow_events import finalize_interrupted_subflow - - for subflow_step in finalize_interrupted_subflow( - workspace_step, - runtime, - peak_memory_mb, - ): - _notify_flow_observer( - observer, - "on_subflow_stage", - workspace_step, - subflow_step, - ) - except (Exception, SystemExit): - logger.exception("Failed to finalize subflow after %s", workspace_step.name) - - -def _refresh_signoff_checklist(workspace: Workspace, workspace_step: WorkspaceStep) -> None: - """Replace step/home checklists after the step's terminal flow state is saved.""" - try: - from chipcompiler.tools.ecc.signoff_checklist import refresh_step_checklist - - refresh_step_checklist(workspace, workspace_step) - except (Exception, SystemExit): - logger.exception( - "Failed to refresh signoff checklist after %s/%s", - workspace_step.name, - workspace_step.tool, - ) - - -def _notify_flow_observer(observer, method_name: str, *args) -> None: - """Keep optional GUI observers outside the flow engine's failure domain.""" - if observer is None: - return - callback = getattr(observer, method_name, None) - if not callable(callback): - return - try: - callback(*args) - except (Exception, SystemExit): - # Runtime observers must never turn a completed tool execution into a - # failed flow. The coordinator records transport failures separately. - logging.getLogger(__name__).exception("flow observer callback failed: %s", method_name) - - -def _wait_for_step_rendered(observer, workspace_step: WorkspaceStep, state: StateEnum) -> bool: - if observer is None or state != StateEnum.Success: - return True - callback = getattr(observer, "wait_for_step_rendered", None) - if not callable(callback): - return True - try: - return bool(callback(workspace_step, state)) - except Exception: - # Fail-open: observer bugs must not invalidate successful tool results. - logging.getLogger(__name__).exception("flow observer render gate failed") - return True diff --git a/chipcompiler/engine/flow_completion.py b/chipcompiler/engine/flow_completion.py new file mode 100644 index 000000000..d99dc9624 --- /dev/null +++ b/chipcompiler/engine/flow_completion.py @@ -0,0 +1,96 @@ +"""Shared EngineFlow completion hooks.""" + +import logging +from copy import deepcopy + +logger = logging.getLogger(__name__) + + +def normalize_legacy_terminal_state(flow, workspace_step, step_tag) -> None: + """Reset terminal states from pre-guard workspaces before rerun.""" + old_step = flow.get_step(name=workspace_step.name, tool=workspace_step.tool) + if old_step is None: + return + persisted = old_step.get("state") + if persisted in {"Incomplete", "Invalid", "Warning"}: + logger.warning( + "Normalizing legacy %s state '%s' -> Unstart", + step_tag, + persisted, + ) + old_step["state"] = "Unstart" + old_step["runtime"] = "" + old_step["peak memory (mb)"] = 0 + + +def finalize_interrupted_subflow(observer, workspace_step, runtime, peak_memory_mb) -> None: + try: + from chipcompiler.runtime.subflow_events import finalize_interrupted_subflow as finalize + + for subflow_step in finalize(workspace_step, runtime, peak_memory_mb): + notify_flow_observer(observer, "on_subflow_stage", workspace_step, subflow_step) + except (Exception, SystemExit): + logger.exception("Failed to finalize subflow after %s", workspace_step.name) + + +def refresh_signoff_checklist(workspace, workspace_step) -> None: + """Replace step/home checklists after the step's terminal flow state is saved.""" + try: + from chipcompiler.tools.ecc.signoff_checklist import refresh_step_checklist + + refresh_step_checklist(workspace, workspace_step) + except (Exception, SystemExit): + logger.exception( + "Failed to refresh signoff checklist after %s/%s", + workspace_step.name, + workspace_step.tool, + ) + + +def refresh_qor_report(workspace, step_tag: str) -> None: + """Refresh the inspection report without changing flow completion policy.""" + try: + from chipcompiler.analysis.qor import refresh_workspace_qor_report + + refresh_workspace_qor_report(workspace) + except Exception: + workspace.logger.exception("[QOR] %s failed to refresh the workspace QoR report", step_tag) + + +def notify_flow_observer(observer, method_name: str, *args) -> None: + """Keep optional GUI observers outside the flow engine's failure domain.""" + if observer is None: + return + callback = getattr(observer, method_name, None) + if not callable(callback): + return + try: + callback(*args) + except (Exception, SystemExit): + if getattr(observer, "fatal_observer", False): + raise + logger.exception("flow observer callback failed: %s", method_name) + + +def notify_step_completed( + flow, + observer, + workspace_step, + state, + error, + previous_step: dict | None, +) -> None: + """Publish completion and restore Ongoing state when a fatal commit fails.""" + try: + notify_flow_observer(observer, "on_step_completed", workspace_step, state, error) + except (Exception, SystemExit): + if previous_step is not None: + current = flow.get_step(workspace_step.name, workspace_step.tool) + if current is not None: + current.clear() + current.update(deepcopy(previous_step)) + if not flow.save(): + raise RuntimeError( + "failed to roll back flow state after completion commit failure" + ) from None + raise diff --git a/chipcompiler/engine/pdk_binding.py b/chipcompiler/engine/pdk_binding.py new file mode 100644 index 000000000..f26134ac5 --- /dev/null +++ b/chipcompiler/engine/pdk_binding.py @@ -0,0 +1,64 @@ +import hashlib +import json +from pathlib import Path +from typing import Any + + +def pdk_binding_content_hash( + spec: dict[str, Any], + bindings: dict[str, Any], +) -> str: + digest = hashlib.sha256() + digest.update(str(spec.get("familyId", "")).encode()) + digest.update(str(spec.get("version") or bindings.get("version") or "").encode()) + digest.update(json.dumps(spec.get("overrides", {}), sort_keys=True).encode()) + for path in _selected_paths(spec, bindings): + digest.update(path.read_bytes()) + return digest.hexdigest() + + +def _selected_paths(spec: dict[str, Any], bindings: dict[str, Any]) -> list[Path]: + if spec.get("mode") == "manual": + files = bindings.get("files") + if not isinstance(files, dict): + return [] + return [ + Path(files[ref["fileId"]]) + for ref in spec.get("files", []) + if isinstance(ref, dict) + and isinstance(ref.get("fileId"), str) + and ref["fileId"] in files + and Path(files[ref["fileId"]]).is_file() + ] + + try: + from chipcompiler.data import get_pdk + + pdk = get_pdk( + spec.get("familyId", ""), + pdk_root=bindings.get("root", ""), + overrides=pdk_overrides_from_binding(spec, bindings), + ) + except (OSError, ValueError): + return [] + candidates = [pdk.tech, *pdk.lefs, *pdk.libs, pdk.mapping_file] + return [Path(path) for path in candidates if path is not None and Path(path).is_file()] + + +def pdk_overrides_from_binding(spec: dict[str, Any], bindings: dict[str, Any]) -> dict[str, Any]: + overrides = dict(spec.get("overrides", {})) + files = bindings.get("files", {}) + by_role: dict[str, list[str]] = {} + for ref in spec.get("files", []): + path = files.get(ref.get("fileId")) if isinstance(files, dict) else None + if isinstance(path, str): + by_role.setdefault(ref["role"], []).append(path) + if by_role.get("tech"): + overrides["tech"] = by_role["tech"][0] + if by_role.get("lef"): + overrides["lefs"] = by_role["lef"] + if by_role.get("liberty"): + overrides["libs"] = by_role["liberty"] + if by_role.get("mapping"): + overrides["mapping_file"] = by_role["mapping"][0] + return overrides diff --git a/chipcompiler/engine/qor.py b/chipcompiler/engine/qor.py new file mode 100644 index 000000000..c54788cb2 --- /dev/null +++ b/chipcompiler/engine/qor.py @@ -0,0 +1,129 @@ +"""Assemble Snapshot `qorAssessment` from committed analysis. + +Scoring rules live in `qor_scoring`; this module only validates metric +records, calls `score_qor`, and attaches gate status from step summaries. +""" + +import math +from typing import Any + +from .qor_scoring import ( + DIMENSION_WEIGHTS, + QOR_SCORE_THRESHOLD, + QorScoringMetric, + score_qor, +) + + +def build_workspace_qor_assessment(analysis: dict[str, Any]) -> dict[str, Any]: + metrics = [] + step_summaries = [] + metric_steps = [] + for step in analysis["steps"]: + if step["flowState"] != "Success": + continue + step_id = step["stepId"] + metrics_file = step["metrics"] + payload = metrics_file["data"] if metrics_file["status"] == "available" else {} + records = payload.get("metrics") + valid_records = [record for record in records or [] if _valid_metric(record)] + metrics.extend(valid_records) + metric_steps.extend((step_id, record) for record in valid_records) + summary_file = step["summary"] + summary = summary_file["data"] if summary_file["status"] == "available" else {} + summary_status = ( + str(summary.get("quality_status", "incomplete")) + if summary.get("schema_version") == 4 + else "unavailable" + ) + step_summaries.append( + { + "stepId": step_id, + "order": step["order"], + "name": step_id, + "status": summary_status, + "summaryMetricCount": len(valid_records), + } + ) + + if not metrics: + return { + "status": "unavailable", + "score": {"value": None, "threshold": QOR_SCORE_THRESHOLD, "gate": "unavailable"}, + "areaScoringStep": None, + "dimensionScores": {}, + "metrics": [], + "steps": step_summaries, + } + + gate = _gate_status(step_summaries) + scoring = score_qor( + [ + QorScoringMetric( + step=step_id, + metric_id=record["id"], + value=float(record["value"]), + dimension=record["category"], + direction=record["direction"], + scope=record["scope"], + corner=record.get("corner"), + project_role=record["project_role"], + rating_score=record["rating"]["score"], + ) + for step_id, record in metric_steps + ] + ) + return { + "status": "ready", + "score": { + "value": scoring.overall_score, + "threshold": QOR_SCORE_THRESHOLD, + "gate": gate, + }, + "areaScoringStep": scoring.area_scoring_step, + "dimensionScores": { + dimension: score for dimension, (score, _count) in scoring.dimensions.items() + }, + "metrics": metrics, + "steps": step_summaries, + } + + +def _valid_metric(record: Any) -> bool: + if not isinstance(record, dict): + return False + value = record.get("value") + rating = record.get("rating") + corner = record.get("corner") + corner_context = record.get("corner_context") + return ( + isinstance(record.get("id"), str) + and isinstance(record.get("display_name"), str) + and isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and record.get("category") in DIMENSION_WEIGHTS + and record.get("direction") + in {"higher_is_better", "lower_is_better", "target_range", "trend_only"} + and isinstance(record.get("scope"), str) + and (corner is None or isinstance(corner, str)) + and (corner_context is None or isinstance(corner_context, dict)) + and isinstance(record.get("analysis_group"), str) + and isinstance(rating, dict) + and isinstance(rating.get("gate"), bool) + and isinstance(rating.get("score"), bool) + and isinstance(rating.get("trend"), bool) + and record.get("project_role") in {"final", "trend", "gate", "none"} + and record.get("step_role") in {"primary", "secondary", "detail", "hidden"} + and record.get("confidence") in {"high", "medium", "low"} + and isinstance(record.get("source"), dict) + ) + + +def _gate_status(steps: list[dict[str, Any]]) -> str: + statuses = {step["status"] for step in steps} + if "blocked" in statuses: + return "blocked" + if statuses & {"incomplete", "unavailable"}: + return "incomplete" + return "pass" diff --git a/chipcompiler/engine/qor_scoring.py b/chipcompiler/engine/qor_scoring.py new file mode 100644 index 000000000..d3de78d53 --- /dev/null +++ b/chipcompiler/engine/qor_scoring.py @@ -0,0 +1,200 @@ +"""Shared QoR scoring rules. + +Owns fail thresholds, dimension weights, metric selection, and overall +score. Callers: Snapshot assessment (`qor.py`) and CLI report +(`qor_report.py`). Studio GUI consumes the Snapshot result; it does not +keep a second rule table. +""" + +from dataclasses import dataclass +from typing import Any + +QOR_SCORE_THRESHOLD = 60 + +DIMENSION_WEIGHTS = { + "timing": 0.35, + "power_integrity": 0.25, + "routability_physical": 0.2, + "area_cost": 0.1, + "clock_robustness_dfm": 0.1, + "runtime": 0.0, +} + +METRIC_FAIL_VALUES = { + "drc_count": 10, + "lvs_count": 10, + "route_wirelength": 6000, + "route_via_count": 2000, + "cts_buffer_count": 20, + "cts_buffer_area": 40, + "clock_wirelength": 400000, + "cts_clock_wirelength_max": 100000, + "cts_clock_tree_max_level": 20, + "die_area": 3000, + "core_area": 2500, + "core_utilization": 0.85, + "synthesis_cell_area": 3000, + "fanout_max": 100, + "place_hpwl": 10000, + "place_grwl": 12000, + "place_flute_wirelength": 10000, + "place_congestion_egr_overflow_total": 100, + "place_congestion_egr_overflow_max": 20, + "place_rudy_utilization_max": 1, + "place_lutrudy_utilization_max": 1, + "route_dr_total_violation_count": 50, + "route_dr_total_patch_count": 100, + "route_dr_total_wirelength": 6000, + "route_dr_total_via_count": 2000, + "route_la_total_overflow": 100, + "rcx_missing_corner_count": 9, + "sta_setup_wns": -0.2, + "sta_setup_tns": -1, + "sta_hold_wns": -0.2, + "sta_hold_tns": -1, + "sta_frequency_mhz": 100, + "sta_setup_violation_count": 1, + "sta_hold_violation_count": 1, + "sta_missing_corner_count": 1, + "harden_artifact_missing_count": 6, +} + +_SLACK_METRICS = {"sta_setup_wns", "sta_setup_tns", "sta_hold_wns", "sta_hold_tns"} +_ROLE_PRIORITY = {"final": 0, "gate": 1, "trend": 2, "none": 3} +CORE_UTILIZATION_TARGET = (0.45, 0.70) + + +@dataclass(frozen=True) +class QorScoringMetric: + step: str + metric_id: str + value: float + dimension: str + direction: str + scope: str + corner: str | None + project_role: str + rating_score: bool + payload: Any = None + + +@dataclass(frozen=True) +class ScoredQorMetric: + metric: QorScoringMetric + score: float | None + + +@dataclass(frozen=True) +class QorScoringResult: + area_scoring_step: str | None + metrics: tuple[ScoredQorMetric, ...] + dimensions: dict[str, tuple[float, int]] + overall_score: float | None + + +def score_qor( + records: list[QorScoringMetric], *, flow_order: tuple[str, ...] = () +) -> QorScoringResult: + area_step = _area_scoring_step(records, flow_order) + selected: dict[tuple[str, str, str], tuple[int, int, QorScoringMetric]] = {} + for record in records: + if record.project_role == "none": + continue + if record.dimension == "area_cost" and record.step != area_step: + continue + key = (record.metric_id, record.scope, record.corner or "") + candidate = ( + _ROLE_PRIORITY.get(record.project_role, 3), + -_step_rank(record.step, flow_order), + record, + ) + if key not in selected or candidate[:2] < selected[key][:2]: + selected[key] = candidate + + scored = tuple( + ScoredQorMetric(record, score_metric(record) if record.rating_score else None) + for _role, _rank, record in sorted(selected.values(), key=lambda item: item[2].metric_id) + ) + by_dimension: dict[str, list[float]] = {} + for item in scored: + if item.score is not None: + by_dimension.setdefault(item.metric.dimension, []).append(item.score) + dimensions = { + dimension: ( + round(sum(by_dimension[dimension]) / len(by_dimension[dimension]), 1), + len(by_dimension[dimension]), + ) + for dimension in DIMENSION_WEIGHTS + if dimension in by_dimension + } + weighted = sum( + score * DIMENSION_WEIGHTS[dimension] + for dimension, (score, _count) in dimensions.items() + if DIMENSION_WEIGHTS[dimension] > 0 + ) + overall = ( + round(weighted, 1) + if any(DIMENSION_WEIGHTS[dimension] > 0 for dimension in dimensions) + else None + ) + return QorScoringResult(area_step, scored, dimensions, overall) + + +def score_metric(record: QorScoringMetric) -> float | None: + if record.direction == "trend_only": + return None + fail = METRIC_FAIL_VALUES.get(record.metric_id) + if fail is None: + return None + if record.metric_id in _SLACK_METRICS: + if fail >= 0: + return None + return 100.0 if record.value >= 0 else _clamp(100 * (record.value - fail) / -fail) + if record.direction == "target_range": + if record.metric_id != "core_utilization": + return None + low, high = CORE_UTILIZATION_TARGET + if low <= record.value <= high: + return 100.0 + if record.value < low: + return _clamp(100 * record.value / low) + return _clamp(100 * (fail - record.value) / (fail - high)) + if fail <= 0: + return None + if record.direction == "lower_is_better": + return _clamp(100 * (fail - record.value) / fail) + if record.direction == "higher_is_better": + return _clamp(100 * record.value / fail) + return None + + +def _area_scoring_step(records: list[QorScoringMetric], flow_order: tuple[str, ...]) -> str | None: + if flow_order: + by_step = { + record.step + for record in records + if record.dimension == "area_cost" and record.rating_score + } + for step in reversed(flow_order): + if step in by_step: + return step + return None + return next( + ( + record.step + for record in reversed(records) + if record.dimension == "area_cost" and record.rating_score + ), + None, + ) + + +def _step_rank(step: str, flow_order: tuple[str, ...]) -> int: + try: + return flow_order.index(step) + except ValueError: + return -1 + + +def _clamp(value: float) -> float: + return max(0.0, min(100.0, value)) diff --git a/chipcompiler/engine/reconcile.py b/chipcompiler/engine/reconcile.py index addf79db3..1eb77e990 100644 --- a/chipcompiler/engine/reconcile.py +++ b/chipcompiler/engine/reconcile.py @@ -33,7 +33,6 @@ when the flow is compatible but an append/adopt is due. """ -import fcntl import logging from contextlib import contextmanager from dataclasses import dataclass, field @@ -123,15 +122,10 @@ def _persisted_flow_data(workspace_dir: Path, json_read) -> dict: @contextmanager def _workspace_lock(workspace_dir: Path): - # The lock lives NEXT TO the workspace (never inside it): an overwrite - # deleting the tree cannot invalidate the lock's inode, so a waiter - # always serializes against the run that replaces the directory. - lock_path = workspace_dir.parent / f"{workspace_dir.name}.lock" - workspace_dir.parent.mkdir(parents=True, exist_ok=True) - with open(lock_path, "a") as lock_file: - fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + from chipcompiler.utility.workspace_lock import workspace_lock + + with workspace_lock(workspace_dir): yield - fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) def resolve_target_section(project_flow: dict | None, workspace_flow: dict | None) -> dict: diff --git a/chipcompiler/engine/rerun.py b/chipcompiler/engine/rerun.py index 4de57a694..080d56670 100644 --- a/chipcompiler/engine/rerun.py +++ b/chipcompiler/engine/rerun.py @@ -177,29 +177,88 @@ def _invalidate_suffix(flow: "EngineFlow", index: int, last_index: int | None = def _run_selected(flow: "EngineFlow", selected: list[tuple[WorkspaceStep, Path]]) -> StepRunResult: - """Run the selected steps in order, each with a fresh output directory.""" + """Run selected steps through the shared Engine execution contract.""" # a new selection must not inherit a DB positioned by an earlier run if flow.engine_db is not None: flow.engine_db.close() - executed = [] - for workspace_step, output_dir in selected: - flow.workspace.logger.log_section( - f"{workspace_step.tool} - begin step - {workspace_step.name}" - ) - _reset_output_dir(output_dir) - _redirect_to_step_log(workspace_step) - flow.init_db_engine_for_step(workspace_step) - state = flow.run_step(workspace_step, rerun=True) - log_flow(workspace=flow.workspace) - flow.workspace.logger.log_section( - f"{workspace_step.tool} - end step - {workspace_step.name}" - ) - if state is not StateEnum.Success: - # A persisted Incomplete blocks the rerun. - return StepRunResult(ok=False, executed=tuple(executed), failed=workspace_step.name) - executed.append(workspace_step.name) - return StepRunResult(ok=True, executed=tuple(executed)) + output_dirs = {workspace_step.name: output_dir for workspace_step, output_dir in selected} + + class PreparedFlow: + workspace = flow.workspace + + def get_workspace_step(self, step_id): + return flow.get_workspace_step(step_id) + + def run_step(self, workspace_step, *, rerun=False, observer=None): + output_dir = output_dirs[workspace_step.name] + flow.workspace.logger.log_section( + f"{workspace_step.tool} - begin step - {workspace_step.name}" + ) + _reset_output_dir(output_dir) + _redirect_to_step_log(workspace_step) + flow.init_db_engine_for_step(workspace_step) + if _callable_accepts_keyword(flow.run_step, "observer"): + state = flow.run_step(workspace_step, rerun=rerun, observer=observer) + else: + state = flow.run_step(workspace_step, rerun=rerun) + log_flow(workspace=flow.workspace) + flow.workspace.logger.log_section( + f"{workspace_step.tool} - end step - {workspace_step.name}" + ) + return state + + from chipcompiler.engine.execution import ExecutionPlan, execute + + result = execute( + PreparedFlow(), + ExecutionPlan(intent="rerun", step_ids=tuple(output_dirs)), + ) + executed = result.executed_steps or _successful_prefix( + flow, + selected, + succeeded=result.succeeded, + ) + return StepRunResult( + ok=result.succeeded, + executed=executed, + failed=result.failed_step + or ( + None + if result.succeeded + else tuple(output_dirs)[min(len(executed), len(output_dirs) - 1)] + ), + ) + + +def _callable_accepts_keyword(callback, keyword: str) -> bool: + import inspect + + try: + parameters = inspect.signature(callback).parameters.values() + except (TypeError, ValueError): + return False + return any( + parameter.name == keyword or parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in parameters + ) + + +def _successful_prefix(flow, selected, *, succeeded: bool) -> tuple[str, ...]: + if succeeded: + return tuple(step.name for step, _output_dir in selected) + persisted = getattr(getattr(flow.workspace, "flow", None), "data", {}) + states = { + str(step.get("name")): step.get("state") + for step in persisted.get("steps", []) + if isinstance(step, dict) + } + completed = [] + for step, _output_dir in selected: + if states.get(step.name) != StateEnum.Success.value: + break + completed.append(step.name) + return tuple(completed) def _validated_output_dirs(workspace: Workspace, steps: list[WorkspaceStep]) -> list[Path]: diff --git a/chipcompiler/engine/signoff/collector.py b/chipcompiler/engine/signoff/collector.py index ba4fac133..a34c7cd6f 100644 --- a/chipcompiler/engine/signoff/collector.py +++ b/chipcompiler/engine/signoff/collector.py @@ -578,6 +578,7 @@ def add_file( checklist_data = rebuild_home_checklist( self.workspace, resource_issues=[*issues, *analysis_issues], + persist=options.materialize, ) add_file( "status.checklist", diff --git a/chipcompiler/engine/signoff_assessment.py b/chipcompiler/engine/signoff_assessment.py new file mode 100644 index 000000000..35c71bf03 --- /dev/null +++ b/chipcompiler/engine/signoff_assessment.py @@ -0,0 +1,188 @@ +from pathlib import Path +from typing import Any, TypedDict + +from chipcompiler.utility import json_read + +_REVIEW_GROUPS: tuple[tuple[str, str], ...] = ( + ("initial", "Initial"), + ("config", "Config"), + ("harden", "Harden"), + ("final_design", "Final Design"), + ("sta", "STA"), + ("spef", "SPEF"), + ("reports", "Reports"), +) + + +class _ReviewGroup(TypedDict): + id: str + label: str + available: int + expected: int + blocked: list[dict] + attention: list[dict] + + +def build_signoff_assessment(workspace: Any) -> dict[str, Any]: + flow = getattr(workspace, "flow", None) + steps_fn = getattr(flow, "steps", None) + if callable(steps_fn): + steps = steps_fn() + else: + flow_data = getattr(flow, "data", None) + steps = flow_data.get("steps", []) if isinstance(flow_data, dict) else [] + if steps and any(str(step.get("state", "")) not in {"Success", "Skipped"} for step in steps): + return _unavailable_assessment() + checklist = json_read(Path(workspace.directory) / "home" / "checklist.json") + if ( + not isinstance(checklist, dict) + or checklist.get("schema_version") != 3 + or checklist.get("kind") != "signoff_checklist" + or not isinstance(checklist.get("checklist"), list) + ): + return _unavailable_assessment() + + groups: dict[str, _ReviewGroup] = {} + for group_id, label in _REVIEW_GROUPS: + groups[group_id] = { + "id": group_id, + "label": label, + "available": 0, + "expected": 0, + "blocked": [], + "attention": [], + } + for item in checklist.get("checklist", []): + if not isinstance(item, dict): + continue + group = groups[_group_for(item)] + group["expected"] += 1 + if item.get("state") == "pass": + group["available"] += 1 + elif item.get("blocked") is True: + group["blocked"].append(_detail(item)) + else: + group["attention"].append(_detail(item)) + + review_groups = [] + risks = [] + for group_id, _label in _REVIEW_GROUPS: + group = groups[group_id] + blocked = group["blocked"] + attention = group["attention"] + if blocked: + status = "blocked" + summary = f"{len(blocked)} blocking checklist requirements" + risks.append(_risk(group, "blocked", blocked, summary)) + if attention: + risks.append( + _risk( + group, + "warning", + attention, + f"{len(attention)} attention-only checklist requirements", + ) + ) + elif attention: + status = "attention" + summary = f"{len(attention)} attention-only checklist requirements" + risks.append(_risk(group, "warning", attention, summary)) + else: + status = "ready" + summary = ( + f"{group['available']} of {group['expected']} requirements ready" + if group["expected"] + else "No requirements" + ) + review_groups.append( + { + "id": group_id, + "label": group["label"], + "status": status, + "available": group["available"], + "expected": group["expected"], + "summary": summary, + } + ) + + status = checklist.get("status") + return { + "status": status if status in {"ready", "attention", "blocked"} else "blocked", + "groups": review_groups, + "risks": sorted(risks, key=lambda risk: risk["severity"] != "blocked"), + } + + +def _risk(group: _ReviewGroup, severity: str, details: list[dict], summary: str) -> dict: + result = "requirements block export" if severity == "blocked" else "attention" + return { + "severity": severity, + "title": f"{group['label']} signoff {result}", + "summary": summary, + "details": details, + } + + +def _detail(item: dict) -> dict: + source = item.get("source", {}) + source = source if isinstance(source, dict) else {} + evidence = item.get("evidence", []) + return { + "kind": item.get("category", "checklist"), + "label": item.get("title", "Checklist item"), + "location": source.get("path", "home/checklist.json"), + "reason": item.get("summary", ""), + "owner": item.get("owner", "checklist"), + "policy": item.get("policy", "warn"), + "state": item.get("state", "unavailable"), + "evidence": evidence if isinstance(evidence, list) else [], + } + + +def _group_for(item: dict) -> str: + step = str(item.get("step", "")) + category = str(item.get("category", "")) + source = item.get("source", {}) + path = source.get("path", "") if isinstance(source, dict) else "" + if category == "configuration" or path.startswith("config/"): + return "config" + if category == "provenance" or path.startswith(("origin/", "initial/")): + return "initial" + if step == "Harden" or path.startswith(("Harden_ecc/", "harden/")): + return "harden" + if step == "sta" or path.startswith(("sta_ecc/", "final/timing/sta/")): + return "sta" + if step == "RCX" or path.startswith(("RCX_ecc/", "final/timing/spef/")): + return "spef" + if step in {"Route", "drc", "lvs", "filler"} or path.startswith( + ("route_ecc/", "drc_ecc/", "lvs_ecc/", "filler_ecc/", "final/design/") + ): + return "final_design" + return "reports" + + +def _unavailable_assessment() -> dict[str, Any]: + return { + "status": "blocked", + "groups": [ + { + "id": group_id, + "label": label, + "status": "blocked" if group_id == "reports" else "ready", + "available": 0, + "expected": 0, + "summary": ( + "Checklist unavailable" if group_id == "reports" else "No requirements" + ), + } + for group_id, label in _REVIEW_GROUPS + ], + "risks": [ + { + "severity": "blocked", + "title": "Signoff checklist unavailable", + "summary": "Re-run signoff inspection after current-output analysis completes.", + "details": [], + } + ], + } diff --git a/chipcompiler/engine/signoff_export.py b/chipcompiler/engine/signoff_export.py new file mode 100644 index 000000000..19afb6b1b --- /dev/null +++ b/chipcompiler/engine/signoff_export.py @@ -0,0 +1,116 @@ +import os +import shutil +import tempfile +from pathlib import Path + +from chipcompiler.engine import EngineFlow, SignoffPackageOptions +from chipcompiler.engine.signoff_assessment import build_signoff_assessment + + +class SignoffExportError(RuntimeError): + pass + + +def inspect_signoff_package(workspace) -> dict: + """Return the current Signoff Assessment without mutating the Workspace.""" + return build_signoff_assessment(workspace) + + +def export_signoff_package_archive( + workspace, + output_path: str, + additional_files: list[dict[str, str]] | None = None, + *, + include_debug: bool = False, +) -> str: + raw_destination = Path(output_path).expanduser() + destination = raw_destination.parent.resolve() / raw_destination.name + + with tempfile.TemporaryDirectory(prefix="ecc-signoff-") as temporary_root: + try: + result = EngineFlow(workspace).collect_signoff_package( + SignoffPackageOptions( + output_dir=temporary_root, + archive=False, + include_debug=include_debug, + materialize=False, + refresh_analysis=False, + ) + ) + except (OSError, ValueError, TypeError, KeyError) as exc: + raise SignoffExportError(str(exc)) from exc + if not result.ok: + missing = ", ".join(result.missing_required) or "unknown required resources" + raise SignoffExportError(f"signoff package is incomplete: {missing}") + if not result.package_dir: + raise SignoffExportError("signoff package directory was not created") + + package_dir = Path(result.package_dir) + + if additional_files is not None and not isinstance(additional_files, list): + raise SignoffExportError("additional_files must be a list") + if additional_files: + for file_info in additional_files: + if not isinstance(file_info, dict): + raise SignoffExportError("additional file entries must be objects") + archive_path = file_info.get("archivePath") + content = file_info.get("content") + if not isinstance(archive_path, str) or not archive_path: + raise SignoffExportError("additional file entries require archivePath") + if not isinstance(content, str): + raise SignoffExportError("additional file entries require string content") + try: + path = _additional_file_path(package_dir, archive_path) + path.parent.mkdir(parents=True, exist_ok=True) + if _has_symlink_parent(package_dir, path): + raise SignoffExportError("additional file path contains a symlink") + path.write_text(content, encoding="utf-8") + except SignoffExportError: + raise + except (OSError, ValueError, TypeError) as exc: + raise SignoffExportError(str(exc)) from exc + + archive = package_dir.with_suffix(".tar.gz") + import tarfile + + with tarfile.open(archive, "w:gz") as tar: + tar.add(package_dir, arcname=package_dir.name) + + destination.parent.mkdir(parents=True, exist_ok=True) + descriptor, staged_name = tempfile.mkstemp( + dir=destination.parent, + prefix=f".{destination.name}.", + ) + os.close(descriptor) + staged_path = Path(staged_name) + try: + shutil.copy2(archive, staged_path) + os.replace(staged_path, destination) + finally: + staged_path.unlink(missing_ok=True) + + return str(destination) + + +def _additional_file_path(package_dir: Path, archive_path: str) -> Path: + if not archive_path or "\x00" in archive_path: + raise SignoffExportError("additional file path must name a relative file") + relative = Path(archive_path) + if relative == Path(".") or relative.is_absolute() or ".." in relative.parts: + raise SignoffExportError("additional file path must stay inside the signoff package") + try: + destination = (package_dir / relative).resolve() + except (OSError, ValueError) as exc: + raise SignoffExportError("additional file path must name a relative file") from exc + if not destination.is_relative_to(package_dir.resolve()): + raise SignoffExportError("additional file path must stay inside the signoff package") + return destination + + +def _has_symlink_parent(package_dir: Path, path: Path) -> bool: + current = package_dir + for component in path.relative_to(package_dir).parts: + current /= component + if current.is_symlink(): + return True + return False diff --git a/chipcompiler/engine/snapshot.py b/chipcompiler/engine/snapshot.py new file mode 100644 index 000000000..a1c55b58f --- /dev/null +++ b/chipcompiler/engine/snapshot.py @@ -0,0 +1,436 @@ +import hashlib +from copy import deepcopy +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from chipcompiler.engine.snapshot_qor import ( + build_qor_snapshot_extension, + unavailable_qor_snapshot_extension, + validate_qor_snapshot_extension, +) +from chipcompiler.utility import JsonReadError, file_digest, json_read, json_read_strict, json_write + +SNAPSHOT_SCHEMA_VERSION = 2 +SNAPSHOT_V3_SCHEMA_VERSION = 3 +SUPPORTED_SNAPSHOT_SCHEMA_VERSIONS = frozenset( + {SNAPSHOT_SCHEMA_VERSION, SNAPSHOT_V3_SCHEMA_VERSION} +) +SNAPSHOT_FILENAME = "engineering-snapshot.json" +STALE_SNAPSHOT_FILENAME = "engineering-snapshot.stale.json" +SNAPSHOT_V2_TO_V3_MIGRATION_CAUSE = "snapshot.migrated.v2_to_v3" + + +class EngineeringSnapshotError(RuntimeError): + pass + + +def create_engineering_snapshot( + workspace: Any, + *, + workspace_id: str | None = None, + workspace_revision: int = 1, + cause: str = "workspace.created", +) -> dict[str, Any]: + snapshot = _build_snapshot( + workspace, + workspace_id=workspace_id or f"workspace-{uuid4().hex}", + workspace_revision=workspace_revision, + cause=cause, + ) + _write_snapshot(_snapshot_path(workspace), snapshot) + return snapshot + + +def ensure_engineering_snapshot(workspace: Any) -> dict[str, Any]: + path = _snapshot_path(workspace) + if path.is_file(): + snapshot = _read_snapshot(path) + if snapshot["schemaVersion"] != SNAPSHOT_SCHEMA_VERSION: + raise EngineeringSnapshotError("production Snapshot schema is still v2") + return snapshot + return create_engineering_snapshot(workspace, cause="workspace.migrated") + + +def read_engineering_snapshot( + workspace: Any, + *, + expected_workspace_id: str | None = None, + expected_workspace_revision: int | None = None, + validate_artifacts: bool = True, +) -> dict[str, Any]: + snapshot = _read_snapshot(_snapshot_path(workspace), validate_artifacts=validate_artifacts) + if expected_workspace_id is not None and snapshot["workspaceId"] != expected_workspace_id: + raise EngineeringSnapshotError("Engineering Snapshot workspace identity mismatch") + if ( + expected_workspace_revision is not None + and snapshot["workspaceRevision"] != expected_workspace_revision + ): + raise EngineeringSnapshotError("Engineering Snapshot Workspace Revision mismatch") + return snapshot + + +def read_engineering_snapshot_from_directory(directory: str | Path) -> dict[str, Any]: + return _read_snapshot(Path(directory).expanduser().resolve() / "home" / SNAPSHOT_FILENAME) + + +def read_stale_engineering_snapshot(workspace: Any) -> dict[str, Any] | None: + path = _stale_snapshot_path(workspace) + return _read_snapshot(path, validate_artifacts=False) if path.is_file() else None + + +def migrate_engineering_snapshot( + workspace: Any, + *, + expected_workspace_revision: int | None = None, + cause: str = SNAPSHOT_V2_TO_V3_MIGRATION_CAUSE, +) -> dict[str, Any]: + """Explicitly migrate one v2 Snapshot to the prepared v3 contract. + + Normal Snapshot producers remain pinned to v2. This write-only seam is the + only path that emits v3 until ECC and Studio switch the production contract. + """ + current = _read_snapshot(_snapshot_path(workspace)) + if current["schemaVersion"] != SNAPSHOT_SCHEMA_VERSION: + raise EngineeringSnapshotError("Snapshot migration requires schemaVersion 2") + if ( + expected_workspace_revision is not None + and current["workspaceRevision"] != expected_workspace_revision + ): + raise EngineeringSnapshotError( + "Workspace Revision does not match before Snapshot migration" + ) + try: + snapshot = _build_snapshot( + workspace, + workspace_id=current["workspaceId"], + workspace_revision=current["workspaceRevision"] + 1, + cause=cause, + schema_version=SNAPSHOT_V3_SCHEMA_VERSION, + strict_qor=True, + ) + except Exception as exc: + raise EngineeringSnapshotError( + "failed to regenerate QoR facts for Snapshot migration" + ) from exc + if isinstance(current.get("stalePredecessor"), dict): + snapshot["stalePredecessor"] = deepcopy(current["stalePredecessor"]) + _write_snapshot(_snapshot_path(workspace), snapshot) + return snapshot + + +migrate_engineering_snapshot_v2_to_v3 = migrate_engineering_snapshot + + +def commit_engineering_snapshot( + workspace: Any, + *, + workspace_id: str, + cause: str, +) -> dict[str, Any]: + current = read_engineering_snapshot(workspace, validate_artifacts=False) + if current["schemaVersion"] != SNAPSHOT_SCHEMA_VERSION: + raise EngineeringSnapshotError("production Snapshot schema is still v2") + if current["workspaceId"] != workspace_id: + raise EngineeringSnapshotError("Workspace identity changed before commit") + snapshot = _build_snapshot( + workspace, + workspace_id=workspace_id, + workspace_revision=current["workspaceRevision"] + 1, + cause=cause, + ) + stale = current.get("stalePredecessor") + if isinstance(stale, dict): + states = { + str(step.get("name")): step.get("state") + for step in snapshot.get("flow", {}).get("steps", []) + if isinstance(step, dict) and step.get("name") + } + remaining = [ + step_id + for step_id in stale.get("invalidatedStepIds", []) + if states.get(step_id) not in {"Success", "Skipped"} + ] + if remaining: + snapshot["stalePredecessor"] = {**deepcopy(stale), "invalidatedStepIds": remaining} + _write_snapshot(_snapshot_path(workspace), snapshot) + if isinstance(stale, dict) and "stalePredecessor" not in snapshot: + _stale_snapshot_path(workspace).unlink(missing_ok=True) + return snapshot + + +def invalidate_engineering_snapshot( + workspace: Any, + *, + workspace_id: str, + cause: str, + first_invalidated_step: str | None = None, +) -> dict[str, Any]: + current = read_engineering_snapshot(workspace, validate_artifacts=False) + if current["schemaVersion"] != SNAPSHOT_SCHEMA_VERSION: + raise EngineeringSnapshotError("production Snapshot schema is still v2") + if current["workspaceId"] != workspace_id: + raise EngineeringSnapshotError("Workspace identity changed before invalidation") + flow = deepcopy(current.get("flow", {})) + steps = flow.get("steps", []) if isinstance(flow, dict) else [] + invalidated_index = 0 + if first_invalidated_step is not None: + invalidated_index = next( + ( + index + for index, step in enumerate(steps) + if isinstance(step, dict) + and str(step.get("name", "")).casefold() == first_invalidated_step.casefold() + ), + -1, + ) + if invalidated_index < 0: + raise EngineeringSnapshotError(f"Flow Step not found: {first_invalidated_step}") + for step in steps[invalidated_index:]: + if isinstance(step, dict): + step["state"] = "Unstart" + step.pop("runtime", None) + step.pop("peak memory (mb)", None) + stale_path = _stale_snapshot_path(workspace) + if not stale_path.is_file(): + _write_snapshot(stale_path, current) + invalidated = [ + str(step["name"]) + for step in steps[invalidated_index:] + if isinstance(step, dict) and step.get("name") + ] + snapshot = _build_snapshot( + workspace, + workspace_id=workspace_id, + workspace_revision=current["workspaceRevision"] + 1, + cause=cause, + ) + snapshot["flow"] = flow + snapshot["stalePredecessor"] = { + "workspaceRevision": current["workspaceRevision"], + "invalidatedStepIds": invalidated, + } + _write_snapshot(_snapshot_path(workspace), snapshot) + return snapshot + + +def _build_snapshot( + workspace: Any, + *, + workspace_id: str, + workspace_revision: int, + cause: str, + schema_version: int = SNAPSHOT_SCHEMA_VERSION, + strict_qor: bool = False, +) -> dict[str, Any]: + flow_owner = getattr(workspace, "flow", None) + flow = _data_mapping(flow_owner) + if not flow and flow_owner is not None: + steps = flow_owner.steps() + flow = {"steps": deepcopy(steps)} if steps else {} + home = _data_mapping(getattr(workspace, "home", None)) + checklist_path = home.get("checklist") + checklist = json_read(checklist_path) if isinstance(checklist_path, (str, Path)) else {} + from chipcompiler.engine.analysis import build_workspace_analysis + from chipcompiler.engine.qor import build_workspace_qor_assessment + from chipcompiler.engine.signoff_assessment import build_signoff_assessment + + analysis, artifacts = build_workspace_analysis(workspace, workspace_id) + qor_assessment = build_workspace_qor_assessment(analysis) + try: + from chipcompiler.analysis.qor import build_qor_analysis + + qor_extension = build_qor_snapshot_extension( + build_qor_analysis(workspace), + artifacts, + ) + except Exception as exc: + if strict_qor: + raise + qor_extension = unavailable_qor_snapshot_extension(str(exc)) + return { + "schemaVersion": schema_version, + "workspaceId": workspace_id, + "workspaceRevision": workspace_revision, + "cause": cause, + "flow": flow, + "parameters": _data_mapping(getattr(workspace, "parameters", None)), + "checklist": checklist if isinstance(checklist, dict) else {}, + "analysis": analysis, + "metrics": deepcopy(qor_assessment["metrics"]), + "qorAssessment": qor_assessment, + "qorSnapshotExtension": qor_extension, + "signoffAssessment": build_signoff_assessment(workspace), + "artifacts": artifacts, + } + + +def _snapshot_path(workspace: Any) -> Path: + return Path(workspace.directory) / "home" / SNAPSHOT_FILENAME + + +def _stale_snapshot_path(workspace: Any) -> Path: + return Path(workspace.directory) / "home" / STALE_SNAPSHOT_FILENAME + + +def _data_mapping(owner: Any) -> dict[str, Any]: + data = getattr(owner, "data", {}) + return deepcopy(data) if isinstance(data, dict) else {} + + +def _write_snapshot(path: Path, snapshot: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not json_write(path, snapshot): + raise EngineeringSnapshotError(f"failed to persist Engineering Snapshot: {path}") + + +def _read_snapshot(path: Path, *, validate_artifacts: bool = True) -> dict[str, Any]: + try: + snapshot = json_read_strict(path) + except (OSError, JsonReadError) as exc: + raise EngineeringSnapshotError(f"invalid Engineering Snapshot: {path}") from exc + if ( + not isinstance(snapshot, dict) + or snapshot.get("schemaVersion") not in SUPPORTED_SNAPSHOT_SCHEMA_VERSIONS + or not isinstance(snapshot.get("workspaceId"), str) + or not snapshot["workspaceId"] + or isinstance(snapshot.get("workspaceRevision"), bool) + or not isinstance(snapshot.get("workspaceRevision"), int) + or snapshot["workspaceRevision"] < 1 + or ( + snapshot.get("schemaVersion") == SNAPSHOT_V3_SCHEMA_VERSION + and not validate_qor_snapshot_extension(snapshot.get("qorSnapshotExtension")) + ) + ): + raise EngineeringSnapshotError(f"invalid Engineering Snapshot: {path}") + _validate_snapshot_sections( + snapshot, + path.parent.parent, + validate_artifacts=validate_artifacts, + ) + return snapshot + + +def _validate_snapshot_sections( + snapshot: dict[str, Any], + workspace_root: Path, + *, + validate_artifacts: bool, +) -> None: + for key in ( + "flow", + "parameters", + "checklist", + "analysis", + "qorAssessment", + "signoffAssessment", + ): + if not isinstance(snapshot.get(key), dict): + raise EngineeringSnapshotError(f"invalid Engineering Snapshot section: {key}") + if "steps" in snapshot["flow"] and not isinstance(snapshot["flow"]["steps"], list): + raise EngineeringSnapshotError("invalid Engineering Snapshot section: flow.steps") + if "steps" in snapshot["analysis"] and not isinstance(snapshot["analysis"]["steps"], list): + raise EngineeringSnapshotError("invalid Engineering Snapshot section: analysis.steps") + artifacts = snapshot.get("artifacts") + if not isinstance(artifacts, list) or len(artifacts) > 4096: + raise EngineeringSnapshotError("invalid Engineering Snapshot section: artifacts") + workspace_root = workspace_root.resolve() + metadata_id = _workspace_metadata_id(workspace_root) + if metadata_id is not None and metadata_id != snapshot["workspaceId"]: + raise EngineeringSnapshotError("Engineering Snapshot workspace identity mismatch") + for artifact in artifacts: + _validate_snapshot_artifact( + artifact, + snapshot["workspaceId"], + workspace_root, + validate_artifacts=validate_artifacts, + ) + stale = snapshot.get("stalePredecessor") + if stale is not None and ( + not isinstance(stale, dict) + or type(stale.get("workspaceRevision")) is not int + or stale["workspaceRevision"] < 1 + or not isinstance(stale.get("invalidatedStepIds"), list) + or not all(isinstance(step_id, str) and step_id for step_id in stale["invalidatedStepIds"]) + ): + raise EngineeringSnapshotError("invalid Engineering Snapshot stale predecessor") + + +def _validate_snapshot_artifact( + artifact: object, + workspace_id: str, + workspace_root: Path, + *, + validate_artifacts: bool, +) -> None: + if not isinstance(artifact, dict): + raise EngineeringSnapshotError("invalid Engineering Snapshot artifact") + artifact_id = artifact.get("artifactId") + reference = artifact.get("reference") + availability = artifact.get("availability") + if ( + not isinstance(artifact_id, str) + or not isinstance(reference, str) + or not reference + or Path(reference).is_absolute() + or ".." in Path(reference).parts + or artifact_id != _artifact_id(workspace_id, reference) + or availability not in {"missing", "available", "stale"} + ): + raise EngineeringSnapshotError("invalid Engineering Snapshot artifact reference") + if availability != "available" or not validate_artifacts: + return + digest = artifact.get("sha256") + size = artifact.get("sizeBytes") + if ( + not isinstance(digest, str) + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest.lower()) + or type(size) is not int + or size < 0 + ): + raise EngineeringSnapshotError("invalid Engineering Snapshot artifact fingerprint") + candidate = workspace_root / reference + try: + candidate.relative_to(workspace_root) + except ValueError as exc: + raise EngineeringSnapshotError("invalid Engineering Snapshot artifact path") from exc + if _contains_symlink(candidate, workspace_root): + raise EngineeringSnapshotError("invalid Engineering Snapshot artifact path") + if file_digest(candidate) != (digest, size): + raise EngineeringSnapshotError("Engineering Snapshot artifact fingerprint mismatch") + + +def _contains_symlink(path: Path, root: Path) -> bool: + current = path + while current != root: + if current.is_symlink() or current.parent == current: + return True + current = current.parent + return root.is_symlink() + + +def _workspace_metadata_id(workspace_root: Path) -> str | None: + command_path = workspace_root / "home" / "workspace-commands.json" + try: + payload = json_read_strict(command_path) + except (OSError, JsonReadError): + return None + commands = payload.get("commands") if isinstance(payload, dict) else None + if not isinstance(commands, dict): + return None + workspace_ids = { + result.get("workspaceId") + for command in commands.values() + if isinstance(command, dict) + and isinstance(result := command.get("result"), dict) + and isinstance(result.get("workspaceId"), str) + } + if len(workspace_ids) > 1: + raise EngineeringSnapshotError("invalid Workspace command identity metadata") + return next(iter(workspace_ids), None) + + +def _artifact_id(workspace_id: str, reference: str) -> str: + digest = hashlib.sha256(f"{workspace_id}\0{reference}".encode()).hexdigest() + return f"artifact-{digest[:32]}" diff --git a/chipcompiler/engine/snapshot_qor.py b/chipcompiler/engine/snapshot_qor.py new file mode 100644 index 000000000..d4ac7883c --- /dev/null +++ b/chipcompiler/engine/snapshot_qor.py @@ -0,0 +1,414 @@ +"""Bounded QoR v3 facts embedded in an Engineering Snapshot.""" + +from math import isfinite +from typing import Any + +from chipcompiler.analysis.qor.schema import ( + CONFIDENCES, + DIMENSION_KEYS, + DIMENSION_STATES, + EVIDENCE_STATES, + FEASIBILITY_STATUSES, + GATE_STATES, + SCALAR_STATUSES, + TIERS, +) + +QOR_SNAPSHOT_EXTENSION_SCHEMA_VERSION = 1 +_MAX_DIAGNOSES = 64 +_MAX_INTERVENTIONS = 4 +_MAX_ARTIFACT_IDS = 512 +_MAX_TEXT = 512 + + +def build_qor_snapshot_extension(analysis: Any, artifacts: list[dict[str, Any]]) -> dict[str, Any]: + """Project the report into bounded, path-free committed QoR facts.""" + report = analysis.to_dict() if hasattr(analysis, "to_dict") else analysis + if not isinstance(report, dict): + raise ValueError("QoR analysis is not an object") + + dimensions = {} + for key, dimension in report.get("qor_record", {}).items(): + if not isinstance(key, str) or not isinstance(dimension, dict): + continue + features = dimension.get("features", []) + dimensions[key] = { + "value": _number_or_none(dimension.get("value")), + "state": _text(dimension.get("state"), "UNKNOWN"), + "featureIds": [ + feature["feature_id"] + for feature in features[:32] + if isinstance(feature, dict) and isinstance(feature.get("feature_id"), str) + ], + } + + feasibility = report.get("feasibility") + feasibility = feasibility if isinstance(feasibility, dict) else {} + gates = [] + for gate in feasibility.get("gates", []): + if not isinstance(gate, dict) or not isinstance(gate.get("id"), str): + continue + gates.append( + { + "id": gate["id"], + "stage": _text(gate.get("stage")), + "state": _text(gate.get("state"), "unavailable"), + "blocksTapeout": bool(gate.get("blocks_tapeout")), + "metrics": [ + metric for metric in gate.get("metrics", [])[:32] if isinstance(metric, str) + ], + "availability": gate.get("availability") + if isinstance(gate.get("availability"), str) + else None, + } + ) + + summary = report.get("scalar_summary") + summary = summary if isinstance(summary, dict) else {} + evidence = report.get("evidence") + evidence = evidence if isinstance(evidence, dict) else {} + inflation = report.get("inflation") + inflation = inflation if isinstance(inflation, dict) else {} + power = report.get("power") + power = power if isinstance(power, dict) else {} + + return { + "schemaVersion": QOR_SNAPSHOT_EXTENSION_SCHEMA_VERSION, + "scoringEngine": "qor-v3", + "status": "available", + "score": _number_or_none(summary.get("score")), + "scalarStatus": _text(summary.get("status"), "NOT_RATED"), + "profile": _text(summary.get("profile"), "balanced"), + "qphys": dimensions, + "feasibility": { + "status": _text(feasibility.get("status"), "UNKNOWN"), + "gates": gates[:32], + }, + "evidence": { + "index": _number_or_none(evidence.get("index")), + "state": _text(evidence.get("state"), "NOT_VERIFIED"), + "integrity": _number_or_none(evidence.get("integrity")), + "coverage": _number_or_none(evidence.get("coverage")), + "consistency": _number_or_none(evidence.get("consistency")), + }, + "diagnoses": [_diagnosis(item) for item in report.get("diagnoses", [])[:_MAX_DIAGNOSES]], + "inflation": { + "iPlace": _number_or_none(inflation.get("i_place")), + "iRoute": _number_or_none(inflation.get("i_route")), + "iTotal": _number_or_none(inflation.get("i_total")), + "congestionSeverity": _number_or_none(inflation.get("congestion_severity")), + "compatibilityStatus": _text(inflation.get("compatibility_status")), + }, + "power": { + "totalUw": _number_or_none(power.get("total_uw")), + "budgetUw": _number_or_none(power.get("budget_uw")), + "sourceKind": power.get("source_kind") + if isinstance(power.get("source_kind"), str) + else None, + "corner": power.get("corner") if isinstance(power.get("corner"), str) else None, + }, + "artifactIds": _artifact_ids(artifacts), + } + + +def unavailable_qor_snapshot_extension(reason: str) -> dict[str, Any]: + """Return an explicit unavailable extension without inventing QoR values.""" + return { + "schemaVersion": QOR_SNAPSHOT_EXTENSION_SCHEMA_VERSION, + "scoringEngine": "qor-v3", + "status": "unavailable", + "reason": _text(reason, "QoR analysis unavailable"), + "score": None, + "scalarStatus": "NOT_RATED", + "profile": "balanced", + "qphys": {}, + "feasibility": {"status": "UNKNOWN", "gates": []}, + "evidence": { + "index": None, + "state": "NOT_VERIFIED", + "integrity": None, + "coverage": None, + "consistency": None, + }, + "diagnoses": [], + "inflation": { + "iPlace": None, + "iRoute": None, + "iTotal": None, + "congestionSeverity": None, + "compatibilityStatus": "UNAVAILABLE", + }, + "power": {"totalUw": None, "budgetUw": None, "sourceKind": None, "corner": None}, + "artifactIds": [], + } + + +def validate_qor_snapshot_extension(value: object) -> bool: + """Validate the bounded, path-free QoR extension fail-closed.""" + if not isinstance(value, dict): + return False + status = value.get("status") + required = { + "schemaVersion", + "scoringEngine", + "status", + "score", + "scalarStatus", + "profile", + "qphys", + "feasibility", + "evidence", + "diagnoses", + "inflation", + "power", + "artifactIds", + } + if status == "unavailable": + required.add("reason") + if ( + set(value) != required + or value.get("schemaVersion") != QOR_SNAPSHOT_EXTENSION_SCHEMA_VERSION + or value.get("scoringEngine") != "qor-v3" + or status not in {"available", "unavailable"} + or not _bounded_text( + value.get("profile"), + values=("balanced", "timing_critical", "low_power", "area_optimized"), + ) + or not _bounded_number(value.get("score"), 0, 100, nullable=True) + or value.get("scalarStatus") not in SCALAR_STATUSES + ): + return False + if status == "unavailable" and not _bounded_text(value.get("reason")): + return False + + qphys = value.get("qphys") + if not isinstance(qphys, dict) or len(qphys) > len(DIMENSION_KEYS): + return False + for key, dimension in qphys.items(): + if key not in DIMENSION_KEYS or not isinstance(dimension, dict): + return False + if set(dimension) != {"value", "state", "featureIds"}: + return False + if not _bounded_number(dimension["value"], 0, 100, nullable=True): + return False + if dimension["state"] not in DIMENSION_STATES or not _bounded_strings( + dimension["featureIds"], 32 + ): + return False + + feasibility = value.get("feasibility") + if not isinstance(feasibility, dict) or set(feasibility) != {"status", "gates"}: + return False + if ( + feasibility["status"] not in FEASIBILITY_STATUSES + or not isinstance(feasibility["gates"], list) + or len(feasibility["gates"]) > 32 + ): + return False + for gate in feasibility["gates"]: + if not isinstance(gate, dict) or set(gate) != { + "id", + "stage", + "state", + "blocksTapeout", + "metrics", + "availability", + }: + return False + if ( + not _bounded_text(gate["id"]) + or not _bounded_text(gate["stage"]) + or gate["state"] not in GATE_STATES + or not isinstance(gate["blocksTapeout"], bool) + or not _bounded_strings(gate["metrics"], 32) + or not _bounded_text(gate["availability"], nullable=True) + ): + return False + + evidence = value.get("evidence") + if not isinstance(evidence, dict) or set(evidence) != { + "index", + "state", + "integrity", + "coverage", + "consistency", + }: + return False + if ( + evidence["state"] not in EVIDENCE_STATES + or not _bounded_number(evidence["index"], 0, 100, nullable=True) + or not all( + _bounded_number(evidence[field], 0, 1, nullable=True) + for field in ("integrity", "coverage", "consistency") + ) + ): + return False + + diagnoses = value.get("diagnoses") + if not isinstance(diagnoses, list) or len(diagnoses) > _MAX_DIAGNOSES: + return False + for diagnosis in diagnoses: + if not isinstance(diagnosis, dict) or set(diagnosis) != { + "diagnosisId", + "state", + "severity", + "confidence", + "triggerFeatures", + "affectedDimensions", + "interventions", + "interventionConfidence", + "validationRequired", + }: + return False + if ( + not _bounded_text(diagnosis["diagnosisId"]) + or not _bounded_text(diagnosis["state"]) + or not _bounded_number(diagnosis["severity"], 0, 1) + or diagnosis["confidence"] not in CONFIDENCES + or not _bounded_strings(diagnosis["triggerFeatures"], 32) + or not _bounded_strings(diagnosis["affectedDimensions"], 16) + or not isinstance(diagnosis["interventions"], list) + or len(diagnosis["interventions"]) > _MAX_INTERVENTIONS + or diagnosis["interventionConfidence"] not in CONFIDENCES + or not _bounded_text(diagnosis["validationRequired"], nullable=True) + ): + return False + for intervention in diagnosis["interventions"]: + if not isinstance(intervention, dict) or set(intervention) != { + "hypothesis", + "tier", + "confidence", + "parameterKnob", + "validationProcedure", + }: + return False + if ( + not _bounded_text(intervention["hypothesis"]) + or intervention["tier"] not in TIERS + or intervention["confidence"] not in CONFIDENCES + or not _bounded_text(intervention["parameterKnob"], nullable=True) + or not _bounded_text(intervention["validationProcedure"], nullable=True) + ): + return False + + inflation = value.get("inflation") + if not isinstance(inflation, dict) or set(inflation) != { + "iPlace", + "iRoute", + "iTotal", + "congestionSeverity", + "compatibilityStatus", + }: + return False + if not all( + _bounded_number(inflation[field], 0, None, nullable=True) + for field in ("iPlace", "iRoute", "iTotal", "congestionSeverity") + ) or inflation["compatibilityStatus"] not in { + "EXACT_COMPATIBLE", + "MAPPED_COMPATIBLE", + "INCOMPATIBLE", + "UNAVAILABLE", + }: + return False + + power = value.get("power") + if not isinstance(power, dict) or set(power) != {"totalUw", "budgetUw", "sourceKind", "corner"}: + return False + if ( + not _bounded_number(power["totalUw"], 0, None, nullable=True) + or not _bounded_number(power["budgetUw"], 0, None, nullable=True) + or power["sourceKind"] not in {None, "signoff", "synthesis"} + or not _bounded_text(power["corner"], nullable=True) + ): + return False + + return _bounded_strings(value["artifactIds"], _MAX_ARTIFACT_IDS) + + +def _bounded_number( + value: object, low: float | None, high: float | None, *, nullable: bool +) -> bool: + if value is None: + return nullable + if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value): + return False + return (low is None or value >= low) and (high is None or value <= high) + + +def _bounded_text( + value: object, *, nullable: bool = False, values: tuple[str, ...] | None = None +) -> bool: + if value is None: + return nullable + return ( + isinstance(value, str) + and bool(value) + and len(value) <= _MAX_TEXT + and (values is None or value in values) + ) + + +def _bounded_strings(value: object, maximum: int) -> bool: + return ( + isinstance(value, list) + and len(value) <= maximum + and all(_bounded_text(item) for item in value) + ) + + +def _diagnosis(value: object) -> dict[str, Any]: + if not isinstance(value, dict): + return {"diagnosisId": "unknown", "state": "UNKNOWN", "severity": None} + interventions = [] + for item in value.get("interventions", [])[:_MAX_INTERVENTIONS]: + if not isinstance(item, dict): + continue + interventions.append( + { + "hypothesis": _text(item.get("hypothesis")), + "tier": _text(item.get("tier")), + "confidence": _text(item.get("confidence"), "LOW"), + "parameterKnob": item.get("parameter_knob") + if isinstance(item.get("parameter_knob"), str) + else None, + "validationProcedure": item.get("validation_procedure") + if isinstance(item.get("validation_procedure"), str) + else None, + } + ) + return { + "diagnosisId": _text(value.get("diagnosis_id"), "unknown"), + "state": _text(value.get("state"), "UNKNOWN"), + "severity": _number_or_none(value.get("severity")), + "confidence": _text(value.get("diagnosis_confidence"), "LOW"), + "triggerFeatures": [ + item for item in value.get("trigger_features", [])[:32] if isinstance(item, str) + ], + "affectedDimensions": [ + item for item in value.get("affected_dimensions", [])[:16] if isinstance(item, str) + ], + "interventions": interventions, + "interventionConfidence": _text(value.get("intervention_confidence"), "LOW"), + "validationRequired": value.get("validation_required") + if isinstance(value.get("validation_required"), str) + else None, + } + + +def _artifact_ids(artifacts: list[dict[str, Any]]) -> list[str]: + ids = { + item["artifactId"] + for item in artifacts + if isinstance(item, dict) and isinstance(item.get("artifactId"), str) + } + return sorted(ids)[:_MAX_ARTIFACT_IDS] + + +def _number_or_none(value: object) -> float | int | None: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value): + return None + return value + + +def _text(value: object, default: str = "") -> str: + return value[:_MAX_TEXT] if isinstance(value, str) and value else default diff --git a/chipcompiler/engine/workspace_configuration.py b/chipcompiler/engine/workspace_configuration.py new file mode 100644 index 000000000..d4e1eb8f4 --- /dev/null +++ b/chipcompiler/engine/workspace_configuration.py @@ -0,0 +1,473 @@ +from copy import deepcopy +from pathlib import Path +from typing import Any + +from chipcompiler.data import load_workspace, save_parameter +from chipcompiler.data.parameter_schema import ( + list_schemas, + lookup_schema, + validate_schema_type, + validate_value, +) +from chipcompiler.data.workspace import workspace_config_paths +from chipcompiler.data.workspace_parameters import ( + update_workspace_param_value, + workspace_param_value, +) +from chipcompiler.data.workspace_transaction import WorkspaceFileTransaction +from chipcompiler.engine.flow import EngineFlow +from chipcompiler.engine.rerun import invalidate_from +from chipcompiler.engine.snapshot import ( + SNAPSHOT_FILENAME, + STALE_SNAPSHOT_FILENAME, + EngineeringSnapshotError, + ensure_engineering_snapshot, + invalidate_engineering_snapshot, + read_engineering_snapshot, +) +from chipcompiler.rtl2gds import get_flow_builders, normalize_flow_step + +from .workspace_lifecycle import ( + WorkspaceLifecycleError, + _command_retry_matches, + _load_committed_workspace, + _workspace_command_fingerprint, + _write_workspace_command, +) + + +def update_workspace_configuration( + target_directory: str | Path, + expected_workspace_revision: int, + configuration: object, + workspace_bindings: object, + command_id: str = "", +): + target = Path(target_directory).expanduser().resolve() + if not target.is_dir(): + raise WorkspaceLifecycleError("workspace_missing", f"Workspace not found: {target}") + return _run_file_transaction( + target, + lambda: _update_workspace_configuration( + target, + expected_workspace_revision, + configuration, + workspace_bindings, + command_id, + ), + ) + + +def _update_workspace_configuration( + target: Path, + expected_workspace_revision: int, + configuration: object, + workspace_bindings: object, + command_id: str, +): + if not isinstance(configuration, dict) or not isinstance(workspace_bindings, dict): + raise WorkspaceLifecycleError( + "workspace_spec_invalid", "Workspace configuration and bindings must be objects" + ) + config = {str(key): value for key, value in configuration.items()} + unknown_sections = config.keys() - {"design", "pdk", "parameters"} + if unknown_sections: + raise WorkspaceLifecycleError( + "workspace_spec_invalid", f"Unknown configuration section: {min(unknown_sections)}" + ) + if config.get("design") or config.get("pdk"): + raise WorkspaceLifecycleError( + "workspace_structure_change_requires_update", + "Design and PDK changes require a structural Workspace Update", + ) + requested = config.get("parameters", {}) + if not isinstance(requested, dict): + raise WorkspaceLifecycleError("workspace_spec_invalid", "parameters must be an object") + parameters = {str(key): value for key, value in requested.items()} + fingerprint = _workspace_command_fingerprint( + "configuration", config, workspace_bindings, expected_workspace_revision + ) + if command_id and _command_retry_matches(target, command_id, fingerprint): + return _load_committed_workspace(target) + + workspace = _load_committed_workspace(target) + current = ensure_engineering_snapshot(workspace) + if current["workspaceRevision"] != expected_workspace_revision: + raise WorkspaceLifecycleError( + "revision_conflict", + "Workspace Revision does not match", + { + "expectedWorkspaceRevision": expected_workspace_revision, + "actualWorkspaceRevision": current["workspaceRevision"], + }, + ) + + changed = False + for parameter, value in parameters.items(): + schema = lookup_schema(parameter) + if schema is None or schema.pdk_target is not None: + raise WorkspaceLifecycleError("unknown_parameter", f"Unknown parameter: {parameter}") + normalized, type_error = validate_schema_type(value, schema) + errors = [type_error] if type_error else validate_value(normalized, schema) + if errors: + raise WorkspaceLifecycleError("invalid_parameter", str(errors[0])) + if workspace_param_value(workspace, schema) != normalized: + update_workspace_param_value(workspace, schema, normalized) + changed = True + + if not changed: + _write_workspace_command( + target, + command_id, + fingerprint, + current["workspaceId"], + current["workspaceRevision"], + ) + return workspace + if not save_parameter(workspace.parameters): + raise OSError("Failed to save Workspace parameters") + from chipcompiler.data import refresh_workspace_config + + refresh_workspace_config(workspace) + flow = EngineFlow(workspace) + steps = workspace.flow.steps() + if steps: + invalidate_from(flow, str(steps[0]["name"])) + updated = invalidate_engineering_snapshot( + workspace, + workspace_id=current["workspaceId"], + cause="workspace.configuration_updated", + ) + _write_workspace_command( + target, + command_id, + fingerprint, + updated["workspaceId"], + updated["workspaceRevision"], + ) + return _load_committed_workspace(target) + + +def read_workspace_configuration(workspace: Any) -> dict[str, Any]: + parameters = {} + for schema in list_schemas(): + if schema.pdk_target is not None: + continue + try: + parameters[schema.param] = workspace_param_value(workspace, schema) + except (OSError, ValueError): + continue + steps = workspace.flow.steps() + flow_names = [str(step.get("name", "")) for step in steps if step.get("name")] + flow = {"flowId": _flow_id(flow_names)} + if flow_names: + flow.update({"fromStepId": flow_names[0], "throughStepId": flow_names[-1]}) + inputs, input_bindings = _workspace_inputs(workspace) + pdk_mode = "manual" if workspace.parameters.data.get("pdk_config") else "default" + pdk_files, pdk_file_bindings = ( + _workspace_pdk_files(workspace) if pdk_mode == "manual" else ([], {}) + ) + snapshot = _read_snapshot_metadata(workspace) + return { + "workspaceId": snapshot["workspaceId"], + "workspaceRevision": snapshot["workspaceRevision"], + "workspaceSpec": { + "schemaVersion": 1, + "design": { + "name": workspace.design.name, + "topModule": workspace.design.top_module, + "clockPort": str(workspace.parameters.data.get("clock", "")), + }, + "inputMode": workspace.parameters.data.get("_input_mode") + if workspace.parameters.data.get("_input_mode") in {"rtl", "postSynthesis"} + else ("postSynthesis" if workspace.design.origin_def is not None else "rtl"), + "inputs": inputs, + "pdk": { + "familyId": workspace.pdk.name, + "version": workspace.pdk.version, + "mode": pdk_mode, + **({"files": pdk_files} if pdk_files else {}), + }, + "flow": flow, + "parameters": parameters, + }, + "workspaceBindings": { + "inputs": input_bindings, + "pdk": { + "root": str(workspace.pdk.root or ""), + "version": workspace.pdk.version, + **({"files": pdk_file_bindings} if pdk_file_bindings else {}), + }, + }, + } + + +def read_step_configuration(workspace: Any, step_id: str) -> dict[str, Any]: + step, schemas = _step_catalog(workspace, step_id) + snapshot = _read_snapshot_metadata(workspace) + if not schemas: + raise WorkspaceLifecycleError( + "step_configuration_unavailable", + f"Flow Step has no configurable parameters: {step}", + { + "workspaceId": snapshot["workspaceId"], + "workspaceRevision": snapshot["workspaceRevision"], + }, + ) + return { + "step": step, + "stepId": step, + "parameters": [_public_parameter_record(workspace, schema) for schema in schemas], + "workspaceId": snapshot["workspaceId"], + "workspaceRevision": snapshot["workspaceRevision"], + } + + +def read_step_configuration_from_directory( + target_directory: str | Path, + step_id: str, +) -> dict[str, Any]: + workspace = _load_committed_workspace(Path(target_directory).expanduser().resolve()) + return read_step_configuration(workspace, step_id) + + +def update_workspace_step_configuration( + target_directory: str | Path, + expected_workspace_revision: int, + step_id: str, + parameters: object, + command_id: str = "", +): + target = Path(target_directory).expanduser().resolve() + if not target.is_dir(): + raise WorkspaceLifecycleError("workspace_missing", f"Workspace not found: {target}") + return _run_file_transaction( + target, + lambda: _update_workspace_step_configuration( + target, + expected_workspace_revision, + step_id, + parameters, + command_id, + ), + ) + + +def _update_workspace_step_configuration( + target: Path, + expected_workspace_revision: int, + step_id: str, + parameters: object, + command_id: str, +): + if not isinstance(step_id, str) or not step_id.strip() or not isinstance(parameters, dict): + raise WorkspaceLifecycleError( + "workspace_spec_invalid", "Step identity and parameters object are required" + ) + patch = {str(key): value for key, value in parameters.items()} + fingerprint = _workspace_command_fingerprint( + "step_configuration", + {"stepId": step_id, "parameters": patch}, + {}, + expected_workspace_revision, + ) + if command_id and _command_retry_matches(target, command_id, fingerprint): + return _load_committed_workspace(target) + + workspace = _load_committed_workspace(target) + snapshot = ensure_engineering_snapshot(workspace) + if snapshot["workspaceRevision"] != expected_workspace_revision: + raise WorkspaceLifecycleError( + "revision_conflict", + "Workspace Revision does not match", + { + "expectedWorkspaceRevision": expected_workspace_revision, + "actualWorkspaceRevision": snapshot["workspaceRevision"], + }, + ) + step, schemas = _step_catalog(workspace, step_id) + allowed = {schema.param: schema for schema in schemas} + changed = False + for parameter, value in patch.items(): + schema = lookup_schema(parameter) + if schema is None or schema.pdk_target is not None: + raise WorkspaceLifecycleError("unknown_parameter", f"Unknown parameter: {parameter}") + if parameter not in allowed: + raise WorkspaceLifecycleError( + "parameter_not_applicable", + f"Parameter {parameter} is not configurable at {step}", + ) + normalized, type_error = validate_schema_type(value, schema) + errors = [type_error] if type_error else validate_value(normalized, schema) + if errors: + raise WorkspaceLifecycleError("invalid_parameter", str(errors[0])) + if workspace_param_value(workspace, schema) != normalized: + update_workspace_param_value(workspace, schema, normalized) + changed = True + + if not changed: + _write_workspace_command( + target, + command_id, + fingerprint, + snapshot["workspaceId"], + snapshot["workspaceRevision"], + ) + return workspace + if not save_parameter(workspace.parameters): + raise OSError("Failed to save Workspace parameters") + from chipcompiler.data import refresh_workspace_config + + refresh_workspace_config(workspace) + invalidate_from(EngineFlow(workspace), step) + updated = invalidate_engineering_snapshot( + workspace, + workspace_id=snapshot["workspaceId"], + cause="workspace.step_configuration_updated", + first_invalidated_step=step, + ) + _write_workspace_command( + target, + command_id, + fingerprint, + updated["workspaceId"], + updated["workspaceRevision"], + ) + return _load_committed_workspace(target) + + +def _step_catalog(workspace: Any, step_id: str): + identity = normalize_flow_step(step_id).casefold() + steps = [str(step.get("name", "")) for step in workspace.flow.steps() if step.get("name")] + step = next( + (candidate for candidate in steps if normalize_flow_step(candidate).casefold() == identity), + None, + ) + if step is None: + raise WorkspaceLifecycleError("unknown_flow_step", f"Flow Step not found: {step_id}") + first = normalize_flow_step(steps[0]).casefold() + schemas = tuple( + schema + for schema in list_schemas() + if schema.pdk_target is None + and ( + normalize_flow_step(schema.applies).casefold() == identity + or (schema.applies == "all" and identity == first) + ) + ) + return step, schemas + + +def _public_parameter_record(workspace: Any, schema) -> dict[str, Any]: + record = { + "param": schema.param, + "type": schema.type, + "value": workspace_param_value(workspace, schema), + "default": deepcopy(schema.default), + "applies": schema.applies, + "description": schema.description, + } + for field in ("range", "choices", "unit"): + value = getattr(schema, field) + if value is not None: + record[field] = list(value) if isinstance(value, tuple) else value + return record + + +def read_workspace_configuration_from_directory( + target_directory: str | Path, +) -> dict[str, Any]: + target = Path(target_directory).expanduser().resolve() + workspace = load_workspace(target, read_only=True) + if workspace is None: + raise WorkspaceLifecycleError("workspace_missing", f"Workspace not found: {target}") + return read_workspace_configuration(workspace) + + +def _workspace_inputs(workspace: Any) -> tuple[list[dict[str, str]], dict[str, str]]: + values = [] + input_mode = workspace.parameters.data.get("_input_mode") + if input_mode == "postSynthesis" and workspace.design.origin_verilog is not None: + values.append(("netlist", workspace.design.origin_verilog)) + elif workspace.design.input_filelist is not None: + values.append(("filelist", workspace.design.input_filelist)) + elif workspace.design.origin_verilog is not None: + role = "netlist" if workspace.design.origin_def is not None else "rtl" + values.append((role, workspace.design.origin_verilog)) + for role, path in ( + ("def", workspace.design.origin_def), + ("goldenNetlist", workspace.design.golden_verilog), + ("sdc", workspace.pdk.sdc), + ("spef", workspace.pdk.spef), + ): + if path is not None: + values.append((role, path)) + return ( + [{"inputId": role, "role": role} for role, _path in values], + {role: str(path) for role, path in values}, + ) + + +def _workspace_pdk_files(workspace: Any) -> tuple[list[dict[str, str]], dict[str, str]]: + grouped = { + "tech": [workspace.pdk.tech] if workspace.pdk.tech else [], + "lef": list(workspace.pdk.lefs), + "liberty": list(workspace.pdk.libs), + "mapping": [workspace.pdk.mapping_file] if workspace.pdk.mapping_file else [], + } + refs = [] + bindings = {} + for role, paths in grouped.items(): + for index, path in enumerate(paths): + file_id = role if len(paths) == 1 else f"{role}-{index}" + refs.append({"fileId": file_id, "role": role}) + bindings[file_id] = str(path) + return refs, bindings + + +def _read_snapshot_metadata(workspace: Any) -> dict[str, Any]: + try: + snapshot = read_engineering_snapshot(workspace) + except (EngineeringSnapshotError, OSError): + return {"workspaceId": None, "workspaceRevision": None} + return { + "workspaceId": snapshot["workspaceId"], + "workspaceRevision": snapshot["workspaceRevision"], + } + + +def _flow_id(names: list[str]) -> str: + for flow_id, builder in get_flow_builders().items(): + candidate = [str(getattr(step, "value", step)) for step, _tool, _state in builder()] + if candidate == names: + return flow_id + if flow_id == "rtl2gds" and names: + start = next( + (index for index, value in enumerate(candidate) if value == names[0]), None + ) + if start is not None and candidate[start : start + len(names)] == names: + return flow_id + return "custom" + + +def _run_file_transaction(target: Path, apply): + transaction = WorkspaceFileTransaction.begin(target, _workspace_file_paths(target)) + try: + result = apply() + transaction.commit() + return result + except BaseException: + transaction.rollback() + raise + + +def _workspace_file_paths(target: Path) -> list[Path]: + paths = [ + target / "home" / "params.toml", + target / "home" / "flow.json", + target / "home" / SNAPSHOT_FILENAME, + target / "home" / STALE_SNAPSHOT_FILENAME, + target / "home" / "workspace-commands.json", + ] + paths.extend(path for key, path in workspace_config_paths(target).items() if key != "dir") + return paths diff --git a/chipcompiler/engine/workspace_flow.py b/chipcompiler/engine/workspace_flow.py new file mode 100644 index 000000000..0d10bd13d --- /dev/null +++ b/chipcompiler/engine/workspace_flow.py @@ -0,0 +1,89 @@ +from typing import Any + + +def build_flow_for_workspace(workspace, *, create_step_workspaces: bool = True): + import chipcompiler.engine as engine_api + + engine_flow = engine_api.EngineFlow(workspace=workspace) + if not engine_flow.has_init(): + raise ValueError("Workspace has no committed Flow") + + if create_step_workspaces: + engine_flow.create_step_workspaces() + return engine_flow + + +def workspace_step_from_flow(workspace, name: str): + previous_step = None + loader = getattr(workspace.flow, "steps", None) + steps = loader() if callable(loader) else workspace.flow.data.get("steps", []) + for flow_step in steps: + workspace_step = _build_workspace_step_for_info(workspace, flow_step, previous_step) + if flow_step.get("name") == name: + return workspace_step + if workspace_step is not None: + previous_step = workspace_step + return None + + +def _build_workspace_step_for_info(workspace, flow_step: dict, previous_step): + step_name = flow_step.get("name") + tool = flow_step.get("tool") + if not step_name or not tool: + return None + + if previous_step is None: + input_def = workspace.design.origin_def + input_verilog = workspace.design.origin_verilog + input_db = None + else: + input_def = previous_step.output.def_ or "" + input_verilog = previous_step.output.verilog or "" + input_db = previous_step.output.db or "" + + builder = _load_tool_builder(tool) + if builder is None or not hasattr(builder, "build_step"): + return None + + return builder.build_step( + workspace=workspace, + step_name=step_name, + input_def=input_def, + input_verilog=input_verilog, + input_db=input_db, + ) + + +def _load_tool_builder(tool: str): + import importlib + + module_alias = { + "klayout": "klayout_tool", + "dreamplace": "ecc_dreamplace", + "sizer": "ecc_sizer", + } + module_name = module_alias.get(tool, tool) + return importlib.import_module(f"chipcompiler.tools.{module_name}.builder") + + +def init_db_engine_for_workspace_step(engine_flow, workspace_step): + engine_db = getattr(engine_flow, "engine_db", None) + if engine_db is None: + from chipcompiler.engine import EngineDB + + engine_db = EngineDB(workspace=engine_flow.workspace) + engine_flow.engine_db = engine_db + elif engine_db.has_init(): + return True + + return engine_db.create_db_engine(step=workspace_step) + + +def success_state(): + from chipcompiler.data import StateEnum + + return StateEnum.Success + + +def state_value(state: Any) -> str: + return getattr(state, "value", str(state)) diff --git a/chipcompiler/engine/workspace_lifecycle.py b/chipcompiler/engine/workspace_lifecycle.py new file mode 100644 index 000000000..726c7cd66 --- /dev/null +++ b/chipcompiler/engine/workspace_lifecycle.py @@ -0,0 +1,476 @@ +import ctypes +import errno +import hashlib +import json +import os +import shutil +import tempfile +from pathlib import Path +from typing import Any + +from chipcompiler.data import PDK, create_workspace, get_pdk, load_workspace +from chipcompiler.data.parameter_schema import ( + build_backend_overrides, + build_config_overrides, + resolve_parameters, +) +from chipcompiler.data.workspace.config_overrides import CONFIG_OVERRIDES_KEY +from chipcompiler.engine.snapshot import create_engineering_snapshot +from chipcompiler.engine.workspace_spec import validate_workspace_spec +from chipcompiler.rtl2gds import get_flow_builders + + +class WorkspaceLifecycleError(RuntimeError): + def __init__(self, code: str, message: str, details: dict[str, Any] | None = None): + super().__init__(message) + self.code = code + self.details = details or {} + + +def describe_workspace_binding_requirement( + workspace_directory: str | Path, +) -> dict[str, Any]: + workspace = _load_committed_workspace(Path(workspace_directory).expanduser().resolve()) + return { + "familyId": workspace.pdk.name, + "version": workspace.pdk.version or "unversioned", + "mode": "manual" if workspace.parameters.data.get("pdk_config") else "default", + } + + +def assess_execution_readiness( + workspace_directory: str | Path, + bindings: object | None, +) -> dict[str, Any]: + try: + workspace = _load_committed_workspace(Path(workspace_directory).expanduser().resolve()) + except (OSError, ValueError, WorkspaceLifecycleError): + return {"ready": False, "code": "workspace_invalid"} + binding_map = _string_keyed_dict(bindings) + pdk_binding = _string_keyed_dict(binding_map.get("pdk")) + root = pdk_binding.get("root") + if not isinstance(root, str) or not Path(root).is_dir(): + return {"ready": False, "code": "pdk_binding_missing"} + try: + if workspace.parameters.data.get("pdk_config"): + if workspace.pdk.root is None or Path(root).resolve() != workspace.pdk.root.resolve(): + return {"ready": False, "code": "pdk_binding_mismatch"} + workspace.pdk.validate() + else: + get_pdk(workspace.pdk.name, pdk_root=root).validate() + except (OSError, ValueError): + return {"ready": False, "code": "pdk_binding_mismatch"} + return {"ready": True} + + +def apply_workspace_bindings(workspace, bindings: object) -> None: + binding_map = _string_keyed_dict(bindings) + pdk_binding = _string_keyed_dict(binding_map.get("pdk")) + root = pdk_binding.get("root") + if not isinstance(root, str) or not Path(root).is_dir(): + raise WorkspaceLifecycleError("pdk_binding_missing", "PDK binding root is required") + sdc, spef = workspace.pdk.sdc, workspace.pdk.spef + if workspace.parameters.data.get("pdk_config"): + if workspace.pdk.root is None or Path(root).resolve() != workspace.pdk.root.resolve(): + raise WorkspaceLifecycleError("pdk_binding_mismatch", "PDK binding root does not match") + workspace.pdk.validate() + else: + workspace.pdk = get_pdk(workspace.pdk.name, pdk_root=root) + workspace.pdk.sdc, workspace.pdk.spef = sdc, spef + from chipcompiler.data import refresh_workspace_config + + refresh_workspace_config(workspace) + + +def create_workspace_from_spec( + target_directory: str | Path, + spec: object, + bindings: object, + command_id: str = "", +): + """Create a Workspace Spec target while serializing sibling creators.""" + from chipcompiler.engine.reconcile import _workspace_lock + + target = Path(target_directory).expanduser().resolve() + with _workspace_lock(target): + return _create_workspace_from_spec(target, spec, bindings, command_id) + + +def _create_workspace_from_spec( + target_directory: str | Path, + spec: object, + bindings: object, + command_id: str = "", +): + target = Path(target_directory).expanduser().resolve() + fingerprint = _workspace_command_fingerprint("create", spec, bindings) + if target.exists(): + if command_id and _command_retry_matches(target, command_id, fingerprint): + return _load_committed_workspace(target) + raise WorkspaceLifecycleError("workspace_exists", f"Workspace already exists: {target}") + validation = validate_workspace_spec(spec, bindings) + issues = validation["issues"] + if issues: + raise WorkspaceLifecycleError( + "workspace_spec_invalid", + "Workspace Spec validation failed", + {"issues": issues}, + ) + resolved = validation["resolvedWorkspaceSpec"] + binding_map = _string_keyed_dict(bindings) + input_bindings = _string_keyed_dict(binding_map.get("inputs")) + input_paths = { + item["role"]: str(input_bindings[item["inputId"]]) for item in resolved["inputs"] + } + rtl_paths = [ + str(input_bindings[item["inputId"]]) for item in resolved["inputs"] if item["role"] == "rtl" + ] + parameters, errors = resolve_parameters(toml_overrides=resolved["parameters"]) + if errors: + raise WorkspaceLifecycleError( + "workspace_spec_invalid", + "Workspace parameters are invalid", + {"issues": errors}, + ) + backend_parameters = build_backend_overrides(parameters, include_defaults=True) + config_overrides = build_config_overrides(parameters) + if config_overrides: + backend_parameters[CONFIG_OVERRIDES_KEY] = config_overrides + backend_parameters.update( + { + "design": resolved["design"]["name"], + "top_module": resolved["design"]["topModule"], + "clock": resolved["design"].get("clockPort", ""), + } + ) + flow_steps = get_flow_builders()[resolved["flow"]["flowId"]]() + flow_start = resolved["flow"].get("fromStepId") + flow_end = resolved["flow"].get("throughStepId") + flow_config = { + "start_step": flow_start or str(getattr(flow_steps[0][0], "value", flow_steps[0][0])), + "end_step": flow_end or str(getattr(flow_steps[-1][0], "value", flow_steps[-1][0])), + } + pdk, pdk_root, pdk_overrides = _bound_pdk( + resolved["pdk"], _string_keyed_dict(binding_map["pdk"]) + ) + + generated_filelist: str | None = None + generated_pdk_config: str | None = None + if len(rtl_paths) > 1: + with tempfile.NamedTemporaryFile( + "w", encoding="utf-8", suffix=".f", delete=False + ) as handle: + handle.write("\n".join(rtl_paths) + "\n") + generated_filelist = handle.name + if isinstance(pdk, PDK): + with tempfile.NamedTemporaryFile( + "w", encoding="utf-8", suffix=".json", delete=False + ) as handle: + json.dump(_pdk_config(pdk), handle) + generated_pdk_config = handle.name + + try: + workspace = create_workspace( + directory=target, + origin_def=input_paths.get("def", ""), + origin_verilog=( + input_paths.get("netlist", "") or (rtl_paths[0] if len(rtl_paths) == 1 else "") + ), + input_filelist=input_paths.get("filelist", "") or generated_filelist or "", + golden_verilog=input_paths.get("goldenNetlist", ""), + sdc=input_paths.get("sdc", ""), + spef=input_paths.get("spef", ""), + pdk=pdk.name if isinstance(pdk, PDK) else pdk, + pdk_root=pdk_root, + pdk_json=generated_pdk_config or "", + pdk_overrides=pdk_overrides, + parameters=backend_parameters, + flow_config=flow_config, + ) + if workspace is None: + raise WorkspaceLifecycleError( + "workspace_create_failed", f"Workspace creation failed: {target}" + ) + workspace.parameters.data["_input_mode"] = resolved["inputMode"] + from chipcompiler.data import save_parameter + + if not save_parameter(workspace.parameters): + raise OSError(f"Failed to persist input mode: {workspace.parameters.path}") + snapshot = create_engineering_snapshot(workspace) + _write_workspace_command( + target, + command_id, + fingerprint, + snapshot["workspaceId"], + snapshot["workspaceRevision"], + ) + return workspace + except Exception: + shutil.rmtree(target, ignore_errors=True) + raise + finally: + if generated_filelist is not None: + Path(generated_filelist).unlink(missing_ok=True) + if generated_pdk_config is not None: + Path(generated_pdk_config).unlink(missing_ok=True) + + +def update_workspace_from_spec( + target_directory: str | Path, + expected_workspace_revision: int, + spec: object, + bindings: object, + command_id: str = "", +): + target = Path(target_directory).expanduser().resolve() + if not target.is_dir(): + raise WorkspaceLifecycleError("workspace_missing", f"Workspace not found: {target}") + from chipcompiler.engine.reconcile import _workspace_lock + + with _workspace_lock(target): + return _update_workspace_from_spec( + target, expected_workspace_revision, spec, bindings, command_id + ) + + +def _update_workspace_from_spec( + target_directory: str | Path, + expected_workspace_revision: int, + spec: object, + bindings: object, + command_id: str = "", +): + from chipcompiler.engine.snapshot import ( + EngineeringSnapshotError, + ensure_engineering_snapshot, + read_engineering_snapshot, + ) + + target = Path(target_directory).expanduser().resolve() + fingerprint = _workspace_command_fingerprint( + "update", spec, bindings, expected_workspace_revision + ) + if command_id and _command_retry_matches(target, command_id, fingerprint): + return _load_committed_workspace(target) + current = _load_committed_workspace(target) + snapshot_path = target / "home" / "engineering-snapshot.json" + try: + snapshot = read_engineering_snapshot(current) + except EngineeringSnapshotError as exc: + if snapshot_path.exists() or snapshot_path.is_symlink(): + raise WorkspaceLifecycleError( + "workspace_invalid", "Invalid Engineering Snapshot" + ) from exc + if expected_workspace_revision != 1: + raise WorkspaceLifecycleError( + "revision_conflict", + "Workspace Revision does not match", + { + "expectedWorkspaceRevision": expected_workspace_revision, + "actualWorkspaceRevision": 1, + }, + ) from exc + snapshot = ensure_engineering_snapshot(current) + if snapshot["workspaceRevision"] != expected_workspace_revision: + raise WorkspaceLifecycleError( + "revision_conflict", + "Workspace Revision does not match", + { + "expectedWorkspaceRevision": expected_workspace_revision, + "actualWorkspaceRevision": snapshot["workspaceRevision"], + }, + ) + + staging = Path(tempfile.mkdtemp(prefix=f".{target.name}.staging-", dir=target.parent)) + staging.rmdir() + try: + staged = create_workspace_from_spec(staging, spec, bindings) + create_engineering_snapshot( + staged, + workspace_id=snapshot["workspaceId"], + workspace_revision=snapshot["workspaceRevision"] + 1, + cause="workspace.updated", + ) + _copy_workspace_commands(target, staging) + _write_workspace_command( + staging, + command_id, + fingerprint, + snapshot["workspaceId"], + snapshot["workspaceRevision"] + 1, + ) + _rewrite_workspace_paths(staging, target) + _exchange_directories(target, staging) + try: + return _load_committed_workspace(target) + except Exception: + _exchange_directories(target, staging) + raise + finally: + shutil.rmtree(staging, ignore_errors=True) + + +def _pdk_config(pdk: PDK) -> dict[str, Any]: + return { + "name": pdk.name, + "version": pdk.version, + "root": str(pdk.root or ""), + "tech": str(pdk.tech or ""), + "lefs": [str(path) for path in pdk.lefs], + "libs": [str(path) for path in pdk.libs], + "mapping_file": str(pdk.mapping_file or ""), + "dont_use": pdk.dont_use, + "abc_load": pdk.abc_load, + } + + +def _bound_pdk(spec: dict, binding: dict) -> tuple[PDK | str, str, dict | None]: + root = Path(str(binding["root"])) + overrides = spec.get("overrides", {}) + if spec["mode"] != "manual": + return ( + spec["familyId"], + str(root), + { + **({"dont_use": overrides["dont_use"]} if "dont_use" in overrides else {}), + **({"abc_load": overrides["abc_load"]} if "abc_load" in overrides else {}), + }, + ) + files = binding.get("files", {}) + by_role: dict[str, list[Path]] = {} + for item in spec.get("files", []): + by_role.setdefault(item["role"], []).append(Path(str(files[item["fileId"]]))) + return ( + PDK( + name=spec["familyId"], + version=spec.get("version", ""), + root=root, + tech=by_role["tech"][0], + lefs=by_role.get("lef", []), + libs=by_role.get("liberty", []), + mapping_file=(by_role.get("mapping") or [None])[0], + dont_use=overrides.get("dont_use", []), + abc_load=float(overrides.get("abc_load", 0.015)), + ), + "", + None, + ) + + +def _string_keyed_dict(value: object) -> dict[str, Any]: + if not isinstance(value, dict): + return {} + return {str(key): item for key, item in value.items()} + + +def _load_committed_workspace(path: Path): + workspace = load_workspace(path) + if workspace is None: + raise WorkspaceLifecycleError("workspace_invalid", f"Workspace cannot be opened: {path}") + return workspace + + +def _workspace_command_fingerprint( + kind: str, + spec: object, + bindings: object, + expected_revision: int | None = None, +) -> str: + payload = json.dumps( + { + "kind": kind, + "spec": spec, + "bindings": bindings, + "expectedWorkspaceRevision": expected_revision, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + return hashlib.sha256(payload).hexdigest() + + +def _workspace_commands(path: Path) -> dict[str, Any]: + command_path = path / "home" / "workspace-commands.json" + try: + payload = json.loads(command_path.read_text(encoding="utf-8")) + except FileNotFoundError: + return {"schemaVersion": 1, "commands": {}} + if payload.get("schemaVersion") != 1 or not isinstance(payload.get("commands"), dict): + raise WorkspaceLifecycleError("workspace_invalid", "Invalid Workspace command ledger") + return payload + + +def _command_retry_matches(path: Path, command_id: str, fingerprint: str) -> bool: + record = _workspace_commands(path)["commands"].get(command_id) + if record is None: + return False + if not isinstance(record, dict) or record.get("fingerprint") != fingerprint: + raise WorkspaceLifecycleError( + "idempotency_conflict", f"command id reused with different input: {command_id}" + ) + return True + + +def _write_workspace_command( + path: Path, + command_id: str, + fingerprint: str, + workspace_id: str, + workspace_revision: int, +) -> None: + if not command_id: + return + from chipcompiler.utility import json_write + + payload = _workspace_commands(path) + payload["commands"][command_id] = { + "fingerprint": fingerprint, + "result": { + "workspaceId": workspace_id, + "workspaceRevision": workspace_revision, + }, + } + if not json_write(path / "home" / "workspace-commands.json", payload): + raise OSError("Failed to persist Workspace command ledger") + + +def _copy_workspace_commands(source: Path, destination: Path) -> None: + source_path = source / "home" / "workspace-commands.json" + if source_path.is_file(): + shutil.copy2(source_path, destination / "home" / "workspace-commands.json") + + +def _rewrite_workspace_paths(source_root: Path, target_root: Path) -> None: + from chipcompiler.utility import json_write + + source = str(source_root) + target = str(target_root) + for path in source_root.rglob("*.json"): + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + rewritten = _replace_string_prefix(value, source, target) + if rewritten != value and not json_write(path, rewritten): + raise OSError(f"Failed to rewrite staged Workspace path: {path}") + + +def _replace_string_prefix(value: Any, source: str, target: str) -> Any: + if isinstance(value, str): + return target + value[len(source) :] if value.startswith(source) else value + if isinstance(value, list): + return [_replace_string_prefix(item, source, target) for item in value] + if isinstance(value, dict): + return {key: _replace_string_prefix(item, source, target) for key, item in value.items()} + return value + + +def _exchange_directories(left: Path, right: Path) -> None: + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = getattr(libc, "renameat2", None) + if os.name != "posix" or renameat2 is None: + raise OSError(errno.ENOTSUP, "atomic Workspace Update is unavailable") + if renameat2(-100, os.fsencode(left), -100, os.fsencode(right), 2) == 0: + return + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) diff --git a/chipcompiler/engine/workspace_spec.py b/chipcompiler/engine/workspace_spec.py new file mode 100644 index 000000000..c25cf232a --- /dev/null +++ b/chipcompiler/engine/workspace_spec.py @@ -0,0 +1,433 @@ +from copy import deepcopy +from pathlib import Path +from typing import Any + +from chipcompiler.data.parameter_schema import list_schemas, resolve_parameters +from chipcompiler.engine.pdk_binding import pdk_binding_content_hash +from chipcompiler.rtl2gds import get_flow_builders, normalize_flow_step +from chipcompiler.utility.filelist import validate_filelist + +SCHEMA_VERSION = 1 +_INPUT_ROLES = frozenset({"rtl", "filelist", "netlist", "goldenNetlist", "def", "sdc", "spef"}) +_PDK_FILE_ROLES = frozenset({"tech", "lef", "liberty", "mapping"}) +_SPEC_FIELDS = frozenset( + {"schemaVersion", "design", "inputMode", "inputs", "pdk", "flow", "mpc", "parameters"} +) + + +def describe_workspace_spec() -> dict[str, Any]: + flows = [] + for flow_id, builder in sorted(get_flow_builders().items()): + steps = [_enum_value(step) for step, _tool, _state in builder()] + flows.append({"flowId": flow_id, "stepIds": steps}) + return { + "schemaVersion": SCHEMA_VERSION, + "parameterCatalog": [ + { + "id": schema.param, + "type": schema.type, + "default": deepcopy(schema.default), + "appliesTo": schema.applies, + "backendMapping": deepcopy(schema.maps_to), + **({"range": list(schema.range)} if schema.range else {}), + **({"choices": list(schema.choices)} if schema.choices else {}), + **({"unit": schema.unit} if schema.unit else {}), + } + for schema in list_schemas() + ], + "flowDefinitions": flows, + "inputRoleRules": { + "roles": sorted(_INPUT_ROLES), + "rtl": {"rtl": "oneOrMore", "filelist": "exactlyOneAlternative"}, + "postSynthesis": {"netlist": "exactlyOne"}, + "manualPdk": { + "tech": "exactlyOne", + "lef": "oneOrMore", + "liberty": "oneOrMore", + "mapping": "zeroOrOne", + }, + }, + } + + +def validate_workspace_spec(spec: object, bindings: object) -> dict[str, Any]: + issues: list[dict[str, Any]] = [] + if not isinstance(spec, dict): + return {"issues": [_issue("invalid_type", "", expected="object")]} + spec = {str(key): value for key, value in spec.items()} + if not isinstance(bindings, dict): + bindings = {} + issues.append(_issue("invalid_type", "/bindings", expected="object")) + else: + bindings = {str(key): value for key, value in bindings.items()} + + _unknown_fields(spec, _SPEC_FIELDS, "", issues) + if spec.get("schemaVersion") != SCHEMA_VERSION or isinstance(spec.get("schemaVersion"), bool): + issues.append( + _issue( + "unsupported_schema_version", + "/schemaVersion", + supported=SCHEMA_VERSION, + ) + ) + + design = _mapping(spec.get("design"), "/design", issues) + _unknown_fields(design, {"name", "topModule", "clockPort"}, "/design", issues) + _required_string(design, "name", "/design/name", issues) + _required_string(design, "topModule", "/design/topModule", issues) + if "clockPort" in design and not _nonempty_string(design["clockPort"]): + issues.append(_issue("invalid_value", "/design/clockPort")) + + input_mode = spec.get("inputMode") + if input_mode not in {"rtl", "postSynthesis"}: + issues.append(_issue("invalid_input_mode", "/inputMode")) + inputs = _refs(spec.get("inputs"), "inputId", _INPUT_ROLES, "/inputs", issues) + input_bindings = _input_bindings(bindings.get("inputs"), issues) + _validate_binding_set(inputs, input_bindings, issues) + _validate_input_roles(input_mode, inputs, issues) + _validate_input_files(inputs, input_bindings, issues) + + pdk = _mapping(spec.get("pdk"), "/pdk", issues) + _unknown_fields(pdk, {"familyId", "version", "mode", "files", "overrides"}, "/pdk", issues) + _required_string(pdk, "familyId", "/pdk/familyId", issues) + mode = pdk.get("mode") + if mode not in {"default", "manual"}: + issues.append(_issue("invalid_pdk_mode", "/pdk/mode")) + overrides = pdk.get("overrides", {}) + if not isinstance(overrides, dict): + issues.append(_issue("invalid_type", "/pdk/overrides", expected="object")) + else: + _unknown_fields(overrides, {"dont_use", "abc_load"}, "/pdk/overrides", issues) + if "dont_use" in overrides and not ( + isinstance(overrides["dont_use"], list) + and all(isinstance(value, str) for value in overrides["dont_use"]) + ): + issues.append(_issue("invalid_type", "/pdk/overrides/dont_use", expected="array")) + if "abc_load" in overrides and ( + isinstance(overrides["abc_load"], bool) + or not isinstance(overrides["abc_load"], (int, float)) + ): + issues.append(_issue("invalid_type", "/pdk/overrides/abc_load", expected="number")) + pdk_bindings = _mapping(bindings.get("pdk"), "/bindings/pdk", issues) + root = pdk_bindings.get("root") + if not isinstance(root, str) or not root.strip() or not Path(root).is_dir(): + issues.append(_issue("pdk_binding_missing", "/bindings/pdk/root")) + _validate_pdk_files(pdk, pdk_bindings, issues) + requested_version = pdk.get("version") + bound_version = pdk_bindings.get("version") + if requested_version and bound_version and requested_version != bound_version: + issues.append( + _issue( + "pdk_binding_mismatch", + "/bindings/pdk/version", + expected=requested_version, + actual=bound_version, + ) + ) + + flow = _mapping(spec.get("flow"), "/flow", issues) + _unknown_fields(flow, {"flowId", "fromStepId", "throughStepId"}, "/flow", issues) + flow_steps = _validate_flow(flow, issues) + + parameters = spec.get("parameters", {}) + if not isinstance(parameters, dict): + issues.append(_issue("invalid_type", "/parameters", expected="object")) + parameters = {} + else: + parameters = {str(key): value for key, value in parameters.items()} + known_parameters = {schema.param for schema in list_schemas()} + for parameter_id in parameters: + if parameter_id not in known_parameters: + issues.append(_issue("unknown_parameter", f"/parameters/{_pointer(parameter_id)}")) + resolved_parameters, parameter_errors = resolve_parameters( + toml_overrides={key: value for key, value in parameters.items() if key in known_parameters} + ) + for error in parameter_errors: + parameter_id = next((key for key in parameters if key in error), "") + issues.append( + _issue( + "invalid_parameter", + f"/parameters/{_pointer(parameter_id)}" if parameter_id else "/parameters", + reason=error, + ) + ) + _validate_parameter_applicability(parameters, flow_steps, issues) + + mpc = spec.get("mpc") + if mpc is not None: + mpc = _mapping(mpc, "/mpc", issues) + _unknown_fields(mpc, {"resourceId", "version", "designId"}, "/mpc", issues) + for key in ("resourceId", "version", "designId"): + _required_string(mpc, key, f"/mpc/{key}", issues) + mpc_binding = _mapping(bindings.get("mpc"), "/bindings/mpc", issues) + if not isinstance(mpc_binding.get("template"), dict): + issues.append(_issue("mpc_binding_missing", "/bindings/mpc/template")) + + if any(issue["severity"] == "error" for issue in issues): + return {"issues": issues} + + resolved = deepcopy(spec) + resolved["parameters"] = _effective_parameter_values(resolved_parameters, flow_steps) + resolved["pdk"] = { + **deepcopy(pdk), + "version": requested_version or bound_version or "unversioned", + "contentHash": pdk_binding_content_hash(pdk, pdk_bindings), + } + return {"resolvedWorkspaceSpec": resolved, "issues": issues} + + +def _validate_input_roles( + input_mode: object, + inputs: list[dict[str, str]], + issues: list[dict[str, Any]], +) -> None: + roles = [item["role"] for item in inputs] + if input_mode == "rtl": + if ("rtl" in roles) == ("filelist" in roles): + issues.append(_issue("conflicting_input_roles", "/inputs")) + if roles.count("filelist") > 1: + issues.append(_issue("filelist_cardinality", "/inputs")) + if "netlist" in roles: + issues.append(_issue("input_role_not_allowed", "/inputs", mode="rtl")) + elif input_mode == "postSynthesis": + if roles.count("netlist") != 1: + issues.append(_issue("netlist_cardinality", "/inputs")) + if any(role in {"rtl", "filelist"} for role in roles): + issues.append(_issue("input_role_not_allowed", "/inputs", mode="postSynthesis")) + for role in ("netlist", "goldenNetlist", "def", "sdc", "spef"): + if roles.count(role) > 1: + issues.append(_issue(f"{role}_cardinality", "/inputs")) + + +def _validate_input_files( + refs: list[dict[str, str]], + bindings: dict[str, str], + issues: list[dict[str, Any]], +) -> None: + for ref in refs: + input_id = ref["inputId"] + path = bindings.get(input_id) + if not path: + continue + source = Path(path) + pointer = f"/bindings/inputs/{_pointer(input_id)}" + if not source.is_file(): + issues.append(_issue("input_unreadable", pointer)) + continue + if ref["role"] != "filelist": + continue + try: + _existing, missing = validate_filelist(str(source)) + except (OSError, ValueError) as exc: + issues.append(_issue("filelist_invalid", pointer, reason=str(exc))) + continue + if missing: + issues.append(_issue("filelist_source_missing", pointer, missing=missing)) + + +def _validate_binding_set( + refs: list[dict[str, str]], + bindings: dict[str, str], + issues: list[dict[str, Any]], +) -> None: + ref_ids = {item["inputId"] for item in refs} + for input_id in sorted(ref_ids - bindings.keys()): + issues.append(_issue("input_binding_missing", f"/bindings/inputs/{_pointer(input_id)}")) + for input_id in sorted(bindings.keys() - ref_ids): + issues.append(_issue("unknown_input_binding", f"/bindings/inputs/{_pointer(input_id)}")) + + +def _validate_pdk_files( + pdk: dict[str, Any], + bindings: dict[str, Any], + issues: list[dict[str, Any]], +) -> None: + mode = pdk.get("mode") + refs = _refs(pdk.get("files", []), "fileId", _PDK_FILE_ROLES, "/pdk/files", issues) + roles = [item["role"] for item in refs] + rules = ( + ( + ("tech", roles.count("tech") == 1), + ("lef", roles.count("lef") >= 1), + ("liberty", roles.count("liberty") >= 1), + ("mapping", roles.count("mapping") <= 1), + ) + if mode == "manual" + else ( + ("tech", roles.count("tech") <= 1), + ("mapping", roles.count("mapping") <= 1), + ) + ) + for role, valid in rules: + if not valid: + issues.append(_issue(f"pdk_{role}_cardinality", "/pdk/files")) + raw_file_bindings = bindings.get("files", {}) + file_bindings = raw_file_bindings if isinstance(raw_file_bindings, dict) else {} + ref_ids = {item["fileId"] for item in refs} + for file_id in sorted(ref_ids - file_bindings.keys()): + issues.append( + _issue("pdk_file_binding_missing", f"/bindings/pdk/files/{_pointer(file_id)}") + ) + for file_id in sorted(file_bindings.keys() - ref_ids): + issues.append( + _issue("unknown_pdk_file_binding", f"/bindings/pdk/files/{_pointer(file_id)}") + ) + for file_id, path in file_bindings.items(): + if file_id in ref_ids and (not isinstance(path, str) or not Path(path).is_file()): + issues.append(_issue("pdk_file_unreadable", f"/bindings/pdk/files/{_pointer(file_id)}")) + + +def _validate_flow(flow: dict[str, Any], issues: list[dict[str, Any]]) -> set[str]: + flow_id = flow.get("flowId") + builder = get_flow_builders().get(flow_id) if isinstance(flow_id, str) else None + if builder is None: + issues.append(_issue("unknown_flow", "/flow/flowId")) + return set() + steps = [_enum_value(step) for step, _tool, _state in builder()] + start = flow.get("fromStepId") + end = flow.get("throughStepId") + if (start is None) != (end is None): + issues.append(_issue("flow_boundaries_incomplete", "/flow")) + return set(steps) + if start is None: + return set(steps) + if start not in steps: + issues.append(_issue("unknown_flow_boundary", "/flow/fromStepId")) + if end not in steps: + issues.append(_issue("unknown_flow_boundary", "/flow/throughStepId")) + if start in steps and end in steps: + start_index, end_index = steps.index(start), steps.index(end) + if start_index > end_index: + issues.append(_issue("flow_boundary_reversed", "/flow")) + else: + return set(steps[start_index : end_index + 1]) + return set() + + +def _validate_parameter_applicability( + explicit: dict[str, Any], + flow_steps: set[str], + issues: list[dict[str, Any]], +) -> None: + for schema in list_schemas(): + if schema.param not in explicit: + continue + if schema.applies == "all": + continue + if flow_steps and not _parameter_applies_to_flow(schema.applies, flow_steps): + issues.append( + _issue( + "inapplicable_parameter", + f"/parameters/{_pointer(schema.param)}", + appliesTo=schema.applies, + ) + ) + + +def _effective_parameter_values(resolved, steps: set[str]) -> dict[str, object]: + return { + parameter.param: deepcopy(parameter.value) + for parameter in resolved + if parameter.schema.pdk_target is None + and ( + parameter.schema.applies == "all" + or _parameter_applies_to_flow(parameter.schema.applies, steps) + ) + } + + +def _parameter_applies_to_flow(applies: str, steps: set[str]) -> bool: + normalized = {normalize_flow_step(step).casefold() for step in steps} + aliases = { + "synthesis": {"synthesis"}, + "floorplan": {"prefloorplan", "macroplacement", "postfloorplan"}, + "placement": {"placement", "place", "macroplacement"}, + "routing": {"routing", "route"}, + "fixfanout": {"fixfanout"}, + } + candidates = aliases.get(applies.casefold(), {normalize_flow_step(applies).casefold()}) + return bool(normalized & candidates) + + +def _refs( + value: object, + id_key: str, + roles: frozenset[str], + path: str, + issues: list[dict[str, Any]], +) -> list[dict[str, str]]: + if not isinstance(value, list): + issues.append(_issue("invalid_type", path, expected="array")) + return [] + refs = [] + seen = set() + for index, value_item in enumerate(value): + item_path = f"{path}/{index}" + if not isinstance(value_item, dict): + issues.append(_issue("invalid_type", item_path, expected="object")) + continue + item = {str(key): field for key, field in value_item.items()} + _unknown_fields(item, {id_key, "role"}, item_path, issues) + item_id = item.get(id_key) + role = item.get("role") + if not _nonempty_string(item_id): + issues.append(_issue("invalid_value", f"{item_path}/{id_key}")) + continue + if item_id in seen: + issues.append(_issue("duplicate_ref", f"{item_path}/{id_key}")) + continue + seen.add(item_id) + if role not in roles: + issues.append(_issue("unknown_role", f"{item_path}/role")) + continue + refs.append({id_key: item_id, "role": role}) + return refs + + +def _input_bindings(value: object, issues: list[dict[str, Any]]) -> dict[str, str]: + if not isinstance(value, dict): + issues.append(_issue("invalid_type", "/bindings/inputs", expected="object")) + return {} + return { + key: path for key, path in value.items() if isinstance(key, str) and isinstance(path, str) + } + + +def _mapping(value: object, path: str, issues: list[dict[str, Any]]) -> dict[str, Any]: + if not isinstance(value, dict): + issues.append(_issue("invalid_type", path, expected="object")) + return {} + return {str(key): item for key, item in value.items()} + + +def _unknown_fields( + value: dict[str, Any], + allowed: set[str] | frozenset[str], + path: str, + issues: list[dict[str, Any]], +) -> None: + for key in value.keys() - allowed: + issues.append(_issue("unknown_field", f"{path}/{_pointer(key)}")) + + +def _required_string( + value: dict[str, Any], key: str, path: str, issues: list[dict[str, Any]] +) -> None: + if not _nonempty_string(value.get(key)): + issues.append(_issue("required", path)) + + +def _nonempty_string(value: object) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _enum_value(value: object) -> str: + return str(getattr(value, "value", value)) + + +def _pointer(value: str) -> str: + return value.replace("~", "~0").replace("/", "~1") + + +def _issue(code: str, path: str, **details: Any) -> dict[str, Any]: + return {"code": code, "path": path, "severity": "error", "details": details} diff --git a/chipcompiler/project/__init__.py b/chipcompiler/project/__init__.py new file mode 100644 index 000000000..98ff93c2e --- /dev/null +++ b/chipcompiler/project/__init__.py @@ -0,0 +1,15 @@ +from .api import ( + create_project_manifest, + create_project_workspace, + discover_project_manifest, + load_project_manifest, + mutate_project_manifest, +) + +__all__ = [ + "create_project_manifest", + "create_project_workspace", + "discover_project_manifest", + "load_project_manifest", + "mutate_project_manifest", +] diff --git a/chipcompiler/project/api.py b/chipcompiler/project/api.py new file mode 100644 index 000000000..7dfc6782b --- /dev/null +++ b/chipcompiler/project/api.py @@ -0,0 +1,292 @@ +import json +import shutil +from copy import deepcopy +from datetime import UTC, datetime +from pathlib import Path + +from chipcompiler.project.manifest import ( + _CANONICAL_TO_MANIFEST_STEP, + ManifestError, + load_manifest, +) +from chipcompiler.project.manifest_write import ( + build_project_document, + manifest_lock, + manifest_workspace_entry, + update_manifest, + update_manifest_locked, + write_manifest_if_absent, +) + + +def load_project_manifest(project_dir: str | Path) -> dict: + manifest = load_manifest(str(Path(project_dir).expanduser().resolve())) + document = deepcopy(manifest.raw) + document.update( + { + "schema_version": 1, + "project_id": manifest.project_id, + "name": manifest.name, + "design_name": manifest.design_name, + "root_path": manifest.project_dir, + "base_design": deepcopy(manifest.base_design), + "objectives": deepcopy(manifest.objectives), + "workspaces": [_workspace_document(item) for item in manifest.workspaces], + "qor_baseline": deepcopy(manifest.qor_baseline), + } + ) + return document + + +def discover_project_manifest(directory: str | Path) -> tuple[Path, dict] | None: + current = Path(directory).expanduser().resolve() + if not current.is_dir(): + current = current.parent + for project in (current, *current.parents): + if (project / "project.json").exists() or (project / "project.json").is_symlink(): + return project, load_project_manifest(project) + return None + + +def create_project_manifest( + project_dir: str | Path, + name: str, + design_name: str, + *, + now: str | None = None, + mpc: dict | None = None, +) -> dict: + project = Path(project_dir).expanduser().resolve() + project.mkdir(parents=True, exist_ok=True) + document = build_project_document( + str(project), + design_name=design_name, + base_design={"rtl_list": [], "parameters": {"design": design_name}}, + name=name, + now=now, + mpc=mpc, + ) + if not write_manifest_if_absent(str(project), document): + raise ManifestError(f"Project Manifest already exists: {project / 'project.json'}") + try: + return load_project_manifest(project) + except Exception: + (project / "project.json").unlink(missing_ok=True) + raise + + +def mutate_project_manifest(project_dir: str | Path, mutation: dict) -> dict: + project = Path(project_dir).expanduser().resolve() + load_project_manifest(project) + + apply = _project_manifest_mutator(project, mutation) + + if not update_manifest(str(project), apply): + raise ManifestError("Project Manifest update failed") + return load_project_manifest(project) + + +def _project_manifest_mutator(project: Path, mutation: dict): + def apply(document: dict) -> None: + kind = mutation.get("type") + if kind == "register_workspace": + _register_workspace(document, mutation, project) + elif kind in {"select_qor_baseline", "select_best_workspace"}: + _select_workspace(document, mutation) + elif kind == "archive_workspace": + _archive_workspace(document, mutation) + elif kind == "delete_workspace": + _delete_workspace(document, mutation) + else: + raise ManifestError(f"unsupported Project mutation: {kind}") + + return apply + + +def create_project_workspace( + project_dir: str | Path, + target_directory: str | Path, + spec: object, + bindings: object, + *, + command_id: str, + workspace_id: str | None = None, + name: str | None = None, + source_workspace_id: str | None = None, + expected_project_id: str | None = None, + now: str | None = None, +): + from chipcompiler.engine.reconcile import _workspace_lock + from chipcompiler.engine.workspace_lifecycle import _create_workspace_from_spec + + project = Path(project_dir).expanduser().resolve() + target = Path(target_directory).expanduser().resolve() + try: + target.relative_to(project) + except ValueError as exc: + raise ManifestError("Workspace must be inside the Project root") from exc + existed = target.exists() + with manifest_lock(project): + manifest = load_project_manifest(project) + if expected_project_id is not None and manifest["project_id"] != expected_project_id: + raise ManifestError("Project identity does not match") + try: + with _workspace_lock(target): + workspace = _create_workspace_from_spec(str(target), spec, bindings, command_id) + if workspace is None: + raise ManifestError("Workspace creation returned no Workspace") + identity = workspace_id or target.name + mutation = { + "type": "register_workspace", + "workspace_id": identity, + "name": name or identity, + "workspace_path": str(target), + "source_workspace_id": source_workspace_id, + "created_at": now, + "updated_at": now, + } + if not update_manifest_locked( + project, _project_manifest_mutator(project, mutation) + ): + raise ManifestError("Project Manifest update failed") + except Exception: + if not existed: + shutil.rmtree(target, ignore_errors=True) + raise + return workspace + + +def _workspace_document(workspace) -> dict: + document = deepcopy(workspace.raw) + document.update( + { + "workspace_id": workspace.workspace_id, + "workspace_path": workspace.workspace_path, + "start_step": workspace.start_step, + "end_step": workspace.end_step, + "status": workspace.status, + "parameter_patch": deepcopy(workspace.parameter_patch), + } + ) + return document + + +def _register_workspace(document: dict, mutation: dict, project: Path) -> None: + workspace_id = _required_string(mutation, "workspace_id") + workspace_path = _resolve_workspace(project, _required_string(mutation, "workspace_path")) + for existing in document.get("workspaces", []): + same_id = existing.get("workspace_id") == workspace_id + same_path = ( + _resolve_workspace(project, str(existing.get("workspace_path", ""))) == workspace_path + ) + if same_id and same_path: + return + if same_id or same_path: + raise ManifestError(f"Workspace registration conflicts with: {workspace_id}") + start_step, end_step = _workspace_range(workspace_path, mutation) + timestamp = str( + mutation.get("updated_at") or mutation.get("created_at") or datetime.now(UTC).isoformat() + ) + entry = manifest_workspace_entry( + workspace_id, + name=str(mutation.get("name") or workspace_id), + workspace_path=str(workspace_path), + start_step=start_step, + end_step=end_step, + status="archived" if mutation.get("lifecycle") == "archived" else "not_started", + now=timestamp, + ) + for key in ( + "source_workspace_id", + "branch_from", + "parameter_patch", + "metrics_summary", + "step_metrics", + ): + if key in mutation: + entry[key] = deepcopy(mutation[key]) + document.setdefault("workspaces", []).append(entry) + document["updated_at"] = timestamp + + +def _workspace_range(workspace: Path, mutation: dict) -> tuple[str, str]: + start = mutation.get("start_step") + end = mutation.get("end_step") + if isinstance(start, str) and isinstance(end, str) and start and end: + return start, end + try: + ledger = json.loads((workspace / "home" / "flow.json").read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return "Synth", "Harden" + names = [ + step.get("name") + for step in ledger.get("steps", []) + if isinstance(step, dict) and isinstance(step.get("name"), str) + ] + if not names: + return "Synth", "Harden" + return ( + _CANONICAL_TO_MANIFEST_STEP.get(names[0], names[0]), + _CANONICAL_TO_MANIFEST_STEP.get(names[-1], names[-1]), + ) + + +def _select_workspace(document: dict, mutation: dict) -> None: + workspace_id = _required_string(mutation, "workspace_id") + workspace = _find_workspace(document, workspace_id) + if workspace.get("status") == "archived": + raise ManifestError(f"Workspace is not active: {workspace_id}") + field = "qor_baseline" if mutation["type"] == "select_qor_baseline" else "best_workspace" + document[field] = {"workspace_id": workspace_id, "reason": str(mutation.get("reason") or "")} + document["updated_at"] = str(mutation.get("updated_at") or datetime.now(UTC).isoformat()) + + +def _archive_workspace(document: dict, mutation: dict) -> None: + workspace = _find_workspace(document, _required_string(mutation, "workspace_id")) + workspace["status"] = "archived" + timestamp = str(mutation.get("updated_at") or datetime.now(UTC).isoformat()) + workspace["updated_at"] = timestamp + document["updated_at"] = timestamp + + +def _delete_workspace(document: dict, mutation: dict) -> None: + workspace_id = _required_string(mutation, "workspace_id") + _find_workspace(document, workspace_id) + document["workspaces"] = [ + workspace + for workspace in document.get("workspaces", []) + if workspace.get("workspace_id") != workspace_id + ] + for workspace in document["workspaces"]: + if workspace.get("source_workspace_id") == workspace_id: + workspace["source_workspace_id"] = None + for field in ("qor_baseline", "best_workspace"): + selection = document.get(field) + if isinstance(selection, dict) and selection.get("workspace_id") == workspace_id: + document[field] = None + document["updated_at"] = str(mutation.get("updated_at") or datetime.now(UTC).isoformat()) + + +def _find_workspace(document: dict, workspace_id: str) -> dict: + for workspace in document.get("workspaces", []): + if isinstance(workspace, dict) and workspace.get("workspace_id") == workspace_id: + return workspace + raise ManifestError(f"Workspace is not registered: {workspace_id}") + + +def _required_string(value: dict, key: str) -> str: + result = value.get(key) + if not isinstance(result, str) or not result.strip(): + raise ManifestError(f"Project Manifest {key} is required") + return result.strip() + + +def _resolve_workspace(project: Path, declared: str) -> Path: + workspace = Path(declared) + workspace = workspace if workspace.is_absolute() else project / workspace + resolved = workspace.resolve() + try: + resolved.relative_to(project) + except ValueError as exc: + raise ManifestError("Workspace must be inside the Project root") from exc + return resolved diff --git a/chipcompiler/project/locking.py b/chipcompiler/project/locking.py new file mode 100644 index 000000000..6a8f4082e --- /dev/null +++ b/chipcompiler/project/locking.py @@ -0,0 +1,14 @@ +"""Project-domain file lock helper.""" + +import fcntl +from contextlib import contextmanager + + +@contextmanager +def flock_file(path: str, *, exclusive: bool = True): + with open(path, "a") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) + try: + yield lock_file + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) diff --git a/chipcompiler/project/manifest.py b/chipcompiler/project/manifest.py new file mode 100644 index 000000000..1df09e9ba --- /dev/null +++ b/chipcompiler/project/manifest.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python + +"""``project.json`` manifest support for the CLI. + +The manifest is the GUI's project descriptor (schema v1). The CLI reads it +for configuration layering and run discovery, and projects it into +configuration payloads. Write operations (generation, status write-back, +registration) live in chipcompiler.project.manifest_write, which +routes every write through one read-modify-write helper. + +This module sits on the CLI startup path (imported by +cli/core/invocation.py): keep module-level imports cheap — no +chipcompiler.data imports here. +""" + +import json +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +MANIFEST_FILENAME = "project.json" + +# GUI display names for the canonical rtl2gds chain. +MANIFEST_FLOW_STEPS = ( + "Synth", + "LEC", + "PreFloorplan", + "MacroPlacement", + "PostFloorplan", + "Place", + "CTS", + "Legal", + "TimingOpt", + "Route", + "Filler", + "RCX", + "STA", + "LVS", + "PostRouteLEC", + "DRC", + "Harden", +) + +# ``Floor`` was the public manifest value before floorplanning was split into +# pre/macro/post stages. Read it as the completed handoff stage so old ranges +# keep their original start/end meaning without rewriting project.json. +_MANIFEST_STEP_ALIASES = {"Floor": "PostFloorplan"} + +PRESET_MANIFEST_RANGE = { + "syn_sta": ("Synth", "Synth"), + "rtl2gds": ("Synth", "Harden"), + "synthesis_lec": ("Synth", "LEC"), + # Legacy presets removed from the builder; keep resolving them so + # persisted projects still load. + "rcx": ("Synth", "STA"), + "harden": ("Synth", "Harden"), +} + +_CANONICAL_TO_MANIFEST_STEP = { + "Synthesis": "Synth", + "lec": "LEC", + "preFloorplan": "PreFloorplan", + "macroPlacement": "MacroPlacement", + "postFloorplan": "PostFloorplan", + "Floorplan": "PostFloorplan", + "place": "Place", + "CTS": "CTS", + "legalization": "Legal", + "Timing optimization": "TimingOpt", + "route": "Route", + "filler": "Filler", + "RCX": "RCX", + "sta": "STA", + "lvs": "LVS", + "postRouteLec": "PostRouteLEC", + "drc": "DRC", + "Harden": "Harden", +} + +_WORKSPACE_STATUSES = frozenset( + {"success", "failed", "running", "in_progress", "not_started", "archived"} +) + + +class ManifestError(ValueError): + """Raised when a project.json manifest cannot be used (manifest_invalid).""" + + +@dataclass(frozen=True) +class ManifestWorkspace: + workspace_id: str + workspace_path: str + start_step: str + end_step: str + status: str + parameter_patch: dict = field(default_factory=dict) + raw: dict = field(default_factory=dict) + + +@dataclass(frozen=True) +class ProjectManifest: + project_dir: str + path: str + project_id: str + name: str + design_name: str + base_design: dict + objectives: dict + workspaces: tuple[ManifestWorkspace, ...] + qor_baseline: dict | None + raw: dict + + def active_workspaces(self) -> list[ManifestWorkspace]: + return [w for w in self.workspaces if w.status != "archived"] + + def find_workspace(self, workspace_id: str) -> ManifestWorkspace | None: + """Match a managed workspace by its declared identifier only.""" + for workspace in self.workspaces: + if workspace.workspace_id == workspace_id: + return workspace + return None + + +def _optional_str(value: Any) -> str: + return value.strip() if isinstance(value, str) else "" + + +def _record(value: Any) -> dict: + return value if isinstance(value, dict) else {} + + +def _normalize_workspace_entry(value: Any, index: int, project_dir: str) -> ManifestWorkspace: + source = _record(value) + workspace_id = _optional_str(source.get("workspace_id")) + workspace_path = _optional_str(source.get("workspace_path")) + if not workspace_id or not workspace_path: + raise ManifestError(f"workspaces[{index}] requires workspace_id and workspace_path") + resolved = Path(workspace_path) + if not resolved.is_absolute(): + resolved = Path(project_dir) / resolved + try: + canonical = resolved.resolve() + canonical.relative_to(Path(project_dir).resolve()) + except ValueError: + raise ManifestError( + f"workspaces[{index}] workspace_path escapes the project root: {workspace_path}" + ) from None + except RuntimeError as exc: + # e.g. a symlink loop inside the spelled path: invalid manifest + # input, never a traceback. + raise ManifestError( + f"workspaces[{index}] workspace_path cannot be resolved: {workspace_path}" + ) from exc + status = source.get("status") + if not isinstance(status, str) or status not in _WORKSPACE_STATUSES: + status = "not_started" + start_step = _optional_str(source.get("start_step")) or "Synth" + end_step = _optional_str(source.get("end_step")) or "Harden" + start_step = _MANIFEST_STEP_ALIASES.get(start_step, start_step) + end_step = _MANIFEST_STEP_ALIASES.get(end_step, end_step) + for step_name, field_name in ((start_step, "start_step"), (end_step, "end_step")): + if step_name not in MANIFEST_FLOW_STEPS: + raise ManifestError( + f"workspaces[{index}] {field_name} is not on the canonical flow chain: {step_name}" + ) + if MANIFEST_FLOW_STEPS.index(start_step) > MANIFEST_FLOW_STEPS.index(end_step): + raise ManifestError( + f"workspaces[{index}] flow range is reversed: {start_step} -> {end_step}" + ) + return ManifestWorkspace( + workspace_id=workspace_id, + workspace_path=str(canonical), + start_step=start_step, + end_step=end_step, + status=status, + parameter_patch=_record(source.get("parameter_patch")), + raw=dict(source), + ) + + +def _validate_mpc(value: Any) -> None: + """Mirror the GUI parser's mpc rules: null or a well-formed MPC record.""" + if value is None: + return + source = _record(value) + if not source: + raise ManifestError("invalid project manifest: mpc must be an object or null") + resource_id = _optional_str(source.get("resource_id")) + if not resource_id.startswith("mpc:") or len(resource_id) == 4: + raise ManifestError("invalid project manifest: mpc.resource_id must be an MPC id") + for field_name in ("display_name", "installed_version", "path", "spec_path"): + if not _optional_str(source.get(field_name)): + raise ManifestError(f"invalid project manifest: mpc.{field_name} is required") + mpc_path = source["path"].rstrip("/") + if source["spec_path"] != f"{mpc_path}/spec/spec.json.in": + raise ManifestError( + "invalid project manifest: mpc.spec_path must reference spec/spec.json.in" + ) + design = _record(source.get("design")) + index = design.get("index") + if ( + not design + or isinstance(index, bool) # a JSON boolean is not an index (True == 1 in Python) + or not isinstance(index, (int, float)) + # The GUI accepts integral numbers (0.0). is_integer() applies to + # floats only: float(huge_int) raises OverflowError, and ints are + # integral by construction. + or (isinstance(index, float) and not index.is_integer()) + or index < 0 + or not _optional_str(design.get("design_name")) + ): + raise ManifestError( + "invalid project manifest: mpc.design requires a non-negative index and design_name" + ) + if not isinstance(source.get("core_template"), dict): + raise ManifestError("invalid project manifest: mpc.core_template must be an object") + + +def load_manifest(project_dir: str) -> ProjectManifest: + """Load and tolerantly normalize ``/project.json``. + + Mirrors the GUI parser's contract: schema_version 1 and a workspaces + array are required, everything else is default-filled. Raises + ManifestError on parse failure, root_path mismatch, or a workspace + path outside the project root. + """ + path = os.path.join(project_dir, MANIFEST_FILENAME) + try: + with open(path, encoding="utf-8") as f: + source = json.load(f) + except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc: + raise ManifestError(f"invalid project manifest: {path}: {exc}") from exc + + if not isinstance(source, dict): + raise ManifestError(f"invalid project manifest: {path}: top level must be an object") + schema_version = source.get("schema_version") + # A JSON boolean is not the schema version (True == 1 in Python); the + # GUI parser rejects it, and the CLI must not open what the GUI cannot. + if isinstance(schema_version, bool) or schema_version != 1: + raise ManifestError("invalid project manifest: schema_version 1 is required") + raw_workspaces = source.get("workspaces") + if not isinstance(raw_workspaces, list): + raise ManifestError("invalid project manifest: workspaces must be an array") + + root_path = _optional_str(source.get("root_path")) + if not root_path: + raise ManifestError("invalid project manifest: root_path is required") + if os.path.realpath(root_path) != os.path.realpath(project_dir): + raise ManifestError( + f"invalid project manifest: root_path {root_path} does not match {project_dir}" + ) + design_name = _optional_str(source.get("design_name")) + if not design_name: + raise ManifestError("invalid project manifest: design_name is required") + + name = _optional_str(source.get("name")) or os.path.basename(project_dir) or "project" + base_design = _record(source.get("base_design")) + base_design = {**base_design, "parameters": _record(base_design.get("parameters"))} + # Mirror the GUI parser: primary defaults to "timing", directions keep + # only maximize/minimize entries from the source (no default fill). + objectives_raw = _record(source.get("objectives")) + objectives = dict(objectives_raw) + objectives["primary"] = _optional_str(objectives_raw.get("primary")) or "timing" + objectives["directions"] = { + key: value + for key, value in _record(objectives_raw.get("directions")).items() + if value in ("maximize", "minimize") + } + qor_baseline_raw = _record(source.get("qor_baseline")) + qor_baseline = None + if _optional_str(qor_baseline_raw.get("workspace_id")): + qor_baseline = { + "workspace_id": qor_baseline_raw["workspace_id"], + "reason": _optional_str(qor_baseline_raw.get("reason")) or "Project QoR baseline", + } + + _validate_mpc(source.get("mpc")) + + return ProjectManifest( + project_dir=project_dir, + path=path, + project_id=_optional_str(source.get("project_id")) or f"proj_{_slugify(name)}", + name=name, + design_name=design_name, + base_design=base_design, + objectives=objectives, + workspaces=tuple( + _normalize_workspace_entry(entry, index, project_dir) + for index, entry in enumerate(raw_workspaces) + ), + qor_baseline=qor_baseline, + raw=source, + ) + + +def find_manifest(project_dir: str) -> str | None: + """The manifest path when the entry lexically exists. + + Lexical presence, not readability: a directory, symlink loop, or + dangling symlink still counts as PRESENT so ``load_manifest`` fails + loud (manifest_invalid) instead of silently demoting the project to + the virgin or legacy layout. + """ + path = os.path.join(project_dir, MANIFEST_FILENAME) + return path if os.path.lexists(path) else None + + +def has_legacy_runs_layout(project_dir: str) -> bool: + runs_dir = os.path.join(project_dir, "runs") + if not os.path.isdir(runs_dir): + return False + try: + return any(os.path.isdir(os.path.join(runs_dir, entry)) for entry in os.listdir(runs_dir)) + except OSError: + return False + + +def classify_project(project_dir: str) -> str: + """Classify a project directory: manifest | legacy | virgin.""" + if find_manifest(project_dir) is not None: + return "manifest" + if has_legacy_runs_layout(project_dir): + return "legacy" + return "virgin" + + +def assemble_config(manifest: ProjectManifest, workspace: ManifestWorkspace | None) -> dict: + """Flatten the manifest into a parameter payload (lowest precedence layer). + + ``base_design.parameters`` plus the workspace's ``parameter_patch`` form + the base layer beneath project ecc.toml and --set overrides. + """ + parameters = dict(manifest.base_design.get("parameters") or {}) + if workspace is not None: + for key, change in workspace.parameter_patch.items(): + parameters[key] = ( + change["to"] if isinstance(change, dict) and "to" in change else change + ) + if manifest.design_name and not _optional_str(parameters.get("design")): + parameters["design"] = manifest.design_name + rtl_list = manifest.base_design.get("rtl_list") + if not isinstance(rtl_list, list): + rtl_list = [] + return { + "pdk": _optional_str(manifest.base_design.get("pdk")), + "pdk_root": _optional_str(manifest.base_design.get("pdk_root")), + "design_name": manifest.design_name, + "top_module": _optional_str(manifest.base_design.get("top_module")), + "clock": _optional_str(manifest.base_design.get("clock")), + "rtl_list": [item for item in rtl_list if isinstance(item, str)], + "origin_verilog": _optional_str(manifest.base_design.get("origin_verilog")), + "origin_def": _optional_str(manifest.base_design.get("origin_def")), + "netlist": _optional_str(manifest.base_design.get("netlist")), + "golden_netlist": _optional_str(manifest.base_design.get("golden_netlist")), + "sdc": _optional_str(manifest.base_design.get("sdc")), + "spef": _optional_str(manifest.base_design.get("spef")), + "parameters": parameters, + } + + +def resolved_base_parameters(cfg) -> dict: + """The ecc.toml-resolved base_design.parameters for a generated manifest. + + GUI-flat vocabulary: identity fields plus the [params] overrides, + projected through the geometry converter so positional values surface + as the wizard's aliases (die_width, utilitization, margin, ...). + --set values are run-scoped and never included. + """ + canonical: dict = { + "design": cfg.design_name, + "top_module": cfg.design_top, + "clock": cfg.design_clock_port, + "frequency_max": cfg.design_frequency_mhz, + } + if cfg.params_overrides: + from chipcompiler.data.parameter_schema import ( + build_backend_overrides, + resolve_parameters, + ) + + resolved, _ = resolve_parameters(toml_overrides=cfg.params_overrides) + canonical.update(build_backend_overrides(resolved)) + + from chipcompiler.data.parameter_keys import parameters_to_geometry + + flat = parameters_to_geometry(canonical) + # Exclusive GUI-flat shape: geometry lives only in the aliases — + # the canonical die/core subtrees are consumed, not duplicated. + # Non-positional members (e.g. aspect_ratio) hoist to flat top-level + # keys; positional members are covered by the aliases. + for subtree_name in ("die", "core"): + subtree = flat.pop(subtree_name, None) + if not isinstance(subtree, dict): + continue + for member, value in subtree.items(): + if member in ("size", "utilitization", "margin"): + continue + flat.setdefault(member, value) + return flat + + +def base_design_from_config(cfg, pdk_root: str) -> dict: + """The base_design document for a generated manifest. + + Identity and sources come from the ecc.toml-resolved config with the + DECLARED project source spellings preserved: ``rtl_list`` verbatim, + and ``origin_verilog`` when the single source is plain RTL (empty for + a filelist source; the document builder drops empty keys). Parameters + are the GUI-flat projection. Shared by virgin generation and first + migration so the two writers cannot drift. + """ + from chipcompiler.utility.filelist import FILELIST_SUFFIXES + + first_rtl = cfg.design_rtl[0] if cfg.design_rtl else "" + suffix = os.path.splitext(first_rtl)[1].lower() + origin_verilog = first_rtl if first_rtl and suffix not in FILELIST_SUFFIXES else "" + return { + "pdk": cfg.pdk_name, + "pdk_root": pdk_root, + "top_module": cfg.design_top, + "clock": cfg.design_clock_port, + "rtl_list": cfg.design_rtl, + "origin_verilog": cfg.design_rtl[0] if origin_verilog else "", + "origin_def": cfg.design_def, + "netlist": cfg.design_netlist, + "golden_netlist": cfg.design_golden_netlist, + "sdc": cfg.design_sdc, + "spef": cfg.design_spef, + "parameters": resolved_base_parameters(cfg), + } + + +def _slugify(value: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "_", value.strip().lower()).strip("_") + return slug or "project" diff --git a/chipcompiler/project/manifest_write.py b/chipcompiler/project/manifest_write.py new file mode 100644 index 000000000..71714bb11 --- /dev/null +++ b/chipcompiler/project/manifest_write.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python + +"""Manifest write, mutation, and registration operations for the CLI. + +All ``project.json`` writes go through one read-modify-write helper so +status write-back and migration entry-append share the same atomicity +story. Loading and normalization live in +chipcompiler.project.manifest; this module imports from it, never +the reverse. + +Like manifest.py, this module sits on the CLI startup path (imported by +run dispatch and migration flows): keep module-level imports cheap — no +chipcompiler.data imports here. +""" + +import json +import logging +import os +import tempfile +from contextlib import contextmanager +from copy import deepcopy +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from chipcompiler.project.manifest import ( + _CANONICAL_TO_MANIFEST_STEP, + MANIFEST_FILENAME, + PRESET_MANIFEST_RANGE, + ManifestError, + _record, + _slugify, + base_design_from_config, +) + +logger = logging.getLogger(__name__) + +DEFAULT_OBJECTIVES = { + "primary": "timing", + "directions": { + "wns": "maximize", + "tns": "maximize", + "area": "minimize", + "drc_count": "minimize", + "lvs_count": "minimize", + "power": "minimize", + }, +} + + +def _now_iso() -> str: + return datetime.now(UTC).isoformat() + + +def manifest_workspace_entry( + workspace_id: str, + *, + name: str, + workspace_path: str, + start_step: str, + end_step: str, + status: str, + now: str, +) -> dict: + """One complete schema-v1 workspaces[] entry, every field materialized. + + The single builder for generated manifests and migration previews, so + the previewed entry and the applied entry are the same object shape. + """ + return { + "workspace_id": workspace_id, + "name": name, + "workspace_path": workspace_path, + "source_workspace_id": None, + "branch_from": None, + "start_step": start_step, + "end_step": end_step, + "status": status, + "created_at": now, + "updated_at": now, + "parameter_patch": {}, + "metrics_summary": {}, + "step_metrics": {}, + } + + +def build_manifest_document( + project_dir: str, + *, + design_name: str, + base_design: dict, + workspace_id: str, + workspace_path: str, + start_step: str, + end_step: str, + status: str = "running", +) -> dict: + """Assemble a schema-v1 manifest for a virgin project's first run.""" + now = _now_iso() + document = build_project_document( + project_dir, + design_name=design_name, + base_design=base_design, + now=now, + ) + document["workspaces"] = [ + manifest_workspace_entry( + workspace_id, + name=design_name, + workspace_path=workspace_path, + start_step=start_step, + end_step=end_step, + status=status, + now=now, + ) + ] + document["qor_baseline"] = { + "workspace_id": workspace_id, + "reason": "Default project QoR baseline", + } + return document + + +def build_project_document( + project_dir: str, + *, + design_name: str, + base_design: dict, + name: str | None = None, + now: str | None = None, + mpc: dict | None = None, +) -> dict[str, Any]: + """Assemble a schema-v1 Project Manifest without a Workspace.""" + timestamp = now or _now_iso() + project_name = name or os.path.basename(os.path.normpath(project_dir)) or "project" + return { + "schema_version": 1, + "project_id": f"proj_{_slugify(project_name)}", + "name": project_name, + "design_name": design_name, + "description": "", + "root_path": project_dir, + "created_at": timestamp, + "updated_at": timestamp, + "base_design": { + **{key: value for key, value in base_design.items() if key != "parameters" and value}, + "parameters": _record(base_design.get("parameters")), + "rtl_list": [ + item for item in base_design.get("rtl_list") or [] if isinstance(item, str) + ], + }, + "objectives": json.loads(json.dumps(DEFAULT_OBJECTIVES)), + "workspaces": [], + "mpc": deepcopy(mpc), + "best_workspace": None, + "qor_baseline": None, + } + + +def write_manifest_if_absent(project_dir: str, document: dict) -> bool: + """Write the manifest only when it does not exist (virgin generation race). + + Fully written and fsynced at a temp path, then linked into place: + readers never see a partial file, and a concurrent creator wins the + link — ours is discarded and the caller continues read-only. + """ + path = os.path.join(project_dir, MANIFEST_FILENAME) + content = json.dumps(document, indent=2) + "\n" + tmp_path = None + try: + with tempfile.NamedTemporaryFile( + "w", + dir=project_dir, + delete=False, + prefix=f".{MANIFEST_FILENAME}.", + suffix=".tmp", + encoding="utf-8", + ) as f: + tmp_path = f.name + f.write(content) + f.flush() + os.fsync(f.fileno()) + # Mode stays the tempfile default (0600), matching json_write's + # convention for newly created state files. + os.link(tmp_path, path) + return True + except FileExistsError: + return False + except OSError as exc: + logger.warning("manifest write failed: %s: %s", path, exc) + return False + finally: + if tmp_path is not None: + Path(tmp_path).unlink(missing_ok=True) + + +def _read_manifest_document(path: str): + try: + with open(path, encoding="utf-8") as f: + document = json.load(f) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return None + return document if isinstance(document, dict) else None + + +def update_manifest(project_dir: str, mutator) -> bool: + """Read-modify-write the manifest atomically (locked re-read + patch + replace). + + The whole read-modify-replace runs under ``.manifest.lock`` (flock): + two cooperating writers can no longer both complete the fresh read + before either replaces, so neither loses the other's update. The + mutator receives the parsed document and edits it in place. When an + unrelated change lands between the read and the write, the mutator is + re-applied to the freshest document instead of overwriting the change. + Project-level fields (including updated_at) are owned by the mutator. + Returns False (with a warning) when the manifest is missing, unreadable, + the lock cannot be taken, or the write fails — callers degrade to a + warning, never a run failure. + """ + path = os.path.join(project_dir, MANIFEST_FILENAME) + try: + with manifest_lock(project_dir): + return update_manifest_locked(project_dir, mutator) + except OSError as exc: + # An untakeable lock (e.g. a directory at the lock path) degrades + # like any write failure: a warning, never an uncaught exception — + # the migration registration path relies on False to roll back. + logger.warning("manifest update failed: %s: %s", path, exc) + return False + + +@contextmanager +def manifest_lock(project_dir: str | Path): + from chipcompiler.project.locking import flock_file + + yield_lock_path = os.path.join(str(project_dir), ".manifest.lock") + with flock_file(yield_lock_path, exclusive=True): + yield + + +def update_manifest_locked(project_dir: str | Path, mutator) -> bool: + """Apply a manifest mutation while the caller owns ``manifest_lock``.""" + path = os.path.join(str(project_dir), MANIFEST_FILENAME) + return _update_manifest_locked(path, mutator) + + +def _update_manifest_locked(path: str, mutator) -> bool: + base = _read_manifest_document(path) + if base is None: + logger.warning("manifest update skipped (unreadable): %s", path) + return False + + document = deepcopy(base) + mutator(document) + + fresh = _read_manifest_document(path) + if fresh is not None and fresh != base: + # An unrelated edit landed after our read: re-apply the mutator to + # the freshest document so the interleaved change survives. + document = fresh + mutator(document) + + target = Path(path) + tmp_path = None + try: + with tempfile.NamedTemporaryFile( + "w", + dir=target.parent, + delete=False, + prefix=f".{target.name}.", + suffix=".tmp", + encoding="utf-8", + ) as f: + tmp_path = Path(f.name) + json.dump(document, f, indent=2) + f.write("\n") + f.flush() + os.fsync(f.fileno()) + # Preserve the existing manifest's permissions: mkstemp's 0600 must + # not silently narrow a shared project.json. + if target.exists(): + os.chmod(tmp_path, target.stat().st_mode & 0o7777) + os.replace(tmp_path, target) + return True + except OSError as exc: + logger.warning("manifest update failed: %s: %s", path, exc) + if tmp_path is not None: + tmp_path.unlink(missing_ok=True) + return False + + +def write_back_workspace_status(project_dir: str, workspace_id: str, status: str) -> bool: + """Update one workspace entry's status (and updated_at) after a run.""" + + def mutate(document: dict) -> None: + for entry in document.get("workspaces", []): + if isinstance(entry, dict) and entry.get("workspace_id") == workspace_id: + entry["status"] = status + entry["updated_at"] = _now_iso() + + return update_manifest(project_dir, mutate) + + +def remove_workspace_registration(project_dir: str, workspace_id: str) -> bool: + """Roll back a pre-registration: drop the freshly added entry. + + Used when an overwrite run against an undeclared workspace fails before + the replacement is constructed: the restored previous workspace must not + be shadowed by a stale ``not_started`` entry this invocation created. + """ + + def mutate(document: dict) -> None: + workspaces = document.get("workspaces") + if isinstance(workspaces, list): + document["workspaces"] = [ + entry + for entry in workspaces + if not (isinstance(entry, dict) and entry.get("workspace_id") == workspace_id) + ] + document["updated_at"] = _now_iso() + + return update_manifest(project_dir, mutate) + + +def manifest_range_for_flow(cfg, flow_config: dict | None) -> tuple[str, str]: + """Return the GUI manifest range for a workspace's effective target.""" + if isinstance(flow_config, dict) and flow_config.get("start_step"): + from chipcompiler.rtl2gds import normalize_flow_step + + start = normalize_flow_step(flow_config["start_step"]) + end = normalize_flow_step(flow_config.get("end_step") or start) + try: + return (_CANONICAL_TO_MANIFEST_STEP[start], _CANONICAL_TO_MANIFEST_STEP[end]) + except KeyError as exc: + raise ManifestError(f"unknown workspace flow step: {exc.args[0]}") from None + return PRESET_MANIFEST_RANGE.get(cfg.flow_preset, ("Synth", "Harden")) + + +def pre_register_workspace( + project_dir: str, + *, + cfg, + pdk_root: str, + workspace_id: str, + workspace_path: str, + flow_config: dict | None, +) -> str: + """Atomically register a fresh managed workspace before filesystem creation. + + Returns ``registered``, ``existing``, ``conflict``, or ``failed``. A + workspace entry intentionally contains no input snapshot: copied files and + the workspace config are the reproducibility boundary. + """ + try: + start_step, end_step = manifest_range_for_flow(cfg, flow_config) + except ManifestError: + return "failed" + now = _now_iso() + manifest_path = os.path.join(project_dir, MANIFEST_FILENAME) + if not os.path.lexists(manifest_path): + document = build_manifest_document( + project_dir, + design_name=cfg.design_name, + base_design=base_design_from_config(cfg, pdk_root), + workspace_id=workspace_id, + workspace_path=workspace_path, + start_step=start_step, + end_step=end_step, + status="not_started", + ) + if write_manifest_if_absent(project_dir, document): + return "registered" + # A concurrent creator won the link race: fall through and apply the + # same registration mutation under the manifest lock instead of + # aborting this run. A genuine I/O failure fails again below. + + outcome = "registered" + + def mutate(document: dict) -> None: + nonlocal outcome + workspaces = document.get("workspaces") + if not isinstance(workspaces, list): + outcome = "failed" + return + for entry in workspaces: + if not isinstance(entry, dict) or entry.get("workspace_id") != workspace_id: + continue + if os.path.realpath(str(entry.get("workspace_path", ""))) == os.path.realpath( + workspace_path + ): + outcome = "existing" + else: + outcome = "conflict" + return + workspaces.append( + manifest_workspace_entry( + workspace_id, + name=cfg.design_name, + workspace_path=workspace_path, + start_step=start_step, + end_step=end_step, + status="not_started", + now=now, + ) + ) + document["updated_at"] = now + + if not update_manifest(project_dir, mutate): + return "failed" + return outcome diff --git a/chipcompiler/rtl2gds/builder.py b/chipcompiler/rtl2gds/builder.py index 0b09cfc82..79f89d7c0 100644 --- a/chipcompiler/rtl2gds/builder.py +++ b/chipcompiler/rtl2gds/builder.py @@ -41,6 +41,9 @@ def normalize_flow_step(value: str | StepEnum) -> str: "synth": StepEnum.SYNTHESIS.value, "synthesis": StepEnum.SYNTHESIS.value, "prefloorplan": StepEnum.PRE_FLOORPLAN.value, + "floorplan": StepEnum.PRE_FLOORPLAN.value, + "floor": StepEnum.POST_FLOORPLAN.value, + "macro": StepEnum.MACRO_PLACEMENT.value, "macroplace": StepEnum.MACRO_PLACEMENT.value, "macroplacement": StepEnum.MACRO_PLACEMENT.value, "postfloorplan": StepEnum.POST_FLOORPLAN.value, diff --git a/chipcompiler/runtime/errors.py b/chipcompiler/runtime/errors.py new file mode 100644 index 000000000..64ace25c2 --- /dev/null +++ b/chipcompiler/runtime/errors.py @@ -0,0 +1,6 @@ +class RuntimeApiError(RuntimeError): + def __init__(self, code: str, message: str, data: dict | None = None): + super().__init__(message) + self.code = code + self.message = message + self.data = data or {} diff --git a/chipcompiler/runtime/methods.py b/chipcompiler/runtime/methods.py index f17f4e507..340aa8f1d 100644 --- a/chipcompiler/runtime/methods.py +++ b/chipcompiler/runtime/methods.py @@ -4,6 +4,7 @@ from chipcompiler.runtime.requests import ( DbEnsureRequest, DbReleaseRequest, + EmptyRequest, FloorplanEditInspectRequest, FloorplanEditRunAutoRequest, FloorplanEditValidateRequest, @@ -17,7 +18,11 @@ OperationIdRequest, OperationStartFlowRequest, OperationStartStepRequest, + ProjectManifestDiscoverRequest, + ProjectManifestLoadRequest, + ProjectManifestMutationRequest, WorkspaceCloseRequest, + WorkspaceConfigurationUpdateRequest, WorkspaceCreateRequest, WorkspaceExportSignoffRequest, WorkspaceIdRequest, @@ -25,7 +30,12 @@ WorkspaceInspectSignoffRequest, WorkspaceOpenRequest, WorkspaceRecoverInterruptedRequest, + WorkspaceSpecOpenRequest, + WorkspaceSpecValidateRequest, + WorkspaceStepConfigurationReadRequest, + WorkspaceStepConfigurationUpdateRequest, WorkspaceSyncConfigRequest, + WorkspaceUpdateRequest, ) RequestT = TypeVar("RequestT") @@ -39,6 +49,31 @@ class RuntimeMethodSpec(Generic[RequestT]): RUNTIME_METHODS: Final[tuple[RuntimeMethodSpec[Any], ...]] = ( + RuntimeMethodSpec( + method_name="workspace_spec.describe", + request_model=EmptyRequest, + handler_name="describe_workspace_spec", + ), + RuntimeMethodSpec( + method_name="workspace_spec.validate", + request_model=WorkspaceSpecValidateRequest, + handler_name="validate_workspace_spec", + ), + RuntimeMethodSpec( + method_name="project.discover", + request_model=ProjectManifestDiscoverRequest, + handler_name="discover_project", + ), + RuntimeMethodSpec( + method_name="project.manifest.load", + request_model=ProjectManifestLoadRequest, + handler_name="load_project_manifest", + ), + RuntimeMethodSpec( + method_name="project.manifest.mutate", + request_model=ProjectManifestMutationRequest, + handler_name="mutate_project_manifest", + ), RuntimeMethodSpec( method_name="workspace.create", request_model=WorkspaceCreateRequest, @@ -49,6 +84,36 @@ class RuntimeMethodSpec(Generic[RequestT]): request_model=WorkspaceOpenRequest, handler_name="open_workspace", ), + RuntimeMethodSpec( + method_name="workspace.binding_requirement", + request_model=WorkspaceSpecOpenRequest, + handler_name="workspace_binding_requirement", + ), + RuntimeMethodSpec( + method_name="workspace.update", + request_model=WorkspaceUpdateRequest, + handler_name="update_workspace", + ), + RuntimeMethodSpec( + method_name="workspace.configuration.update", + request_model=WorkspaceConfigurationUpdateRequest, + handler_name="update_workspace_configuration", + ), + RuntimeMethodSpec( + method_name="workspace.configuration.read", + request_model=WorkspaceOpenRequest, + handler_name="read_workspace_configuration", + ), + RuntimeMethodSpec( + method_name="workspace.step_configuration.update", + request_model=WorkspaceStepConfigurationUpdateRequest, + handler_name="update_workspace_step_configuration", + ), + RuntimeMethodSpec( + method_name="workspace.step_configuration.read", + request_model=WorkspaceStepConfigurationReadRequest, + handler_name="read_workspace_step_configuration", + ), RuntimeMethodSpec( method_name="workspace.close", request_model=WorkspaceCloseRequest, @@ -129,6 +194,11 @@ class RuntimeMethodSpec(Generic[RequestT]): request_model=WorkspaceIdRequest, handler_name="workspace_snapshot", ), + RuntimeMethodSpec( + method_name="workspace.engineering_snapshot", + request_model=WorkspaceIdRequest, + handler_name="engineering_snapshot", + ), RuntimeMethodSpec( method_name="workspace.recover_interrupted", request_model=WorkspaceRecoverInterruptedRequest, @@ -136,6 +206,26 @@ class RuntimeMethodSpec(Generic[RequestT]): ), ) +# These methods were added with the v1 workspace contract. Older embedders may +# provide only the legacy runtime API; expose a stable invalid-request result +# until they upgrade instead of making the whole RPC server unstartable. +OPTIONAL_RUNTIME_METHOD_NAMES: Final[frozenset[str]] = frozenset( + { + "workspace_spec.describe", + "workspace_spec.validate", + "project.discover", + "project.manifest.load", + "project.manifest.mutate", + "workspace.binding_requirement", + "workspace.update", + "workspace.configuration.update", + "workspace.configuration.read", + "workspace.step_configuration.update", + "workspace.step_configuration.read", + "workspace.engineering_snapshot", + } +) + PERSISTENT_DB_METHODS: Final[tuple[RuntimeMethodSpec[Any], ...]] = ( RuntimeMethodSpec( diff --git a/chipcompiler/runtime/operations.py b/chipcompiler/runtime/operations.py index eed9292a2..c18653dc7 100644 --- a/chipcompiler/runtime/operations.py +++ b/chipcompiler/runtime/operations.py @@ -1,5 +1,8 @@ from __future__ import annotations +import hashlib +import json +import logging import threading import time from collections.abc import Callable @@ -11,10 +14,10 @@ _LOG_POLL_INTERVAL_SECONDS = 0.25 _MAX_LOG_CHUNK_BYTES = 16 * 1024 _MAX_FINAL_LOG_BYTES = 64 * 1024 -_RENDER_ACK_RETRY_SECONDS = 5.0 -_RENDER_ACK_PAUSE_SECONDS = 30.0 -_RENDER_ACK_ABORT_SECONDS = 300.0 -_TERMINAL_OPERATION_STATES = frozenset({"succeeded", "failed", "cancelled"}) +_TERMINAL_OPERATION_STATES = frozenset({"succeeded", "failed", "cancelled", "interrupted"}) +_MAX_TERMINAL_OPERATIONS = 256 +_LEDGER_SCHEMA_VERSION = 1 +logger = logging.getLogger(__name__) @dataclass @@ -38,6 +41,10 @@ class RuntimeOperationCancelled(RuntimeError): """Cancellation was accepted at a safe step boundary.""" +class RuntimeOperationIdempotencyConflict(RuntimeError): + """A command ID was reused with different immutable input.""" + + @dataclass class RuntimeOperation: operation_id: str @@ -49,6 +56,7 @@ class RuntimeOperation: rerun: bool step: str = "" idempotency_key: str = "" + command_fingerprint: str = "" state: str = "queued" current_step: str = "" current_tool: str = "" @@ -63,9 +71,7 @@ class RuntimeOperation: workspace_revision: int = 0 render_sync_state: str = "idle" render_retry_count: int = 0 - render_wait_started_at: float | None = None last_render_ack_at: float | None = None - render_sync_degraded: bool = False acked_event_ids: set[str] = field(default_factory=set) cancel_requested: bool = False interruptibility: str = "deferred" @@ -77,13 +83,14 @@ class RuntimeOperationManager: def __init__(self, publisher: Callable[[dict[str, Any]], None] | None = None): self._publisher = publisher self._lock = threading.RLock() - self._render_gate = threading.Condition(self._lock) self._operations: dict[str, RuntimeOperation] = {} self._active_by_workspace: dict[str, str] = {} - self._idempotency: dict[tuple[str, str], str] = {} + self._idempotency: dict[tuple[str, str], tuple[str, str]] = {} self._step_log_tails: dict[str, _StepLogTail] = {} self._runtime_instance_id = uuid4().hex self._workspace_sequences: dict[str, int] = {} + self._ledger_paths: dict[str, Path] = {} + self._loaded_ledgers: set[Path] = set() def set_publisher(self, publisher: Callable[[dict[str, Any]], None] | None) -> None: with self._lock: @@ -99,16 +106,39 @@ def start( step: str, idempotency_key: str, runner: Callable[[RuntimeFlowObserver], dict[str, Any]], + workspace_revision: int = 0, + snapshot_committer: Callable[[Any, Any, str | None], int] | None = None, + command_input: dict[str, Any] | None = None, + ledger_path: str | Path | None = None, + precondition: Callable[[], None] | None = None, ) -> dict[str, Any]: + fingerprint = _command_fingerprint( + kind=kind, + origin=origin, + rerun=rerun, + step=step, + workspace_revision=workspace_revision, + command_input=command_input, + ) + if ledger_path is not None: + self.load_workspace_ledger(workspace_id, ledger_path) with self._lock: if idempotency_key: - known_id = self._idempotency.get((workspace_id, idempotency_key)) - if known_id is not None: + known = self._idempotency.get((workspace_id, idempotency_key)) + if known is not None: + known_id, known_fingerprint = known + if known_fingerprint and known_fingerprint != fingerprint: + raise RuntimeOperationIdempotencyConflict( + f"command id reused with different input: {idempotency_key}" + ) return { **self._operation_payload(self._operations[known_id]), "deduplicated": True, } + if precondition is not None: + precondition() + active_id = self._active_by_workspace.get(workspace_id) if active_id is not None: active = self._operations[active_id] @@ -126,17 +156,23 @@ def start( rerun=rerun, step=step, idempotency_key=idempotency_key, + command_fingerprint=fingerprint, + workspace_revision=workspace_revision, ) self._operations[operation.operation_id] = operation self._active_by_workspace[workspace_id] = operation.operation_id if idempotency_key: - self._idempotency[(workspace_id, idempotency_key)] = operation.operation_id + self._idempotency[(workspace_id, idempotency_key)] = ( + operation.operation_id, + fingerprint, + ) + self._persist_workspace_locked(workspace_id) queued_event = self._new_event_locked(operation, "operation.queued", {}) self._publish(queued_event) thread = threading.Thread( target=self._run, - args=(operation.operation_id, runner), + args=(operation.operation_id, runner, snapshot_committer), name=f"ecc-runtime-{operation.operation_id}", daemon=True, ) @@ -153,8 +189,90 @@ def operation_status(self, operation_id: str) -> dict[str, Any]: def is_active(self, operation_id: str) -> bool: with self._lock: operation = self._operations.get(operation_id) + return ( + operation is not None + and operation.state not in _TERMINAL_OPERATION_STATES + and self._active_by_workspace.get(operation.workspace_id) == operation_id + ) + + def has_active_workspace(self, workspace_id: str) -> bool: + with self._lock: + operation_id = self._active_by_workspace.get(workspace_id) + if operation_id is None: + return False + operation = self._operations.get(operation_id) return operation is not None and operation.state not in _TERMINAL_OPERATION_STATES + def load_workspace_ledger( + self, + workspace_id: str, + ledger_path: str | Path, + *, + recover: bool = True, + ) -> list[str]: + """Load the bounded execution ledger, optionally marking live work interrupted.""" + from chipcompiler.utility import JsonReadError, json_read_strict + + path = Path(ledger_path).expanduser().resolve() + with self._lock: + if path in self._loaded_ledgers: + if recover: + return self._recover_loaded_workspace_locked(workspace_id) + return [] + self._loaded_ledgers.add(path) + self._ledger_paths[workspace_id] = path + try: + payload = json_read_strict(path) + except (OSError, JsonReadError): + return [] + entries = payload.get("operations", []) if isinstance(payload, dict) else payload + if not isinstance(entries, list): + return [] + restored: list[str] = [] + for entry in entries: + if not isinstance(entry, dict) or entry.get("workspaceId") != workspace_id: + continue + operation = _operation_from_payload(entry, self._runtime_instance_id) + if operation is None: + continue + self._operations[operation.operation_id] = operation + if operation.idempotency_key: + self._idempotency[(workspace_id, operation.idempotency_key)] = ( + operation.operation_id, + operation.command_fingerprint, + ) + self._workspace_sequences[workspace_id] = max( + self._workspace_sequences.get(workspace_id, 0), operation.sequence + ) + restored.append(operation.operation_id) + if recover: + self._recover_loaded_workspace_locked(workspace_id) + return restored + + def _recover_loaded_workspace_locked(self, workspace_id: str) -> list[str]: + recovered: list[str] = [] + for operation in self._operations.values(): + if ( + operation.workspace_id != workspace_id + or operation.state in _TERMINAL_OPERATION_STATES + ): + continue + operation.state = "interrupted" + operation.error = { + "code": "interrupted", + "message": "Runtime process ended before the Operation completed", + } + operation.updated_at = time.time() + operation.awaiting_event_id = None + operation.awaiting_event = None + operation.awaiting_step_commit_id = None + operation.render_sync_state = "idle" + self._active_by_workspace.pop(workspace_id, None) + recovered.append(operation.operation_id) + self._prune_terminal_locked() + self._persist_workspace_locked(workspace_id) + return recovered + def workspace_snapshot(self, workspace_id: str) -> dict[str, Any]: with self._lock: operations = [ @@ -179,7 +297,7 @@ def acknowledge_step_rendered( step_commit_id: str = "", workspace_revision: int | None = None, ) -> dict[str, Any]: - with self._render_gate: + with self._lock: operation = self._operations.get(operation_id) if operation is None: raise KeyError(operation_id) @@ -219,10 +337,9 @@ def acknowledge_step_rendered( operation.awaiting_event = None operation.awaiting_step_commit_id = None operation.render_sync_state = "idle" - operation.render_wait_started_at = None operation.last_render_ack_at = time.time() operation.updated_at = time.time() - self._render_gate.notify_all() + self._persist_workspace_locked(operation.workspace_id) return { "accepted": True, "duplicate": False, @@ -231,19 +348,26 @@ def acknowledge_step_rendered( } def request_cancel(self, operation_id: str) -> dict[str, Any]: - with self._render_gate: + with self._lock: operation = self._operations.get(operation_id) if operation is None: raise KeyError(operation_id) if operation.state in _TERMINAL_OPERATION_STATES: return {"accepted": False, "operationId": operation_id, "state": operation.state} operation.cancel_requested = True + operation.state = "cancelling" operation.updated_at = time.time() + self._persist_workspace_locked(operation.workspace_id) event = self._new_event_locked(operation, "operation.cancel_requested", {}) - self._render_gate.notify_all() self._publish(event) return {"accepted": True, "operationId": operation_id, "state": operation.state} + def raise_if_cancel_requested(self, operation_id: str) -> None: + with self._lock: + operation = self._operations[operation_id] + if operation.cancel_requested: + raise RuntimeOperationCancelled("operation cancelled at a step boundary") + def shutdown_barrier(self) -> dict[str, Any] | None: with self._lock: for operation_id in self._active_by_workspace.values(): @@ -263,15 +387,37 @@ def _run( self, operation_id: str, runner: Callable[[RuntimeFlowObserver], dict[str, Any]], + snapshot_committer: Callable[[Any, Any, str | None], int] | None, ) -> None: with self._lock: operation = self._operations[operation_id] - operation.state = "running" - operation.updated_at = time.time() - started_event = self._new_event_locked(operation, "operation.started", {}) - observer = RuntimeFlowObserver(self, operation_id) + if operation.cancel_requested: + operation.state = "cancelled" + operation.error = { + "message": "operation cancelled before execution", + "code": "cancelled", + } + operation.updated_at = time.time() + event = self._new_event_locked( + operation, + "operation.cancelled", + {"error": operation.error}, + ) + self._prune_terminal_locked() + self._persist_workspace_locked(operation.workspace_id) + self._active_by_workspace.pop(operation.workspace_id, None) + publish_before_return = True + else: + operation.state = "running" + operation.updated_at = time.time() + event = self._new_event_locked(operation, "operation.started", {}) + publish_before_return = False + if publish_before_return: + self._publish(event) + return + observer = RuntimeFlowObserver(self, operation_id, snapshot_committer) try: - self._publish(started_event) + self._publish(event) try: result = runner(observer) with self._lock: @@ -286,6 +432,8 @@ def _run( "operation.completed", {"result": result}, ) + self._prune_terminal_locked() + self._persist_workspace_locked(operation.workspace_id) except RuntimeOperationCancelled as exc: with self._lock: operation = self._operations[operation_id] @@ -305,6 +453,8 @@ def _run( event_type, {"error": operation.error}, ) + self._prune_terminal_locked() + self._persist_workspace_locked(operation.workspace_id) except Exception as exc: with self._lock: operation = self._operations[operation_id] @@ -314,10 +464,7 @@ def _run( event_type = "operation.cancelled" else: operation.state = "failed" - operation.error = operation.error or { - "message": str(exc), - "code": "command_failed", - } + operation.error = operation.error or _operation_error_from_exception(exc) event_type = "operation.failed" operation.updated_at = time.time() payload = {"error": operation.error} @@ -330,6 +477,8 @@ def _run( } ) event = self._new_event_locked(operation, event_type, payload) + self._prune_terminal_locked() + self._persist_workspace_locked(operation.workspace_id) self._publish(event) finally: try: @@ -365,6 +514,7 @@ def step_started(self, operation_id: str, workspace_step: Any) -> None: ) if log_tail is not None: self._step_log_tails[operation_id] = log_tail + self._persist_workspace_locked(operation.workspace_id) self._publish(event) if log_tail is not None: thread = threading.Thread( @@ -383,20 +533,27 @@ def rerun_prepared( affected_steps: list[str], scope: str, target_step: str = "", + workspace_revision: int | None = None, ) -> None: """Publish the idempotent GUI reset boundary before a rerun starts.""" with self._lock: operation = self._operations[operation_id] + if workspace_revision is not None: + operation.workspace_revision = workspace_revision operation.updated_at = time.time() + payload = { + "affectedSteps": affected_steps, + "scope": scope, + "targetStep": target_step, + } + if workspace_revision is not None: + payload["workspaceRevision"] = workspace_revision event = self._new_event_locked( operation, "operation.rerun_prepared", - { - "affectedSteps": affected_steps, - "scope": scope, - "targetStep": target_step, - }, + payload, ) + self._persist_workspace_locked(operation.workspace_id) self._publish(event) def step_completed( @@ -405,11 +562,12 @@ def step_completed( workspace_step: Any, state: Any, error: str | None = None, + workspace_revision: int | None = None, ) -> None: self._stop_step_log_tail(operation_id) state_value = str(getattr(state, "value", state)) final_log = _read_final_log(getattr(workspace_step, "log", None)) - with self._render_gate: + with self._lock: operation = self._operations[operation_id] operation.current_step = str(getattr(workspace_step, "name", "")) operation.current_tool = str(getattr(workspace_step, "tool", "")) @@ -422,7 +580,7 @@ def step_completed( } if error: log_file = str(getattr(getattr(workspace_step, "log", None), "file", "") or "") - operation.error = { + operation.error = operation.error or { "code": "tool_failed", "message": error, "step": operation.current_step, @@ -430,21 +588,59 @@ def step_completed( "logFile": log_file, } payload["error"] = operation.error - payload["logFile"] = log_file + payload["logFile"] = str(operation.error.get("logFile", log_file)) event = self._new_event_locked(operation, "step.completed", payload) - if state_value == "Success": + if workspace_revision is None: operation.workspace_revision += 1 - step_commit_id = f"{operation.operation_id}:step:{operation.workspace_revision}" - payload["stepCommitId"] = step_commit_id - payload["workspaceRevision"] = operation.workspace_revision - if not operation.render_sync_degraded: - operation.awaiting_event_id = event["eventId"] - operation.awaiting_event = event - operation.awaiting_step_commit_id = step_commit_id - operation.render_sync_state = "waiting_for_gui_sync" - operation.render_retry_count = 0 - operation.render_wait_started_at = time.monotonic() - operation.state = "waiting_for_gui_sync" + else: + operation.workspace_revision = workspace_revision + step_commit_id = f"{operation.operation_id}:step:{operation.workspace_revision}" + payload["stepCommitId"] = step_commit_id + payload["workspaceRevision"] = operation.workspace_revision + self._persist_workspace_locked(operation.workspace_id) + self._publish(event) + + def step_diagnostic( + self, + operation_id: str, + workspace_step: Any, + diagnostic: dict[str, Any], + ) -> None: + message = str(diagnostic.get("message", "tool failed")) + log_file = str(getattr(getattr(workspace_step, "log", None), "file", "") or "") + with self._lock: + operation = self._operations[operation_id] + operation.error = { + "code": "tool_failed", + "message": message, + "step": str(getattr(workspace_step, "name", "")), + "tool": str(getattr(workspace_step, "tool", "")), + "logFile": log_file, + **diagnostic, + } + operation.updated_at = time.time() + revision = operation.workspace_revision + operation.error.update( + { + "snapshotRevision": revision, + "eventRevision": revision, + "operationRevision": revision, + } + ) + event = self._new_event_locked( + operation, + "step.diagnostic", + { + "step": str(getattr(workspace_step, "name", "")), + "tool": str(getattr(workspace_step, "tool", "")), + "diagnostic": operation.error, + "snapshotRevision": revision, + "eventRevision": revision, + "operationRevision": revision, + "workspaceRevision": revision, + }, + ) + self._persist_workspace_locked(operation.workspace_id) self._publish(event) def subflow_stage( @@ -470,6 +666,7 @@ def subflow_stage( "tool": tool, }, ) + self._persist_workspace_locked(operation.workspace_id) self._publish(event) def step_skipped(self, operation_id: str, workspace_step: Any) -> None: @@ -491,81 +688,8 @@ def step_skipped(self, operation_id: str, workspace_step: Any) -> None: self._publish(event) def wait_for_step_rendered(self, operation_id: str) -> bool: - while True: - degraded_event: dict[str, Any] | None = None - replay_event: dict[str, Any] | None = None - pause_event: dict[str, Any] | None = None - with self._render_gate: - operation = self._operations[operation_id] - if operation.cancel_requested: - return False - if not operation.awaiting_event_id: - if operation.state in { - "waiting_for_gui_sync", - "paused_for_gui_recovery", - }: - operation.state = "running" - operation.updated_at = time.time() - return True - - started_at = operation.render_wait_started_at or time.monotonic() - elapsed = time.monotonic() - started_at - if elapsed >= _RENDER_ACK_ABORT_SECONDS: - awaiting_event_id = operation.awaiting_event_id - awaiting_step_commit_id = operation.awaiting_step_commit_id - operation.awaiting_event_id = None - operation.awaiting_event = None - operation.awaiting_step_commit_id = None - operation.render_sync_state = "gui_sync_degraded" - operation.render_sync_degraded = True - operation.state = "running" - operation.updated_at = time.time() - degraded_event = self._new_event_locked( - operation, - "operation.gui_sync_degraded", - { - "eventId": awaiting_event_id, - "stepCommitId": awaiting_step_commit_id, - "workspaceRevision": operation.workspace_revision, - }, - ) - elif ( - elapsed >= _RENDER_ACK_PAUSE_SECONDS - and operation.render_sync_state != "paused_for_gui_recovery" - ): - operation.render_sync_state = "paused_for_gui_recovery" - operation.state = "paused_for_gui_recovery" - operation.updated_at = time.time() - pause_event = self._new_event_locked( - operation, - "operation.gui_sync_paused", - { - "eventId": operation.awaiting_event_id, - "stepCommitId": operation.awaiting_step_commit_id, - "workspaceRevision": operation.workspace_revision, - }, - ) - - if degraded_event is None: - operation.render_retry_count += 1 - if operation.awaiting_event is not None: - replay_event = { - **operation.awaiting_event, - "payload": { - **operation.awaiting_event["payload"], - "replayed": True, - "retryCount": operation.render_retry_count, - }, - } - self._render_gate.wait(timeout=_RENDER_ACK_RETRY_SECONDS) - - if degraded_event is not None: - self._publish(degraded_event) - return True - if pause_event is not None: - self._publish(pause_event) - if replay_event is not None: - self._publish(replay_event) + with self._lock: + return not self._operations[operation_id].cancel_requested def _tail_step_log(self, log_tail: _StepLogTail) -> None: while not log_tail.stopped.is_set(): @@ -642,6 +766,43 @@ def _new_event_locked( "payload": payload, } + def _prune_terminal_locked(self) -> None: + terminal = [ + operation + for operation in self._operations.values() + if operation.state in _TERMINAL_OPERATION_STATES + ] + if len(terminal) <= _MAX_TERMINAL_OPERATIONS: + return + terminal.sort(key=lambda operation: (operation.updated_at, operation.operation_id)) + removed = terminal[: len(terminal) - _MAX_TERMINAL_OPERATIONS] + removed_ids = {operation.operation_id for operation in removed} + for operation_id in removed_ids: + self._operations.pop(operation_id, None) + self._idempotency = { + key: record for key, record in self._idempotency.items() if record[0] not in removed_ids + } + + def _persist_workspace_locked(self, workspace_id: str) -> None: + path = self._ledger_paths.get(workspace_id) + if path is None: + return + from chipcompiler.utility import json_write + + operations = [ + self._operation_payload(operation) + for operation in self._operations.values() + if operation.workspace_id == workspace_id + ] + json_write( + path, + { + "schemaVersion": _LEDGER_SCHEMA_VERSION, + "workspaceId": workspace_id, + "operations": operations, + }, + ) + @staticmethod def _operation_payload(operation: RuntimeOperation) -> dict[str, Any]: return { @@ -653,6 +814,8 @@ def _operation_payload(operation: RuntimeOperation) -> dict[str, Any]: "origin": operation.origin, "rerun": operation.rerun, "step": operation.step, + "idempotencyKey": operation.idempotency_key, + "commandFingerprint": operation.command_fingerprint, "state": operation.state, "currentStep": operation.current_step, "currentTool": operation.current_tool, @@ -670,18 +833,31 @@ def _operation_payload(operation: RuntimeOperation) -> dict[str, Any]: "shutdownBarrier": operation.state not in _TERMINAL_OPERATION_STATES, "createdAt": operation.created_at, "updatedAt": operation.updated_at, + "sequence": operation.sequence, } def _publish(self, event: dict[str, Any]) -> None: publisher = self._publisher if publisher is not None: - publisher(event) + try: + publisher(event) + except Exception: + logger.exception("runtime event consumer failed: %s", event.get("type")) class RuntimeFlowObserver: - def __init__(self, manager: RuntimeOperationManager, operation_id: str): + fatal_observer = True + + def __init__( + self, + manager: RuntimeOperationManager, + operation_id: str, + snapshot_committer: Callable[[Any, Any, str | None], int] | None = None, + ): self._manager = manager self._operation_id = operation_id + self._snapshot_committer = snapshot_committer + self._committed_revision: int | None = None @property def runtime_operation(self) -> dict[str, Any]: @@ -700,21 +876,46 @@ def on_rerun_prepared( affected_steps: list[str], scope: str, target_step: str = "", + workspace_revision: int | None = None, ) -> None: self._manager.rerun_prepared( self._operation_id, affected_steps=affected_steps, scope=scope, target_step=target_step, + workspace_revision=workspace_revision, ) + def raise_if_cancelled(self) -> None: + self._manager.raise_if_cancel_requested(self._operation_id) + def on_step_completed( self, workspace_step: Any, state: Any, error: str | None = None, ) -> None: - self._manager.step_completed(self._operation_id, workspace_step, state, error) + if self._committed_revision is None and self._snapshot_committer is not None: + self.commit_step(workspace_step, state, error) + self._manager.step_completed( + self._operation_id, + workspace_step, + state, + error, + self._committed_revision, + ) + self._committed_revision = None + + def on_step_diagnostic( + self, + workspace_step: Any, + diagnostic: dict[str, Any], + ) -> None: + self._manager.step_diagnostic(self._operation_id, workspace_step, diagnostic) + + def commit_step(self, workspace_step: Any, state: Any, error: str | None = None) -> None: + if self._snapshot_committer is not None: + self._committed_revision = self._snapshot_committer(workspace_step, state, error) def on_subflow_stage(self, workspace_step: Any, subflow_step: dict[str, Any]) -> None: self._manager.subflow_stage(self._operation_id, workspace_step, subflow_step) @@ -761,3 +962,92 @@ def _step_log_tail_for( tool=tool, cursor=cursor, ) + + +def _operation_from_payload( + payload: dict[str, Any], runtime_instance_id: str +) -> RuntimeOperation | None: + operation_id = payload.get("operationId") + run_session_id = payload.get("runSessionId") + workspace_id = payload.get("workspaceId") + kind = payload.get("kind") + origin = payload.get("origin") + if not all( + isinstance(value, str) and value + for value in (operation_id, run_session_id, workspace_id, kind, origin) + ): + return None + return RuntimeOperation( + operation_id=operation_id, + run_session_id=run_session_id, + runtime_instance_id=runtime_instance_id, + workspace_id=workspace_id, + kind=kind, + origin=origin, + rerun=bool(payload.get("rerun", False)), + step=str(payload.get("step", "")), + idempotency_key=str(payload.get("idempotencyKey", "")), + command_fingerprint=str(payload.get("commandFingerprint", "")), + state=str(payload.get("state", "interrupted")), + current_step=str(payload.get("currentStep", "")), + current_tool=str(payload.get("currentTool", "")), + error=payload.get("error") if isinstance(payload.get("error"), dict) else None, + result=payload.get("result") if isinstance(payload.get("result"), dict) else None, + created_at=_number(payload.get("createdAt")), + updated_at=_number(payload.get("updatedAt")), + sequence=int(payload.get("sequence", 0) or 0), + workspace_revision=int(payload.get("workspaceRevision", 0) or 0), + render_sync_state=str(payload.get("renderSyncState", "idle")), + render_retry_count=int(payload.get("renderRetryCount", 0) or 0), + last_render_ack_at=_optional_number(payload.get("lastRenderAckAt")), + cancel_requested=bool(payload.get("cancelRequested", False)), + interruptibility=str(payload.get("interruptibility", "deferred")), + ) + + +def _number(value: Any) -> float: + return ( + float(value) + if isinstance(value, (int, float)) and not isinstance(value, bool) + else time.time() + ) + + +def _command_fingerprint( + *, + kind: str, + origin: str, + rerun: bool, + step: str, + workspace_revision: int, + command_input: dict[str, Any] | None = None, +) -> str: + encoded = json.dumps( + { + "kind": kind, + "origin": origin, + "rerun": rerun, + "step": step, + "workspaceRevision": workspace_revision, + "commandInput": command_input or {}, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _operation_error_from_exception(exc: Exception) -> dict[str, Any]: + code = getattr(exc, "code", None) + data = getattr(exc, "data", None) + error: dict[str, Any] = { + "message": str(exc), + "code": code if isinstance(code, str) and code else "command_failed", + } + if isinstance(data, dict): + error.update(data) + return error + + +def _optional_number(value: Any) -> float | None: + return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None diff --git a/chipcompiler/runtime/requests.py b/chipcompiler/runtime/requests.py index 8eb6116de..b648684cc 100644 --- a/chipcompiler/runtime/requests.py +++ b/chipcompiler/runtime/requests.py @@ -4,7 +4,7 @@ @dataclass(frozen=True) class WorkspaceCreateRequest: - directory: str + directory: str = "" pdk: str = "" pdk_root: str = "" pdk_json: Any = None @@ -15,16 +15,112 @@ class WorkspaceCreateRequest: rtl_list: list[str] | None = None sdc: str = "" flow_config: dict[str, Any] | None = None + command_id: str = "" + target_directory: str = "" + workspace_spec: dict[str, Any] | None = None + workspace_bindings: dict[str, Any] | None = None + project_id: str = "" + project_root: str = "" @dataclass(frozen=True) class WorkspaceOpenRequest: directory: str + workspace_bindings: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class EmptyRequest: + pass + + +@dataclass(frozen=True) +class WorkspaceSpecOpenRequest: + directory: str + workspace_bindings: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class WorkspaceSpecValidateRequest: + workspace_spec: dict[str, Any] + workspace_bindings: dict[str, Any] + + +@dataclass(frozen=True) +class ProjectManifestLoadRequest: + project_root: str + + +@dataclass(frozen=True) +class ProjectManifestDiscoverRequest: + directory: str + + +@dataclass(frozen=True) +class ProjectManifestMutationRequest: + project_root: str + mutation: dict[str, Any] + + +@dataclass(frozen=True) +class WorkspaceSpecCreateRequest: + command_id: str = "" + target_directory: str = "" + workspace_spec: dict[str, Any] | None = None + workspace_bindings: dict[str, Any] | None = None + project_id: str = "" + project_root: str = "" + directory: str = "" + pdk: str = "" + pdk_root: str = "" + pdk_json: Any = None + parameters: dict[str, Any] | None = None + origin_def: str = "" + origin_verilog: str = "" + filelist: str = "" + rtl_list: list[str] | None = None + sdc: str = "" + flow_config: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class WorkspaceUpdateRequest: + command_id: str + workspace_id: str + expected_workspace_revision: int + workspace_spec: dict[str, Any] + workspace_bindings: dict[str, Any] + + +@dataclass(frozen=True) +class WorkspaceConfigurationUpdateRequest: + command_id: str + workspace_id: str + expected_workspace_revision: int + configuration: dict[str, Any] + workspace_bindings: dict[str, Any] + + +@dataclass(frozen=True) +class WorkspaceStepConfigurationUpdateRequest: + command_id: str + workspace_id: str + expected_workspace_revision: int + step_id: str + parameters: dict[str, Any] + + +@dataclass(frozen=True) +class WorkspaceStepConfigurationReadRequest: + step: str + workspace_id: str = "" + directory: str = "" @dataclass(frozen=True) class WorkspaceIdRequest: workspace_id: str + expected_workspace_revision: int = 1 @dataclass(frozen=True) @@ -66,6 +162,7 @@ class WorkspaceInfoRequest: @dataclass(frozen=True) class FlowRunRequest: workspace_id: str + expected_workspace_revision: int | None = None rerun: bool = False @@ -73,12 +170,14 @@ class FlowRunRequest: class FlowRunStepRequest: workspace_id: str step: str + expected_workspace_revision: int | None = None rerun: bool = False @dataclass(frozen=True) class OperationStartFlowRequest: workspace_id: str + expected_workspace_revision: int | None = None rerun: bool = False origin: str = "gui" idempotency_key: str = "" @@ -88,6 +187,7 @@ class OperationStartFlowRequest: class OperationStartStepRequest: workspace_id: str step: str + expected_workspace_revision: int | None = None rerun: bool = False reset_dependents: bool = False origin: str = "gui" @@ -137,6 +237,13 @@ class LayoutEditApplyRequest: class LayoutEditSaveRequest: edit_session_id: str expected_revision: int + expected_workspace_revision: int = 1 + + +@dataclass(frozen=True) +class WorkspaceMutationRequest: + workspace_id: str + expected_workspace_revision: int = 1 @dataclass(frozen=True) @@ -178,6 +285,7 @@ def __init__(self, reason: str): "paramJson": "parameters", "rtlList": "rtl_list", "workspaceId": "workspace_id", + "expectedWorkspaceRevision": "expected_workspace_revision", "operationId": "operation_id", "eventId": "event_id", "stepCommitId": "step_commit_id", @@ -194,6 +302,12 @@ def __init__(self, reason: str): "expectedSourceFingerprint": "expected_source_fingerprint", "id": "info_id", "additionalFiles": "additional_files", + "workspaceSpec": "workspace_spec", + "workspaceBindings": "workspace_bindings", + "targetDirectory": "target_directory", + "projectId": "project_id", + "projectRoot": "project_root", + "stepId": "step_id", } diff --git a/chipcompiler/runtime/server.py b/chipcompiler/runtime/server.py index 70167ea3a..5589b11ae 100644 --- a/chipcompiler/runtime/server.py +++ b/chipcompiler/runtime/server.py @@ -60,7 +60,7 @@ def set_notification_sink(self, sink: Callable[[str, dict], None] | None) -> Non def _publish_runtime_event(self, event: dict) -> None: sink = self._notification_sink if sink is not None: - sink("runtime.event", event) + sink("runtime.event", _project_runtime_event(event)) def _register_base_methods(self) -> None: self.dispatcher.add_method("rpc.hello", self._hello) @@ -76,6 +76,7 @@ def _hello(self, version: int): ) return { "version": PROTOCOL_VERSION, + "protocolVersion": PROTOCOL_VERSION, "eccVersion": getattr(chipcompiler, "__version__", "unknown"), "capabilities": list(self.capabilities), } @@ -101,9 +102,13 @@ def _register_runtime_methods(self) -> None: ): api_method = getattr(self.api, spec.handler_name, None) if not callable(api_method): - raise TypeError( - f"runtime method {spec.method_name} handler {spec.handler_name} is not callable" + if spec.method_name not in methods.OPTIONAL_RUNTIME_METHOD_NAMES: + raise TypeError(f"runtime API handler is not callable: {spec.handler_name}") + self.dispatcher.add_method( + spec.method_name, + self._missing_optional_method_handler(spec), ) + continue self.dispatcher.add_method( spec.method_name, self._runtime_method_handler(spec, api_method), @@ -129,6 +134,16 @@ def handler(**params): {"message": exc.message, **exc.data}, ) except Exception as exc: + stable_code = getattr(exc, "code", None) + if isinstance(stable_code, str): + details = getattr(exc, "details", None) + if not isinstance(details, dict): + details = getattr(exc, "data", None) + return Error( + ERROR_CODES.get(stable_code, -32000), + stable_code, + {"message": str(exc), **(details if isinstance(details, dict) else {})}, + ) return Error( ERROR_CODES["command_failed"], "command_failed", @@ -136,3 +151,51 @@ def handler(**params): ) return handler + + @staticmethod + def _missing_optional_method_handler(spec): + def handler(**_params): + return Error( + -32602, + "invalid_request", + {"message": f"runtime API does not support {spec.method_name}"}, + ) + + return handler + + +def _project_runtime_event(event: dict) -> dict: + source_type = str(event.get("type", "")) + payload = {**event.get("payload", {}), "sourceType": source_type} + if source_type == "step.completed": + event_type = "workspace.committed" + elif source_type in { + "step.started", + "step.log", + "subflow.stage", + "operation.rerun_prepared", + }: + event_type = "execution.progress" + else: + event_type = "operation.changed" + state = { + "operation.queued": "queued", + "operation.started": "running", + "operation.cancel_requested": "cancelling", + "operation.completed": "succeeded", + "operation.failed": "failed", + "operation.cancelled": "cancelled", + "operation.interrupted": "interrupted", + }.get(source_type) + if state: + payload["state"] = state + return { + **event, + "type": event_type, + "payload": payload, + **( + {"workspaceRevision": payload["workspaceRevision"]} + if "workspaceRevision" in payload + else {} + ), + } diff --git a/chipcompiler/runtime/sessions.py b/chipcompiler/runtime/sessions.py index c975ac104..1bfeda45e 100644 --- a/chipcompiler/runtime/sessions.py +++ b/chipcompiler/runtime/sessions.py @@ -11,6 +11,9 @@ class WorkspaceSession: workspace_id: str directory: Path workspace: Any + workspace_revision: int = 0 + execution_readiness: dict[str, Any] = field(default_factory=lambda: {"ready": True}) + workspace_bindings: dict[str, Any] | None = None db_handle: Any = None layout_edit_session: "LayoutEditSession | None" = None mutation_lock: threading.Lock = field(default_factory=threading.Lock) @@ -38,6 +41,7 @@ class LayoutEditSession: requires_verilog: bool = False used_floorplan_editor: bool = False validation_diagnostics: list[dict[str, Any]] = field(default_factory=list) + ownership_lock: Any = None class WorkspaceSessionNotFound(KeyError): @@ -58,22 +62,54 @@ def __init__(self, db_releaser: Callable[[Any], None] | None = _close_db_handle) self._db_releaser = db_releaser self._lock = threading.Lock() - def create_session(self, directory: str | Path, *, workspace: Any) -> WorkspaceSession: + def create_session( + self, + directory: str | Path, + *, + workspace: Any, + workspace_id: str | None = None, + workspace_revision: int = 0, + execution_readiness: dict[str, Any] | None = None, + workspace_bindings: dict[str, Any] | None = None, + ) -> WorkspaceSession: resolved_directory = Path(directory).resolve() with self._lock: existing_id = self._sessions_by_directory.get(resolved_directory) if existing_id is not None: self._remove_session(existing_id) - return self._create_session(resolved_directory, workspace=workspace) - - def open_session(self, directory: str | Path, *, workspace: Any) -> WorkspaceSession: + return self._create_session( + resolved_directory, + workspace=workspace, + workspace_id=workspace_id, + workspace_revision=workspace_revision, + execution_readiness=execution_readiness, + workspace_bindings=workspace_bindings, + ) + + def open_session( + self, + directory: str | Path, + *, + workspace: Any, + workspace_id: str | None = None, + workspace_revision: int = 0, + execution_readiness: dict[str, Any] | None = None, + workspace_bindings: dict[str, Any] | None = None, + ) -> WorkspaceSession: resolved_directory = Path(directory).resolve() with self._lock: existing_id = self._sessions_by_directory.get(resolved_directory) if existing_id is not None: return self._sessions[existing_id] - return self._create_session(resolved_directory, workspace=workspace) + return self._create_session( + resolved_directory, + workspace=workspace, + workspace_id=workspace_id, + workspace_revision=workspace_revision, + execution_readiness=execution_readiness, + workspace_bindings=workspace_bindings, + ) def get_session(self, workspace_id: str) -> WorkspaceSession: try: @@ -96,13 +132,26 @@ def close_all(self) -> None: def release_session_db(self, session: WorkspaceSession) -> bool: return self._release_session_db(session) - def _create_session(self, directory: Path, *, workspace: Any) -> WorkspaceSession: - workspace_id = f"workspace-{self._next_id}" - self._next_id += 1 + def _create_session( + self, + directory: Path, + *, + workspace: Any, + workspace_id: str | None, + workspace_revision: int, + execution_readiness: dict[str, Any] | None, + workspace_bindings: dict[str, Any] | None, + ) -> WorkspaceSession: + if workspace_id is None: + workspace_id = f"workspace-{self._next_id}" + self._next_id += 1 session = WorkspaceSession( workspace_id=workspace_id, directory=directory, workspace=workspace, + workspace_revision=workspace_revision, + execution_readiness=execution_readiness or {"ready": True}, + workspace_bindings=workspace_bindings, ) self._sessions[workspace_id] = session self._sessions_by_directory[directory] = workspace_id diff --git a/chipcompiler/runtime/workspace_api.py b/chipcompiler/runtime/workspace_api.py index 5c256c221..ebb26ec7d 100644 --- a/chipcompiler/runtime/workspace_api.py +++ b/chipcompiler/runtime/workspace_api.py @@ -6,13 +6,21 @@ import tempfile import threading from collections.abc import Callable +from contextlib import contextmanager from copy import copy, deepcopy from dataclasses import replace from pathlib import Path from typing import Any, TypeVar +try: + import fcntl +except ImportError: # pragma: no cover - Windows uses the in-process lock. + fcntl = None + +from chipcompiler.runtime.errors import RuntimeApiError from chipcompiler.runtime.operations import ( RuntimeOperationConflict, + RuntimeOperationIdempotencyConflict, RuntimeOperationManager, ) from chipcompiler.runtime.requests import ( @@ -38,6 +46,9 @@ WorkspaceInspectSignoffRequest, WorkspaceOpenRequest, WorkspaceRecoverInterruptedRequest, + WorkspaceSpecCreateRequest, + WorkspaceSpecOpenRequest, + WorkspaceStepConfigurationReadRequest, WorkspaceSyncConfigRequest, ) from chipcompiler.runtime.sessions import ( @@ -56,15 +67,10 @@ _T = TypeVar("_T") -class RuntimeApiError(RuntimeError): - def __init__(self, code: str, message: str, data: dict | None = None): - super().__init__(message) - self.code = code - self.message = message - self.data = data or {} +from chipcompiler.runtime.workspace_spec_api import WorkspaceSpecRuntimeMixin # noqa: E402 -class WorkspaceRuntimeApi: +class WorkspaceRuntimeApi(WorkspaceSpecRuntimeMixin): def __init__( self, sessions: WorkspaceSessionRegistry | None = None, @@ -86,7 +92,40 @@ def __init__( def set_event_publisher(self, publisher: Callable[[dict[str, Any]], None] | None) -> None: self.operations.set_publisher(publisher) - def create_workspace(self, request: WorkspaceCreateRequest) -> dict: + def create_workspace( + self, request: WorkspaceCreateRequest | WorkspaceSpecCreateRequest + ) -> dict: + if getattr(request, "workspace_spec", None) is not None: + spec_request = ( + request + if isinstance(request, WorkspaceSpecCreateRequest) + else WorkspaceSpecCreateRequest( + command_id=request.command_id, + target_directory=request.target_directory, + workspace_spec=request.workspace_spec, + workspace_bindings=request.workspace_bindings or {}, + project_id=request.project_id, + project_root=request.project_root, + ) + ) + return self._create_workspace_from_spec(spec_request) + if isinstance(request, WorkspaceSpecCreateRequest): + request = WorkspaceCreateRequest( + directory=request.directory, + pdk=request.pdk, + pdk_root=request.pdk_root, + pdk_json=request.pdk_json, + parameters=request.parameters, + origin_def=request.origin_def, + origin_verilog=request.origin_verilog, + filelist=request.filelist, + rtl_list=request.rtl_list, + sdc=request.sdc, + flow_config=request.flow_config, + ) + return self._create_legacy_workspace(request) + + def _create_legacy_workspace(self, request: WorkspaceCreateRequest) -> dict: if not request.directory: raise RuntimeApiError("invalid_request", "missing required field: directory") @@ -127,24 +166,64 @@ def create_workspace(self, request: WorkspaceCreateRequest) -> dict: build_flow_for_workspace(workspace) session = self.sessions.create_session(workspace.directory, workspace=workspace) + self.operations.load_workspace_ledger( + session.workspace_id, + session.directory / "home" / "runtime-commands.json", + ) return _workspace_session_result(session) - def open_workspace(self, request: WorkspaceOpenRequest) -> dict: + def open_workspace(self, request: WorkspaceOpenRequest | WorkspaceSpecOpenRequest) -> dict: + if isinstance(request, WorkspaceSpecOpenRequest) or request.workspace_bindings is not None: + spec_request = ( + request + if isinstance(request, WorkspaceSpecOpenRequest) + else WorkspaceSpecOpenRequest( + directory=request.directory, + workspace_bindings=request.workspace_bindings, + ) + ) + return WorkspaceSpecRuntimeMixin.open_workspace(self, spec_request) + return self._open_legacy_workspace(request) + + def _open_legacy_workspace(self, request: WorkspaceOpenRequest) -> dict: workspace = self._load_workspace(request.directory) build_flow_for_workspace(workspace, create_step_workspaces=False) - session = self.sessions.open_session(workspace.directory, workspace=workspace) + from chipcompiler.engine.snapshot import EngineeringSnapshotError, read_engineering_snapshot + + try: + snapshot = read_engineering_snapshot(workspace) + except EngineeringSnapshotError: + snapshot = None + session = self.sessions.open_session( + workspace.directory, + workspace=workspace, + workspace_id=snapshot["workspaceId"] if snapshot else None, + workspace_revision=snapshot["workspaceRevision"] if snapshot else 0, + ) + self.operations.load_workspace_ledger( + session.workspace_id, + session.directory / "home" / "runtime-commands.json", + recover=False, + ) return _workspace_session_result(session) def recover_interrupted(self, request: WorkspaceRecoverInterruptedRequest) -> dict: from chipcompiler.runtime.recovery import recover_interrupted_operation - return self._with_session_mutation_lock( - request.workspace_id, - lambda session: recover_interrupted_operation( + def recover(session: WorkspaceSession) -> dict: + result = recover_interrupted_operation( session.workspace, self.operations, request.operation_id, - ), + ) + snapshot_path = Path(session.workspace.directory) / "home" / "engineering-snapshot.json" + if result["recovered"] and snapshot_path.is_file(): + self._commit_workspace_snapshot(session, "operation.recovered") + return result + + return self._with_session_mutation_lock( + request.workspace_id, + recover, ) def workspace_home(self, request: WorkspaceIdRequest) -> dict: @@ -173,6 +252,47 @@ def workspace_info(self, request: WorkspaceInfoRequest) -> dict: "info": stringify_paths(info or {}), } + def read_workspace_step_configuration( + self, request: WorkspaceStepConfigurationReadRequest + ) -> dict: + if bool(request.workspace_id) == bool(request.directory): + raise RuntimeApiError( + "invalid_request", + "Exactly one of workspaceId or directory is required", + ) + + from chipcompiler.engine import ( + WorkspaceLifecycleError, + read_step_configuration, + read_step_configuration_from_directory, + ) + + try: + if request.directory: + result = read_step_configuration_from_directory(request.directory, request.step) + else: + session = self._get_session(request.workspace_id) + result = read_step_configuration(session.workspace, request.step) + except WorkspaceLifecycleError as exc: + if exc.code == "step_configuration_unavailable": + return { + "status": "unavailable", + "step": request.step, + "reason": exc.code, + **exc.details, + **( + { + "workspaceId": session.workspace_id, + "workspaceRevision": session.workspace_revision, + } + if request.workspace_id + else {} + ), + } + raise RuntimeApiError(exc.code, str(exc), exc.details) from exc + + return {"status": "available", **result} + def refresh_config(self, request: WorkspaceIdRequest) -> dict: def refresh(session: WorkspaceSession) -> dict: self._release_session_db(session) @@ -262,15 +382,19 @@ def _flow_run( preserve_user_inputs: bool = False, ) -> dict: def run(session: WorkspaceSession) -> dict: + self._ensure_execution_ready(session) + self._validate_workspace_revision(session, request.expected_workspace_revision) + stale_step_ids = self._stale_step_ids(session.workspace) + requires_preparation = request.rerun or bool(stale_step_ids) should_capture = self._should_capture_session_db(session) previous_db = session.db_handle if should_capture else None - if request.rerun and should_capture: + if requires_preparation and should_capture: self._release_session_db(session) previous_db = None engine_flow = self._build_flow_for_session( session, - attach_session_db=should_capture and not request.rerun, + attach_session_db=should_capture and not requires_preparation, ) if request.rerun: affected_steps = list(getattr(engine_flow, "workspace_steps", [])) @@ -279,13 +403,44 @@ def run(session: WorkspaceSession) -> dict: engine_flow, preserve_user_inputs=preserve_user_inputs, ) + reset_revision = self._commit_rerun_snapshot( + session, + "flow.rerun_prepared", + ) self._notify_rerun_prepared( observer, affected_steps, scope="flow", + workspace_revision=reset_revision, + ) + elif stale_step_ids: + affected_steps = [ + step + for step in getattr(engine_flow, "workspace_steps", []) + if str(getattr(step, "name", "")) in stale_step_ids + ] + self._refresh_workspace_config(session.workspace) + self._prepare_steps_for_rerun( + session.workspace, + engine_flow, + affected_steps, + ) + reset_revision = self._commit_rerun_snapshot( + session, + "flow.rerun_prepared", + ) + self._notify_rerun_prepared( + observer, + affected_steps, + scope="flow", + workspace_revision=reset_revision, ) try: - ok = _run_engine_flow_steps(engine_flow, rerun=request.rerun, observer=observer) + ok = _run_engine_flow_steps( + engine_flow, + rerun=request.rerun, + observer=observer, + ) finally: if should_capture: self._capture_flow_db( @@ -316,17 +471,30 @@ def _flow_run_step( reset_dependents: bool = False, ) -> dict: def run_step(session: WorkspaceSession) -> dict: + self._ensure_execution_ready(session) + self._validate_workspace_revision(session, request.expected_workspace_revision) + stale_step_ids = self._stale_step_ids(session.workspace) should_capture = self._should_capture_session_db(session) previous_db = session.db_handle if should_capture else None - if request.rerun and should_capture: + stale_target_index = -1 + if request.step in stale_step_ids: + stale_target_index = stale_step_ids.index(request.step) + requires_preparation = request.rerun or stale_target_index == 0 + if stale_target_index > 0: + raise RuntimeApiError( + "stale_dependency", + f"rerun {stale_step_ids[0]} before {request.step}", + {"requiredStep": stale_step_ids[0]}, + ) + if requires_preparation and should_capture: self._release_session_db(session) previous_db = None engine_flow = self._build_flow_for_session( session, - attach_session_db=should_capture and not request.rerun, + attach_session_db=should_capture and not requires_preparation, ) - if request.rerun: + if requires_preparation: if session.layout_edit_session is not None: raise RuntimeApiError( "layout_edit_active", @@ -337,26 +505,39 @@ def run_step(session: WorkspaceSession) -> dict: workspace_step = engine_flow.get_workspace_step(request.step) if workspace_step is None: raise RuntimeApiError("command_failed", f"step not found: {request.step}") - if request.rerun: - affected_steps = self._rerun_affected_steps( - engine_flow, - workspace_step, - reset_dependents=reset_dependents, + if requires_preparation: + affected_steps = ( + [ + step + for step in getattr(engine_flow, "workspace_steps", []) + if str(getattr(step, "name", "")) in stale_step_ids + ] + if stale_target_index == 0 + else self._rerun_affected_steps( + engine_flow, + workspace_step, + reset_dependents=reset_dependents, + ) ) self._prepare_steps_for_rerun( session.workspace, engine_flow, affected_steps, ) + reset_revision = self._commit_rerun_snapshot( + session, + "flow.rerun_prepared", + ) self._notify_rerun_prepared( observer, affected_steps, scope="step", target_step=workspace_step.name, + workspace_revision=reset_revision, ) try: - step_already_succeeded = not request.rerun and engine_flow.check_state( + step_already_succeeded = not requires_preparation and engine_flow.check_state( name=workspace_step.name, tool=workspace_step.tool, state=_success_state(), @@ -366,7 +547,7 @@ def run_step(session: WorkspaceSession) -> dict: state = _run_engine_flow_step( engine_flow, workspace_step, - rerun=request.rerun, + rerun=requires_preparation, observer=observer, ) finally: @@ -393,8 +574,17 @@ def run_step(session: WorkspaceSession) -> dict: def start_flow_operation(self, request: OperationStartFlowRequest) -> dict: self._require_gui_operation_origin(request.origin) - self._get_session(request.workspace_id) + session = self._get_session(request.workspace_id) + expected_revision = request.expected_workspace_revision + operation_revision = ( + session.workspace_revision if expected_revision is None else expected_revision + ) try: + + def validate_start() -> None: + self._ensure_execution_ready(session) + self._validate_workspace_revision(session, request.expected_workspace_revision) + return self.operations.start( workspace_id=request.workspace_id, kind="flow", @@ -402,19 +592,43 @@ def start_flow_operation(self, request: OperationStartFlowRequest) -> dict: rerun=request.rerun, step="", idempotency_key=request.idempotency_key, + workspace_revision=operation_revision, + snapshot_committer=lambda step, state, error: self._commit_step_snapshot( + session, + step, + state, + error, + ), + ledger_path=session.directory / "home" / "runtime-commands.json", + precondition=validate_start, runner=lambda observer: self._flow_run( - FlowRunRequest(workspace_id=request.workspace_id, rerun=request.rerun), + FlowRunRequest( + workspace_id=request.workspace_id, + expected_workspace_revision=expected_revision, + rerun=request.rerun, + ), observer=observer, preserve_user_inputs=request.rerun, ), ) + except RuntimeOperationIdempotencyConflict as exc: + raise RuntimeApiError("idempotency_conflict", str(exc)) from exc except RuntimeOperationConflict as exc: - raise RuntimeApiError("command_failed", str(exc)) from exc + raise RuntimeApiError("operation_conflict", str(exc)) from exc def start_step_operation(self, request: OperationStartStepRequest) -> dict: self._require_gui_operation_origin(request.origin) - self._get_session(request.workspace_id) + session = self._get_session(request.workspace_id) + expected_revision = request.expected_workspace_revision + operation_revision = ( + session.workspace_revision if expected_revision is None else expected_revision + ) try: + + def validate_start() -> None: + self._ensure_execution_ready(session) + self._validate_workspace_revision(session, request.expected_workspace_revision) + return self.operations.start( workspace_id=request.workspace_id, kind="step", @@ -422,18 +636,31 @@ def start_step_operation(self, request: OperationStartStepRequest) -> dict: rerun=request.rerun, step=request.step, idempotency_key=request.idempotency_key, + workspace_revision=operation_revision, + command_input={"resetDependents": request.reset_dependents}, + snapshot_committer=lambda step, state, error: self._commit_step_snapshot( + session, + step, + state, + error, + ), + ledger_path=session.directory / "home" / "runtime-commands.json", + precondition=validate_start, runner=lambda observer: self._flow_run_step( FlowRunStepRequest( workspace_id=request.workspace_id, step=request.step, + expected_workspace_revision=expected_revision, rerun=request.rerun, ), observer=observer, reset_dependents=request.reset_dependents, ), ) + except RuntimeOperationIdempotencyConflict as exc: + raise RuntimeApiError("idempotency_conflict", str(exc)) from exc except RuntimeOperationConflict as exc: - raise RuntimeApiError("command_failed", str(exc)) from exc + raise RuntimeApiError("operation_conflict", str(exc)) from exc def operation_status(self, request: OperationIdRequest) -> dict: try: @@ -469,8 +696,15 @@ def acknowledge_step_rendered(self, request: OperationAckStepRenderedRequest) -> def workspace_snapshot(self, request: WorkspaceIdRequest) -> dict: session = self._get_session(request.workspace_id) - flow_data = getattr(getattr(session.workspace, "flow", None), "data", {}) + from chipcompiler.engine import read_workspace_configuration + + flow = getattr(session.workspace, "flow", None) + flow_data = getattr(flow, "data", {}) raw_steps = flow_data.get("steps", []) if isinstance(flow_data, dict) else [] + if not raw_steps: + loader = getattr(flow, "steps", None) + if callable(loader): + raw_steps = loader() steps = [ { "name": str(step.get("name", "")), @@ -504,14 +738,49 @@ def workspace_snapshot(self, request: WorkspaceIdRequest) -> dict: parameter_path = workspace_config_path(session.directory) home_data["parameters"] = str(parameter_path) + engineering_snapshot = self._read_engineering_snapshot(session) + try: + configuration = read_workspace_configuration(session.workspace) + except (OSError, ValueError): + configuration = None + if ( + not isinstance(configuration, dict) + or configuration.get("workspaceId") != session.workspace_id + or not isinstance(configuration.get("workspaceSpec"), dict) + or not isinstance(configuration.get("workspaceBindings"), dict) + ): + configuration = None + return { **self.operations.workspace_snapshot(request.workspace_id), + "engineeringSnapshot": engineering_snapshot, "directory": str(session.directory), "flow": {"steps": steps}, "home": stringify_paths(home_data), "parameters": stringify_paths(deepcopy(parameters_data)), + "configuration": stringify_paths(configuration) if configuration else None, } + def engineering_snapshot(self, request: WorkspaceIdRequest) -> dict: + """Read the committed Snapshot for the Phase 1 wire contract.""" + session = self._get_session(request.workspace_id) + from chipcompiler.engine.snapshot import EngineeringSnapshotError, read_engineering_snapshot + + try: + return read_engineering_snapshot( + session.workspace, + expected_workspace_id=session.workspace_id, + expected_workspace_revision=( + session.workspace_revision if session.workspace_revision > 0 else None + ), + ) + except EngineeringSnapshotError as exc: + raise RuntimeApiError( + "engineering_snapshot_unavailable", + str(exc), + {"workspaceId": request.workspace_id}, + ) from exc + def db_ensure(self, request: DbEnsureRequest) -> dict: self._require_persistent_db() @@ -563,6 +832,11 @@ def release(session: WorkspaceSession) -> dict: def layout_edit_begin(self, request: LayoutEditBeginRequest) -> dict: self._require_persistent_db() + session = self._get_session(request.workspace_id) + ownership_lock = None + if session.layout_edit_session is None: + ownership_lock = _workspace_ownership_lock(session.directory) + ownership_lock.__enter__() def begin(session: WorkspaceSession) -> dict: with self._layout_edit_lock: @@ -669,7 +943,19 @@ def begin(session: WorkspaceSession) -> dict: self._layout_edit_sessions[edit_session.edit_session_id] = edit_session return _layout_edit_begin_result(edit_session, reused=False) - return self._with_session_mutation_lock(request.workspace_id, begin) + try: + result = self._with_session_mutation_lock(request.workspace_id, begin) + except BaseException as exc: + if ownership_lock is not None: + ownership_lock.__exit__(type(exc), exc, exc.__traceback__) + raise + if ownership_lock is not None: + edit_session = session.layout_edit_session + if edit_session is not None and result.get("reused") is not True: + edit_session.ownership_lock = ownership_lock + else: + ownership_lock.__exit__(None, None, None) + return result def layout_edit_apply(self, request: LayoutEditApplyRequest) -> dict: def apply(session: WorkspaceSession, edit_session: LayoutEditSession) -> dict: @@ -744,7 +1030,9 @@ def save(session: WorkspaceSession, edit_session: LayoutEditSession) -> dict: }, ) if not edit_session.dirty: - return _layout_edit_save_result(edit_session, saved=False) + result = _layout_edit_save_result(edit_session, saved=False) + _release_layout_edit_ownership_lock(edit_session) + return result current_fingerprint = _artifact_fingerprint(edit_session.source_paths) if current_fingerprint != edit_session.source_fingerprint: @@ -780,8 +1068,19 @@ def save(session: WorkspaceSession, edit_session: LayoutEditSession) -> dict: edit_session.source_kind = "db" edit_session.source_paths = (output_db,) edit_session.source_fingerprint = _artifact_fingerprint(edit_session.source_paths) + workspace_revision = None + snapshot_path = Path(session.workspace.directory) / "home" / "engineering-snapshot.json" + if snapshot_path.is_file(): + workspace_revision = WorkspaceSpecRuntimeMixin._commit_workspace_snapshot( + session, + "layout.edit.save", + ) edit_session.dirty = False - return _layout_edit_save_result(edit_session, saved=True, artifacts=artifacts) + result = _layout_edit_save_result(edit_session, saved=True, artifacts=artifacts) + if workspace_revision is not None: + result["workspaceRevision"] = workspace_revision + _release_layout_edit_ownership_lock(edit_session) + return result return self._with_layout_edit_session_mutation_lock(request.edit_session_id, save) @@ -862,7 +1161,9 @@ def _discard_layout_edit_session(self, session: WorkspaceSession) -> bool: if edit_session is None: return False self._layout_edit_sessions.pop(edit_session.edit_session_id, None) - return self.sessions.release_layout_edit_session(session) + released = self.sessions.release_layout_edit_session(session) + _release_layout_edit_ownership_lock(edit_session) + return released def _release_session_db(self, session: WorkspaceSession) -> bool: return self.sessions.release_session_db(session) @@ -906,10 +1207,74 @@ def _with_session_mutation_lock( self, workspace_id: str, operation: Callable[[WorkspaceSession], _T], + *, + reject_active_operation: bool = False, ) -> _T: session = self._get_session(workspace_id) + if reject_active_operation: + self._ensure_no_active_operation(session) with session.mutation_lock: - return operation(session) + if reject_active_operation: + self._ensure_no_active_operation(session) + with _workspace_ownership_lock(session.directory): + return operation(session) + + @staticmethod + def _validate_workspace_revision( + session: WorkspaceSession, + expected_workspace_revision: int | None, + ) -> None: + if ( + expected_workspace_revision is None + or session.workspace_revision == 0 + or expected_workspace_revision == session.workspace_revision + ): + return + raise RuntimeApiError( + "revision_conflict", + "Workspace Revision does not match", + { + "expectedRevision": expected_workspace_revision, + "actualRevision": session.workspace_revision, + }, + ) + + def _ensure_no_active_operation(self, session: WorkspaceSession) -> None: + if self.operations.has_active_workspace(session.workspace_id): + raise RuntimeApiError( + "operation_conflict", + "Workspace has an active Operation", + ) + + @staticmethod + def _stale_step_ids(workspace) -> list[str]: + from chipcompiler.engine.snapshot import ( + EngineeringSnapshotError, + read_engineering_snapshot, + ) + + try: + snapshot = read_engineering_snapshot(workspace) + except EngineeringSnapshotError: + snapshot_path = Path(workspace.directory) / "home" / "engineering-snapshot.json" + if snapshot_path.is_file(): + raise + return [] + predecessor = snapshot.get("stalePredecessor") + if not isinstance(predecessor, dict): + return [] + return [ + step_id + for step_id in predecessor.get("invalidatedStepIds", []) + if isinstance(step_id, str) and step_id + ] + + @staticmethod + def _commit_rerun_snapshot(session: WorkspaceSession, cause: str) -> int | None: + snapshot_path = Path(session.directory) / "home" / "engineering-snapshot.json" + if not snapshot_path.is_file(): + return None + return WorkspaceSpecRuntimeMixin._commit_workspace_snapshot(session, cause) def _refresh_workspace_config(self, workspace) -> None: import chipcompiler.data as data_api @@ -948,16 +1313,20 @@ def _notify_rerun_prepared( workspace_steps, *, scope: str, + workspace_revision: int | None, target_step: str = "", ) -> None: callback = getattr(observer, "on_rerun_prepared", None) if callback is None: return - callback( - affected_steps=[str(getattr(step, "name", "")) for step in workspace_steps], - scope=scope, - target_step=target_step, - ) + payload = { + "affected_steps": [str(getattr(step, "name", "")) for step in workspace_steps], + "scope": scope, + "target_step": target_step, + } + if workspace_revision is not None: + payload["workspace_revision"] = workspace_revision + callback(**payload) @staticmethod def _prepare_step_for_rerun(workspace, engine_flow, workspace_step) -> None: @@ -1113,6 +1482,32 @@ def _validate_step_artifact_dir( return resolved +@contextmanager +def _workspace_ownership_lock(directory: Path): + """Acquire the CLI-compatible sibling lock without waiting in Runtime.""" + if fcntl is None: + yield + return + from chipcompiler.utility.workspace_lock import workspace_lock + + try: + with workspace_lock(directory, blocking=False): + yield + except BlockingIOError as exc: + raise RuntimeApiError( + "workspace_busy", + f"Workspace is busy: {directory}", + {"directory": str(directory)}, + ) from exc + + +def _release_layout_edit_ownership_lock(edit_session: LayoutEditSession) -> None: + ownership_lock = edit_session.ownership_lock + edit_session.ownership_lock = None + if ownership_lock is not None: + ownership_lock.__exit__(None, None, None) + + def _layout_edit_begin_result(edit_session: LayoutEditSession, *, reused: bool) -> dict: return { "editSessionId": edit_session.edit_session_id, @@ -2076,7 +2471,10 @@ def build_flow_for_workspace(workspace, *, create_step_workspaces: bool = True): def _workspace_session_result(session: WorkspaceSession) -> dict: - return {"workspaceId": session.workspace_id, "directory": str(session.directory)} + result = {"workspaceId": session.workspace_id, "directory": str(session.directory)} + if session.workspace_revision > 0: + result["workspaceRevision"] = session.workspace_revision + return result def _db_ensure_result( @@ -2232,25 +2630,25 @@ def _state_value(state: Any) -> str: def _run_engine_flow_steps(engine_flow, *, rerun: bool, observer) -> bool: - run_steps = engine_flow.run_steps - if observer is not None and _callable_accepts_keyword(run_steps, "observer"): - return run_steps(rerun=rerun, observer=observer) - return run_steps(rerun=rerun) + from chipcompiler.engine import ExecutionPlan, execute + result = execute( + engine_flow, + ExecutionPlan(intent="rerun" if rerun else "run"), + event_sink=observer, + ) + return result.succeeded -def _run_engine_flow_step(engine_flow, workspace_step, *, rerun: bool, observer): - run_step = engine_flow.run_step - if observer is not None and _callable_accepts_keyword(run_step, "observer"): - return run_step(workspace_step, rerun=rerun, observer=observer) - return run_step(workspace_step, rerun=rerun) +def _run_engine_flow_step(engine_flow, workspace_step, *, rerun: bool, observer): + from chipcompiler.engine import ExecutionPlan, execute -def _callable_accepts_keyword(callback, keyword: str) -> bool: - try: - parameters = inspect.signature(callback).parameters.values() - except (TypeError, ValueError): - return False - return any( - parameter.name == keyword or parameter.kind == inspect.Parameter.VAR_KEYWORD - for parameter in parameters + result = execute( + engine_flow, + ExecutionPlan( + intent="rerun" if rerun else "run", + step_id=str(getattr(workspace_step, "name", "")), + ), + event_sink=observer, ) + return _success_state() if result.succeeded else result.state diff --git a/chipcompiler/runtime/workspace_spec_api.py b/chipcompiler/runtime/workspace_spec_api.py new file mode 100644 index 000000000..ba22530df --- /dev/null +++ b/chipcompiler/runtime/workspace_spec_api.py @@ -0,0 +1,490 @@ +import os +from datetime import UTC, datetime +from pathlib import Path + +from chipcompiler.runtime.errors import RuntimeApiError +from chipcompiler.runtime.requests import ( + ProjectManifestDiscoverRequest, + ProjectManifestLoadRequest, + ProjectManifestMutationRequest, + WorkspaceConfigurationUpdateRequest, + WorkspaceOpenRequest, + WorkspaceSpecCreateRequest, + WorkspaceSpecOpenRequest, + WorkspaceSpecValidateRequest, + WorkspaceStepConfigurationUpdateRequest, + WorkspaceUpdateRequest, +) +from chipcompiler.runtime.sessions import ( + WorkspaceSession, + WorkspaceSessionNotFound, +) + + +class WorkspaceSpecRuntimeMixin: + def describe_workspace_spec(self, _request) -> dict: + from chipcompiler.engine import describe_workspace_spec + + return describe_workspace_spec() + + def validate_workspace_spec(self, request: WorkspaceSpecValidateRequest) -> dict: + from chipcompiler.engine import validate_workspace_spec + + return validate_workspace_spec(request.workspace_spec, request.workspace_bindings) + + def workspace_binding_requirement(self, request: WorkspaceSpecOpenRequest) -> dict: + from chipcompiler.engine import describe_workspace_binding_requirement + + try: + return describe_workspace_binding_requirement(request.directory) + except (OSError, ValueError) as exc: + raise RuntimeApiError("workspace_descriptor_invalid", str(exc)) from exc + + def read_workspace_configuration(self, request: WorkspaceOpenRequest) -> dict: + from chipcompiler.engine import ( + WorkspaceLifecycleError, + read_workspace_configuration_from_directory, + ) + + try: + return read_workspace_configuration_from_directory(request.directory) + except WorkspaceLifecycleError as exc: + raise RuntimeApiError(exc.code, str(exc), exc.details) from exc + + def discover_project(self, request: ProjectManifestDiscoverRequest) -> dict | None: + from chipcompiler.project import discover_project_manifest + + try: + discovered = discover_project_manifest(request.directory) + except (OSError, ValueError) as exc: + raise RuntimeApiError("project_manifest_invalid", str(exc)) from exc + if discovered is None: + return None + project_root, manifest = discovered + return { + "projectRoot": str(project_root), + "projectId": manifest["project_id"], + } + + def load_project_manifest(self, request: ProjectManifestLoadRequest) -> dict: + from chipcompiler.project import load_project_manifest + + try: + return load_project_manifest(request.project_root) + except (OSError, ValueError) as exc: + raise RuntimeApiError("project_manifest_invalid", str(exc)) from exc + + def mutate_project_manifest(self, request: ProjectManifestMutationRequest) -> dict: + from chipcompiler.project import ( + create_project_manifest, + mutate_project_manifest, + ) + + mutation = request.mutation + kind = mutation.get("type") + if kind == "create": + try: + return create_project_manifest( + request.project_root, + str(mutation.get("name") or "project"), + str(mutation.get("designName") or ""), + mpc=(mutation.get("mpc") if isinstance(mutation.get("mpc"), dict) else None), + ) + except (OSError, ValueError) as exc: + raise RuntimeApiError("project_manifest_invalid", str(exc)) from exc + now = str(mutation.get("now") or datetime.now(UTC).isoformat()) + source = mutation.get("input") if isinstance(mutation.get("input"), dict) else mutation + translated = { + "type": str(kind).replace("-", "_"), + **{ + target: source[key] + for key, target in ( + ("workspaceId", "workspace_id"), + ("workspacePath", "workspace_path"), + ("sourceWorkspaceId", "source_workspace_id"), + ("startStep", "start_step"), + ("endStep", "end_step"), + ("lifecycle", "lifecycle"), + ("name", "name"), + ("reason", "reason"), + ) + if key in source + }, + "updated_at": now, + } + if source.get("sourceStep"): + translated["branch_from"] = { + "source_workspace_id": str(source.get("sourceWorkspaceId") or ""), + "source_step": str(source["sourceStep"]), + **( + {"source_output_path": str(source["sourceOutputPath"])} + if source.get("sourceOutputPath") + else {} + ), + **( + {"source_output_type": str(source["sourceOutputType"])} + if source.get("sourceOutputType") + else {} + ), + } + if translated["type"] == "register_workspace": + workspace_path = str(translated.get("workspace_path") or "") + workspace_id = str(translated.get("workspace_id") or Path(workspace_path).name) + translated.update( + { + "workspace_id": workspace_id, + "name": str(translated.get("name") or workspace_id), + "created_at": now, + } + ) + try: + return mutate_project_manifest(request.project_root, translated) + except (OSError, ValueError) as exc: + raise RuntimeApiError("project_manifest_invalid", str(exc)) from exc + + def create_workspace( + self, + request: WorkspaceSpecCreateRequest, + ) -> dict: + return self._create_workspace_from_spec(request) + + def _create_workspace_from_spec(self, request: WorkspaceSpecCreateRequest) -> dict: + from chipcompiler.engine import ( + WorkspaceLifecycleError, + create_workspace_from_spec, + ) + + try: + if request.project_root: + from chipcompiler.project import create_project_workspace + + workspace = create_project_workspace( + request.project_root, + request.target_directory, + request.workspace_spec, + request.workspace_bindings, + command_id=request.command_id, + expected_project_id=request.project_id or None, + ) + else: + workspace = create_workspace_from_spec( + request.target_directory, + request.workspace_spec, + request.workspace_bindings, + request.command_id, + ) + except WorkspaceLifecycleError as exc: + raise RuntimeApiError(exc.code, str(exc), exc.details) from exc + except (OSError, ValueError) as exc: + raise RuntimeApiError("project_manifest_invalid", str(exc)) from exc + snapshot = self._read_engineering_snapshot(workspace) + session = self.sessions.create_session( + workspace.directory, + workspace=workspace, + workspace_id=snapshot["workspaceId"], + workspace_revision=snapshot["workspaceRevision"], + execution_readiness={"ready": True}, + workspace_bindings=request.workspace_bindings, + ) + self.operations.load_workspace_ledger( + session.workspace_id, + session.directory / "home" / "runtime-commands.json", + ) + return _workspace_session_result(session) + + def open_workspace( + self, + request: WorkspaceOpenRequest | WorkspaceSpecOpenRequest, + ) -> dict: + from chipcompiler.runtime.workspace_api import _workspace_ownership_lock + + with _workspace_ownership_lock(Path(request.directory)): + return self._open_workspace_unlocked(request) + + def _open_workspace_unlocked( + self, + request: WorkspaceOpenRequest | WorkspaceSpecOpenRequest, + ) -> dict: + workspace = self._load_workspace(request.directory) + snapshot = self._ensure_engineering_snapshot(workspace) + bindings = ( + request.workspace_bindings if isinstance(request, WorkspaceSpecOpenRequest) else None + ) + if isinstance(request, WorkspaceSpecOpenRequest): + from chipcompiler.engine import assess_execution_readiness + + readiness = assess_execution_readiness(workspace.directory, bindings) + if readiness.get("ready") is True: + from chipcompiler.engine import apply_workspace_bindings + + apply_workspace_bindings(workspace, bindings) + else: + readiness = {"ready": True} + + from chipcompiler.runtime.workspace_api import build_flow_for_workspace + + build_flow_for_workspace(workspace, create_step_workspaces=False) + session = self.sessions.open_session( + workspace.directory, + workspace=workspace, + workspace_id=snapshot["workspaceId"], + workspace_revision=snapshot["workspaceRevision"], + execution_readiness=readiness, + workspace_bindings=bindings, + ) + session.execution_readiness = readiness + session.workspace_bindings = bindings + self.operations.load_workspace_ledger( + session.workspace_id, + session.directory / "home" / "runtime-commands.json", + recover=False, + ) + return _workspace_session_result(session) + + def update_workspace(self, request: WorkspaceUpdateRequest) -> dict: + from chipcompiler.engine import ( + WorkspaceLifecycleError, + update_workspace_from_spec, + ) + + def update(session: WorkspaceSession) -> dict: + self._validate_workspace_revision( + session, + request.expected_workspace_revision, + ) + self._release_session_db(session) + try: + workspace = update_workspace_from_spec( + session.directory, + request.expected_workspace_revision, + request.workspace_spec, + request.workspace_bindings, + request.command_id, + ) + except WorkspaceLifecycleError as exc: + raise RuntimeApiError(exc.code, str(exc), exc.details) from exc + except (OSError, ValueError) as exc: + raise RuntimeApiError("workspace_update_failed", str(exc)) from exc + snapshot = self._read_engineering_snapshot(workspace) + session.workspace = workspace + session.workspace_revision = snapshot["workspaceRevision"] + session.workspace_bindings = request.workspace_bindings + session.execution_readiness = {"ready": True} + return _workspace_session_result(session) + + return self._with_session_mutation_lock( + request.workspace_id, + update, + reject_active_operation=True, + ) + + def update_workspace_configuration(self, request: WorkspaceConfigurationUpdateRequest) -> dict: + from chipcompiler.engine import ( + WorkspaceLifecycleError, + update_workspace_configuration, + ) + + def update(session: WorkspaceSession) -> dict: + self._validate_workspace_revision( + session, + request.expected_workspace_revision, + ) + self._release_session_db(session) + try: + workspace = update_workspace_configuration( + session.directory, + request.expected_workspace_revision, + request.configuration, + request.workspace_bindings, + request.command_id, + ) + except WorkspaceLifecycleError as exc: + raise RuntimeApiError(exc.code, str(exc), exc.details) from exc + except (OSError, ValueError) as exc: + raise RuntimeApiError("workspace_configuration_update_failed", str(exc)) from exc + snapshot = self._read_engineering_snapshot(workspace) + session.workspace = workspace + session.workspace_revision = snapshot["workspaceRevision"] + session.workspace_bindings = request.workspace_bindings + session.execution_readiness = {"ready": True} + return _workspace_session_result(session) + + return self._with_session_mutation_lock( + request.workspace_id, + update, + reject_active_operation=True, + ) + + def update_workspace_step_configuration( + self, request: WorkspaceStepConfigurationUpdateRequest + ) -> dict: + from chipcompiler.engine import ( + WorkspaceLifecycleError, + update_workspace_step_configuration, + ) + + def update(session: WorkspaceSession) -> dict: + self._validate_workspace_revision( + session, + request.expected_workspace_revision, + ) + self._release_session_db(session) + try: + workspace = update_workspace_step_configuration( + session.directory, + request.expected_workspace_revision, + request.step_id, + request.parameters, + request.command_id, + ) + except WorkspaceLifecycleError as exc: + raise RuntimeApiError(exc.code, str(exc), exc.details) from exc + except (OSError, ValueError) as exc: + raise RuntimeApiError( + "workspace_step_configuration_update_failed", str(exc) + ) from exc + snapshot = self._read_engineering_snapshot(workspace) + session.workspace = workspace + session.workspace_revision = snapshot["workspaceRevision"] + return _workspace_session_result(session) + + return self._with_session_mutation_lock( + request.workspace_id, + update, + reject_active_operation=True, + ) + + def _load_workspace(self, directory: str): + if not directory: + raise RuntimeApiError("invalid_request", "missing required field: directory") + if not os.path.isdir(directory): + raise RuntimeApiError( + "invalid_request", + f"invalid workspace directory: {directory}", + ) + + import chipcompiler.data as data_api + + workspace = data_api.load_workspace(directory=directory) + if workspace is None: + raise RuntimeApiError( + "command_failed", + f"load workspace failed : {directory}", + ) + return workspace + + def _get_session(self, workspace_id: str) -> WorkspaceSession: + try: + return self.sessions.get_session(workspace_id) + except WorkspaceSessionNotFound as exc: + raise RuntimeApiError( + "workspace_session_not_found", + f"workspace session not found: {workspace_id}", + ) from exc + + @staticmethod + def _ensure_execution_ready(session: WorkspaceSession) -> None: + readiness = session.execution_readiness + if readiness.get("ready") is True: + return + code = str(readiness.get("code") or "pdk_binding_missing") + raise RuntimeApiError(code, "Workspace is not ready for execution", readiness) + + @staticmethod + def _ensure_engineering_snapshot(workspace) -> dict: + from chipcompiler.engine.snapshot import ( + EngineeringSnapshotError, + read_engineering_snapshot, + ) + + try: + return read_engineering_snapshot(workspace) + except EngineeringSnapshotError as exc: + raise RuntimeApiError("engineering_snapshot_unavailable", str(exc)) from exc + + @staticmethod + def _create_engineering_snapshot(workspace) -> dict: + from chipcompiler.engine.snapshot import ( + EngineeringSnapshotError, + create_engineering_snapshot, + ) + + try: + return create_engineering_snapshot(workspace) + except EngineeringSnapshotError as exc: + raise RuntimeApiError("engineering_snapshot_commit_failed", str(exc)) from exc + + @staticmethod + def _read_engineering_snapshot(owner) -> dict: + from chipcompiler.engine.snapshot import ( + EngineeringSnapshotError, + read_engineering_snapshot, + ) + + try: + workspace = getattr(owner, "workspace", owner) + revision = getattr(owner, "workspace_revision", None) + return read_engineering_snapshot( + workspace, + expected_workspace_id=getattr(owner, "workspace_id", None), + expected_workspace_revision=revision + if isinstance(revision, int) and revision > 0 + else None, + ) + except EngineeringSnapshotError as exc: + raise RuntimeApiError("engineering_snapshot_unavailable", str(exc)) from exc + + @staticmethod + def _commit_step_snapshot( + session: WorkspaceSession, + workspace_step, + state, + error: str | None, + ) -> int: + state_value = str(getattr(state, "value", state)).lower() + try: + return WorkspaceSpecRuntimeMixin._commit_workspace_snapshot( + session, + f"flow_step.{state_value}", + ) + except RuntimeApiError as exc: + raise RuntimeApiError( + "engineering_snapshot_commit_failed", + str(exc), + { + "step": str(getattr(workspace_step, "name", "")), + "error": error or "", + }, + ) from exc + + @staticmethod + def _commit_workspace_snapshot(session: WorkspaceSession, cause: str) -> int: + from chipcompiler.engine.snapshot import ( + EngineeringSnapshotError, + commit_engineering_snapshot, + ) + + try: + snapshot = commit_engineering_snapshot( + session.workspace, + workspace_id=session.workspace_id, + cause=cause, + ) + except EngineeringSnapshotError as exc: + raise RuntimeApiError("engineering_snapshot_commit_failed", str(exc)) from exc + session.workspace_revision = snapshot["workspaceRevision"] + return session.workspace_revision + + +def _workspace_session_result(session: WorkspaceSession) -> dict: + return { + "workspaceId": session.workspace_id, + "workspaceRevision": session.workspace_revision, + "directory": str(session.directory), + **( + {"executionReadiness": session.execution_readiness} + if session.workspace_bindings is not None + or session.execution_readiness.get("ready") is not True + else {} + ), + } diff --git a/chipcompiler/tools/ecc/checklist_render.py b/chipcompiler/tools/ecc/checklist_render.py new file mode 100644 index 000000000..03c381e68 --- /dev/null +++ b/chipcompiler/tools/ecc/checklist_render.py @@ -0,0 +1,13 @@ +"""Non-persisting checklist rendering for read-only package inspection.""" + +from pathlib import Path + +from chipcompiler.data import Checklist + + +def render_checklist(path: Path | str, items, *, persist: bool = False) -> dict: + if persist: + checklist = Checklist(path) + checklist.replace(list(items)) + return checklist.data + return Checklist.from_items(path, items) diff --git a/chipcompiler/tools/ecc/metrics.py b/chipcompiler/tools/ecc/metrics.py index 33077a7b8..fc2131e05 100644 --- a/chipcompiler/tools/ecc/metrics.py +++ b/chipcompiler/tools/ecc/metrics.py @@ -5,6 +5,7 @@ from pathlib import Path from chipcompiler.data import EccStep, StateEnum, StepEnum, StepMetrics, Workspace, WorkspaceStep +from chipcompiler.tools.ecc.qor_detail_facts import database_fact_summary, lvs_detail_summary from chipcompiler.tools.ecc.sta_qor import ( POST_SYNTHESIS_STA_CORNER, STA_POWER_SUMMARY_FILENAME, @@ -54,7 +55,7 @@ "name": "synthesis_power_internal_uw", "display_name": "Synthesis Internal Power", "unit": "uW", - "dimension": "power", + "dimension": "power_integrity", "polarity": "trend_only", "confidence": "medium", }, @@ -62,7 +63,7 @@ "name": "synthesis_power_switching_uw", "display_name": "Synthesis Switching Power", "unit": "uW", - "dimension": "power", + "dimension": "power_integrity", "polarity": "trend_only", "confidence": "medium", }, @@ -70,7 +71,7 @@ "name": "synthesis_power_dynamic_uw", "display_name": "Synthesis Dynamic Power", "unit": "uW", - "dimension": "power", + "dimension": "power_integrity", "polarity": "trend_only", "confidence": "medium", }, @@ -78,7 +79,7 @@ "name": "synthesis_power_leakage_uw", "display_name": "Synthesis Leakage Power", "unit": "uW", - "dimension": "power", + "dimension": "power_integrity", "polarity": "trend_only", "confidence": "medium", }, @@ -138,6 +139,13 @@ "dimension": "area_cost", "polarity": "trend_only", }, + "instance_area": { + "name": "instance_area", + "display_name": "Instance Area", + "unit": "um^2", + "dimension": "area_cost", + "polarity": "trend_only", + }, "Total nets": { "name": "net_count", "display_name": "Net Count", @@ -145,6 +153,62 @@ "dimension": "routability_physical", "polarity": "trend_only", }, + "macro_count": { + "name": "macro_count", + "display_name": "Macro Count", + "unit": "count", + "dimension": "area_cost", + "polarity": "trend_only", + }, + "macro_area": { + "name": "macro_area", + "display_name": "Macro Area", + "unit": "um^2", + "dimension": "area_cost", + "polarity": "trend_only", + }, + "std_cell_count": { + "name": "std_cell_count", + "display_name": "Standard Cell Count", + "unit": "count", + "dimension": "area_cost", + "polarity": "trend_only", + }, + "std_cell_area": { + "name": "std_cell_area", + "display_name": "Standard Cell Area", + "unit": "um^2", + "dimension": "area_cost", + "polarity": "trend_only", + }, + "clock_count": { + "name": "clock_count", + "display_name": "Clock Cell Count", + "unit": "count", + "dimension": "area_cost", + "polarity": "trend_only", + }, + "clock_area": { + "name": "clock_area", + "display_name": "Clock Cell Area", + "unit": "um^2", + "dimension": "area_cost", + "polarity": "trend_only", + }, + "io_pad_count": { + "name": "io_pad_count", + "display_name": "IO Pad Count", + "unit": "count", + "dimension": "routability_physical", + "polarity": "trend_only", + }, + "io_pad_area": { + "name": "io_pad_area", + "display_name": "IO Pad Area", + "unit": "um^2", + "dimension": "area_cost", + "polarity": "trend_only", + }, "GP HPWL": { "name": "place_hpwl", "display_name": "Place HPWL", @@ -1856,7 +1920,16 @@ def _sta_corner_context(step: WorkspaceStep, corner: str | None) -> dict | None: "core_utilization": "/Design Layout/core_usage", "io_pin_count": "/Design Statis/num_iopins", "instance_count": "/Design Statis/num_instances", + "instance_area": "/Instances/total/area", "net_count": "/Design Statis/num_nets", + "macro_count": "/Instances/macros/num", + "macro_area": "/Instances/macros/area", + "std_cell_count": "/Instances/logic/num", + "std_cell_area": "/Instances/logic/area", + "clock_count": "/Instances/clock/num", + "clock_area": "/Instances/clock/area", + "io_pad_count": "/Instances/iopads/num", + "io_pad_area": "/Instances/iopads/area", } @@ -2099,6 +2172,22 @@ def _metric_feature_source( def _qor_detail_records(step: WorkspaceStep, step_metrics: StepMetrics) -> list[dict]: details = [] + database_path = getattr(step.feature, "db", None) + database_summary = database_fact_summary(database_path) + database_source = _relative_step_path(step, database_path) + if database_summary is not None and database_source is not None: + details.append( + { + "id": "database_facts", + "presentation": "database_facts", + "summary": database_summary, + "feature_source": { + "kind": "feature", + "path": database_source, + "selector": "", + }, + } + ) detail_specs = ( ("place_map_metrics", "place_map_summary", getattr(step.feature, "map", None)), ("cts_clock_skew_metrics", "cts_clock_skew_table", getattr(step.feature, "step", None)), @@ -2165,6 +2254,23 @@ def _qor_detail_records(step: WorkspaceStep, step_metrics: StepMetrics) -> list[ }, } ) + if step.name == StepEnum.LVS.value: + feature_path = getattr(step.feature, "step", None) + source_path = _relative_step_path(step, feature_path) + summary = lvs_detail_summary(feature_path) + if source_path is not None and summary is not None: + details.append( + { + "id": "lvs_connectivity_summary", + "presentation": "lvs_connectivity_tables", + "summary": summary, + "feature_source": { + "kind": "feature", + "path": source_path, + "selector": "/lvs", + }, + } + ) return details @@ -3505,6 +3611,23 @@ def build_metrics_db(workspace: Workspace, step: EccStep) -> dict: ): _add_number_metric(metrics, label, statistics.get(key)) + instances = data.get("Instances", {}) + instances = instances if isinstance(instances, dict) else {} + for metric_id, instance_kind, key in ( + ("instance_area", "total", "area"), + ("macro_count", "macros", "num"), + ("macro_area", "macros", "area"), + ("std_cell_count", "logic", "num"), + ("std_cell_area", "logic", "area"), + ("clock_count", "clock", "num"), + ("clock_area", "clock", "area"), + ("io_pad_count", "iopads", "num"), + ("io_pad_area", "iopads", "area"), + ): + values = instances.get(instance_kind, {}) + if isinstance(values, dict): + _add_number_metric(metrics, metric_id, values.get(key)) + metrics.update(build_metrics_timing(workspace=workspace, step=step)) return metrics diff --git a/chipcompiler/tools/ecc/qor_detail_facts.py b/chipcompiler/tools/ecc/qor_detail_facts.py new file mode 100644 index 000000000..7777d1343 --- /dev/null +++ b/chipcompiler/tools/ecc/qor_detail_facts.py @@ -0,0 +1,210 @@ +"""Bounded dashboard detail facts derived from step feature JSON.""" + +from math import isfinite +from typing import Any + +from chipcompiler.utility import json_read + +_INSTANCE_CLASS_LIMIT = 32 +_LAYER_RECORD_LIMIT = 64 +_PIN_DISTRIBUTION_LIMIT = 64 +_LVS_RECORD_LIMIT = 100 + + +def qor_number(value: Any) -> int | float | None: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) if value.is_integer() else value if isfinite(value) else None + if isinstance(value, str): + text = value.strip() + if not text: + return None + percent = text.endswith("%") + if percent: + text = text[:-1].strip() + text = text.replace(",", "") + try: + number = float(text) + except ValueError: + return None + if percent: + number = number / 100.0 + if not isfinite(number): + return None + return int(number) if number.is_integer() else number + return None + + +def database_fact_summary(feature_path) -> dict | None: + data = json_read(feature_path or "") + if not isinstance(data, dict): + return None + layout = data.get("Design Layout") + statistics = data.get("Design Statis") + instances = data.get("Instances") + pins = data.get("Pins") + layers = data.get("Layers") + nets = data.get("Nets") + layout = layout if isinstance(layout, dict) else {} + statistics = statistics if isinstance(statistics, dict) else {} + instances = instances if isinstance(instances, dict) else {} + pins = pins if isinstance(pins, dict) else {} + layers = layers if isinstance(layers, dict) else {} + nets = nets if isinstance(nets, dict) else {} + + def instance_record(kind, value): + value = value if isinstance(value, dict) else {} + return { + "kind": kind, + "count": qor_number(value.get("num")), + "area": qor_number(value.get("area")), + "pin_count": qor_number(value.get("pin_num")), + } + + def layer_records(value, source_metric, output_metric): + if not isinstance(value, list): + return [] + records = [] + for index, item in enumerate(value[:_LAYER_RECORD_LIMIT]): + if not isinstance(item, dict): + continue + layer = item.get("layer_name") + records.append( + { + "layer": layer if isinstance(layer, str) and layer else f"Layer {index + 1}", + output_metric: qor_number(item.get(source_metric)), + } + ) + return records + + pin_distribution = pins.get("pin_distribution") + summary = { + "schema_version": 1, + "layout": { + "die_area": qor_number(layout.get("die_area")), + "die_usage": qor_number(layout.get("die_usage")), + "die_width": qor_number(layout.get("die_bounding_width")), + "die_height": qor_number(layout.get("die_bounding_height")), + "core_area": qor_number(layout.get("core_area")), + "core_usage": qor_number(layout.get("core_usage")), + "core_width": qor_number(layout.get("core_bounding_width")), + "core_height": qor_number(layout.get("core_bounding_height")), + "dbu": qor_number(layout.get("design_dbu")), + }, + "statistics": { + "io_pins": qor_number(statistics.get("num_iopins")), + "instances": qor_number(statistics.get("num_instances")), + "nets": qor_number(statistics.get("num_nets")), + "pdn": qor_number(statistics.get("num_pdn")), + }, + "instance_classes": [ + instance_record(kind, value) + for kind, value in sorted(instances.items()) + if kind != "total" and isinstance(kind, str) and isinstance(value, dict) + ][:_INSTANCE_CLASS_LIMIT], + "instance_total": instance_record("total", instances.get("total")), + "pin_distribution": [ + { + "pin_count": int(pin_count), + "instance_count": qor_number(item.get("inst_num")), + "net_count": qor_number(item.get("net_num")), + } + for item in ( + pin_distribution[:_PIN_DISTRIBUTION_LIMIT] + if isinstance(pin_distribution, list) + else [] + ) + if isinstance(item, dict) + and (pin_count := qor_number(item.get("pin_num"))) is not None + and pin_count >= 0 + and float(pin_count).is_integer() + ], + "cut_layers": layer_records(layers.get("cut_layers"), "via_num", "via_count"), + "routing_layers": layer_records(layers.get("routing_layers"), "wire_len", "wire_length"), + "wire_length": qor_number(nets.get("wire_len")), + "via_count": qor_number(nets.get("num_via")), + } + scalar_groups = (summary["layout"], summary["statistics"]) + has_instance_total = any( + summary["instance_total"][key] is not None for key in ("count", "area", "pin_count") + ) + if ( + not has_instance_total + and not any(value is not None for group in scalar_groups for value in group.values()) + and not any( + summary[key] + for key in ("instance_classes", "pin_distribution", "cut_layers", "routing_layers") + ) + and summary["wire_length"] is None + and summary["via_count"] is None + ): + return None + return summary + + +def lvs_text(value) -> str: + if isinstance(value, list): + return ", ".join(str(item).strip() for item in value if str(item).strip()) + return value.strip() if isinstance(value, str) else "" + + +def lvs_detail_summary(feature_path) -> dict | None: + feature = json_read(feature_path or "") + if not isinstance(feature, dict): + return None + section = feature.get("lvs", feature) + if not isinstance(section, dict): + return None + entities = [] + raw_entities = section.get("entity", []) + for item in raw_entities[:_LVS_RECORD_LIMIT] if isinstance(raw_entities, list) else []: + if not isinstance(item, dict) or not lvs_text(item.get("entity")): + continue + entities.append( + { + "entity": lvs_text(item.get("entity")), + "netlist": qor_number(item.get("netlist")), + "def": qor_number(item.get("def")), + "difference": qor_number(item.get("difference")), + } + ) + connectivity = [] + raw_connectivity = section.get("connectivity", []) + for item in raw_connectivity[:_LVS_RECORD_LIMIT] if isinstance(raw_connectivity, list) else []: + if not isinstance(item, dict) or not lvs_text(item.get("connectivity")): + continue + connectivity.append( + { + "connectivity": lvs_text(item.get("connectivity")), + "open": qor_number(item.get("open")), + "short": qor_number(item.get("short")), + "connected": qor_number(item.get("connected")), + "total": qor_number(item.get("total")), + } + ) + violations = [] + raw_violations = section.get("violations", []) + for item in raw_violations[:_LVS_RECORD_LIMIT] if isinstance(raw_violations, list) else []: + if not isinstance(item, dict): + continue + violation_type = lvs_text(item.get("type")) + if not violation_type: + continue + violations.append( + { + "type": violation_type, + "net": lvs_text(item.get("net")), + "instance": lvs_text(item.get("instance")), + "terminals": lvs_text(item.get("terminals")), + "components": lvs_text(item.get("components")), + } + ) + return { + "schema_version": 1, + "entities": entities, + "connectivity": connectivity, + "violations": violations, + } diff --git a/chipcompiler/tools/ecc/runner.py b/chipcompiler/tools/ecc/runner.py index 188fcc925..5fbda7f54 100644 --- a/chipcompiler/tools/ecc/runner.py +++ b/chipcompiler/tools/ecc/runner.py @@ -906,7 +906,7 @@ def run_sta(workspace: Workspace, step: EccStep, ecc_module: ECCToolsModule | No sub_flow.update_step(step_name=EccSubFlowEnum.run_sta.value, state=StateEnum.Imcomplete) return False - if not os.path.exists(workspace.pdk.sdc): + if not workspace.pdk.sdc or not os.path.exists(workspace.pdk.sdc): workspace.logger.error("STA SDC does not exist: %s", workspace.pdk.sdc) sub_flow.update_step(step_name=EccSubFlowEnum.run_sta.value, state=StateEnum.Imcomplete) return False diff --git a/chipcompiler/tools/ecc/signoff_checklist.py b/chipcompiler/tools/ecc/signoff_checklist.py index 25cd09847..5c9ab93a5 100644 --- a/chipcompiler/tools/ecc/signoff_checklist.py +++ b/chipcompiler/tools/ecc/signoff_checklist.py @@ -639,7 +639,9 @@ def _package_items(resource_issues) -> list[dict]: return items -def rebuild_home_checklist(workspace: Workspace, resource_issues=None) -> dict: +def rebuild_home_checklist( + workspace: Workspace, resource_issues=None, *, persist: bool = True +) -> dict: """Replace the aggregate workspace checklist from current step snapshots.""" workspace_directory = getattr(workspace, "directory", None) if not workspace_directory: @@ -690,8 +692,8 @@ def rebuild_home_checklist(workspace: Workspace, resource_issues=None) -> dict: # Recover home.json files whose checklist path was cleared by an older # home.reset(); persist so later checklist updates resolve as well. checklist_path = workspace_dir / "home" / "checklist.json" - if workspace.home.path is not None: + if persist and workspace.home.path is not None: workspace.home.set_checklist(checklist_path) - checklist = Checklist(checklist_path) - checklist.replace(list(deduplicated.values())) - return checklist.data + from chipcompiler.tools.ecc.checklist_render import render_checklist + + return render_checklist(checklist_path, deduplicated.values(), persist=persist) diff --git a/chipcompiler/tools/ecc_sizer/builder.py b/chipcompiler/tools/ecc_sizer/builder.py index 6009ebebd..0723038a6 100644 --- a/chipcompiler/tools/ecc_sizer/builder.py +++ b/chipcompiler/tools/ecc_sizer/builder.py @@ -2,7 +2,7 @@ import shutil from pathlib import Path -from chipcompiler.data import EccStep, Workspace +from chipcompiler.data import EccStep, Workspace, step_storage_name from chipcompiler.tools.ecc import builder as ecc_builder from .utility import find_sizer_root @@ -22,7 +22,7 @@ def step_shape( Dependency-free (no rosettakit import at module scope of this module's callers) so deferred creation and selected creation share one shape. """ - safe_step_name = "_".join(step_name.split()).lower() + safe_step_name = step_storage_name(step_name, "sizer") step_directory = Path(workspace.directory) / f"{safe_step_name}_sizer" if output_def is None: output_def = step_directory / "output" / f"{workspace.design.name}_{safe_step_name}.def.gz" diff --git a/chipcompiler/utility/workspace_lock.py b/chipcompiler/utility/workspace_lock.py new file mode 100644 index 000000000..84a841ae5 --- /dev/null +++ b/chipcompiler/utility/workspace_lock.py @@ -0,0 +1,54 @@ +import threading +from contextlib import contextmanager +from pathlib import Path + +try: + import fcntl +except ImportError: # pragma: no cover - Windows has no sibling flock. + fcntl = None + +_local = threading.local() +_locks_guard = threading.Lock() +_locks: dict[str, threading.RLock] = {} + + +@contextmanager +def workspace_lock(directory: str | Path, *, blocking: bool = True): + """Share the sibling Workspace lock across Engine and Runtime callers.""" + path = Path(directory).expanduser().resolve() + key = str(path) + held = getattr(_local, "held", set()) + if key in held: + yield + return + + with _locks_guard: + lock = _locks.setdefault(key, threading.RLock()) + if not lock.acquire(blocking=blocking): + raise BlockingIOError(f"Workspace lock is busy: {path}") + + lock_file = None + file_locked = False + try: + path.parent.mkdir(parents=True, exist_ok=True) + if fcntl is not None: + lock_file = (path.parent / f"{path.name}.lock").open("a") + flags = fcntl.LOCK_EX if blocking else fcntl.LOCK_EX | fcntl.LOCK_NB + fcntl.flock(lock_file.fileno(), flags) + file_locked = True + held = set(held) + held.add(key) + _local.held = held + yield + finally: + if key in getattr(_local, "held", set()): + remaining = set(_local.held) + remaining.remove(key) + _local.held = remaining + if lock_file is not None: + try: + if file_locked: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + finally: + lock_file.close() + lock.release() diff --git a/docs/development.cn.md b/docs/development.cn.md index 0b14b99a0..afeed60cb 100644 --- a/docs/development.cn.md +++ b/docs/development.cn.md @@ -247,8 +247,7 @@ chipcompiler/cli/commands/ # typer 命令定义层（薄） ├── project_config.py # project 子应用（set/unset/add/remove/show） ├── workspace.py # workspace 子应用（refresh） ├── signoff.py # signoff 子应用（inspect/export） - ├── report.py # report 子应用（summary/qor/checklist/step） - └── rpc.py # rpc 子应用（serve） + └── report.py # report 子应用（summary/qor/checklist/step） chipcompiler/cli/command_handlers/ # 业务处理层（唯一的处理器包，有状态/重逻辑） ├── project.py # init / check / run / migrate / workspace refresh（含 preset 解析与环境预检） ├── inspect.py # status / log / config @@ -274,7 +273,8 @@ chipcompiler/cli/inspection/ # 只读探查逻辑 chipcompiler/cli/project/ # config.py（ecc.toml 解析校验）/ config_fields.py（`ecc project` 的项目声明 schema）/ params.py（参数注册表）/ workspace_params.py（workspace 局部覆盖记录）/ manifest.py（项目形态分类）/ effective_config.py / config_params/（直配参数 schema）/ migrate*.py（旧布局迁移）/ run_*.py（run 目标解析与分发） chipcompiler/cli/rendering/ # 输出渲染（render / renderers / pretty / progress） chipcompiler/engine/signoff/ # 签核收集器 + 设计/checklist 报告（包，见下文） -chipcompiler/engine/qor_report.py # QoR 总分计分（GUI 规则移植） +chipcompiler/analysis/qor/ # QoR v3 唯一分析、评分与报告契约 +chipcompiler/engine/qor_report.py # CLI QoR facade，委托 analysis.qor ``` 模块归属由 `test/cli/test_cli_module_layout.py` 强制：核心框架必须在 `cli/core/`、命令注册在 `cli/commands/`、全部处理器在唯一的 `cli/command_handlers/` 包、只读探查在 `cli/inspection/`、渲染在 `cli/rendering/`；旧的 `chipcompiler/cli/*.py` 平铺模块必须不可导入。新增文件时放进对应子包，不要在 `cli/` 根下新建模块。 @@ -414,7 +414,7 @@ config_param( - **新建 workspace**：解析 `[design]` 输入声明、PDK、参数与请求入口步骤；只校验入口步骤所需文件；先原子登记受管名称到 `project.json`（`not_started`）；预检工具；在 `/` 调用 `create_workspace`。`create_workspace` 将输入复制到 `origin/` 并产出全部步骤配置，CLI 后续不改写配置。正常新建 flow 用 preset；`--from A --to B` 改用 `rtl2gds.build_flow_range(A, B)` 动态构建包含式规范范围。新范围不能与 `--preset`、`--overwrite`、`--resume`、`--only`、`--force` 组合。 - **已有 workspace**：先由 `chipcompiler/engine/reconcile.py` 把持久化 flow 与目标对齐（前缀 → 追加扩展；超集且全成 → `no_op`；分叉 → `flow_mismatch`），再 `load_workspace` 后由 `chipcompiler.engine.rerun` 的 `run_resume`、`run_from` 或 `run_only` 原地复跑。`--from A --to B` 是已有 flow 的包含式范围，会将其后的步骤状态失效但保留其输出文件。已有 workspace 不会重新预检输入，也不会改写已复制输入或配置。 -项目 preset 的步骤序列定义在 `chipcompiler/rtl2gds/builder.py`（`build_*_flow()` / `get_flow_builders()`），不在 CLI 层。`build_flow_range()` 对规范的 `build_rtl2gds_flow()` 结果切片，步骤别名和顺序只有一份来源。修改序列时须同步引擎默认 flow、`StepEnum` 与 manifest 范围映射；CLI 只负责参数解析、输入契约、进度渲染选择与结果映射。 +项目 preset 的步骤序列定义在 `chipcompiler/rtl2gds/builder.py`（`build_*_flow()` / `get_flow_builders()`），不在 CLI 层。`build_flow_range()` 对规范的 `build_rtl2gds_flow()` 结果切片，步骤别名和顺序只有一份来源。修改序列时须同步引擎默认 flow、`StepEnum` 与 manifest 范围映射；CLI 只负责参数解析、输入契约、进度渲染选择与结果映射。交互式 TTY 的 `ecc run` 走 `run_flow_with_progress()`，`--plain` 与 GUI 走 `execute()`；两条路径挂同一套 Engineering Snapshot 提交 observer，每完成一步都会更新 `home/engineering-snapshot.json`。 #### 扩展环境探查（doctor / 预检） @@ -428,7 +428,9 @@ config_param( #### 扩展报告（`ecc report summary/qor/checklist/step`） - **设计总结**：`ecc report summary` 调用 `chipcompiler.engine.signoff.generate_text_report`。其实现按职责分模块（`report.py` 编排 / `report_data.py` 数据契约 / `report_extract.py` 解析器+workspace 收集 / `report_sections.py` 分区抽取 / `report_timing.py` timing 链 / `report_text.py` 格式化），全部经包 `__init__` 对外暴露。新增报告分区时，在 `report_sections.py`（或 timing 链）增加 `_extract_(q)`，并在 `report.py` 编排处注册。 -- `engine/qor_report.py`：GUI `projectQorTrend.ts` 的单 workspace 移植——常量表（`METRIC_FAIL_VALUES`/`DIMENSION_WEIGHTS`/`QOR_SCORE_THRESHOLD`）+ 归一化 + 项目级记录选择（role 优先级 final>gate>trend、area_cost 只取最后成功的 area 步）+ `score_record` 计分公式 + 维度加权（不重归一化）。新增可计分指标 = 在 GUI 与 `METRIC_FAIL_VALUES` 同步加阈值。 +- `analysis/qor/`：QoR v3 唯一分析引擎，负责指标加载、feature/维度计算、feasibility gates、evidence、评分、diagnosis、intervention、有界报告 schema 和文本渲染。新的工程结论只能在这里实现，GUI/CLI 不得复制阈值或公式。 +- `engine/qor_report.py`：CLI `ecc report qor` facade，委托 `analysis.qor`，不拥有第二套评分实现。 +- `engine/qor_scoring.py` 与 `engine/qor.py`：仅为生产 Snapshot v2 兼容保留；ECC-only 阶段不要让 QoR v3 消费这条路径。 - `engine/signoff/report_checklist.py`：只读渲染 `home/checklist.json`（不合法时报 unavailable，绝不回写文件）。 - CLI：`cli/commands/report.py` + `cli/command_handlers/report.py`；workspace 解析复用 `inspection/discovery.py`（`resolve_workspace_path` 是无副作用核心，`resolve_command_workspace` 是核心加 `load_workspace`；signoff、report 与只读的 status/log/config 共用）。 @@ -490,7 +492,7 @@ uv run ecc run --project gcd --preset rtl2gds ### 报告 -`ecc report qor` 按 GUI 项目看板相同的方式给 workspace 打分（每指标对固定 fail 阈值计分、维度求均值、加权总分——缺失维度不做权重重归一化）；`ecc report checklist` 渲染签核清单状态；`ecc report summary` 写出与 GUI 一致的文本设计总结。三者默认写入 `/signoff/`，接受 `-o` 以及常规的 `--project` 和可选的受管 `--workspace NAME` 选择器： +`ecc report qor` 委托 `chipcompiler.analysis.qor` QoR v3 引擎。生产 Snapshot v2 的 `qorAssessment` 仅为当前 GUI 兼容保留，不是 v3 工程结论的第二来源。`ecc report checklist` 渲染签核清单状态；`ecc report summary` 写出与 GUI 一致的文本设计总结。三者默认写入 `/signoff/`，接受 `-o` 以及常规的 `--project` 和可选的受管 `--workspace NAME` 选择器： ```bash uv run ecc report qor --project gcd diff --git a/docs/development.md b/docs/development.md index bb23cba1d..5e09dfe21 100644 --- a/docs/development.md +++ b/docs/development.md @@ -303,7 +303,8 @@ chipcompiler/cli/inspection/ # read-only probing logic chipcompiler/cli/project/ # config.py (ecc.toml parsing and validation) / config_fields.py (project declaration schema for `ecc project`) / params.py (parameter registry) / workspace_params.py (workspace-local override records) / manifest.py (project-state classification) / effective_config.py / config_params/ (direct-config schemas) / migrate*.py (legacy-layout migration) / run_*.py (workspace target resolution and dispatch) chipcompiler/cli/rendering/ # output rendering (render / renderers / pretty / progress) chipcompiler/engine/signoff/ # signoff collector + design/checklist reports (package, see below) -chipcompiler/engine/qor_report.py # overall QoR scoring (port of the GUI rules) +chipcompiler/analysis/qor/ # canonical QoR v3 analysis, scoring, and report contract +chipcompiler/engine/qor_report.py # CLI QoR facade delegating to analysis.qor ``` Module placement is enforced by `test/cli/test_cli_module_layout.py`: the core @@ -533,7 +534,10 @@ Project preset sequences are defined in `chipcompiler/rtl2gds/builder.py` step aliases and ordering have one source of truth. Keep a sequence change coordinated with the engine's default flow, `StepEnum`, and manifest range mappings; the CLI only handles argument parsing, input contracts, -progress-renderer selection, and result mapping. +progress-renderer selection, and result mapping. Interactive TTY +`ecc run` uses `run_flow_with_progress()`; `--plain` and GUI use +`execute()`. Both attach the same Engineering Snapshot commit observer +so each completed step updates `home/engineering-snapshot.json`. #### Extending environment probing (doctor / preflight) @@ -571,13 +575,16 @@ required by doctor. / `report_text.py` formatting), all exposed through the package `__init__`. Add a report section through an `_extract_(q)` in `report_sections.py` (or the timing chain) and register it from `report.py`. -- `engine/qor_report.py`: the single-workspace port of the GUI's - `projectQorTrend.ts` — constant tables - (`METRIC_FAIL_VALUES`/`DIMENSION_WEIGHTS`/`QOR_SCORE_THRESHOLD`) + - normalization + project-level record selection (role priority - final>gate>trend; area_cost only from the last successful area step) + the - `score_record` formulas + dimension weighting (no renormalization). Adding a - scoreable metric = adding its threshold here and in the GUI. +- `analysis/qor/`: the canonical QoR v3 Engine. It owns metric loading, + feature and dimension evaluation, feasibility gates, evidence, scoring, + diagnoses, interventions, the bounded report schema, and text rendering. + New QoR conclusions belong here; do not copy thresholds or formulas into + the GUI or CLI. +- `engine/qor_report.py`: the CLI `ecc report qor` facade. It delegates to + `analysis.qor` and does not own a second scoring implementation. +- `engine/qor_scoring.py` and `engine/qor.py`: legacy v2 Snapshot assessment + kept only while production Snapshot writes remain v2. Do not extend this + path for QoR v3 consumers. - `engine/signoff/report_checklist.py`: read-only rendering of `home/checklist.json` (reports unavailable on an invalid file; never writes back). @@ -677,9 +684,10 @@ uv run ecc run --project gcd --preset rtl2gds ### Reports -`ecc report qor` scores the workspace the same way the GUI project dashboard -does (per-metric scores against fixed fail thresholds, dimension averages, -weighted overall — weights are not renormalized over missing dimensions); +`ecc report qor` delegates to the canonical QoR v3 Engine in +`chipcompiler.analysis.qor`. The production v2 Snapshot `qorAssessment` +projection remains available for GUI compatibility during the ECC-only +rollout; it is not a second source for v3 conclusions. `ecc report checklist` renders the signoff checklist status, and `ecc report summary` writes the GUI-parity text design summary. All three write to `/signoff/` by default and accept `-o` plus the usual diff --git a/docs/specification/cli-design.md b/docs/specification/cli-design.md index d678d39c1..0e845c015 100644 --- a/docs/specification/cli-design.md +++ b/docs/specification/cli-design.md @@ -188,8 +188,8 @@ agent-specific disclosure fields inside core flow APIs. ### Core Commands The current root surface is a Typer command graph. The project-first command -surface stays small, with version reporting and the private runtime sidecar -available as explicit root entries: +surface stays small, with version reporting and layout rendering as explicit +root entries: ```bash ecc --version @@ -208,7 +208,6 @@ ecc project ecc workspace ecc signoff ecc report -ecc rpc ecc layout-image ``` @@ -232,7 +231,6 @@ Responsibilities: | `ecc workspace` | Refresh a declared workspace from current `ecc.toml` without running it | | `ecc signoff` | Inspect package readiness and export the tar.gz package | | `ecc report` | Write design-summary, QoR, and checklist reports; show step evidence | -| `ecc rpc` | Serve the private JSON-RPC runtime sidecar over stdio | | `ecc layout-image` | Render a GDS file into an image | `ecc run` preflights the tools its preset needs (yosys for synthesis, @@ -277,7 +275,7 @@ The command graph follows these rules; new commands must follow them too: `check`, `run`, `status`, `log`, `config`, `doctor`, `migrate`, `version`) plus the frozen tool invocations (`layout-image`). Resource management and reporting live in noun groups (`param`, `pdk`, `project`, `workspace`, - `signoff`, `report`, `rpc`). + `signoff`, `report`). - **Subcommand verbs.** Mutable resources use the CRUD set (`list`, `show`, `set`, `unset`, `diff`). The `report` group names its artifacts instead (`summary`, `qor`, `checklist`, `step`) @@ -767,9 +765,8 @@ NAME` for a workspace declared in the project manifest. It persists a workspace-local override and refreshes the affected configuration without running the flow. PDK resources and input references are project-level values: edit `ecc.toml` through `ecc project` (or the matching `ecc pdk`/`ecc param` -commands), then use `ecc workspace refresh NAME`. Old workspace create/run -automation should use the private JSON-RPC runtime sidecar. The long-term -default is project-oriented and configuration-driven through `ecc.toml` and +commands), then use `ecc workspace refresh NAME`. The supported integration +surface is project-oriented and configuration-driven through `ecc.toml` and subcommands such as `ecc run --project `. The project-level Python APIs should remain compatible with existing Python diff --git a/pyproject.toml b/pyproject.toml index 514f5ab92..7e7a65ad7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "ecc-dreamplace==0.1.0a7", "ecc-tools-bin==0.1.0a13", "fastapi>=0.109", - "jsonrpcserver>=5.0.9", + "jsonrpcserver>=5.0.9,<6", "klayout>=0.30.2", "matplotlib>=3.4", "numpy>=1.21", diff --git a/test/cli/commands/test_signoff.py b/test/cli/commands/test_signoff.py index 9df845104..07947cf93 100644 --- a/test/cli/commands/test_signoff.py +++ b/test/cli/commands/test_signoff.py @@ -63,7 +63,7 @@ def fake_load_workspace(path): def _patch_inspect(monkeypatch, review=REVIEW): monkeypatch.setattr( - "chipcompiler.runtime.signoff_export.inspect_signoff_package", + "chipcompiler.engine.signoff_export.inspect_signoff_package", lambda workspace: review, ) @@ -85,7 +85,7 @@ def fake_export(workspace, output_path, additional_files=None, *, include_debug= return destination monkeypatch.setattr( - "chipcompiler.runtime.signoff_export.export_signoff_package_archive", fake_export + "chipcompiler.engine.signoff_export.export_signoff_package_archive", fake_export ) return calls @@ -226,13 +226,13 @@ def test_export_forwards_include_debug( def test_export_incomplete_maps_to_error( self, tmp_path, capsys, monkeypatch, create_cli_project, workspace_stub, plain_records ): - from chipcompiler.runtime.workspace_api import RuntimeApiError + from chipcompiler.engine.signoff_export import SignoffExportError project_dir = create_cli_project() os.makedirs(os.path.join(project_dir, "default")) _patch_export( monkeypatch, - error=RuntimeApiError("command_failed", "signoff package is incomplete: x"), + error=SignoffExportError("signoff package is incomplete: x"), ) rc = cli_main.run( diff --git a/test/cli/commands/test_workspace_range.py b/test/cli/commands/test_workspace_range.py index 3cfce1a91..fe585e3bf 100644 --- a/test/cli/commands/test_workspace_range.py +++ b/test/cli/commands/test_workspace_range.py @@ -53,6 +53,7 @@ def test_new_workspace_range_uses_ecc_toml_inputs_and_registers_before_execution assert create_kwargs["directory"] == str(Path(project_dir) / "cts-only") assert create_kwargs["origin_def"] == str(design_def) assert create_kwargs["origin_verilog"] == str(netlist) + assert create_kwargs["parameters"]["_input_mode"] == "postSynthesis" assert create_kwargs["flow_config"] == {"start_step": "CTS", "end_step": "CTS"} manifest = json.loads((Path(project_dir) / "project.json").read_text()) entry = manifest["workspaces"][0] diff --git a/test/cli/params/test_flow_filters.py b/test/cli/params/test_flow_filters.py new file mode 100644 index 000000000..e9486dc1b --- /dev/null +++ b/test/cli/params/test_flow_filters.py @@ -0,0 +1,72 @@ +from chipcompiler.cli import main as cli_main + + +def test_param_list_synthesis_includes_global_parameters(capsys, create_cli_project, plain_records): + project_dir = create_cli_project() + rc = cli_main.run(["param", "list", "--step", "synthesis", "--project", project_dir, "--plain"]) + + assert rc == 0 + assert [record["param"] for record in plain_records(capsys.readouterr().out)] == [ + "design.frequency_mhz", + "flow.run_analysis", + ] + + +def test_param_set_rejects_parameter_outside_project_flow( + tmp_path, capsys, create_cli_project, plain_records +): + project_dir = create_cli_project() + config_path = tmp_path / "gcd" / "ecc.toml" + config_path.write_text( + config_path.read_text().replace('preset = "rtl2gds"', 'preset = "syn_sta"') + ) + + rc = cli_main.run( + [ + "param", + "set", + "place.target_density", + "0.65", + "--project", + project_dir, + "--plain", + ] + ) + + assert rc == 1 + assert plain_records(capsys.readouterr().out)[0]["error"] == "parameter_not_in_flow" + assert "target_density" not in config_path.read_text() + + +def test_param_list_excludes_steps_absent_from_selected_flow( + tmp_path, capsys, create_cli_project, plain_records +): + project_dir = create_cli_project() + config_path = tmp_path / "gcd" / "ecc.toml" + config_path.write_text( + config_path.read_text().replace('preset = "rtl2gds"', 'preset = "synthesis_lec"') + ) + + rc = cli_main.run(["param", "list", "--step", "floorplan", "--project", project_dir, "--plain"]) + + assert rc == 0 + assert plain_records(capsys.readouterr().out) == [] + + +def test_param_list_default_excludes_explicit_absent_step_parameters( + tmp_path, capsys, create_cli_project, plain_records +): + project_dir = create_cli_project() + config_path = tmp_path / "gcd" / "ecc.toml" + config_path.write_text( + config_path.read_text().replace('preset = "rtl2gds"', 'preset = "synthesis_lec"') + + "\n[params.place]\ntarget_density = 0.65\n" + ) + + rc = cli_main.run(["param", "list", "--project", project_dir, "--plain"]) + + assert rc == 0 + assert all( + record["param"] != "place.target_density" + for record in plain_records(capsys.readouterr().out) + ) diff --git a/test/cli/params/test_workspace_commands.py b/test/cli/params/test_workspace_commands.py index 684988f2d..cdfa0aa92 100644 --- a/test/cli/params/test_workspace_commands.py +++ b/test/cli/params/test_workspace_commands.py @@ -200,3 +200,34 @@ def test_workspace_param_list_honors_step_filter( assert plain_records(capsys.readouterr().out) == [ {"param": "list", "status": "clean", "workspace": "baseline"} ] + + +def test_workspace_param_list_matches_first_step_configuration( + capsys, create_cli_project, monkeypatch, plain_records +): + project_dir = create_cli_project() + workspace_dir = Path(project_dir) / "baseline" + _write_manifest(project_dir) + workspace = _workspace(workspace_dir) + monkeypatch.setattr("chipcompiler.data.load_workspace", lambda _path: workspace) + + rc = cli_main.run( + [ + "param", + "list", + "--workspace", + "baseline", + "--step", + "synthesis", + "--all", + "--project", + project_dir, + "--plain", + ] + ) + + assert rc == 0 + assert [record["param"] for record in plain_records(capsys.readouterr().out)] == [ + "design.frequency_mhz", + "flow.run_analysis", + ] diff --git a/test/cli/project/test_manifest.py b/test/cli/project/test_manifest.py index 4832d8ab6..40a823263 100644 --- a/test/cli/project/test_manifest.py +++ b/test/cli/project/test_manifest.py @@ -73,6 +73,27 @@ def test_load_manifest_normalizes_workspace_entries(tmp_path): assert manifest.active_workspaces() == [entry] +def test_load_manifest_maps_legacy_floor_range_to_post_floorplan(tmp_path): + _write_manifest( + tmp_path, + _minimal_document( + tmp_path, + workspaces=[ + { + "workspace_id": "legacy", + "workspace_path": str(tmp_path / "legacy"), + "start_step": "Floor", + "end_step": "Place", + } + ], + ), + ) + + entry = load_manifest(str(tmp_path)).workspaces[0] + + assert (entry.start_step, entry.end_step) == ("PostFloorplan", "Place") + + def test_load_manifest_relative_workspace_path_resolves_inside_root(tmp_path): _write_manifest( tmp_path, diff --git a/test/cli/rendering/test_progress.py b/test/cli/rendering/test_progress.py index e3aa90f5f..9384ff5e9 100644 --- a/test/cli/rendering/test_progress.py +++ b/test/cli/rendering/test_progress.py @@ -658,6 +658,10 @@ def check_state_fn(self, name, tool, state): class TestRunFlowWithProgress: + @pytest.fixture(autouse=True) + def _disable_snapshot_sink(self, monkeypatch): + monkeypatch.setattr(progress, "event_sink_for_workspace", lambda workspace: None) + def test_success_summary_format(self, tmp_path): flow = _make_flow( _make_ws(str(tmp_path)), @@ -1125,6 +1129,65 @@ def test_color_disabled_for_non_tty(self): for code in (BOLD, CYAN, GREEN, RED, DIM): assert code not in output + def test_progress_run_commits_snapshot_after_each_step(self, tmp_path, monkeypatch): + committed = [] + sink = SimpleNamespace( + on_step_completed=lambda step, state, error=None: committed.append( + (step.name, state, error) + ) + ) + monkeypatch.setattr(progress, "event_sink_for_workspace", lambda workspace: sink) + + def fake_run_step(self, step, *, rerun=False, observer=None): + assert rerun is False + observer.on_step_completed(step, StateEnum.Success) + return StateEnum.Success + + flow = _make_flow( + _make_ws(str(tmp_path)), + [ + _make_step("Synthesis", "yosys"), + _make_step("Floorplan", "ecc"), + ], + fake_run_step, + ) + + buf = FakeTTYStderr(isatty_value=True) + result = run_flow_with_progress(flow, _make_ctx(), None, buf) + + assert result is True + assert committed == [ + ("Synthesis", StateEnum.Success, None), + ("Floorplan", StateEnum.Success, None), + ] + + def test_progress_run_commits_snapshot_for_failed_step(self, monkeypatch): + committed = [] + sink = SimpleNamespace( + on_step_completed=lambda step, state, error=None: committed.append((step.name, state)) + ) + monkeypatch.setattr(progress, "event_sink_for_workspace", lambda workspace: sink) + + def fake_run_step(self, step, *, rerun=False, observer=None): + state = StateEnum.Success if step.name == "Synthesis" else StateEnum.Imcomplete + observer.on_step_completed(step, state) + return state + + flow = _make_flow( + _make_ws(), + [_make_step("Synthesis", "yosys"), _make_step("Floorplan", "ecc")], + fake_run_step, + ) + + buf = FakeTTYStderr(isatty_value=True) + result = run_flow_with_progress(flow, _make_ctx(), None, buf) + + assert result is False + assert committed == [ + ("Synthesis", StateEnum.Success), + ("Floorplan", StateEnum.Imcomplete), + ] + # --------------------------------------------------------------------------- # Failure context block formatting (AC-5) diff --git a/test/cli/test_help_rendering.py b/test/cli/test_help_rendering.py index c8fc586ea..ee7a03471 100644 --- a/test/cli/test_help_rendering.py +++ b/test/cli/test_help_rendering.py @@ -28,6 +28,8 @@ def test_help_renders_for_every_command(path, capsys): def test_help_keeps_styles_when_color_is_forced(monkeypatch, capsys): monkeypatch.setattr("typer.rich_utils.FORCE_TERMINAL", True) + monkeypatch.delenv("NO_COLOR", raising=False) + monkeypatch.setenv("TERM", "xterm-256color") rc = cli_main.run(["--help"]) diff --git a/test/data/test_step_storage_name.py b/test/data/test_step_storage_name.py new file mode 100644 index 000000000..1785b1ad7 --- /dev/null +++ b/test/data/test_step_storage_name.py @@ -0,0 +1,7 @@ +from chipcompiler.data import StepEnum, step_storage_name + + +def test_step_storage_name_sanitizes_sizer_timing_opt(): + assert step_storage_name(StepEnum.TIMING_OPT.value, "sizer") == "timing_optimization" + assert step_storage_name(StepEnum.TIMING_OPT.value, "Sizer") == "timing_optimization" + assert step_storage_name(StepEnum.FLOORPLAN.value, "ecc") == StepEnum.FLOORPLAN.value diff --git a/test/data/test_workspace.py b/test/data/test_workspace.py index 3635c121b..57e795f34 100644 --- a/test/data/test_workspace.py +++ b/test/data/test_workspace.py @@ -19,7 +19,9 @@ prepare_workspace_for_rerun, refresh_workspace_config, sync_workspace_config_to_parameters, + update_step_config, ) +from chipcompiler.data.workspace.layout import EccData, EccOutput, EccStep, StepInput from chipcompiler.utility import json_read, json_write EXPECTED_WORKSPACE_CONFIG_FILENAMES = { @@ -1172,6 +1174,38 @@ def test_refresh_workspace_config_reapplies_direct_config_overrides( assert json_read(workspace.config["dreamplace"])["num_threads"] == 12 +def test_update_step_config_preserves_floorplan_mode_override_after_result_backfill( + tmp_path, minimal_ics55_pdk_factory, default_ics55_parameters +): + workspace_dir, workspace = _create_loaded_ics55_workspace( + tmp_path, + "workspace_floorplan_mode_override", + minimal_ics55_pdk_factory, + default_ics55_parameters, + ) + parameters = _read_parameters(workspace_dir / "home" / "params.toml") + parameters["die"] = {"size": [31.8, 32.0], "area": 1017.6} + parameters["config_overrides"] = { + "Floorplan": {"die_builder": {"mode": "die_util"}}, + } + _write_parameters(workspace_dir / "home" / "params.toml", parameters) + workspace.parameters.data = parameters + + step = EccStep( + name=StepEnum.POST_FLOORPLAN.value, + input=StepInput(), + output=EccOutput(dir=workspace_dir / "postFloorplan_ecc" / "output"), + data=EccData( + steps={StepEnum.POST_FLOORPLAN.value: workspace_dir / "postFloorplan_ecc" / "data"} + ), + ) + + update_step_config(workspace, step) + + floorplan = json_read(workspace.config[StepEnum.FLOORPLAN.value]) + assert floorplan["die_builder"]["mode"] == "die_util" + + def test_sync_workspace_config_to_parameters_updates_routing_layers_and_refreshes_peers( tmp_path, minimal_ics55_pdk_factory, default_ics55_parameters ): diff --git a/test/data/test_workspace_config.py b/test/data/test_workspace_config.py index 94ec050f4..790f56633 100644 --- a/test/data/test_workspace_config.py +++ b/test/data/test_workspace_config.py @@ -144,7 +144,7 @@ def test_flow_validation_accepts_canonical_names(): def test_flow_validation_rejects_display_name_aliases(): # Workspace files carry canonical names only; aliases translate at the - # manifest/RPC boundary. + # manifest/adapter boundary. with pytest.raises(WorkspaceFlowTargetError): validate_flow_config({"start": "Synth", "end": "Filler"}) diff --git a/test/data/test_workspace_filelist.py b/test/data/test_workspace_filelist.py index 1fad257fe..749507b45 100644 --- a/test/data/test_workspace_filelist.py +++ b/test/data/test_workspace_filelist.py @@ -86,9 +86,49 @@ def test_workspace_with_nested_filelist(self, tmp_path, test_parameters, pdk): assert (origin_dir / "rtl" / "core" / "alu.v").exists() assert (origin_dir / "rtl" / "core" / "ctrl.v").exists() - def test_filelist_absolute_entries_rewritten_to_frozen_sources( - self, tmp_path, test_parameters, pdk - ): + def test_filelist_survives_workspace_reload(self, tmp_path, minimal_ics55_pdk_factory): + from chipcompiler.data import load_workspace + from chipcompiler.data.parameter import load_parameter, save_parameter + from chipcompiler.data.workspace_config import workspace_config_path + + project_dir = tmp_path / "project" + project_dir.mkdir() + _write_rtl_file(project_dir / "gcd.v", "gcd") + + filelist = project_dir / "design.f" + _create_filelist(filelist, "gcd.v") + + pdk_root = minimal_ics55_pdk_factory(tmp_path / "ics55") + workspace_dir = tmp_path / "workspace" + create_workspace( + directory=str(workspace_dir), + origin_def="", + origin_verilog="", + pdk="ics55", + parameters={ + "design": "gcd", + "top_module": "gcd", + "clock": "clk", + "frequency_max": 100, + }, + input_filelist=str(filelist), + pdk_root=pdk_root, + ) + + frozen = workspace_dir / "origin" / "design.f" + assert frozen.is_file() + + parameters = load_parameter(workspace_config_path(workspace_dir)) + assert parameters.data["file_list"] == "origin/design.f" + + reloaded = load_workspace(workspace_dir) + assert reloaded.design.input_filelist == frozen + + parameters.data["file_list"] = str(frozen) + assert save_parameter(parameters) + assert load_workspace(workspace_dir).design.input_filelist == frozen + + def test_filelist_absolute_entries_rewritten_to_frozen_sources(self, tmp_path): from chipcompiler.data.workspace import copy_filelist_with_sources project_dir = tmp_path / "project" @@ -111,9 +151,7 @@ def test_filelist_absolute_entries_rewritten_to_frozen_sources( assert (workspace_dir / "origin" / "rtl" / "b.v").exists() assert installed == str(workspace_dir / "origin" / "design.f") - def test_filelist_rewrite_handles_quoted_and_commented_entries( - self, tmp_path, test_parameters, pdk - ): + def test_filelist_rewrite_handles_quoted_and_commented_entries(self, tmp_path): from chipcompiler.data.workspace import copy_filelist_with_sources project_dir = tmp_path / "project" @@ -129,9 +167,7 @@ def test_filelist_rewrite_handles_quoted_and_commented_entries( assert lines == ['"a.v" # top'] assert (workspace_dir / "origin" / "a.v").exists() - def test_filelist_absolute_duplicate_basenames_are_disambiguated( - self, tmp_path, test_parameters, pdk - ): + def test_filelist_absolute_duplicate_basenames_are_disambiguated(self, tmp_path): from chipcompiler.data.workspace import copy_filelist_with_sources dir_a = tmp_path / "a" @@ -152,7 +188,7 @@ def test_filelist_absolute_duplicate_basenames_are_disambiguated( assert (workspace_dir / "origin" / "foo.v").read_text() == "module foo_a; endmodule\n" assert (workspace_dir / "origin" / "b_foo.v").read_text() == "module foo_b; endmodule\n" - def test_filelist_absolute_incdir_is_frozen_inside_origin(self, tmp_path, test_parameters, pdk): + def test_filelist_absolute_incdir_is_frozen_inside_origin(self, tmp_path): from chipcompiler.data.workspace.filelist_copy import copy_filelist_with_sources include_dir = tmp_path / "proj" / "include" @@ -169,7 +205,7 @@ def test_filelist_absolute_incdir_is_frozen_inside_origin(self, tmp_path, test_p assert lines[0] == "+incdir+include" assert (workspace_dir / "origin" / "include" / "defs.svh").exists() - def test_load_workspace_rejects_symlinked_params_toml(self, tmp_path, test_parameters, pdk): + def test_load_workspace_rejects_symlinked_params_toml(self, tmp_path): from chipcompiler.data.workspace import load_workspace from chipcompiler.data.workspace_config import WorkspaceConfigError, save_workspace_config @@ -185,9 +221,7 @@ def test_load_workspace_rejects_symlinked_params_toml(self, tmp_path, test_param with pytest.raises(WorkspaceConfigError): load_workspace(str(workspace_dir)) - def test_filelist_absolute_incdirs_with_same_basename_are_disambiguated( - self, tmp_path, test_parameters, pdk - ): + def test_filelist_absolute_incdirs_with_same_basename_are_disambiguated(self, tmp_path): from chipcompiler.data.workspace.filelist_copy import copy_filelist_with_sources dir_a = tmp_path / "a" / "include" diff --git a/test/engine/test_execution.py b/test/engine/test_execution.py new file mode 100644 index 000000000..ad8d705ef --- /dev/null +++ b/test/engine/test_execution.py @@ -0,0 +1,151 @@ +from types import SimpleNamespace + +import pytest + +from chipcompiler.data import StateEnum +from chipcompiler.engine.execution import ExecutionPlan, event_sink_for_workspace, execute + + +def test_execution_plan_dispatches_full_flow_and_single_step(): + calls = [] + + class Flow: + def run_steps(self, *, rerun=False, observer=None): + calls.append(("flow", rerun, observer)) + return True + + def get_workspace_step(self, step_id): + return SimpleNamespace(name=step_id) + + def run_step(self, step, *, rerun=False, observer=None): + calls.append((step.name, rerun, observer)) + return StateEnum.Success + + observer = object() + flow = Flow() + + assert execute(flow, ExecutionPlan(intent="run"), event_sink=observer).succeeded + step_result = execute( + flow, + ExecutionPlan(intent="rerun", step_id="Floorplan"), + event_sink=observer, + ) + + assert step_result == step_result.__class__( + succeeded=True, + state=StateEnum.Success.value, + step_id="Floorplan", + ) + assert [call[:2] for call in calls] == [("flow", False), ("Floorplan", True)] + assert all(call[2].delegate is observer for call in calls) + + +def test_execution_observer_does_not_hide_unknown_callbacks(): + from chipcompiler.engine.execution import ExecutionObserver + + with pytest.raises(AttributeError): + _ = ExecutionObserver(object()).unknown_callback + + +def test_execution_plan_checks_cancel_before_each_selected_step(): + calls = [] + + class Cancelled(RuntimeError): + pass + + class Observer: + fatal_observer = True + + def __init__(self): + self.completed = 0 + + def on_step_completed(self, _step, _state, _error=None): + self.completed += 1 + + def raise_if_cancelled(self): + if self.completed: + raise Cancelled + + class Flow: + def get_workspace_step(self, step_id): + return SimpleNamespace(name=step_id) + + def run_step(self, step, *, rerun=False, observer=None): + calls.append(step.name) + observer.on_step_completed(step, StateEnum.Success) + return StateEnum.Success + + with pytest.raises(Cancelled): + execute( + Flow(), + ExecutionPlan(intent="run", step_ids=("Synthesis", "Floorplan")), + event_sink=Observer(), + ) + + assert calls == ["Synthesis"] + + +def test_execution_failure_keeps_main_blocking_semantics(): + class Flow: + workspace = SimpleNamespace(flow=SimpleNamespace(data={"steps": []})) + + def run_steps(self, *, rerun=False, observer=None): + return False + + result = execute(Flow(), ExecutionPlan(intent="run"), event_sink=object()) + + assert not result.succeeded + assert result.state == StateEnum.Imcomplete.value + + +def test_default_execution_observer_commits_completed_steps(monkeypatch, tmp_path): + from chipcompiler.engine import execution + + committed = [] + + class Flow: + workspace = SimpleNamespace(directory=tmp_path) + + def run_steps(self, *, rerun=False, observer=None): + observer.on_step_completed(SimpleNamespace(name="synthesis"), StateEnum.Success) + return True + + monkeypatch.setattr( + execution, + "_EngineeringCommitSink", + lambda workspace: SimpleNamespace( + snapshot={"workspaceId": "workspace-1", "workspaceRevision": 1}, + on_step_completed=lambda step, state, error=None: committed.append((step.name, state)), + ), + ) + + result = execution.execute(Flow(), execution.ExecutionPlan(intent="run")) + + assert result.succeeded + assert committed == [("synthesis", StateEnum.Success)] + + +def test_execution_reports_ordered_steps_and_completed_default_is_noop(): + calls = [] + + class Flow: + workspace = SimpleNamespace( + flow=SimpleNamespace(data={"steps": [{"name": "synthesis", "state": "Success"}]}) + ) + + def run_steps(self, **_kwargs): + calls.append("run") + return True + + result = execute(Flow(), ExecutionPlan(intent="run")) + + assert result.succeeded + assert result.no_op + assert result.executed_steps == () + assert result.failed_step is None + assert calls == [] + + +def test_event_sink_for_workspace_is_none_without_directory(): + assert event_sink_for_workspace(SimpleNamespace()) is None + assert event_sink_for_workspace(SimpleNamespace(directory=None)) is None diff --git a/test/engine/test_qor_scoring.py b/test/engine/test_qor_scoring.py new file mode 100644 index 000000000..2f01abc23 --- /dev/null +++ b/test/engine/test_qor_scoring.py @@ -0,0 +1,94 @@ +from chipcompiler.engine.qor_scoring import QorScoringMetric, score_metric, score_qor + + +def _metric(step, metric_id, value, dimension, direction="lower_is_better", **kwargs): + return QorScoringMetric( + step=step, + metric_id=metric_id, + value=value, + dimension=dimension, + direction=direction, + scope="workspace", + corner=None, + project_role=kwargs.get("project_role", "final"), + rating_score=kwargs.get("rating_score", True), + ) + + +def test_qor_scoring_selects_latest_area_and_combines_dimension_weights(): + result = score_qor( + [ + _metric("Floor", "die_area", 300, "area_cost"), + _metric("STA", "sta_setup_wns", -0.1, "timing", "higher_is_better"), + _metric("Harden", "die_area", 1500, "area_cost"), + _metric("DRC", "drc_count", 0, "routability_physical"), + ] + ) + + assert result.area_scoring_step == "Harden" + assert result.dimensions == { + "timing": (50.0, 1), + "routability_physical": (100.0, 1), + "area_cost": (50.0, 1), + } + assert result.overall_score == 42.5 + + +def test_qor_scoring_ignores_forward_version_dimensions_and_steps(): + result = score_qor( + [ + _metric("future-step", "future_metric", 1, "future_dimension"), + _metric("Harden", "die_area", 1500, "area_cost"), + ] + ) + + assert result.dimensions == {"area_cost": (50.0, 1)} + + +def test_qor_scoring_uses_flow_order_for_area_step_when_provided(): + result = score_qor( + [ + _metric("Harden", "die_area", 1500, "area_cost"), + _metric("DRC", "die_area", 300, "area_cost"), + ], + flow_order=("STA", "DRC", "Harden"), + ) + + assert result.area_scoring_step == "Harden" + assert result.dimensions == {"area_cost": (50.0, 1)} + + +def test_analysis_maps_legacy_power_category_to_power_integrity(tmp_path): + from chipcompiler.engine.analysis import _analysis_file + from chipcompiler.utility import json_write + + path = tmp_path / "qor_metrics.json" + json_write( + path, + { + "schema_version": 3, + "metrics": [ + { + "id": "synthesis_power_dynamic_uw", + "display_name": "Synthesis Dynamic Power", + "value": 18.5, + "category": "power", + "direction": "trend_only", + "scope": "synthesis", + "rating": {"gate": False, "score": False, "trend": True}, + } + ], + }, + ) + + payload = _analysis_file(path, "artifact-metrics", 3, tmp_path) + assert payload["status"] == "available" + assert payload["data"]["metrics"][0]["category"] == "power_integrity" + + +def test_score_metric_matches_fail_threshold_formulas(): + slack = _metric("STA", "sta_setup_wns", -0.1, "timing", "higher_is_better") + assert score_metric(slack) == 50.0 + assert score_metric(_metric("DRC", "drc_count", 0, "routability_physical")) == 100.0 + utilization = _metric("Harden", "core_utilization", 0.55, "area_cost", "target_range") + assert score_metric(utilization) == 100.0 diff --git a/test/engine/test_signoff_assessment.py b/test/engine/test_signoff_assessment.py new file mode 100644 index 000000000..d237b3a80 --- /dev/null +++ b/test/engine/test_signoff_assessment.py @@ -0,0 +1,52 @@ +import json +from pathlib import Path +from types import SimpleNamespace + +from chipcompiler.engine.signoff_assessment import build_signoff_assessment +from chipcompiler.engine.signoff_export import SignoffExportError, _additional_file_path + + +def test_stale_checklist_cannot_make_incomplete_flow_ready(tmp_path): + home = tmp_path / "home" + home.mkdir() + (home / "checklist.json").write_text( + json.dumps( + { + "schema_version": 3, + "kind": "signoff_checklist", + "status": "ready", + "checklist": [], + } + ) + ) + workspace = SimpleNamespace( + directory=Path(tmp_path), + flow=SimpleNamespace(steps=lambda: [{"name": "Synthesis", "state": "Unstart"}]), + ) + + result = build_signoff_assessment(workspace) + + assert result["status"] == "blocked" + + +def test_malformed_checklist_is_reported_as_unavailable(tmp_path): + home = tmp_path / "home" + home.mkdir() + (home / "checklist.json").write_text("[]") + workspace = SimpleNamespace(directory=Path(tmp_path), flow=None) + + result = build_signoff_assessment(workspace) + + assert result["status"] == "blocked" + + +def test_signoff_additional_file_path_rejects_escape(tmp_path): + package = tmp_path / "package" + package.mkdir() + + for value in ("/tmp/escape", "../escape", "", ".", "bad\x00path"): + try: + _additional_file_path(package, value) + except SignoffExportError: + continue + raise AssertionError(f"unsafe path accepted: {value!r}") diff --git a/test/engine/test_sizer_snapshot_directory.py b/test/engine/test_sizer_snapshot_directory.py new file mode 100644 index 000000000..e52c120a2 --- /dev/null +++ b/test/engine/test_sizer_snapshot_directory.py @@ -0,0 +1,89 @@ +import json +from types import SimpleNamespace + +from chipcompiler.engine.snapshot import create_engineering_snapshot + + +def test_snapshot_resolves_canonical_sizer_step_directory(tmp_path): + step_dir = tmp_path / "timing_optimization_sizer" + analysis = step_dir / "analysis" + output = step_dir / "output" + report = step_dir / "report" + analysis.mkdir(parents=True) + output.mkdir() + report.mkdir() + metric = { + "id": "instance_count", + "display_name": "Instance Count", + "value": 298, + "unit": "count", + "category": "area_cost", + "direction": "trend_only", + "scope": "timing_optimization", + "corner": None, + "analysis_group": "timing optimization_metrics", + "rating": {"gate": False, "score": False, "trend": True}, + "project_role": "trend", + "step_role": "primary", + "confidence": "high", + "source": {}, + } + (analysis / "qor_metrics.json").write_text( + json.dumps({"schema_version": 3, "metrics": [metric]}), encoding="utf-8" + ) + (analysis / "qor_summary.json").write_text( + json.dumps( + { + "schema_version": 4, + "analysis_status": "valid", + "quality_status": "pass", + "gates": [], + "missing_metrics": [], + } + ), + encoding="utf-8", + ) + (analysis / "qor_hotspots.json").write_text( + json.dumps({"schema_version": 3, "hotspots": []}), encoding="utf-8" + ) + (output / "gcd_Timing optimization.png").write_bytes(b"layout") + (report / "Timing optimization.db.rpt").write_text("report", encoding="utf-8") + (step_dir / "subflow.json").write_text( + json.dumps({"steps": [{"name": "run sizer", "state": "Success"}]}), + encoding="utf-8", + ) + workspace = SimpleNamespace( + directory=tmp_path, + design=SimpleNamespace(name="gcd"), + flow=SimpleNamespace( + data={ + "steps": [ + { + "name": "Timing optimization", + "tool": "sizer", + "state": "Success", + } + ] + } + ), + parameters=SimpleNamespace(data={}), + home=SimpleNamespace(data={}), + ) + + snapshot = create_engineering_snapshot(workspace, workspace_id="engineering-a") + + step = snapshot["analysis"]["steps"][0] + assert step["stepId"] == "Timing optimization" + assert step["metrics"]["data"]["metrics"] == [metric] + assert step["subflow"]["status"] == "available" + artifacts = {artifact["kind"]: artifact for artifact in snapshot["artifacts"]} + assert artifacts["qor_metrics"]["reference"] == ( + "timing_optimization_sizer/analysis/qor_metrics.json" + ) + assert artifacts["qor_metrics"]["availability"] == "available" + assert artifacts["layout_image"]["availability"] == "available" + assert artifacts["layout_image"]["reference"] == ( + "timing_optimization_sizer/output/gcd_Timing optimization.png" + ) + reports = [artifact for artifact in snapshot["artifacts"] if artifact["kind"] == "report_text"] + assert reports[0]["availability"] == "available" diff --git a/test/engine/test_snapshot_migration.py b/test/engine/test_snapshot_migration.py new file mode 100644 index 000000000..b0a23fc76 --- /dev/null +++ b/test/engine/test_snapshot_migration.py @@ -0,0 +1,174 @@ +import hashlib +import json +from copy import deepcopy +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from chipcompiler.engine.snapshot import ( + SNAPSHOT_V3_SCHEMA_VERSION, + EngineeringSnapshotError, + create_engineering_snapshot, + ensure_engineering_snapshot, + migrate_engineering_snapshot, + read_engineering_snapshot, +) +from chipcompiler.engine.snapshot_qor import ( + unavailable_qor_snapshot_extension, + validate_qor_snapshot_extension, +) + + +def _workspace(tmp_path): + root = tmp_path / "workspace" + (root / "home").mkdir(parents=True) + steps = [{"name": "Synthesis", "tool": "yosys", "state": "Success"}] + (root / "home" / "flow.json").write_text(json.dumps({"steps": steps}), encoding="utf-8") + return SimpleNamespace( + directory=root, + flow=SimpleNamespace(data={"steps": steps}), + home=SimpleNamespace(data={}), + parameters=SimpleNamespace(data={"design": "gcd"}), + design=SimpleNamespace(name="gcd"), + ) + + +def test_migration_preserves_identity_advances_revision_and_projects_qor(tmp_path): + workspace = _workspace(tmp_path) + current = create_engineering_snapshot(workspace, workspace_id="engineering-gcd") + assert current["schemaVersion"] == 2 + assert str(workspace.directory) not in json.dumps(current["qorSnapshotExtension"]) + + migrated = migrate_engineering_snapshot( + workspace, + expected_workspace_revision=current["workspaceRevision"], + ) + + assert migrated["schemaVersion"] == SNAPSHOT_V3_SCHEMA_VERSION + assert migrated["workspaceId"] == "engineering-gcd" + assert migrated["workspaceRevision"] == current["workspaceRevision"] + 1 + assert migrated["cause"] == "snapshot.migrated.v2_to_v3" + assert migrated["qorSnapshotExtension"]["scoringEngine"] == "qor-v3" + assert read_engineering_snapshot(workspace)["schemaVersion"] == SNAPSHOT_V3_SCHEMA_VERSION + + +def test_migration_preserves_stale_predecessor_metadata(tmp_path): + workspace = _workspace(tmp_path) + current = create_engineering_snapshot(workspace, workspace_id="engineering-gcd") + path = Path(workspace.directory) / "home" / "engineering-snapshot.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["stalePredecessor"] = {"workspaceRevision": 1, "invalidatedStepIds": ["route"]} + path.write_text(json.dumps(payload), encoding="utf-8") + + migrated = migrate_engineering_snapshot(workspace) + + assert migrated["workspaceRevision"] == current["workspaceRevision"] + 1 + assert migrated["stalePredecessor"] == payload["stalePredecessor"] + + +def test_migration_failure_keeps_previous_snapshot(tmp_path, monkeypatch): + workspace = _workspace(tmp_path) + create_engineering_snapshot(workspace, workspace_id="engineering-gcd") + path = Path(workspace.directory) / "home" / "engineering-snapshot.json" + before = path.read_bytes() + + def fail(_workspace): + raise RuntimeError("broken analysis") + + monkeypatch.setattr("chipcompiler.analysis.qor.build_qor_analysis", fail) + with pytest.raises(EngineeringSnapshotError, match="regenerate QoR facts"): + migrate_engineering_snapshot(workspace) + + assert path.read_bytes() == before + + +def test_unsupported_schema_is_not_migrated(tmp_path): + workspace = _workspace(tmp_path) + path = Path(workspace.directory) / "home" / "engineering-snapshot.json" + path.write_text(json.dumps({"schemaVersion": 99}), encoding="utf-8") + + with pytest.raises(EngineeringSnapshotError, match="invalid Engineering Snapshot"): + migrate_engineering_snapshot(workspace) + + +def test_production_write_paths_reject_migrated_v3_snapshot(tmp_path): + workspace = _workspace(tmp_path) + create_engineering_snapshot(workspace, workspace_id="engineering-gcd") + migrate_engineering_snapshot(workspace) + + with pytest.raises(EngineeringSnapshotError, match="production Snapshot schema is still v2"): + ensure_engineering_snapshot(workspace) + + +def test_qor_extension_validator_rejects_missing_and_invalid_nested_fields(): + extension = unavailable_qor_snapshot_extension("analysis unavailable") + assert validate_qor_snapshot_extension(extension) + + missing_score = deepcopy(extension) + missing_score.pop("score") + assert not validate_qor_snapshot_extension(missing_score) + + invalid_power = deepcopy(extension) + invalid_power["power"]["sourceKind"] = "raw_file" + assert not validate_qor_snapshot_extension(invalid_power) + + invalid_gate = deepcopy(extension) + invalid_gate["feasibility"]["gates"].append({"id": "broken"}) + assert not validate_qor_snapshot_extension(invalid_gate) + + +def test_snapshot_read_validates_expected_identity_and_revision(tmp_path): + workspace = _workspace(tmp_path) + create_engineering_snapshot(workspace, workspace_id="engineering-gcd") + + with pytest.raises(EngineeringSnapshotError, match="identity mismatch"): + read_engineering_snapshot(workspace, expected_workspace_id="other") + with pytest.raises(EngineeringSnapshotError, match="Revision mismatch"): + read_engineering_snapshot(workspace, expected_workspace_revision=2) + + (workspace.directory / "home" / "workspace-commands.json").write_text( + json.dumps( + { + "schemaVersion": 1, + "commands": { + "create-1": {"result": {"workspaceId": "other"}}, + }, + } + ), + encoding="utf-8", + ) + with pytest.raises(EngineeringSnapshotError, match="identity mismatch"): + read_engineering_snapshot(workspace) + + +def test_snapshot_read_rejects_invalid_sections_and_artifact_fingerprints(tmp_path): + workspace = _workspace(tmp_path) + create_engineering_snapshot(workspace, workspace_id="engineering-gcd") + path = Path(workspace.directory) / "home" / "engineering-snapshot.json" + payload = json.loads(path.read_text(encoding="utf-8")) + + invalid_section = deepcopy(payload) + invalid_section["flow"] = [] + path.write_text(json.dumps(invalid_section), encoding="utf-8") + with pytest.raises(EngineeringSnapshotError, match="section: flow"): + read_engineering_snapshot(workspace) + + path.write_text(json.dumps(payload), encoding="utf-8") + artifact = payload["artifacts"][0] + artifact_path = workspace.directory / artifact["reference"] + artifact_path.parent.mkdir(parents=True, exist_ok=True) + artifact_path.write_bytes(b"before") + artifact.update( + { + "availability": "available", + "sizeBytes": 6, + "sha256": hashlib.sha256(b"before").hexdigest(), + } + ) + path.write_text(json.dumps(payload), encoding="utf-8") + assert read_engineering_snapshot(workspace)["artifacts"][0]["availability"] == "available" + + artifact_path.write_bytes(b"after") + with pytest.raises(EngineeringSnapshotError, match="fingerprint mismatch"): + read_engineering_snapshot(workspace) diff --git a/test/engine/test_workspace_configuration.py b/test/engine/test_workspace_configuration.py new file mode 100644 index 000000000..96b9f9ddb --- /dev/null +++ b/test/engine/test_workspace_configuration.py @@ -0,0 +1,266 @@ +import json +import os +from copy import deepcopy +from pathlib import Path + +import pytest + +from chipcompiler.data import load_workspace +from chipcompiler.engine import ( + EngineFlow, + WorkspaceLifecycleError, + create_workspace_from_spec, + read_step_configuration, + read_workspace_configuration_from_directory, + update_workspace_configuration, + update_workspace_step_configuration, +) +from chipcompiler.engine.snapshot import create_engineering_snapshot, read_engineering_snapshot + + +def _workspace_spec_fixture() -> tuple[dict, dict]: + root = Path(__file__).parents[1] / "fixtures" / "workspace_spec" + payload = json.loads((root / "valid.json").read_text(encoding="utf-8")) + bindings = deepcopy(payload["workspaceBindings"]) + bindings["inputs"] = {key: str(root / value) for key, value in bindings["inputs"].items()} + return payload["workspaceSpec"], bindings + + +def test_workspace_configuration_update_is_revision_aware_and_idempotent( + tmp_path, minimal_ics55_pdk_factory +): + spec, bindings = _workspace_spec_fixture() + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings, "create-1") + initial = read_engineering_snapshot(workspace) + + updated = update_workspace_configuration( + workspace.directory, + initial["workspaceRevision"], + {"parameters": {"design.frequency_mhz": 250.0}}, + bindings, + "configuration-1", + ) + current = read_engineering_snapshot(updated) + repeated = update_workspace_configuration( + workspace.directory, + initial["workspaceRevision"], + {"parameters": {"design.frequency_mhz": 250.0}}, + bindings, + "configuration-1", + ) + + assert initial["workspaceRevision"] == 1 + assert current["workspaceRevision"] == 2 + assert read_engineering_snapshot(repeated)["workspaceRevision"] == 2 + assert repeated.parameters.data["frequency_max"] == 250.0 + assert ( + read_workspace_configuration_from_directory(workspace.directory)["workspaceSpec"][ + "parameters" + ]["design.frequency_mhz"] + == 250.0 + ) + + with pytest.raises(WorkspaceLifecycleError) as conflict: + update_workspace_configuration( + workspace.directory, + 1, + {"parameters": {"design.frequency_mhz": 300.0}}, + bindings, + "configuration-2", + ) + assert conflict.value.code == "revision_conflict" + assert read_engineering_snapshot(updated)["workspaceRevision"] == 2 + + +def test_workspace_configuration_update_rolls_back_refresh_failure( + tmp_path, minimal_ics55_pdk_factory, monkeypatch +): + spec, bindings = _workspace_spec_fixture() + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings, "create-1") + before_config = (workspace.directory / "home" / "params.toml").read_bytes() + before_snapshot = read_engineering_snapshot(workspace) + + def fail_refresh(_workspace): + raise RuntimeError("config regeneration failed") + + monkeypatch.setattr("chipcompiler.data.refresh_workspace_config", fail_refresh) + with pytest.raises(RuntimeError, match="config regeneration failed"): + update_workspace_configuration( + workspace.directory, + before_snapshot["workspaceRevision"], + {"parameters": {"design.frequency_mhz": 250.0}}, + bindings, + "configuration-1", + ) + + assert (workspace.directory / "home" / "params.toml").read_bytes() == before_config + assert read_engineering_snapshot(workspace) == before_snapshot + + +def test_configuration_update_recovers_after_process_exit_before_snapshot( + tmp_path, minimal_ics55_pdk_factory +): + spec, bindings = _workspace_spec_fixture() + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings, "create-1") + before = read_engineering_snapshot(workspace) + + child = os.fork() + if child == 0: + import chipcompiler.engine.workspace_configuration as configuration_module + + configuration_module.invalidate_engineering_snapshot = lambda *_args, **_kwargs: os._exit( + 23 + ) + update_workspace_configuration( + workspace.directory, + before["workspaceRevision"], + {"parameters": {"design.frequency_mhz": 250.0}}, + bindings, + "configuration-crash", + ) + os._exit(99) + + _pid, status = os.waitpid(child, 0) + assert os.waitstatus_to_exitcode(status) == 23 + + updated = update_workspace_configuration( + workspace.directory, + before["workspaceRevision"], + {"parameters": {"design.frequency_mhz": 250.0}}, + bindings, + "configuration-crash", + ) + + assert updated.parameters.data["frequency_max"] == 250.0 + assert read_engineering_snapshot(updated)["workspaceRevision"] == ( + before["workspaceRevision"] + 1 + ) + + +def test_configuration_update_recovers_after_process_exit_during_commit_cleanup( + tmp_path, minimal_ics55_pdk_factory +): + spec, bindings = _workspace_spec_fixture() + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings, "create-1") + before = read_engineering_snapshot(workspace) + + child = os.fork() + if child == 0: + import chipcompiler.data.workspace_transaction as transaction_module + + original_rmtree = transaction_module.shutil.rmtree + + def crash_cleanup(path, *args, **kwargs): + if Path(path).name == ".workspace-configuration-backup.discard": + os._exit(24) + return original_rmtree(path, *args, **kwargs) + + transaction_module.shutil.rmtree = crash_cleanup + update_workspace_configuration( + workspace.directory, + before["workspaceRevision"], + {"parameters": {"design.frequency_mhz": 250.0}}, + bindings, + "configuration-cleanup-crash", + ) + os._exit(99) + + _pid, status = os.waitpid(child, 0) + assert os.waitstatus_to_exitcode(status) == 24 + + recovered = load_workspace(workspace.directory) + assert recovered.parameters.data["frequency_max"] == 250.0 + assert read_engineering_snapshot(recovered)["workspaceRevision"] == ( + before["workspaceRevision"] + 1 + ) + + +def test_read_workspace_configuration_does_not_materialize_missing_home_files( + tmp_path, minimal_ics55_pdk_factory +): + import shutil + + spec, bindings = _workspace_spec_fixture() + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings) + home_file = workspace.directory / "home" / "home.json" + checklist_file = workspace.directory / "home" / "checklist.json" + log_dir = workspace.directory / "log" + home_file.unlink() + checklist_file.unlink() + if log_dir.exists(): + shutil.rmtree(log_dir) + shutil.rmtree(Path(bindings["pdk"]["root"])) + + configuration = read_workspace_configuration_from_directory(workspace.directory) + readonly_workspace = load_workspace(workspace.directory, read_only=True) + from chipcompiler.tools.ecc.signoff_checklist import rebuild_home_checklist + + rebuild_home_checklist(readonly_workspace, persist=False) + + assert configuration["workspaceSpec"]["pdk"]["familyId"] == "ics55" + assert not home_file.exists() + assert not checklist_file.exists() + assert not log_dir.exists() + + +def test_step_configuration_update_invalidates_only_target_suffix( + tmp_path, minimal_ics55_pdk_factory +): + spec, bindings = _workspace_spec_fixture() + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings, "create-1") + flow = EngineFlow(workspace) + for step in flow.workspace.flow.data["steps"][:3]: + step["state"] = "Success" + assert flow.save() + initial = read_engineering_snapshot(workspace) + create_engineering_snapshot( + workspace, + workspace_id=initial["workspaceId"], + workspace_revision=initial["workspaceRevision"], + ) + + configuration = read_step_configuration(workspace, "floor-plan") + synthesis = read_step_configuration(workspace, "synthesis") + updated = update_workspace_step_configuration( + workspace.directory, + configuration["workspaceRevision"], + "Floorplan", + {"floorplan.core_util": 0.55}, + "step-configuration-1", + ) + + records = {record["param"]: record for record in configuration["parameters"]} + synthesis_params = {record["param"] for record in synthesis["parameters"]} + assert {"design.frequency_mhz", "flow.run_analysis"} <= synthesis_params + assert "floorplan.core_util" in records + assert set(records["floorplan.core_util"]) == { + "param", + "type", + "value", + "default", + "applies", + "description", + "range", + } + states = {step["name"]: step["state"] for step in updated.flow.steps()} + assert states["Synthesis"] == "Success" + assert states["preFloorplan"] == "Unstart" + assert states["macroPlacement"] == "Unstart" + assert read_engineering_snapshot(updated)["workspaceRevision"] == 2 + + with pytest.raises(WorkspaceLifecycleError) as inapplicable: + update_workspace_step_configuration( + workspace.directory, + 2, + "Floorplan", + {"place.target_density": 0.6}, + "step-configuration-2", + ) + assert inapplicable.value.code == "parameter_not_applicable" + assert read_engineering_snapshot(updated)["workspaceRevision"] == 2 diff --git a/test/engine/test_workspace_flow.py b/test/engine/test_workspace_flow.py new file mode 100644 index 000000000..06d54bf7f --- /dev/null +++ b/test/engine/test_workspace_flow.py @@ -0,0 +1,14 @@ +from chipcompiler.engine.workspace_flow import build_flow_for_workspace + + +def test_build_flow_requires_committed_flow(tmp_path): + from chipcompiler.data import Workspace + + workspace = Workspace(directory=tmp_path) + + try: + build_flow_for_workspace(workspace, create_step_workspaces=False) + except ValueError as error: + assert str(error) == "Workspace has no committed Flow" + else: + raise AssertionError("missing committed Flow was accepted") diff --git a/test/engine/test_workspace_spec.py b/test/engine/test_workspace_spec.py new file mode 100644 index 000000000..d3cffaa11 --- /dev/null +++ b/test/engine/test_workspace_spec.py @@ -0,0 +1,377 @@ +import json +from copy import deepcopy +from pathlib import Path + +import pytest + +from chipcompiler.engine.workspace_spec import ( + describe_workspace_spec, + validate_workspace_spec, +) + + +def _valid_spec(): + return { + "schemaVersion": 1, + "design": {"name": "gcd", "topModule": "gcd", "clockPort": "clk"}, + "inputMode": "rtl", + "inputs": [{"inputId": "rtl-main", "role": "rtl"}], + "pdk": {"familyId": "ics55", "mode": "default"}, + "flow": {"flowId": "rtl2gds"}, + "parameters": {"design.frequency_mhz": 200.0}, + } + + +def _shared_fixture(name: str): + root = Path(__file__).parents[1] / "fixtures" / "workspace_spec" + payload = json.loads((root / name).read_text(encoding="utf-8")) + bindings = deepcopy(payload["workspaceBindings"]) + bindings["inputs"] = {key: str(root / value) for key, value in bindings["inputs"].items()} + bindings["pdk"]["root"] = str(root / bindings["pdk"]["root"]) + return payload, bindings + + +def test_shared_workspace_spec_fixtures_cover_valid_and_invalid_contracts(): + valid, valid_bindings = _shared_fixture("valid.json") + assert validate_workspace_spec(valid["workspaceSpec"], valid_bindings)["issues"] == [] + + invalid, invalid_bindings = _shared_fixture("invalid.json") + result = validate_workspace_spec(invalid["workspaceSpec"], invalid_bindings) + assert {issue["code"] for issue in result["issues"]} >= set(invalid["expectedIssueCodes"]) + + +def test_workspace_spec_discovery_and_validation_are_canonical_and_side_effect_free(tmp_path): + rtl = tmp_path / "gcd.v" + rtl.write_text("module gcd(input clk); endmodule\n") + pdk = tmp_path / "pdk" + pdk.mkdir() + before = sorted(str(path.relative_to(tmp_path)) for path in tmp_path.rglob("*")) + + discovery = describe_workspace_spec() + result = validate_workspace_spec( + _valid_spec(), + {"inputs": {"rtl-main": str(rtl)}, "pdk": {"root": str(pdk)}}, + ) + + assert discovery["schemaVersion"] == 1 + assert {item["id"] for item in discovery["parameterCatalog"]} >= { + "design.frequency_mhz", + "floorplan.core_util", + } + assert result["issues"] == [] + assert result["resolvedWorkspaceSpec"]["parameters"]["design.frequency_mhz"] == 200.0 + assert result["resolvedWorkspaceSpec"]["parameters"]["floorplan.core_util"] == 0.4 + assert len(result["resolvedWorkspaceSpec"]["pdk"]["contentHash"]) == 64 + assert str(pdk) not in json.dumps(result["resolvedWorkspaceSpec"]) + assert sorted(str(path.relative_to(tmp_path)) for path in tmp_path.rglob("*")) == before + + +def test_workspace_spec_reports_structured_binding_and_input_cardinality_issues(tmp_path): + filelist = tmp_path / "sources.f" + filelist.write_text("missing.v\n") + spec = _valid_spec() + spec["inputs"] = [ + {"inputId": "rtl-main", "role": "rtl"}, + {"inputId": "sources", "role": "filelist"}, + ] + + result = validate_workspace_spec( + spec, + { + "inputs": {"sources": str(filelist), "unknown": str(filelist)}, + "pdk": {"root": str(tmp_path)}, + }, + ) + + assert {(issue["code"], issue["path"]) for issue in result["issues"]} >= { + ("conflicting_input_roles", "/inputs"), + ("input_binding_missing", "/bindings/inputs/rtl-main"), + ("unknown_input_binding", "/bindings/inputs/unknown"), + ("filelist_source_missing", "/bindings/inputs/sources"), + } + assert "resolvedWorkspaceSpec" not in result + + +def test_workspace_spec_rejects_multiple_manual_pdk_technology_files(tmp_path): + rtl = tmp_path / "gcd.v" + rtl.write_text("module gcd; endmodule\n") + for name in ("a.tech.lef", "b.tech.lef", "cells.lef", "typ.lib"): + (tmp_path / name).write_text(name) + spec = _valid_spec() + spec["pdk"] = { + "familyId": "ics55", + "mode": "manual", + "files": [ + {"fileId": "tech-a", "role": "tech"}, + {"fileId": "tech-b", "role": "tech"}, + {"fileId": "cells", "role": "lef"}, + {"fileId": "lib", "role": "liberty"}, + ], + } + + result = validate_workspace_spec( + json.loads(json.dumps(spec)), + { + "inputs": {"rtl-main": str(rtl)}, + "pdk": { + "root": str(tmp_path), + "files": { + "tech-a": str(tmp_path / "a.tech.lef"), + "tech-b": str(tmp_path / "b.tech.lef"), + "cells": str(tmp_path / "cells.lef"), + "lib": str(tmp_path / "typ.lib"), + }, + }, + }, + ) + + assert any(issue["code"] == "pdk_tech_cardinality" for issue in result["issues"]) + + +def test_validated_workspace_spec_creates_main_compatible_workspace( + tmp_path, minimal_ics55_pdk_factory +): + from chipcompiler.data import load_workspace + from chipcompiler.engine import create_workspace_from_spec + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + target = tmp_path / "workspace" + + created = create_workspace_from_spec( + target, + payload["workspaceSpec"], + bindings, + "create-1", + ) + reopened = load_workspace(target) + + assert created.directory == target + assert reopened.directory == target + assert reopened.design.name == "gcd" + assert reopened.parameters.data["frequency_max"] == 200.0 + assert reopened.flow.steps()[0]["name"] == "Synthesis" + + +def test_workspace_spec_applies_config_target_parameters(tmp_path, minimal_ics55_pdk_factory): + from chipcompiler.data import load_workspace + from chipcompiler.engine import create_workspace_from_spec + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + spec = deepcopy(payload["workspaceSpec"]) + spec["parameters"]["cts.skew_bound"] = "0.12" + + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings) + reopened = load_workspace(workspace.directory) + cts = json.loads(reopened.config["CTS"].read_text(encoding="utf-8")) + + assert cts["skew_bound"] == "0.12" + + +def test_manual_pdk_workspace_reopens_through_main_persistence(tmp_path): + from chipcompiler.data import load_workspace + from chipcompiler.engine import ( + assess_execution_readiness, + create_workspace_from_spec, + describe_workspace_binding_requirement, + ) + + rtl = tmp_path / "gcd.v" + rtl.write_text("module gcd(input clk); endmodule\n") + pdk_root = tmp_path / "pdk" + pdk_root.mkdir() + for name in ("tech.lef", "cells.lef", "typ.lib"): + (pdk_root / name).write_text(name) + spec = _valid_spec() + spec["flow"] = {"flowId": "syn_sta"} + spec["pdk"] = { + "familyId": "ics55", + "mode": "manual", + "files": [ + {"fileId": "tech", "role": "tech"}, + {"fileId": "cells", "role": "lef"}, + {"fileId": "lib", "role": "liberty"}, + ], + } + bindings = { + "inputs": {"rtl-main": str(rtl)}, + "pdk": { + "root": str(pdk_root), + "files": { + "tech": str(pdk_root / "tech.lef"), + "cells": str(pdk_root / "cells.lef"), + "lib": str(pdk_root / "typ.lib"), + }, + }, + } + target = tmp_path / "workspace" + + create_workspace_from_spec(target, spec, bindings) + reopened = load_workspace(target) + + assert reopened.pdk.tech == pdk_root / "tech.lef" + assert reopened.pdk.lefs == [pdk_root / "cells.lef"] + assert describe_workspace_binding_requirement(target)["mode"] == "manual" + assert assess_execution_readiness(target, bindings) == {"ready": True} + + +def test_partial_flow_spec_preserves_requested_boundaries(tmp_path, minimal_ics55_pdk_factory): + from chipcompiler.data import load_workspace + from chipcompiler.engine import create_workspace_from_spec + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + spec = deepcopy(payload["workspaceSpec"]) + spec["flow"] = { + "flowId": "rtl2gds", + "fromStepId": "Synthesis", + "throughStepId": "preFloorplan", + } + spec["parameters"] = {"design.frequency_mhz": 200.0} + workspace = create_workspace_from_spec(tmp_path / "workspace", spec, bindings) + + assert [step["name"] for step in load_workspace(workspace.directory).flow.steps()] == [ + "Synthesis", + "preFloorplan", + ] + + +def test_workspace_spec_update_is_atomic_revisioned_and_idempotent( + tmp_path, minimal_ics55_pdk_factory +): + from chipcompiler.engine import ( + WorkspaceLifecycleError, + create_workspace_from_spec, + update_workspace_from_spec, + ) + from chipcompiler.engine.snapshot import read_engineering_snapshot + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + target = tmp_path / "workspace" + created = create_workspace_from_spec(target, payload["workspaceSpec"], bindings, "create-1") + before = read_engineering_snapshot(created) + updated_spec = deepcopy(payload["workspaceSpec"]) + updated_spec["parameters"]["design.frequency_mhz"] = 250.0 + + updated = update_workspace_from_spec( + target, + before["workspaceRevision"], + updated_spec, + bindings, + "update-1", + ) + after = read_engineering_snapshot(updated) + repeated = update_workspace_from_spec( + target, + before["workspaceRevision"], + updated_spec, + bindings, + "update-1", + ) + + assert after["workspaceId"] == before["workspaceId"] + assert after["workspaceRevision"] == before["workspaceRevision"] + 1 + assert read_engineering_snapshot(repeated) == after + assert repeated.parameters.data["frequency_max"] == 250.0 + + with pytest.raises(WorkspaceLifecycleError) as conflict: + update_workspace_from_spec( + target, + before["workspaceRevision"], + payload["workspaceSpec"], + bindings, + "update-2", + ) + assert conflict.value.code == "revision_conflict" + assert read_engineering_snapshot(repeated) == after + + +def test_workspace_spec_update_keeps_generated_filelist_relocatable( + tmp_path, minimal_ics55_pdk_factory +): + from chipcompiler.data import load_workspace + from chipcompiler.data.parameter import load_parameter + from chipcompiler.engine import create_workspace_from_spec, update_workspace_from_spec + from chipcompiler.engine.snapshot import read_engineering_snapshot + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + target = tmp_path / "workspace" + created = create_workspace_from_spec(target, payload["workspaceSpec"], bindings) + + second_rtl = tmp_path / "helper.v" + second_rtl.write_text("module helper; endmodule\n", encoding="utf-8") + updated_spec = deepcopy(payload["workspaceSpec"]) + updated_spec["inputs"].append({"inputId": "rtl-helper", "role": "rtl"}) + updated_bindings = deepcopy(bindings) + updated_bindings["inputs"]["rtl-helper"] = str(second_rtl) + + update_workspace_from_spec( + target, + read_engineering_snapshot(created)["workspaceRevision"], + updated_spec, + updated_bindings, + ) + + persisted = Path(load_parameter(target / "home" / "params.toml").data["file_list"]) + reopened = load_workspace(target) + + assert not persisted.is_absolute() + assert (target / persisted).is_file() + assert reopened.design.input_filelist == target / persisted + + +def test_workspace_spec_stale_revision_does_not_create_missing_snapshot( + tmp_path, minimal_ics55_pdk_factory +): + from chipcompiler.engine import WorkspaceLifecycleError, create_workspace_from_spec + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + target = tmp_path / "workspace" + create_workspace_from_spec(target, payload["workspaceSpec"], bindings) + snapshot = target / "home" / "engineering-snapshot.json" + snapshot.unlink() + + with pytest.raises(WorkspaceLifecycleError) as conflict: + from chipcompiler.engine import update_workspace_from_spec + + update_workspace_from_spec( + target, + 2, + payload["workspaceSpec"], + bindings, + "stale-update", + ) + + assert conflict.value.code == "revision_conflict" + assert not snapshot.exists() + + +def test_workspace_spec_rejects_invalid_existing_snapshot(tmp_path, minimal_ics55_pdk_factory): + from chipcompiler.engine import ( + WorkspaceLifecycleError, + create_workspace_from_spec, + update_workspace_from_spec, + ) + + payload, bindings = _shared_fixture("valid.json") + bindings["pdk"]["root"] = str(minimal_ics55_pdk_factory(tmp_path / "pdk")) + target = tmp_path / "workspace" + create_workspace_from_spec(target, payload["workspaceSpec"], bindings) + snapshot = target / "home" / "engineering-snapshot.json" + snapshot.write_text('{"schemaVersion": 2}', encoding="utf-8") + + with pytest.raises(WorkspaceLifecycleError) as invalid: + update_workspace_from_spec( + target, + 1, + payload["workspaceSpec"], + bindings, + "invalid-snapshot-update", + ) + + assert invalid.value.code == "workspace_invalid" + assert snapshot.read_text(encoding="utf-8") == '{"schemaVersion": 2}' diff --git a/test/fixtures/workspace_spec/input/gcd.v b/test/fixtures/workspace_spec/input/gcd.v new file mode 100644 index 000000000..5705cb05d --- /dev/null +++ b/test/fixtures/workspace_spec/input/gcd.v @@ -0,0 +1,3 @@ +module gcd(input clk, output y); + assign y = clk; +endmodule diff --git a/test/fixtures/workspace_spec/input/sources.f b/test/fixtures/workspace_spec/input/sources.f new file mode 100644 index 000000000..369c2e562 --- /dev/null +++ b/test/fixtures/workspace_spec/input/sources.f @@ -0,0 +1 @@ +missing.v diff --git a/test/fixtures/workspace_spec/invalid.json b/test/fixtures/workspace_spec/invalid.json new file mode 100644 index 000000000..b179d6795 --- /dev/null +++ b/test/fixtures/workspace_spec/invalid.json @@ -0,0 +1,67 @@ +{ + "cliConfig": { + "design_name": "gcd", + "design_rtl": ["input/gcd.v"], + "design_top": "gcd", + "flow_preset": "rtl2gds", + "pdk_name": "ics55", + "pdk_overrides": { + "mystery": "unsupported" + }, + "pdk_root": "pdk" + }, + "expectedCliError": "unsupported_legacy_field: pdk.overrides.mystery", + "expectedIssueCodes": [ + "conflicting_input_roles", + "filelist_source_missing" + ], + "expectedStudioError": "unsupported_legacy_field: parameters.mystery_knob", + "studioDraft": { + "commandId": "workspace-create-invalid-shared-fixture", + "designInputMode": "rtl", + "directory": "workspace", + "originVerilog": "input/gcd.v", + "parameters": { + "design": "gcd", + "mystery_knob": 3, + "top_module": "gcd" + }, + "pdk": "ics55", + "pdkRoot": "pdk" + }, + "workspaceBindings": { + "inputs": { + "filelist": "input/sources.f", + "rtl-main": "input/gcd.v" + }, + "pdk": { + "root": "pdk" + } + }, + "workspaceSpec": { + "design": { + "name": "gcd", + "topModule": "gcd" + }, + "flow": { + "flowId": "rtl2gds" + }, + "inputMode": "rtl", + "inputs": [ + { + "inputId": "rtl-main", + "role": "rtl" + }, + { + "inputId": "filelist", + "role": "filelist" + } + ], + "parameters": {}, + "pdk": { + "familyId": "ics55", + "mode": "default" + }, + "schemaVersion": 1 + } +} diff --git a/test/fixtures/workspace_spec/pdk/.keep b/test/fixtures/workspace_spec/pdk/.keep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/test/fixtures/workspace_spec/pdk/.keep @@ -0,0 +1 @@ + diff --git a/test/fixtures/workspace_spec/valid.json b/test/fixtures/workspace_spec/valid.json new file mode 100644 index 000000000..94a43fe0c --- /dev/null +++ b/test/fixtures/workspace_spec/valid.json @@ -0,0 +1,68 @@ +{ + "cliConfig": { + "design_clock_port": "clk", + "design_frequency_mhz": 200, + "design_name": "gcd", + "design_rtl": ["input/gcd.v"], + "design_top": "gcd", + "flow_preset": "rtl2gds", + "params_overrides": { + "floorplan.core_util": 0.6, + "place.target_density": 0.7 + }, + "pdk_name": "ics55", + "pdk_root": "pdk" + }, + "studioDraft": { + "commandId": "workspace-create-shared-fixture", + "designInputMode": "rtl", + "directory": "workspace", + "originVerilog": "input/gcd.v", + "parameters": { + "clock": "clk", + "core_utilization": 0.6, + "design": "gcd", + "frequency_max": 200, + "target_density": 0.7, + "top_module": "gcd" + }, + "pdk": "ics55", + "pdkConfigMode": "default", + "pdkRoot": "pdk" + }, + "workspaceBindings": { + "inputs": { + "rtl-main": "input/gcd.v" + }, + "pdk": { + "root": "pdk" + } + }, + "workspaceSpec": { + "design": { + "clockPort": "clk", + "name": "gcd", + "topModule": "gcd" + }, + "flow": { + "flowId": "rtl2gds" + }, + "inputMode": "rtl", + "inputs": [ + { + "inputId": "rtl-main", + "role": "rtl" + } + ], + "parameters": { + "design.frequency_mhz": 200.0, + "floorplan.core_util": 0.6, + "place.target_density": 0.7 + }, + "pdk": { + "familyId": "ics55", + "mode": "default" + }, + "schemaVersion": 1 + } +} diff --git a/test/project/__init__.py b/test/project/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/project/test_manifest.py b/test/project/test_manifest.py new file mode 100644 index 000000000..0d768760a --- /dev/null +++ b/test/project/test_manifest.py @@ -0,0 +1,110 @@ +import json +from contextlib import contextmanager + +import chipcompiler.project.api as project_api +from chipcompiler.cli.project.manifest import load_manifest +from chipcompiler.project import ( + create_project_manifest, + discover_project_manifest, + load_project_manifest, + mutate_project_manifest, +) + + +def test_domain_created_manifest_is_cli_readable(tmp_path): + created = create_project_manifest(tmp_path, "Demo", "gcd", now="2026-01-01T00:00:00Z") + + loaded = load_manifest(str(tmp_path)) + + assert created["root_path"] == str(tmp_path.resolve()) + assert loaded.project_id == created["project_id"] + assert loaded.design_name == "gcd" + assert loaded.workspaces == () + + +def test_project_manifest_is_discovered_from_nested_workspace(tmp_path): + workspace = tmp_path / "experiment" + workspace.mkdir() + created = create_project_manifest(tmp_path, "Demo", "gcd", now="2026-01-01T00:00:00Z") + + assert discover_project_manifest(workspace) == (tmp_path.resolve(), created) + + +def test_register_workspace_uses_main_manifest_shape(tmp_path): + workspace = tmp_path / "experiment" + (workspace / "home").mkdir(parents=True) + (workspace / "home" / "flow.json").write_text( + json.dumps({"steps": [{"name": "Synthesis", "state": "Unstart"}]}) + ) + create_project_manifest(tmp_path, "Demo", "gcd", now="2026-01-01T00:00:00Z") + + updated = mutate_project_manifest( + tmp_path, + { + "type": "register_workspace", + "workspace_id": "experiment", + "workspace_path": str(workspace), + "name": "Experiment", + "created_at": "2026-02-01T00:00:00Z", + "updated_at": "2026-02-01T00:00:00Z", + }, + ) + + assert updated == load_project_manifest(tmp_path) + assert updated["workspaces"][0]["start_step"] == "Synth" + assert updated["workspaces"][0]["end_step"] == "Synth" + + +def test_manifest_mutation_without_timestamp_keeps_audit_timestamp(tmp_path): + workspace = tmp_path / "experiment" + (workspace / "home").mkdir(parents=True) + (workspace / "home" / "flow.json").write_text(json.dumps({"steps": []})) + create_project_manifest(tmp_path, "Demo", "gcd", now="2026-01-01T00:00:00Z") + mutate_project_manifest( + tmp_path, + { + "type": "register_workspace", + "workspace_id": "experiment", + "workspace_path": str(workspace), + "name": "Experiment", + }, + ) + + updated = mutate_project_manifest( + tmp_path, + {"type": "archive_workspace", "workspace_id": "experiment"}, + ) + + assert updated["updated_at"] + assert updated["workspaces"][0]["updated_at"] + + +def test_project_workspace_acquires_manifest_lock_before_workspace_lock(tmp_path, monkeypatch): + create_project_manifest(tmp_path, "Demo", "gcd", now="2026-01-01T00:00:00Z") + events = [] + + @contextmanager + def marked_lock(name): + events.append(f"{name}:acquire") + yield + + monkeypatch.setattr(project_api, "manifest_lock", lambda _project: marked_lock("manifest")) + monkeypatch.setattr( + "chipcompiler.engine.reconcile._workspace_lock", + lambda _target: marked_lock("workspace"), + ) + monkeypatch.setattr( + "chipcompiler.engine.workspace_lifecycle._create_workspace_from_spec", + lambda *_args: object(), + ) + monkeypatch.setattr(project_api, "update_manifest_locked", lambda *_args: True) + + project_api.create_project_workspace( + tmp_path, + tmp_path / "experiment", + {}, + {}, + command_id="create-1", + ) + + assert events == ["manifest:acquire", "workspace:acquire"] diff --git a/test/runtime/test_methods.py b/test/runtime/test_methods.py index b3fb0d126..b5a0a27d4 100644 --- a/test/runtime/test_methods.py +++ b/test/runtime/test_methods.py @@ -16,8 +16,19 @@ def test_runtime_method_registry_contains_current_methods_once(): from chipcompiler.runtime.methods import RUNTIME_METHODS, runtime_method_names expected_methods = ( + "workspace_spec.describe", + "workspace_spec.validate", + "project.discover", + "project.manifest.load", + "project.manifest.mutate", "workspace.create", "workspace.open", + "workspace.binding_requirement", + "workspace.update", + "workspace.configuration.update", + "workspace.configuration.read", + "workspace.step_configuration.update", + "workspace.step_configuration.read", "workspace.close", "workspace.home", "workspace.info", @@ -34,9 +45,9 @@ def test_runtime_method_registry_contains_current_methods_once(): "operation.cancel", "operation.ack_step_rendered", "workspace.snapshot", + "workspace.engineering_snapshot", "workspace.recover_interrupted", ) - assert runtime_method_names() == expected_methods assert len(runtime_method_names()) == len(set(runtime_method_names())) assert len(RUNTIME_METHODS) == len(expected_methods) diff --git a/test/runtime/test_operation_parity.py b/test/runtime/test_operation_parity.py new file mode 100644 index 000000000..de8f1261d --- /dev/null +++ b/test/runtime/test_operation_parity.py @@ -0,0 +1,278 @@ +import threading +from types import SimpleNamespace + +import pytest + +from chipcompiler.data import StateEnum +from chipcompiler.runtime.errors import RuntimeApiError +from chipcompiler.runtime.operations import ( + RuntimeOperationIdempotencyConflict, + RuntimeOperationManager, +) + + +def test_terminal_step_uses_the_committed_snapshot_revision_for_failures(): + events = [] + commits = [] + manager = RuntimeOperationManager(events.append) + step = SimpleNamespace(name="LEC", tool="yosys_lec", log=SimpleNamespace(file="")) + + def runner(observer): + observer.on_step_completed(step, StateEnum.Imcomplete, "missing output") + raise RuntimeError("run step LEC failed with state Incomplete") + + started = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=False, + step="LEC", + idempotency_key="failed-lec-revision", + workspace_revision=6, + snapshot_committer=lambda *_args: commits.append(7) or 7, + runner=runner, + ) + + status = _wait_for_terminal(manager, started["operationId"]) + completed = _wait_for_event(events, "step.completed") + + assert status["workspaceRevision"] == 7 + assert commits == [7] + assert completed["payload"]["workspaceRevision"] == 7 + assert completed["payload"]["stepCommitId"].endswith(":step:7") + + +def test_rerun_prepared_adopts_and_publishes_the_reset_revision(): + events = [] + manager = RuntimeOperationManager(events.append) + + def runner(observer): + observer.on_rerun_prepared( + affected_steps=["preFloorplan", "place"], + scope="flow", + workspace_revision=8, + ) + return {"rerun": True} + + started = manager.start( + workspace_id="workspace-1", + kind="flow", + origin="gui", + rerun=True, + step="", + idempotency_key="rerun-revision", + workspace_revision=7, + runner=runner, + ) + + status = _wait_for_terminal(manager, started["operationId"]) + prepared = _wait_for_event(events, "operation.rerun_prepared") + + assert prepared["payload"]["workspaceRevision"] == 8 + assert status["workspaceRevision"] == 8 + + +def test_reusing_a_command_id_with_different_parameters_conflicts(): + release = threading.Event() + manager = RuntimeOperationManager() + first = manager.start( + workspace_id="workspace-1", + kind="flow", + origin="gui", + rerun=False, + step="", + idempotency_key="request-1", + runner=lambda _observer: release.wait(timeout=1) or {}, + ) + + try: + with pytest.raises(RuntimeOperationIdempotencyConflict): + manager.start( + workspace_id="workspace-1", + kind="flow", + origin="gui", + rerun=True, + step="", + idempotency_key="request-1", + runner=lambda _observer: {}, + ) + finally: + release.set() + _wait_for_terminal(manager, first["operationId"]) + + +def test_command_fingerprint_includes_command_specific_input(): + release = threading.Event() + manager = RuntimeOperationManager() + first = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=True, + step="Place", + idempotency_key="request-1", + command_input={"resetDependents": False}, + runner=lambda _observer: release.wait(timeout=1) or {}, + ) + + try: + with pytest.raises(RuntimeOperationIdempotencyConflict): + manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=True, + step="Place", + idempotency_key="request-1", + command_input={"resetDependents": True}, + runner=lambda _observer: {}, + ) + finally: + release.set() + _wait_for_terminal(manager, first["operationId"]) + + +def test_step_diagnostic_persists_structured_tool_error(): + events = [] + manager = RuntimeOperationManager(events.append) + step = SimpleNamespace(name="place", tool="dreamplace", log=SimpleNamespace(file="place.log")) + + def runner(observer): + observer.on_step_diagnostic( + step, + {"message": "overflow exceeded", "details": {"overflow": 1.2}}, + ) + return {"rerun": False} + + started = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=False, + step="place", + idempotency_key="diagnostic", + runner=runner, + ) + + status = _wait_for_terminal(manager, started["operationId"]) + + assert status["error"] == { + "code": "tool_failed", + "message": "overflow exceeded", + "step": "place", + "tool": "dreamplace", + "logFile": "place.log", + "details": {"overflow": 1.2}, + "snapshotRevision": 0, + "eventRevision": 0, + "operationRevision": 0, + } + diagnostic = next(event for event in events if event["type"] == "step.diagnostic") + assert diagnostic["payload"]["workspaceRevision"] == status["workspaceRevision"] == 0 + assert diagnostic["payload"]["diagnostic"]["operationRevision"] == 0 + + +def test_terminal_completion_preserves_a_prior_structured_diagnostic(): + manager = RuntimeOperationManager() + step = SimpleNamespace(name="place", tool="dreamplace", log=SimpleNamespace(file="place.log")) + + def runner(observer): + observer.on_step_diagnostic(step, {"message": "overflow exceeded", "overflow": 1.2}) + observer.on_step_completed(step, StateEnum.Imcomplete, "tool exited with failure") + + started = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=False, + step="place", + idempotency_key="diagnostic-terminal", + runner=runner, + ) + + status = _wait_for_terminal(manager, started["operationId"]) + + assert status["error"] == { + "code": "tool_failed", + "message": "overflow exceeded", + "step": "place", + "tool": "dreamplace", + "logFile": "place.log", + "overflow": 1.2, + "snapshotRevision": 0, + "eventRevision": 0, + "operationRevision": 0, + } + + +def test_snapshot_commit_failure_keeps_a_stable_operation_error_code(): + manager = RuntimeOperationManager() + step = SimpleNamespace(name="Synthesis", tool="yosys", log=SimpleNamespace(file="")) + + def fail_commit(*_args): + raise RuntimeApiError( + "engineering_snapshot_commit_failed", + "snapshot write failed", + {"step": "Synthesis"}, + ) + + started = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=False, + step="Synthesis", + idempotency_key="snapshot-failure", + snapshot_committer=fail_commit, + runner=lambda observer: observer.on_step_completed(step, StateEnum.Success), + ) + + status = _wait_for_terminal(manager, started["operationId"]) + + assert status["state"] == "failed" + assert status["error"] == { + "code": "engineering_snapshot_commit_failed", + "message": "snapshot write failed", + "step": "Synthesis", + } + + +def test_event_consumer_failure_does_not_rollback_a_durable_step_commit(): + manager = RuntimeOperationManager( + lambda _event: (_ for _ in ()).throw(RuntimeError("IPC down")) + ) + step = SimpleNamespace(name="Synthesis", tool="yosys", log=SimpleNamespace(file="")) + + started = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=False, + step="Synthesis", + idempotency_key="publisher-failure", + snapshot_committer=lambda *_args: 12, + runner=lambda observer: observer.on_step_completed(step, StateEnum.Success), + ) + + status = _wait_for_terminal(manager, started["operationId"]) + + assert status["state"] == "succeeded" + assert status["workspaceRevision"] == 12 + + +def _wait_for_event(events: list[dict], event_type: str) -> dict: + for _ in range(200): + for event in events: + if event["type"] == event_type: + return event + threading.Event().wait(0.01) + raise AssertionError(f"event not received: {event_type}") + + +def _wait_for_terminal(manager: RuntimeOperationManager, operation_id: str) -> dict: + for _ in range(100): + status = manager.operation_status(operation_id) + if status["state"] in {"succeeded", "failed", "cancelled", "interrupted"}: + return status + threading.Event().wait(0.01) + return manager.operation_status(operation_id) diff --git a/test/runtime/test_operations.py b/test/runtime/test_operations.py index 70c9e77e6..7ca9d4150 100644 --- a/test/runtime/test_operations.py +++ b/test/runtime/test_operations.py @@ -1,14 +1,14 @@ +import json import threading from types import SimpleNamespace from chipcompiler.data import StateEnum -from chipcompiler.runtime import operations +from chipcompiler.engine.flow import EngineFlow from chipcompiler.runtime.operations import RuntimeOperationManager -def test_successful_step_waits_for_matching_render_ack_before_completing(): +def test_successful_step_does_not_wait_for_render_ack_before_completing(): events = [] - entered_render_gate = threading.Event() completed = threading.Event() manager = RuntimeOperationManager(events.append) step = SimpleNamespace(name="Synthesis", tool="yosys", log=SimpleNamespace(file="")) @@ -16,7 +16,6 @@ def test_successful_step_waits_for_matching_render_ack_before_completing(): def runner(observer): observer.on_step_started(step) observer.on_step_completed(step, StateEnum.Success) - entered_render_gate.set() assert observer.wait_for_step_rendered(step, StateEnum.Success) completed.set() return {"rerun": False} @@ -31,33 +30,109 @@ def runner(observer): runner=runner, ) - assert started["state"] in {"queued", "running", "waiting_for_gui_sync"} - assert entered_render_gate.wait(timeout=1) - assert not completed.wait(timeout=0.05) + assert started["state"] in {"queued", "running", "succeeded"} + assert completed.wait(timeout=1) step_completed = next(event for event in events if event["type"] == "step.completed") assert step_completed["payload"]["stepCommitId"] assert step_completed["payload"]["workspaceRevision"] == 1 - assert not manager.acknowledge_step_rendered( - started["operationId"], - step_completed["eventId"], - "wrong-step-commit", - step_completed["payload"]["workspaceRevision"], - )["accepted"] - - assert manager.acknowledge_step_rendered( - started["operationId"], - step_completed["eventId"], - step_completed["payload"]["stepCommitId"], - step_completed["payload"]["workspaceRevision"], - ) == { + status = manager.operation_status(started["operationId"]) + assert status["state"] == "succeeded" + assert status["renderSyncState"] == "idle" + assert status["awaitingEventId"] is None + assert events[-1]["type"] == "operation.completed" + + +def test_cancel_stops_before_next_engine_flow_step(monkeypatch): + events = [] + first_step_running = threading.Event() + release_first_step = threading.Event() + manager = RuntimeOperationManager(events.append) + steps = [ + SimpleNamespace(name=name, tool="mock", log=SimpleNamespace(file="")) + for name in ("Synthesis", "Floorplan") + ] + workspace = SimpleNamespace( + flow=SimpleNamespace(data={"steps": [{}, {}]}), + logger=SimpleNamespace(log_section=lambda *_args: None, error=lambda *_args: None), + ) + flow = EngineFlow(workspace=None) + flow.workspace = workspace + flow.workspace_steps = steps + flow.init_db_engine = lambda: True + executed = [] + + def run_step(step, *, rerun=False, observer=None): + executed.append(step.name) + observer.on_step_started(step) + if step.name == "Synthesis": + first_step_running.set() + assert release_first_step.wait(timeout=2) + observer.on_step_completed(step, StateEnum.Success) + return StateEnum.Success + + flow.run_step = run_step + monkeypatch.setattr("chipcompiler.engine.flow.log_flow", lambda **_kwargs: None) + revisions = [] + + def commit_step(*_args): + revisions.append(len(revisions) + 1) + return revisions[-1] + + started = manager.start( + workspace_id="workspace-1", + kind="flow", + origin="gui", + rerun=False, + step="", + idempotency_key="cancel-at-step-boundary", + snapshot_committer=commit_step, + runner=lambda observer: {"succeeded": flow.run_steps(observer=observer)}, + ) + assert first_step_running.wait(timeout=1) + cancellation = manager.request_cancel(started["operationId"]) + assert cancellation == { "accepted": True, - "duplicate": False, "operationId": started["operationId"], - "eventId": step_completed["eventId"], + "state": "cancelling", } - assert completed.wait(timeout=1) - assert manager.operation_status(started["operationId"])["state"] == "succeeded" - assert events[-1]["type"] == "operation.completed" + release_first_step.set() + + status = _wait_for_terminal(manager, started["operationId"]) + + assert status["state"] == "cancelled" + assert status["workspaceRevision"] == 1 + assert executed == ["Synthesis"] + assert revisions == [1] + event_types = [event["type"] for event in events] + assert event_types.index("step.completed") < event_types.index("operation.cancelled") + + +def test_queued_cancel_finishes_without_leaving_active_workspace(monkeypatch): + class DeferredThread: + def __init__(self, target, args, **_kwargs): + self._target = target + self._args = args + + def start(self): + return None + + monkeypatch.setattr("chipcompiler.runtime.operations.threading.Thread", DeferredThread) + manager = RuntimeOperationManager() + started = manager.start( + workspace_id="workspace-1", + kind="flow", + origin="gui", + rerun=False, + step="", + idempotency_key="queued-cancel", + runner=lambda _observer: {"ok": True}, + ) + + assert manager.request_cancel(started["operationId"])["state"] == "cancelling" + manager._run(started["operationId"], lambda _observer: {"ok": True}, None) + + assert manager.operation_status(started["operationId"])["state"] == "cancelled" + assert manager.shutdown_barrier() is None def test_subflow_stage_is_emitted_for_the_active_workspace_step(): @@ -104,56 +179,6 @@ def runner(observer): assert _wait_for_terminal(manager, started["operationId"])["state"] == "succeeded" -def test_render_ack_replays_one_commit_then_pauses_before_a_bounded_timeout(monkeypatch): - monkeypatch.setattr(operations, "_RENDER_ACK_RETRY_SECONDS", 0.01) - monkeypatch.setattr(operations, "_RENDER_ACK_PAUSE_SECONDS", 0.02) - monkeypatch.setattr(operations, "_RENDER_ACK_ABORT_SECONDS", 0.5) - events = [] - entered_render_gate = threading.Event() - manager = RuntimeOperationManager(events.append) - step = SimpleNamespace(name="Synthesis", tool="yosys", log=SimpleNamespace(file="")) - - def runner(observer): - observer.on_step_started(step) - observer.on_step_completed(step, StateEnum.Success) - entered_render_gate.set() - assert observer.wait_for_step_rendered(step, StateEnum.Success) - return {"rerun": False} - - started = manager.start( - workspace_id="workspace-1", - kind="flow", - origin="gui", - rerun=False, - step="", - idempotency_key="request-replay", - runner=runner, - ) - assert entered_render_gate.wait(timeout=1) - step_completed = _wait_for_event(events, "step.completed") - paused = _wait_for_event(events, "operation.gui_sync_paused") - replays = [ - event - for event in events - if event["type"] == "step.completed" and event["payload"].get("replayed") - ] - assert paused["payload"]["stepCommitId"] == step_completed["payload"]["stepCommitId"] - assert replays - assert all(event["eventId"] == step_completed["eventId"] for event in replays) - assert ( - manager.operation_status(started["operationId"])["renderSyncState"] - == "paused_for_gui_recovery" - ) - - assert manager.acknowledge_step_rendered( - started["operationId"], - step_completed["eventId"], - step_completed["payload"]["stepCommitId"], - step_completed["payload"]["workspaceRevision"], - )["accepted"] - assert _wait_for_terminal(manager, started["operationId"])["state"] == "succeeded" - - def test_ack_and_start_requests_are_idempotent(): events = [] release = threading.Event() @@ -269,49 +294,6 @@ def runner(observer): assert len([event for event in events if event["type"] == "operation.rerun_prepared"]) == 1 -def test_render_ack_timeout_degrades_and_allows_the_flow_to_continue(monkeypatch): - monkeypatch.setattr(operations, "_RENDER_ACK_RETRY_SECONDS", 0.01) - monkeypatch.setattr(operations, "_RENDER_ACK_PAUSE_SECONDS", 0.02) - monkeypatch.setattr(operations, "_RENDER_ACK_ABORT_SECONDS", 0.04) - events = [] - manager = RuntimeOperationManager(events.append) - step = SimpleNamespace(name="Synthesis", tool="yosys", log=SimpleNamespace(file="")) - - def runner(observer): - observer.on_step_started(step) - observer.on_step_completed(step, StateEnum.Success) - assert observer.wait_for_step_rendered(step, StateEnum.Success) - observer.on_step_started(step) - observer.on_step_completed(step, StateEnum.Success) - assert observer.wait_for_step_rendered(step, StateEnum.Success) - return {"rerun": False} - - started = manager.start( - workspace_id="workspace-1", - kind="flow", - origin="gui", - rerun=False, - step="", - idempotency_key="request-degraded-sync", - runner=runner, - ) - - degraded = _wait_for_event(events, "operation.gui_sync_degraded") - assert degraded["payload"]["stepCommitId"] - assert _wait_for_terminal(manager, started["operationId"])["state"] == "succeeded" - completed_events = [ - event - for event in events - if event["type"] == "step.completed" and not event["payload"].get("replayed") - ] - assert len(completed_events) == 2 - assert completed_events[1]["payload"]["stepCommitId"] - assert ( - manager.operation_status(started["operationId"])["renderSyncState"] == "gui_sync_degraded" - ) - assert not any(event["type"] == "operation.failed" for event in events) - - def test_active_operation_reports_a_shutdown_barrier_and_safe_boundary(): release = threading.Event() manager = RuntimeOperationManager() @@ -336,34 +318,6 @@ def test_active_operation_reports_a_shutdown_barrier_and_safe_boundary(): release.set() -def test_cancel_at_render_ack_boundary_releases_the_waiting_flow(): - entered_render_gate = threading.Event() - manager = RuntimeOperationManager() - step = SimpleNamespace(name="Synthesis", tool="yosys", log=SimpleNamespace(file="")) - - def runner(observer): - observer.on_step_started(step) - observer.on_step_completed(step, StateEnum.Success) - entered_render_gate.set() - if not observer.wait_for_step_rendered(step, StateEnum.Success): - raise RuntimeError("operation cancelled at a render boundary") - return {"rerun": False} - - started = manager.start( - workspace_id="workspace-1", - kind="flow", - origin="gui", - rerun=False, - step="", - idempotency_key="request-cancel-at-gate", - runner=runner, - ) - assert entered_render_gate.wait(timeout=1) - - assert manager.request_cancel(started["operationId"])["accepted"] is True - assert _wait_for_terminal(manager, started["operationId"])["state"] == "cancelled" - - def test_step_log_events_stream_only_new_log_bytes_and_keep_final_tail(tmp_path): events = [] step_started = threading.Event() @@ -405,9 +359,6 @@ def runner(observer): complete_step.set() step_complete = _wait_for_event(events, "step.completed") assert step_complete["payload"]["finalLog"] == ("previous run\nlive line one\nlive line two\n") - assert manager.acknowledge_step_rendered(started["operationId"], step_complete["eventId"])[ - "accepted" - ] assert _wait_for_terminal(manager, started["operationId"])["state"] == "succeeded" @@ -500,6 +451,122 @@ def runner(observer): } +def test_operation_manager_keeps_only_latest_terminal_window(): + manager = RuntimeOperationManager() + + for index in range(257): + started = manager.start( + workspace_id="workspace-1", + kind="step", + origin="gui", + rerun=False, + step="step", + idempotency_key=f"command-{index}", + runner=lambda _observer: {"ok": True}, + ) + assert _wait_for_terminal(manager, started["operationId"])["state"] == "succeeded" + + assert len(manager.workspace_snapshot("workspace-1")["operations"]) == 256 + + +def test_operation_ledger_recovers_unfinished_operations_as_interrupted(tmp_path): + ledger = tmp_path / "runtime-commands.json" + manager = RuntimeOperationManager() + manager.load_workspace_ledger("workspace-1", ledger) + release = threading.Event() + started = manager.start( + workspace_id="workspace-1", + kind="flow", + origin="gui", + rerun=False, + step="", + idempotency_key="running-command", + runner=lambda _observer: release.wait(timeout=2), + ) + for _ in range(100): + if ledger.exists() and started["operationId"] in ledger.read_text(): + break + threading.Event().wait(0.01) + + restored = RuntimeOperationManager() + restored_ids = restored.load_workspace_ledger("workspace-1", ledger) + assert restored_ids == [started["operationId"]] + assert restored.operation_status(started["operationId"])["state"] == "interrupted" + release.set() + + +def test_read_only_ledger_load_does_not_recover_or_rewrite(tmp_path): + ledger = tmp_path / "runtime-commands.json" + ledger.write_text( + json.dumps( + { + "schemaVersion": 1, + "workspaceId": "workspace-1", + "operations": [ + { + "operationId": "operation-running", + "runSessionId": "session-running", + "runtimeInstanceId": "runtime-old", + "workspaceId": "workspace-1", + "kind": "flow", + "origin": "gui", + "state": "running", + } + ], + } + ), + encoding="utf-8", + ) + before = ledger.read_bytes() + + manager = RuntimeOperationManager() + loaded = manager.load_workspace_ledger("workspace-1", ledger, recover=False) + + assert loaded == ["operation-running"] + assert manager.operation_status("operation-running")["state"] == "running" + assert not manager.is_active("operation-running") + assert ledger.read_bytes() == before + + assert manager.load_workspace_ledger("workspace-1", ledger) == ["operation-running"] + assert manager.operation_status("operation-running")["state"] == "interrupted" + + +def test_operation_ledger_clears_legacy_render_wait_state(tmp_path): + ledger = tmp_path / "runtime-commands.json" + ledger.write_text( + json.dumps( + { + "schemaVersion": 1, + "workspaceId": "workspace-1", + "operations": [ + { + "operationId": "operation-legacy", + "runSessionId": "session-legacy", + "runtimeInstanceId": "runtime-legacy", + "workspaceId": "workspace-1", + "kind": "flow", + "origin": "gui", + "state": "waiting_for_gui_sync", + "awaitingEventId": "event-legacy", + "awaitingStepCommitId": "commit-legacy", + "renderSyncState": "waiting_for_gui_sync", + } + ], + } + ), + encoding="utf-8", + ) + + manager = RuntimeOperationManager() + manager.load_workspace_ledger("workspace-1", ledger) + + status = manager.operation_status("operation-legacy") + assert status["state"] == "interrupted" + assert status["renderSyncState"] == "idle" + assert status["awaitingEventId"] is None + assert status["awaitingStepCommitId"] is None + + def _wait_for_event(events: list[dict], event_type: str) -> dict: for _ in range(200): for event in events: @@ -512,7 +579,7 @@ def _wait_for_event(events: list[dict], event_type: str) -> dict: def _wait_for_terminal(manager: RuntimeOperationManager, operation_id: str) -> dict: for _ in range(100): status = manager.operation_status(operation_id) - if status["state"] in {"succeeded", "failed", "cancelled"}: + if status["state"] in {"succeeded", "failed", "cancelled", "interrupted"}: return status threading.Event().wait(0.01) return manager.operation_status(operation_id) diff --git a/test/runtime/test_server.py b/test/runtime/test_server.py index 34393f85e..7b275bcf0 100644 --- a/test/runtime/test_server.py +++ b/test/runtime/test_server.py @@ -10,7 +10,7 @@ WorkspaceInspectSignoffRequest, WorkspaceOpenRequest, ) -from chipcompiler.runtime.server import RuntimeServer +from chipcompiler.runtime.server import RuntimeServer, _project_runtime_event from chipcompiler.runtime.workspace_api import RuntimeApiError @@ -136,6 +136,18 @@ def test_rpc_hello_reports_persistent_db_capabilities_when_enabled(): assert "db.release" in response["result"]["capabilities"] +def test_cancel_requested_event_projects_cancelling_state(): + projected = _project_runtime_event( + { + "type": "operation.cancel_requested", + "payload": {}, + } + ) + + assert projected["type"] == "operation.changed" + assert projected["payload"]["state"] == "cancelling" + + def test_rpc_hello_rejects_incompatible_version(): server = RuntimeServer() @@ -383,7 +395,7 @@ def test_first_slice_methods_are_registered(method): response = _dispatch(server, f'{{"jsonrpc":"2.0","method":"{method}","id":1}}') - assert response["error"]["code"] != -32601 + assert response.get("error", {}).get("code") != -32601 @pytest.mark.parametrize( diff --git a/test/runtime/test_workspace_api.py b/test/runtime/test_workspace_api.py index be63e65d4..427fdc7ce 100644 --- a/test/runtime/test_workspace_api.py +++ b/test/runtime/test_workspace_api.py @@ -18,12 +18,27 @@ WorkspaceInfoRequest, WorkspaceOpenRequest, WorkspaceRecoverInterruptedRequest, + WorkspaceStepConfigurationReadRequest, WorkspaceSyncConfigRequest, ) -from chipcompiler.runtime.sessions import WorkspaceSessionRegistry +from chipcompiler.runtime.sessions import WorkspaceSession, WorkspaceSessionRegistry from chipcompiler.runtime.workspace_api import RuntimeApiError, WorkspaceRuntimeApi +def test_legacy_flow_request_without_revision_remains_compatible(tmp_path): + session = WorkspaceSession( + workspace_id="workspace-1", + directory=tmp_path, + workspace=object(), + workspace_revision=2, + ) + + WorkspaceRuntimeApi._validate_workspace_revision( + session, + FlowRunRequest(workspace_id="workspace-1").expected_workspace_revision, + ) + + class DummyEngineDB: def __init__(self, flow): self.flow = flow @@ -403,6 +418,53 @@ def test_open_workspace_loads_without_creating_step_workspaces(monkeypatch, tmp_ assert not DummyFlow.instances[0].created +def test_open_workspace_reuses_engineering_snapshot_identity(monkeypatch, tmp_path): + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + monkeypatch.setattr( + "chipcompiler.engine.snapshot.read_engineering_snapshot", + lambda _workspace: {"workspaceId": "cli-workspace", "workspaceRevision": 7}, + ) + api = WorkspaceRuntimeApi() + + result = api.open_workspace(WorkspaceOpenRequest(directory=str(ws))) + + assert result == { + "workspaceId": "cli-workspace", + "workspaceRevision": 7, + "directory": str(ws.resolve()), + } + + +def test_step_configuration_keeps_cli_workspace_identity_after_open(monkeypatch, tmp_path): + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + monkeypatch.setattr( + "chipcompiler.engine.snapshot.read_engineering_snapshot", + lambda _workspace: {"workspaceId": "cli-workspace", "workspaceRevision": 7}, + ) + monkeypatch.setattr( + "chipcompiler.engine.read_step_configuration", + lambda _workspace, _step: { + "step": "Synthesis", + "stepId": "Synthesis", + "parameters": [], + "workspaceId": "cli-workspace", + "workspaceRevision": 7, + }, + ) + api = WorkspaceRuntimeApi() + opened = api.open_workspace(WorkspaceOpenRequest(directory=str(ws))) + + result = api.read_workspace_step_configuration( + WorkspaceStepConfigurationReadRequest( + step="Synthesis", + workspace_id=opened["workspaceId"], + ) + ) + + assert result["workspaceId"] == opened["workspaceId"] + assert result["workspaceRevision"] == opened["workspaceRevision"] + + def test_recover_interrupted_is_marker_scoped_and_idempotent(monkeypatch, tmp_path): _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) api = WorkspaceRuntimeApi() @@ -923,6 +985,76 @@ def test_runtime_modules_do_not_import_typer_or_click(): assert "import click" not in source +def test_workspace_snapshot_includes_configuration_and_engineering_snapshot(monkeypatch, tmp_path): + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + configuration = { + "workspaceId": "workspace-1", + "workspaceRevision": 2, + "workspaceSpec": {"design": {"name": "gcd"}}, + "workspaceBindings": {}, + } + monkeypatch.setattr( + "chipcompiler.engine.read_workspace_configuration", + lambda _workspace: configuration, + ) + api = WorkspaceRuntimeApi() + monkeypatch.setattr( + api, + "_read_engineering_snapshot", + lambda _owner: {"workspaceId": "workspace-1", "workspaceRevision": 2}, + ) + workspace_id = api.open_workspace(WorkspaceOpenRequest(directory=str(ws)))["workspaceId"] + session = api.sessions.get_session(workspace_id) + session.workspace.parameters = SimpleNamespace(data={}, path=ws / "home" / "params.toml") + session.workspace.home.data = {} + + snapshot = api.workspace_snapshot(WorkspaceIdRequest(workspace_id)) + + assert snapshot["configuration"] == configuration + assert snapshot["engineeringSnapshot"] == { + "workspaceId": "workspace-1", + "workspaceRevision": 2, + } + + +def test_workspace_snapshot_falls_back_to_persisted_flow_steps(monkeypatch, tmp_path): + _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) + api = WorkspaceRuntimeApi() + workspace_id = api.open_workspace(WorkspaceOpenRequest(directory=str(ws)))["workspaceId"] + workspace = api.sessions.get_session(workspace_id).workspace + flow = workspace.flow + flow.data = {} + workspace.parameters = SimpleNamespace(data={}, path=ws / "home" / "parameters.json") + workspace.home = SimpleNamespace(data={}) + monkeypatch.setattr( + flow, + "steps", + lambda: [{"name": "Synthesis", "tool": "yosys", "state": "Success"}], + raising=False, + ) + monkeypatch.setattr( + api, + "_read_engineering_snapshot", + lambda _owner: {"workspaceId": "workspace-1", "workspaceRevision": 1}, + ) + monkeypatch.setattr( + "chipcompiler.engine.read_workspace_configuration", + lambda _workspace: {}, + ) + + snapshot = api.workspace_snapshot(WorkspaceIdRequest(workspace_id)) + + assert snapshot["flow"]["steps"] == [ + { + "name": "Synthesis", + "tool": "yosys", + "state": "Success", + "runtime": "", + "peakMemory": 0, + } + ] + + def test_flow_run_uses_run_steps_and_prepare_on_rerun(monkeypatch, tmp_path): _capture, ws = _install_runtime_mocks(monkeypatch, tmp_path) prepared = [] diff --git a/test/test_engine_flow.py b/test/test_engine_flow.py index ad7b9101e..068dc0171 100644 --- a/test/test_engine_flow.py +++ b/test/test_engine_flow.py @@ -41,6 +41,25 @@ def test_engine_flow_missing_path_is_not_initialized(): assert engine_flow.has_init() is False +def test_run_step_without_runtime_operation_marker_does_not_expand_function(monkeypatch, tmp_path): + from chipcompiler.engine.execution import ExecutionObserver + + workspace = Workspace() + workspace.flow.data = { + "steps": [{"name": "route", "tool": "ecc", "state": "Unstart"}], + } + engine_flow = EngineFlow(workspace) + workspace_step = EccStep(name="route", directory=tmp_path, tool="ecc") + engine_flow.workspace_steps = [workspace_step] + engine_flow.engine_db = SimpleNamespace(engine=None) + + monkeypatch.setattr(tools, "run_step", lambda **_kwargs: True) + monkeypatch.setattr(engine_flow, "check_step_result", lambda **_kwargs: True) + + observer = ExecutionObserver(object()) + assert engine_flow.run_step(workspace_step, observer=observer) == StateEnum.Success + + def test_engine_flow_default_steps_include_synthesis_lec(tmp_path): workspace = Workspace() workspace.flow.path = tmp_path / "flow.json" @@ -522,6 +541,39 @@ class Observer: assert step["state"] == StateEnum.Ongoing.value assert step["info"]["runtime_operation"]["operation_id"] == "operation-1" + def test_fatal_completion_commit_restores_ongoing_flow_marker(self, monkeypatch, tmp_path): + workspace = Workspace() + workspace.flow.path = tmp_path / "flow.json" + workspace.flow.data = { + "steps": [{"name": "place", "tool": "dreamplace", "state": "Unstart", "info": {}}], + } + workspace.flow.path.write_text(json.dumps(workspace.flow.data), encoding="utf-8") + engine_flow = EngineFlow(workspace) + workspace_step = EccStep(name="place", directory=tmp_path, tool="dreamplace") + engine_flow.workspace_steps = [workspace_step] + engine_flow.engine_db = SimpleNamespace(engine=None) + + class Observer: + fatal_observer = True + runtime_operation = { + "schema": 1, + "operation_id": "operation-1", + "runtime_instance_id": "runtime-1", + } + + def on_step_completed(self, _step, _state, _error=None): + raise RuntimeError("snapshot commit failed") + + monkeypatch.setattr(tools, "run_step", lambda **_kwargs: True) + monkeypatch.setattr(engine_flow, "check_step_result", lambda **_kwargs: True) + + with pytest.raises(RuntimeError, match="snapshot commit failed"): + engine_flow.run_step(workspace_step, observer=Observer()) + + persisted = json.loads(workspace.flow.path.read_text(encoding="utf-8"))["steps"][0] + assert persisted["state"] == StateEnum.Ongoing.value + assert persisted["info"]["runtime_operation"]["operation_id"] == "operation-1" + def test_result_check_system_exit_still_finalizes_step(self, monkeypatch, tmp_path): workspace = Workspace() workspace.flow.path = tmp_path / "flow.json" diff --git a/test/test_signoff_package.py b/test/test_signoff_package.py index c2c2bfec3..1de074f14 100644 --- a/test/test_signoff_package.py +++ b/test/test_signoff_package.py @@ -301,6 +301,18 @@ def test_collect_signoff_package_requires_synthesis_verilog(tmp_path): ) +def test_read_only_signoff_collection_does_not_rewrite_home_checklist(tmp_path): + workspace_dir = _make_signoff_workspace(tmp_path) + checklist = workspace_dir / "home" / "checklist.json" + before = file_digest(checklist) + + _make_engine_flow(workspace_dir).collect_signoff_package( + SignoffPackageOptions(archive=False, materialize=False) + ) + + assert file_digest(checklist) == before + + def test_collect_signoff_package_rejects_stale_post_route_lec_proof(tmp_path): workspace_dir = _make_signoff_workspace(tmp_path) gate = workspace_dir / "lvs_ecc" / "output" / "gcd_lvs.v.gz" diff --git a/test/tools/ecc/test_qor_detail_facts.py b/test/tools/ecc/test_qor_detail_facts.py new file mode 100644 index 000000000..f4fe8537d --- /dev/null +++ b/test/tools/ecc/test_qor_detail_facts.py @@ -0,0 +1,239 @@ +import json + +from chipcompiler.data import OriginDesign, StepEnum, Workspace +from chipcompiler.tools.ecc.builder import build_step, build_step_space +from chipcompiler.tools.ecc.metrics import build_metrics_floorplan, build_metrics_lvs +from chipcompiler.tools.ecc.qor_detail_facts import database_fact_summary + + +def test_database_fact_summary_extracts_layout_and_instance_classes(tmp_path): + path = tmp_path / "Floorplan.db.json" + path.write_text( + json.dumps( + { + "Design Layout": { + "die_area": 2259.861, + "core_area": 1778.432, + "die_bounding_width": 47.538, + "die_bounding_height": 47.538, + "die_usage": 0.34, + "core_usage": 0.42, + }, + "Design Statis": { + "num_iopins": 58, + "num_instances": 615, + "num_nets": 361, + }, + "Instances": { + "total": {"area": 1200.0, "num": 615}, + "clock": {"area": 20.0, "num": 10}, + "iopads": {"area": 10.0, "num": 2}, + "logic": {"area": 709.25, "num": 286}, + "macros": {"area": 401.5, "num": 3}, + }, + } + ), + encoding="utf-8", + ) + + summary = database_fact_summary(path) + + assert summary is not None + assert summary["layout"]["core_area"] == 1778.432 + assert summary["instance_total"]["count"] == 615 + assert {item["kind"] for item in summary["instance_classes"]} == { + "clock", + "iopads", + "logic", + "macros", + } + + +def test_floorplan_qor_metrics_include_instance_classes_and_database_facts(tmp_path): + workspace = Workspace( + directory=tmp_path, + design=OriginDesign(name="gcd", top_module="gcd"), + ) + step = build_step( + workspace=workspace, + step_name=StepEnum.FLOORPLAN.value, + input_def=tmp_path / "input.def", + input_verilog=tmp_path / "input.v", + ) + build_step_space(step) + assert step.feature.db is not None + step.feature.db.write_text( + json.dumps( + { + "Design Layout": { + "die_area": 2259.861, + "core_area": 1778.432, + "die_bounding_width": 47.538, + "die_bounding_height": 47.538, + "die_usage": 0.34, + "core_usage": 0.42, + }, + "Design Statis": { + "num_iopins": 58, + "num_instances": 615, + "num_nets": 361, + }, + "Instances": { + "total": {"area": 1200.0, "num": 615}, + "clock": {"area": 20.0, "num": 10}, + "iopads": {"area": 10.0, "num": 2}, + "logic": {"area": 709.25, "num": 286}, + "macros": {"area": 401.5, "num": 3}, + }, + } + ), + encoding="utf-8", + ) + + metrics = build_metrics_floorplan(workspace, step) + + assert metrics is not None + assert step.analysis.qor_metrics is not None + assert step.analysis.qor_metrics.exists() + qor_metrics = json.loads(step.analysis.qor_metrics.read_text(encoding="utf-8")) + assert qor_metrics["schema_version"] == 3 + assert qor_metrics["tool"] == "ecc" + assert qor_metrics["step"] == StepEnum.FLOORPLAN.value + assert qor_metrics["design"] == "gcd" + + records = {record["id"]: record for record in qor_metrics["metrics"]} + assert records["core_utilization"]["value"] == 0.42 + assert records["core_utilization"]["direction"] == "target_range" + assert records["core_area"]["value"] == 1778.432 + assert records["die_area"]["unit"] == "um^2" + assert { + metric_id: ( + records[metric_id]["value"], + records[metric_id]["source"]["selector"], + records[metric_id]["rating"]["score"], + ) + for metric_id in ( + "macro_count", + "macro_area", + "std_cell_count", + "std_cell_area", + "clock_count", + "clock_area", + "io_pad_count", + "io_pad_area", + "instance_area", + ) + } == { + "macro_count": (3, "/Instances/macros/num", False), + "macro_area": (401.5, "/Instances/macros/area", False), + "std_cell_count": (286, "/Instances/logic/num", False), + "std_cell_area": (709.25, "/Instances/logic/area", False), + "clock_count": (10, "/Instances/clock/num", False), + "clock_area": (20, "/Instances/clock/area", False), + "io_pad_count": (2, "/Instances/iopads/num", False), + "io_pad_area": (10, "/Instances/iopads/area", False), + "instance_area": (1200, "/Instances/total/area", False), + } + details = {detail["id"]: detail["summary"] for detail in qor_metrics["details"]} + assert details["database_facts"]["layout"]["core_area"] == 1778.432 + assert details["database_facts"]["instance_total"]["count"] == 615 + assert {item["kind"] for item in details["database_facts"]["instance_classes"]} == { + "clock", + "iopads", + "logic", + "macros", + } + + +def test_lvs_qor_metrics_include_connectivity_summary(tmp_path): + workspace = Workspace( + directory=tmp_path, + design=OriginDesign(name="gcd", top_module="gcd"), + ) + step = build_step( + workspace=workspace, + step_name=StepEnum.LVS.value, + input_def=tmp_path / "input.def", + input_verilog=tmp_path / "input.v", + ) + build_step_space(step) + assert step.feature.step is not None + step.feature.step.write_text( + json.dumps( + { + "entity": [{"entity": "nets", "netlist": 10, "def": 9, "difference": 1}], + "connectivity": [ + {"connectivity": "signals", "open": 1, "short": 0, "connected": 9, "total": 10} + ], + "violations": [ + {"type": "open", "net": ["n1"], "terminals": ["A", "B"]}, + {"type": "short", "net": "n2", "components": ["u1", "u2"]}, + ], + } + ), + encoding="utf-8", + ) + + metrics = build_metrics_lvs(workspace, step) + + assert metrics is not None + assert metrics.data["lvs_count"] == 2 + assert step.analysis.qor_summary is not None + summary = json.loads(step.analysis.qor_summary.read_text(encoding="utf-8")) + assert summary["quality_status"] == "blocked" + assert summary["gates"] == [ + { + "id": "qor.lvs.clean", + "title": "Final LVS clean", + "state": "failed", + "blocking": True, + "metrics": [ + { + "id": "lvs_count", + "actual": 2, + "operator": "==", + "expected": 0, + "source": { + "kind": "feature", + "path": "feature/lvs.step.json", + "selector": "/violations", + }, + } + ], + "evidence": [ + { + "kind": "feature", + "path": "feature/lvs.step.json", + "selector": "/violations", + } + ], + } + ] + assert step.analysis.qor_metrics is not None + details = { + detail["id"]: detail + for detail in json.loads(step.analysis.qor_metrics.read_text(encoding="utf-8"))["details"] + } + assert details["lvs_connectivity_summary"]["summary"] == { + "schema_version": 1, + "entities": [{"entity": "nets", "netlist": 10, "def": 9, "difference": 1}], + "connectivity": [ + {"connectivity": "signals", "open": 1, "short": 0, "connected": 9, "total": 10} + ], + "violations": [ + { + "type": "open", + "net": "n1", + "instance": "", + "terminals": "A, B", + "components": "", + }, + { + "type": "short", + "net": "n2", + "instance": "", + "terminals": "", + "components": "u1, u2", + }, + ], + } diff --git a/test/tools/ecc/test_runner.py b/test/tools/ecc/test_runner.py index 3477516a9..eaa7909fa 100644 --- a/test/tools/ecc/test_runner.py +++ b/test/tools/ecc/test_runner.py @@ -953,6 +953,50 @@ def test_run_sta_uses_matched_report_and_feature_corner_directories(tmp_path, mo ] +def test_run_sta_returns_false_when_sdc_is_missing(tmp_path, monkeypatch): + config_dir = tmp_path / "config" + config_dir.mkdir() + max_lib = tmp_path / "pdk" / "max.lib" + spef = tmp_path / "RCX_ecc" / "output" / "gcd_RCworst_125C.spef" + for path in (max_lib, spef): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("", encoding="utf-8") + sta_config = config_dir / "sta_ecc.json" + sta_config.write_text( + json.dumps( + { + "liberty": [{"corner": "MAX", "temperature": 125, "path": [str(max_lib)]}], + "signoff": [{"MAX": ["RCworst"]}], + } + ), + encoding="utf-8", + ) + rcx_config = config_dir / "rcx_ecc.json" + rcx_config.write_text( + json.dumps({"output": str(tmp_path / "RCX_ecc" / "data")}), + encoding="utf-8", + ) + logger = FakeLogger() + workspace = Workspace( + directory=tmp_path, + design=OriginDesign(name="gcd", top_module="gcd"), + pdk=PDK(libs=[max_lib], sdc=None), + config={StepEnum.STA.value: sta_config, StepEnum.RCX.value: rcx_config}, + logger=logger, + ) + step = EccStep( + name=StepEnum.STA.value, + data=EccData(steps={StepEnum.STA.value: tmp_path / "sta_ecc" / "data" / "sta"}), + report=EccReport(dir=tmp_path / "sta_ecc" / "report"), + feature=EccFeature(dir=tmp_path / "sta_ecc" / "feature"), + ) + monkeypatch.setattr(ecc_runner, "EccSubFlow", FakeSubFlow) + monkeypatch.setattr(ecc_runner, "get_eda_instance", lambda **kwargs: FakeSynthesisStaModule()) + + assert ecc_runner.run_sta(workspace, step) is False + assert logger.errors[0][0] == "STA SDC does not exist: %s" + + def test_rcx_checklist_strips_top_module_from_spef_corner(tmp_path): checklist = EccRcxChecklist.__new__(EccRcxChecklist) checklist.workspace = Workspace( diff --git a/test/tools/yosys/test_metrics.py b/test/tools/yosys/test_metrics.py index f9185a230..5e0900ef3 100644 --- a/test/tools/yosys/test_metrics.py +++ b/test/tools/yosys/test_metrics.py @@ -98,7 +98,7 @@ def test_synthesis_metrics_write_v2_qor_files_without_legacy_metrics(tmp_path): "display_name": "Synthesis Dynamic Power", "value": 62.7968, "unit": "uW", - "category": "power", + "category": "power_integrity", "direction": "trend_only", "scope": "synthesis", "corner": None, diff --git a/uv.lock b/uv.lock index aedf194bc..1c2ee175c 100644 --- a/uv.lock +++ b/uv.lock @@ -493,7 +493,7 @@ requires-dist = [ { name = "ecc-dreamplace", editable = "chipcompiler/thirdparty/ecc-dreamplace" }, { name = "ecc-tools-bin", editable = "chipcompiler/thirdparty/ecc-tools" }, { name = "fastapi", specifier = ">=0.109" }, - { name = "jsonrpcserver", specifier = ">=5.0.9" }, + { name = "jsonrpcserver", specifier = ">=5.0.9,<6" }, { name = "klayout", specifier = ">=0.30.2" }, { name = "matplotlib", specifier = ">=3.4" }, { name = "numpy", specifier = ">=1.21" },